def add_task(cls, dataset_id, module_id, user_ID, module_version, dataset_version, task_type='classification', data_type='image', evaluation=0, desc='', user_token=None, device='cpu'): # insert module module = ProjectBusiness.get_by_id(module_id) dataset = ProjectBusiness.get_by_id(dataset_id) # 数据插入 config = cls._insert_module_and_parse_json(dataset, module, dataset_version) task = CSTaskBusiness.add_task(dataset_id, user_ID, 0, 0, module_id, task_type, data_type, evaluation, desc, module_version, dataset_version, classes=config.get('classes'), device=device) # create pickle insert_version = '' if dataset_version == 'current' else dataset_version data_path = os.path.join(module.path, 'results') des_pickle_path = os.path.join(data_path, str(task.id)) if not os.path.exists(des_pickle_path): os.makedirs(des_pickle_path) des_pickle_path = os.path.join(des_pickle_path, 'create.pkl') with open(des_pickle_path, 'wb') as file: user_module_read_dataset_path = os.path.join( './datasets', dataset.user.user_ID + '-' + dataset.name + '-' + insert_version) pickle.dump({'filename': user_module_read_dataset_path}, file) print('dump to pickle:', user_module_read_dataset_path) print('pickle path:', des_pickle_path) # create done # 后期修改, 要删除, 只为了暂时测试, 不需要返回值的 cls._create_job_for_get_probability(task, module, user_ID, user_token, des_pickle_path, device) # 制造 # tasnform data format return task
def tb_proxy(project_id): port = ProjectBusiness.get_by_id(project_id).tb_port resp = requests.post('{localhost}:{port}'. format(localhost='http://localhost', port=port) ) print(resp.content) return resp.content
def upload_file_module(): # check if the post request has the file part if 'file' not in request.files: return jsonify({"response": "No file part"}), 399 file = request.files['file'] # if user does not select file, browser also # submit a empty part without filename if file.filename == '': return jsonify({"response": "No selected file'"}), 399 data = dict(request.form.items()) user_ID = data["user_ID"] project_id = data["project_id"] project = ProjectBusiness.get_by_id(project_id) store_path = os.path.join(USER_DIR, user_ID, project.name) # 如果路径不存在创建路径 if os.path.isdir(store_path): pass else: os.mkdir(store_path) if file and file_service.allowed_file(file.filename): filename = secure_filename(file.filename) file.save(os.path.join(store_path, filename)) if file.filename.split('.')[-1] == 'zip': # 解压缩 file_size, file_uri, folder_name = \ file_service.extract_files_and_get_size(file, store_path) print(file_size) return jsonify({"response": "upload success"}), 200
def update_user_request(cls, user_request_id, **kwargs): tags = kwargs.pop('tags', []) if not isinstance(tags, list): tags = str_utility.split_without_empty(tags) select_project_id = kwargs.pop('select_project') if select_project_id: select_project = ProjectBusiness.get_by_id(select_project_id) kwargs.update({'select_project': select_project}) request = UserRequestBusiness.update_by_id( user_request_id=user_request_id, **kwargs) # todo 判断是否更新到Elasticsearch 那边的数据 if 'description' in kwargs or 'title' in kwargs: try: SearchService.add_request(request.title, request.description, request.type, request.user.username, request.id) except: pass oldtags = [tag.id for tag in request.tags] # update tags cls.update_tags(oldtags=oldtags, newtags=tags, entity_type='request', entities=[request]) # request.add_tags(tags) return request
def parse_select_project(object_info, object): if 'select_project' in object_info: # 获取commit try: # commits = ProjectBusiness.get_commits( # request_answer[index].select_project.path) select_project = object['select_project'] if isinstance(select_project, str) or isinstance( select_project, ObjectId): from server3.business.project_business import ProjectBusiness select_project = ProjectBusiness.get_by_id(select_project) # select_project.commits = [{ # 'message': c.message, # 'time': datetime.fromtimestamp(c.time[0] + c.time[1]), # } for c in commits] object_info['select_project'] = convert_to_json( select_project.to_mongo()) object_info['select_project']['commits'].reverse() object_info['select_project'][ 'username'] = select_project.user.username object_info['select_project'][ 'user_ID'] = select_project.user.user_ID object_info['select_project'][ 'avatar_url'] = select_project.user.avatar_url except Exception as e: object_info['select_project'] = {'deleted': True}
def fulfil_task(cls, task_id, user_ID): task = CSTaskBusiness.get_task_by_id(task_id) dataset = ProjectBusiness.get_by_id(task.dataset.id) # complete task.status = 2 task.save() # file_path csv_file = cls._generate_csv_file_for_results(task, dataset, user_ID) # print(csv_file) dest_path = os.path.join(task.module.path, CS_PATH, str(task.id)) if not os.path.exists(dest_path): os.makedirs(dest_path) # dest_path = os.path.join(module_path, task.task_type + '_' + str(task.evaluation)) dest_path = os.path.join(dest_path, csv_file.split('/')[-1]) shutil.copyfile(csv_file, dest_path) # print('error:', task.to_mongo()) results = CSResultBusiness.read_all_by_id(task.id, task.evaluation) sample_marked_for_user, need_marked_times, marked_times = \ cls._compute_sample_marked_count(results) # 结束 if task.evaluation >= 3 or sample_marked_for_user < task.total_count: task.user_operation = True task.save() # add the info task = cls._add_extra_info_with_task(task) print('back') return csv_file, task, dest_path
def get_downloading_list(cls, project_id): projects = ProjectBusiness.get_by_id(project_id) user = UserBusiness.get_by_user_ID('momodel') try: downloading_list = DatasetBusiness.filter( user=user, mount_kaggle_projects=projects, is_finished=False) return downloading_list except: return False
def create_user_request(cls, title, user_ID, **kwargs): tags = kwargs.pop('tags', []) select_project_id = kwargs.pop('select_project') if select_project_id: select_project = ProjectBusiness.get_by_id(select_project_id) kwargs.update({'select_project': select_project}) # create a new user_request object user = UserBusiness.get_by_user_ID(user_ID) created_user_request = UserRequestBusiness.add_user_request( title=title, user=user, status=0, **kwargs) # todo 添加Elasticsearch的Request数据 SearchService.add_request(created_user_request.title, created_user_request.description, created_user_request.type, created_user_request.user.username, created_user_request.id) # update tags cls.update_tags(oldtags=[], newtags=tags, entity_type='request', entities=[created_user_request]) # created_user_request.add_tags(tags) EventBusiness.create_event(user=user, target=created_user_request, target_type="request", action="create") # 记录世界频道消息 # 推送消息 # world = WorldService.system_send( # channel=CHANNEL.request, # message=f"用户{created_user_request.user.user_ID}" + # f"发布了需求{created_user_request.title}") # admin_user = UserBusiness.get_by_user_ID('admin') # # text = f"<http://localhost:8899/discussion/{created_user_request.id}?type={created_user_request.type}|" \ # f"用户 {created_user_request.user.user_ID} 发布了需求 {created_user_request.title}>" # # SlackBusiness.send_message(user=admin_user, text=text, channel='general') if ENV in ['PROD', 'MO']: try: text = f"[{created_user_request.user.username}]({WEB_ADDR}/profile/{created_user_request.user.user_ID})" \ f" 发布了话题 [{created_user_request.title}]({WEB_ADDR}/discussion/{created_user_request.id}?type={created_user_request.type})" RocketChatBusiness.post_official_message(text=text) except: print('rc error') # msg = json_utility.convert_to_json({ # 'user_ID': user_ID, # 'content': '发布了新需求,快去看看吧', # }) # socketio.emit('world_message', msg, # namespace='/log/%s' % user_ID) return created_user_request
def get_crowdsoucring_list(cls, dataset_id): dataset = ProjectBusiness.get_by_id(dataset_id) # get the path dataset_path = dataset.dataset_path # concanated crowdsoucing_path = os.path.join(dataset_path, 'crowdsourcing') # print(crowdsoucing_path) if os.path.exists(crowdsoucing_path): # read the all files return cls._read_directory(crowdsoucing_path) else: return False
def commit_broadcast(project_id): project = ProjectBusiness.get_by_id(project_id) # ProjectBusiness.commit(project_id, commit_msg) receivers = project.favor_users # get app subscriber # commits = ProjectBusiness.get_commits(project.path) admin_user = UserBusiness.get_by_user_ID('admin') message_service.create_message(admin_user, 'commit', receivers, project.user, project_type=project.type, project_id=project_id) return jsonify({"response": 1})
def get_project(project_id): if not project_id: return jsonify({'response': 'no project_id arg'}), 400 project = ProjectBusiness.get_by_id(project_id) # if request.args.get('commits') == 'true': # commits = ProjectBusiness.get_commits(project.path) # project.commits = [{ # 'message': c.message, # 'time': datetime.fromtimestamp(c.time[0] + c.time[1]), # } for c in commits] project = json_utility.convert_to_json(project.to_mongo()) project['commits'].reverse() return make_response(jsonify({'response': project}), 200)
def _generate_csv_file_for_results(cls, task, dataset, user_ID): # init the path moudle = ProjectBusiness.get_by_id(task.module.id) dataset_path = dataset.dataset_path save_path = os.path.join('crowdsourcing', user_ID, moudle.name, task.dataset_version) csv_file_name = str(task.id) + '_' + str(task.evaluation) + '.csv' # read the result results = CSResultBusiness.read_all_by_id(task, task.evaluation) # create the path store_path = os.path.join(dataset_path, save_path) if not os.path.exists(store_path): os.makedirs(store_path) # get the statistics result full_path = os.path.join(store_path, csv_file_name) _, transform_tasks = cls._analyse_labels(results) # get the labels save_key = [] for sample in transform_tasks: # print(sample) max_count = 0 label = '' for key in sample['label'].keys(): if key != '': if label == '': max_count = sample['label'][key] label = key elif max_count < sample['label'][key]: max_count = sample['label'][key] label = key label = 'None' if label == '' else label save_key.append({'sample': sample['sample'], 'label': label}) # save if len(save_key) <= 0: save_key.append({'sample': 'no samples', 'label': 'no labels'}) # print(save_key) pd.DataFrame(save_key, index=np.arange(len(save_key))).to_csv(full_path, index=None) # move to module # path return full_path
def need_csv_file(cls, task_id, user_ID): # what is time for generating this file. task = CSTaskBusiness.get_task_by_id(task_id) dataset = ProjectBusiness.get_by_id(task.dataset.id) # generate file, will overwrite when the next time. csv_file = cls._generate_csv_file_for_results(task, dataset, task.sponsor.user_ID) # if the sponsor click it if task.sponsor.user_ID == user_ID: # copy it dest_path = os.path.join(task.module.path, CS_PATH, str(task.id)) if not os.path.exists(dest_path): os.makedirs(dest_path) # dest_path = os.path.join(module_path, task.task_type + '_' + str(task.evaluation)) dest_path = os.path.join(dest_path, csv_file.split('/')[-1]) shutil.copyfile(csv_file, dest_path) return csv_file, task, dest_path # print(csv_file) return csv_file, task
def related_projects(cls, **args): if ElasticSearchSearvice.is_available(): page_no = args.get('page_no') page_size = args.get('page_size') project_id = args.get('project_id') project_type = args.get('project_type') project = ProjectBusiness.get_by_id(project_id) tags = [tag.id for tag in project.tags] projects = [] for tag in tags: ps = bus_type_mapper[project_type].repo.search( tag, {'tags': 'icontains'}) ps = ps(status='active') for p in ps: if 'tutorial' not in p.display_name.lower(): projects.append(p) # projects.extend(ps) np.random.shuffle(projects) if not isinstance(projects, list): projects = projects.tolist() return projects[(page_no - 1) * page_size:page_no * page_size], len(projects) return [], 0
def commit_broadcast(project_id): project = ProjectBusiness.get_by_id(project_id) ProjectService.send_message(project, m_type='commit') return jsonify({"response": 1})
def create_job(project_id, script_path): working_path = ProjectBusiness.get_by_id(project_id).path print(working_path, script_path)
def job_call_the_filter(cls, task_id, if_success=True): task = CSTaskBusiness.get_task_by_id(task_id) # print(task_id, if_success) # print(task.to_mongo()) if if_success: # read pickle module = ProjectBusiness.get_by_id(task.module.id) file_path = os.path.join(module.path, 'results', str(task.id), 'sample.pkl') try: with open(file_path, 'rb') as file: non_exactly = pickle.load(file) # file.close() # print('non_exactly:', non_exactly) if len(non_exactly.values()) > 2: # raise RuntimeError('The pickle file is not valid.') task.status = 2 task.user_operation = True task.save() _ = MessageService.create_message( UserBusiness.get_by_user_ID('admin'), 'cs_task_pickle_file_invalid', [task.sponsor], user=task.sponsor, task=task, project=task.module) return if len(non_exactly['results']) > 0: # task.status = 2 task.user_operation = True task.save() _ = MessageService.create_message( UserBusiness.get_by_user_ID('admin'), 'cs_task_done', [task.sponsor], user=task.sponsor, task=task, project=task.module) return # print(file_path, non_exactly) # print(non_exactly) unmarked_set = cls._filter_probabilities( [non_exactly['results'], non_exactly['max_choose']]) # print(unmarked_set) if len(unmarked_set) > 0: task.status = 0 task.evaluation += 1 task.total_count += len(unmarked_set) cls._dispatch(unmarked_set, task_id, task.evaluation) task.save() # send notification to user message = MessageService.create_message( UserBusiness.get_by_user_ID('admin'), 'cs_task_start', [task.sponsor], user=task.sponsor, task=task, project=task.module) # print('success:', message.to_mongo()) return task else: task.status = 2 task.user_operation = True # return {'msg': 'This task has occurred error when executed, ' # 'you can check it in your module job page.'} # send notification to user task.save() message = MessageService.create_message( UserBusiness.get_by_user_ID('admin'), 'cs_task_done', [task.sponsor], user=task.sponsor, task=task, project=task.module) print('no unlabeled:', message.to_mongo()) except FileNotFoundError as e: # return {'msg': 'your module don`t create the pickle file.'} # send notification to user task.status = 2 task.user_operation = True task.save() task = cls._add_extra_info_with_task(task) message = MessageService.create_message( UserBusiness.get_by_user_ID('admin'), 'cs_task_pickle_file_not_found', [task.sponsor], task=task, project=task.module, ) # , 'project': task.module print("file not found:", message.to_mongo()) except Exception as e: # return {'msg': 'error occur.'} print(e) task.status = 2 task.user_operation = True task.save() task = cls._add_extra_info_with_task(task) message = MessageService.create_message( UserBusiness.get_by_user_ID('admin'), 'cs_task_job_error', [task.sponsor], task=task, user=task.sponsor, project=task.module) # , 'project': task.module print('exception:', e, 'message:', message.to_mongo()) raise RuntimeError(e) else: task.status = 2 task.user_operation = True # task.evaluation -= 1 if task.evaluation > 1: message = MessageService.create_message( UserBusiness.get_by_user_ID('admin'), 'cs_task_job_error', [task.sponsor], user=task.sponsor, task=task, project=task.module, ) # , 'project': task.module print('train, has exception:', message.to_mongo()) task.save()