def send_job():
# copy job to staging folder
staging_path = new_job.staging_path
staging_project_path = '/'.join(staging_path.split('/')[:-1])
if not os.path.exists(staging_project_path):
os.makedirs(staging_project_path)
time.sleep(2)
ProjectBusiness.copy_to_ignore_env(new_job.project,
staging_path,
no_site_packages=False,
no_remove=True)
project_path = f'{user.user_ID}/{project.name}'
# check if running or pending jobs which makes this job to wait
wait = False
if cls.all_running_pending_jobs_num() > 1 and env == 'gpu':
wait = True
print('need wait')
# send job to celery
celery.send_task('celery_conf.create_job',
args=(str(new_job.id),
new_job.script_path,
project_path, env,
user_token, function,
str(task.id) if task else None,
False, PY_SERVER, args, wait,
0),
queue=env)
def nb_to_script(project_id):
data = request.get_json()
optimise = data.get('optimise')
nb_path = data.get('nb_path')
try:
# ProjectBusiness.nb_to_script(project_id, nb_path, optimise)
ProjectBusiness.nb_to_py_script(project_id, nb_path, optimise)
except FileNotFoundError as e:
return jsonify({"response": 'Try save your notebook and convert '
'again.'}), \
399
return jsonify({"response": 1})
def add_task(cls,
dataset_id,
module_id,
user_ID,
module_version,
dataset_version,
task_type='classification',
data_type='image',
evaluation=0,
desc='',
user_token=None,
device='cpu'):
# insert module
module = ProjectBusiness.get_by_id(module_id)
dataset = ProjectBusiness.get_by_id(dataset_id)
# 数据插入
config = cls._insert_module_and_parse_json(dataset, module,
dataset_version)
task = CSTaskBusiness.add_task(dataset_id,
user_ID,
0,
0,
module_id,
task_type,
data_type,
evaluation,
desc,
module_version,
dataset_version,
classes=config.get('classes'),
device=device)
# create pickle
insert_version = '' if dataset_version == 'current' else dataset_version
data_path = os.path.join(module.path, 'results')
des_pickle_path = os.path.join(data_path, str(task.id))
if not os.path.exists(des_pickle_path):
os.makedirs(des_pickle_path)
des_pickle_path = os.path.join(des_pickle_path, 'create.pkl')
with open(des_pickle_path, 'wb') as file:
user_module_read_dataset_path = os.path.join(
'./datasets', dataset.user.user_ID + '-' + dataset.name + '-' +
insert_version)
pickle.dump({'filename': user_module_read_dataset_path}, file)
print('dump to pickle:', user_module_read_dataset_path)
print('pickle path:', des_pickle_path)
# create done
# 后期修改, 要删除, 只为了暂时测试, 不需要返回值的
cls._create_job_for_get_probability(task, module, user_ID, user_token,
des_pickle_path, device)
# 制造
# tasnform data format
return task
def update_project(project_identity):
data = request.get_json()
tags = data.get('tags')
if not isinstance(tags, list) and tags is not None:
data['tags'] = str_utility.split_without_empty(tags)
if request.args.get('by') == 'name':
project = ProjectBusiness.update_project_by_identity(
project_identity, **data)
else:
project = ProjectBusiness.update_project(project_identity, **data)
project = json_utility.convert_to_json(project.to_mongo())
return jsonify({'response': project}), 200
def get_project(project_id):
if not project_id:
return jsonify({'response': 'no project_id arg'}), 400
project = ProjectBusiness.get_by_id(project_id)
if request.args.get('commits') == 'true':
commits = ProjectBusiness.get_commits(project.path)
project.commits = [{
'message':
c.message,
'time':
datetime.fromtimestamp(c.time[0] + c.time[1]),
} for c in commits]
project = json_utility.convert_to_json(project.to_mongo())
return make_response(jsonify({'response': project}), 200)
def fulfil_task(cls, task_id, user_ID):
task = CSTaskBusiness.get_task_by_id(task_id)
dataset = ProjectBusiness.get_by_id(task.dataset.id)
# complete
task.status = 2
task.save()
# file_path
csv_file = cls._generate_csv_file_for_results(task, dataset, user_ID)
# print(csv_file)
dest_path = os.path.join(task.module.path, CS_PATH, str(task.id))
if not os.path.exists(dest_path):
os.makedirs(dest_path)
# dest_path = os.path.join(module_path, task.task_type + '_' + str(task.evaluation))
dest_path = os.path.join(dest_path, csv_file.split('/')[-1])
shutil.copyfile(csv_file, dest_path)
# print('error:', task.to_mongo())
results = CSResultBusiness.read_all_by_id(task.id, task.evaluation)
sample_marked_for_user, need_marked_times, marked_times = \
cls._compute_sample_marked_count(results)
# 结束
if task.evaluation >= 3 or sample_marked_for_user < task.total_count:
task.user_operation = True
task.save()
# add the info
task = cls._add_extra_info_with_task(task)
print('back')
return csv_file, task, dest_path
def parse_select_project(object_info, object):
if 'select_project' in object_info:
# 获取commit
try:
# commits = ProjectBusiness.get_commits(
# request_answer[index].select_project.path)
select_project = object['select_project']
if isinstance(select_project, str) or isinstance(
select_project, ObjectId):
from server3.business.project_business import ProjectBusiness
select_project = ProjectBusiness.get_by_id(select_project)
# select_project.commits = [{
# 'message': c.message,
# 'time': datetime.fromtimestamp(c.time[0] + c.time[1]),
# } for c in commits]
object_info['select_project'] = convert_to_json(
select_project.to_mongo())
object_info['select_project']['commits'].reverse()
object_info['select_project'][
'username'] = select_project.user.username
object_info['select_project'][
'user_ID'] = select_project.user.user_ID
object_info['select_project'][
'avatar_url'] = select_project.user.avatar_url
except Exception as e:
object_info['select_project'] = {'deleted': True}
def tb_proxy(project_id):
port = ProjectBusiness.get_by_id(project_id).tb_port
resp = requests.post('{localhost}:{port}'.
format(localhost='http://localhost', port=port)
)
print(resp.content)
return resp.content
def upload_file_module():
# check if the post request has the file part
if 'file' not in request.files:
return jsonify({"response": "No file part"}), 399
file = request.files['file']
# if user does not select file, browser also
# submit a empty part without filename
if file.filename == '':
return jsonify({"response": "No selected file'"}), 399
data = dict(request.form.items())
user_ID = data["user_ID"]
project_id = data["project_id"]
project = ProjectBusiness.get_by_id(project_id)
store_path = os.path.join(USER_DIR, user_ID, project.name)
# 如果路径不存在创建路径
if os.path.isdir(store_path):
pass
else:
os.mkdir(store_path)
if file and file_service.allowed_file(file.filename):
filename = secure_filename(file.filename)
file.save(os.path.join(store_path, filename))
if file.filename.split('.')[-1] == 'zip':
# 解压缩
file_size, file_uri, folder_name = \
file_service.extract_files_and_get_size(file, store_path)
print(file_size)
return jsonify({"response": "upload success"}), 200
def update_user_request(cls, user_request_id, **kwargs):
tags = kwargs.pop('tags', [])
if not isinstance(tags, list):
tags = str_utility.split_without_empty(tags)
select_project_id = kwargs.pop('select_project')
if select_project_id:
select_project = ProjectBusiness.get_by_id(select_project_id)
kwargs.update({'select_project': select_project})
request = UserRequestBusiness.update_by_id(
user_request_id=user_request_id, **kwargs)
# todo 判断是否更新到Elasticsearch 那边的数据
if 'description' in kwargs or 'title' in kwargs:
try:
SearchService.add_request(request.title, request.description,
request.type, request.user.username,
request.id)
except:
pass
oldtags = [tag.id for tag in request.tags]
# update tags
cls.update_tags(oldtags=oldtags,
newtags=tags,
entity_type='request',
entities=[request])
# request.add_tags(tags)
return request
def remove_project(project_id):
user_ID = request.args.get('user_ID')
if not project_id:
return jsonify({'response': 'no project_id arg'}), 400
if not user_ID:
return jsonify({'response': 'no user_ID arg'}), 400
result = ProjectBusiness.remove_project_by_id(project_id, user_ID)
return jsonify({'response': result}), 200
def get_downloading_list(cls, project_id):
projects = ProjectBusiness.get_by_id(project_id)
user = UserBusiness.get_by_user_ID('momodel')
try:
downloading_list = DatasetBusiness.filter(
user=user, mount_kaggle_projects=projects, is_finished=False)
return downloading_list
except:
return False
def list_request_answer():
user_request_id = request.args.get("user_request_id")
user_ID = request.args.get("user_ID")
page_no = int(request.args.get("page_no", 1))
page_size = int(request.args.get("page_size", 10))
type = request.args.get("type")
search_query = request.args.get("search_query")
if user_request_id:
request_answer = request_answer_service. \
get_all_answer_of_this_user_request(user_request_id)
for each_one in request_answer:
# todo 删除历史数据后，可删除此判断
if each_one.answer_user:
each_one.answer_user_ID = each_one.answer_user.user_ID
request_answer_info = json_utility. \
me_obj_list_to_json_list(request_answer)
# 得到每一个answer下的comments 和 selcet project
for index, answer in enumerate(request_answer_info):
answer_comment = comments_service.get_comments_of_this_answer(
answer['_id'])
answer_comment_info = json_utility. \
me_obj_list_to_json_list(answer_comment)
answer['comment'] = answer_comment_info
if 'select_project' in answer:
# 获取commit
try:
commits = ProjectBusiness.get_commits(
request_answer[index].select_project.path)
select_project = request_answer[index].select_project
select_project.commits = [{
'message':
c.message,
'time':
datetime.fromtimestamp(c.time[0] + c.time[1]),
} for c in commits]
answer['select_project'] = json_utility.convert_to_json(
select_project.to_mongo())
except:
print('selected app has been deleted')
answer['select_project'] = {'deleted': True}
return jsonify({'response': request_answer_info}), 200
elif user_ID:
request_answer, total_number = request_answer_service. \
get_all_answer_by_user_ID(user_ID, page_no, page_size,
type, search_query)
request_answer_info = json_utility. \
me_obj_list_to_json_list(request_answer)
return jsonify({
'response': {
'request_answer_info': request_answer_info,
'total_number': total_number
}
}), 200
else:
return jsonify({'response': 'insufficient arguments'}), 400
def create_user_request(cls, title, user_ID, **kwargs):
tags = kwargs.pop('tags', [])
select_project_id = kwargs.pop('select_project')
if select_project_id:
select_project = ProjectBusiness.get_by_id(select_project_id)
kwargs.update({'select_project': select_project})
# create a new user_request object
user = UserBusiness.get_by_user_ID(user_ID)
created_user_request = UserRequestBusiness.add_user_request(
title=title, user=user, status=0, **kwargs)
# todo 添加Elasticsearch的Request数据
SearchService.add_request(created_user_request.title,
created_user_request.description,
created_user_request.type,
created_user_request.user.username,
created_user_request.id)
# update tags
cls.update_tags(oldtags=[],
newtags=tags,
entity_type='request',
entities=[created_user_request])
# created_user_request.add_tags(tags)
EventBusiness.create_event(user=user,
target=created_user_request,
target_type="request",
action="create")
# 记录世界频道消息 # 推送消息
# world = WorldService.system_send(
# channel=CHANNEL.request,
# message=f"用户{created_user_request.user.user_ID}" +
# f"发布了需求{created_user_request.title}")
# admin_user = UserBusiness.get_by_user_ID('admin')
#
# text = f"<http://localhost:8899/discussion/{created_user_request.id}?type={created_user_request.type}|" \
# f"用户 {created_user_request.user.user_ID} 发布了需求 {created_user_request.title}>"
#
# SlackBusiness.send_message(user=admin_user, text=text, channel='general')
if ENV in ['PROD', 'MO']:
try:
text = f"[{created_user_request.user.username}]({WEB_ADDR}/profile/{created_user_request.user.user_ID})" \
f" 发布了话题 [{created_user_request.title}]({WEB_ADDR}/discussion/{created_user_request.id}?type={created_user_request.type})"
RocketChatBusiness.post_official_message(text=text)
except:
print('rc error')
# msg = json_utility.convert_to_json({
# 'user_ID': user_ID,
# 'content': '发布了新需求，快去看看吧',
# })
# socketio.emit('world_message', msg,
# namespace='/log/%s' % user_ID)
return created_user_request
def get_crowdsoucring_list(cls, dataset_id):
dataset = ProjectBusiness.get_by_id(dataset_id)
# get the path
dataset_path = dataset.dataset_path
# concanated
crowdsoucing_path = os.path.join(dataset_path, 'crowdsourcing')
# print(crowdsoucing_path)
if os.path.exists(crowdsoucing_path):
# read the all files
return cls._read_directory(crowdsoucing_path)
else:
return False
def commit_broadcast(project_id):
project = ProjectBusiness.get_by_id(project_id)
# ProjectBusiness.commit(project_id, commit_msg)
receivers = project.favor_users # get app subscriber
# commits = ProjectBusiness.get_commits(project.path)
admin_user = UserBusiness.get_by_user_ID('admin')
message_service.create_message(admin_user,
'commit',
receivers,
project.user,
project_type=project.type,
project_id=project_id)
return jsonify({"response": 1})
def _generate_csv_file_for_results(cls, task, dataset, user_ID):
# init the path
moudle = ProjectBusiness.get_by_id(task.module.id)
dataset_path = dataset.dataset_path
save_path = os.path.join('crowdsourcing', user_ID, moudle.name,
task.dataset_version)
csv_file_name = str(task.id) + '_' + str(task.evaluation) + '.csv'
# read the result
results = CSResultBusiness.read_all_by_id(task, task.evaluation)
# create the path
store_path = os.path.join(dataset_path, save_path)
if not os.path.exists(store_path):
os.makedirs(store_path)
# get the statistics result
full_path = os.path.join(store_path, csv_file_name)
_, transform_tasks = cls._analyse_labels(results)
# get the labels
save_key = []
for sample in transform_tasks:
# print(sample)
max_count = 0
label = ''
for key in sample['label'].keys():
if key != '':
if label == '':
max_count = sample['label'][key]
label = key
elif max_count < sample['label'][key]:
max_count = sample['label'][key]
label = key
label = 'None' if label == '' else label
save_key.append({'sample': sample['sample'], 'label': label})
# save
if len(save_key) <= 0:
save_key.append({'sample': 'no samples', 'label': 'no labels'})
# print(save_key)
pd.DataFrame(save_key,
index=np.arange(len(save_key))).to_csv(full_path,
index=None)
# move to module
# path
return full_path
def need_csv_file(cls, task_id, user_ID):
# what is time for generating this file.
task = CSTaskBusiness.get_task_by_id(task_id)
dataset = ProjectBusiness.get_by_id(task.dataset.id)
# generate file, will overwrite when the next time.
csv_file = cls._generate_csv_file_for_results(task, dataset,
task.sponsor.user_ID)
# if the sponsor click it
if task.sponsor.user_ID == user_ID:
# copy it
dest_path = os.path.join(task.module.path, CS_PATH, str(task.id))
if not os.path.exists(dest_path):
os.makedirs(dest_path)
# dest_path = os.path.join(module_path, task.task_type + '_' + str(task.evaluation))
dest_path = os.path.join(dest_path, csv_file.split('/')[-1])
shutil.copyfile(csv_file, dest_path)
return csv_file, task, dest_path
# print(csv_file)
return csv_file, task
def related_projects(cls, **args):
if ElasticSearchSearvice.is_available():
page_no = args.get('page_no')
page_size = args.get('page_size')
project_id = args.get('project_id')
project_type = args.get('project_type')
project = ProjectBusiness.get_by_id(project_id)
tags = [tag.id for tag in project.tags]
projects = []
for tag in tags:
ps = bus_type_mapper[project_type].repo.search(
tag, {'tags': 'icontains'})
ps = ps(status='active')
for p in ps:
if 'tutorial' not in p.display_name.lower():
projects.append(p)
# projects.extend(ps)
np.random.shuffle(projects)
if not isinstance(projects, list):
projects = projects.tolist()
return projects[(page_no - 1) * page_size:page_no *
page_size], len(projects)
return [], 0
def commit(project_id):
data = request.get_json()
commit_msg = data.get('commit_msg')
ProjectBusiness.commit(project_id, commit_msg)
return jsonify({"response": 1})
def job_call_the_filter(cls, task_id, if_success=True):
task = CSTaskBusiness.get_task_by_id(task_id)
# print(task_id, if_success)
# print(task.to_mongo())
if if_success:
# read pickle
module = ProjectBusiness.get_by_id(task.module.id)
file_path = os.path.join(module.path, 'results', str(task.id),
'sample.pkl')
try:
with open(file_path, 'rb') as file:
non_exactly = pickle.load(file)
# file.close()
# print('non_exactly:', non_exactly)
if len(non_exactly.values()) > 2:
# raise RuntimeError('The pickle file is not valid.')
task.status = 2
task.user_operation = True
task.save()
_ = MessageService.create_message(
UserBusiness.get_by_user_ID('admin'),
'cs_task_pickle_file_invalid', [task.sponsor],
user=task.sponsor,
task=task,
project=task.module)
return
if len(non_exactly['results']) > 0:
#
task.status = 2
task.user_operation = True
task.save()
_ = MessageService.create_message(
UserBusiness.get_by_user_ID('admin'),
'cs_task_done', [task.sponsor],
user=task.sponsor,
task=task,
project=task.module)
return
# print(file_path, non_exactly)
# print(non_exactly)
unmarked_set = cls._filter_probabilities(
[non_exactly['results'], non_exactly['max_choose']])
# print(unmarked_set)
if len(unmarked_set) > 0:
task.status = 0
task.evaluation += 1
task.total_count += len(unmarked_set)
cls._dispatch(unmarked_set, task_id, task.evaluation)
task.save()
# send notification to user
message = MessageService.create_message(
UserBusiness.get_by_user_ID('admin'),
'cs_task_start', [task.sponsor],
user=task.sponsor,
task=task,
project=task.module)
# print('success:', message.to_mongo())
return task
else:
task.status = 2
task.user_operation = True
# return {'msg': 'This task has occurred error when executed, '
# 'you can check it in your module job page.'}
# send notification to user
task.save()
message = MessageService.create_message(
UserBusiness.get_by_user_ID('admin'),
'cs_task_done', [task.sponsor],
user=task.sponsor,
task=task,
project=task.module)
print('no unlabeled:', message.to_mongo())
except FileNotFoundError as e:
# return {'msg': 'your module don`t create the pickle file.'}
# send notification to user
task.status = 2
task.user_operation = True
task.save()
task = cls._add_extra_info_with_task(task)
message = MessageService.create_message(
UserBusiness.get_by_user_ID('admin'),
'cs_task_pickle_file_not_found',
[task.sponsor],
task=task,
project=task.module,
)
# , 'project': task.module
print("file not found:", message.to_mongo())
except Exception as e:
# return {'msg': 'error occur.'}
print(e)
task.status = 2
task.user_operation = True
task.save()
task = cls._add_extra_info_with_task(task)
message = MessageService.create_message(
UserBusiness.get_by_user_ID('admin'),
'cs_task_job_error', [task.sponsor],
task=task,
user=task.sponsor,
project=task.module)
# , 'project': task.module
print('exception:', e, 'message:', message.to_mongo())
raise RuntimeError(e)
else:
task.status = 2
task.user_operation = True
# task.evaluation -= 1
if task.evaluation > 1:
message = MessageService.create_message(
UserBusiness.get_by_user_ID('admin'),
'cs_task_job_error',
[task.sponsor],
user=task.sponsor,
task=task,
project=task.module,
)
# , 'project': task.module
print('train, has exception:', message.to_mongo())
task.save()
def create_job(project_id, script_path):
working_path = ProjectBusiness.get_by_id(project_id).path
print(working_path, script_path)
def create_project(cls, *args, **kwargs):
ProjectBusiness.repo = Repo(project.App)
return ProjectBusiness.create_project(*args,
status='inactive',
**kwargs)
def nb_to_script(project_id):
data = request.get_json()
optimise = data.get('optimise')
nb_path = data.get('nb_path')
ProjectBusiness.nb_to_script(project_id, nb_path, optimise)
return jsonify({"response": 1})
def commit_broadcast(project_id):
project = ProjectBusiness.get_by_id(project_id)
ProjectService.send_message(project, m_type='commit')
return jsonify({"response": 1})