def init_project_path(user_key):
'''
create project directories
'''
logger.info("Init project: init project path for {}...".format(user_key))
user_path = os.path.join(PROJECT_DIR, 'usercase/' + user_key)
user_upload_path = os.path.join(user_path, 'upload')
user_download_path = os.path.join(user_path, 'download')
user_log_path = os.path.join(user_path, 'log')
utils.create_dir(user_path)
utils.create_dir(user_upload_path)
utils.create_dir(user_download_path)
utils.create_dir(user_log_path)
# create an empty log file
user_log_file_path = os.path.join(user_log_path, 'mb_pipe.log')
if not os.path.exists(user_log_file_path):
with open(user_log_file_path, 'w'):
pass
logger.info("Init project: send user key to Amazon SQS...")
# utils.send_sqs_message(user_key) # TODO: uncomment for online testing
return user_path
def init_user_config(user_path, user_data):
logger.info("Save data: save data to user.config...")
logger.info(user_data)
# init username.config and save config data
config_file = os.path.join(user_path, 'user.config')
with open(config_file, 'w') as fopen:
yaml.safe_dump(user_data,
fopen,
encoding='utf-8',
allow_unicode=True,
default_flow_style=False)
def show_key():
user_key = request.args['key']
if request.method == 'POST':
if 'result_button' in request.form:
user_key = request.form['result_button']
# when key is null, refresh the website
if user_key == "":
return render_template('key_demonstration.html',
key=request.args['key'])
logger.info("Retrieve result: for " + user_key)
if parseIO.is_user_key_exists(user_key):
logger.info("Retrieve result: user exists.")
return redirect(url_for('get_result', user_key=user_key))
else:
logger.error("Retrieve result: did not find the result.")
err_msg = "Job does not exist, make sure you enter the right key."
return render_template('key_demonstration.html',
key=request.args['key'])
# navbar result button
if 'navbar_button' in request.form:
logger.info("Retrieve result...")
user_key = request.form['navbar_button']
if parseIO.is_user_key_exists(user_key):
logger.info("Retrieve result: user exists.")
return redirect(url_for('get_result', user_key=user_key))
else:
logger.error("Retrieve result: did not find the result.")
err_msg = "Job does not exist, make sure you enter the right key."
return redirect(url_for('error_page', msg=err_msg))
return render_template('key_demonstration.html', key=user_key)
def get_result():
if request.method == 'POST':
# navbar result button
if 'navbar_button' in request.form:
logger.info("Retrieve result...")
user_key = request.form['navbar_button']
if parseIO.is_user_key_exists(user_key):
logger.info("Retrieve result: user exists.")
return redirect(url_for('get_result', user_key=user_key))
else:
logger.error("Retrieve result: did not find the result.")
err_msg = "Job does not exist, make sure you enter the right key."
return redirect(url_for('error_page', msg=err_msg))
else:
user_key = request.args['user_key']
user_data = parseIO.get_user_data(user_key)
logger.info('Get result: for ' + user_key)
logger.info(user_data)
results = parseIO.generate_results(user_data)
results['sample'] = False
return render_template('result_demonstration.html', results=results)
def get_user_data(user_key):
logger.info(
"Get data: get data from user.config for {}...".format(user_key))
user_path = os.path.join(PROJECT_DIR, 'usercase/' + user_key)
config_file = os.path.join(user_path, 'user.config')
# handle file not exists problem
if not os.path.exists(config_file):
return None
with open(config_file, 'r') as fopen:
user_data = yaml.load(fopen)
return user_data
def help():
if request.method == 'POST':
# navbar result button
if 'navbar_button' in request.form:
logger.info("Retrieve result...")
user_key = request.form['navbar_button']
if parseIO.is_user_key_exists(user_key):
logger.info("Retrieve result: user exists.")
return redirect(url_for('get_result', user_key=user_key))
else:
logger.error("Retrieve result: did not find the result.")
err_msg = "Job does not exist, make sure you enter the right key."
return redirect(url_for('error_page', msg=err_msg))
return render_template('help.html')
def send_mail():
usercase_dir = os.path.join(PROJECT_DIR, 'usercase')
for userkey in os.listdir(usercase_dir):
user_path = os.path.join(usercase_dir, userkey)
# iterate the user path
if not os.path.isdir(user_path):
continue
user_data = do_process.get_user_data(userkey)
if not user_data:
continue
if 'status' not in user_data: # not yet processing
continue
if 'user_email' in user_data:
if user_data['user_email'] == '':
continue
if user_data['status'] == 'Done':
logger.info('Send Email: send success e-mail to user {}'.format(userkey))
send_flag, send_msg = utils.send_user_key(user_data['user_email'], userkey, user_data['status'])
if send_flag:
logger.info('Send Email: ' + send_msg)
else:
logger.error('Send Email: ' + send_msg)
user_data['status'] = 'Sent'
do_process.init_user_config(user_path, user_data)
if user_data['status'] == 'Error':
logger.info('Job Finish: send error e-mail to user {}'.format(userkey))
send_flag, send_msg = utils.send_user_key(user_data['user_email'], userkey, user_data['status'])
if send_flag:
logger.info('Send Email: ' + send_msg)
else:
logger.error('Send Email: ' + send_msg)
user_data['status'] = 'Sent'
do_process.init_user_config(user_path, user_data)
def generate_user_key(username, jobname):
'''
generate key according to e-mail or jobname
'''
logger.info("Init project: generate key ...")
import time
tstamp = time.time() # userkey: name_timestamp
user_mail = username
# if jobname is null
if jobname == '':
if username == '':
logger.info(
"Init project: user does not input e-mail or jobname...")
username = '******'
else:
logger.info("Init project: user e-mail is {}...".format(username))
username = username.split('@')[0]
# if jobname is not null, generate key according to jobname
else:
logger.info("Init project: user jobname is {}...".format(jobname))
username = jobname.replace(' ', '') # get rid of spaces
key = username + '_' + str(tstamp)
logger.info("Init project: user key is {}...".format(key))
# send key to user's e-mail
if username != "":
logger.info("Init project: send e-mail to {} for {}".format(
user_mail, key))
send_flag, send_msg = utils.send_email(user_mail, key, 'Submit')
if send_flag:
logger.info("Init project: " + send_msg)
else:
logger.error("Init project:" + send_msg)
else:
logger.error("Init project: username is null")
return key
def index():
if request.method == 'POST':
# submit job button
if 'submit_button' in request.form:
# use user email or job name to generate unique job path
username = request.form['username']
jobname = request.form['jobname']
user_key = parseIO.generate_user_key(username, jobname)
# docker user path
user_path = parseIO.init_project_path(user_key)
# init user data
user_data = {}
user_data['user_email'] = username
user_data['user_job'] = jobname
user_data['user_key'] = user_key
user_data['user_path'] = user_path
user_data['dataType'] = request.form['dataType']
user_data['assembly'] = request.form['species']
user_data['files'] = ""
if user_data['dataType'] == "ChIP-seq":
allowed_extensions = set(['bam', 'bed'])
if user_data['dataType'] == "Geneset":
allowed_extensions = set(['txt'])
# get pasted genes and save to upload/genelist.txt
if request.form.get('uploadList', None):
gene_list = request.form['uploadList']
gene_list_file = 'genelist.txt'
gene_list_file_path = os.path.join(user_path,
'upload/' + gene_list_file)
with open(gene_list_file_path, 'w') as fopen:
for gene in gene_list:
fopen.write(gene)
user_data['files'] = gene_list_file
if request.form['dataType'] == 'ChIP-seq' or \
(request.form['dataType'] == 'Geneset' and request.form['geneType'] == 'geneFile'):
# process what user has uploaded
if 'uploadFiles' not in request.files:
flash('Please choose a file')
return redirect(request.url)
file = request.files['uploadFiles']
# if user does not select file, browser also submits an empty part without filename
if file.filename == '':
flash('One of the files does not have a legal file name.')
return redirect(request.url)
# make sure the suffix of filename in [.txt, .bam, .bed]
if file and allowed_file(file.filename, allowed_extensions):
filename = secure_filename(file.filename)
upload_path = os.path.join(user_path, 'upload')
filename_abs_path = os.path.join(upload_path, filename)
file.save(filename_abs_path)
user_data[
'files'] = filename # only save file name, since the uploaded path is always the same
parseIO.init_user_config(user_path, user_data)
parseIO.prepare_bart(user_data)
return redirect(url_for('show_key', key=user_key))
# get result button
if 'result_button' in request.form:
user_key = request.form['result_button']
# when key is null, refresh the website
if user_key == "":
return render_template('index.html')
logger.info("Retrieve result: for " + user_key)
if parseIO.is_user_key_exists(user_key):
logger.info("Retrieve result: user exists.")
return redirect(url_for('get_result', user_key=user_key))
else:
logger.error("Retrieve result: did not find the result.")
err_msg = "Job does not exist, make sure you enter the right key."
return redirect(url_for('error_page', msg=err_msg))
# navbar result button
if 'navbar_button' in request.form:
logger.info("Retrieve result...")
user_key = request.form['navbar_button']
if parseIO.is_user_key_exists(user_key):
logger.info("Retrieve result: user exists.")
return redirect(url_for('get_result', user_key=user_key))
else:
logger.error("Retrieve result: did not find the result.")
err_msg = "Job does not exist, make sure you enter the right key."
return redirect(url_for('error_page', msg=err_msg))
return render_template('index.html')
