def playlist_add(name, filename):
if not utils.verify_file(filename):
return False
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as client:
# connecting client to server
client.connect(ADDR)
# will sending operation flag
client.send(flags.CREATE_OBJ)
# sending name
client.send(utils.padded_message(name.encode('utf-8')))
# sending file name
client.send(
utils.padded_message(os.path.basename(filename).encode('utf-8')))
# sending file
file = open(filename, 'rb')
data = file.read(settings.EXCHANGE_SIZE)
while data:
client.send(data)
data = file.read(settings.EXCHANGE_SIZE)
file.close()
return True
def generate_html():
calenders = ['July-2018', 'August-2018', 'September-2018']
file_path = self.app.pargs.file_path
if verify_file(file_path):
self.app.log.info('Parsing spreadsheet %s' % file_path)
show_calender_prompt(calenders)
else:
self.app.log.error('%s does not exist' % file_path)
def handle_subtitles():
u"""
Handler function for root url.
It shows the page in with an empty form in the case of a 'GET' request and.
In the case of a 'POST' request, returns the repaired subtitles file when the form validates and successfully
repairs it. When any step fails, returns the page with the form containing the error messages.
"""
if request.method == "GET":
# in the case of a 'GET' request, gets an instance of the form and renders the page with it
subtitle_form = SubtitleUploadForm()
return render_template('app.html', subtitle_form=subtitle_form)
elif request.method == "POST":
# in the case of a 'POST' request, gets an instance of the form populated with the request parameters on the
# request and initializes auxiliary variables as None
subtitle_form = SubtitleUploadForm(request.form)
error = None
filename = None
result_content = None
if subtitle_form.validate():
# if the form contains valid data
try:
# reads uploaded file content
file_data = request.files[subtitle_form.subtitle_file.name].read()
# gets its filename
filename = request.files[subtitle_form.subtitle_file.name].filename
# and gets from the form the value in seconds to add or subtract from the time pattern matches
seconds = int(request.form['adjustment'])
if verify_file(file_data, filename):
# if the file content and its extension are compatible with the expectations, calls function to
# proceed with the file repairing and stores its results (repaired string with file content)
result_content = fix_file(file_data, seconds)
else:
# otherwise, creates the error message to be shown to the user
error = "File is not in the expected format and/or does not contain expected content."
except (AttributeError, KeyError):
# with any error occurs, composes another erro message to be shown to the user
error = "Something went wrong on your submission, please again to submit a valid file."
if error:
# if there is an error message, add it to the form errors and render the page with the form containing
# the errors to be shown
subtitle_form.errors['subtitle_file'] = error
return render_template('app.html', subtitle_form=subtitle_form)
else:
# otherwise, returns the repaired file as an attachment
response = make_response(result_content)
response.headers["Content-Disposition"] = "attachment; filename={0}".format("fixed_" + filename)
return response
# renders the page again with the form erros if form does not validate
return render_template('app.html', subtitle_form=subtitle_form)
def _run(starting_from_here=False):
fasta_to_blast = new_good_proteomes or config.good_proteins
blast_out = config.blast_out + "_2" if new_good_proteomes else config.blast_out
res = 10
if not on_cluster:
# threads
res = _blast(fasta_to_blast, blast_out, threads=max_jobs)
else:
qsub = which("qsub")
if not qsub:
log.warn("No qsub in system: running multuthreaded")
res = _blast(fasta_to_blast, blast_out, threads=max_jobs)
else:
total_seqs = sum(1 for _ in SeqIO.parse(fasta_to_blast, "fasta"))
num_seqs_for_one_job = max(500, total_seqs / max_jobs)
num_jobs = total_seqs / num_seqs_for_one_job or 1
# num_seqs_for_one_job = total_seqs/2 # DEBUG
# num_jobs = 2 # DEBUG
if num_jobs == 1:
# one single threaded run
res = _blast(fasta_to_blast, blast_out, threads=1)
else:
# jobs
timestamp = str(datetime.now()).replace("-", "_").replace(":", "_").replace(" ", "_")
log.info("Splitting data for " + str(num_jobs) + " cluster jobs.")
class BlastJob:
def __init__(self, i):
self.i = i
self.job_name = workflow_id + "_" + timestamp + "_" + str(i)
self.prot_fpath = join(config.intermediate_dir, "proteins_" + str(i) + ".fasta")
self.out_fpath = join(config.intermediate_dir, "blasted_part_" + str(i) + ".tsv")
self.log_fpath = join(config.intermediate_dir, "run_blast_" + str(i) + ".log")
self.runner_fpath = join(config.intermediate_dir, "run_blast_" + str(i) + ".sh")
cmd = (
"blastp "
+ " ".join(map(str, _blast_basic_params))
+ " -query "
+ realpath(self.prot_fpath)
+ " -out "
+ realpath(self.out_fpath)
)
with open(self.runner_fpath, "w") as f:
f.write("#!/bin/bash\n")
f.write(". /etc/profile.d/modules.sh\n")
f.write("module load blast\n")
f.write(cmd + "\n")
f.write("date\n")
def submit(self):
cmdl = "-pe pe_smp 1 -S /bin/bash -cwd -j y -q batch.q -N {0} -o {1} " "{2}".format(
self.job_name, realpath(self.log_fpath), realpath(self.runner_fpath)
)
# cmdl = '-pe pe_smp 1 -S /bin/bash -cwd -j y -o {0} -q batch.q ' \
# '{1}'.format(self.log_fpath, self.runner_fpath)
log.debug("submitting job " + str(self.i))
res = cmdline("qsub", parameters=cmdl.split())()
log.debug("submitted, res = " + str(res))
log.info("")
return res
blast_jobs = []
i, i_recs = 1, []
for rec in SeqIO.parse(fasta_to_blast, "fasta"):
i_recs.append(rec)
if len(i_recs) > num_seqs_for_one_job:
blast_job = BlastJob(i)
blast_jobs.append(blast_job)
SeqIO.write(i_recs, blast_job.prot_fpath, "fasta")
i, i_recs = i + 1, []
if i_recs:
blast_job = BlastJob(i)
blast_jobs.append(blast_job)
SeqIO.write(i_recs, blast_job.prot_fpath, "fasta")
for bj in blast_jobs:
res = bj.submit()
if res != 0:
log.info("qsub returned exit code " + str(res))
return res
results_script_fpath = join(config.intermediate_dir, "collect_blasted" + ".sh")
collect_log_fpath = join(config.intermediate_dir, "collect_blasted.log")
if isfile(collect_log_fpath):
os.remove(collect_log_fpath)
with open(results_script_fpath, "w") as f:
f.write("#!/bin/bash\n")
f.write("touch " + collect_log_fpath + "\n")
cmdl = "-hold_jid {0} -S /bin/bash -cwd -j y -q batch.q {1}".format(
",".join(j.job_name for j in blast_jobs), realpath(results_script_fpath)
)
log.debug("wating for jobs...")
res = cmdline("qsub", parameters=cmdl.split())()
if res != 0:
return res
log.info("Waiting for blast jobs to finish...")
while not isfile(collect_log_fpath):
sleep(3)
log.info("All blast finished, proceeding.")
# cat_params = ''
ok = True
for bj in blast_jobs:
if not verify_file(bj.out_fpath):
ok = False
else:
log.debug(bj.out_fpath + " exists, ok")
# cat_params += ' ' + bj.prot_fpath
if not ok:
return 3
# res = cmdline('cat',
# parameters=cat_params,
# stdout=blast_out)
with open(blast_out, "w") as out:
for bj in blast_jobs:
with open(bj.out_fpath) as bjout:
out.write(bjout.read())
if not verify_file(blast_out):
log.debug(blast_out + " not exist, return 4")
return 4
print res
if res != 0:
return res
if new_good_proteomes:
log.info(" Appending " + config.blast_out + "_2 to " + config.blast_out)
with open(config.blast_out, "a") as b_out:
with open(config.blast_out + "_2") as b_out_2:
b_out.write(b_out_2.read())
return res
def _run(starting_from_here=False):
fasta_to_blast = new_good_proteomes or config.good_proteins
blast_out = config.blast_out + '_2' if new_good_proteomes else config.blast_out
res = 10
if not on_cluster:
# threads
res = _blast(fasta_to_blast, blast_out, threads=max_jobs)
else:
qsub = which('qsub')
if not qsub:
log.warn('No qsub in system: running multuthreaded')
res = _blast(fasta_to_blast, blast_out, threads=max_jobs)
else:
total_seqs = sum(1 for _ in SeqIO.parse(fasta_to_blast, 'fasta'))
num_seqs_for_one_job = max(500, total_seqs/max_jobs)
num_jobs = total_seqs/num_seqs_for_one_job or 1
# num_seqs_for_one_job = total_seqs/2 # DEBUG
# num_jobs = 2 # DEBUG
if num_jobs == 1:
# one single threaded run
res = _blast(fasta_to_blast, blast_out, threads=1)
else:
# jobs
timestamp = str(datetime.now()).replace('-', '_').replace(':', '_').replace(' ', '_')
log.info('Splitting data for ' + str(num_jobs) + ' cluster jobs.')
class BlastJob:
def __init__(self, i):
self.i = i
self.job_name = workflow_id + '_' + timestamp + '_' + str(i)
self.prot_fpath = join(config.intermediate_dir, 'proteins_' + str(i) + '.fasta')
self.out_fpath = join(config.intermediate_dir, 'blasted_part_' + str(i) + '.tsv')
self.log_fpath = join(config.intermediate_dir, 'run_blast_' + str(i) + '.log')
self.runner_fpath = join(config.intermediate_dir, 'run_blast_' + str(i) + '.sh')
cmd = ('blastp ' +
' '.join(map(str, _blast_basic_params)) +
' -query ' + realpath(self.prot_fpath) +
' -out ' + realpath(self.out_fpath))
with open(self.runner_fpath, 'w') as f:
f.write('#!/bin/bash\n')
f.write('. /etc/profile.d/modules.sh\n')
f.write('module load blast\n')
f.write(cmd + '\n')
f.write('date\n')
def submit(self):
cmdl = '-pe pe_smp 1 -S /bin/bash -cwd -j y -q batch.q -N {0} -o {1} ' \
'{2}'.format(self.job_name, realpath(self.log_fpath), realpath(self.runner_fpath))
# cmdl = '-pe pe_smp 1 -S /bin/bash -cwd -j y -o {0} -q batch.q ' \
# '{1}'.format(self.log_fpath, self.runner_fpath)
log.debug('submitting job ' + str(self.i))
res = cmdline('qsub', parameters=cmdl.split())()
log.debug('submitted, res = ' + str(res))
log.info('')
return res
blast_jobs = []
i, i_recs = 1, []
for rec in SeqIO.parse(fasta_to_blast, 'fasta'):
i_recs.append(rec)
if len(i_recs) > num_seqs_for_one_job:
blast_job = BlastJob(i)
blast_jobs.append(blast_job)
SeqIO.write(i_recs, blast_job.prot_fpath, 'fasta')
i, i_recs = i + 1, []
if i_recs:
blast_job = BlastJob(i)
blast_jobs.append(blast_job)
SeqIO.write(i_recs, blast_job.prot_fpath, 'fasta')
for bj in blast_jobs:
res = bj.submit()
if res != 0:
log.info('qsub returned exit code ' + str(res))
return res
results_script_fpath = join(config.intermediate_dir, 'collect_blasted' + '.sh')
collect_log_fpath = join(config.intermediate_dir, 'collect_blasted.log')
if isfile(collect_log_fpath):
os.remove(collect_log_fpath)
with open(results_script_fpath, 'w') as f:
f.write('#!/bin/bash\n')
f.write('touch ' + collect_log_fpath + '\n')
cmdl = '-hold_jid {0} -S /bin/bash -cwd -j y -q batch.q {1}'.format(
','.join(j.job_name for j in blast_jobs), realpath(results_script_fpath))
log.debug('wating for jobs...')
res = cmdline('qsub', parameters=cmdl.split())()
if res != 0:
return res
log.info('Waiting for blast jobs to finish...')
while not isfile(collect_log_fpath):
sleep(3)
log.info('All blast finished, proceeding.')
# cat_params = ''
ok = True
for bj in blast_jobs:
if not verify_file(bj.out_fpath):
ok = False
else:
log.debug(bj.out_fpath + ' exists, ok')
# cat_params += ' ' + bj.prot_fpath
if not ok:
return 3
# res = cmdline('cat',
# parameters=cat_params,
# stdout=blast_out)
with open(blast_out, 'w') as out:
for bj in blast_jobs:
with open(bj.out_fpath) as bjout:
out.write(bjout.read())
if not verify_file(blast_out):
log.debug(blast_out + ' not exist, return 4')
return 4
print res
if res != 0:
return res
if new_good_proteomes:
log.info(' Appending ' + config.blast_out + '_2 to ' + config.blast_out)
with open(config.blast_out, 'a') as b_out:
with open(config.blast_out + '_2') as b_out_2:
b_out.write(b_out_2.read())
return res