def test_batch_unicode_email(self):
"""
Submit a batch form with non-ASCII unicode characters in the email
address.
"""
file = '\n'.join(
['AB026906.1:c.274G>T', 'AL449423.14(CDKN2A_v002):c.5_400del'])
expected = [['AB026906.1:c.274G>T', 'OK'],
['AL449423.14(CDKN2A_v002):c.5_400del', 'OK']]
data = {
'job_type': 'syntax-checker',
'email': 'pe\[email protected]',
'file': (BytesIO(file.encode('utf-8')), 'test.txt')
}
r = self.app.post('/batch-jobs', data=data)
progress_url = '/' + r.location.split('/')[-1]
assert models.BatchJob.query.first().email == 'pe\[email protected]'
scheduler = Scheduler.Scheduler()
scheduler.process()
r = self.app.get(progress_url)
dom = lxml.html.fromstring(r.data)
result_url = dom.cssselect('#ifnot_items_left a')[0].attrib['href']
r = self.app.get(result_url)
assert 'text/plain' in r.headers['Content-Type']
result = r.get_data(as_text=True).strip().split('\n')[1:]
assert expected == [line.split('\t') for line in result]
def _batch_job(batch_file, expected, job_type, argument=None):
file_instance = File.File(output.Output('test'))
scheduler = Scheduler.Scheduler()
job, columns = file_instance.parseBatchFile(batch_file)
result_id = scheduler.addJob('*****@*****.**',
job,
columns,
job_type,
argument=argument)
batch_job = BatchJob.query.filter_by(result_id=result_id).one()
left = batch_job.batch_queue_items.count()
assert left == len(expected)
scheduler.process()
left = batch_job.batch_queue_items.count()
assert left == 0
filename = 'batch-job-%s.txt' % result_id
result = io.open(os.path.join(settings.CACHE_DIR, filename),
encoding='utf-8')
next(result) # Header.
assert expected == [line.strip().split('\t') for line in result]
def test_batchjob_unicode(api):
"""
Submit a batch job with non-ASCII unicode characters in the input
file.
"""
variants = ['\u2026AB026906.1:c.274G>T',
'\u2026AL449423.14(CDKN2A_v002):c.5_400del']
expected = [['\u2026AB026906.1:c.274G>T',
'(grammar): Expected W:(0123...) (at char 0), (line:1, col:1)'],
['\u2026AL449423.14(CDKN2A_v002):c.5_400del',
'(grammar): Expected W:(0123...) (at char 0), (line:1, col:1)']]
data = '\n'.join(variants) + '\n' # .encode('base64')
result = api('submitBatchJob', data.encode('utf-8'), 'SyntaxChecker')
job_id = unicode(result)
result = api('monitorBatchJob', job_id)
assert int(result) == len(variants)
scheduler = Scheduler.Scheduler()
scheduler.process()
result = api('monitorBatchJob', job_id)
assert int(result) == 0
result = api('getBatchJob', job_id)
result = result.decode('base64').decode('utf-8').strip().split('\n')[1:]
assert expected == [line.split('\t') for line in result]
def test_batchjob(api):
"""
Submit a batch job.
"""
variants = [
'AB026906.1(SDHD):g.7872G>T', 'NM_003002.2:c.3_4insG',
'AL449423.14(CDKN2A_v002):c.5_400del'
]
data = '\n'.join(variants) + '\n' #.encode('base64')
result = api('submitBatchJob', data.encode('utf-8'), 'NameChecker')
job_id = unicode(result)
result = api('monitorBatchJob', job_id)
assert int(result) == len(variants)
scheduler = Scheduler.Scheduler()
scheduler.process()
result = api('monitorBatchJob', job_id)
assert int(result) == 0
result = api('getBatchJob', job_id)
assert len(
result.decode('base64').strip().split('\n')) - 1 == len(variants)
def _batch(self,
job_type='name-checker',
assembly_name_or_alias=None,
file="",
size=0,
header='',
lines=None):
"""
Submit a batch form.
@kwarg batch_type: Type of batch job to test. One of name-checker,
syntax-checker, position-converter.
@kwarg argument: Optional extra argument for the batch job.
@kwarg file: String with variants to use as input for the batch job.
@kwarg size: Number of variants in input.
@kwarg header: Message that must be found in the batch job result.
@kwarg lines: Number of result rows expected.
@return: The batch result document.
@rtype: string
"""
data = {
'job_type': job_type,
'email': '*****@*****.**',
'file': (BytesIO(file.encode('utf-8')), 'test.txt')
}
if assembly_name_or_alias is not None:
data['assembly_name_or_alias'] = assembly_name_or_alias
r = self.app.post('/batch-jobs', data=data)
progress_url = '/' + r.location.split('/')[-1]
r = self.app.get(progress_url)
assert '<div id="if_items_left">' in r.data
assert '<div id="ifnot_items_left" style="display:none">' in r.data
assert ('<span id="items_left">%d</span>' % size) in r.data
scheduler = Scheduler.Scheduler()
scheduler.process()
r = self.app.get(progress_url)
assert '<div id="if_items_left" style="display:none">' in r.data
assert '<div id="ifnot_items_left">' in r.data
dom = lxml.html.fromstring(r.data)
result_url = dom.cssselect('#ifnot_items_left a')[0].attrib['href']
if not lines:
lines = size
r = self.app.get(result_url)
assert 'text/plain' in r.headers['Content-Type']
assert header in r.data
assert len(r.data.strip().split('\n')) - 1 == lines
return r.data
def test_batchjob_newlines_windows(api):
"""
Submit a batch job with Windows newlines.
"""
variants = ['AB026906.1(SDHD):g.7872G>T',
'NM_003002.2:c.3_4insG',
'AL449423.14(CDKN2A_v002):c.5_400del']
data = '\r\n'.join(variants) + '\r\n'
result = api('submitBatchJob', data.encode('utf-8'), 'SyntaxChecker')
job_id = unicode(result)
result = api('monitorBatchJob', job_id)
assert int(result) == len(variants)
scheduler = Scheduler.Scheduler()
scheduler.process()
result = api('monitorBatchJob', job_id)
assert int(result) == 0
def batch_jobs_submit():
"""
Run batch jobs and render batch checker HTML form. The batch jobs are
added to the database by the scheduler and ran by the BatchChecker
daemon.
"""
job_type = request.form.get('job_type')
email = request.form.get('email')
# Note that this is always a seekable binary file object.
batch_file = request.files.get('file')
assemblies = Assembly.query \
.order_by(*Assembly.order_by_criteria) \
.all()
assembly_name_or_alias = request.form.get('assembly_name_or_alias',
settings.DEFAULT_ASSEMBLY)
errors = []
if not email:
email = '{}@website.mutalyzer'.format(request.remote_addr)
if job_type not in BATCH_JOB_TYPES:
errors.append('Invalid batch job type.')
if not file:
errors.append('Please select a local file for upload.')
if job_type == 'position-converter':
try:
Assembly.by_name_or_alias(assembly_name_or_alias)
except NoResultFound:
errors.append('Not a valid assembly.')
argument = assembly_name_or_alias
else:
argument = None
output = Output(__file__)
if not errors:
stats.increment_counter('batch-job/website')
scheduler = Scheduler.Scheduler()
file_instance = File.File(output)
job, columns = file_instance.parseBatchFile(batch_file)
if job is None:
errors.append('Could not parse input file, please check your '
'file format.')
else:
result_id = scheduler.addJob(email, job, columns, job_type,
argument=argument)
# Todo: We now assume that the job was not scheduled if there are
# messages, which is probably not correct.
if not output.getMessages():
return redirect(url_for('.batch_job_progress',
result_id=result_id))
for error in errors:
output.addMessage(__file__, 3, 'EBATCHJOB', error)
messages = map(util.message_info, output.getMessages())
return render_template('batch-jobs.html',
assemblies=assemblies,
assembly_name_or_alias=assembly_name_or_alias,
job_type=job_type,
max_file_size=settings.MAX_FILE_SIZE // 1048576,
messages=messages)