def _batch_job(batch_file, expected, job_type, argument=None): file_instance = File.File(output.Output('test')) scheduler = Scheduler.Scheduler() job, columns = file_instance.parseBatchFile(batch_file) result_id = scheduler.addJob('*****@*****.**', job, columns, job_type, argument=argument) batch_job = BatchJob.query.filter_by(result_id=result_id).one() left = batch_job.batch_queue_items.count() assert left == len(expected) scheduler.process() left = batch_job.batch_queue_items.count() assert left == 0 filename = 'batch-job-%s.txt' % result_id result = io.open(os.path.join(settings.CACHE_DIR, filename), encoding='utf-8') next(result) # Header. assert expected == [line.strip().split('\t') for line in result]
def test_invalid_zip_file(): """ Random zip file input for batch job (invalid). """ path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data', 'image.zip') batch_file = open(path, 'rb') file_instance = File.File(output.Output('test')) job, columns = file_instance.parseBatchFile(batch_file) assert job is None
def batch_jobs_submit(): """ Run batch jobs and render batch checker HTML form. The batch jobs are added to the database by the scheduler and ran by the BatchChecker daemon. """ job_type = request.form.get('job_type') email = request.form.get('email') # Note that this is always a seekable binary file object. batch_file = request.files.get('file') assemblies = Assembly.query \ .order_by(*Assembly.order_by_criteria) \ .all() assembly_name_or_alias = request.form.get('assembly_name_or_alias', settings.DEFAULT_ASSEMBLY) errors = [] if not email: email = '{}@website.mutalyzer'.format(request.remote_addr) if job_type not in BATCH_JOB_TYPES: errors.append('Invalid batch job type.') if not file: errors.append('Please select a local file for upload.') if job_type == 'position-converter': try: Assembly.by_name_or_alias(assembly_name_or_alias) except NoResultFound: errors.append('Not a valid assembly.') argument = assembly_name_or_alias else: argument = None output = Output(__file__) if not errors: stats.increment_counter('batch-job/website') scheduler = Scheduler.Scheduler() file_instance = File.File(output) job, columns = file_instance.parseBatchFile(batch_file) if job is None: errors.append('Could not parse input file, please check your ' 'file format.') else: result_id = scheduler.addJob(email, job, columns, job_type, argument=argument) # Todo: We now assume that the job was not scheduled if there are # messages, which is probably not correct. if not output.getMessages(): return redirect(url_for('.batch_job_progress', result_id=result_id)) for error in errors: output.addMessage(__file__, 3, 'EBATCHJOB', error) messages = map(util.message_info, output.getMessages()) return render_template('batch-jobs.html', assemblies=assemblies, assembly_name_or_alias=assembly_name_or_alias, job_type=job_type, max_file_size=settings.MAX_FILE_SIZE // 1048576, messages=messages)