def pop_batch_queue_item(batch_job):
"""
Get the next batch queue item for the given batch job. Return its fields
as a tuple `item`, `flags` and remove it from the database.
If no batch queue item could be found for this batch job, return `None`.
.. note:: Originally, finding the next batch queue item was done using a
more complicated query::
SELECT QueueID, Input, Flags
FROM BatchQueue
WHERE QueueID = (
SELECT MIN(QueueID)
FROM BatchQueue
GROUP BY JobID
HAVING JobID = {batch_job.id}
);
However, I couldn't see any significant performance difference in my
latest benchmarks, so we stick with the more obvious query for now.
"""
batch_queue_item = BatchQueueItem.query \
.filter_by(batch_job=batch_job) \
.order_by(BatchQueueItem.id.asc()) \
.first()
if batch_queue_item is None:
return None
item, flags = batch_queue_item.item, batch_queue_item.flags
session.delete(batch_queue_item)
session.commit()
return item, flags
def pop_batch_queue_item(batch_job):
"""
Get the next batch queue item for the given batch job. Return its fields
as a tuple `item`, `flags` and remove it from the database.
If no batch queue item could be found for this batch job, return `None`.
.. note:: Originally, finding the next batch queue item was done using a
more complicated query::
SELECT QueueID, Input, Flags
FROM BatchQueue
WHERE QueueID = (
SELECT MIN(QueueID)
FROM BatchQueue
GROUP BY JobID
HAVING JobID = {batch_job.id}
);
However, I couldn't see any significant performance difference in my
latest benchmarks, so we stick with the more obvious query for now.
"""
batch_queue_item = BatchQueueItem.query \
.filter_by(batch_job=batch_job) \
.order_by(BatchQueueItem.id.asc()) \
.first()
if batch_queue_item is None:
return None
item, flags = batch_queue_item.item, batch_queue_item.flags
session.delete(batch_queue_item)
session.commit()
return item, flags
def process(self):
"""
Start the mutalyzer Batch Processing. This method retrieves all jobs
jobs from the database and processes them in a roundrobin fashion.
After each round, the process checks if new jobs are added during the
last processing round and repeats. This continue until no jobs are
left to process.
If during this process the {stop} method is called, the current
job item is completed and we return.
This method uses two database tables, BatchJob and BatchQueue.
The jobList is an array of tuples with three elements
- jobID ; The ID of the job
- jobType ; The type of the job
- argument1 ; Currently only used for the ConversionChecker
to send the build version.
If the jobList is not empty, the method will iterate once over the
list and fetch the first entry of a job from the database table
BatchQueue. This request returns both the input for the batch and
the flags for the job.
#Flags
A job can be flagged in three ways:
- A ; Altered - this means that the input is altered
before execution. This could be the case if an
entry uses an accession number without a version.
If a version is retrieved from the NCBI, all
further occurences of that accession will be
replaced by the accession with version number.
- S ; Skipped - this means that this batchentry will be
skipped by the batchprocess. This could be the
case if the user made a mistake that could not be
auto fixed and henceforth all occurences of the
mistake will be skipped.
- C ; Continue - this means the input does not end the
current row, so no new row in the output should
be started.
A Flag consists of either an A, S or C followed by a digit, which
refers to the reason of alteration / skip.
"""
while not self.stopped():
batch_jobs = BatchJob.query
if batch_jobs.count() == 0:
break
for batch_job in batch_jobs:
if self.stopped():
break
batch_queue_item = queries.pop_batch_queue_item(batch_job)
if batch_queue_item is not None:
item, flags = batch_queue_item
if batch_job.job_type == 'name-checker':
self._processNameBatch(batch_job, item, flags)
elif batch_job.job_type == 'syntax-checker':
self._processSyntaxCheck(batch_job, item, flags)
elif batch_job.job_type == 'position-converter':
self._processConversion(batch_job, item, flags)
elif batch_job.job_type == 'snp-converter':
self._processSNP(batch_job, item, flags)
else:
# Unknown job type, should never happen.
# Todo: Log some screaming message.
pass
else:
print ('Job %s finished, email %s file %s'
% (batch_job.id, batch_job.email, batch_job.id))
self.__sendMail(batch_job.email, batch_job.download_url)
session.delete(batch_job)
session.commit()
def process(self):
"""
Start the mutalyzer Batch Processing. This method retrieves all jobs
jobs from the database and processes them in a roundrobin fashion.
After each round, the process checks if new jobs are added during the
last processing round and repeats. This continue until no jobs are
left to process.
If during this process the {stop} method is called, the current
job item is completed and we return.
This method uses two database tables, BatchJob and BatchQueue.
The jobList is an array of tuples with three elements
- jobID ; The ID of the job
- jobType ; The type of the job
- argument1 ; Currently only used for the ConversionChecker
to send the build version.
If the jobList is not empty, the method will iterate once over the
list and fetch the first entry of a job from the database table
BatchQueue. This request returns both the input for the batch and
the flags for the job.
#Flags
A job can be flagged in three ways:
- A ; Altered - this means that the input is altered
before execution. This could be the case if an
entry uses an accession number without a version.
If a version is retrieved from the NCBI, all
further occurences of that accession will be
replaced by the accession with version number.
- S ; Skipped - this means that this batchentry will be
skipped by the batchprocess. This could be the
case if the user made a mistake that could not be
auto fixed and henceforth all occurences of the
mistake will be skipped.
- C ; Continue - this means the input does not end the
current row, so no new row in the output should
be started.
A Flag consists of either an A, S or C followed by a digit, which
refers to the reason of alteration / skip.
"""
while not self.stopped():
batch_jobs = BatchJob.query
if batch_jobs.count() == 0:
break
for batch_job in batch_jobs:
if self.stopped():
break
batch_queue_item = queries.pop_batch_queue_item(batch_job)
if batch_queue_item is not None:
item, flags = batch_queue_item
if batch_job.job_type == 'name-checker':
self._processNameBatch(batch_job, item, flags)
elif batch_job.job_type == 'syntax-checker':
self._processSyntaxCheck(batch_job, item, flags)
elif batch_job.job_type == 'position-converter':
self._processConversion(batch_job, item, flags)
elif batch_job.job_type == 'snp-converter':
self._processSNP(batch_job, item, flags)
else:
# Unknown job type, should never happen.
# Todo: Log some screaming message.
pass
else:
print('Job %s finished, email %s file %s' %
(batch_job.id, batch_job.email, batch_job.id))
self.__sendMail(batch_job.email, batch_job.download_url)
session.delete(batch_job)
session.commit()