def shmir_from_sirna_score(seq1, seq2, shift_left, shift_right):
"""Main function takes string input and returns the best results depending
on scoring. Single result include sh-miR sequence,
score and link to 2D structure from mfold program
Args:
input_str(str): Input string contains one or two sequences.
Returns:
List of sh-miR(s) sorted by score.
"""
original_frames = frames_by_scaffold('all')
frames = adjusted_frames(seq1, seq2,
shift_left, shift_right,
deepcopy(original_frames))
shmirs = [frame.template() for frame in frames]
# folding via mfold
with allow_join_result():
foldings = group(
fold.s(
shmir
).set(queue="subtasks") for shmir in shmirs
).apply_async().get()
# scoring results
with allow_join_result():
scores = group(
score_from_sirna.s(
frame,
original,
folding['ss']
).set(queue="subtasks")
for frame, original, folding in izip(frames, original_frames, foldings)
).apply_async().get()
full_reference = [
{
'score': score,
'shmir': shmir,
'scaffold_name': frame.name,
'pdf_reference': folding['path_id'],
'sequences': (frame.siRNA1, frame.siRNA2),
}
for score, shmir, frame, folding in izip(scores, shmirs, frames, foldings)
if score['all'] > 60
]
return sorted(
full_reference,
key=lambda elem: elem['score']['all'],
reverse=True
)[:SIRNA_RESULT_LIMIT]
def wait_for_ping(ping_task_timeout=10.0):
"""
Wait for the celery worker to respond to a ping.
This should ensure that any other running tasks are done.
"""
with allow_join_result():
assert ping.delay().get(timeout=ping_task_timeout) == 'pong'
def refresh_vulns_product_versions_task(self):
logger.debug("Entering 'refresh_vulns_product_versions_task'")
vulns = Vuln.objects.all().order_by('-id')
job = group([refresh_vuln_product_versions_task.s(vuln.id) for vuln in vulns])
result = job.apply_async()
with allow_join_result():
return result.get()
def start_worker(app: Celery,
concurrency: int = None,
pool: str = None,
loglevel: Union[str, int] = None,
logfile: str = None,
perform_ping_check: bool = True,
ping_task_timeout: float = 10.0,
**kwargs: Any) -> Iterable:
"""Start embedded worker.
Yields:
celery.app.worker.Worker: worker instance.
"""
worker_starting.send(sender=app)
with _start_worker_thread(app,
concurrency=concurrency,
pool=pool,
loglevel=loglevel,
logfile=logfile,
**kwargs) as _worker:
if perform_ping_check:
with allow_join_result():
assert ping.delay().get(timeout=ping_task_timeout) == 'pong'
yield _worker
worker_stopped.send(sender=app, worker=worker)
def trigger_callback(app, callback, group_result):
"""Add the callback to the queue or mark the callback as failed
Implementation borrowed from `celery.app.builtins.unlock_chord`
"""
j = (
group_result.join_native
if group_result.supports_native_join
else group_result.join
)
try:
with allow_join_result():
ret = j(timeout=app.conf.result_chord_join_timeout, propagate=True)
except Exception as exc: # pylint: disable=broad-except
try:
culprit = next(group_result._failed_join_report())
reason = "Dependency {0.id} raised {1!r}".format(culprit, exc)
except StopIteration:
reason = repr(exc)
logger.exception("Chord %r raised: %r", group_result.id, exc)
app.backend.chord_error_from_stack(callback, ChordError(reason))
else:
try:
callback.delay(ret)
except Exception as exc: # pylint: disable=broad-except
logger.exception("Chord %r raised: %r", group_result.id, exc)
app.backend.chord_error_from_stack(
callback, exc=ChordError("Callback error: {0!r}".format(exc))
)
def recurse_pipe():
# TODO: refactor this to update once every night as a background task
# We use a mem-cache lock to avoid duplicate pipes.
# lock_string = str(Coin.objects.get(id=1).last_updated)
# lock_utf8 = lock_string.encode('utf-8')
# lock_id = md5(lock_utf8).hexdigest()
# if not memcache_lock(lock_id, 'recurse-pipe'):
# print('Exiting recurse_pipe', lock_string)
# else:
try:
with allow_join_result():
timestamp_last = pipe()
reference_timestamp = datetime.fromtimestamp(timestamp_last)
eta = reference_timestamp + timedelta(seconds=100)
# recurse_pipe.apply_async(eta=eta)
if eta < datetime.now():
# recurse_pipe.si().apply_async()
recurse_pipe.apply_async(countdown=30)
else:
recurse_pipe.apply_async(eta=eta)
except Exception as e:
print(e)
def start_worker(app,
concurrency=1,
pool='solo',
loglevel=WORKER_LOGLEVEL,
logfile=None,
perform_ping_check=True,
ping_task_timeout=10.0,
**kwargs):
# type: (Celery, int, str, Union[str, int],
# str, bool, float, **Any) -> # Iterable
"""Start embedded worker.
Yields:
celery.app.worker.Worker: worker instance.
"""
test_worker_starting.send(sender=app)
with _start_worker_thread(app,
concurrency=concurrency,
pool=pool,
loglevel=loglevel,
logfile=logfile,
**kwargs) as worker:
if perform_ping_check:
from .tasks import ping
with allow_join_result():
assert ping.delay().get(timeout=ping_task_timeout) == 'pong'
yield worker
test_worker_stopped.send(sender=app, worker=worker)
def deploy_failed(task_uuid, driverCls, provider, identity, instance_id,
**celery_task_args):
from core.models.instance import Instance
from core.email import send_deploy_failed_email
try:
logger.debug("deploy_failed task started at %s." % datetime.now())
logger.info("task_uuid=%s" % task_uuid)
result = app.AsyncResult(task_uuid)
with allow_join_result():
exc = result.get(propagate=False)
err_str = "DEPLOYERROR::%s" % (result.traceback, )
logger.error(err_str)
driver = get_driver(driverCls, provider, identity)
instance = driver.get_instance(instance_id)
update_instance_metadata(driver,
instance,
data={'tmp_status': 'deploy_error'},
replace=False)
#Send deploy email
core_instance = Instance.objects.get(provider_alias=instance_id)
send_deploy_failed_email(core_instance, err_str)
logger.debug("deploy_failed task finished at %s." % datetime.now())
except Exception as exc:
logger.warn(exc)
deploy_failed.retry(exc=exc)
def mount_failed(task_uuid,
driverCls,
provider,
identity,
volume_id,
unmount=False,
**celery_task_args):
try:
logger.debug("mount_failed task started at %s." % datetime.now())
logger.info("task_uuid=%s" % task_uuid)
result = app.AsyncResult(task_uuid)
with allow_join_result():
exc = result.get(propagate=False)
err_str = "Mount Error Traceback:%s" % (result.traceback, )
logger.error(err_str)
driver = get_driver(driverCls, provider, identity)
volume = driver.get_volume(volume_id)
if unmount:
tmp_status = 'umount_error'
else:
tmp_status = 'mount_error'
return volume_service.update_volume_metadata(
driver, volume, metadata={'tmp_status': tmp_status})
logger.debug("mount_failed task finished at %s." % datetime.now())
except Exception as exc:
logger.warn(exc)
mount_failed.retry(exc=exc)
def start_worker(
app, # type: Celery
concurrency=1, # type: int
pool="solo", # type: str
loglevel=WORKER_LOGLEVEL, # type: Union[str, int]
logfile=None, # type: str
perform_ping_check=True, # type: bool
ping_task_timeout=10.0, # type: float
**kwargs # type: Any
):
# type: (...) -> Iterable
"""Start embedded worker.
Yields:
celery.app.worker.Worker: worker instance.
"""
test_worker_starting.send(sender=app)
with _start_worker_thread(
app,
concurrency=concurrency,
pool=pool,
loglevel=loglevel,
logfile=logfile,
perform_ping_check=perform_ping_check,
**kwargs
) as worker:
if perform_ping_check:
from .tasks import ping
with allow_join_result():
assert ping.delay().get(timeout=ping_task_timeout) == "pong"
yield worker
test_worker_stopped.send(sender=app, worker=worker)
def start_worker(app,
concurrency=1,
pool='solo',
loglevel=WORKER_LOGLEVEL,
logfile=None,
perform_ping_check=True,
ping_task_timeout=10.0,
**kwargs):
# type: (Celery, int, str, Union[str, int],
# str, bool, float, **Any) -> # Iterable
"""Start embedded worker.
Yields:
celery.app.worker.Worker: worker instance.
"""
test_worker_starting.send(sender=app)
with _start_worker_thread(app,
concurrency=concurrency,
pool=pool,
loglevel=loglevel,
logfile=logfile,
**kwargs) as worker:
if perform_ping_check:
from .tasks import ping
with allow_join_result():
assert ping.delay().get(timeout=ping_task_timeout) == 'pong'
yield worker
test_worker_stopped.send(sender=app, worker=worker)
def get_info(self):
if self.result.ready():
success = self.result.successful()
with allow_join_result():
return {
'complete':
True,
'success':
success,
'progress':
_get_completed_progress(),
'result':
self.result.get(self.task_id)
if success else self.result.result.__str__(),
}
elif self.result.state == PROGRESS_STATE:
return {
'complete': False,
'success': None,
'progress': self.result.info,
}
elif self.result.state in ['PENDING', 'STARTED']:
return {
'complete': False,
'success': None,
'progress': _get_unknown_progress(),
}
return self.result.info
def sync_s3_data(full_sync=False):
"""
Sync XML data from S3.
If full_sync is false, only sync files where the VolumeXML with that md5 has not already been successfully ingested.
"""
if full_sync:
raise Exception(
"""Full sync is not currently supported. We currently cannot distinguish between volumes that
have been imported and edited in the database, and those that have been imported and had a second version
uploaded to S3, so we will only import a volume from S3 a single time unless it is marked as
import_status='pending' to trigger re-import.""")
# pre-run setup
wipe_redis_db()
# Round One:
# - Find the latest manifest.json and get list of inventory files to process.
# - Call read_inventory_file() as a celery task for each inventory file. For each file:
# - Add name of each unique volume folder to the "volumes" queue.
# - Add list of volume files to "volume:<volume_folder>" queue.
# - Call sync_s3_data_step_two when all celery tasks are complete.
# This "with" is required only because celery 4.2 falsely detects the following line as a problem when CELERY_TASK_ALWAYS_EAGER=True
# See https://github.com/celery/celery/issues/4576
with allow_join_result():
read_inventory_files()(sync_s3_data_step_two.si(full_sync))
def get_mlp(tagger_group_id: int,
text: str,
lemmatize: bool = False,
use_ner: bool = True):
"""
Retrieves lemmas.
Retrieves tags predicted by MLP NER and present in models.
:return: string, list
"""
tags = []
hybrid_tagger_object = TaggerGroup.objects.get(pk=tagger_group_id)
taggers = {
t.description.lower(): {
"tag": t.description,
"id": t.id
}
for t in hybrid_tagger_object.taggers.all()
}
if lemmatize or use_ner:
logging.getLogger(INFO_LOGGER).info(
f"[Get MLP] Applying lemmatization and NER...")
with allow_join_result():
mlp = apply_mlp_on_list.apply_async(
kwargs={
"texts": [text],
"analyzers": ["all"]
},
queue=CELERY_MLP_TASK_QUEUE).get()
mlp_result = mlp[0]
logging.getLogger(INFO_LOGGER).info(
f"[Get MLP] Finished applying MLP.")
# lemmatize
if lemmatize and mlp_result:
text = mlp_result["text_mlp"]["lemmas"]
lemmas_exists = True if text.strip() else False
logging.getLogger(INFO_LOGGER).info(
f"[Get MLP] Lemmatization result exists: {lemmas_exists}")
# retrieve tags
if use_ner and mlp_result:
seen_tags = {}
for fact in mlp_result["texta_facts"]:
fact_val = fact["str_val"].lower().strip()
if fact_val in taggers and fact_val not in seen_tags:
fact_val_dict = {
"tag": taggers[fact_val]["tag"],
"probability": 1.0,
"tagger_id": taggers[fact_val]["id"],
"ner_match": True
}
tags.append(fact_val_dict)
seen_tags[fact_val] = True
logging.getLogger(INFO_LOGGER).info(
f"[Get MLP] Detected {len(tags)} with NER.")
return text, tags
def apply_async(self, args=(), kwargs={}, **options):
# python is best at unpacking kwargs, so .run is here to do that.
app = self.app
if app.conf.task_always_eager:
with allow_join_result():
return self.apply(args, kwargs, **options)
return self.run(args, kwargs, app=app, **(
dict(self.options, **options) if options else self.options))
def apply_async(self, args=(), kwargs={}, **options):
# python is best at unpacking kwargs, so .run is here to do that.
app = self.app
if app.conf.task_always_eager:
with allow_join_result():
return self.apply(args, kwargs, **options)
return self.run(args, kwargs, app=app, **(
dict(self.options, **options) if options else self.options))
def main_publisher_distributed():
n_data = []
for x in range(0, 100000):
res = add.delay(x)
with allow_join_result():
n_data.append(res.get())
logger.info("Result data : {} ".format(n_data))
def on_chord_part_return(self, task, state, result, propagate=None):
if not self.implements_incr:
return
app = self.app
if propagate is None:
propagate = app.conf.CELERY_CHORD_PROPAGATES
gid = task.request.group
if not gid:
return
key = self.get_key_for_chord(gid)
try:
deps = GroupResult.restore(gid, backend=task.backend)
except Exception as exc:
callback = maybe_signature(task.request.chord, app=app)
return self.chord_error_from_stack(
callback,
ChordError('Cannot restore group: {0!r}'.format(exc)),
)
if deps is None:
try:
raise ValueError(gid)
except ValueError as exc:
callback = maybe_signature(task.request.chord, app=app)
return self.chord_error_from_stack(
callback,
ChordError('GroupResult {0} no longer exists'.format(gid)),
)
val = self.incr(key)
if val >= len(deps):
callback = maybe_signature(task.request.chord, app=app)
j = deps.join_native if deps.supports_native_join else deps.join
try:
with allow_join_result():
ret = j(timeout=3.0, propagate=propagate)
except Exception as exc:
try:
culprit = next(deps._failed_join_report())
reason = 'Dependency {0.id} raised {1!r}'.format(
culprit,
exc,
)
except StopIteration:
reason = repr(exc)
self.chord_error_from_stack(callback, ChordError(reason))
else:
try:
callback.delay(ret)
except Exception as exc:
self.chord_error_from_stack(
callback,
ChordError('Callback error: {0!r}'.format(exc)),
)
finally:
deps.delete()
self.client.delete(key)
else:
self.expire(key, 86400)
def unlock_chord(group_id,
callback,
interval=None,
propagate=None,
max_retries=None,
result=None,
Result=app.AsyncResult,
GroupResult=app.GroupResult,
result_from_tuple=result_from_tuple):
# if propagate is disabled exceptions raised by chord tasks
# will be sent as part of the result list to the chord callback.
# Since 3.1 propagate will be enabled by default, and instead
# the chord callback changes state to FAILURE with the
# exception set to ChordError.
propagate = default_propagate if propagate is None else propagate
if interval is None:
interval = unlock_chord.default_retry_delay
# check if the task group is ready, and if so apply the callback.
callback = maybe_signature(callback, app)
deps = GroupResult(
group_id,
[result_from_tuple(r, app=app) for r in result],
)
j = deps.join_native if deps.supports_native_join else deps.join
if deps.ready():
callback = maybe_signature(callback, app=app)
try:
with allow_join_result():
ret = j(timeout=3.0, propagate=propagate)
except Exception as exc:
try:
culprit = next(deps._failed_join_report())
reason = 'Dependency {0.id} raised {1!r}'.format(
culprit,
exc,
)
except StopIteration:
reason = repr(exc)
logger.error('Chord %r raised: %r', group_id, exc, exc_info=1)
app.backend.chord_error_from_stack(callback,
ChordError(reason))
else:
try:
callback.delay(ret)
except Exception as exc:
logger.error('Chord %r raised: %r',
group_id,
exc,
exc_info=1)
app.backend.chord_error_from_stack(
callback,
exc=ChordError('Callback error: {0!r}'.format(exc)),
)
else:
raise unlock_chord.retry(countdown=interval,
max_retries=max_retries)
def unlock_chord(self,
group_id,
callback,
interval=None,
max_retries=None,
result=None,
Result=app.AsyncResult,
GroupResult=app.GroupResult,
result_from_tuple=result_from_tuple,
**kwargs):
if interval is None:
interval = self.default_retry_delay
# check if the task group is ready, and if so apply the callback.
callback = maybe_signature(callback, app)
deps = GroupResult(
group_id,
[result_from_tuple(r, app=app) for r in result],
app=app,
)
j = deps.join_native if deps.supports_native_join else deps.join
try:
ready = deps.ready()
except Exception as exc:
raise self.retry(
exc=exc,
countdown=interval,
max_retries=max_retries,
)
else:
if not ready:
raise self.retry(countdown=interval, max_retries=max_retries)
callback = maybe_signature(callback, app=app)
try:
with allow_join_result():
ret = j(timeout=3.0, propagate=True)
except Exception as exc:
try:
culprit = next(deps._failed_join_report())
reason = 'Dependency {0.id} raised {1!r}'.format(
culprit,
exc,
)
except StopIteration:
reason = repr(exc)
logger.error('Chord %r raised: %r', group_id, exc, exc_info=1)
app.backend.chord_error_from_stack(callback, ChordError(reason))
else:
try:
callback.delay(ret)
except Exception as exc:
logger.error('Chord %r raised: %r', group_id, exc, exc_info=1)
app.backend.chord_error_from_stack(
callback,
exc=ChordError('Callback error: {0!r}'.format(exc)),
)
def convert(request_data, base_path):
"""
Converts the given assets into outputs desired formats. It receives
a dictionary request_data with something like this:
{
"input": "testbook.epub",
"assets": {
"testbook.epub": "http://127.0.0.1:8000/bla-foo/_export/"
},
"outputs": {
"two": {
"profile": "epub",
"output": "testbook.epub",
"config": {
"project_id": "bla-foo"
}
}
}
}
"""
# TODO we should use a chain of tasks
assets = AssetCollection(base_path)
assets.add_urls(request_data.assets)
assets.add_files(request_data.files)
subtasks = []
for (name, output) in request_data.outputs.iteritems():
sandbox_path = os.path.join(base_path, name)
output_path = os.path.join(sandbox_path, output.output)
subtask_args = (output.profile, request_data.input, output_path)
subtask_kwargs = {
"assets": assets,
"config": output.config,
"sandbox_path": sandbox_path
}
custom_task_id = '%s:%s' % (name, str(uuid4()))
subtask = convert_one.subtask(
args=subtask_args,
kwargs=subtask_kwargs,
task_id=custom_task_id
)
subtasks.append(subtask)
job = group(subtasks, disable_sync_subtasks=False)
result = job.apply_async()
# TODO we should use chain here
# http://docs.celeryq.org/en/latest/userguide/tasks.html#task-synchronous-subtasks
with allow_join_result():
result.join(propagate=False)
subtasks_info = {async.task_id: async for async in result.children}
def on_chord_part_return(self, task, state, result, propagate=None):
if not self.implements_incr:
return
app = self.app
if propagate is None:
propagate = app.conf.CELERY_CHORD_PROPAGATES
gid = task.request.group
if not gid:
return
key = self.get_key_for_chord(gid)
try:
deps = GroupResult.restore(gid, backend=task.backend)
except Exception as exc:
callback = maybe_signature(task.request.chord, app=app)
return self.chord_error_from_stack(
callback,
ChordError('Cannot restore group: {0!r}'.format(exc)),
)
if deps is None:
try:
raise ValueError(gid)
except ValueError as exc:
callback = maybe_signature(task.request.chord, app=app)
return self.chord_error_from_stack(
callback,
ChordError('GroupResult {0} no longer exists'.format(gid)),
)
val = self.incr(key)
if val >= len(deps):
callback = maybe_signature(task.request.chord, app=app)
j = deps.join_native if deps.supports_native_join else deps.join
try:
with allow_join_result():
ret = j(timeout=3.0, propagate=propagate)
except Exception as exc:
try:
culprit = next(deps._failed_join_report())
reason = 'Dependency {0.id} raised {1!r}'.format(
culprit, exc,
)
except StopIteration:
reason = repr(exc)
self.chord_error_from_stack(callback, ChordError(reason))
else:
try:
callback.delay(ret)
except Exception as exc:
self.chord_error_from_stack(
callback,
ChordError('Callback error: {0!r}'.format(exc)),
)
finally:
deps.delete()
self.client.delete(key)
else:
self.expire(key, 86400)
def _execute_run_command_body(task_id, recon_pipeline, pipeline_run_id,
instance, write_stream_fn):
# we need to send but the fact that we have loaded the args so the calling
# process knows it is safe to clean up the temp input file
# write_stream_fn(ExecuteRunArgsLoadComplete())
pipeline_run = instance.get_run_by_id(pipeline_run_id)
instance.report_engine_event(
message=f"Pipeline execution starting (task: {task_id})",
pipeline_run=pipeline_run,
)
try:
with allow_join_result():
for event in execute_run_iterator(recon_pipeline, pipeline_run,
instance):
write_stream_fn(event)
except KeyboardInterrupt:
instance.report_engine_event(
message="Pipeline execution terminated by interrupt",
pipeline_run=pipeline_run,
)
except DagsterSubprocessError as err:
if not all([
err_info.cls_name == "KeyboardInterrupt"
for err_info in err.subprocess_error_infos
]):
instance.report_engine_event(
"An exception was thrown during execution that is likely a framework error, "
"rather than an error in user code.",
pipeline_run,
EngineEventData.engine_error(
serializable_error_info_from_exc_info(sys.exc_info())),
)
except Exception as exc: # pylint: disable=broad-except
if isinstance(
exc, CheckError
) and 'in state PipelineRunStatus.STARTED, expected PipelineRunStatus.NOT_STARTED' in str(
exc):
# TODO Should this log?
return
instance.report_engine_event(
"An exception was thrown during execution that is likely a framework error, "
"rather than an error in user code.",
pipeline_run,
EngineEventData.engine_error(
serializable_error_info_from_exc_info(sys.exc_info())),
)
finally:
instance.report_engine_event(
"Task for pipeline completed (task: {task_id}).".format(
task_id=task_id),
pipeline_run,
)
def unlock_chord(self, group_id, callback, interval=None, propagate=None,
max_retries=None, result=None,
Result=app.AsyncResult, GroupResult=app.GroupResult,
result_from_tuple=result_from_tuple):
# if propagate is disabled exceptions raised by chord tasks
# will be sent as part of the result list to the chord callback.
# Since 3.1 propagate will be enabled by default, and instead
# the chord callback changes state to FAILURE with the
# exception set to ChordError.
propagate = default_propagate if propagate is None else propagate
if interval is None:
interval = self.default_retry_delay
# check if the task group is ready, and if so apply the callback.
callback = maybe_signature(callback, app)
deps = GroupResult(
group_id,
[result_from_tuple(r, app=app) for r in result],
app=app,
)
j = deps.join_native if deps.supports_native_join else deps.join
try:
ready = deps.ready()
except Exception as exc:
raise self.retry(
exc=exc, countdown=interval, max_retries=max_retries,
)
else:
if not ready:
raise self.retry(countdown=interval, max_retries=max_retries)
callback = maybe_signature(callback, app=app)
try:
with allow_join_result():
ret = j(timeout=3.0, propagate=propagate)
except Exception as exc:
try:
culprit = next(deps._failed_join_report())
reason = 'Dependency {0.id} raised {1!r}'.format(
culprit, exc,
)
except StopIteration:
reason = repr(exc)
logger.error('Chord %r raised: %r', group_id, exc, exc_info=1)
app.backend.chord_error_from_stack(callback,
ChordError(reason))
else:
try:
callback.delay(ret)
except Exception as exc:
logger.error('Chord %r raised: %r', group_id, exc, exc_info=1)
app.backend.chord_error_from_stack(
callback,
exc=ChordError('Callback error: {0!r}'.format(exc)),
)
def unlock_graph(result, callback,
interval=1, propagate=False, max_retries=None):
if result.ready():
second_level_res = result.get()
if second_level_res.ready():
with allow_join_result():
signature(callback).delay(list(joinall(
second_level_res, propagate=propagate)))
else:
unlock_graph.retry(countdown=interval, max_retries=max_retries)
def check_realtime_broker():
res = check_broker_connection.delay()
# wait to sync result
try:
with allow_join_result():
update_indicator.delay(res.get())
return res.get()
except BaseException as e:
LOGGER.exception(e)
return False
def track_job(*, task_id: str) -> bool:
task_result = AsyncResult(task_id, app=celery_app)
scan = Scan.objects.get(uuid=task_id)
scan.state = task_result.state
finished = task_result.ready()
if finished:
with allow_join_result():
scan.output = task_result.get()
scan.save()
return finished
def lemmatize(self, text):
with allow_join_result():
mlp = apply_mlp_on_list.apply_async(
kwargs={
"texts": [text],
"analyzers": ["lemmas"]
},
queue=CELERY_MLP_TASK_QUEUE).get()
lemmas = mlp[0]["text_mlp"]["lemmas"]
return lemmas
def check_realtime_broker():
res = check_broker_connection.delay()
# wait to sync result
try:
with allow_join_result():
update_indicator.delay(res.get())
return res.get()
except BaseException as e:
LOGGER.exception(e)
return False
def unlock_graph(result, callback,
interval=1, propagate=False, max_retries=None):
if result.ready():
second_level_res = result.get()
if second_level_res.ready():
with allow_join_result():
signature(callback).delay(list(joinall(
second_level_res, propagate=propagate)))
else:
unlock_graph.retry(countdown=interval, max_retries=max_retries)
def run(self, ip=None, xmlfile=None, validate_fileformat=True, validate_integrity=True, rootdir=None):
step = ProcessStep.objects.create(
name="Validate Files",
parallel=True,
parent_step=self.taskobj.processstep
)
if any([validate_fileformat, validate_integrity]):
if rootdir is None:
rootdir = ip.ObjectPath
doc = etree.ElementTree(file=xmlfile)
for elname, props in settings.FILE_ELEMENTS.iteritems():
for f in doc.xpath('.//*[local-name()="%s"]' % elname):
fpath = get_value_from_path(f, props["path"])
if fpath:
fpath = remove_prefix(fpath, props.get("pathprefix", ""))
fformat = get_value_from_path(f, props.get("format"))
checksum = get_value_from_path(f, props.get("checksum"))
algorithm = get_value_from_path(f, props.get("checksumtype"))
if validate_fileformat and fformat is not None:
step.tasks.add(ProcessTask.objects.create(
name=self.fileformat_task,
params={
"filename": os.path.join(rootdir, fpath),
"fileformat": fformat,
},
log=self.taskobj.log,
information_package=ip,
responsible=self.taskobj.responsible,
))
if validate_integrity and checksum is not None:
step.tasks.add(ProcessTask.objects.create(
name=self.checksum_task,
params={
"filename": os.path.join(rootdir, fpath),
"checksum": checksum,
"algorithm": algorithm,
},
log=self.taskobj.log,
information_package=ip,
responsible=self.taskobj.responsible,
))
self.taskobj.log = None
self.taskobj.save(update_fields=['log'])
self.set_progress(100, total=100)
with allow_join_result():
return step.run().get()
def convert(request_data, base_path):
"""
Converts the given assets into outputs desired formats. It receives
a dictionary request_data with something like this:
{
"input": "testbook.epub",
"assets": {
"testbook.epub": "http://127.0.0.1:8000/bla-foo/_export/"
},
"outputs": {
"two": {
"profile": "epub",
"output": "testbook.epub",
"config": {
"project_id": "bla-foo"
}
}
}
}
"""
# TODO we should use a chain of tasks
assets = AssetCollection(base_path)
assets.add_urls(request_data.assets)
assets.add_files(request_data.files)
subtasks = []
for (name, output) in request_data.outputs.iteritems():
sandbox_path = os.path.join(base_path, name)
output_path = os.path.join(sandbox_path, output.output)
subtask_args = (output.profile, request_data.input, output_path)
subtask_kwargs = {
"assets": assets,
"config": output.config,
"sandbox_path": sandbox_path
}
custom_task_id = '%s:%s' % (name, str(uuid4()))
subtask = convert_one.subtask(args=subtask_args,
kwargs=subtask_kwargs,
task_id=custom_task_id)
subtasks.append(subtask)
job = group(subtasks, disable_sync_subtasks=False)
result = job.apply_async()
# TODO we should use chain here
# http://docs.celeryq.org/en/latest/userguide/tasks.html#task-synchronous-subtasks
with allow_join_result():
result.join(propagate=False)
subtasks_info = {async .task_id: async for async in result.children}
def destroy_redirector(db: Session = Depends(get_db), *, redirector_id: int):
redirector_obj = crud_redirector.get(db_session=db, id=redirector_id)
destroy_task = celery_app.send_task("destroy_vps",
args=[redirector_obj.vps_id])
with allow_join_result():
destroy_task.get()
delete_result = {
'status': crud_redirector.remove(db_session=db, id=redirector_id)
}
return dict(result=delete_result)
def export_request_error(task_uuid, export_request_id):
celery_logger.info("export_request_id=%s" % export_request_id)
celery_logger.info("task_uuid=%s" % task_uuid)
result = app.AsyncResult(task_uuid)
with allow_join_result():
exc = result.get(propagate=False)
err_str = "ERROR - %r Exception:%r" % (result.result, result.traceback,)
celery_logger.error(err_str)
export_request = ExportRequest.objects.get(id=export_request_id)
export_request.status = err_str
export_request.save()
def parse_async(asin, num_workers=1, need_parse=True, url=''):
# set the state != "PENDING", to notify that the task has started
current_task.update_state(state="PROGRESS", meta={"page": 1})
# celery only accepts json serializable objects as parameters
# so we cannot pass "prod", "properties" directly
# instead, we have to query the database again for these objects
prod = Product.objects.get(pk=asin)
properties = list(Property.objects.filter(prod=prod))
pc = 1
while True:
# variable indicating whether the process needs to break
need_break = False
if need_parse:
gp = group(worker.s(asin, pc + w) for w in range(num_workers))
res = gp.apply_async()
ret_list = None
with allow_join_result():
try:
ret_list = res.get(timeout=10)
need_break = correct_terminate(ret_list)
except Exception as e:
raise ValueError("%s in worker" % e)
reviews = [r for x in ret_list for r in x[0]]
save_review(reviews, prod, pc)
else:
tmp = Review.objects.filter(page__gte=pc + num_workers)
need_break = (len(tmp) == 0)
saved_reviews = list(
Review.objects.filter(
page__gte=pc,
page__lt=pc + num_workers,
))
property_data = [convert_property(p) for p in properties]
review_data = [convert_review(r) for r in saved_reviews]
rels = []
if url == "":
rels = matcher.keyword_match(property_data, review_data)
else:
rels = fetch_from_api(url, property_data, review_data)
save_relationship(rels, prod, url)
# update task state to reflect the increment in page count
# "meta" can be accessed in main process as "AsyncResult.info"
# which is not pointed out clearly in celery documentation
if need_break: break
pc += num_workers
current_task.update_state(state="PROGRESS", meta={"page": pc})
def export_request_error(task_uuid, export_request_id):
celery_logger.info("export_request_id=%s" % export_request_id)
celery_logger.info("task_uuid=%s" % task_uuid)
result = app.AsyncResult(task_uuid)
with allow_join_result():
exc = result.get(propagate=False)
err_str = "ERROR - %r Exception:%r" % (result.result, result.traceback,)
celery_logger.error(err_str)
export_request = ExportRequest.objects.get(id=export_request_id)
export_request.status = err_str
export_request.save()
def run(self, *args, **kwargs):
with session_manager() as db_session:
domain_list = crud_domain.get_domain_list(db_session)
load_domain_extra_data_task = group([
LoadDomainExtraDataTask().s(domain.id)
for domain in domain_list
])
load_result = load_domain_extra_data_task.delay()
with allow_join_result():
load_result.join()
return self.set_result()
def get_info(self):
response = {'state': self.result.state}
if self.result.state in ['SUCCESS', 'FAILURE']:
success = self.result.successful()
with allow_join_result():
response.update({
'complete': True,
'success': success,
'progress': _get_completed_progress(),
'result': self.result.get(self.result.id) if success else str(self.result.info),
})
elif self.result.state in ['RETRY', 'REVOKED']:
if self.result.state == 'RETRY':
retry = self.result.info
when = str(retry.when) if isinstance(retry.when, datetime.datetime) else str(
datetime.datetime.now() + datetime.timedelta(seconds=retry.when))
result = {'when': when, 'message': retry.message or str(retry.exc)}
else:
result = 'Task ' + str(self.result.info)
response.update({
'complete': True,
'success': False,
'progress': _get_completed_progress(),
'result': result,
})
elif self.result.state == 'IGNORED':
response.update({
'complete': True,
'success': None,
'progress': _get_completed_progress(),
'result': str(self.result.info)
})
elif self.result.state == PROGRESS_STATE:
response.update({
'complete': False,
'success': None,
'progress': self.result.info,
})
elif self.result.state in ['PENDING', 'STARTED']:
response.update({
'complete': False,
'success': None,
'progress': _get_unknown_progress(self.result.state),
})
else:
logger.error('Task %s has unknown state %s with metadata %s', self.result.id, self.result.state, self.result.info)
response.update({
'complete': True,
'success': False,
'progress': _get_unknown_progress(self.result.state),
'result': 'Unknown state {}'.format(self.result.state),
})
return response
def sync_s3_data_step_two(full_sync=False):
# Round Two:
# - Filter down "volumes" queue to most recent copy of each volume.
# - Call ingest_volume_from_redis() as a celery task for each volume. For each volume:
# - Ingest volume XML, case XML, and alto XML, if not already in database.
# - If we are adding/updating volume XML, check that METS inventory is valid.
# - Call sync_s3_data_step_three when all celery tasks are complete.
# This "with" is required only because celery 4.2 falsely detects the following line as a problem when CELERY_TASK_ALWAYS_EAGER=True
# See https://github.com/celery/celery/issues/4576
with allow_join_result():
ingest_volumes(full_sync)(sync_s3_data_step_three.si())
def machine_request_error(task_uuid, machine_request_id):
logger.info("machine_request_id=%s" % machine_request_id)
logger.info("task_uuid=%s" % task_uuid)
result = app.AsyncResult(task_uuid)
with allow_join_result():
exc = result.get(propagate=False)
err_str = "ERROR - %r Exception:%r" % (result.result, result.traceback)
logger.error(err_str)
send_image_request_failed_email(machine_request, err_str)
machine_request = MachineRequest.objects.get(id=machine_request_id)
machine_request.status = err_str
machine_request.save()
def on_chord_part_return(self, request, state, result, **kwargs):
if not self.implements_incr:
return
app = self.app
gid = request.group
if not gid:
return
key = self.get_key_for_chord(gid)
try:
deps = GroupResult.restore(gid, backend=self)
except Exception as exc:
callback = maybe_signature(request.chord, app=app)
logger.error("Chord %r raised: %r", gid, exc, exc_info=1)
return self.chord_error_from_stack(callback, ChordError("Cannot restore group: {0!r}".format(exc)))
if deps is None:
try:
raise ValueError(gid)
except ValueError as exc:
callback = maybe_signature(request.chord, app=app)
logger.error("Chord callback %r raised: %r", gid, exc, exc_info=1)
return self.chord_error_from_stack(callback, ChordError("GroupResult {0} no longer exists".format(gid)))
val = self.incr(key)
size = len(deps)
if val > size: # pragma: no cover
logger.warning("Chord counter incremented too many times for %r", gid)
elif val == size:
callback = maybe_signature(request.chord, app=app)
j = deps.join_native if deps.supports_native_join else deps.join
try:
with allow_join_result():
ret = j(timeout=3.0, propagate=True)
except Exception as exc:
try:
culprit = next(deps._failed_join_report())
reason = "Dependency {0.id} raised {1!r}".format(culprit, exc)
except StopIteration:
reason = repr(exc)
logger.error("Chord %r raised: %r", gid, reason, exc_info=1)
self.chord_error_from_stack(callback, ChordError(reason))
else:
try:
callback.delay(ret)
except Exception as exc:
logger.error("Chord %r raised: %r", gid, exc, exc_info=1)
self.chord_error_from_stack(callback, ChordError("Callback error: {0!r}".format(exc)))
finally:
deps.delete()
self.client.delete(key)
else:
self.expire(key, 86400)
def unlock_chord(self, group_id, callback, interval=None,
max_retries=None, result=None,
Result=app.AsyncResult, GroupResult=app.GroupResult,
result_from_tuple=result_from_tuple, **kwargs):
if interval is None:
interval = self.default_retry_delay
# check if the task group is ready, and if so apply the callback.
callback = maybe_signature(callback, app)
deps = GroupResult(
group_id,
[result_from_tuple(r, app=app) for r in result],
app=app,
)
j = deps.join_native if deps.supports_native_join else deps.join
try:
ready = deps.ready()
except Exception as exc:
raise self.retry(
exc=exc, countdown=interval, max_retries=max_retries,
)
else:
if not ready:
raise self.retry(countdown=interval, max_retries=max_retries)
callback = maybe_signature(callback, app=app)
try:
with allow_join_result():
ret = j(timeout=3.0, propagate=True)
except Exception as exc:
try:
culprit = next(deps._failed_join_report())
reason = 'Dependency {0.id} raised {1!r}'.format(
culprit, exc,
)
except StopIteration:
reason = repr(exc)
logger.error('Chord %r raised: %r', group_id, exc, exc_info=1)
app.backend.chord_error_from_stack(callback,
ChordError(reason))
else:
try:
callback.delay(ret)
except Exception as exc:
logger.error('Chord %r raised: %r', group_id, exc, exc_info=1)
app.backend.chord_error_from_stack(
callback,
exc=ChordError('Callback error: {0!r}'.format(exc)),
)
def unlock_chord(group_id, callback, interval=None, propagate=None,
max_retries=None, result=None,
Result=app.AsyncResult, GroupResult=app.GroupResult,
result_from_tuple=result_from_tuple):
# if propagate is disabled exceptions raised by chord tasks
# will be sent as part of the result list to the chord callback.
# Since 3.1 propagate will be enabled by default, and instead
# the chord callback changes state to FAILURE with the
# exception set to ChordError.
propagate = default_propagate if propagate is None else propagate
if interval is None:
interval = unlock_chord.default_retry_delay
# check if the task group is ready, and if so apply the callback.
deps = GroupResult(
group_id,
[result_from_tuple(r, app=app) for r in result],
)
j = deps.join_native if deps.supports_native_join else deps.join
if deps.ready():
callback = signature(callback, app=app)
try:
with allow_join_result():
ret = j(propagate=propagate)
except Exception as exc:
try:
culprit = next(deps._failed_join_report())
reason = 'Dependency {0.id} raised {1!r}'.format(
culprit, exc,
)
except StopIteration:
reason = repr(exc)
app._tasks[callback.task].backend.fail_from_current_stack(
callback.id, exc=ChordError(reason),
)
else:
try:
callback.delay(ret)
except Exception as exc:
app._tasks[callback.task].backend.fail_from_current_stack(
callback.id,
exc=ChordError('Callback error: {0!r}'.format(exc)),
)
else:
raise unlock_chord.retry(countdown=interval,
max_retries=max_retries)
def shmir_from_fasta(siRNA, offtarget, regexp, original_frames, prefix):
siRNA2 = reverse_complement(siRNA)
frames = adjusted_frames(siRNA, siRNA2, 0, 0, deepcopy(original_frames)) # we do not have shifts here
shmirs = [frame.template() for frame in frames]
with allow_join_result():
foldings = group(fold.s(shmir, prefix=prefix).set(queue="subtasks") for shmir in shmirs).apply_async().get()
results = []
iter_frames = izip(frames, original_frames, foldings)
for frame, original_frame, folding in iter_frames:
score = score_from_transcript(frame, original_frame, folding["ss"], offtarget, regexp)
if validate_transcript_by_score(score):
results.append({"score": score, "frame": frame, "folding": folding, "found_sequence": siRNA})
return results
def apply_async(self, args=(), kwargs={}, task_id=None,
producer=None, publisher=None, connection=None,
router=None, result_cls=None, **options):
kwargs = kwargs or {}
args = (tuple(args) + tuple(self.args)
if args and not self.immutable else self.args)
body = kwargs.pop('body', None) or self.kwargs['body']
kwargs = dict(self.kwargs['kwargs'], **kwargs)
body = body.clone(**options)
app = self._get_app(body)
tasks = (self.tasks.clone() if isinstance(self.tasks, group)
else group(self.tasks, app=app))
if app.conf.task_always_eager:
with allow_join_result():
return self.apply(args, kwargs,
body=body, task_id=task_id, **options)
# chord([A, B, ...], C)
return self.run(tasks, body, args, task_id=task_id, **options)
def fetch_and_populate_company(name, linkedin_id, callback_url=None, crunchbase_url=None):
print 'TASK! fetch company ' + str(name.encode('utf8')) + ', ' + str(linkedin_id)
# 1) Try to find company in database by linkedin id or by name
if not linkedin_id:
# TODO make intelligent disambiguiation based on name
return 'Sorry no linkedin id'
company = Company.from_linkedin_id(linkedin_id)
if not company:
print ' linkedin id not found; fetching by name...'
company = Company.from_name(name)
# 2) fetch it from crunchbase if it doesn't exist or is outdated
company_info = None
if company:
print ' ...company already exists as ' + str(company.name.encode('utf8')) + ', ' + str(company.linkedin_id)
if not company.last_crunchbase_update or company.last_crunchbase_update < datetime.now() - timedelta(days=EXPIRATION_DAYS):
print ' crunchbase not updated for ' + str(EXPIRATION_DAYS) + ' days => updating'
company_info = fetch_company_info_from_crunchbase.delay(company)
elif crunchbase_url and company.crunchbase_url != crunchbase_url:
print ' new crunchbase url ' + crunchbase_url.encode('utf8') + ' (old was ' + str(company.crunchbase_url) + ')'
company.crunchbase_url = crunchbase_url
company_info = fetch_company_info_from_crunchbase.delay(company)
else:
print ' crunchbase updated recently => no crunchbase update necessary'
else:
print ' ...' + str(name.encode('utf8')) + ', ' + str(linkedin_id) + ' does not exist => new company'
company = Company(name, linkedin_id)
company.crunchbase_url = crunchbase_url
db.session.add(company)
company_info = fetch_company_info_from_crunchbase.delay(company)
with allow_join_result():
if company_info:
print ' waiting for crunchbase result'
company_info = company_info.wait()
update_company_with_crunchbase_info(company, company_info)
db.session.commit()
# 3) send back company to the mother ship
print 'callback url = ' + str(callback_url)
if callback_url:
serialize_and_send_company_to_callback_url(company, callback_url)
return 'Done?' # TODO return meaningful result
def deploy_failed(task_uuid, driverCls, provider, identity, instance_id, **celery_task_args):
from core.models.instance import Instance
from core.email import send_deploy_failed_email
try:
logger.debug("deploy_failed task started at %s." % datetime.now())
logger.info("task_uuid=%s" % task_uuid)
result = app.AsyncResult(task_uuid)
with allow_join_result():
exc = result.get(propagate=False)
err_str = "DEPLOYERROR::%s" % (result.traceback,)
logger.error(err_str)
driver = get_driver(driverCls, provider, identity)
instance = driver.get_instance(instance_id)
update_instance_metadata(driver, instance, data={"tmp_status": "deploy_error"}, replace=False)
logger.debug("deploy_failed task finished at %s." % datetime.now())
except Exception as exc:
logger.warn(exc)
deploy_failed.retry(exc=exc)
def handle_staging_result(staging_task, sf):
with allow_join_result():
logger.debug('getting staging result')
try:
staging_task.get() # returns None, but more importantly propagates any exception in the task
except Exception, e:
logger.warning('%s failed: %s' % (staging_task, e))
# consider resubmitting the task a limited amount since it
# should only fail in rare cases (if so then do this by
# raising a RetryException or similar here)
logger.debug('deregistering %s from %s' % (sf.staging_task, sf))
tasks.session.delete(staging_task)
sf.staging_task = None
assert sf.staging_task is None, sf
logger.debug('db commit')
tasks.session.commit()
for r in dispatched_requests(sf):
logger.info('request %s failed' % r.uuid)
fail_request(r, 'Staging of %s failed: %s' % (sf, str(e)))
return # we deliberately only handle exceptions from the staging task, any other exception will propagate
def celery_worker(_celery_app):
if not NO_WORKER:
on_started = threading.Event()
def on_worker_ready(consumer):
on_started.set()
_celery_app.set_current()
_celery_app.set_default()
_celery_app.finalize()
_celery_app.log.setup()
# Make sure we can connect to the broker
with _celery_app.connection() as conn:
conn.default_channel.queue_declare
if celery.VERSION >= (4,):
pool_args = {'pool': 'solo'}
else:
pool_args = {'pool_cls': 'solo'}
worker = _celery_app.WorkController(
concurrency=1,
loglevel=WORKER_LOGLEVEL,
logfile=None,
ready_callback=on_worker_ready,
**pool_args)
t = threading.Thread(target=worker.start)
t.start()
assert any(t.startswith('thorn.') for t in _celery_app.tasks)
assert 'cyanide.tasks.add' in _celery_app.tasks
on_started.wait()
with allow_join_result():
assert add.delay(2, 2).get(timeout=3) == 4
_set_task_join_will_block(False)
yield worker
worker.stop()
t.join()
def shmir_from_sirna_score(input_str):
"""Main function takes string input and returns the best results depending
on scoring. Single result include sh-miR sequence,
score and link to 2D structure from mfold program
Args:
input_str(str): Input string contains one or two sequences.
Returns:
List of sh-miR(s) sorted by score.
"""
seq1, seq2, shift_left, shift_right = check_input(input_str)
if not seq2:
seq2 = reverse_complement(seq1)
original_frames = db_session.query(Backbone).all()
frames = get_frames(seq1, seq2,
shift_left, shift_right,
deepcopy(original_frames))
with allow_join_result():
frames_with_score = group(
fold_and_score.s(
seq1, seq2,
frame_tuple,
original,
score_from_sirna,
(seq1,)
).set(queue="subtasks")
for frame_tuple, original in zip(frames, original_frames)
).apply_async().get()
sorted_frames = [
elem[:-1] for elem in sorted(
frames_with_score, key=operator.itemgetter(0), reverse=True
) if elem[0] > 60
][:3]
return sorted_frames
def machine_request_error(task_request, *args, **kwargs):
#Args format: (exception, ?, subtask_args...)
exception = args[0]
machine_request_id = args[2]
task_uuid = task_request.id
celery_logger.info("machine_request_id=%s" % machine_request_id)
celery_logger.info("task_uuid=%s" % (task_uuid,) )
celery_logger.info("exception=%s" % (exception,) )
celery_logger.info("task_kwargs=%s" % kwargs)
machine_request = MachineRequest.objects.get(id=machine_request_id)
result = app.AsyncResult(task_uuid)
with allow_join_result():
exc = result.get(propagate=False)
err_str = _status_to_error(machine_request.old_status, result.result, result.traceback)
celery_logger.info("traceback=%s" % (result.traceback,) )
celery_logger.error(err_str)
machine_request = MachineRequest.objects.get(id=machine_request_id)
machine_request.old_status = err_str
machine_request.save()
send_image_request_failed_email(machine_request, err_str)
def mount_failed(task_uuid, driverCls, provider, identity, volume_id, unmount=False, **celery_task_args):
from service import volume as volume_service
try:
celery_logger.debug("mount_failed task started at %s." % datetime.now())
celery_logger.info("task_uuid=%s" % task_uuid)
result = app.AsyncResult(task_uuid)
with allow_join_result():
exc = result.get(propagate=False)
err_str = "Mount Error Traceback:%s" % (result.traceback,)
celery_logger.error(err_str)
driver = get_driver(driverCls, provider, identity)
volume = driver.get_volume(volume_id)
if unmount:
tmp_status = "umount_error"
else:
tmp_status = "mount_error"
return volume_service.update_volume_metadata(driver, volume, metadata={"tmp_status": tmp_status})
celery_logger.debug("mount_failed task finished at %s." % datetime.now())
except Exception as exc:
celery_logger.warn(exc)
mount_failed.retry(exc=exc)
def join_sizing_tasks(files):
with allow_join_result():
for sf in files:
try:
assert sf.size is None, sf
logger.debug('<= awaiting size estimation result for %s, '
'task is %s' % (sf, sf.sizing_task))
if sf.sizing_task is None:
# this means it failed before and the task got
# unregistered from sf
raise TaskFailure('size estimation has failed - a new task '
'will be created on the next request '
'containing this file')
size = sf.sizing_task.get()
logger.debug('size estimated to %d bytes' % size)
assert size >= 0, size
sf.size = size
except Exception, e:
logger.warning('size estimation task %s failed: %s' % \
(sf.sizing_task, e))
raise e
finally:
def shmir_from_fasta_string(fasta_string, original_frames,
actual_offtarget, regexp_type, path):
"""Generating function of shmir from fasta string.
Args:
fasta_string(str): Sequence.
original_frames(Backbone): original Backbone object.
actual_offtarget(int): offtarget value
regexp_type(int): Number of a regex from database.
Returns:
list of sh-miR(s)
"""
seq2 = reverse_complement(fasta_string)
frames = get_frames(fasta_string, seq2, 0, 0, deepcopy(original_frames))
with allow_join_result():
frames_with_score = group(
fold_and_score.s(
fasta_string,
seq2,
frame_tuple,
original,
score_from_transcript,
(actual_offtarget, regexp_type),
path
).set(queue="subtasks")
for frame_tuple, original in zip(frames, original_frames)
).apply_async().get()
filtered_frames = []
for frame in frames_with_score:
notes = frame[0]
if notes['frame'] > 60 and notes['all'] > 100:
frame[0] = notes['all']
filtered_frames.append(frame)
return sorted(filtered_frames, key=operator.itemgetter(0), reverse=True) or None
def validate_sequences(sequences, regexp, name, minimum_CG, maximum_CG, maximum_offtarget, immuno):
"""
Here we remove all bad sequences (siRNA) by validators
"""
# filter sequences here by no expensive features
preprocessed = filter(
lambda sequence: all(
[
validate_gc_content(sequence, minimum_CG, maximum_CG),
validate_immuno(sequence, immuno),
validate_thermostability(sequence),
]
),
sequences,
)
# uncomment if debuging
# return {
# name: [{
# "sequence": seq,
# "regexp": int(regexp),
# "offtarget": 0}
# for seq in preprocessed]
# }
# counting offtarget is expensive
with allow_join_result():
offtarget = (
group(offtarget_seed.s(sequence).set(queue="blast") for sequence in preprocessed).apply_async().get()
)
return {
name: [
{"sequence": sequence, "regexp": int(regexp), "offtarget": actual_offtarget}
for sequence, actual_offtarget in izip(preprocessed, offtarget)
if actual_offtarget <= maximum_offtarget
]
}
def wait_for_celery(timeout=10):
with allow_join_result():
ping.delay().get(timeout=timeout)
def shmir_from_transcript_sequence(
transcript_name, minimum_CG, maximum_CG, maximum_offtarget, scaffold,
stimulatory_sequences
):
"""Generating function of shmir from transcript sequence.
Args:
transcript_name(str): Name of transcipt.
minimum_CG(int): Minimum number of 'C' and 'G' nucleotide in sequence.
maximum_CG(int): Maximum number of 'C' and 'G' nucleotide in sequence.
maximum_offtarget(int): Maximum offtarget.
scaffold(str): Name of frame of miRNA or 'all'.
stimulatory_sequences(str): One of 'yes', 'no', 'no_difference'.
Returns:
list of sh-miR(s).
"""
# check if results are in database
try:
stored_input = db_session.query(InputData).filter(
func.lower(InputData.transcript_name) == transcript_name.lower(),
InputData.minimum_CG == minimum_CG,
InputData.maximum_CG == maximum_CG,
InputData.maximum_offtarget == maximum_offtarget,
func.lower(InputData.scaffold) == scaffold.lower(),
func.lower(
InputData.stimulatory_sequences
) == stimulatory_sequences.lower()
).outerjoin(InputData.results).one()
except NoResultFound:
pass
else:
return [result.as_json() for result in stored_input.results]
# create path string
path = "_".join(
map(
str,
[transcript_name, minimum_CG, maximum_CG, maximum_offtarget,
scaffold, stimulatory_sequences]
)
)
mRNA = ncbi_api.get_mRNA(transcript_name)
if scaffold == 'all':
original_frames = db_session.query(Backbone).all()
else:
original_frames = db_session.query(Backbone).filter(
func.lower(Backbone.name) == scaffold.lower()
).all()
frames_by_name = {frame.name: frame for frame in original_frames}
patterns = {
frame.name: OrderedDict(
sorted(
json.loads(frame.regexp).items(),
reverse=True
)
) for frame in original_frames
}
best_sequences = defaultdict(list)
for name, patterns_dict in patterns.iteritems():
for regexp_type, sequences in find_by_patterns(patterns_dict, mRNA).iteritems():
with allow_join_result():
is_empty, sequences = generator_is_empty(sequences)
if not is_empty:
best_sequences[name] = remove_none(
group(
validate_and_offtarget.s(
sequence,
maximum_offtarget,
minimum_CG,
maximum_CG,
stimulatory_sequences,
int(regexp_type)
).set(queue="blast")
for sequence in sequences
).apply_async().get()
)
results = []
for name, seq_dict in unpack_dict_to_list(best_sequences):
if len(results) == 20:
break
with allow_join_result():
shmir_result = shmir_from_fasta_string.s(
seq_dict['sequence'],
[frames_by_name[name]],
seq_dict['offtarget'],
seq_dict['regexp'],
path
).set(queue="score").apply_async().get()
if shmir_result:
results.extend(shmir_result)
if not results:
best_sequences = []
sequences = all_possible_sequences(mRNA, 19, 21)
with allow_join_result():
is_empty, sequences = generator_is_empty(sequences)
if not is_empty:
best_sequences = remove_none(
group(
validate_and_offtarget.s(
sequence,
maximum_offtarget,
minimum_CG,
maximum_CG,
stimulatory_sequences,
0
).set(queue="blast")
for sequence in sequences
).apply_async().get()
)
if best_sequences:
with allow_join_result():
results = chain(*remove_none(
group(
shmir_from_fasta_string.s(
seq_dict['sequence'], original_frames,
seq_dict['offtarget'], seq_dict['regexp'], path
).set(queue="score")
for seq_dict in best_sequences
).apply_async().get()
))
sorted_results = sorted(
results,
key=operator.itemgetter(0),
reverse=True
)[:10]
db_results = [Result(
score=score,
sh_mir=shmir,
pdf=path_id,
backbone=frames_by_name[frame_name].id,
sequence=found_sequences[0],
) for score, shmir, frame_name, path_id, found_sequences in sorted_results]
remove_bad_foldings(path, (result.get_task_id() for result in db_results))
db_input = InputData(
transcript_name=transcript_name,
minimum_CG=minimum_CG,
maximum_CG=maximum_CG,
maximum_offtarget=maximum_offtarget,
scaffold=scaffold,
stimulatory_sequences=stimulatory_sequences,
results=db_results
)
db_session.add(db_input)
db_session.add_all(db_results)
db_session.commit()
return [result.as_json() for result in db_results]
def shmir_from_transcript_sequence(
transcript_name, minimum_CG, maximum_CG, maximum_offtarget, scaffold, immunostimulatory
):
"""Generating function of shmir from transcript sequence.
Args:
transcript_name(str): Name of transcipt.
minimum_CG(int): Minimum number of 'C' and 'G' nucleotide in sequence.
maximum_CG(int): Maximum number of 'C' and 'G' nucleotide in sequence.
maximum_offtarget(int): Maximum offtarget.
scaffold(str): Name of frame of miRNA or 'all'.
stimulatory_sequences(str): One of 'yes', 'no', 'no_difference'.
Returns:
list of sh-miR(s).
"""
# check if results are in database
results = get_results(transcript_name, minimum_CG, maximum_CG, maximum_offtarget, scaffold, immunostimulatory)
# sometimes results is an empty list
if results is not None:
return results
path = create_path_string(transcript_name, minimum_CG, maximum_CG, maximum_offtarget, scaffold, immunostimulatory)
mRNA = ncbi_api.get_mRNA(transcript_name)
reversed_mRNA = reverse_complement(mRNA)
original_frames = frames_by_scaffold(scaffold)
frames_by_name = {frame.name: frame for frame in original_frames}
# best patters should be choosen first
patterns = {
frame.name: OrderedDict(sorted(json.loads(frame.regexp).items(), reverse=True)) for frame in original_frames
}
with allow_join_result():
validated = (
group(
validate_sequences.s(
list(sequences), # generators are not serializable
regexp_type,
name,
minimum_CG,
maximum_CG,
maximum_offtarget,
immunostimulatory,
).set(queue="score")
for name, patterns_dict in patterns.iteritems()
for regexp_type, sequences in find_by_patterns(patterns_dict, reversed_mRNA).iteritems()
)
.apply_async()
.get()
)
best_sequences = merge_results(validated)
with allow_join_result():
results = (
group(
shmir_from_fasta.s(
siRNA["sequence"], siRNA["offtarget"], siRNA["regexp"], [frames_by_name[name]], path
).set(queue="score")
for name, siRNA in unpack_dict_to_list(best_sequences)
)
.apply_async()
.get()
)
# merge
results = list(chain(*results))
if not results:
with allow_join_result():
validated = (
validate_sequences.s(
list(all_possible_sequences(reversed_mRNA, 21)), # not serializable
0,
"all",
minimum_CG,
maximum_CG,
maximum_offtarget,
immunostimulatory,
)
.apply_async(queue="subtasks")
.get()
)
best_sequences = merge_results([validated])
with allow_join_result():
results = (
group(
shmir_from_fasta.s(
siRNA["sequence"], siRNA["offtarget"], siRNA["regexp"], original_frames, path
).set(queue="score")
for name, siRNA in unpack_dict_to_list(best_sequences)
)
.apply_async()
.get()
)
# merge
results = chain(*results)
sorted_results = sorted(results, key=lambda result: result["score"]["all"], reverse=True)[:TRANSCRIPT_RESULT_LIMIT]
db_results = store_results(
transcript_name, minimum_CG, maximum_CG, maximum_offtarget, scaffold, immunostimulatory, sorted_results
)
remove_bad_foldings(path, [result.get_task_id() for result in db_results])
return [result.as_json() for result in db_results]
