def deploy_contract(
self,
uuid: UUID,
contract_name: str,
args: Optional[tuple] = None, kwargs: Optional[dict] = None,
signing_address: Optional[str] = None, encrypted_private_key: Optional[str]=None,
gas_limit: Optional[int] = None,
prior_tasks: Optional[UUIDList] = None
):
"""
The main deploy contract entrypoint for the processor.
:param uuid: the celery generated uuid for the task
:param contract_name: System will attempt to fetched abi and bytecode from this
:param args: arguments for the constructor
:param kwargs: keyword arguments for the constructor
:param signing_address: address of the wallet signing the txn
:param encrypted_private_key: private key of the wallet making the transaction, encrypted using key from settings
:param gas_limit: limit on the amount of gas txn can use. Overrides system default
:param prior_tasks: a list of task uuid that must succeed before this task will be attempted
"""
signing_wallet_obj = self.get_signing_wallet_object(signing_address, encrypted_private_key)
task = self.persistence_interface.create_deploy_contract_task(uuid,
signing_wallet_obj,
contract_name,
args, kwargs,
gas_limit,
prior_tasks)
# Attempt Create Async Transaction
signature(utils.eth_endpoint('_attempt_transaction'), args=(task.uuid,)).delay()
def send_newsletter() -> None:
users_sig = signature("users.get_all", queue="users")
newsletters_sig = signature("mailing.send_newsletter_all", queue="mailing")
task_chain = chain(users_sig | newsletters_sig)
task_chain.apply_async()
def send_eth(self,
uuid: UUID,
amount_wei: int,
recipient_address: str,
signing_address: Optional[str] = None, encrypted_private_key: Optional[str] = None,
prior_tasks: Optional[UUIDList] = None,
posterior_tasks: Optional[UUIDList] = None):
"""
The main entrypoint sending eth.
:param uuid: the celery generated uuid for the task
:param amount_wei: the amount in WEI to send
:param recipient_address: the recipient address
:param signing_address: address of the wallet signing the txn
:param encrypted_private_key: private key of the wallet making the transaction, encrypted using key from settings
:param prior_tasks: a list of task uuids that must succeed before this task will be attempted
:param posterior_tasks: a uuid list of tasks for which this task must succeed before they will be attempted
:return: task_id
"""
signing_wallet_obj = self.get_signing_wallet_object(signing_address, encrypted_private_key)
task = self.persistence_interface.create_send_eth_task(uuid,
signing_wallet_obj,
recipient_address, amount_wei,
prior_tasks,
posterior_tasks)
# Attempt Create Async Transaction
signature(utils.eth_endpoint('_attempt_transaction'), args=(task.uuid,)).delay()
def send_tasks(self,
from_loc: str,
to_loc: str,
start_date: str,
nb_passenger: str,
workers: list = None) -> AsyncResult:
if workers is None:
workers = self.workers
kwargs = {
"from_loc": from_loc,
"to_loc": to_loc,
"start_date": start_date,
"nb_passenger": nb_passenger
}
sigs = [
signature(
"worker",
kwargs=kwargs,
routing_key=f"journey.{k}",
exchange="bonvoyage",
) for k in workers
]
print(sigs)
broker = signature("broker",
kwargs=kwargs,
routing_key="journey.broker",
exchange="bonvoyage")
r = chord(sigs)(broker)
return r
def trigger_event(group_name, event_name):
event_source = Group.objects.get(name=group_name)
event_definition = EventDefinition.objects.get(name=event_name)
event_write = Event(event_definition=event_definition,
device_group=event_source)
event_write.save()
# completion of automation writes an event which could trigger another automation
automations = Automation.objects.filter(cause=event_definition,
source_group=event_source)
for automation in automations:
dummy_result = [None]
event_target = automation.target_group
config_id = str(event_target.get_config_id())
# initialize automation chain with triggered automation
automation_chain = signature(automation.effect,
args=dummy_result,
kwargs={'config_id': config_id})
a = automation
while a.next:
a = a.next
automation_chain |= (signature(
'events.tasks.evaluate_event',
kwargs={
'group_name': group_name,
'event_name': event_name
}) | signature(a.effect, kwargs={'config_id': config_id}))
automation_chain.apply_async()
def check_transaction_response(self, celery_task, transaction_id):
def transaction_response_countdown():
t = lambda retries: ETH_CHECK_TRANSACTION_BASE_TIME * 2**retries
# If the system has been longer than the max retry period
# if previous_result:
# submitted_at = datetime.strptime(previous_result['submitted_date'], "%Y-%m-%d %H:%M:%S.%f")
# if (datetime.utcnow() - submitted_at).total_seconds() > ETH_CHECK_TRANSACTION_RETRIES_TIME_LIMIT:
# if self.request.retries != self.max_retries:
# self.request.retries = self.max_retries - 1
#
# return 0
return t(celery_task.request.retries)
try:
transaction_object = self.persistence_interface.get_transaction(
transaction_id)
task = transaction_object.task
transaction_hash = transaction_object.hash
result = self.check_transaction_hash(transaction_hash)
self.persistence_interface.update_transaction_data(
transaction_id, result)
status = result.get('status')
print(
f'Status for transaction {transaction_object.id} of task UUID {task.uuid} is:'
f'\n {status}')
if status == 'SUCCESS':
unstarted_posteriors = self.get_unstarted_posteriors(task)
for dep_task in unstarted_posteriors:
print('Starting posterior task: {}'.format(dep_task.uuid))
signature(utils.eth_endpoint('_attempt_transaction'),
args=(dep_task.uuid, )).delay()
self.persistence_interface.set_task_status_text(
task, 'SUCCESS')
if status == 'PENDING':
celery_task.request.retries = 0
raise Exception("Need Retry")
if status == 'FAILED':
self.new_transaction_attempt(task)
except TaskRetriesExceededError as e:
pass
except Exception as e:
print(e)
celery_task.retry(countdown=transaction_response_countdown())
def create_and_store_estimator_from_all_eligible(
minimum_verifications=ff_conf.ML_MINIMUM_TAG_VERIFICATIONS_DURING_STAGE_1):
return (
celery.signature('django.training_eligible_data',
args=(minimum_verifications,)) |
celery.signature('learn.create_estimator') |
celery.signature('results.store_estimator')
).apply_async()
def send(self, message, extra={}, tags={}, sentry_data={}, crash_obj=None):
event_id = self.client.capture('raven.events.Message',
message=message,
extra=extra,
tags=tags,
data=data)
signature("tasks.get_sentry_link",
args=(crash_obj.pk, event_id)).apply_async(queue='private',
countdown=1)
def test_link_error_using_signature(self):
fail = signature('t.integration.tasks.fail', args=("test", ))
retrun_exception = signature('t.integration.tasks.return_exception')
fail.link_error(retrun_exception)
exception = ExpectedException("Task expected to fail", "test")
assert (fail.delay().get(timeout=TIMEOUT,
propagate=False), True) == (exception, True)
def unlock_graph(result, callback,
interval=1, propagate=False, max_retries=None):
if result.ready():
second_level_res = result.get()
if second_level_res.ready():
signature(callback).delay(list(joinall(
second_level_res, propagate=propagate)))
else:
unlock_graph.retry(countdown=interval, max_retries=max_retries)
def send(self, message, extra={}, tags={}, sentry_data={}, crash_obj=None):
event_id = self.client.capture(
'raven.events.Message',
message=message,
extra=extra,
tags=tags,
data=sentry_data
)
signature("tasks.get_sentry_link", args=(crash_obj.pk, event_id)).apply_async(queue='private', countdown=1)
def upgrade(self, request, pk=None):
"""
更新节点
"""
data = request.data
data['version'] = data.get('version') or 'null'
TbCetusNodeInfo.objects.filter(pk=data['id']).update(status=2)
signature('upgrade_node', kwargs=data).delay()
return Response('更新请求发送成功', status=status.HTTP_200_OK)
def get_submission(self, submission_id):
json_metrics = []
try:
start_time = time.time()
submission = reddit.submission(id=submission_id)
fetch_time = time.time() - start_time
except prawcore.exceptions.PrawcoreException:
logging.info('Cannot get submission from Reddit API.')
finally:
submission = RedditSubmission(
original_URL=submission.url,
author_name=submission.author.name,
subreddit_name=submission.subreddit.name,
subbreddit_display_name=submission.subreddit.display_name,
post_title=submission.title,
post_title_embedding=[],
post_text=submission.selftext,
post_text_embedding=[],
upvote_ratio=submission.upvote_ratio,
up_votes_number=submission.score,
comments_number=submission.num_comments,
nsfw=submission.over_18,
spoiler=submission.spoiler,
original=submission.is_original_content,
distinguished=submission.distinguished,
locked=submission.locked,
fetch_time=fetch_time)
title = submission.post_title
title_to_log = (title[:15] + '..') if len(title) > 15 else title
logging.info('SUBMISSION TITLE: ' + title_to_log)
x = signature('put_embeddings', args=[submission
]) | signature('send_to_mongo')
x.apply_async()
json_metrics = [{
"measurement": "submissions",
"time": str(datetime.now()),
"fields": {
"fetch_time": fetch_time,
"title_len": len(submission.post_title),
"text_len": len(submission.post_text),
"nsfw": str(submission.nsfw),
"spoiler": str(submission.spoiler),
"original_content": str(submission.original),
}
}]
try:
influxdb_client.write_points(json_metrics)
except influxdb.exceptions.InfluxDBServerError:
logging.info('Cannot send metrics to InfluxDB - Server Error.')
except influxdb.exceptions.InfluxDBClientError:
logging.info('Cannot send metrics to InfluxDB - Client Error.')
def notify_api_status(analysis_pk, task_status):
logging.info("Notify API: analysis_id={}, status={}".format(
analysis_pk,
task_status
))
signature(
'set_task_status',
args=(analysis_pk, task_status),
queue='celery'
).delay()
def register_worker(sender, **k):
m_supplier = os.environ.get('OASIS_MODEL_SUPPLIER_ID')
m_name = os.environ.get('OASIS_MODEL_ID')
m_id = os.environ.get('OASIS_MODEL_VERSION_ID')
logging.info(
'register_worker: SUPPLIER_ID={}, MODEL_ID={}, VERSION_ID={}'.format(
m_supplier, m_name, m_id))
signature('run_register_worker',
args=(m_supplier, m_name, m_id),
queue='celery').delay()
def upgrade(self, request, pk=None):
"""
更新服务
"""
data = request.data
data['version'] = data.get('version') or 'null'
for item in TbCetusNodeInfo.objects.filter(group_id=pk):
data.update(id=item.id)
TbCetusNodeInfo.objects.filter(pk=item.id).update(status=2)
signature('upgrade_node', kwargs=data).delay()
return Response('更新请求发送成功', status=status.HTTP_200_OK)
def install(self, request):
"""
安装服务
"""
data = request.data
data['version'] = data.get('version') or 'null'
data['path'] = data.get('path') or '/home'
cetus_info = self.create(request)
data['id'] = cetus_info.data.id
signature('install_cetus', kwargs=data).delay()
return Response('安装请求发送成功', status=status.HTTP_200_OK)
def run_copr_build_handler(self, event_data: dict, number_of_builds: int):
for _ in range(number_of_builds):
self.pushgateway.copr_builds_queued.inc()
signature(
TaskName.copr_build.value,
kwargs={
"package_config": dump_package_config(self.package_config),
"job_config": dump_job_config(self.job_config),
"event": event_data,
},
).apply_async()
def on_error(request, ex, traceback, record_task_name, analysis_pk,
initiator_pk):
"""
Because of how celery works we need to include a celery task registered in the
current app to pass to the `link_error` function on a chain.
This function takes the error and passes it on back to the server so that it can store
the info on the analysis.
"""
signature(record_task_name,
args=(analysis_pk, initiator_pk, traceback),
queue='celery').delay()
def _batch_by_size(collection_name, documents):
"""Groups a list of documents into batches which are below the size limit,
so that they can be sent to Solr together
"""
file_names = [path.json_text_path(d["pk"], d["slug"]) for d in documents]
text_sizes = storage.async_size(file_names)
docs_with_sizes = list(zip(documents, text_sizes))
docs_with_sizes.sort(key=lambda x: x[1], reverse=True)
batch = []
batch_size = 0
tasks = []
for document, size in docs_with_sizes:
if size > settings.SOLR_INDEX_MAX_SIZE:
# if the isngle document is too large to index at once, index it by itself
# so it can be indexed in pieces
logger.info("[SOLR INDEX] batching single document %s size %s",
document["pk"], size)
tasks.append(
signature(
"documentcloud.documents.tasks.solr_reindex_single",
args=[collection_name, document["pk"]],
))
elif (batch_size + size > settings.SOLR_INDEX_MAX_SIZE
or len(batch) >= settings.SOLR_INDEX_BATCH_LIMIT):
# if adding the next document would make the batch larger than the max
# size, or if the batch is already at the max size,
# then send the current batch to be indexed and start a new batch
logger.info("[SOLR INDEX] batch of %s size %s", batch, batch_size)
tasks.append(
signature(
"documentcloud.documents.tasks.solr_reindex_batch",
args=[collection_name, batch],
))
batch = [document["pk"]]
batch_size = size
else:
# otherwise add the current document to the current batch
batch.append(document["pk"])
batch_size += size
logger.info("[SOLR INDEX] batch of %s size %s", batch, batch_size)
tasks.append(
signature(
"documentcloud.documents.tasks.solr_reindex_batch",
args=[collection_name, batch],
))
return tasks
def _broadcast(**kwargs):
"""Broadcasts an event by calling the registered tasks."""
app_name = kwargs.pop('app_name', None)
event_name = kwargs.pop('event_name', None)
if app_name and event_name:
event = app.registry.event(app_name,
event_name,
raise_does_not_exist=True)
app.update_local_event(event)
for task in event.tasks:
signature(task.name, kwargs=kwargs,
queue=task.queue).delay()
def loadbook(bag,
mmsid=None,
outformat="JPEG",
filter="ANTIALIAS",
scale=0.4,
crop=None,
collection='islandora:bookCollection'):
"""
Generate derivative of Bag and load into S3.
args:
bag: name of bag to load
mmsid: MMS ID is needed to obtain MARC XML
outformat - string representation of image format - default is "JPEG".
Available Formats: http://pillow.readthedocs.io/en/3.4.x/handbook/image-file-formats.html
scale - percentage to scale by represented as a decimal
filter - string representing filter to apply to resized image - default is "ANTIALIAS"
crop - list of coordinates to crop from - i.e. [10, 10, 200, 200]
collection: Name of Islandora collection to ingest to. Default is: islandora:bookCollection
"""
# Generate derivatives and store in s3 an local
deriv_gen = signature(
"imageq.tasks.tasks.derivative_generation",
kwargs={
'bags': bag,
's3_bucket': s3_bucket,
's3_source': s3_source,
's3_destination': s3_derivative,
'outformat': outformat,
'filter': filter,
'scale': scale,
'crop': crop,
'upload_s3':
False # derivatives will be uploaded later during recipewriterq's process_derivative task
})
# generate recipe files and process derivatives into bags
process_derivs = signature("recipewriterq.tasks.tasks.process_derivative",
kwargs={'mmsid': mmsid})
# add entries to data catalog
# load into islandora
#ingest_recipe = signature("islandoraq.tasks.tasks.ingest_recipe", kwargs={'collection': collection})
#chain = (deriv_gen | process_derivs | ingest_recipe)
chain = (deriv_gen | process_derivs)
result = chain()
return "Kicked off tasks to generate derivative for {0}".format(bag)
def send_stacktrace_sentry(crash):
stacktrace = crash.stacktrace_json
exception = {
"values": [{
"type":
stacktrace.get('crash_info', {}).get('type', 'unknown exception'),
"value":
stacktrace.get('crash_info', {}).get('crash_address', '0x0'),
"stacktrace":
stacktrace['crashing_thread']
}]
}
data = {'sentry.interfaces.Exception': exception}
if crash.userid:
data['sentry.interfaces.User'] = dict(id=crash.userid)
extra = dict(
crash_admin_panel_url='http://{}{}'.format(
settings.HOST_NAME, '/admin/crash/crash/%s/' % crash.pk),
crashdump_url=crash.upload_file_minidump.url,
)
tags = {}
if crash.meta:
extra.update(crash.meta)
ver = crash.meta.get('ver')
if ver:
tags['ver'] = ver
if crash.channel:
tags['channel'] = crash.channel
if crash.archive:
extra['archive_url'] = crash.archive.url
tags.update(stacktrace.get('system_info', {}))
if crash.appid:
tags['appid'] = crash.appid
event_id = client.capture('raven.events.Message',
message=crash.signature,
extra=extra,
tags=tags,
data=data)
signature("tasks.get_sentry_link",
args=(crash.pk, event_id)).apply_async(queue='private',
countdown=1)
def process_video(video_name: str):
video = VideoCapture(str(Config.VIDEO_STORAGE / video_name))
frames = _frame_from_video(video)
chord_obj = chord(
[
signature(
"detector.detector.detect_frames",
args=(chunk, ),
queue="detector",
) for chunk in _chunkit(frames)
],
body=signature("detector.processor.merge_chunks", queue="processor"),
)
chord_res = chord_obj()
return chord_res.id
def group_on_results(
iterable: Iterable[Any],
task_signature: Dict[str, Any],
callback: Dict[str, Any],
task_signature_errback: Dict[str, Any] = None,
final: Optional[Dict[str, Any]] = None,
) -> Union[AsyncResult]:
task_group = group(*(chain(
signature(task_signature).clone((item, )).set(
link_error=task_signature_errback),
signature(callback),
) for item in iterable))
if final is not None:
return chord(task_group)(signature(final))
return task_group.apply_async()
def handle(self, *args, **options):
filter_qs = {}
# Don't include disabled projects by default
if not options['disabled_projects']:
filter_qs['entity__resource__project__disabled'] = False
# Don't include obsolete by default
if not options['obsolete_entities']:
filter_qs['entity__obsolete'] = False
translations_pks = (
Translation.objects
.filter(
entity__resource__format__in=DB_FORMATS,
**filter_qs
)
.values_list('pk', flat=True)
)
# Split translations into even batches and send them to Celery workers
batch_size = int(options['batch_size'])
group(
signature(
check_translations,
args=(translations_pks[i:i + batch_size],)
)
for i in range(0, len(translations_pks), batch_size)
).apply_async()
def transact_with_function_task(self,
signing_address,
contract_address,
contract_type,
func,
args=None,
gas_limit=None,
prior_tasks=None,
reverses_task=None):
kwargs = {
'signing_address': signing_address,
'contract_address': contract_address,
'abi_type': contract_type,
'function': func,
'args': args,
'prior_tasks': prior_tasks,
'reverses_task': reverses_task
}
if gas_limit:
kwargs['gas_limit'] = gas_limit
return signature(eth_endpoint('transact_with_contract_function'),
kwargs=kwargs)
def send_eth_task(self, signing_address, amount_wei, recipient_address):
return signature(eth_endpoint('send_eth'),
kwargs={
'signing_address': signing_address,
'amount_wei': amount_wei,
'recipient_address': recipient_address
})
def test_on_chord_part_return(self, restore):
tb = CacheBackend(backend='memory://', app=self.app)
deps = Mock()
deps.__len__ = Mock()
deps.__len__.return_value = 2
restore.return_value = deps
task = Mock()
task.name = 'foobarbaz'
self.app.tasks['foobarbaz'] = task
task.request.chord = signature(task)
result = self.app.GroupResult(
uuid(),
[self.app.AsyncResult(uuid()) for _ in range(3)],
)
task.request.group = result.id
tb.apply_chord(result, None)
deps.join_native.assert_not_called()
tb.on_chord_part_return(task.request, 'SUCCESS', 10)
deps.join_native.assert_not_called()
tb.on_chord_part_return(task.request, 'SUCCESS', 10)
deps.join_native.assert_called_with(propagate=True, timeout=3.0)
deps.delete.assert_called_with()
def test_on_chord_part_return(self, restore):
tb = CacheBackend(backend='memory://', app=self.app)
deps = Mock()
deps.__len__ = Mock()
deps.__len__.return_value = 2
restore.return_value = deps
task = Mock()
task.name = 'foobarbaz'
self.app.tasks['foobarbaz'] = task
task.request.chord = signature(task)
result = self.app.GroupResult(
uuid(),
[self.app.AsyncResult(uuid()) for _ in range(3)],
)
task.request.group = result.id
tb.apply_chord(result, None)
deps.join_native.assert_not_called()
tb.on_chord_part_return(task.request, 'SUCCESS', 10)
deps.join_native.assert_not_called()
tb.on_chord_part_return(task.request, 'SUCCESS', 10)
deps.join_native.assert_called_with(propagate=True, timeout=3.0)
deps.delete.assert_called_with()
def update_unconfigured_nodes():
DBSession = get_sql_session()
devices = DBSession.query(OpenWrt).filter(OpenWrt.configured==False)
for device in devices:
arguments = ( device.uuid, )
update_device_task = signature('openwifi.jobserver.tasks.get_config',args=arguments)
update_device_task.delay()
def get_provider_responses(provider=None):
"""
Get the price responses from the various providers
"""
providers_names = providers.__all__
provider_list = []
for provider_name in providers_names:
if provider is not None and provider_name != provider:
continue
# generate a task signature for this provider
provider_sig = signature(
getattr(tasks, 'get_provider_response'),
kwargs={
'provider_name': provider_name
},
immutable=True
)
# append to the list of signatures
provider_list.append(provider_sig)
# turn the list of signatures into a Celery group
provider_group = group(provider_list)
# then run the group
provider_group.apply_async()
def apply_async(self, task_signature, ignore_result=False,
additional_params=None, task_args=None, task_kwargs=None):
"""
Вызывает таск на стороне RabbitMQ источника
:param task_signature: string сигнатура вызова таска
:param ignore_result: bool признак "игнорировать результат от внешнего
приложения"
:param additional_params: dict дополнительные параметры (можно задать
queue, routing_key, ...)
:param task_args: list неименованные параметры
:param task_kwargs: dict именованные параметры
:return: результат выполнения таска
"""
signature_task = signature(task_signature, app=self.app)
async_result = signature_task.apply_async(
args=task_args or (),
kwargs=task_kwargs or {},
**self.get_default_celery_settings(additional_params)
)
if not ignore_result:
# мы ожидаем окончание выполнение таска и возвращаем его результат
# для дальнейшего логирования
external_task_result = async_result.get()
if async_result.state == 'FAILURE':
# при возникновении ошибки на стороне внешнего приложения,
# async_result.state == 'FAILURE', а в async_result.traceback
# содержится ее traceback
raise Exception(async_result.traceback)
return external_task_result
# не возвращаем результат выполнения внешнего таска, если явно
# игнорируем результат
return
def get_ccxt_responses():
"""
Get the ticker responses from the CCXT wrapper
"""
exchange_list = []
for exchange in ccxt.exchanges:
try:
wrapper = getattr(ccxt, exchange)()
except Exception:
continue
if not wrapper.has['fetchTicker']:
logger.info('No ticker available for {}'.format(exchange.title()))
continue
# generate a task signature for this exchange
exchange_sig = signature(
getattr(tasks, 'get_ccxt_response'),
kwargs={
'exchange': exchange
},
immutable=True
)
# append to the list of signatures
exchange_list.append(exchange_sig)
# turn the list of signatures into a Celery group
exchange_group = group(exchange_list)
# then run the group
exchange_group.apply_async()
def test_state_is_ready___run_is_started(self, status, task_id):
with TemporaryDirectory() as d:
with override_settings(MEDIA_ROOT=d):
res_factory = FakeAsyncResultFactory(target_task_id=task_id)
analysis = fake_analysis(status=status,
run_task_id=task_id,
input_file=fake_related_file(),
settings_file=fake_related_file())
initiator = fake_user()
sig_res = Mock()
sig_res.delay.return_value = res_factory(task_id)
with patch(
'src.server.oasisapi.analyses.models.Analysis.run_analysis_signature',
PropertyMock(return_value=sig_res)):
analysis.run(initiator)
sig_res.link.assert_called_once_with(
record_run_analysis_result.s(analysis.pk,
initiator.pk))
sig_res.link_error.assert_called_once_with(
signature('on_error',
args=('record_run_analysis_failure',
analysis.pk, initiator.pk),
queue=analysis.model.queue_name))
sig_res.delay.assert_called_once_with()
def create_signature(vertex):
task_name = 'tasks.{}.{}'.format(
vertex['domain'],
vertex['id']
)
task_config = vertex['parameters']
return signature(task_name, kwargs=task_config)
def send_stacktrace_sentry(crash):
stacktrace = crash.stacktrace_json
exception = {
"values": [
{
"type": stacktrace.get('crash_info', {}).get('type', 'unknown exception'),
"value": stacktrace.get('crash_info', {}).get('crash_address', '0x0'),
"stacktrace": stacktrace['crashing_thread']
}
]
}
data = {'sentry.interfaces.Exception': exception}
if crash.userid:
data['sentry.interfaces.User'] = dict(id=crash.userid)
extra = dict(
crash_admin_panel_url='http://{}{}'.format(
settings.HOST_NAME,
'/admin/crash/crash/%s/' % crash.pk),
crashdump_url=crash.upload_file_minidump.url,
)
tags = {}
if crash.meta:
extra.update(crash.meta)
ver = crash.meta.get('ver')
if ver:
tags['ver'] = ver
if crash.archive:
extra['archive_url'] = crash.archive.url
tags.update(stacktrace.get('system_info', {}))
if crash.appid:
tags['appid'] = crash.appid
event_id = client.capture(
'raven.events.Message',
message=crash.signature,
extra=extra,
tags=tags,
data=data
)
signature("tasks.get_sentry_link", args=(crash.pk, event_id)).apply_async(queue='private', countdown=1)
def form_valid(self, form):
email_sender = getattr(settings, 'EMAIL_SENDER', None)
email_recipients = getattr(settings, 'EMAIL_RECIPIENTS', None)
obj = form.save()
if email_sender and email_recipients:
(signature("tasks.send_email_feedback",
args=(obj.pk, email_sender, email_recipients))
.apply_async(queue='private', countdown=1))
return HttpResponse(obj.pk, status=200)
def create_task(self):
tid = uuid()
task = Mock(name='task-{0}'.format(tid))
task.name = 'foobarbaz'
self.app.tasks['foobarbaz'] = task
task.request.chord = signature(task)
task.request.id = tid
task.request.chord['chord_size'] = 10
task.request.group = 'group_id'
return task
def create_task():
tid = uuid()
task = Mock(name="task-{0}".format(tid))
task.name = "foobarbaz"
self.app.tasks["foobarbaz"] = task
task.request.chord = signature(task)
task.request.id = tid
task.request.chord["chord_size"] = 10
task.request.group = "group_id"
return task
def test_unregistered_task_can_be_used_as_error_callback(self, mock_group):
b = BaseBackend(app=self.app)
b._store_result = Mock()
request = Mock(name='request')
request.errbacks = [signature('doesnotexist',
immutable=True)]
exc = KeyError()
b.mark_as_failure('id', exc, request=request)
mock_group.assert_called_once_with(request.errbacks, app=self.app)
def automatically_tag_with_stored_estimator(all_analysis_ids, stored_estimator_id):
for analysis_ids in chunkify(all_analysis_ids, 100):
analyses = [(
analysis.id,
analysis.hu_moments,
analysis.centroid,
analysis.orientation_from_moments,
) for analysis in dm.ImageAnalysis.objects.filter(id__in=analysis_ids)]
estimator_object = dm.KMeansEstimator.objects.get(pk=stored_estimator_id)
estimator = estimator_object.rebuilt_estimator
scaler = estimator_object.rebuilt_scaler
label_deltas = estimator_object.label_deltas_defaultdict
(
celery.signature('drone.compute_automatic_tags_with_estimator',
args=(analyses, estimator, scaler, label_deltas)) |
celery.signature('results.store_automatic_analysis_tags')
).apply_async()
def automatically_tag_by_ellipse_search(all_image_ids, per_chunk=16):
results = list()
for image_ids in chunkify(all_image_ids, per_chunk):
taggables = list()
cals = dict()
for image in list(dm.Image.objects.filter(id__in=image_ids)):
data = image.jpeg
cal_name = image.cjr.cal_image.image_file.file.name
if cal_name not in cals:
cals[cal_name] = image.cal_jpeg
taggables.append((image.id, data, cal_name, image.search_envelope))
results.append((
celery.signature('drone.compute_automatic_tags_with_ellipse_search',
args=(taggables, cals)) |
celery.signature('results.store_ellipse_search_tags')
).apply_async())
return results
def trigger_org_task(task_name, queue="celery"):
"""
Triggers the given org task to be run for all active orgs
:param task_name: the full task name, e.g. 'myproj.myapp.tasks.do_stuff'
:param queue: the name of the queue to send org sub-tasks to
"""
active_orgs = apps.get_model("orgs", "Org").objects.filter(is_active=True)
for org in active_orgs:
sig = signature(task_name, args=[org.pk])
sig.apply_async(queue=queue)
logger.info("Requested task '%s' for %d active orgs" % (task_name, len(active_orgs)))
def handle_callback(callbacks=[]):
'''
A generic function that handles a list of callback functions, execute each callback in order
:param list(dict) callbacks: a list of celery signature objects
'''
if any(callbacks):
callbacks = [signature(x) for x in callbacks]
callbacks[0].kwargs['callbacks'] = callbacks[1:] #Pass the rest callbacks into callbacks[0] in keyword arguments 'callbacks'
logger.debug("Task: {0!r}".format(callbacks[0].get('task','Task name not found in handle_callback')))
should_delay = callbacks[0].get('options',{}).get('should_delay', True)
logger.debug("Should delay: {0!r}".format(should_delay))
callbacks[0].delay() if should_delay else callbacks[0]()
else:
logger.debug("No callbacks")
def create_task_status(task_name, args=None, kwargs=None):
sig = str(signature(task_name, args=args, kwargs=kwargs)) # args and kwargs required?
# Set status to PENDING since it's not running yet
init_status = PENDING
# Check for timelimit
annotations_for_task = resolve_annotations(task_name)
timelimit = annotations_for_task.get('time_limit')
# Determine when this task should expire
utc_before = datetime.now(tzutc())
expires_at = utc_before + timedelta(seconds=timelimit)
status = TaskStatus.objects.create(
status=init_status,
signature=sig,
expires_at=expires_at,
)
return status
def test_on_chord_part_return(self, restore):
b = self.MockBackend(app=self.app)
deps = Mock()
deps.__len__ = Mock()
deps.__len__.return_value = 10
restore.return_value = deps
b.client.incr.return_value = 1
task = Mock()
task.name = 'foobarbaz'
self.app.tasks['foobarbaz'] = task
task.request.chord = signature(task)
task.request.group = 'group_id'
b.on_chord_part_return(task)
self.assertTrue(b.client.incr.call_count)
b.client.incr.return_value = len(deps)
b.on_chord_part_return(task)
deps.join_native.assert_called_with(propagate=True, timeout=3.0)
deps.delete.assert_called_with()
self.assertTrue(b.client.expire.call_count)
def lock_task(task_name, *args, **kwargs):
"""
Try to lock a task: check whether or not a status entry exists for given
parameters or create one (preventing others from being created).
Returns whether or not the task was locked and a status instance.
"""
locked, status = False, None
# MUST use `str()`, otherwise it will invoke the task instead in the
# query up ahead!
sig = str(signature(task_name, args=args, kwargs=kwargs))
# Create a status if allowed
if not TaskStatus.objects\
.filter(Q(expires_at__gte=datetime.now(tzutc())) | Q(expires_at__isnull=True),
signature=sig)\
.filter(status__in=[PENDING, RECEIVED, STARTED, REVOKED, RETRY, IGNORED, REJECTED])\
.exists():
# Set status to PENDING since it's not running yet
init_status = PENDING
# Check for timelimit
annotations_for_task = resolve_annotations(task_name)
timelimit = annotations_for_task.get('time_limit')
# Determine when this task should expire
utc_before = datetime.now(tzutc())
expires_at = utc_before + timedelta(seconds=timelimit)
status = TaskStatus.objects.create(
status=init_status,
signature=sig,
expires_at=expires_at,
)
locked = True
return locked, status
def test_on_chord_part_return(self, restore):
tb = CacheBackend(backend='memory://', app=self.app)
deps = Mock()
deps.__len__ = Mock()
deps.__len__.return_value = 2
restore.return_value = deps
task = Mock()
task.name = 'foobarbaz'
self.app.tasks['foobarbaz'] = task
task.request.chord = signature(task)
gid, res = uuid(), [self.app.AsyncResult(uuid()) for _ in range(3)]
task.request.group = gid
tb.on_chord_apply(gid, {}, result=res)
self.assertFalse(deps.join_native.called)
tb.on_chord_part_return(task)
self.assertFalse(deps.join_native.called)
tb.on_chord_part_return(task)
deps.join_native.assert_called_with(propagate=True)
deps.delete.assert_called_with()
def beat_init_handler(sender=None, **kwargs):
# def get_run_times(trigger, first_run_time, end_time):
# if end_time == None:
# end_time = datetime.datetime.now(pytz.utc)
# run_times = []
# while first_run_time and first_run_time <= end_time:
# run_times.append(first_run_time)
# first_run_time = trigger.get_next_fire_time(first_run_time, first_run_time)
# return run_times
for v in CELERYBEAT_SCHEDULE.itervalues():
trigger = CronTrigger(hour=','.join(str(h) for h in v['schedule'].hour),
start_date=v['options']['eta'],
end_date=v['options'].get('expires', None),
timezone=pytz.utc)
next_fire_time = trigger.start_date
while next_fire_time and next_fire_time <= (trigger.end_date if trigger.end_date else datetime.datetime.now(pytz.utc)):
task = signature(v['task'], v.get('args', ()) + (next_fire_time,), v.get('kwargs', {}), v.get('options', {}), app)
try:
task()
except Exception as e:
logger.exception(e)
next_fire_time = trigger.get_next_fire_time(next_fire_time, next_fire_time)
def monitor():
return celery.group([celery.signature('psu.cached_report'),
celery.signature('cjc.queues_ping'),
celery.signature('cjc.thread_states')])()
def web_home_view(request):
LOG.error("web_home_view")
subtask = celery.signature('business.tasks.foo', args=(3,))
callback = celery.signature('business.tasks.foo_callback')
subtask.apply_async(link=callback)
return {'ping': 'pong'}
def add_cb(x, y, callback=None):
result = x + y
if callback:
return signature(callback).apply_async(result)
return result
def crash_post_save(sender, instance, created, *args, **kwargs):
if created and instance.upload_file_minidump:
signature("tasks.processing_crash_dump", args=(instance.pk,)).apply_async(queue='private', countdown=1)
def monitor_process(task, cluster, job, pid, nohup_out_path,
log_write_url=None, on_complete=None,
output_message='Job download/upload error: %s',
girder_token=None):
job_url = '%s/jobs/%s/log' % (cumulus.config.girder.baseUrl, job['_id'])
log = get_post_logger(job['_id'], girder_token, job_url)
headers = {'Girder-Token': girder_token}
job_id = job['_id']
status_url = '%s/jobs/%s' % (cumulus.config.girder.baseUrl, job_id)
try:
# if terminating break out
if _is_terminating(job, girder_token):
return
with get_connection(girder_token, cluster) as conn:
# See if the process is still running
output = conn.execute('ps %s | grep %s' % (pid, pid),
ignore_exit_status=True,
source_profile=False)
if len(output) > 0:
# Process is still running so schedule self again in about 5
# secs
# N.B. throw=False to prevent Retry exception being raised
task.retry(throw=False, countdown=5)
else:
try:
nohup_out_file_name = os.path.basename(nohup_out_path)
# Log the output
with conn.get(nohup_out_path) as fp:
output = fp.read()
if output.strip():
log.error(output_message % output)
# If we have output then set the error state on the
# job and return
r = requests.patch(status_url, headers=headers,
json={'status': JobState.ERROR})
check_status(r)
return
finally:
if nohup_out_file_name and \
os.path.exists(nohup_out_file_name):
os.remove(nohup_out_file_name)
# Fire off the on_compete task if we have one
if on_complete:
signature(on_complete).delay()
# If we where uploading move job to the complete state
if job['status'] == JobState.UPLOADING:
job_status = from_string(job['status'], task=task,
cluster=cluster, job=job,
log_write_url=log_write_url,
girder_token=girder_token,
conn=conn)
job_status = Complete(job_status)
job_status = job_status.next(JobQueueState.COMPLETE)
job_status.run()
r = requests.patch(status_url, headers=headers,
json={'status': str(job_status)})
check_status(r)
except EOFError:
# Try again
task.retry(throw=False, countdown=5)
except Exception as ex:
r = requests.patch(status_url, headers=headers,
json={'status': JobState.UNEXPECTEDERROR})
check_status(r)
get_job_logger(job, girder_token).exception(ex.message)
raise
def create_and_store_estimator(ted):
return (
celery.signature('learn.create_estimator', args=(ted,)) |
celery.signature('results.store_estimator')
).apply_async()
def regenerate_stacktrace(self, request, queryset):
for i in queryset:
signature("tasks.processing_crash_dump", args=(i.pk,)).apply_async(queue='default')
def cleanup(self):
model = self.initial['type'].split('__')
task_kwargs = self.get_task_kwargs()
signature("tasks.deferred_manual_cleanup", args=(model,), kwargs=task_kwargs).apply_async(queue='limitation')
def upload_job_output_to_item(cluster, job, log_write_url=None, job_dir=None,
girder_token=None):
headers = {'Girder-Token': girder_token}
job_id = job['_id']
status_url = '%s/jobs/%s' % (cumulus.config.girder.baseUrl, job_id)
try:
# if terminating break out
if _is_terminating(job, girder_token):
return
with get_connection(girder_token, cluster) as conn:
# First put girder client on master
path = inspect.getsourcefile(cumulus.girderclient)
with open(path, 'r') as fp:
conn.put(fp,
os.path.normpath(os.path.join(job_dir, '..',
os.path.basename(path))))
cmds = ['cd %s' % job_dir]
upload_cmd = 'python ../girderclient.py --token %s --url "%s" ' \
'upload --job %s' \
% (girder_token,
cumulus.config.girder.baseUrl, job['_id'])
upload_output = '%s.upload.out' % job_id
upload_output_path = os.path.normpath(os.path.join(job_dir, '..',
upload_output))
cmds.append('nohup %s &> ../%s &\n' % (upload_cmd, upload_output))
upload_cmd = _put_script(conn, '\n'.join(cmds))
output = conn.execute(upload_cmd)
# Remove upload script
conn.remove(upload_cmd)
if len(output) != 1:
raise Exception('PID not returned by execute command')
try:
pid = int(output[0])
except ValueError:
raise Exception('Unable to extract PID from: %s' % output)
on_complete = None
if _get_on_complete(job) == 'terminate':
cluster_log_url = '%s/clusters/%s/log' % \
(cumulus.config.girder.baseUrl, cluster['_id'])
on_complete = signature(
'cumulus.tasks.cluster.terminate_cluster',
args=(cluster,), kwargs={'log_write_url': cluster_log_url,
'girder_token': girder_token})
monitor_process.delay(cluster, job, pid, upload_output_path,
log_write_url=log_write_url,
on_complete=on_complete,
girder_token=girder_token)
except Exception as ex:
r = requests.patch(status_url, headers=headers,
json={'status': JobState.UNEXPECTEDERROR})
check_status(r)
get_job_logger(job, girder_token).exception(ex.message)
