def update_endpoints(self, endpoint_list):
# for now we process the enpoint even if they were already processed
endpoint_to_process = endpoint_list.endpoint_set.filter(processed=False)
total = endpoint_to_process.count()
count = 0
if not settings.SKIP_CELERY_TASK:
# the task workflow must be a group of serials requests to each endpoint.
# The celery chains links together signatures so that one is called after the other.
tasks = []
for endpoint in endpoint_to_process:
# use immutable signatures, we dont want the result of the previous
# task in the celery chain for the next task.
tasks.append(update_endpoint.si(endpoint))
chain(tasks)()
# update state
if not self.request.called_directly:
self.update_state(
state='PROGRESS',
meta={'current': count, 'total': total}
)
else:
for endpoint in endpoint_to_process:
update_endpoint(endpoint)
return True
def check_service(self, service):
# total is determined (and updated) exactly after service.update_layers
total = 100
def status_update(count):
if not self.request.called_directly:
self.update_state(
state='PROGRESS',
meta={'current': count, 'total': total}
)
status_update(0)
service.update_layers()
# we count 1 for update_layers and 1 for service check for simplicity
layer_to_process = service.layer_set.all()
total = layer_to_process.count() + 2
status_update(1)
service.check_available()
status_update(2)
count = 3
if not settings.SKIP_CELERY_TASK:
tasks = []
for layer in layer_to_process:
# update state
status_update(count)
tasks.append(check_layer.si(layer))
count += 1
chain(tasks)()
else:
for layer in layer_to_process:
status_update(count)
check_layer(layer)
count += 1
def import_task():
"""
Enter all of the daily import tasks into the queue, where they can grind away from there.
The import is broken up into tasks for a few reasons: it can be paused by stopping the sims queue if necessary;
works around the celery task time limit.
"""
if not settings.DO_IMPORTING_HERE:
return
tasks = [
daily_cleanup.si(),
fix_unknown_emplids.si(),
get_role_people.si(),
import_grads.si(),
get_update_grads_task(),
import_offerings.si(continue_import=True),
import_semester_info.si(),
import_active_grad_gpas.si(),
#get_import_offerings_task(),
#import_combined_sections.si(),
#send_report.si()
]
chain(*tasks).apply_async()
def give_to_pool(cls, pool, custom_limit=None):
"""Give appliances from shepherd to the pool where the maximum count is specified by pool
or you can specify a custom limit
"""
from appliances.tasks import (
appliance_power_on, mark_appliance_ready, wait_appliance_ready, appliance_yum_update,
appliance_reboot)
limit = custom_limit if custom_limit is not None else pool.total_count
appliances = []
with transaction.atomic():
for template in pool.possible_templates:
for appliance in cls.unassigned().filter(
template=template).all()[:limit - len(appliances)]:
with appliance.kill_lock:
appliance.appliance_pool = pool
appliance.save()
appliance.set_status("Given to pool {}".format(pool.id))
tasks = [appliance_power_on.si(appliance.id)]
if pool.yum_update:
tasks.append(appliance_yum_update.si(appliance.id))
tasks.append(
appliance_reboot.si(appliance.id, if_needs_restarting=True))
if appliance.preconfigured:
tasks.append(wait_appliance_ready.si(appliance.id))
else:
tasks.append(mark_appliance_ready.si(appliance.id))
chain(*tasks)()
appliances.append(appliance)
if len(appliances) == limit:
break
return len(appliances)
def main():
logging.basicConfig(level=logging.INFO, format="[%(asctime)s] %(module)s.%(funcName)s - %(message)s")
R = praw.Reddit(USER_AGENT)
R.login(os.environ["PT_REDDIT_USERNAME"], os.environ["PT_REDDIT_PASSWORD"])
logging.info("Starting check loop")
while True:
for message in R.get_unread():
message.mark_as_read()
if "/u/playlist-this!" not in message.body or not message.was_comment:
continue
if message.is_root:
parent_permalink = message.submission.permalink
else:
parent_permalink = message.submission.permalink + message.parent_id[3:]
chain(
tasks.get_songs_from.s(parent_permalink),
tasks.create_playlist.s(),
tasks.generate_playlist_comment.s(),
tasks.post_comment.s(message.permalink),
).apply_async()
logging.info("Sleeping for %s seconds", CHECK_INTERVAL)
time.sleep(CHECK_INTERVAL)
def add_bot_v(request, form = 0):
if form == 0:
form = BotForm()
if request.method == 'POST':
form = BotForm(request.POST, request.FILES)
if form.is_valid():
# get challenge
challenge = Challenge.objects.get( id = form.data["target_challenge"] )
# create directory for a bot
directory = challenge.directory + "bots/" + form.data["name"].replace(" ", "_") + "/"
os.makedirs(directory)
# upload source file
upload_file(directory, request.FILES['source_file'])
# gather data
program = Program(compiler = Compiler.objects.get(id = form.data["compiler"]),
source_file = directory + request.FILES['source_file'].name )
program.save()
bot = Bot( name = form.data["name"],
playing_program = program,
directory = directory,
owner = request.user,
target_challenge = challenge)
# delay compilation
recent_action = RecentAction(owner = request.user,
message = "Bot validation: " + bot.name
+ " (for challenge: " + challenge.title + ")",
state = ActionState.objects.get(name = 'IN_QUEUE'))
recent_action.save()
chain(compile_bot.si(bot, recent_action), enqueue_bots_battles.si(bot)).apply_async()
return HttpResponseRedirect('/successful/')
return render_to_response('ChallengeManagement/add_bot.xhtml',
{ "form": form, "title" : "Add Bot" },
context_instance = RequestContext(request));
def cache_and_train(*args, **kwargs):
"""
Cache plexon file (if using plexon system) and train BMI.
"""
# import config
if config.recording_sys['make'] == 'plexon':
print "cache and train"
entry = kwargs['entry']
print entry
plxfile = models.DataFile.objects.get(system__name='plexon', entry=entry)
print plxfile
if not plxfile.has_cache():
cache = cache_plx.si(plxfile.get_path())
train = make_bmi.si(*args, **kwargs)
chain(cache, train)()
else:
print "calling"
make_bmi.delay(*args, **kwargs)
elif config.recording_sys['make'] == 'blackrock':
make_bmi.delay(*args, **kwargs)
else:
raise Exception('Unknown recording_system!')
def serve(app_name, uuid):
client = xmlrpclib.Server(settings.EMPRESS_SERVICE_URL)
app_info = client.private.app_info(app_name)
if app_info.get('celery', {}).get('enabled'):
chain(
provide_virtualenv.s((app_info, uuid)),
pull_source_code.s(),
install_requirements.s(),
syncdb_and_migrate.s(),
render_uwsgi_config.s(),
render_nginx_config.s(),
reload_nginx.s(),
render_supervisor_config.s(),
reload_supervisor.s(),
end.s(),
).apply_async()
else:
chain(
provide_virtualenv.s((app_info, uuid)),
pull_source_code.s(),
install_requirements.s(),
syncdb_and_migrate.s(),
render_uwsgi_config.s(),
render_nginx_config.s(),
reload_nginx.s(),
end.s(),
).apply_async()
def fetch_with_chain(uid):
user = User.objects.select_related().get(uid=uid)
user_path = os.path.join(tempfile.tempdir, user.uid)
if not os.path.exists(user_path):
os.mkdir(user_path)
working_tasks = get_subdirectories(user_path)
if len(working_tasks):
working_task = working_tasks[-1]
task_result = AsyncResult(working_task)
return task_result.status
else:
task_id = str(uuid.uuid4())
task_path = os.path.join(user_path, task_id)
os.mkdir(task_path)
status = chain(
fetch_status.s(user.password),
parse_status.s(user.get_full_name(), task_path)
)
photos = chain(
fetch_photos.s(user.password),
download_photos.s(task_path)
)
friends = chain(
fetch_friends.s(user.password),
download_friends.s(task_path)
)
return chord(group(status, photos, friends), compress_and_upload.s(uid, task_path)).apply_async(task_id=task_id)
def liberate_collect_emails(results, mail_path, options):
""" Send off data mining tasks """
msg_tasks = []
results = results or []
for result in results:
inbox = [(mail_path, result['folder'], email_id) for email_id in result['ids']]
msg_tasks.extend(inbox)
task_len = len(msg_tasks)
if task_len > 0:
msg_tasks = liberate_message.chunks(msg_tasks, 100).group()
msg_tasks.skew(step=10)
msg_tasks = chain(
msg_tasks,
liberate_convert_box.s(mail_path, options),
liberate_fetch_info.s(options),
liberate_tarball.s(options),
liberation_finish.s(options)
)
else:
options["noEmails"] = True
data = {"results": []}
msg_tasks = chain(
liberate_convert_box.s(data, mail_path, options),
liberate_fetch_info.s(options),
liberate_tarball.s(options),
liberation_finish.s(options)
)
async_result = msg_tasks.apply_async()
lib_status = get_user_model().objects.get(id=options["user"]).liberation
lib_status.async_result = async_result.id
lib_status.save()
def _start_back_end(record, priority=False):
"""
kick off the asynchronous licence lookup process. There is no need for this to return
anything, although a handle on the asynchronous request object is provided for convenience of
testing
arguments:
record -- an OAG record object, see the module documentation for details
returns:
AsyncRequest object from the Celery framework
"""
log.debug("injecting record into asynchronous processing chain: " + str(record))
# ask the record to prep itself for injection into the processing chain
# this will return just the record dictionary (not the message object), and will
# have removed the existing journal information and added any processing flags
# that are relevant
chainable = record.prep_for_backend()
log.debug("record prepped for chain: " + str(chainable))
if priority:
ch = chain(priority_detect_provider.s(chainable), priority_provider_licence.s(), priority_store_results.s())
r = ch.apply_async()
return r
else:
ch = chain(detect_provider.s(chainable), provider_licence.s(), store_results.s())
r = ch.apply_async()
return r
def taskPeriodicTasks():
haveLock = False
Lock = redis.Redis().lock("project")
try:
haveLock = Lock.acquire(blocking=False)
logger.info(haveLock)
if haveLock:
logger.info("Locked")
chain(
scraper.getProjectsList(),
scraper.getRepos(),
scraper.getComponents(),
scraper.getBugStatus(),
scraper.getBugSeverity(),
scraper.getBugPriority(),
scraper.getTestCoverage(),
scraper.getSuccessDensity(),
scraper.getCommitCountTotal(),
scraper.getContributorsCount(),
scraper.getCommitCountLastWeek()
)
finally:
if haveLock:
logger.info("Released")
Lock.release()
def _execute_callback(self, lab_name, **kwargs):
action = kwargs["action"]
response = hresponse.HandlerResponse()
response.status_code = 202
response.reason = "Request accepted and being processed"
if action in ["boot", "test"]:
tasks = [
taskqueue.tasks.callback.lava_test.s(
self.json_obj, self.job_meta, lab_name),
]
if action == "test":
tasks.append(taskqueue.tasks.test.find_regression.s())
chain(tasks).apply_async(
link_error=taskqueue.tasks.error_handler.s())
else:
response.status_code = 404
response.reason = "Unsupported LAVA action: {}".format(action)
# Also run the legacy boot callback to generate boot entries
if action == "boot":
tasks = [
taskqueue.tasks.callback.lava_boot.s(
self.json_obj, self.job_meta, lab_name),
taskqueue.tasks.boot.find_regression.s(),
]
chain(tasks).apply_async(
link_error=taskqueue.tasks.error_handler.s())
return response
def test_mutable_chain_with_multiple_return_values():
"""See what happens when a task in a chain returns
multiple values
#NOTE: scoping tasks in a function is purely for readability,
these tasks will still b global and will still persist after
this test has completed.
RESULT: all return values are provided in a tuple in args[0]
"""
@task
def task_that_returns_multiple_values():
return 1, 2, 3, {'one': 1}
@task
def task_that_accepts_values(*args, **kwargs):
print(args)
print(kwargs)
chain(
task_that_returns_multiple_values.s(),
task_that_accepts_values.s(),
).apply_async().get()
assert 1==1
def test_chord_in_chords_with_chains(self, manager):
try:
manager.app.backend.ensure_chords_allowed()
except NotImplementedError as e:
raise pytest.skip(e.args[0])
c = chord(
group([
chain(
add.si(1, 2),
chord(
group([add.si(1, 2), add.si(1, 2)]),
add.si(1, 2),
),
),
chain(
add.si(1, 2),
chord(
group([add.si(1, 2), add.si(1, 2)]),
add.si(1, 2),
),
),
]),
add.si(2, 2)
)
r = c.delay()
assert r.get(timeout=TIMEOUT) == 4
def handle_analysis(analysis_result, event_id, locale='en'):
"""Handle analysis products"""
flood = Flood.objects.get(id=event_id)
flood.inspected_language = locale
task_state = 'FAILURE'
if analysis_result['status'] == RESULT_SUCCESS:
try:
flood.impact_file_path = analysis_result['output'][
'analysis_summary']
task_state = 'SUCCESS'
process_impact_layer.delay(flood)
chain(
get_keywords.s(
flood.impact_file_path,
keyword='keyword_version'
).set(queue=get_keywords.queue),
handle_keyword_version.s(
flood.id
).set(queue=handle_keyword_version.queue)
).delay()
except BaseException as e:
LOGGER.exception(e)
else:
LOGGER.error(analysis_result['message'])
flood.analysis_task_status = task_state
flood.analysis_task_result = json.dumps(analysis_result)
flood.save()
return analysis_result
def worker(self):
res_pastie = chain(task_pastie_grabber.s() |
task_check_link_redis.s() |
task_download_pastes.s() |
task_add_downloaded_link_redis.s()
)
res_nopaste = chain(task_nopaste_grabber.s() |
task_check_link_redis.s() |
task_download_pastes.s() |
task_add_downloaded_link_redis.s()
)
res_pastebin = chain(task_pastebin_grabber.s() |
task_check_link_redis.s() |
task_download_pastes.s() |
task_add_downloaded_link_redis.s()
)
# res_pastesite = chain(task_pastesite_grabber.s() |
# task_check_link_redis.s() |
# task_download_pastes.s() |
# task_add_downloaded_link_redis.s()
# )
g_res = group(res_nopaste,
res_pastie,
res_pastebin,
# res_pastesite
)
g_res.apply_async()
def reply_list(self, sms_wrapper, msgs):
"""Simple wrapper for returning a text message response given a list of texts to return
sms_wrapper: SmsMessage object (used for metadata)
msg: Iterable containing the messages to send. It is the caller's responsibility
to ensure they're the proper length.
"""
if not msgs or len(msgs) < 1:
return Response()
callback_url = self.request.build_absolute_uri(reverse('sms-callback'))
args = {
'callback': callback_url,
'from_': sms_wrapper.to_phone,
'to': sms_wrapper.from_phone,
}
# First item uses different call/args
# See http://docs.celeryproject.org/en/latest/userguide/canvas.html#chains
msg_chain = [sms_bunny.s(args, msgs[0])]
for m in msgs[1:]:
msg_chain.append(sms_bunny.subtask((m,), countdown=self._sms_delay))
try:
chain(*msg_chain).apply_async()
except socket.error:
# Not finding celery counts as 502: bad gateway
return HttpResponse('Couldn\'t connect to celery to send SMS messages', status=502)
return Response()
def UpdateRepo(repo):
# find our repository
repoRoot = CFGDATA.get("RepoRoot")
repoPath = os.path.join(repoRoot, repo)
if not os.path.exists(repoPath):
return
# lift the repo description as we might need to create the repo first
repoDesc = "This repository has no description"
repoDescFile = os.path.join(repoPath, "description")
if os.path.exists(repoDescFile):
with open(repoDescFile) as f:
repoDesc = f.read().strip()
# spawn push to github task first
if CFGDATA.get("GithubEnabled") and (not isExcluded(repo, CFGDATA.get("GithubExcepts"))):
githubPrefix = CFGDATA.get("GithubPrefix")
githubUser = CFGDATA.get("GithubUser")
githubRemote = "%[email protected]:%s/%s" % (githubUser, githubPrefix, repo)
createTask = CeleryWorkers.CreateRepoGithub.si(repo, repoDesc)
syncTask = CeleryWorkers.SyncRepo.si(repoPath, githubRemote, True)
celery.chain(createTask, syncTask)()
# now spawn all push to anongit tasks
if CFGDATA.get("AnongitEnabled") and (not isExcluded(repo, CFGDATA.get("AnongitExcepts"))):
anonUser = CFGDATA.get("AnongitUser")
anonPrefix = CFGDATA.get("AnongitPrefix")
for server in CFGDATA.get("AnongitServers"):
anonRemote = "%s@%s:%s/%s" % (anonUser, server, anonPrefix, repo)
createTask = CeleryWorkers.CreateRepoAnongit.si(repo, server, repoDesc)
syncTask = CeleryWorkers.SyncRepo.si(repoPath, anonRemote, False)
celery.chain(createTask, syncTask)()
def create_workflow_task(self, ctx):
"""create_workflow_task celery task
:param self: celery task instance
:param ctx: workflow state
:type ctx: dict
:return: dict(ctx, workflow)
"""
assert ctx['workflow_cycle_count'] > 0, 'workflow_cycle_count have to be bigger then 0'
assert ctx['operation_concurrent_count'] > 0, 'operation_concurrent_count have to be bigger then 0'
logger.info('creating workflow')
ctx.setdefault('workflow_count', 0)
ctx['workflow_count'] += 1
if ctx['workflow_count'] < ctx['workflow_cycle_count']:
logger.info('need to repeat workflow')
chord_callback_tasks = chain(
operation_summery_task.subtask((), {'ctx': ctx}),
create_workflow_task.subtask())
else:
logger.info('do not need to repeat workflow')
chord_callback_tasks = chain(
operation_summery_task.subtask((), {'ctx': ctx}),
cleanup_task.subtask(()),
)
logger.info('workflow count: {ctx[workflow_count]}'.format(ctx=ctx))
return {
'ctx': ctx,
'workflow': chord(
operation_task.s(index=index, ctx=ctx)
for index in xrange(ctx['operation_concurrent_count'])
)(chord_callback_tasks),
}
def task_group():
# 100 random num(group) -> filter -> sum (chain)
# execute several tasks in parallel.
g = group(rand.s(x) for x in range(100))
# chain a group together with another task will auto upgrade to be a chord
gchain = chain(g, filter.s(), xsum.s())
gchain.apply_async(countdown=1)
# logger.warning(datetime.now() + timedelta(seconds=5))
# gchain.apply_async(eta=datetime.now() + timedelta(seconds=5))
# gchain.apply_async(countdown=5)
job = group([
add.s(1, 2),
add.s(2, 3),
add.s(3, 4),
add.s(4, 5),
])
chain(job, xsum.s()).apply_async()
# chord(job, xsum.s()).apply_async()
# sync call
if settings.CELERY_ALWAYS_EAGER:
res = job.delay()
cnt = 0
while not res.ready():
print('wait job ready%s' % '.' * cnt)
# async call will stucked here
print('job result: %s' % sum(res.get(timeout=2)))
def test_tr55_job_error_in_chain(self):
model_input = {
'inputs': [],
'area_of_interest': {
'type': 'MultiPolygon',
'coordinates': [[
[[-75.06271362304688, 40.15893480687665],
[-75.2728271484375, 39.97185812402586],
[-74.99130249023438, 40.10958807474143],
[-75.06271362304688, 40.15893480687665]]
]]
},
'modification_pieces': [],
'modifications': [],
'modification_hash': 'j39fj9fg7yshb399h4nsdhf'
}
job_chain = views._construct_tr55_job_chain(model_input,
self.job.id)
self.assertTrue('tasks.start_histograms_job' in str(job_chain[0]))
self.assertTrue('tasks.get_histogram_job_results' in str(job_chain[1]))
job_chain = [get_test_histogram.s()] + job_chain[2:]
with self.assertRaises(Exception) as context:
chain(job_chain).apply_async()
self.assertEqual(str(context.exception),
'No precipitation value defined',
'Unexpected exception occurred')
def import_all(force_update_all=False):
"""Start all the importers sequentially and attempt
to move the result index to production
"""
app = create_app()
with app.blueprint_context(BLUEPRINT_NAME):
solr_server = app.config['PLACES_SOLR_SERVER']
staging_core = app.config['PLACES_SOLR_CORE_STAGING']
staging_core_url = '{server}/{core}/update'.format(server=solr_server, core=staging_core)
# make sure the "staging" index is empty
delete_response = requests.post(staging_core_url, '<delete><query>*:*</query></delete>', headers={'Content-type': 'text/xml'})
commit_response = requests.post(staging_core_url, '<commit/>', headers={'Content-type': 'text/xml'})
if delete_response.ok and commit_response.ok:
logger.info("Deleted all documents from staging, launching importers")
# Using a chain (seq) so tasks execute in order
chain(import_oxpoints.s(force_update=force_update_all),
import_oxpoints_organisation_descendants.s(force_update=force_update_all),
import_osm.s(force_update=force_update_all),
import_naptan.s(force_update=force_update_all),
import_ox_library_data.s(force_update=force_update_all),
swap_places_cores.s())()
return True
else:
logger.warning("Staging core not deleted correctly, aborting", extra={
'delete_response': delete_response.status_code,
'commit_response': commit_response.status_code
})
return False
def _using_lock(self, name, do_task, cleanup_tasks=None,
error_tasks=None):
"""
Applies lock for the deployment
:return: Lock object (dictionary)
:rtype: dict
"""
try:
lock = LockService().apply_lock(name)
except ResourceLockedException as lock_error:
raise self.retry(exc=lock_error)
_release_lock_s = _release_lock.si(lock)
cleanup_tasks = cleanup_tasks or []
if not isinstance(cleanup_tasks, list):
cleanup_tasks = [cleanup_tasks]
error_tasks = error_tasks or []
if not isinstance(error_tasks, list):
error_tasks = [error_tasks]
error_tasks.append(_release_lock_s)
cleanup_tasks.append(_release_lock_s)
return (
do_task
).apply_async(
link=chain(cleanup_tasks),
link_error=chain(error_tasks)
)
def get_district_attachment_pages(options, rd_pks, tag_names, session):
"""Get the attachment page information for all of the items selected
:param options: The options returned by argparse. Should have the following
keys:
- queue: The celery queue to use
- offset: The offset to skip
- limit: The limit to stop after
:param rd_pks: A list or ValuesList of RECAPDocument PKs to get attachment
pages for.
:param tag_names: A list of tags to associate with the downloaded items.
:param session: A PACER logged-in PacerSession object
:return None
"""
q = options['queue']
recap_user = User.objects.get(username='******')
throttle = CeleryThrottle(queue_name=q)
for i, rd_pk in enumerate(rd_pks):
if i < options['offset']:
continue
if i >= options['limit'] > 0:
break
throttle.maybe_wait()
chain(
get_attachment_page_by_rd.s(rd_pk, session.cookies).set(queue=q),
make_attachment_pq_object.s(rd_pk, recap_user.pk).set(queue=q),
process_recap_attachment.s(tag_names=tag_names).set(queue=q),
).apply_async()
def update_search_pattern_sets():
"""Update all records affected by search-patterned OAISets.
In order to avoid racing condition when editing the records, all
OAISet task groups are chained.
"""
oaisets = OAISet.query.filter(OAISet.search_pattern.isnot(None))
chain(make_oai_task_group(oais) for oais in oaisets).apply_async()
def test_report(self):
chain(tasks.requests_fetch.s(), tasks.requests_report.s()).delay()
self.assertEqual(len(mail.outbox), 1)
self.assertIn("Amount: 200", mail.outbox[0].body)
self.assertIn("User: %s" % (self.user.username), mail.outbox[0].body)
self.assertIn("Date:", mail.outbox[0].body)
self.assertIn("Current: %s" % (self.user.userprofile.pool_amount,), mail.outbox[0].body)
def handle(self, text):
try:
chain(capture_image.s(),
send_twitter_status.s(text),
text_back.s(self.msg.connection.identity)).apply_async()
except Exception as ex:
logger.warn(ex)
return
def _run_reindex_tasks(self, models, queue):
apply_async_kwargs = {'priority': 0}
if queue:
log.info('Adding indexing tasks to queue {0}'.format(queue))
apply_async_kwargs['queue'] = queue
else:
log.info('Adding indexing tasks to default queue')
index_time = timezone.now()
timestamp = datetime.datetime.now().strftime('%Y%m%d%H%M%S')
for doc in registry.get_documents(models):
queryset = doc().get_queryset()
# Get latest object from the queryset
app_label = queryset.model._meta.app_label
model_name = queryset.model.__name__
index_name = doc._doc_type.index
new_index_name = "{}_{}".format(index_name, timestamp)
# Set index temporarily for indexing,
# this will only get set during the running of this command
doc._doc_type.index = new_index_name
pre_index_task = create_new_es_index.si(app_label=app_label,
model_name=model_name,
index_name=index_name,
new_index_name=new_index_name)
indexing_tasks = self._get_indexing_tasks(app_label=app_label, model_name=model_name,
queryset=queryset,
index_name=new_index_name,
document_class=str(doc))
post_index_task = switch_es_index.si(app_label=app_label, model_name=model_name,
index_name=index_name,
new_index_name=new_index_name)
# Task to run in order to add the objects
# that has been inserted into database while indexing_tasks was running
# We pass the creation time of latest object, so its possible to index later items
missed_index_task = index_missing_objects.si(app_label=app_label,
model_name=model_name,
document_class=str(doc),
index_generation_time=index_time)
# http://celery.readthedocs.io/en/latest/userguide/canvas.html#chords
chord_tasks = chord(header=indexing_tasks, body=post_index_task)
if queue:
pre_index_task.set(queue=queue)
chord_tasks.set(queue=queue)
missed_index_task.set(queue=queue)
# http://celery.readthedocs.io/en/latest/userguide/canvas.html#chain
chain(pre_index_task, chord_tasks, missed_index_task).apply_async(**apply_async_kwargs)
message = ("Successfully issued tasks for {}.{}, total {} items"
.format(app_label, model_name, queryset.count()))
log.info(message)
def call_check(data):
join_room(data["transaction_id"])
url = data["url"]
if not re.match(r"^http\://", url):
data["url"] = "http://%s" % url
for location in app.config["LOCATIONS"]:
for testsuite in location["testsuites"]:
input_data = {
"transaction_id": data["transaction_id"],
"location": location["location"],
"country": location["country"],
"ISP": location["ISP"],
"url": data["url"],
}
location_queue = "%s_%s" % (location["location"].lower().replace(" ", "_"), location["ISP"].lower().replace(" ", "_"))
logging.warn(location_queue)
input_data["test_type"] = testsuite
if testsuite in ("dns_google", "dns_TM", "dns_opendns"):
for server in app.config["TESTSUITES"][testsuite]["servers"]:
input_data["task_id"] = str(uuid.uuid4())
extra_attr = {
"provider": app.config["TESTSUITES"][testsuite]["provider"],
"server": server
}
input_data["extra_attr"] = extra_attr
logging.warn("DNS Check")
description = "%s server: %s " % (app.config["TESTSUITES"][testsuite]["description"], server)
input_data["description"] = description
result_data = ResultData.from_json(input_data, extra_attr=extra_attr)
task = chain(
call_dns_task.s(result_data.to_json()).set(queue=location_queue),
update_entry.s().set(queue="basecamp"),
post_update.s().set(queue="basecamp")
).apply_async()
else:
input_data["task_id"] = str(uuid.uuid4())
input_data["description"] = app.config["TESTSUITES"][testsuite]["description"]
result_data = ResultData.from_json(input_data)
if testsuite == "http":
task = chain(
call_http_task.s(result_data.to_json()).set(queue=location_queue),
update_entry.s().set(queue="basecamp"),
post_update.s().set(queue="basecamp")
).apply_async()
elif testsuite == "http_dpi_tampering":
task = chain(
call_http_dpi_tampering_task.s(result_data.to_json()).set(queue=location_queue),
update_entry.s().set(queue="basecamp"),
post_update.s().set(queue="basecamp")
).apply_async()
emit("result_received", result_data.to_json(), room=result_data.transaction_id)
def create_job(**body):
"""
This function creates a new Transfer job.
TODO add some logic
"""
msg = body['body']
tempfile = 'temp/' + str(uuid.uuid1())
def _in():
if 'http' in msg['input']['type']:
LOGGER.info(msg['input']['type']['http'])
_input = 'fromUrl'
return msg['input']['type']['http'], _input
if 'ftp' in msg['input']['type']:
_input = 'fromFtp'
return msg['input']['type']['ftp'], _input
def _out():
if 'filesystem' in msg['output']['type']:
_output = 'toFilesystem'
return msg['output']['type']['filesystem'], _output
if 'ftp' in msg['output']['type']:
_output = 'toFtp'
return msg['output']['type']['ftp'], _output
worker_msg = {**_in()[0], **_out()[0]}
_input = _in()[1]
_output = _out()[1]
LOGGER.info(worker_msg)
if _output == 'toFilesystem':
LOGGER.info('output set to filesystem path')
if _output == 'toFtp':
LOGGER.info('output set to ftp')
# worker logic comes here
# Todo refactor to something useful
if _input == 'fromUrl' and _output == 'toFilesystem':
if 'user' in worker_msg and 'passwd' in worker_msg:
job = transferFromUrl.s(uri=worker_msg['uri'],
destPath=worker_msg['destPath'],
user=worker_msg['user'],
password=worker_msg['passwd'])
task = job.apply_async(retry=True)
job_id = task.id
else:
job = transferFromUrl.s(uri=worker_msg['uri'],
destPath=worker_msg['destPath'])
task = job.apply_async()
job_id = task.id
worker_msg['job_id'] = job_id
return worker_msg
if _input == 'fromFtp' and _output == 'toFilesystem':
job = transferFromFtp.s(password=worker_msg['passwd'],
host=worker_msg['host'],
user=worker_msg['user'],
path=worker_msg['path'],
destpath=worker_msg['destPath'])
task = job.apply_async(retry=True)
job_id = task.id
worker_msg['job_id'] = job_id
return worker_msg
if _input == 'fromFtp' and _output == 'toFtp':
job = chain(
transferFromFtp.s(password=worker_msg['passwd'],
host=worker_msg['host'],
user=worker_msg['user'],
path=worker_msg['path'],
destpath=tempfile),
ftp_result.s(server=worker_msg['ftpHost'],
destfile=worker_msg['ftpPath'],
user=worker_msg['ftpUser'],
passwd=worker_msg['ftpPasswd']))
task = job.apply_async(retry=True)
job_id = task.id
worker_msg['job_id'] = job_id
delta_time = datetime.utcnow() + timedelta(days=1)
LOGGER.info('scheduled deletion of %s in 1 day', tempfile)
deleteFile.apply_async((tempfile, ), eta=delta_time)
return worker_msg
if _input == 'fromUrl' and _output == 'toFtp':
if 'user' in worker_msg and 'passwd' in worker_msg:
job = chain(
transferFromUrl.s(uri=worker_msg['uri'],
user=worker_msg['user'],
password=worker_msg['passwd'],
destPath=tempfile),
ftp_result.s(server=worker_msg['ftpHost'],
destfile=worker_msg['ftpPath'],
user=worker_msg['ftpUser'],
passwd=worker_msg['ftpPasswd']))
else:
job = chain(
transferFromUrl.s(uri=worker_msg['uri'], destPath=tempfile),
ftp_result.s(server=worker_msg['ftpHost'],
destfile=worker_msg['ftpPath'],
user=worker_msg['ftpUser'],
passwd=worker_msg['ftpPasswd']))
task_chain = job.apply_async(retry=True)
job_id = task_chain.id
worker_msg['job_id'] = job_id
delta_time = datetime.utcnow() + timedelta(days=1)
LOGGER.info('scheduled deletion of %s in 1 day', tempfile)
deleteFile.apply_async((tempfile, ), eta=delta_time)
return worker_msg
from celery import chain, group, chord
from tasks import add, xsum, raise_error, on_chord_error
if __name__ == '__main__':
## chain primitive
res = chain(add.s(2, 2), add.s(4), add.s(8))()
print('Chain result: %s' % res.get())
# shortcut of above
res = (add.s(2, 2) | add.s(4) | add.s(8))()
print('Chain shortcut: %s' % res.get())
res = chain(add.si(2, 2), add.si(4, 5), add.si(8, 8))()
print('Chain with independent task: %s' % res.get()) # 8 + 8
print('Parent result: %s' % res.parent.get()) # 4 + 5
print('Parent of parent result: %s' % res.parent.parent.get()) # 2 + 2
## group primitive
res = group(add.s(i, i) for i in range(10))()
print('Group result: %s' % res.get())
## chord primitive
res = chord((add.s(i, i) for i in range(10)), xsum.s())()
# is equal to: group(add.s(i, i) for i in range(10)) | xsum.s()
print('Chord result: %s' % res.get())
res = chord([add.s(2, 2), raise_error.s(), add.s(4, 4)], xsum.s())()
def task_delete_ibm_subnet_workflow(self, task_id, cloud_id, region,
subnet_id):
"""
This request deletes a VPC Subnet and its attached resources
such as vpn gateways and its attached resources,load balancers
and its attached resources,and the instances and its floating ip
@return:
"""
workflow_steps, subnet_tasks_list, floating_ip_tasks_list = list(), list(
), list()
vpn_instance_tasks_list, load_balancer_task_list = list(), list()
ibm_subnet = doosradb.session.query(IBMSubnet).filter_by(
id=subnet_id).first()
if not ibm_subnet:
return
for lb in ibm_subnet.ibm_load_balancers:
load_balancer_task_list.append(
task_delete_ibm_load_balancer.si(task_id=task_id,
cloud_id=cloud_id,
region=region,
load_balancer_id=lb.id))
for vpn in ibm_subnet.vpn_gateways.all():
vpn_instance_tasks_list.append(
task_delete_ibm_vpn_gateway.si(task_id=task_id,
cloud_id=cloud_id,
region=region,
vpn_id=vpn.id))
for network_interface in ibm_subnet.network_interfaces.all():
if network_interface.ibm_instance:
if network_interface.floating_ip:
floating_ip_tasks_list.append(
task_delete_ibm_floating_ip.si(
task_id=task_id,
cloud_id=cloud_id,
region=region,
floating_ip_id=network_interface.floating_ip.id))
vpn_instance_tasks_list.append(
task_delete_ibm_instance.si(
task_id=task_id,
cloud_id=cloud_id,
region=region,
instance_id=network_interface.ibm_instance.id))
if load_balancer_task_list and len(load_balancer_task_list) == 1:
workflow_steps.extend(load_balancer_task_list)
elif load_balancer_task_list:
workflow_steps.append(
chord(
group(load_balancer_task_list),
update_group_tasks.si(
task_id=task_id,
cloud_id=cloud_id,
region=region,
message="Load Balancers Tasks Chord Finisher")))
if floating_ip_tasks_list and len(floating_ip_tasks_list) == 1:
workflow_steps.extend(floating_ip_tasks_list)
elif floating_ip_tasks_list:
workflow_steps.append(
chord(
group(floating_ip_tasks_list),
update_group_tasks.si(
task_id=task_id,
cloud_id=cloud_id,
region=region,
message="Floating IP's Tasks Chord Finisher")))
if vpn_instance_tasks_list and len(vpn_instance_tasks_list) == 1:
workflow_steps.extend(vpn_instance_tasks_list)
elif vpn_instance_tasks_list:
workflow_steps.append(
chord(
group(vpn_instance_tasks_list),
update_group_tasks.si(
task_id=task_id,
cloud_id=cloud_id,
region=region,
message="VPN/VSI's Tasks Chord Finisher")))
workflow_steps.append(
task_delete_ibm_subnet.si(task_id=task_id,
cloud_id=cloud_id,
region=region,
subnet_id=ibm_subnet.id))
workflow_steps.append(update_ibm_task.si(task_id=task_id))
chain(workflow_steps).delay()
def post(self, request):
"""
Create a new Project. Projects and the root File directory for a Project should
be owned by the portal, with roles/permissions granted to the creating user.
1. Create the metadata record for the project
2. Create a directory on the projects storage system named after the metadata uuid
3. Associate the metadata uuid and file uuid
:param request:
:return: The newly created project
:rtype: JsonResponse
"""
# portal service account needs to create the objects on behalf of the user
ag = get_service_account_client()
if request.is_ajax():
post_data = json.loads(request.body)
else:
post_data = request.POST.copy()
# create Project (metadata)
metrics.info('projects',
extra={
'user': request.user.username,
'sessionId': getattr(request.session, 'session_key',
''),
'operation': 'project_create',
'info': {
'postData': post_data
}
})
prj_model = project_lookup_model({
'name': 'designsafe.project',
'value': post_data
})
prj = prj_model(value=post_data)
project_uuid = prj.uuid
prj.manager().set_client(ag)
prj.save(ag)
# create Project Directory on Managed system
metrics.info('projects',
extra={
'user': request.user.username,
'sessionId': getattr(request.session, 'session_key',
''),
'operation': 'base_directory_create',
'info': {
'systemId': Project.STORAGE_SYSTEM_ID,
'uuid': prj.uuid
}
})
project_storage_root = BaseFileResource(ag, Project.STORAGE_SYSTEM_ID,
'/')
project_storage_root.mkdir(prj.uuid)
# Wrap Project Directory as private system for project
project_system_tmpl = template_project_storage_system(prj)
project_system_tmpl['storage']['rootDir'] = \
project_system_tmpl['storage']['rootDir'].format(project_uuid)
metrics.info('projects',
extra={
'user': request.user.username,
'sessionId': getattr(request.session, 'session_key',
''),
'operation': 'private_system_create',
'info': {
'id':
project_system_tmpl.get('id'),
'site':
project_system_tmpl.get('site'),
'default':
project_system_tmpl.get('default'),
'status':
project_system_tmpl.get('status'),
'description':
project_system_tmpl.get('description'),
'name':
project_system_tmpl.get('name'),
'globalDefault':
project_system_tmpl.get('globalDefault'),
'available':
project_system_tmpl.get('available'),
'public':
project_system_tmpl.get('public'),
'type':
project_system_tmpl.get('type'),
'storage': {
'homeDir':
project_system_tmpl.get('storage',
{}).get('homeDir'),
'rootDir':
project_system_tmpl.get('storage',
{}).get('rootDir')
}
}
})
ag.systems.add(body=project_system_tmpl)
# grant initial permissions for creating user and PI, if exists
metrics.info('projects',
extra={
'user': request.user.username,
'sessionId': getattr(request.session, 'session_key',
''),
'operation': 'initial_pems_create',
'info': {
'collab': request.user.username,
'pi': prj.pi
}
})
if getattr(prj, 'copi', None):
prj.add_co_pis(prj.copi)
elif getattr(prj, 'co_pis', None):
prj.add_co_pis(prj.co_pis)
if getattr(prj, 'team', None):
prj.add_team_members(prj.team)
elif getattr(prj, 'team_members', None):
prj.add_team_members(prj.team_members)
prj._add_team_members_pems([prj.pi])
if request.user.username not in list(
set(prj.co_pis + prj.team_members + [prj.pi])):
# Add creator to project as team member
prj.add_team_members([request.user.username])
# Email collaborators
chain(
tasks.set_project_id.s(prj.uuid).set(queue="api")
| tasks.email_collaborator_added_to_project.s(
prj.uuid, prj.title,
request.build_absolute_uri('{}/projects/{}/'.format(
reverse('designsafe_data:data_depot'), prj.uuid)),
[
u
for u in list(set(prj.co_pis + prj.team_members +
[prj.pi])) if u != request.user.username
], [])).apply_async()
tasks.set_facl_project.apply_async(args=[
prj.uuid,
list(set(prj.co_pis + prj.team_members + [prj.pi]))
],
queue='api')
prj.add_admin('prjadmin')
return JsonResponse(prj.to_body_dict(), safe=False)
def dispatch_faculty_notifications(self, user_id):
"""
Dispatch daily summary notifications that are directed at faculty
:param self:
:param user_id:
:return:
"""
try:
user = db.session.query(User).filter_by(id=user_id).first()
except SQLAlchemyError as e:
current_app.logger.exception("SQLAlchemyError exception",
exc_info=e)
raise self.retry()
if user is None:
self.update_state('FAILURE',
meta='Could not read database records')
raise Ignore()
# create snapshot of notifications list; this is what we use *everywhere* to decide which notifications
# to process, in order to avoid race conditions with other threads adding notifications to the database
raw_list = [(n.id, n.event_type)
for n in user.unheld_email_notifications]
# strip out all notification ids, no matter what type of event we are dealing with
n_ids = [x[0] for x in raw_list]
# strip out all notification ids for newly created events
c_ids = [
x[0] for x in raw_list
if x[1] == EmailNotification.CONFIRMATION_REQUEST_CREATED
]
# if we are not grouping notifications into summaries for this user,
# generate a task to send each email in the queue
task = None
if not user.group_summaries:
if len(n_ids) > 0:
task = group(
dispatch_faculty_single_email.s(user_id, n_id)
for n_id in n_ids)
# we always generate a possible summary, even if the 'group summaries' option is not used,
# to advise of confirmation requests that are not being handled in a timely fashion.
has_summary = False
allow_summary = user.last_email is None
if not allow_summary:
time_since_last_email = datetime.now() - user.last_email
# cutoff is user-specified frequency in days, minus 30 minutes overhead
# eg. messages sent after a 5pm scheduled mailing would be recording as being issued at
# 1701 or 1702. The next day, this value for user.last_email would appear to be 23 hours 59/58 mins
# and prevent another email being issued
frequency = timedelta(days=user.summary_frequency) - timedelta(
hours=0.5)
allow_summary = time_since_last_email > frequency
if allow_summary:
cr_ids = _get_outstanding_faculty_confirmation_requests(user)
summary_ids = n_ids if user.group_summaries else []
if len(summary_ids) > 0 or len(cr_ids) > 0:
has_summary = True
if task is None:
task = dispatch_faculty_summary_email.s(
None, user.id, summary_ids, cr_ids)
else:
task = task | dispatch_faculty_summary_email.s(
user.id, summary_ids, cr_ids)
# if nothing to do, then return
if task is None:
return
if not has_summary:
task = task | no_summary_adapter.s()
# if there *is* something to do, also dispatch paired emails to both faculty and students
if len(c_ids) > 0:
# we have to set up this task as a chain so that we can sequentially pass through any
# list of previously-handled ids
# If we used a group, each dispatch_new_request_notification() would return its own
# copy of this list, so we would end up with a list-of-lists in reset_notifications
if task is None:
c_id_head = c_ids[0]
c_ids_tail = c_ids[1:]
if len(c_ids_tail) > 0:
task = dispatch_new_request_notification.s(
[], c_id_head) | chain(
dispatch_new_request_notification.s(c_id)
for c_id in c_ids_tail)
else:
task = dispatch_new_request_notification.s([], c_id_head)
else:
task = task | chain(
dispatch_new_request_notification.s(c_id)
for c_id in c_ids)
task = task | group(reset_notifications.s(),
reset_last_email_time.s(user_id))
raise self.replace(task)
def by_chain():
# 可以将任务链接在一起，以便在一个任务返回后又调用另一个任务,串行
res = chain(add.s(10) | add.s(20))(10) # add(add(10, 10), 20)
print("**********链式处理结果：{0}***********".format(res.get()))
print("**********链式处理父的结果：{0}***********".format(res.parent.get())) # 一个parent对应一个父结果
def build_index_pipeline(file_path='',
events='',
timeline_name='',
index_name='',
file_extension='',
sketch_id=None,
only_index=False,
timeline_id=None):
"""Build a pipeline for index and analysis.
Args:
file_path: The full path to a file to upload, either a file_path or
or events need to be defined.
events: String with the event data, either file_path or events
needs to be defined.
timeline_name: Name of the timeline to create.
index_name: Name of the index to index to.
file_extension: The file extension of the file.
sketch_id: The ID of the sketch to analyze.
only_index: If set to true then only indexing tasks are run, not
analyzers. This is to be used when uploading data in chunks,
we don't want to run the analyzers until all chunks have been
uploaded.
timeline_id: Optional ID of the timeline object this data belongs to.
Returns:
Celery chain with indexing task (or single indexing task) and analyzer
task group.
"""
if not (file_path or events):
raise RuntimeError(
'Unable to upload data, missing either a file or events.')
index_task_class = _get_index_task_class(file_extension)
index_analyzer_chain = _get_index_analyzers()
sketch_analyzer_chain = None
searchindex = SearchIndex.query.filter_by(index_name=index_name).first()
index_task = index_task_class.s(file_path, events, timeline_name,
index_name, file_extension, timeline_id)
if only_index:
return index_task
if sketch_id:
sketch_analyzer_chain, _ = build_sketch_analysis_pipeline(
sketch_id, searchindex.id, user_id=None)
# If there are no analyzers just run the indexer.
if not index_analyzer_chain and not sketch_analyzer_chain:
return index_task
if sketch_analyzer_chain:
if not index_analyzer_chain:
return chain(index_task, run_sketch_init.s(),
sketch_analyzer_chain)
return chain(index_task, index_analyzer_chain, run_sketch_init.s(),
sketch_analyzer_chain)
if current_app.config.get('ENABLE_EMAIL_NOTIFICATIONS'):
return chain(index_task, index_analyzer_chain,
run_email_result_task.s())
return chain(index_task, index_analyzer_chain)
def run(self):
"""Executes the current batch definition.
Expands the current batch definition to a series of celery chains and
executes them asynchronously. Additionally a batch record is written to
the celery result backend.
Returns:
(unicode): Batch identifier.
"""
if self.lock:
raise NidabaInputException('Executed batch may not be modified')
# reorder task definitions
keys = [
'img', 'binarize', 'segmentation', 'ocr', 'stats',
'postprocessing', 'output', 'archive'
]
tasks = OrderedDict((key, self.tasks[key]) for key in keys)
first = []
prev = None
result_data = {}
self.lock = True
# build chain
root_docs = self.docs
prev = []
for group, step in tasks.iteritems():
# skip groups without tasks
if not step:
continue
sequential = True if self.order[group][0] == 'sequence' else False
mmode = self.order[group][1]
def _repeat(lst, n):
return list(
itertools.chain.from_iterable(
itertools.repeat(x, n) for x in lst))
if sequential:
step = [step]
# multiply number of tasks in this step by number of tasks in
# previous step if not merging
if not mmode:
step = _repeat(step, len(root_docs))
root_docs = root_docs * (len(step) / len(root_docs))
# by number of root docs if doc merging
elif mmode == 'doc':
step = _repeat(step, len(self.docs))
root_docs = self.docs
else:
root_docs = [root_docs] * len(step)
if not sequential:
step = [[x] for x in step]
nprev = []
r = []
for rd_idx, (rdoc, c) in enumerate(zip(root_docs, step)):
if sequential:
r.append([])
for idx, (fun, kwargs) in enumerate(c):
# if idx > 0 (sequential == true) parent is previous task in sequence
if idx > 0:
parents = [task_id]
# if merge mode is 'doc' base parents are tasks n * (len(prev)/len(docs)) to n+1 ...
elif mmode == 'doc':
parents = prev[rd_idx::len(root_docs)]
# if merging everything all tasks in previous step are parents
elif mmode:
parents = prev
# if not merging a single task in previous step is the parent
elif mmode is False:
parents = [prev[rd_idx % len(prev)]] if prev else prev
task_id = uuid.uuid4().get_hex()
# last task in a sequence is entered into new prev array
if idx + 1 == len(c):
nprev.append(task_id)
result_data[task_id] = {
'children': [],
'parents': parents,
'root_documents': rdoc if mmode else [rdoc],
'state': 'PENDING',
'result': None,
'task': (group, fun, kwargs)
}
for parent in parents:
result_data[parent]['children'].append(task_id)
task = self.celery.app.tasks[u'nidaba.{}.{}'.format(
group, fun)]
if sequential:
r[-1].append(
task.s(batch_id=self.id, task_id=task_id,
**kwargs))
else:
r.append(
task.s(batch_id=self.id, task_id=task_id,
**kwargs))
prev = nprev
t = self.celery.app.tasks[u'nidaba.util.barrier'].s(
merging=mmode,
sequential=sequential,
replace=r,
root_docs=self.docs)
first.append(t)
with self.redis.pipeline() as pipe:
while (1):
try:
pipe.watch(self.id)
self._restore_and_create_scratchpad(pipe)
# also deletes the scratchpad
pipe.set(self.id, json.dumps(result_data))
pipe.execute()
break
except WatchError:
continue
chain(first).apply_async(args=[self.docs])
return self.id
def do_transaction(list_signatures):
task_list = []
for sig in list_signatures:
task_list += [sig, monitor.s()]
res = chain(*task_list)
return res()
def test_chain_of_chain_with_a_single_task(self, manager):
sig = signature('any_taskname', queue='any_q')
chain([chain(sig)]).apply_async()
def test_rebuild_nested_group_group(self, manager):
sig = chain(tasks.return_nested_signature_group_group.s(),
tasks.rebuild_signature.s())
sig.delay().get(timeout=TIMEOUT)
def get_task_signature(
cls, tenant, serialized_tenant, pull_security_groups=True, **kwargs
):
""" Create tenant, add user to it, create internal network, pull quotas """
# we assume that tenant one network and subnet after creation
network = tenant.networks.first()
subnet = network.subnets.first()
serialized_network = core_utils.serialize_instance(network)
serialized_subnet = core_utils.serialize_instance(subnet)
creation_tasks = [
core_tasks.BackendMethodTask().si(
serialized_tenant,
'create_tenant_safe',
state_transition='begin_creating',
),
core_tasks.BackendMethodTask().si(
serialized_tenant, 'add_admin_user_to_tenant'
),
core_tasks.BackendMethodTask().si(serialized_tenant, 'create_tenant_user'),
core_tasks.BackendMethodTask().si(
serialized_network, 'create_network', state_transition='begin_creating'
),
core_tasks.BackendMethodTask().si(
serialized_subnet, 'create_subnet', state_transition='begin_creating'
),
]
quotas = tenant.quotas.all()
quotas = {
q.name: int(q.limit) if q.limit.is_integer() else q.limit for q in quotas
}
creation_tasks.append(
core_tasks.BackendMethodTask().si(
serialized_tenant, 'push_tenant_quotas', quotas
)
)
# handle security groups
# XXX: Create default security groups that was connected to SPL earlier.
for security_group in tenant.security_groups.all():
creation_tasks.append(
SecurityGroupCreateExecutor.as_signature(security_group)
)
if pull_security_groups:
creation_tasks.append(
core_tasks.BackendMethodTask().si(
serialized_tenant, 'pull_tenant_security_groups'
)
)
# initialize external network if it defined in service settings
service_settings = tenant.service_project_link.service.settings
customer = tenant.service_project_link.project.customer
external_network_id = service_settings.get_option('external_network_id')
try:
customer_openstack = models.CustomerOpenStack.objects.get(
settings=service_settings, customer=customer
)
external_network_id = customer_openstack.external_network_id
except models.CustomerOpenStack.DoesNotExist:
pass
if external_network_id and not kwargs.get('skip_connection_extnet'):
creation_tasks.append(
core_tasks.BackendMethodTask().si(
serialized_tenant,
'connect_tenant_to_external_network',
external_network_id=external_network_id,
)
)
creation_tasks.append(
core_tasks.BackendMethodTask().si(
serialized_tenant, backend_method='pull_tenant_routers',
)
)
creation_tasks.append(
core_tasks.BackendMethodTask().si(serialized_tenant, 'pull_tenant_quotas')
)
return chain(*creation_tasks)
def lookup_contact(contact_number):
contact = Contact.objects.get(phone_number=contact_number)
return chain(lookup_contact_whitepages.si(contact.id)).apply_async()
def user_created(self, user_name):
self._create_datastore_dir(user_name)
res = chain(create_workspace.si(user_name), create_datastore.si(user_name))
res.delay()
LOGGER.info("Start monitoring datastore of created user [%s]", user_name)
Monitoring().register(self._shapefile_folder_dir(user_name), True, Geoserver)
def run_both(url: str):
"""A task that calls other tasks."""
url += f"?t={int(time.time())}"
chain(first_part.s(url), second_part.s()).delay()
def file_wrap_up_chained(_, snap_id, group_id):
chain(upload_files_to_s3_task.s(snap_id=snap_id, group_id=group_id),
clean_up_files_chained.s(snap_id=snap_id,
group_id=group_id)).apply_async()
def get_stock_info(self, func, param, arg):
try:
chain(get_stock_data.s(func, param),
output_datas.s(arg)).apply_async()
except Exception as exc:
raise self.retry(exc=exc, coutdown=10, max_retries=3)
def create_request():
"""
Submit a request to resolve and cache the given source code and its dependencies.
:param str repo: the URL to the SCM repository
:param str ref: the SCM reference to fetch
:param list<str> pkg_managers: list of package managers to be used for resolving dependencies
:param list<str> flags: list of flag names
:rtype: flask.Response
:raise ValidationError: if required parameters are not supplied
"""
payload = flask.request.get_json()
if not isinstance(payload, dict):
raise ValidationError('The input data must be a JSON object')
request = Request.from_json(payload)
if not re.match(r'^[a-f0-9]{40}', request.ref):
raise ValidationError(
'The "ref" parameter must be a 40 character hex string')
db.session.add(request)
db.session.commit()
if current_user.is_authenticated:
flask.current_app.logger.info('The user %s submitted request %d',
current_user.username, request.id)
else:
flask.current_app.logger.info('An anonymous user submitted request %d',
request.id)
pkg_manager_names = set(pkg_manager.name
for pkg_manager in request.pkg_managers)
auto_detect = len(pkg_manager_names) == 0
if auto_detect:
flask.current_app.logger.info(
'Automatic detection will be used since "pkg_managers" was empty')
# Chain tasks
error_callback = tasks.failed_request_callback.s(request.id)
chain_tasks = [
tasks.fetch_app_source.s(request.repo, request.ref,
request.id).on_error(error_callback),
]
if 'gomod' in pkg_manager_names or auto_detect:
gomod_dependency_replacements = [
dependency_replacement for dependency_replacement in payload.get(
'dependency_replacements', [])
if dependency_replacement['type'] == 'gomod'
]
chain_tasks.append(
tasks.fetch_gomod_source.si(
request.id,
auto_detect,
gomod_dependency_replacements,
).on_error(error_callback))
chain_tasks.extend([
tasks.create_bundle_archive.si(request.id).on_error(error_callback),
tasks.set_request_state.si(request.id, 'complete',
'Completed successfully'),
])
chain(chain_tasks).delay()
flask.current_app.logger.debug('Successfully scheduled request %d',
request.id)
return flask.jsonify(request.to_json()), 201
def extract_addon(self, entry, batch_size=BATCH_SIZE):
"""
This method takes a GitExtractionEntry object and creates a chain of
Celery tasks to extract each version in a git repository that haven't
been extracted yet (including the deleted versions).
It does not run if the add-on is locked for git extraction.
"""
addon = entry.addon
log.info('Starting git extraction of add-on "%s".', addon.pk)
# See: https://github.com/mozilla/addons-server/issues/14289
if addon.type != amo.ADDON_EXTENSION:
log.info(
'Skipping git extraction of add-on "%s": not an extension.',
addon.pk,
)
entry.delete()
return
# We cannot use `entry.in_progress` because we have to be sure of the
# add-on state and `entry` might not reflect the most up-to-date
# database state here.
if addon.git_extraction_is_in_progress:
log.info(
'Aborting extraction of addon "%s" to git storage '
'because it is already in progress.',
addon.pk,
)
return
log.info('Locking add-on "%s" before extraction.', addon.pk)
entry.update(in_progress=True)
# Retrieve all the version pks to extract, sorted by creation date.
versions_to_extract = (addon.versions(
manager='unfiltered_for_relations').filter(
file__is_webextension=True,
git_hash='').order_by('created').values_list('pk', flat=True))
if len(versions_to_extract) == 0:
log.info('No version to git-extract for add-on "%s", exiting.',
addon.pk)
# We can safely delete the entry because there is no version to
# extract.
entry.delete()
return
version_pks = versions_to_extract[0:batch_size]
tasks = [
# Create a task to extract the BATCH_SIZE first versions.
extract_versions_to_git.si(addon_pk=addon.pk,
version_pks=version_pks)
]
if len(version_pks) < len(versions_to_extract):
# If there are more versions to git-extract, let's keep the entry
# in the queue until we're done with this entry/add-on. The
# `continue_git_extraction` task will set the `in_progress` flag to
# `False` and this CRON task will pick the remaining versions to
# git-extract the next time it runs.
tasks.append(continue_git_extraction.si(addon.pk))
else:
# If we do not have more versions to git-extract here, we can
# remove the entry from the queue.
tasks.append(remove_git_extraction_entry.si(addon.pk))
log.info(
'Submitted %s tasks to git-extract %s versions for add-on "%s".',
len(tasks),
len(versions_to_extract),
addon.pk,
)
# Attach an error handler on the chain and run it. The error
# handler should remove the add-on lock (among other things).
chain(*tasks).on_error(on_extraction_error.s(addon.pk)).delay()
def put(self, request):
received_data = request.data
if received_data.get('file') is None or received_data.get('exchange') is None:
raise ParseError('Пропущен один из параметров!')
file = save_uploaded_file(request.FILES['file'])
request_exchange = received_data['exchange'].lower()
exchanges = [item[0] for item in StatsUploadEvent.EXCHANGES]
if request_exchange not in exchanges:
raise ParseError('Параметр exchange невалиден!')
parse_task_id, update_profit_task_id = uuid(), uuid()
StatsUploadEvent.objects.create(
parse_task_id=parse_task_id,
update_profit_task_id=update_profit_task_id,
exchange=request_exchange,
file=file,
uploaded_by=request.user
)
if request_exchange == 'poloniex':
task = chain(
ParsePoloniexStatsTask().subtask(
kwargs={
'file': file,
'task_id': parse_task_id,
},
task_id=parse_task_id,
),
TradeProfitRecalculationTask().subtask(
kwargs={
'task_id': update_profit_task_id,
},
task_id=update_profit_task_id,
)
).apply_async()
elif request_exchange == 'bittrex':
raise ParseError('Парсер Bittrex в разработке!')
# task = chain(
# ParseBittrexStatsTask().subtask(
# kwargs={
# 'file': file,
# 'task_id': parse_task_id,
# },
# task_id=parse_task_id,
# ),
# TradeProfitRecalculationTask().subtask(
# kwargs={
# 'task_id': update_profit_task_id,
# },
# task_id=update_profit_task_id,
# )
# ).apply_async()
elif request_exchange == 'binance':
# raise ParseError('Парсер Binance в разработке!')
task = chain(
ParseBinanceStatsTask().subtask(
kwargs={
'file': file,
'task_id': parse_task_id,
},
task_id=parse_task_id,
),
TradeProfitRecalculationTask().subtask(
kwargs={
'task_id': update_profit_task_id,
},
task_id=update_profit_task_id,
)
).apply_async()
return Response('ok', status=204)
def build_sketch_analysis_pipeline(sketch_id,
searchindex_id,
user_id,
analyzer_names=None,
analyzer_kwargs=None,
timeline_id=None):
"""Build a pipeline for sketch analysis.
If no analyzer_names is passed in then we assume auto analyzers should be
run and get this list from the configuration. Parameters to the analyzers
can be passed in to this function, otherwise they will be taken from the
configuration. Either default kwargs for auto analyzers or defaults for
manually run analyzers.
Args:
sketch_id (int): The ID of the sketch to analyze.
searchindex_id (int): The ID of the searchindex to analyze.
user_id (int): The ID of the user who started the analyzer.
analyzer_names (list): List of analyzers to run.
analyzer_kwargs (dict): Arguments to the analyzers.
timeline_id (int): Optional int of the timeline to run the analyzer on.
Returns:
A tuple with a Celery group with analysis tasks or None if no analyzers
are enabled and an analyzer session ID.
"""
tasks = []
if not analyzer_names:
analyzer_names = current_app.config.get('AUTO_SKETCH_ANALYZERS', [])
if not analyzer_kwargs:
analyzer_kwargs = current_app.config.get(
'AUTO_SKETCH_ANALYZERS_KWARGS', {})
# Exit early if no sketch analyzers are configured to run.
if not analyzer_names:
return None, None
if not analyzer_kwargs:
analyzer_kwargs = current_app.config.get('ANALYZERS_DEFAULT_KWARGS',
{})
if user_id:
user = User.query.get(user_id)
else:
user = None
sketch = Sketch.query.get(sketch_id)
analysis_session = AnalysisSession(user, sketch)
analyzers = manager.AnalysisManager.get_analyzers(analyzer_names)
for analyzer_name, analyzer_cls in analyzers:
if not analyzer_cls.IS_SKETCH_ANALYZER:
continue
kwargs = analyzer_kwargs.get(analyzer_name, {})
searchindex = SearchIndex.query.get(searchindex_id)
timeline = None
if timeline_id:
timeline = Timeline.query.get(timeline_id)
if not timeline:
timeline = Timeline.query.filter_by(
sketch=sketch, searchindex=searchindex).first()
analysis = Analysis(name=analyzer_name,
description=analyzer_name,
analyzer_name=analyzer_name,
parameters=json.dumps(kwargs),
user=user,
sketch=sketch,
timeline=timeline)
analysis.set_status('PENDING')
analysis_session.analyses.append(analysis)
db_session.add(analysis)
db_session.commit()
tasks.append(
run_sketch_analyzer.s(sketch_id,
analysis.id,
analyzer_name,
timeline_id=timeline_id,
**kwargs))
# Commit the analysis session to the database.
db_session.add(analysis_session)
db_session.commit()
if current_app.config.get('ENABLE_EMAIL_NOTIFICATIONS'):
tasks.append(run_email_result_task.s(sketch_id))
if not tasks:
return None, None
return chain(tasks), analysis_session
def __init__(self, file_, addon=None, listed=None, final_task=None):
self.addon = addon
self.file = None
self.prev_file = None
if isinstance(file_, FileUpload):
assert listed is not None
channel = (amo.RELEASE_CHANNEL_LISTED if listed else
amo.RELEASE_CHANNEL_UNLISTED)
is_mozilla_signed = False
# We're dealing with a bare file upload. Try to extract the
# metadata that we need to match it against a previous upload
# from the file itself.
try:
addon_data = parse_addon(file_, minimal=True)
is_mozilla_signed = addon_data.get(
'is_mozilla_signed_extension', False)
except ValidationError as form_error:
log.info('could not parse addon for upload {}: {}'
.format(file_.pk, form_error))
addon_data = None
else:
file_.update(version=addon_data.get('version'))
assert not file_.validation
validation_tasks = self.create_file_upload_tasks(
upload_pk=file_.pk,
channel=channel,
is_mozilla_signed=is_mozilla_signed
)
elif isinstance(file_, File):
# The listed flag for a File object should always come from
# the status of its owner Addon. If the caller tries to override
# this, something is wrong.
assert listed is None
channel = file_.version.channel
is_mozilla_signed = file_.is_mozilla_signed_extension
self.file = file_
self.addon = self.file.version.addon
addon_data = {'guid': self.addon.guid,
'version': self.file.version.version}
validation_tasks = [
tasks.create_initial_validation_results.si(),
tasks.validate_file.s(file_.pk),
tasks.handle_file_validation_result.s(file_.pk)
]
else:
raise ValueError
if final_task:
validation_tasks.append(final_task)
self.task = chain(*validation_tasks)
# Create a cache key for the task, so multiple requests to validate the
# same object do not result in duplicate tasks.
opts = file_._meta
self.cache_key = 'validation-task:{0}.{1}:{2}:{3}'.format(
opts.app_label, opts.object_name, file_.pk, listed)
"""
Models an analysis request for a piece of music including its location
"""
song_fields = api.model(
'SongModel', {
'source_path':
fields.String(description='The path of the song to analyze',
required=True),
'force':
fields.Boolean(description='Should every analysis run', required=False)
})
pipeline = chain(
check_done.s().set(priority=1),
add_bpm.s().set(priority=2),
add_emotions.s().set(priority=3),
add_metering.s().set(priority=4),
add_similarity_features.s().set(priority=5),
)
def add_to_pipeline(data, song_path):
if song_path.endswith(("mp3", "wav")):
id = get_song_id(song_path)
force = data['force'] if 'force' in data else False
config = data['config'] if 'config' in data else {}
song = dict({
'audio_id': id,
'source_path': song_path,
'FORCE': force,
'config': config,
def test_priority(self, manager):
c = chain(return_priority.signature(priority=3))()
assert c.get(timeout=TIMEOUT) == "Priority: 3"
def get_deletion_task(cls, instance, serialized_instance):
serialized_vm = core_utils.serialize_instance(instance.vm)
return chain(
core_tasks.DeletionTask().si(serialized_instance),
VirtualMachineUpdatedNotificationTask().si(serialized_vm),
)
def test_single_chain(self, manager):
c = chain(add.s(3, 4))()
assert c.get(timeout=TIMEOUT) == 7
def appliance_action(request, appliance_id, action, x=None):
if not request.user.is_authenticated():
return go_home(request)
try:
appliance = Appliance.objects.get(id=appliance_id)
except ObjectDoesNotExist:
messages.error(
request,
'Appliance with ID {} does not exist!.'.format(appliance_id))
return go_back_or_home(request)
if not can_operate_appliance_or_pool(appliance, request.user):
messages.error(
request,
'This appliance belongs either to some other user or nobody.')
return go_back_or_home(request)
if action == "start":
if appliance.power_state != Appliance.Power.ON:
chain(appliance_power_on.si(appliance.id),
(wait_appliance_ready if appliance.preconfigured else
mark_appliance_ready).si(appliance.id))()
messages.success(request, 'Initiated launch of appliance.')
return go_back_or_home(request)
else:
messages.info(request, 'Appliance was already powered on.')
return go_back_or_home(request)
elif action == "reboot":
if appliance.power_state == Appliance.Power.ON:
chain(appliance_reboot.si(appliance.id),
(wait_appliance_ready if appliance.preconfigured else
mark_appliance_ready).si(appliance.id))()
messages.success(request, 'Initiated reboot of appliance.')
return go_back_or_home(request)
else:
messages.error(request,
'Only powered on appliances can be rebooted')
return go_back_or_home(request)
elif action == "stop":
if appliance.power_state != Appliance.Power.OFF:
appliance_power_off.delay(appliance.id)
messages.success(request, 'Initiated stop of appliance.')
return go_back_or_home(request)
else:
messages.info(request, 'Appliance was already powered off.')
return go_back_or_home(request)
elif action == "suspend":
if appliance.power_state != Appliance.Power.SUSPENDED:
appliance_suspend.delay(appliance.id)
messages.success(request, 'Initiated suspend of appliance.')
return go_back_or_home(request)
else:
messages.info(request, 'Appliance was already suspended.')
return go_back_or_home(request)
elif action == "kill":
Appliance.kill(appliance)
messages.success(request, 'Kill initiated.')
return go_back_or_home(request)
elif action == "dont_expire":
if not request.user.is_superuser:
messages.error(
request,
'Disabling expiration time is allowed only for superusers.')
return go_back_or_home(request)
with transaction.atomic():
appliance.leased_until = None
appliance.save()
messages.success(request, 'Lease disabled successfully. Be careful.')
return go_back_or_home(request)
elif action == "set_lease":
if not can_operate_appliance_or_pool(appliance, request.user):
messages.error(
request,
'This appliance belongs either to some other user or nobody.')
return go_back_or_home(request)
appliance.prolong_lease(time=int(x))
messages.success(request, 'Lease prolonged successfully.')
return go_back_or_home(request)
else:
messages.error(request, "Unknown action '{}'".format(action))
def _run_reindex_tasks(self, models, queue):
apply_async_kwargs = {'priority': 0}
if queue:
log.info('Adding indexing tasks to queue %s', queue)
apply_async_kwargs['queue'] = queue
else:
log.info('Adding indexing tasks to default queue')
index_time = timezone.now()
timestamp = datetime.datetime.now().strftime('%Y%m%d%H%M%S')
for doc in registry.get_documents(models):
queryset = doc().get_queryset()
# Get latest object from the queryset
app_label = queryset.model._meta.app_label
model_name = queryset.model.__name__
index_name = doc._index._name
new_index_name = "{}_{}".format(index_name, timestamp)
# Set index temporarily for indexing,
# this will only get set during the running of this command
doc._index._name = new_index_name
pre_index_task = create_new_es_index.si(
app_label=app_label,
model_name=model_name,
index_name=index_name,
new_index_name=new_index_name)
indexing_tasks = self._get_indexing_tasks(
app_label=app_label,
model_name=model_name,
queryset=queryset,
index_name=new_index_name,
document_class=str(doc))
post_index_task = switch_es_index.si(app_label=app_label,
model_name=model_name,
index_name=index_name,
new_index_name=new_index_name)
# Task to run in order to add the objects
# that has been inserted into database while indexing_tasks was running
# We pass the creation time of latest object, so its possible to index later items
missed_index_task = index_missing_objects.si(
app_label=app_label,
model_name=model_name,
document_class=str(doc),
index_generation_time=index_time)
# http://celery.readthedocs.io/en/latest/userguide/canvas.html#chords
chord_tasks = chord(header=indexing_tasks, body=post_index_task)
if queue:
pre_index_task.set(queue=queue)
chord_tasks.set(queue=queue)
missed_index_task.set(queue=queue)
# http://celery.readthedocs.io/en/latest/userguide/canvas.html#chain
chain(pre_index_task, chord_tasks,
missed_index_task).apply_async(**apply_async_kwargs)
message = (
"Successfully issued tasks for {}.{}, total {} items".format(
app_label, model_name, queryset.count()))
log.info(message)
def main():
duplicate = multiply.s(2)
task = chain(multiply.s(4, 5), multiply.s(2))
promise = task.delay()
print(promise.get())
def mail(email: str, subject: str, template: Union[str, LazyI18nString],
context: Dict[str, Any]=None, event: Event=None, locale: str=None,
order: Order=None, headers: dict=None, sender: str=None, invoices: list=None,
attach_tickets=False):
"""
Sends out an email to a user. The mail will be sent synchronously or asynchronously depending on the installation.
:param email: The email address of the recipient
:param subject: The email subject. Should be localized to the recipients's locale or a lazy object that will be
localized by being casted to a string.
:param template: The filename of a template to be used. It will be rendered with the locale given in the locale
argument and the context given in the next argument. Alternatively, you can pass a LazyI18nString and
``context`` will be used as the argument to a Python ``.format_map()`` call on the template.
:param context: The context for rendering the template (see ``template`` parameter)
:param event: The event this email is related to (optional). If set, this will be used to determine the sender,
a possible prefix for the subject and the SMTP server that should be used to send this email.
:param order: The order this email is related to (optional). If set, this will be used to include a link to the
order below the email.
:param headers: A dict of custom mail headers to add to the mail
:param locale: The locale to be used while evaluating the subject and the template
:param sender: Set the sender email address. If not set and ``event`` is set, the event's default will be used,
otherwise the system default.
:param invoices: A list of invoices to attach to this email.
:param attach_tickets: Whether to attach tickets to this email, if they are available to download.
:raises MailOrderException: on obvious, immediate failures. Not raising an exception does not necessarily mean
that the email has been sent, just that it has been queued by the email backend.
"""
if email == INVALID_ADDRESS:
return
headers = headers or {}
with language(locale):
if isinstance(context, dict) and event:
for k, v in event.meta_data.items():
context['meta_' + k] = v
if isinstance(context, dict) and order:
try:
context.update({
'invoice_name': order.invoice_address.name,
'invoice_company': order.invoice_address.company
})
except InvoiceAddress.DoesNotExist:
context.update({
'invoice_name': '',
'invoice_company': ''
})
renderer = ClassicMailRenderer(None)
content_plain = body_plain = render_mail(template, context)
subject = str(subject).format_map(context)
sender = sender or (event.settings.get('mail_from') if event else settings.MAIL_FROM)
if event:
sender = formataddr((str(event.name), sender))
else:
sender = formataddr((settings.PRETIX_INSTANCE_NAME, sender))
subject = str(subject)
signature = ""
bcc = []
if event:
renderer = event.get_html_mail_renderer()
if event.settings.mail_bcc:
bcc.append(event.settings.mail_bcc)
if event.settings.mail_from == settings.DEFAULT_FROM_EMAIL and event.settings.contact_mail and not headers.get('Reply-To'):
headers['Reply-To'] = event.settings.contact_mail
prefix = event.settings.get('mail_prefix')
if prefix and prefix.startswith('[') and prefix.endswith(']'):
prefix = prefix[1:-1]
if prefix:
subject = "[%s] %s" % (prefix, subject)
body_plain += "\r\n\r\n-- \r\n"
signature = str(event.settings.get('mail_text_signature'))
if signature:
signature = signature.format(event=event.name)
body_plain += signature
body_plain += "\r\n\r\n-- \r\n"
if order:
if order.testmode:
subject = "[TESTMODE] " + subject
body_plain += _(
"You are receiving this email because you placed an order for {event}."
).format(event=event.name)
body_plain += "\r\n"
body_plain += _(
"You can view your order details at the following URL:\n{orderurl}."
).replace("\n", "\r\n").format(
event=event.name, orderurl=build_absolute_uri(
order.event, 'presale:event.order', kwargs={
'order': order.code,
'secret': order.secret
}
)
)
body_plain += "\r\n"
try:
body_html = renderer.render(content_plain, signature, str(subject), order)
except:
logger.exception('Could not render HTML body')
body_html = None
send_task = mail_send_task.si(
to=[email],
bcc=bcc,
subject=subject,
body=body_plain,
html=body_html,
sender=sender,
event=event.id if event else None,
headers=headers,
invoices=[i.pk for i in invoices] if invoices else [],
order=order.pk if order else None,
attach_tickets=attach_tickets
)
if invoices:
task_chain = [invoice_pdf_task.si(i.pk).on_error(send_task) for i in invoices if not i.file]
else:
task_chain = []
task_chain.append(send_task)
chain(*task_chain).apply_async()
