def _submit(ctx, parent_id, name, url, func, *args, **kwargs):
"""Submit a function to a cluster
Parameters
----------
parent_id : str
The ID of the group that the job is a part of.
name : str
The name of the job
url : str
The handler that can take the results (e.g., /beta_diversity/)
func : function
The function to execute. Any returns from this function will be
serialized and deposited into Redis using the uuid for a key. This
function should raise if the method fails.
args : tuple or None
Any args for ``func``
kwargs : dict or None
Any kwargs for ``func``
Returns
-------
tuple, (str, str, AsyncResult)
The job ID, parent ID and the IPython's AsyncResult object of the job
"""
parent_info = r_client.get(parent_id)
if parent_info is None:
parent_info = create_info('unnamed', 'group', id=parent_id)
parent_id = parent_info['id']
r_client.set(parent_id, json.dumps(parent_info))
parent_pubsub_key = parent_id + ':pubsub'
job_info = create_info(name,
'job',
url=url,
parent=parent_id,
context=ctx.name,
store=True)
job_info['status'] = 'Queued'
job_id = job_info['id']
with r_client.pipeline() as pipe:
pipe.set(job_id, json.dumps(job_info))
pipe.publish(parent_pubsub_key, json.dumps({'add': [job_id]}))
pipe.execute()
ar = ctx.bv.apply_async(_redis_wrap, job_info, func, *args, **kwargs)
return job_id, parent_id, ar
def post(self, analysis_id):
analysis_id = int(analysis_id)
rarefaction_depth = self.get_argument('rarefaction-depth')
# convert to integer if rarefaction level given
if rarefaction_depth:
rarefaction_depth = int(rarefaction_depth)
else:
rarefaction_depth = None
analysis = Analysis(analysis_id)
check_analysis_access(self.current_user, analysis)
command_args = self.get_arguments("commands")
split = [x.split("#") for x in command_args]
moi_user_id = get_id_from_user(self.current_user.id)
moi_group = create_info(analysis_id, 'group', url='/analysis/',
parent=moi_user_id, store=True)
moi_name = 'Creating %s' % analysis.name
moi_result_url = '/analysis/results/%d' % analysis_id
submit(ctx_default, moi_group['id'], moi_name,
moi_result_url, run_analysis, analysis_id, split,
rarefaction_depth=rarefaction_depth)
r_client.hset('analyis-map', analysis_id, moi_group['id'])
self.render("analysis_waiting.html",
group_id=moi_group['id'], aname=analysis.name)
def post(self, analysis_id):
analysis_id = int(analysis_id)
rarefaction_depth = self.get_argument('rarefaction-depth')
mdsi = self.get_argument('merge-duplicated-sample-ids', default=False)
if mdsi == 'on':
mdsi = True
# convert to integer if rarefaction level given
if rarefaction_depth:
rarefaction_depth = int(rarefaction_depth)
else:
rarefaction_depth = None
analysis = Analysis(analysis_id)
check_analysis_access(self.current_user, analysis)
command_args = self.get_arguments("commands")
cmd_split = [x.split("#") for x in command_args]
moi_user_id = get_id_from_user(self.current_user.id)
moi_group = create_info(analysis_id, 'group', url='/analysis/',
parent=moi_user_id, store=True)
moi_name = ("Creating %s... When finished, please click the 'Success' "
"link to the right" % analysis.name)
moi_result_url = '/analysis/results/%d' % analysis_id
submit(ctx_default, moi_group['id'], moi_name,
moi_result_url, run_analysis, analysis_id, cmd_split,
rarefaction_depth=rarefaction_depth,
merge_duplicated_sample_ids=mdsi)
r_client.hset('analyis-map', analysis_id, moi_group['id'])
self.render("analysis_waiting.html",
group_id=moi_group['id'], aname=analysis.name)
def _submit(ctx, parent_id, name, url, func, *args, **kwargs):
"""Submit a function to a cluster
Parameters
----------
parent_id : str
The ID of the group that the job is a part of.
name : str
The name of the job
url : str
The handler that can take the results (e.g., /beta_diversity/)
func : function
The function to execute. Any returns from this function will be
serialized and deposited into Redis using the uuid for a key. This
function should raise if the method fails.
args : tuple or None
Any args for ``func``
kwargs : dict or None
Any kwargs for ``func``
Returns
-------
tuple, (str, str, AsyncResult)
The job ID, parent ID and the IPython's AsyncResult object of the job
"""
parent_info = r_client.get(parent_id)
if parent_info is None:
parent_info = create_info('unnamed', 'group', id=parent_id)
parent_id = parent_info['id']
r_client.set(parent_id, json.dumps(parent_info))
parent_pubsub_key = parent_id + ':pubsub'
job_info = create_info(name, 'job', url=url, parent=parent_id,
context=ctx.name, store=True)
job_info['status'] = 'Queued'
job_id = job_info['id']
with r_client.pipeline() as pipe:
pipe.set(job_id, json.dumps(job_info))
pipe.publish(parent_pubsub_key, json.dumps({'add': [job_id]}))
pipe.execute()
ar = ctx.bv.apply_async(_redis_wrap, job_info, func, *args, **kwargs)
return job_id, parent_id, ar
def post(self, analysis_id):
analysis_id = int(analysis_id)
rarefaction_depth = self.get_argument('rarefaction-depth')
mdsi = self.get_argument('merge-duplicated-sample-ids', default=False)
if mdsi == 'on':
mdsi = True
# convert to integer if rarefaction level given
if rarefaction_depth:
rarefaction_depth = int(rarefaction_depth)
else:
rarefaction_depth = None
analysis = Analysis(analysis_id)
check_analysis_access(self.current_user, analysis)
command_args = self.get_arguments("commands")
cmd_split = [x.split("#") for x in command_args]
moi_user_id = get_id_from_user(self.current_user.id)
moi_group = create_info(analysis_id,
'group',
url='%s/analysis/' % qiita_config.portal_dir,
parent=moi_user_id,
store=True)
moi_name = ("Creating %s... When finished, please click the 'Success' "
"link to the right" % analysis.name)
moi_result_url = '%s/analysis/results/%d' % (qiita_config.portal_dir,
analysis_id)
submit(ctx_default,
moi_group['id'],
moi_name,
moi_result_url,
run_analysis,
analysis_id,
cmd_split,
rarefaction_depth=rarefaction_depth,
merge_duplicated_sample_ids=mdsi)
r_client.hset('analyis-map', analysis_id, moi_group['id'])
self.render("analysis_waiting.html",
group_id=moi_group['id'],
aname=analysis.name)
def post(self):
name = self.get_argument("jobname", None)
group_name = self.get_argument("jobgroup", None)
parent = get_id_from_user("no-user")
job_url = "/result"
if name is not None:
submit(ctx_default, parent, name, job_url, say_hello, name)
else:
group_url = "/group"
group = create_info(group_name, 'group', url=group_url,
parent=parent, store=True)
group_id = group['id']
for i in range(5):
name = group_name + '-%d' % i
submit(ctx_default, group_id, name, job_url, say_hello, name)
self.redirect('/')
def post(self):
name = self.get_argument("jobname", None)
group_name = self.get_argument("jobgroup", None)
parent = get_id_from_user("no-user")
job_url = "/result"
if name is not None:
submit(ctx_default, parent, name, job_url, say_hello, name)
else:
group_url = "/group"
group = create_info(group_name,
'group',
url=group_url,
parent=parent,
store=True)
group_id = group['id']
for i in range(5):
name = group_name + '-%d' % i
submit(ctx_default, group_id, name, job_url, say_hello, name)
self.redirect('/')
def post(self, analysis_id):
analysis_id = int(analysis_id)
rarefaction_depth = self.get_argument('rarefaction-depth')
# convert to integer if rarefaction level given
if rarefaction_depth:
rarefaction_depth = int(rarefaction_depth)
else:
rarefaction_depth = None
analysis = Analysis(analysis_id)
check_analysis_access(self.current_user, analysis)
command_args = self.get_arguments("commands")
split = [x.split("#") for x in command_args]
moi_user_id = get_id_from_user(self.current_user.id)
moi_group = create_info(analysis_id,
'group',
url='/analysis/',
parent=moi_user_id,
store=True)
moi_name = ("Creating %s... When finished, please click the 'Success' "
"link to the right" % analysis.name)
moi_result_url = '/analysis/results/%d' % analysis_id
submit(ctx_default,
moi_group['id'],
moi_name,
moi_result_url,
run_analysis,
analysis_id,
split,
rarefaction_depth=rarefaction_depth)
r_client.hset('analyis-map', analysis_id, moi_group['id'])
self.render("analysis_waiting.html",
group_id=moi_group['id'],
aname=analysis.name)