def cook_recipe(self):
"""
Full pipeline.
"""
# generate a job id
job_id = gen_uuid()
# import the sous chef here to get the timeout
# and raise import errors before it attempts to run
# in the queue
_sc = sc_exec.from_import_path(self.sous_chef_path)
# send it to the queue
if not self.passthrough:
# stash kwargs
kw_key = self.stash_kw(job_id)
# indicate that the recipe is running.
self.recipe.status = "queued"
db.session.add(self.recipe)
db.session.commit()
self.q.enqueue(run,
self.sous_chef_path,
self.recipe.id,
kw_key,
job_id=job_id,
timeout=_sc.timeout,
result_ttl=self.kw_ttl)
# return the job id
return job_id
# directly stream the results out.
return run(self.sous_chef_path,
self.recipe.id,
kw_key=None,
**self.sous_chef_kwargs)
def cook_recipe(self):
"""
Full pipeline.
"""
# generate a job id
job_id = gen_uuid()
# import the sous chef here to get the timeout
# and raise import errors before it attempts to run
# in the queue
_sc = sc_exec.from_import_path(self.sous_chef_path)
# send it to the queue
if not self.passthrough:
# stash kwargs
kw_key = self.stash_kw(job_id)
# indicate that the recipe is running.
self.recipe.status = "queued"
db.session.add(self.recipe)
db.session.commit()
self.q.enqueue(
run, self.sous_chef_path,
self.recipe.id, kw_key,
job_id=job_id, timeout=_sc.timeout,
result_ttl=self.kw_ttl)
# return the job id
return job_id
# directly stream the results out.
return run(self.sous_chef_path,
self.recipe.id, kw_key=None,
**self.sous_chef_kwargs)
def run(sous_chef_path, recipe_id, kw_key, **kw):
"""
Do the work. This exists outside the class
in order to enable pickling for the task queue.
"""
recipe = db.session.query(Recipe).get(recipe_id)
try:
if kw_key:
# load in kwargs
kw = rds.get(kw_key)
if not kw:
raise InternalServerError(
'An unexpected error occurred while attempting to run a Sous Chef.'
)
kw = pickle_to_obj(kw)
# delete them.
rds.delete(kw_key)
# import sous chef
SousChef = sc_exec.from_import_path(sous_chef_path)
# initialize it with kwargs
kw['org'] = db.session\
.query(Org).get(recipe.org.id)\
.to_dict(incl_domains=True)
kw['recipe'] = recipe.to_dict()
sous_chef = SousChef(**kw)
# indicate that the job is running
if not kw.get('passthrough', False):
recipe.status = 'running'
db.session.add(recipe)
db.session.commit()
# cook it.
data = sous_chef.cook()
# passthrough the data.
if kw.get('passthrough', False):
return data
# otherwise just exhaust the generator
if isgenerator(data):
data = list(data)
# teardown this recipe
sous_chef.teardown()
# update status and next job from sous chef.
recipe.status = "stable"
recipe.traceback = None
recipe.last_run = dates.now()
if len(sous_chef.next_job.keys()):
recipe.last_job = sous_chef.next_job
db.session.add(recipe)
db.session.commit()
return True
except:
# always delete the kwargs.
if kw_key:
rds.delete(kw_key)
if not kw.get('passthrough', False):
db.session.rollback()
recipe.status = "error"
recipe.traceback = format_exc()
recipe.last_run = dates.now()
db.session.add(recipe)
db.session.commit()
# notification
tb = format_exc()
error_notification(recipe, tb)
return MerlynneError(tb)
raise MerlynneError(format_exc())
def run(sous_chef_path, recipe_id, kw_key, **kw):
"""
Do the work. This exists outside the class
in order to enable pickling for the task queue.
"""
recipe = db.session.query(Recipe).get(recipe_id)
try:
if kw_key:
# load in kwargs
kw = rds.get(kw_key)
if not kw:
raise InternalServerError(
'An unexpected error occurred while attempting to run a Sous Chef.'
)
kw = pickle_to_obj(kw)
# delete them.
rds.delete(kw_key)
# import sous chef
SousChef = sc_exec.from_import_path(sous_chef_path)
# initialize it with kwargs
kw['org'] = db.session\
.query(Org).get(recipe.org.id)\
.to_dict(incl_domains=True)
kw['recipe'] = recipe.to_dict()
sous_chef = SousChef(**kw)
# indicate that the job is running
if not kw.get('passthrough', False):
recipe.status = 'running'
db.session.add(recipe)
db.session.commit()
# cook it.
data = sous_chef.cook()
# passthrough the data.
if kw.get('passthrough', False):
return data
# otherwise just exhaust the generator
if isgenerator(data):
data = list(data)
# teardown this recipe
sous_chef.teardown()
# update status and next job from sous chef.
recipe.status = "stable"
recipe.traceback = None
recipe.last_run = dates.now()
if len(sous_chef.next_job.keys()):
recipe.last_job = sous_chef.next_job
db.session.add(recipe)
db.session.commit()
return True
except:
# always delete the kwargs.
if kw_key:
rds.delete(kw_key)
if not kw.get('passthrough', False):
db.session.rollback()
recipe.status = "error"
recipe.traceback = format_exc()
recipe.last_run = dates.now()
db.session.add(recipe)
db.session.commit()
# notification
tb = format_exc()
error_notification(recipe, tb)
return MerlynneError(tb)
raise MerlynneError(format_exc())