def run_sous_chef(sous_chef_path, recipe_id, kw_key):
"""
Do the work. This exists outside the class
in order to enable pickling.
"""
recipe = db.session.query(Recipe).get(recipe_id)
try:
# load in kwargs
kw = rds.get(kw_key)
if not kw:
raise InternalServerError(
'An unexpected error occurred while attempting to run a Sous Chef.'
)
kw = pickle_to_obj(kw)
# delete them.
rds.delete(kw_key)
# import sous chef
SousChef = import_sous_chef(sous_chef_path)
# initialize it with kwargs
sc = SousChef(**kw)
# cook it.
sc.cook()
# update status and next job from sous chef.
recipe.status = "stable"
recipe.traceback = None
# if something is set on this object, add it.
if len(sc.next_job.keys()):
recipe.last_job = sc.next_job
db.session.add(recipe)
db.session.commit()
return True
except Exception as e:
# keep track of the error.
db.session.rollback()
recipe.status = "error"
recipe.traceback = format_exc()
db.session.add(recipe)
db.session.commit()
return MerlynneError(e)
def bulkworker(job_id, **qkw):
"""
Fetch a job and execute it.
"""
start = time.time()
try:
k = qkw['job_key_fmt'].format(job_id)
job = rds.get(k)
if not job:
raise InternalServerError(
'An unexpected error occurred while processing bulk upload.'
)
if qkw['serializer'] == 'json':
job = json_to_obj(job)
elif qkw['serializer'] == 'pickle':
job = pickle_to_obj(job)
data = job.pop('data', [])
job = job.pop('kw', {})
# delete them
rds.delete(k)
# chunk list
chunked_data = util.chunk_list(data, qkw.get('chunk_size'))
# partial funtion
load_fx = partial(ingest.source, **job)
# pooled execution
pool = Pool(qkw.get('max_workers', MAX_WORKERS))
for res in pool.imap_unordered(load_fx, chunked_data):
pass
return True
except Exception:
tb = format_exc()
raise RequestError('An Error Ocurred while running {}:\n{}'.format(job_id, tb))
except JobTimeoutException:
end = time.time()
raise InternalServerError(
'Bulk loading timed out after {} seconds'
.format(end-start))
def bulkworker(job_id, **qkw):
"""
Fetch a job and execute it.
"""
start = time.time()
try:
k = qkw['job_key_fmt'].format(job_id)
job = rds.get(k)
if not job:
raise InternalServerError(
'An unexpected error occurred while processing bulk upload.')
if qkw['serializer'] == 'json':
job = json_to_obj(job)
elif qkw['serializer'] == 'pickle':
job = pickle_to_obj(job)
data = job.pop('data', [])
job = job.pop('kw', {})
# delete them
rds.delete(k)
# chunk list
chunked_data = util.chunk_list(data, qkw.get('chunk_size'))
# partial funtion
load_fx = partial(ingest.source, **job)
# pooled execution
pool = Pool(qkw.get('max_workers', MAX_WORKERS))
for res in pool.imap_unordered(load_fx, chunked_data):
pass
return True
except Exception:
tb = format_exc()
raise RequestError('An Error Ocurred while running {}:\n{}'.format(
job_id, tb))
except JobTimeoutException:
end = time.time()
raise InternalServerError(
'Bulk loading timed out after {} seconds'.format(end - start))
def run(sous_chef_path, recipe_id, kw_key, **kw):
"""
Do the work. This exists outside the class
in order to enable pickling for the task queue.
"""
recipe = db.session.query(Recipe).get(recipe_id)
try:
if kw_key:
# load in kwargs
kw = rds.get(kw_key)
if not kw:
raise InternalServerError(
'An unexpected error occurred while attempting to run a Sous Chef.'
)
kw = pickle_to_obj(kw)
# delete them.
rds.delete(kw_key)
# import sous chef
SousChef = sc_exec.from_import_path(sous_chef_path)
# initialize it with kwargs
kw['org'] = db.session\
.query(Org).get(recipe.org.id)\
.to_dict(incl_domains=True)
kw['recipe'] = recipe.to_dict()
sous_chef = SousChef(**kw)
# indicate that the job is running
if not kw.get('passthrough', False):
recipe.status = 'running'
db.session.add(recipe)
db.session.commit()
# cook it.
data = sous_chef.cook()
# passthrough the data.
if kw.get('passthrough', False):
return data
# otherwise just exhaust the generator
if isgenerator(data):
data = list(data)
# teardown this recipe
sous_chef.teardown()
# update status and next job from sous chef.
recipe.status = "stable"
recipe.traceback = None
recipe.last_run = dates.now()
if len(sous_chef.next_job.keys()):
recipe.last_job = sous_chef.next_job
db.session.add(recipe)
db.session.commit()
return True
except:
# always delete the kwargs.
if kw_key:
rds.delete(kw_key)
if not kw.get('passthrough', False):
db.session.rollback()
recipe.status = "error"
recipe.traceback = format_exc()
recipe.last_run = dates.now()
db.session.add(recipe)
db.session.commit()
# notification
tb = format_exc()
error_notification(recipe, tb)
return MerlynneError(tb)
raise MerlynneError(format_exc())
def run_sous_chef(sous_chef_path, recipe_id, kw_key):
"""
Do the work. This exists outside the class
in order to enable pickling.
"""
recipe = db.session.query(Recipe).get(recipe_id)
try:
# load in kwargs
kw = rds.get(kw_key)
if not kw:
raise InternalServerError("An unexpected error occurred while attempting to run a Sous Chef.")
kw = pickle_to_obj(kw)
# delete them.
rds.delete(kw_key)
# import sous chef
SousChef = import_sous_chef(sous_chef_path)
# initialize it with kwargs
sc = SousChef(**kw)
# indicate that the job is running
if not kw.get("passthrough", False):
recipe.status = "running"
db.session.add(recipe)
db.session.commit()
# cook it.
data = sc.cook()
# passthrough the data.
if kw.get("passthrough", False):
return data
# otherwise just exhaust the generator
if isgenerator(data):
data = list(data)
# teardown this recipe
sc.teardown()
# update status and next job from sous chef.
recipe.status = "stable"
recipe.traceback = None
recipe.last_run = dates.now()
if len(sc.next_job.keys()):
recipe.last_job = sc.next_job
db.session.add(recipe)
db.session.commit()
return True
except Exception as e:
# always delete the kwargs.
rds.delete(kw_key)
if kw.get("passthrough", False):
raise MerlynneError(e)
db.session.rollback()
recipe.status = "error"
recipe.traceback = format_exc()
recipe.last_run = dates.now()
db.session.add(recipe)
db.session.commit()
return MerlynneError(e)
def run(sous_chef_path, recipe_id, kw_key, **kw):
"""
Do the work. This exists outside the class
in order to enable pickling for the task queue.
"""
recipe = db.session.query(Recipe).get(recipe_id)
try:
if kw_key:
# load in kwargs
kw = rds.get(kw_key)
if not kw:
raise InternalServerError(
'An unexpected error occurred while attempting to run a Sous Chef.'
)
kw = pickle_to_obj(kw)
# delete them.
rds.delete(kw_key)
# import sous chef
SousChef = sc_exec.from_import_path(sous_chef_path)
# initialize it with kwargs
kw['org'] = db.session\
.query(Org).get(recipe.org.id)\
.to_dict(incl_domains=True)
kw['recipe'] = recipe.to_dict()
sous_chef = SousChef(**kw)
# indicate that the job is running
if not kw.get('passthrough', False):
recipe.status = 'running'
db.session.add(recipe)
db.session.commit()
# cook it.
data = sous_chef.cook()
# passthrough the data.
if kw.get('passthrough', False):
return data
# otherwise just exhaust the generator
if isgenerator(data):
data = list(data)
# teardown this recipe
sous_chef.teardown()
# update status and next job from sous chef.
recipe.status = "stable"
recipe.traceback = None
recipe.last_run = dates.now()
if len(sous_chef.next_job.keys()):
recipe.last_job = sous_chef.next_job
db.session.add(recipe)
db.session.commit()
return True
except:
# always delete the kwargs.
if kw_key:
rds.delete(kw_key)
if not kw.get('passthrough', False):
db.session.rollback()
recipe.status = "error"
recipe.traceback = format_exc()
recipe.last_run = dates.now()
db.session.add(recipe)
db.session.commit()
# notification
tb = format_exc()
error_notification(recipe, tb)
return MerlynneError(tb)
raise MerlynneError(format_exc())
