def run_sous_chef(sous_chef_path, recipe_id, kw_key): """ Do the work. This exists outside the class in order to enable pickling. """ recipe = db.session.query(Recipe).get(recipe_id) try: # load in kwargs kw = rds.get(kw_key) if not kw: raise InternalServerError( 'An unexpected error occurred while attempting to run a Sous Chef.' ) kw = pickle_to_obj(kw) # delete them. rds.delete(kw_key) # import sous chef SousChef = import_sous_chef(sous_chef_path) # initialize it with kwargs sc = SousChef(**kw) # cook it. sc.cook() # update status and next job from sous chef. recipe.status = "stable" recipe.traceback = None # if something is set on this object, add it. if len(sc.next_job.keys()): recipe.last_job = sc.next_job db.session.add(recipe) db.session.commit() return True except Exception as e: # keep track of the error. db.session.rollback() recipe.status = "error" recipe.traceback = format_exc() db.session.add(recipe) db.session.commit() return MerlynneError(e)
def bulkworker(job_id, **qkw): """ Fetch a job and execute it. """ start = time.time() try: k = qkw['job_key_fmt'].format(job_id) job = rds.get(k) if not job: raise InternalServerError( 'An unexpected error occurred while processing bulk upload.' ) if qkw['serializer'] == 'json': job = json_to_obj(job) elif qkw['serializer'] == 'pickle': job = pickle_to_obj(job) data = job.pop('data', []) job = job.pop('kw', {}) # delete them rds.delete(k) # chunk list chunked_data = util.chunk_list(data, qkw.get('chunk_size')) # partial funtion load_fx = partial(ingest.source, **job) # pooled execution pool = Pool(qkw.get('max_workers', MAX_WORKERS)) for res in pool.imap_unordered(load_fx, chunked_data): pass return True except Exception: tb = format_exc() raise RequestError('An Error Ocurred while running {}:\n{}'.format(job_id, tb)) except JobTimeoutException: end = time.time() raise InternalServerError( 'Bulk loading timed out after {} seconds' .format(end-start))
def bulkworker(job_id, **qkw): """ Fetch a job and execute it. """ start = time.time() try: k = qkw['job_key_fmt'].format(job_id) job = rds.get(k) if not job: raise InternalServerError( 'An unexpected error occurred while processing bulk upload.') if qkw['serializer'] == 'json': job = json_to_obj(job) elif qkw['serializer'] == 'pickle': job = pickle_to_obj(job) data = job.pop('data', []) job = job.pop('kw', {}) # delete them rds.delete(k) # chunk list chunked_data = util.chunk_list(data, qkw.get('chunk_size')) # partial funtion load_fx = partial(ingest.source, **job) # pooled execution pool = Pool(qkw.get('max_workers', MAX_WORKERS)) for res in pool.imap_unordered(load_fx, chunked_data): pass return True except Exception: tb = format_exc() raise RequestError('An Error Ocurred while running {}:\n{}'.format( job_id, tb)) except JobTimeoutException: end = time.time() raise InternalServerError( 'Bulk loading timed out after {} seconds'.format(end - start))
def run(sous_chef_path, recipe_id, kw_key, **kw): """ Do the work. This exists outside the class in order to enable pickling for the task queue. """ recipe = db.session.query(Recipe).get(recipe_id) try: if kw_key: # load in kwargs kw = rds.get(kw_key) if not kw: raise InternalServerError( 'An unexpected error occurred while attempting to run a Sous Chef.' ) kw = pickle_to_obj(kw) # delete them. rds.delete(kw_key) # import sous chef SousChef = sc_exec.from_import_path(sous_chef_path) # initialize it with kwargs kw['org'] = db.session\ .query(Org).get(recipe.org.id)\ .to_dict(incl_domains=True) kw['recipe'] = recipe.to_dict() sous_chef = SousChef(**kw) # indicate that the job is running if not kw.get('passthrough', False): recipe.status = 'running' db.session.add(recipe) db.session.commit() # cook it. data = sous_chef.cook() # passthrough the data. if kw.get('passthrough', False): return data # otherwise just exhaust the generator if isgenerator(data): data = list(data) # teardown this recipe sous_chef.teardown() # update status and next job from sous chef. recipe.status = "stable" recipe.traceback = None recipe.last_run = dates.now() if len(sous_chef.next_job.keys()): recipe.last_job = sous_chef.next_job db.session.add(recipe) db.session.commit() return True except: # always delete the kwargs. if kw_key: rds.delete(kw_key) if not kw.get('passthrough', False): db.session.rollback() recipe.status = "error" recipe.traceback = format_exc() recipe.last_run = dates.now() db.session.add(recipe) db.session.commit() # notification tb = format_exc() error_notification(recipe, tb) return MerlynneError(tb) raise MerlynneError(format_exc())
def run_sous_chef(sous_chef_path, recipe_id, kw_key): """ Do the work. This exists outside the class in order to enable pickling. """ recipe = db.session.query(Recipe).get(recipe_id) try: # load in kwargs kw = rds.get(kw_key) if not kw: raise InternalServerError("An unexpected error occurred while attempting to run a Sous Chef.") kw = pickle_to_obj(kw) # delete them. rds.delete(kw_key) # import sous chef SousChef = import_sous_chef(sous_chef_path) # initialize it with kwargs sc = SousChef(**kw) # indicate that the job is running if not kw.get("passthrough", False): recipe.status = "running" db.session.add(recipe) db.session.commit() # cook it. data = sc.cook() # passthrough the data. if kw.get("passthrough", False): return data # otherwise just exhaust the generator if isgenerator(data): data = list(data) # teardown this recipe sc.teardown() # update status and next job from sous chef. recipe.status = "stable" recipe.traceback = None recipe.last_run = dates.now() if len(sc.next_job.keys()): recipe.last_job = sc.next_job db.session.add(recipe) db.session.commit() return True except Exception as e: # always delete the kwargs. rds.delete(kw_key) if kw.get("passthrough", False): raise MerlynneError(e) db.session.rollback() recipe.status = "error" recipe.traceback = format_exc() recipe.last_run = dates.now() db.session.add(recipe) db.session.commit() return MerlynneError(e)