Beispiel #1
0
def run_sous_chef(sous_chef_path, recipe_id, kw_key):
    """
    Do the work. This exists outside the class
    in order to enable pickling.
    """
    recipe = db.session.query(Recipe).get(recipe_id)
    try:
        # load in kwargs
        kw = rds.get(kw_key)
        if not kw:
            raise InternalServerError(
                'An unexpected error occurred while attempting to run a Sous Chef.'
            )
        kw = pickle_to_obj(kw)

        # delete them.
        rds.delete(kw_key)

        # import sous chef
        SousChef = import_sous_chef(sous_chef_path)

        # initialize it with kwargs
        sc = SousChef(**kw)

        # cook it.
        sc.cook()

        # update status and next job from sous chef.
        recipe.status = "stable"
        recipe.traceback = None
        # if something is set on this object, add it.
        if len(sc.next_job.keys()):
            recipe.last_job = sc.next_job
        db.session.add(recipe)
        db.session.commit()
        return True

    except Exception as e:

        # keep track of the error.

        db.session.rollback()
        recipe.status = "error"
        recipe.traceback = format_exc()
        db.session.add(recipe)
        db.session.commit()
        return MerlynneError(e)
Beispiel #2
0
def bulkworker(job_id, **qkw):
    """
    Fetch a job and execute it.
    """
    start = time.time()
    try:
        k = qkw['job_key_fmt'].format(job_id)
        job = rds.get(k)
        if not job:
            raise InternalServerError(
                'An unexpected error occurred while processing bulk upload.'
            )

        if qkw['serializer'] == 'json':
            job = json_to_obj(job)

        elif qkw['serializer'] == 'pickle':
            job = pickle_to_obj(job)

        data = job.pop('data', [])
        job = job.pop('kw', {})

        # delete them
        rds.delete(k)

        # chunk list
        chunked_data = util.chunk_list(data, qkw.get('chunk_size'))

        # partial funtion
        load_fx = partial(ingest.source, **job)

        # pooled execution
        pool = Pool(qkw.get('max_workers', MAX_WORKERS))

        for res in pool.imap_unordered(load_fx, chunked_data):
            pass
        return True

    except Exception:
        tb = format_exc()
        raise RequestError('An Error Ocurred while running {}:\n{}'.format(job_id, tb))

    except JobTimeoutException:
        end = time.time()
        raise InternalServerError(
            'Bulk loading timed out after {} seconds'
            .format(end-start))
Beispiel #3
0
def bulkworker(job_id, **qkw):
    """
    Fetch a job and execute it.
    """
    start = time.time()
    try:
        k = qkw['job_key_fmt'].format(job_id)
        job = rds.get(k)
        if not job:
            raise InternalServerError(
                'An unexpected error occurred while processing bulk upload.')

        if qkw['serializer'] == 'json':
            job = json_to_obj(job)

        elif qkw['serializer'] == 'pickle':
            job = pickle_to_obj(job)

        data = job.pop('data', [])
        job = job.pop('kw', {})

        # delete them
        rds.delete(k)

        # chunk list
        chunked_data = util.chunk_list(data, qkw.get('chunk_size'))

        # partial funtion
        load_fx = partial(ingest.source, **job)

        # pooled execution
        pool = Pool(qkw.get('max_workers', MAX_WORKERS))

        for res in pool.imap_unordered(load_fx, chunked_data):
            pass
        return True

    except Exception:
        tb = format_exc()
        raise RequestError('An Error Ocurred while running {}:\n{}'.format(
            job_id, tb))

    except JobTimeoutException:
        end = time.time()
        raise InternalServerError(
            'Bulk loading timed out after {} seconds'.format(end - start))
Beispiel #4
0
def run(sous_chef_path, recipe_id, kw_key, **kw):
    """
    Do the work. This exists outside the class
    in order to enable pickling for the task queue.
    """
    recipe = db.session.query(Recipe).get(recipe_id)
    try:
        if kw_key:
            # load in kwargs
            kw = rds.get(kw_key)
            if not kw:
                raise InternalServerError(
                    'An unexpected error occurred while attempting to run a Sous Chef.'
                )
            kw = pickle_to_obj(kw)
            # delete them.
            rds.delete(kw_key)

        # import sous chef
        SousChef = sc_exec.from_import_path(sous_chef_path)

        # initialize it with kwargs
        kw['org'] = db.session\
            .query(Org).get(recipe.org.id)\
            .to_dict(incl_domains=True)
        kw['recipe'] = recipe.to_dict()
        sous_chef = SousChef(**kw)

        # indicate that the job is running
        if not kw.get('passthrough', False):
            recipe.status = 'running'
            db.session.add(recipe)
            db.session.commit()

        # cook it.
        data = sous_chef.cook()

        # passthrough the data.
        if kw.get('passthrough', False):
            return data

        # otherwise just exhaust the generator
        if isgenerator(data):
            data = list(data)

        # teardown this recipe
        sous_chef.teardown()

        # update status and next job from sous chef.
        recipe.status = "stable"
        recipe.traceback = None
        recipe.last_run = dates.now()
        if len(sous_chef.next_job.keys()):
            recipe.last_job = sous_chef.next_job
        db.session.add(recipe)
        db.session.commit()
        return True

    except:

        # always delete the kwargs.
        if kw_key:
            rds.delete(kw_key)

        if not kw.get('passthrough', False):
            db.session.rollback()
            recipe.status = "error"
            recipe.traceback = format_exc()
            recipe.last_run = dates.now()
            db.session.add(recipe)
            db.session.commit()

            # notification
            tb = format_exc()
            error_notification(recipe, tb)
            return MerlynneError(tb)

        raise MerlynneError(format_exc())
Beispiel #5
0
def run_sous_chef(sous_chef_path, recipe_id, kw_key):
    """
    Do the work. This exists outside the class
    in order to enable pickling.
    """
    recipe = db.session.query(Recipe).get(recipe_id)
    try:
        # load in kwargs
        kw = rds.get(kw_key)
        if not kw:
            raise InternalServerError("An unexpected error occurred while attempting to run a Sous Chef.")
        kw = pickle_to_obj(kw)

        # delete them.
        rds.delete(kw_key)

        # import sous chef
        SousChef = import_sous_chef(sous_chef_path)

        # initialize it with kwargs
        sc = SousChef(**kw)

        # indicate that the job is running
        if not kw.get("passthrough", False):
            recipe.status = "running"
            db.session.add(recipe)
            db.session.commit()

        # cook it.
        data = sc.cook()

        # passthrough the data.
        if kw.get("passthrough", False):
            return data

        # otherwise just exhaust the generator
        if isgenerator(data):
            data = list(data)

        # teardown this recipe
        sc.teardown()

        # update status and next job from sous chef.
        recipe.status = "stable"
        recipe.traceback = None
        recipe.last_run = dates.now()
        if len(sc.next_job.keys()):
            recipe.last_job = sc.next_job
        db.session.add(recipe)
        db.session.commit()
        return True

    except Exception as e:

        # always delete the kwargs.
        rds.delete(kw_key)
        if kw.get("passthrough", False):
            raise MerlynneError(e)
        db.session.rollback()
        recipe.status = "error"
        recipe.traceback = format_exc()
        recipe.last_run = dates.now()
        db.session.add(recipe)
        db.session.commit()
        return MerlynneError(e)
Beispiel #6
0
def run(sous_chef_path, recipe_id, kw_key, **kw):
    """
    Do the work. This exists outside the class
    in order to enable pickling for the task queue.
    """
    recipe = db.session.query(Recipe).get(recipe_id)
    try:
        if kw_key:
            # load in kwargs
            kw = rds.get(kw_key)
            if not kw:
                raise InternalServerError(
                    'An unexpected error occurred while attempting to run a Sous Chef.'
                )
            kw = pickle_to_obj(kw)
            # delete them.
            rds.delete(kw_key)

        # import sous chef
        SousChef = sc_exec.from_import_path(sous_chef_path)

        # initialize it with kwargs
        kw['org'] = db.session\
            .query(Org).get(recipe.org.id)\
            .to_dict(incl_domains=True)
        kw['recipe'] = recipe.to_dict()
        sous_chef = SousChef(**kw)

        # indicate that the job is running
        if not kw.get('passthrough', False):
            recipe.status = 'running'
            db.session.add(recipe)
            db.session.commit()

        # cook it.
        data = sous_chef.cook()

        # passthrough the data.
        if kw.get('passthrough', False):
            return data

        # otherwise just exhaust the generator
        if isgenerator(data):
            data = list(data)

        # teardown this recipe
        sous_chef.teardown()

        # update status and next job from sous chef.
        recipe.status = "stable"
        recipe.traceback = None
        recipe.last_run = dates.now()
        if len(sous_chef.next_job.keys()):
            recipe.last_job = sous_chef.next_job
        db.session.add(recipe)
        db.session.commit()
        return True

    except:

        # always delete the kwargs.
        if kw_key:
            rds.delete(kw_key)

        if not kw.get('passthrough', False):
            db.session.rollback()
            recipe.status = "error"
            recipe.traceback = format_exc()
            recipe.last_run = dates.now()
            db.session.add(recipe)
            db.session.commit()

            # notification
            tb = format_exc()
            error_notification(recipe, tb)
            return MerlynneError(tb)

        raise MerlynneError(format_exc())