def test_bad_numeric(self): grecipe = copy.copy(good_recipe) grecipe['minutes'] = '014u30214' try: recipe_schema.validate(grecipe, sous_chef) assert False except RecipeSchemaError: assert True
def test_missing_required(self): grecipe = copy.copy(good_recipe) grecipe.pop('owner_screen_name') try: recipe_schema.validate(grecipe, sous_chef) assert False except RecipeSchemaError: assert True
def test_bad_email(self): grecipe = copy.copy(good_recipe) grecipe['notify_email'] = '014u30214 OR asdfasasdf AND flakfsdlakfas' try: recipe_schema.validate(grecipe, sous_chef) assert False except RecipeSchemaError: assert True
def test_bad_regex(self): grecipe = copy.copy(good_recipe) grecipe['user_regex'] = '014.****30214' try: recipe_schema.validate(grecipe, sous_chef) assert False except RecipeSchemaError: assert True
def test_missing_schedules(self): grecipe = copy.copy(good_recipe) grecipe['schedule_by'] = 'crontab' grecipe['crontab'] = None try: recipe_schema.validate(grecipe, sous_chef) assert False except RecipeSchemaError: assert True
def test_partial_update(self): """ Simiulate a partial update process. """ old_recipe = copy.copy(good_recipe) sc = SousChef(**sous_chef) db_session.add(sc) db_session.commit() old_recipe = recipe_schema.validate(old_recipe, sc.to_dict()) old_id = old_recipe['options']['set_content_items'][0]['id'] old_recipe['slug'] += "-{}".format(gen_short_uuid()) r = Recipe(sc, **old_recipe) db_session.add(r) db_session.commit() new_recipe = { 'owner_screen_name': 'johnoliver', 'last_job': { 'foo': 'bar' }, 'status': 'stable', "set_content_items": [{ 'id': 2, 'title': 'foobar' }] } new_recipe = recipe_schema.update(r, new_recipe, sc.to_dict()) assert (new_recipe['options']['owner_screen_name'] == 'johnoliver') assert (new_recipe['last_job']['foo'] == 'bar') assert (new_recipe['status'] == 'stable') assert (new_recipe['options']['set_content_items'][0]['id'] != old_id) db_session.delete(r) db_session.delete(sc) db_session.commit()
def test_partial_update(self): """ Simiulate a partial update process. """ old_recipe = copy.copy(good_recipe) sc = SousChef(**sous_chef) db_session.add(sc) db_session.commit() old_recipe = recipe_schema.validate(old_recipe, sc.to_dict()) old_id = old_recipe['options']['set_content_items'][0]['id'] old_recipe['slug'] += "-{}".format(gen_short_uuid()) r = Recipe(sc, **old_recipe) db_session.add(r) db_session.commit() new_recipe = { 'owner_screen_name': 'johnoliver', 'last_job': {'foo': 'bar'}, 'status': 'stable', "set_content_items": [{'id': 2, 'title': 'foobar'}] } new_recipe = recipe_schema.update( r, new_recipe, sc.to_dict()) assert(new_recipe['options']['owner_screen_name'] == 'johnoliver') assert(new_recipe['last_job']['foo'] == 'bar') assert(new_recipe['status'] == 'stable') assert(new_recipe['options']['set_content_items'][0]['id'] != old_id) db_session.delete(r) db_session.delete(sc) db_session.commit()
def create_recipe(user, org): req_data = request_data() sous_chef = req_data.pop('sous_chef', arg_str('sous_chef', None)) if not sous_chef: raise RequestError( 'You must pass in a SousChef ID or slug to create a recipe.') sc = fetch_by_id_or_field(SousChef, 'slug', sous_chef) if not sc: raise RequestError( 'A SousChef does not exist with ID/slug {}'.format(sous_chef)) # validate the recipe and add it to the database. recipe = recipe_schema.validate(req_data, sc.to_dict()) r = Recipe(sc, user_id=user.id, org_id=org.id, **recipe) db.session.add(r) db.session.commit() # if the recipe creates metrics create them in here. if 'metrics' in sc.creates: for name, params in sc.metrics.items(): m = Metric(name=name, recipe_id=r.id, org_id=org.id, **params) db.session.add(m) try: db.session.commit() except Exception as e: raise ConflictError( "You tried to create a metric that already exists. " "Here's the exact error:\n{}".format(e.message)) return jsonify(r)
def gen_built_in_recipes(org): recipes = [] metrics = [] u = choice(org.users) # add default recipes for recipe in load_default_recipes(): sous_chef_slug = recipe.pop('sous_chef') if not sous_chef_slug: raise RecipeSchemaError( 'Default recipe "{}" is missing a "sous_chef" slug.'.format( recipe.get('name', ''))) sc = SousChef.query\ .filter_by(slug=sous_chef_slug)\ .first() if not sc: raise RecipeSchemaError( '"{}" is not a valid SousChef slug or the SousChef does not yet exist.' .format(sous_chef_slug)) recipe = recipe_schema.validate(recipe, sc.to_dict()) recipe['user_id'] = u.id recipe['org_id'] = org.id r = Recipe.query\ .filter_by(org_id=org.id, name=recipe['name'])\ .first() if not r: # add to database here. r = Recipe(sc, **recipe) else: for name, value in recipe.items(): if name != 'options': setattr(r, name, value) else: r.set_options(value) db.session.add(r) db.session.commit() recipes.append(r) # if the recipe creates metrics add them in here. if 'metrics' in sc.creates: for name, params in sc.metrics.items(): m = Metric.query\ .filter_by(org_id=org.id, recipe_id=r.id, name=name, type=params['type'])\ .first() # print "METRICS PARAMS", params if not m: m = Metric(name=name, recipe_id=r.id, org_id=org.id, **params) else: for k, v in params.items(): setattr(m, k, v) metrics.append(m) db.session.add(m) db.session.commit() return recipes, metrics
def test_good_recipe_no_nesting(self): grecipe = copy.deepcopy(good_recipe) r = recipe_schema.validate(grecipe, sous_chef) o = r.get('options', {}) # make sure name, slug, description, time of day, and interval are # extracted added to the top-level of recipe. assert('slug' in r) assert('description' in r) assert('name' in r) assert('schedule_by' in r) assert('minutes' in r) # make sure user_id is not in recipe or options assert('user_id' not in r) assert('user_id' not in o) # make sure slug gets a new hash assert(sous_chef['slug'] != r['slug']) # make sure interval was parsed to null properly assert(r['crontab'] is None) # make sure event counts is parsed to null assert('event_counts' not in o) # make sure regex is parsed. assert(isinstance(o['user_regex'], RE_TYPE)) # make sure start date is parsed assert(isinstance(o['start_date'], datetime.datetime)) # make boolean is parsed assert(o['filter_bots'] is True) # make sure start date is UTC assert(o['start_date'].tzinfo == pytz.utc) # make sure min followers got filled in with it's defaults. assert(o['min_followers'] == 0) assert(o['link_url'] == "http://example.com/some-url") # assert screen name is list assert(isinstance(o['owner_screen_name'], list)) # make sure search query is a SearchString assert(isinstance(o['search_query'], SearchString)) # make sure first instance of content_items is a dict assert(isinstance(o['set_content_items'][0], dict)) # make sure we can serialize this back to json. obj_to_json(r)
def test_good_recipe_no_nesting(self): grecipe = copy.deepcopy(good_recipe) r = recipe_schema.validate(grecipe, sous_chef) o = r.get('options', {}) # make sure name, slug, description, time of day, and interval are # extracted added to the top-level of recipe. assert ('slug' in r) assert ('description' in r) assert ('name' in r) assert ('schedule_by' in r) assert ('minutes' in r) # make sure user_id is not in recipe or options assert ('user_id' not in r) assert ('user_id' not in o) # make sure slug gets a new hash assert (sous_chef['slug'] != r['slug']) # make sure interval was parsed to null properly assert (r['crontab'] is None) # make sure event counts is parsed to null assert ('event_counts' not in o) # make sure regex is parsed. assert (isinstance(o['user_regex'], RE_TYPE)) # make sure start date is parsed assert (isinstance(o['start_date'], datetime.datetime)) # make boolean is parsed assert (o['filter_bots'] is True) # make sure start date is UTC assert (o['start_date'].tzinfo == pytz.utc) # make sure min followers got filled in with it's defaults. assert (o['min_followers'] == 0) assert (o['link_url'] == "http://example.com/some-url") # assert screen name is list assert (isinstance(o['owner_screen_name'], list)) # make sure search query is a SearchString assert (isinstance(o['search_query'], SearchString)) # make sure first instance of content_items is a dict assert (isinstance(o['set_content_items'][0], dict)) # make sure we can serialize this back to json. obj_to_json(r)
def run(config, **kw): """ Programmatically execute a sous chef given configutations and recipe options. Will not load data back into NewsLynx. This can only be done via Recipes. """ try: # load config file if not isinstance(config, types.DictType): if isinstance(config, basestring) and \ (config.endswith('yaml') or config.endswith('json') or config.endswith('yml')): config = sous_chef_schema.load(config) else: msg = 'Invalid input for config file: {}'.format(config) raise SousChefExecError(msg) else: config = sous_chef_schema.validate(config, None) # parse keyword arguments sc_opts = dict( org=kw.pop('org'), apikey=kw.pop('apikey'), passthrough=True, config=config ) # format all other options as a recipe recipe = dict( status='stable', id=-1 ) recipe.update(kw) sc_opts['recipe'] = recipe_schema.validate(recipe, config) # import sous chef SC = from_import_path(config['runs']) # initialize it with kwargs and cook return SC(**sc_opts).run() # bubble up traceback. except: raise SousChefExecError(format_exc())
def create_recipe(user, org): req_data = request_data() sous_chef = req_data.pop('sous_chef', arg_str('sous_chef', None)) if not sous_chef: raise RequestError( 'You must pass in a SousChef ID or slug to create a recipe.') sc = fetch_by_id_or_field(SousChef, 'slug', sous_chef) if not sc: raise RequestError( 'A SousChef does not exist with ID/slug {}' .format(sous_chef)) # validate the recipe and add it to the database. recipe = recipe_schema.validate(req_data, sc.to_dict()) r = Recipe(sc, user_id=user.id, org_id=org.id, **recipe) db.session.add(r) db.session.commit() # if the recipe creates metrics create them in here. if 'metrics' in sc.creates: for name, params in sc.metrics.items(): m = Metric( name=name, recipe_id=r.id, org_id=org.id, **params) db.session.add(m) try: db.session.commit() except Exception as e: raise ConflictError( "You tried to create a metric that already exists. " "Here's the exact error:\n{}" .format(e.message) ) return jsonify(r)
def run(config, **kw): """ Programmatically execute a sous chef given configutations and recipe options. Will not load data back into NewsLynx. This can only be done via Recipes. """ try: # load config file if not isinstance(config, types.DictType): if isinstance(config, basestring) and \ (config.endswith('yaml') or config.endswith('json') or config.endswith('yml')): config = sous_chef_schema.load(config) else: msg = 'Invalid input for config file: {}'.format(config) raise SousChefExecError(msg) else: config = sous_chef_schema.validate(config, None) # parse keyword arguments sc_opts = dict(org=kw.pop('org'), apikey=kw.pop('apikey'), passthrough=True, config=config) # format all other options as a recipe recipe = dict(status='stable', id=-1) recipe.update(kw) sc_opts['recipe'] = recipe_schema.validate(recipe, config) # import sous chef SC = from_import_path(config['runs']) # initialize it with kwargs and cook return SC(**sc_opts).run() # bubble up traceback. except: raise SousChefExecError(format_exc())
def gen_built_in_recipes(org): recipes = [] metrics = [] u = choice(org.users) # add default recipes for recipe in load_default_recipes(): sous_chef_slug = recipe.pop('sous_chef') if not sous_chef_slug: raise RecipeSchemaError( 'Default recipe "{}" is missing a "sous_chef" slug.' .format(recipe.get('name', ''))) sc = SousChef.query\ .filter_by(slug=sous_chef_slug)\ .first() if not sc: raise RecipeSchemaError( '"{}" is not a valid SousChef slug or the SousChef does not yet exist.' .format(sous_chef_slug)) recipe = recipe_schema.validate(recipe, sc.to_dict()) recipe['user_id'] = u.id recipe['org_id'] = org.id r = Recipe.query\ .filter_by(org_id=org.id, name=recipe['name'])\ .first() if not r: # add to database here. r = Recipe(sc, **recipe) else: for name, value in recipe.items(): if name != 'options': setattr(r, name, value) else: r.set_options(value) db.session.add(r) db.session.commit() recipes.append(r) # if the recipe creates metrics add them in here. if 'metrics' in sc.creates: for name, params in sc.metrics.items(): m = Metric.query\ .filter_by(org_id=org.id, recipe_id=r.id, name=name, type=params['type'])\ .first() # print "METRICS PARAMS", params if not m: m = Metric( name=name, recipe_id=r.id, org_id=org.id, **params) else: for k, v in params.items(): setattr(m, k, v) metrics.append(m) db.session.add(m) db.session.commit() return recipes, metrics
def create_recipe(user, org): req_data = request_data() sous_chef = req_data.pop('sous_chef', arg_str('sous_chef', None)) if not sous_chef: raise NotFoundError( 'You must pass in a SousChef ID or slug to create a recipe.') sc = fetch_by_id_or_field(SousChef, 'slug', sous_chef) if not sc: raise NotFoundError( 'A SousChef does not exist with ID/slug {}' .format(sous_chef)) # validate the recipe and add it to the database. recipe = recipe_schema.validate(req_data, sc.to_dict()) r = Recipe(sc, user_id=user.id, org_id=org.id, **recipe) db.session.add(r) try: db.session.commit() except Exception as e: raise ConflictError( "You tried to create a recipe that already exists. " "Here's the exact error:\n{}" .format(e.message) ) # if the recipe creates metrics create them in here. if 'metrics' in sc.creates: for name, params in sc.metrics.items(): m = Metric.query\ .filter_by(org_id=org.id, recipe_id=r.id, name=name, type=params['type'])\ .first() if not m: m = Metric( name=name, recipe_id=r.id, org_id=org.id, **params) else: for k, v in params.items(): setattr(m, k, v) db.session.add(m) # if the recipe creates a report, create it here. if 'report' in sc.creates and r.status == 'stable': rep = Report.query\ .filter_by(org_id=org.id, recipe_id=r.id, slug=report['slug'])\ .first() if not rep: rep = Report( recipe_id=r.id, org_id=org.id, sous_chef_id=sous_chef_id, user_id=user_id, **report) else: for k, v in report.items(): setattr(rep, k, v) db.session.add(rep) try: db.session.commit() except Exception as e: raise ConflictError( "You tried to create a metric that already exists. " "Here's the exact error:\n{}" .format(e.message) ) return jsonify(r)
def recipes(org): """ (Re)load all default recipes. """ # add default recipes for recipe in init.load_default_recipes(): # fetch it's sous chef. sous_chef_slug = recipe.pop('sous_chef') if not sous_chef_slug: raise RecipeSchemaError( "Default recipe '{}' is missing a 'sous_chef' slug.".format( recipe.get('slug', ''))) sc = SousChef.query\ .filter_by(slug=sous_chef_slug, org_id=org.id)\ .first() if not sc: raise RecipeSchemaError( '"{}" is not a valid SousChef slug or the ' 'SousChef does not yet exist for "{}"'.format( sous_chef_slug, org.slug)) # validate the recipe recipe = recipe_schema.validate(recipe, sc.to_dict()) # fill in relations recipe['user_id'] = org.super_user.id recipe['org_id'] = org.id r = Recipe.query\ .filter_by(org_id=org.id, name=recipe['name'])\ .first() if not r: log.info('Creating recipe: "{slug}"'.format(**recipe)) # add to database here. r = Recipe(sc, **recipe) else: log.warning('Updating recipe: "{slug}"'.format(**recipe)) for name, value in recipe.items(): if name != 'options': setattr(r, name, value) else: r.set_options(value) db.session.add(r) db.session.commit() # if the recipe creates metrics create them here. if 'metrics' in sc.creates and r.status == 'stable': for name, params in sc.metrics.items(): m = Metric.query\ .filter_by(org_id=org.id, recipe_id=r.id, name=name, type=params['type'])\ .first() # print "METRICS PARAMS", params if not m: log.info('Creating metric: "{}"'.format(name)) m = Metric(name=name, recipe_id=r.id, org_id=org.id, **params) else: log.warning('Updating metric: "{}"'.format(name)) for k, v in params.items(): setattr(m, k, v) db.session.add(m) db.session.commit() return org
def org_create(user): req_data = request_data() if not user.super_user: raise ForbiddenError('You must be the super user to create an Org') if 'name' not in req_data \ or 'timezone' not in req_data: raise RequestError("An Org requires a 'name' and 'timezone") org = Org.query\ .filter_by(name=req_data['name'])\ .first() # if the org doesnt exist, create it. if org: raise RequestError("Org '{}' already exists".format(req_data['name'])) # add the requesting user to the org org = Org(name=req_data['name'], timezone=req_data['timezone']) org.users.append(user) db.session.add(org) db.session.commit() # add default tags for tag in load_default_tags(): tag['org_id'] = org.id t = Tag(**tag) db.session.add(t) # add default recipes for recipe in load_default_recipes(): # fetch it's sous chef. sous_chef_slug = recipe.pop('sous_chef') if not sous_chef_slug: raise RecipeSchemaError( "Default recipe '{}' is missing a 'sous_chef' slug.".format( recipe.get('slug', ''))) sc = SousChef.query\ .filter_by(slug=sous_chef_slug)\ .first() if not sc: raise RecipeSchemaError( '"{}" is not a valid SousChef slug or the ' 'SousChef does not yet exist.'.format(sous_chef_slug)) # validate the recipe recipe = recipe_schema.validate(recipe, sc.to_dict()) # fill in relations recipe['user_id'] = user.id recipe['org_id'] = org.id # add to database r = Recipe(sc, **recipe) db.session.add(r) db.session.commit() # if the recipe creates metrics create them here. if 'metrics' in sc.creates: for name, params in sc.metrics.items(): m = Metric(name=name, recipe_id=r.id, org_id=org.id, **params) db.session.add(m) db.session.commit() return jsonify(org)
def org_create(user): req_data = request_data() if not user.super_user: raise ForbiddenError( 'You must be the super user to create an Org') if 'name' not in req_data \ or 'timezone' not in req_data: raise RequestError( "An Org requires a 'name' and 'timezone") org = Org.query\ .filter_by(name=req_data['name'])\ .first() # if the org doesnt exist, create it. if org: raise RequestError( "Org '{}' already exists" .format(req_data['name'])) # add the requesting user to the org org = Org( name=req_data['name'], timezone=req_data['timezone'] ) org.users.append(user) db.session.add(org) db.session.commit() # add default tags for tag in load_default_tags(): tag['org_id'] = org.id t = Tag(**tag) db.session.add(t) # add default recipes for recipe in load_default_recipes(): # fetch it's sous chef. sous_chef_slug = recipe.pop('sous_chef') if not sous_chef_slug: raise RecipeSchemaError( "Default recipe '{}' is missing a 'sous_chef' slug." .format(recipe.get('slug', ''))) sc = SousChef.query\ .filter_by(slug=sous_chef_slug)\ .first() if not sc: raise RecipeSchemaError( '"{}" is not a valid SousChef slug or the ' 'SousChef does not yet exist.' .format(sous_chef_slug)) # validate the recipe recipe = recipe_schema.validate(recipe, sc.to_dict()) # fill in relations recipe['user_id'] = user.id recipe['org_id'] = org.id # add to database r = Recipe(sc, **recipe) db.session.add(r) db.session.commit() # if the recipe creates metrics create them here. if 'metrics' in sc.creates: for name, params in sc.metrics.items(): m = Metric( name=name, recipe_id=r.id, org_id=org.id, **params) db.session.add(m) db.session.commit() return jsonify(org)
def recipes(org): """ (Re)load all default recipes. """ # add default recipes for recipe in init.load_default_recipes(): # fetch it's sous chef. sous_chef_slug = recipe.pop('sous_chef') if not sous_chef_slug: raise RecipeSchemaError( "Default recipe '{}' is missing a 'sous_chef' slug." .format(recipe.get('slug', ''))) sc = SousChef.query\ .filter_by(slug=sous_chef_slug, org_id=org.id)\ .first() if not sc: raise RecipeSchemaError( '"{}" is not a valid SousChef slug or the ' 'SousChef does not yet exist for "{}"' .format(sous_chef_slug, org.slug)) # validate the recipe recipe = recipe_schema.validate(recipe, sc.to_dict()) # fill in relations recipe['user_id'] = org.super_user.id recipe['org_id'] = org.id r = Recipe.query\ .filter_by(org_id=org.id, name=recipe['name'])\ .first() if not r: log.info('Creating recipe: "{slug}"'.format(**recipe)) # add to database here. r = Recipe(sc, **recipe) else: log.warning('Updating recipe: "{slug}"'.format(**recipe)) for name, value in recipe.items(): if name != 'options': setattr(r, name, value) else: r.set_options(value) db.session.add(r) db.session.commit() # if the recipe creates metrics create them here. if 'metrics' in sc.creates and r.status == 'stable': for name, params in sc.metrics.items(): m = Metric.query\ .filter_by(org_id=org.id, recipe_id=r.id, name=name, type=params['type'])\ .first() # print "METRICS PARAMS", params if not m: log.info('Creating metric: "{}"'.format(name)) m = Metric( name=name, recipe_id=r.id, org_id=org.id, **params) else: log.warning('Updating metric: "{}"'.format(name)) for k, v in params.items(): setattr(m, k, v) db.session.add(m) db.session.commit() return org
def create_recipe(user, org): req_data = request_data() sous_chef = req_data.pop('sous_chef', arg_str('sous_chef', None)) if not sous_chef: raise NotFoundError( 'You must pass in a SousChef ID or slug to create a recipe.') sc = fetch_by_id_or_field(SousChef, 'slug', sous_chef) if not sc: raise NotFoundError( 'A SousChef does not exist with ID/slug {}'.format(sous_chef)) # validate the recipe and add it to the database. recipe = recipe_schema.validate(req_data, sc.to_dict()) r = Recipe(sc, user_id=user.id, org_id=org.id, **recipe) db.session.add(r) try: db.session.commit() except Exception as e: raise ConflictError( "You tried to create a recipe that already exists. " "Here's the exact error:\n{}".format(e.message)) # if the recipe creates metrics create them in here. if 'metrics' in sc.creates: for name, params in sc.metrics.items(): m = Metric.query\ .filter_by(org_id=org.id, recipe_id=r.id, name=name, type=params['type'])\ .first() if not m: m = Metric(name=name, recipe_id=r.id, org_id=org.id, **params) else: for k, v in params.items(): setattr(m, k, v) db.session.add(m) # if the recipe creates a report, create it here. if 'report' in sc.creates and r.status == 'stable': rep = Report.query\ .filter_by(org_id=org.id, recipe_id=r.id, slug=report['slug'])\ .first() if not rep: rep = Report(recipe_id=r.id, org_id=org.id, sous_chef_id=sous_chef_id, user_id=user_id, **report) else: for k, v in report.items(): setattr(rep, k, v) db.session.add(rep) try: db.session.commit() except Exception as e: raise ConflictError( "You tried to create a metric that already exists. " "Here's the exact error:\n{}".format(e.message)) return jsonify(r)