def test_metrics_schema_no_metrics(self):
sc = {
"name": "Google Analytics Content Timeseries",
"slug": "twitter-list",
"description": "Grabs timeseries metrics for content items from Google Analytics.",
"runs": "newslynx.sc.events.twitter.List",
"creates": "metrics",
"options": {
"age_of_article": {
"input_type": "number",
"value_types": ["numeric", "nulltype"],
"accepts_list": False,
"required": False,
"default": None,
"help": {
"description": "The age of content-items (in days) which metrics will be pulled for.",
"placeholder": "cspan"
}
}
}
}
try:
sous_chef_schema.validate(sc)
except SousChefSchemaError:
assert True
else:
assert False
def test_good_metrics_schema(self):
sc = {
"name": "Google Analytics Content Timeseries",
"slug": "twitter-list",
"description": "Grabs timeseries metrics for content items from Google Analytics.",
"runs": "newslynx.sc.events.twitter.List",
"creates": "metrics",
"metrics": {
"ga_pageviews": {
"display_name": "Pageviews",
"type": "count",
"content_levels": ['timeseries', 'summary', 'comparison'],
"org_levels": [],
"faceted": False,
"agg": "sum"
}
},
"options": {
"age_of_article": {
"input_type": "number",
"value_types": ["numeric", "nulltype"],
"accepts_list": False,
"required": False,
"default": None,
"help": {
"description": "The age of content-items (in days) which metrics will be pulled for.",
"placeholder": "cspan"
}
}
}
}
sous_chef_schema.validate(sc)
def test_email_non_text(self):
sc = {
"name": "Google Analytics Content Timeseries",
"slug": "twitter_list",
"description": "Grabs timeseries metrics for content items from Google Analytics.",
"runs": "newslynx.sc.events.twitter.List",
"creates": "events",
"options": {
"owner_screen_name": {
"input_type": "paragraph",
"value_types": ["email"],
"accepts_list": False,
"required": True,
"help": {
"placeholder": "cspan"
}
},
"min_followers": {
"input_type": "number",
"value_types": ["number"],
"accepts_list": False,
"required": False,
"default": 0,
"help": {
"placeholder": 1000
}
}
}
}
try:
sous_chef_schema.validate(sc)
except SousChefSchemaError:
assert True
else:
assert False
def test_reserved_name(self):
sc = {
"name": "Twitter List",
"slug": "twitter-list",
"description": "Extracts events from a twitter list.",
"runs": "newslynx.sc.events.twitter.List",
"creates": "events",
"options": {
"user_id": {
"input_type": "text",
"value_types": ["string"],
"accepts_list": True,
"required": True,
"help": {
"placeholder": "cspan"
},
},
"min_folowers": {
"input_type": "number",
"value_types": ["numeric"],
"accepts_list": False,
"required": False,
"default": 0
}
}
}
try:
sous_chef_schema.validate(sc)
except SousChefSchemaError:
assert True
else:
assert False
def test_good_schema(self):
sc = {
"name": "Twitter List",
"slug": "twitter-list",
"description": "Extracts events from a twitter list.",
"runs": "newslynx.sc.events.twitter.List",
"creates": "events",
"options": {
"owner_screen_name": {
"input_type": "text",
"value_types": ["string"],
"accepts_list": True,
"required": True,
"help": {
"placeholder": "cspan"
},
},
"min_followers": {
"input_type": "number",
"value_types": ["numeric"],
"accepts_list": False,
"required": False,
"default": 0,
"help": {
"placeholder": 1000
}
}
}
}
sous_chef_schema.validate(sc)
def test_bad_slug_format(self):
sc = {
"name": "Google Analytics Content Timeseries",
"slug": "twitter_list",
"description": "Grabs timeseries metrics for content items from Google Analytics.",
"runs": "newslynx.sc.events.twitter.List",
"creates": "events",
"options": {
"owner_screen_name": {
"input_type": "text",
"value_types": ["text"],
"accepts_list": True,
"required": True,
"help": {
"placeholder": "cspan"
}
}
}
}
try:
sous_chef_schema.validate(sc)
except SousChefSchemaError:
assert True
else:
assert False
def test_missing_options(self):
sc = {
"name": "Google Analytics Content Timeseries",
"slug": "twitter-list",
"description": "Grabs timeseries metrics for content items from Google Analytics.",
"runs": "newslynx.sc.events.twitter.List",
"creates": "events"
}
try:
sous_chef_schema.validate(sc)
except SousChefSchemaError:
assert True
else:
assert False
def init():
# create the database
db.configure_mappers()
db.create_all()
# create the super user
u = User(name=settings.SUPER_USER,
email=settings.SUPER_USER_EMAIL,
password=settings.SUPER_USER_PASSWORD,
admin=True,
super_user=True)
# optionally add super user apikey
if getattr(settings, 'SUPER_USER_APIKEY', None):
u.apikey = settings.SUPER_USER_APIKEY
db_session.add(u)
# load sql extensions + functions
for sql in load_sql():
db_session.execute(sql)
# load built-in sous-chefs
for sc in load_sous_chefs():
sc = sous_chef_schema.validate(sc)
s = SousChef(**sc)
db_session.add(s)
# commit
db_session.commit()
def test_partial_update(self):
"""
Simiulate a partial update process.
"""
sc = {
"name": "Twitter List",
"slug": "twitter-list",
"description": "Extracts events from a twitter list.",
"runs": "newslynx.sc.events.twitter.List",
"creates": "events",
"options": {
"owner_screen_name": {
"input_type": "text",
"value_types": ["string"],
"accepts_list": True,
"required": True,
"help": {
"placeholder": "cspan"
},
},
"min_followers": {
"input_type": "number",
"value_types": ["numeric"],
"accepts_list": False,
"required": False,
"default": 0,
"help": {
"placeholder": 1000
}
}
}
}
sc = sous_chef_schema.validate(sc)
sc = SousChef(**sc)
db_session.add(sc)
db_session.commit()
new_sc = {
'name': 'Twitter List to Event',
'options': {
'owner_screen_name': {
'accepts_list': False
}
}
}
new_sc = sous_chef_schema.update(sc, new_sc)
assert (new_sc['name'] == 'Twitter List to Event')
assert (new_sc['options']['owner_screen_name']['accepts_list'] is
False)
db_session.delete(sc)
db_session.commit()
def test_partial_update(self):
"""
Simiulate a partial update process.
"""
sc = {
"name": "Twitter List",
"slug": "twitter-list",
"description": "Extracts events from a twitter list.",
"runs": "newslynx.sc.events.twitter.List",
"creates": "events",
"options": {
"owner_screen_name": {
"input_type": "text",
"value_types": ["string"],
"accepts_list": True,
"required": True,
"help": {
"placeholder": "cspan"
},
},
"min_followers": {
"input_type": "number",
"value_types": ["numeric"],
"accepts_list": False,
"required": False,
"default": 0,
"help": {
"placeholder": 1000
}
}
}
}
sc = sous_chef_schema.validate(sc)
sc = SousChef(**sc)
db_session.add(sc)
db_session.commit()
new_sc = {
'name': 'Twitter List to Event',
'options': {
'owner_screen_name': {
'accepts_list': False
}
}
}
new_sc = sous_chef_schema.update(sc, new_sc)
assert(new_sc['name'] == 'Twitter List to Event')
assert(new_sc['options']['owner_screen_name']['accepts_list'] is False)
db_session.delete(sc)
db_session.commit()
def create_sous_chef(user):
req_data = request_data()
# validate the sous chef
sc = sous_chef_schema.validate(req_data)
# add it to the database
sc = SousChef(**sc)
db.session.add(sc)
try:
db.session.commit()
except Exception as e:
raise RequestError(e.message)
return jsonify(sc)
def create_sous_chef(user):
req_data = request_data()
# validate the sous chef
sc = sous_chef_schema.validate(req_data)
# add it to the database
sc = SousChef(**sc)
db.session.add(sc)
try:
db.session.commit()
except Exception as e:
raise RequestError(e.message)
return jsonify(sc)
def run(config, **kw):
"""
Programmatically execute a sous chef given configutations
and recipe options. Will not load data back into NewsLynx. This can
only be done via Recipes.
"""
try:
# load config file
if not isinstance(config, types.DictType):
if isinstance(config, basestring) and \
(config.endswith('yaml') or config.endswith('json') or
config.endswith('yml')):
config = sous_chef_schema.load(config)
else:
msg = 'Invalid input for config file: {}'.format(config)
raise SousChefExecError(msg)
else:
config = sous_chef_schema.validate(config, None)
# parse keyword arguments
sc_opts = dict(
org=kw.pop('org'),
apikey=kw.pop('apikey'),
passthrough=True,
config=config
)
# format all other options as a recipe
recipe = dict(
status='stable',
id=-1
)
recipe.update(kw)
sc_opts['recipe'] = recipe_schema.validate(recipe, config)
# import sous chef
SC = from_import_path(config['runs'])
# initialize it with kwargs and cook
return SC(**sc_opts).run()
# bubble up traceback.
except:
raise SousChefExecError(format_exc())
def sous_chefs(org):
"""
(Re)load all sous chefs.
"""
# load all sous-chefs
for sc, fp in init.load_sous_chefs():
sc = sous_chef_schema.validate(sc, fp)
sc['org_id'] = org.id
sc_obj = db.session.query(SousChef).filter_by(slug=sc['slug']).first()
if not sc_obj:
log.info('Creating sous chef: "{slug}"'.format(**sc))
sc_obj = SousChef(**sc)
else:
log.warning('Updating sous chef: "{slug}"'.format(**sc))
sc = sous_chef_schema.update(sc_obj.to_dict(), sc)
for name, value in sc.items():
setattr(sc_obj, name, value)
db.session.add(sc_obj)
db.session.commit()
return org
def sous_chefs(org):
"""
(Re)load all sous chefs.
"""
# load all sous-chefs
for sc, fp in init.load_sous_chefs():
sc = sous_chef_schema.validate(sc, fp)
sc['org_id'] = org.id
sc_obj = db.session.query(SousChef).filter_by(
slug=sc['slug']).first()
if not sc_obj:
log.info('Creating sous chef: "{slug}"'.format(**sc))
sc_obj = SousChef(**sc)
else:
log.warning('Updating sous chef: "{slug}"'.format(**sc))
sc = sous_chef_schema.update(sc_obj.to_dict(), sc)
for name, value in sc.items():
setattr(sc_obj, name, value)
db.session.add(sc_obj)
db.session.commit()
return org
def run(config, **kw):
"""
Programmatically execute a sous chef given configutations
and recipe options. Will not load data back into NewsLynx. This can
only be done via Recipes.
"""
try:
# load config file
if not isinstance(config, types.DictType):
if isinstance(config, basestring) and \
(config.endswith('yaml') or config.endswith('json') or
config.endswith('yml')):
config = sous_chef_schema.load(config)
else:
msg = 'Invalid input for config file: {}'.format(config)
raise SousChefExecError(msg)
else:
config = sous_chef_schema.validate(config, None)
# parse keyword arguments
sc_opts = dict(org=kw.pop('org'),
apikey=kw.pop('apikey'),
passthrough=True,
config=config)
# format all other options as a recipe
recipe = dict(status='stable', id=-1)
recipe.update(kw)
sc_opts['recipe'] = recipe_schema.validate(recipe, config)
# import sous chef
SC = from_import_path(config['runs'])
# initialize it with kwargs and cook
return SC(**sc_opts).run()
# bubble up traceback.
except:
raise SousChefExecError(format_exc())
def test_schema(self):
for sc, fp in load_sous_chefs(SOUS_CHEF_DIR, incl_internal=False):
sous_chef_schema.validate(sc, fp)
assert True
def test_schema(self):
for sc, fp in load_sous_chefs(SOUS_CHEF_DIR, incl_internal=False):
sous_chef_schema.validate(sc, fp)
assert True
},
"user_regex": {
"input_type": "text",
"value_types": ["regex", "nulltype"],
"required": False
},
"set_content_items": {
"input_type": "checkbox",
"value_types": ["json"],
"accepts_list": True,
"required": False
}
}
}
sous_chef = sous_chef_schema.validate(sous_chef)
good_recipe = {
"name": "My Cool Twitter List Recipe",
"user_id": 1,
"last_job": {},
"owner_screen_name": ["foobar", "uqbar"],
"description": "This is what my cool twitter list recipe does.",
"event_counts": {},
"start_date": "2015-07-08",
"search_query": "~fracking | drilling",
"user_regex": r".*",
"link_url": "http://example.com/some-url",
"notify_email": "*****@*****.**",
"tag_ids": [1, 'this-is-also-a-tag'],
"filter_bots": "t",
},
"user_regex": {
"input_type": "text",
"value_types": ["regex", "nulltype"],
"required": False
},
"set_content_items": {
"input_type": "checkbox",
"value_types": ["json"],
"accepts_list": True,
"required": False
}
}
}
sous_chef = sous_chef_schema.validate(sous_chef)
good_recipe = {
"name": "My Cool Twitter List Recipe",
"user_id": 1,
"last_job": {},
"owner_screen_name": ["foobar", "uqbar"],
"description": "This is what my cool twitter list recipe does.",
"event_counts": {},
"start_date": "2015-07-08",
"search_query": "~fracking | drilling",
"user_regex": r".*",
"link_url": "http://example.com/some-url",
"notify_email": "*****@*****.**",
"tag_ids": [1, 'this-is-also-a-tag'],
"filter_bots": "t",
def run(opts, **kwargs):
# create the database
try:
with app.app_context():
echo('Creating database "{}"'.format(settings.SQLALCHEMY_DATABASE_URI),
no_color=opts.no_color)
db.configure_mappers()
db.create_all()
# create the super user
u = User.query.filter_by(email=settings.SUPER_USER_EMAIL).first()
if not u:
echo('Creating super user "{}"'.format(settings.SUPER_USER_EMAIL),
no_color=opts.no_color)
u = User(name=settings.SUPER_USER,
email=settings.SUPER_USER_EMAIL,
password=settings.SUPER_USER_PASSWORD,
admin=True,
super_user=True)
# optionally add super user apikey
if getattr(settings, 'SUPER_USER_APIKEY', None):
u.apikey = settings.SUPER_USER_APIKEY
else:
echo('Updating super user "{}"'.format(settings.SUPER_USER_EMAIL),
no_color=opts.no_color)
u.name=settings.SUPER_USER,
u.email=settings.SUPER_USER_EMAIL,
u.password=settings.SUPER_USER_PASSWORD,
u.admin=True
super_user=True
db.session.add(u)
echo('(Re)Loading SQL Extensions', no_color=opts.no_color)
# load sql extensions + functions
for sql in load_sql():
db.session.execute(sql)
# load built-in sous-chefs
for sc in load_sous_chefs():
sc = sous_chef_schema.validate(sc)
sc_obj = db.session.query(SousChef).filter_by(slug=sc['slug']).first()
if not sc_obj:
echo('Importing Sous Chef "{}"'.format(sc['slug']),
no_color=opts.no_color)
sc_obj = SousChef(**sc)
else:
echo('Updating Sous Chef "{}"'.format(sc['slug']),
no_color=opts.no_color)
sc = sous_chef_schema.update(sc_obj.to_dict(), sc)
# udpate
for name, value in sc.items():
setattr(sc_obj, name, value)
db.session.add(sc_obj)
# commit
db.session.commit()
db.session.close()
except Exception as e:
db.session.rollback()
db.session.close()
raise e
def load_sous_chef(fp):
"""
Load and validate a sous chef config file.
"""
sc = _load_config_file(fp)
return sous_chef_schema.validate(sc)
def load_sous_chef(fp):
"""
Load and validate a sous chef config file.
"""
sc = _load_config_file(fp)
return sous_chef_schema.validate(sc)
