def import_(self, filepath):
dicts_to_import = RecipeImporter._fetch_data_from_path(filepath)
if len(dicts_to_import) > 1:
self.delete(delete_all=True)
for cocktail_dict in dicts_to_import:
try:
slug = Slug(cocktail_dict['display_name'])
LogService.info("Working %s" % slug)
c = CocktailFactory.raw_to_obj(cocktail_dict, slug)
except KeyError as e:
LogService.error("Something has bad data!")
LogService.error(cocktail_dict)
LogService.error(e)
continue
self.delete(cocktail=c)
db_obj = CocktailModel(**ObjectSerializer.serialize(c, 'dict'))
with self.pgconn.get_session() as session:
session.add(db_obj)
LogService.info("Successfully [re]created %s" % c.slug)
ObjectValidator.validate(db_obj, session=session, fatal=False)
Indexers.get_indexer(c).index(c)
CocktailScanCache.invalidate()
def _build_session(self):
"""
Construct a SQLAlchemy Session() context object. To avoid having to pass session
objects around between Barbados (backend) and Jamaica (frontend) this function
will attempt to determine if we're running inside of a Flask context at the time of
call (scoped session per-request) and use that session. If we're not then generate
one 'cause we're probably running in a script or something weird like that.
Flask sessions will close at the end of the request or when explicitly told to
so to prevent double-committing (which isn't a problem, it just resets the session
an extra time) this will feed back into the caller.
https://docs.sqlalchemy.org/en/13/orm/session_basics.html#closing
:return: Session context object, Boolean of whether to trigger a commit or not.
"""
commit = False
try:
from flask_sqlalchemy_session import current_session as session
if not session:
raise RuntimeError
LogService.debug("Using Flask session")
except RuntimeError as e:
session = self.ScopedSession()
commit = True
LogService.debug("Using thread scoped session")
return session, commit
def __init__(self, path):
connection_string = "sqlite:///%s" % path
LogService.info("connection string is '%s'" % connection_string)
self.engine = sqlalchemy.create_engine(connection_string)
# session.configure(bind=self.engine)
self.Session = sessionmaker(bind=self.engine)
def __init__(self,
hosts=os.getenv('AMARI_ZOOKEEPER_HOSTS', '127.0.0.1:2181'),
read_only=False):
self.hosts = hosts
self.read_only = read_only
LogService.info("Using Zookeeper hosts: \"%s\"" % hosts)
def retrieve(cls):
try:
return pickle.loads(CacheService.get(cls.cache_key))
except KeyError:
LogService.warning("Attempted to retrieve '%s' but it was empty. Repopulating..." % cls.cache_key)
cls.populate()
return pickle.loads(CacheService.get(cls.cache_key))
def get_value(self, skip_registry=False):
"""
Retrieve the value of a setting from the various sources.
Order goes: Registry, Environment Variable, Default.
This potentially enforces a type as well.
:param skip_registry: Boolean to skip looking at the registry.
:return: Value of the setting.
"""
registry_value = RegistryService.get(self.path, default_none=True)
env_value = os.getenv(key=self.env, default=None)
default_value = self.default
potential_values = (registry_value, env_value, default_value)
if skip_registry:
potential_values = (env_value, default_value)
# https://stackoverflow.com/questions/18533620/getting-the-first-non-none-value-from-list
try:
setting_value = next(value for value in potential_values
if value is not None)
except StopIteration:
raise SettingsException("No valid setting found for %s" %
self.path)
LogService.info("Setting %s => %s" % (self.path, setting_value))
return setting_value
def _connect(self):
"""
[re]Connect to the ZooKeeper host(s). Create a local attribute to this class
that acts as the client and does the actual interactions with ZK.
:return: None
"""
# If we don't have a self.zk attribute then we've never connected.
if not hasattr(self, 'zk'):
self.zk = KazooClient(hosts=self.hosts,
read_only=self.read_only,
timeout=5,
connection_retry=self._get_retry())
# Any state not connected is a bad thing. Warn and continue with execution below.
elif self.zk.state != KazooState.CONNECTED:
LogService.warning("ZooKeeper state is %s" % self.zk.state)
pass
# Connected state is good. Do nothing.
elif self.zk.state == KazooState.CONNECTED:
return
# I'm not sure if this actually does anything...
else:
raise Exception("We in a weird state. %s" % self.zk.state)
# Start the connection now that it's guaranteed to exist.
try:
return self.zk.start()
except KazooTimeoutError as e:
raise FatalException("Timeout connecting to ZooKeeper (%s)" % e)
def scrape_recipe(recipe):
url = "%s/%s" % (url_base, endpoints.get('recipe') % recipe)
LogService.info("scraping %s" % url)
parser = UpneatRecipeParser(slug=recipe, url=url)
raw_recipe = parser.parse()
return raw_recipe
def update_obj(cls, obj, id_value, id_attr='slug', commit=True):
"""
Update an existing model based on its current object state.
:param obj: The object to delete.
:param id_value: The ID of the object we should be updating.
:param id_attr: Identity attribute.
:param commit: Whether to commit this transaction now or deal with it yourself. Useful for batches.
:return: New model.
"""
with DatabaseService.get_session() as session:
model = cls._get_model_safe(session, obj, id_attr, id_value)
# This feels unsafe, but should be OK.
# https://stackoverflow.com/questions/9667138/how-to-update-sqlalchemy-row-entry
for key, value in ObjectSerializer.serialize(obj, 'dict').items():
old_value = getattr(model, key)
setattr(model, key, value)
if old_value != value:
LogService.info("Updating %s: '%s'->'%s'" %
(key, old_value, value))
if commit:
session.commit()
return model
def invalidate(cls):
"""
Invalidate (delete) the cache value and key.
:return: None
"""
LogService.info("Invalidating cache key %s" % cls.cache_key)
return CacheService.delete(cls.cache_key)
def delete(self):
LogService.debug("Deleting old data from database")
with self.pgconn.get_session() as session:
deleted = session.query(self.model).delete()
LogService.info("Deleted %s" % deleted)
Indexes.rebuild(ListIndex)
def delete(endpoint, args):
LogService.info("Deleting %s" % args.slug)
if args.slug == 'all':
result = requests.delete(endpoint)
else:
result = requests.delete("%s/%s" % (endpoint, args.slug))
Resource._handle_error(result)
def get(endpoint, args):
LogService.debug("Getting %s" % args.slug)
if args.slug == 'all':
result = requests.get(endpoint)
else:
result = requests.get("%s/%s" % (endpoint, args.slug))
print(json.dumps(result.json()))
Resource._handle_error(result)
def connect():
hosts = elasticsearch_settings.get('hosts')
scheme = elasticsearch_settings.get('scheme')
port = elasticsearch_settings.get('port')
LogService.info("Using ElasticSearch hosts: \"%s\" via %s/%i" %
(hosts, scheme, port))
connections.create_connection(scheme=scheme, hosts=hosts, port=port)
def execute(self, sort='_score'):
"""
Actually talk to ElasticSearch and run the query.
:param sort: ElasticSearch attribute on which to sort the results.
:return: SearchResults child class.
# @TODO address the search range hacks here.
"""
results = self.index_class.search()[0:1000].query(
self.q).sort(sort).execute()
LogService.info("Got %s results." % results.hits.total.value)
return SearchResults(hits=results)
def retrieve(cls):
"""
Retrieve the cache's value
:return: Various
"""
try:
return CacheService.get(cls.cache_key)
except KeyError:
LogService.warning("Attempted to retrieve '%s' but it was empty. Repopulating..." % cls.cache_key)
cls.populate()
return CacheService.get(cls.cache_key)
def init(self):
"""
Re-initialize all indexes. This calls rebuild on every registered
index class. There be dragons here.
:return: None
"""
for name in self._indexes.keys():
LogService.debug("Init on %s" % name)
try:
self.rebuild(self._indexes.get(name))
except NotFoundError or KeyError or AttributeError as e:
LogService.warning("Error re-initing index %s: %s" % (name, e))
def __init__(self, host, port, username, password, ssl, request_timeout,
flask_database_id):
self.host = host
self.port = port
self.username = username
self.password = password
self.ssl = ssl
self.request_timeout = request_timeout
self.flask_database_id = flask_database_id
LogService.info(
"Redis connection: redis://%s:%s@%s:%s?ssl=%s" %
(self.username, self.password, self.host, self.port, self.ssl))
def _get_ingredient_primary_category(ingredient):
category_mappings = IngredientCategoryMappingModel.query.filter(
IngredientCategoryMappingModel.ingredient_id == ingredient.id)
# print([mapping.category_id for mapping in category_mappings])
# exit()
for category_id in [
result.category_id for result in category_mappings
]:
category = IngredientCategoryModel.query.get(category_id)
if category.position and category.position >= 5:
return category.display_name
LogService.error("Could not find category for %s" %
ingredient.canonical_name)
def get(self, path):
"""
Fetch the value from the configuration store for the given key.
:param path: Normalized path in the hierarchy to the key.
:return: str or Exception
"""
self._connect()
try:
data, stat = self.zk.get(path)
return data.decode("utf-8")
except NoNodeError:
raise KeyError("%s does not exist." % path)
except Exception as e:
LogService.error(e.__class__)
LogService.error(e)
def get_recipes():
# character_list = list(range(0, 10)) + list(string.ascii_uppercase)
character_list = string.ascii_uppercase[2:3]
raw_recipes = []
for char in character_list:
# print(UpneatConnector._get_recipes_alpha(char))
slugs = UpneatConnector._get_recipes_alpha(char)
for slug in slugs:
try:
raw_recipes.append(UpneatConnector.scrape_recipe(slug))
except:
LogService.error("ERROR WITH %s " % slug)
return raw_recipes
def _handle_error(result):
"""
Parse an HTTP request for success/failure. Return a count of the success.
:param result:
:return: Integer count of success (THIS IS NOT A RETURN CODE!)
"""
try:
result.raise_for_status()
LogService.debug('Success!')
return 1
except requests.exceptions.RequestException as e:
LogService.error("Error handling URL: %i" % result.status_code)
LogService.error(result.request.body)
LogService.error(result.json().get('message'))
LogService.error(result.json().get('details'))
return 0
def rebuild(self, index_class):
"""
Re-create an index. This deletes the entire index (not just the contents,
but the Whole Damn Thing(tm). and re-creates it.
:param index_class: elasticsearch_dsl.Document child representing this index.
:return: None
"""
try:
index_class._index.delete()
except NotFoundError:
LogService.warning("Index %s did not exist." %
index_class.Index.name)
# Proceed with rebuild.
index_class.init()
LogService.info("Successfully rebuilt index %s" %
index_class.Index.name)
def import_(self, filepath):
data = IngredientImporter._fetch_data_from_path(filepath)
# Delete old data
self.delete()
LogService.info("Starting import")
for ingredient in data:
i = Ingredient(**ingredient)
db_obj = IngredientModel(**ObjectSerializer.serialize(i, 'dict'))
# Test for existing
with self.pgconn.get_session() as session:
# existing = IngredientModel.query.get(i.slug)
existing = session.query(IngredientModel).get(i.slug)
if existing:
if existing.kind == IngredientKinds(
'category'
).value or existing.kind == IngredientKinds(
'family').value:
if i.kind is IngredientKinds('ingredient'):
LogService.error(
"Skipping %s (t:%s) since a broader entry exists (%s)"
% (i.slug, i.kind.value, existing.kind))
else:
LogService.error(
"%s (p:%s) already exists as a %s (p:%s)" %
(i.slug, i.parent, existing.kind,
existing.parent))
else:
LogService.error(
"%s (p:%s) already exists as a %s (p:%s)" %
(i.slug, i.parent, existing.kind, existing.parent))
else:
session.add(db_obj)
Indexers.get_indexer(i).index(i)
LogService.info("Validating")
with self.pgconn.get_session() as session:
objects = session.query(IngredientModel).all()
for db_obj in objects:
# Validate
ObjectValidator.validate(db_obj, session=session, fatal=False)
# Invalidate the cache
IngredientTreeCache.invalidate()
def _kibana_settings():
"""
I am pedantic and want dark mode enabled on the Kibana instance.
This code serves no useful purpose within the app.
:return:
"""
headers = {'kbn-version': '7.5.0', 'Content-Type': 'application/json'}
data = '{"changes":{"theme:darkMode":true}}'
kibana_host = os.getenv('AMARI_KIBANA_HOST', default='localhost')
resp = requests.post("http://%s:5601/api/kibana/settings" %
kibana_host,
headers=headers,
data=data)
if resp.status_code == 200:
LogService.info("Kibana set to dark mode.")
else:
LogService.error("Error setting dark mode: %s" % resp.text)
def delete(cls, obj, fatal=False):
"""
Delete a specific object from its index.
Fatal changed to implicitly be false since if something isn't
in the index (kinda problematic?) whatevs thats the desired state.
:param obj: barbados.objects.base.BarbadosObject child instance.
:param fatal: Should not being in the index be a critical problem.
:return: None
"""
index_obj = cls.factory.obj_to_index(obj)
try:
cls.for_index.delete(index_obj)
except NotFoundError as e:
if fatal:
raise SearchException(e)
LogService.warn(
"Object %s was not found in index on DELETE. This probably isn't a problem?"
% obj)
def get_session(self):
"""
Provide a valid SQLAlchemy Session for use in a context.
Example:
with get_session() as session:
result = session.query(CocktailModel).get('mai-tai')
:return: None
"""
session, commit = self._build_session()
try:
yield session
if commit:
session.commit()
except Exception:
session.rollback()
raise
finally:
LogService.debug("Database with() context complete.")
def import_(self, filepath):
data = MenuImporter._fetch_data_from_path(filepath)
# Delete old data
self.delete()
LogService.info("Starting import")
for menu in data:
m = DrinkListFactory.raw_to_obj(menu)
db_obj = MenuModel(**ObjectSerializer.serialize(m, 'dict'))
# Test for existing
with self.pgconn.get_session() as session:
session.add(db_obj)
Indexers.get_indexer(m).index(m)
# Validate
self.validate()
# Clear Cache and Index
MenuScanCache.invalidate()
def execute(self):
"""
Generate and perform this query
:param session: SQLAlchemy session
:return: List of results.
"""
self._generate_criteria(model=self.model)
LogService.info("QueryBuilder conditions are: %s" % self.criteria)
# I don't get why this is so complicated to build. Feels excessive and
# maybe not scalable but whatever.
# @TODO pagination?
module_name = self.model.__module__
with DatabaseService.get_session() as session:
search_obj = Search(session,
module_name, (self.model, ),
filter_by=self.criteria,
all=True)
return search_obj.results.get('data')
def implies_root(self, node_id):
"""
Return the node id (tag) of the root of the implied tree for the given
node id. This generally means the first family in the parents.
:param node_id: slug of the ingredient to look up.
:return: slug of the root of the tree to use for implied ingredients.
"""
# Families are the root of their implied trees.
try:
self_kind = self.node(node_id).data.kind
except AttributeError:
LogService.warn("Unable to find kind for %s" % node_id)
return
if self_kind in [FamilyKind, CategoryKind]:
return node_id
# Look through all parents until we find one. We can be reasonably sure
# that all Ingredient/Product/Custom kinds have a family parent.
for parent in self.parents(node_id):
if self.node(parent).data.kind == FamilyKind:
return parent