def update(self, session_id, session_update_doc):
# remove _version from the document
session_update_doc.pop('_version', 0)
update_doc = {"doc": session_update_doc}
try:
res = self.es.update(index=self.index,
doc_type=self.doc_type,
id=session_id,
body=update_doc)
version = res['_version']
self.es.indices.refresh(index=self.index)
except elasticsearch.TransportError as e:
if e.status_code == 409:
raise freezer_api_exc.DocumentExists(message=e.error)
raise freezer_api_exc.DocumentNotFound(
message=_i18n._('Unable to update session '
'%(id)s %(e)s') % {
'id': session_id,
'e': e
})
except Exception:
raise freezer_api_exc.StorageEngineError(
message=_i18n._('Unable to update session with id %s') %
session_id)
return version
def add_client(self, user_id, doc):
client_id = doc.get("client_id", None)
if client_id is None:
raise freezer_api_exc.BadDataFormat(message=_i18n._("Missing client ID"))
existing = self.client_manager.search(user_id, client_id)
if existing:
raise freezer_api_exc.DocumentExists(message=_i18n._("Client already registered with ID %s") % client_id)
client_doc = {"client": doc, "user_id": user_id, "uuid": uuid.uuid4().hex}
self.client_manager.insert(client_doc)
logging.info(_i18n._LI("Client registered, client_id: %s") % client_id)
return client_id
def search(self, user_id, doc_id=None, search=None, offset=0, limit=10):
search = search or {}
query_dsl = self.get_search_query(user_id, doc_id, search)
try:
res = self.es.search(index=self.index, doc_type=self.doc_type, size=limit, from_=offset, body=query_dsl)
except elasticsearch.ConnectionError:
raise freezer_api_exc.StorageEngineError(message=_i18n._("unable to connect to db server"))
except Exception as e:
raise freezer_api_exc.StorageEngineError(message=_i18n._("search operation failed: %s") % e)
hit_list = res["hits"]["hits"]
return [x["_source"] for x in hit_list]
def get(self, user_id, doc_id):
try:
res = self.es.get(index=self.index, doc_type=self.doc_type, id=doc_id)
doc = res["_source"]
except elasticsearch.TransportError:
raise freezer_api_exc.DocumentNotFound(message=_i18n._("No document found with ID %s") % doc_id)
except Exception as e:
raise freezer_api_exc.StorageEngineError(message=_i18n._("Get operation failed: %s") % e)
if doc["user_id"] != user_id:
raise freezer_api_exc.AccessForbidden(_i18n._("Document access forbidden"))
if "_version" in res:
doc["_version"] = res["_version"]
return doc
def add_backup(self, user_id, user_name, doc):
# raises if data is malformed (HTTP_400) or already present (HTTP_409)
backup_metadata_doc = BackupMetadataDoc(user_id, user_name, doc)
if not backup_metadata_doc.is_valid():
raise freezer_api_exc.BadDataFormat(message=_i18n._('Bad Data Format'))
backup_id = backup_metadata_doc.backup_id
existing = self.backup_manager.search(user_id, backup_id)
if existing:
raise freezer_api_exc.DocumentExists(
message=_i18n._('Backup data already existing with ID %s') %
backup_id)
self.backup_manager.insert(backup_metadata_doc.serialize())
return backup_id
def add_client(self, user_id, doc):
client_id = doc.get('client_id', None)
if client_id is None:
raise freezer_api_exc.BadDataFormat(message=_('Missing client ID'))
existing = self.client_manager.search(user_id, client_id)
if existing: # len(existing) > 0
raise freezer_api_exc.DocumentExists(
message=_('Client already registered with ID %s') % client_id)
client_doc = {'client': doc,
'user_id': user_id,
'uuid': uuid.uuid4().hex}
self.client_manager.insert(client_doc)
logging.info(_LI('Client registered, client_id: %s') % client_id)
return client_id
def search(self, user_id, doc_id=None, search=None, offset=0, limit=10):
search = search or {}
query_dsl = self.get_search_query(user_id, doc_id, search)
try:
res = self.es.search(index=self.index, doc_type=self.doc_type,
size=limit, from_=offset, body=query_dsl)
except elasticsearch.ConnectionError:
raise freezer_api_exc.StorageEngineError(
message=_i18n._('unable to connect to db server'))
except Exception as e:
raise freezer_api_exc.StorageEngineError(
message=_i18n._('search operation failed: %s') % e)
hit_list = res['hits']['hits']
return [x['_source'] for x in hit_list]
def insert(self, doc, doc_id=None):
try:
# remove _version from the document
doc.pop("_version", None)
res = self.es.index(index=self.index, doc_type=self.doc_type, body=doc, id=doc_id)
created = res["created"]
version = res["_version"]
self.es.indices.refresh(index=self.index)
except elasticsearch.TransportError as e:
if e.status_code == 409:
raise freezer_api_exc.DocumentExists(message=e.error)
raise freezer_api_exc.StorageEngineError(message=_i18n._("index operation failed %s") % e)
except Exception as e:
raise freezer_api_exc.StorageEngineError(message=_i18n._("index operation failed %s") % e)
return (created, version)
def insert(self, doc, doc_id=None):
version = doc.pop('_version', 0)
try:
res = self.es.index(index=self.index, doc_type=self.doc_type,
body=doc, id=doc_id, version=version)
created = res['created']
version = res['_version']
except elasticsearch.TransportError as e:
if e.status_code == 409:
raise freezer_api_exc.DocumentExists(message=e.error)
raise freezer_api_exc.StorageEngineError(
message=_i18n._('index operation failed %s') % e)
except Exception as e:
raise freezer_api_exc.StorageEngineError(
message=_i18n._('index operation failed %s') % e)
return (created, version)
def delete(self, user_id, doc_id):
query_dsl = self.get_search_query(user_id, doc_id)
try:
results = self.es.search(index=self.index, doc_type=self.doc_type, body=query_dsl)
results = results["hits"]["hits"]
except Exception as e:
raise freezer_api_exc.StorageEngineError(message=_i18n._("Scan operation failed: %s") % e)
id = None
for res in results:
id = res.get("_id")
try:
self.es.delete(index=self.index, doc_type=self.doc_type, id=id)
self.es.indices.refresh(index=self.index)
except Exception as e:
raise freezer_api_exc.StorageEngineError(message=_i18n._("Delete operation failed: %s") % e)
return id
def model_query(session, model, args=None, read_deleted='no', project_id=None):
"""Query helper that accounts for context's `read_deleted` field.
:param session: The session to use, sqlalchemy.orm.session.Session
:param model: Model to query. Must be a subclass of ModelBase.
:param args: Arguments to query. If None - model is used.
:param read_deleted: If not None, overrides context's read_deleted field.
Permitted values are 'no', which does not return
deleted values; 'only', which only returns deleted
values; and 'yes', which does not filter deleted
values.
:param project_id: tenant id
"""
query_kwargs = {}
if 'no' == read_deleted:
query_kwargs['deleted'] = False
elif 'only' == read_deleted:
query_kwargs['deleted'] = True
elif 'yes' == read_deleted:
pass
else:
raise ValueError(
_("Unrecognized read_deleted value '%s'") % read_deleted)
query = sqlalchemyutils.model_query(model, session, args, **query_kwargs)
if project_id:
query = query.filter_by(project_id=project_id)
return query
def update(self, job_id, job_update_doc):
version = job_update_doc.pop('_version', 0)
update_doc = {"doc": job_update_doc}
try:
res = self.es.update(index=self.index, doc_type=self.doc_type,
id=job_id, body=update_doc, version=version)
version = res['_version']
except elasticsearch.TransportError as e:
if e.status_code == 409:
raise freezer_api_exc.DocumentExists(message=e.error)
raise freezer_api_exc.DocumentNotFound(
message=_i18n._('Unable to find job to update '
'with id %(id)s. %(e)s') % {'id': job_id, 'e': e})
except Exception:
raise freezer_api_exc.StorageEngineError(
message=_i18n._('Unable to update job with id %s') % job_id)
return version
def delete(self, user_id, doc_id):
query_dsl = self.get_search_query(user_id, doc_id)
try:
results = es_helpers.scan(self.es, index=self.index,
doc_type=self.doc_type, query=query_dsl)
except Exception as e:
raise freezer_api_exc.StorageEngineError(
message=_i18n._('Scan operation failed: %s') % e)
id = None
for res in results:
id = res.get('_id')
try:
self.es.delete(index=self.index, doc_type=self.doc_type, id=id)
except Exception as e:
raise freezer_api_exc.StorageEngineError(
message=_i18n._('Delete operation failed: %s') % e)
return id
def main():
# quick simple server for testing purposes or simple scenarios
ip, port = '127.0.0.1', 9090
if len(sys.argv) > 1:
ip = sys.argv[1]
if ':' in ip:
ip, port = ip.split(':')
httpd = simple_server.make_server(ip, int(port), application)
message = _i18n._('Server listening on %(ip)s:%(port)s'
% {'ip': ip, 'port': port})
print(message)
logging.info(message)
try:
httpd.serve_forever()
except KeyboardInterrupt:
print(_i18n._("\nThanks, Bye"))
sys.exit(0)
def add_backup(self, user_id, user_name, doc):
# raises if data is malformed (HTTP_400) or already present (HTTP_409)
backup_metadata_doc = utils.BackupMetadataDoc(user_id, user_name, doc)
if not backup_metadata_doc.is_valid():
raise freezer_api_exc.BadDataFormat(message=_i18n._("Bad Data Format"))
backup_id = backup_metadata_doc.backup_id
self.backup_manager.insert(backup_metadata_doc.serialize(), backup_id)
return backup_id
def update(self, action_id, action_update_doc):
# remove _version from the document
action_update_doc.pop("_version", 0)
update_doc = {"doc": action_update_doc}
try:
res = self.es.update(index=self.index, doc_type=self.doc_type, id=action_id, body=update_doc)
version = res["_version"]
self.es.indices.refresh(index=self.index)
except elasticsearch.TransportError as e:
if e.status_code == 409:
raise freezer_api_exc.DocumentExists(message=e.error)
raise freezer_api_exc.DocumentNotFound(
message=_i18n._("Unable to find action to update " "with id %s") % action_id
)
except Exception:
raise freezer_api_exc.StorageEngineError(message=_i18n._("Unable to update action with id %s") % action_id)
return version
def get_search_query(user_id, doc_id, search=None):
search = search or {}
try:
base_filter = TypeManager.get_base_search_filter(user_id, search)
query_filter = {"filter": {"bool": {"must": base_filter}}}
return {"query": {"filtered": query_filter}}
except Exception:
raise freezer_api_exc.StorageEngineError(message=_i18n._("search operation failed: query not valid"))
def get_db():
db_engine = CONF.storage.db
if db_engine == 'elasticsearch':
endpoint = CONF.storage.endpoint
logging.info(_LI('Storage backend: Elasticsearch at %s') % endpoint)
db = elastic.ElasticSearchEngine(endpoint)
else:
raise Exception(_('Database Engine %s not supported') % db_engine)
return db
def get_db():
opts = get_options()
db_engine = opts.pop('db')
if db_engine == 'elasticsearch':
logging.debug(_i18n._LI('Elastichsearch config options: %s') % str(opts))
db = elastic.ElasticSearchEngine(**opts)
else:
raise Exception(_i18n._('Database Engine %s not supported') % db_engine)
return db
def insert(self, doc, doc_id=None):
try:
# remove _version from the document
doc.pop('_version', None)
res = self.es.index(index=self.index, doc_type=self.doc_type,
body=doc, id=doc_id)
created = res['created']
version = res['_version']
self.es.indices.refresh(index=self.index)
except elasticsearch.TransportError as e:
if e.status_code == 409:
raise freezer_api_exc.DocumentExists(message=e.error)
raise freezer_api_exc.StorageEngineError(
message=_i18n._('index operation failed %s') % e)
except Exception as e:
raise freezer_api_exc.StorageEngineError(
message=_i18n._('index operation failed %s') % e)
return (created, version)
def update(self, action_id, action_update_doc):
version = action_update_doc.pop('_version', 0)
update_doc = {"doc": action_update_doc}
try:
res = self.es.update(index=self.index, doc_type=self.doc_type,
id=action_id, body=update_doc,
version=version)
version = res['_version']
except elasticsearch.TransportError as e:
if e.status_code == 409:
raise freezer_api_exc.DocumentExists(message=e.error)
raise freezer_api_exc.DocumentNotFound(
message=_('Unable to find action to update '
'with id %s') % action_id)
except Exception:
raise freezer_api_exc.StorageEngineError(
message=_('Unable to update action with id %s') %action_id)
return version
def get_db():
opts = get_options()
db_engine = opts.pop("db")
if db_engine == "elasticsearch":
logging.debug(_i18n._LI("ElasticSearch config options: %s") % str(opts))
db = elastic.ElasticSearchEngine(**opts)
else:
raise Exception(_i18n._("Database Engine %s not supported") % db_engine)
return db
def add_backup(self, user_id, user_name, doc):
# raises if data is malformed (HTTP_400) or already present (HTTP_409)
backup_metadata_doc = utils.BackupMetadataDoc(user_id, user_name, doc)
if not backup_metadata_doc.is_valid():
raise freezer_api_exc.BadDataFormat(
message=_i18n._('Bad Data Format'))
backup_id = backup_metadata_doc.backup_id
self.backup_manager.insert(backup_metadata_doc.serialize(), backup_id)
return backup_id
def get(self, user_id, doc_id):
try:
res = self.es.get(index=self.index,
doc_type=self.doc_type,
id=doc_id)
doc = res['_source']
except elasticsearch.TransportError:
raise freezer_api_exc.DocumentNotFound(
message=_i18n._('No document found with ID %s') % doc_id)
except Exception as e:
raise freezer_api_exc.StorageEngineError(
message=_i18n._('Get operation failed: %s') % e)
if doc['user_id'] != user_id:
raise freezer_api_exc.AccessForbidden(
_i18n._("Document access forbidden"))
if '_version' in res:
doc['_version'] = res['_version']
return doc
def get_search_query(user_id, doc_id, search=None):
search = search or {}
try:
base_filter = TypeManager.get_base_search_filter(user_id, search)
query_filter = {"filter": {"bool": {"must": base_filter}}}
return {'query': {'filtered': query_filter}}
except Exception:
raise freezer_api_exc.StorageEngineError(
message=_i18n._('search operation failed: query not valid'))
def delete(self, user_id, doc_id):
query_dsl = self.get_search_query(user_id, doc_id)
try:
self.es.delete_by_query(index=self.index,
doc_type=self.doc_type,
body=query_dsl)
except Exception as e:
raise freezer_api_exc.StorageEngineError(
message=_('Delete operation failed: %s') % e)
return doc_id
def update(self, session_id, session_update_doc):
# remove _version from the document
session_update_doc.pop('_version', 0)
update_doc = {"doc": session_update_doc}
try:
res = self.es.update(index=self.index, doc_type=self.doc_type,
id=session_id, body=update_doc)
version = res['_version']
self.es.indices.refresh(index=self.index)
except elasticsearch.TransportError as e:
if e.status_code == 409:
raise freezer_api_exc.DocumentExists(message=e.error)
raise freezer_api_exc.DocumentNotFound(
message=_i18n._(('Unable to update session '
'%s. %s') % (session_id, e)))
except Exception:
raise freezer_api_exc.StorageEngineError(
message=_i18n._('Unable to update session with id %s') % session_id)
return version
def delete(self, user_id, doc_id):
query_dsl = self.get_search_query(user_id, doc_id)
try:
results = self.es.search(index=self.index,
doc_type=self.doc_type,
body=query_dsl)
results = results['hits']['hits']
except Exception as e:
raise freezer_api_exc.StorageEngineError(
message=_i18n._('Scan operation failed: %s') % e)
id = None
for res in results:
id = res.get('_id')
try:
self.es.delete(index=self.index, doc_type=self.doc_type, id=id)
self.es.indices.refresh(index=self.index)
except Exception as e:
raise freezer_api_exc.StorageEngineError(
message=_i18n._('Delete operation failed: %s') % e)
return id
def add_client(self, user_id, doc):
client_doc = utils.ClientDoc.create(doc, user_id)
client_id = client_doc['client']['client_id']
existing = self.client_manager.search(user_id, client_id)
if existing:
raise freezer_api_exc.DocumentExists(
message=_i18n._('Client already registered with ID %s') %
client_id)
self.client_manager.insert(client_doc)
logging.info('Client registered, client_id: %s' % client_id)
return client_id
def get_db():
opts = get_options()
db_engine = opts.pop('db')
if db_engine == 'elasticsearch':
logging.debug(
_i18n._LI('ElasticSearch config options: %s') % str(opts))
db = elastic.ElasticSearchEngine(**opts)
else:
raise Exception(
_i18n._('Database Engine %s not supported') % db_engine)
return db
def main():
# setup opts
config.parse_args()
config.setup_logging()
paste_conf = config.find_paste_config()
# quick simple server for testing purposes or simple scenarios
ip = CONF.get('bind_host', '0.0.0.0')
port = CONF.get('bind_port', 9090)
try:
httpserver.serve(
application=deploy.loadapp('config:%s' % paste_conf, name='main'),
host=ip,
port=port)
message = _i18n._('Server listening on %(ip)s:%(port)s' %
{'ip': ip, 'port': port})
_LOG.info(message)
print(message)
except KeyboardInterrupt:
print(_i18n._("Thank You ! \nBye."))
sys.exit(0)
def main():
# setup opts
config.parse_args(args=sys.argv[1:])
config.setup_logging()
paste_conf = config.find_paste_config()
# quick simple server for testing purposes or simple scenarios
ip = CONF.get('bind_host', '0.0.0.0')
port = CONF.get('bind_port', 9090)
try:
httpserver.serve(application=deploy.loadapp('config:%s' % paste_conf,
name='main'),
host=ip,
port=port)
message = (_i18n._('Server listening on %(ip)s:%(port)s') % {
'ip': ip,
'port': port
})
_LOG.info(message)
print(message)
except KeyboardInterrupt:
print(_i18n._("Thank You ! \nBye."))
sys.exit(0)
class Checks(upgradecheck.UpgradeCommands):
"""Various upgrade checks should be added as separate methods in this class
and added to _upgrade_checks tuple.
"""
def _check_placeholder(self):
# This is just a placeholder for upgrade checks, it should be
# removed when the actual checks are added
return upgradecheck.Result(upgradecheck.Code.SUCCESS)
# The format of the check functions is to return an
# oslo_upgradecheck.upgradecheck.Result
# object with the appropriate
# oslo_upgradecheck.upgradecheck.Code and details set.
# If the check hits warnings or failures then those should be stored
# in the returned Result's "details" attribute. The
# summary will be rolled up at the end of the check() method.
_upgrade_checks = (
# In the future there should be some real checks added here
(_('Placeholder'), _check_placeholder), )
def handle(ex, req, resp, params):
raise falcon.HTTPMethodNotAllowed(
title=_("Bad Method"),
description=ex.message)
app = middleware.HealthApp(app=app, path='/v1/health')
return app
config_file = '/etc/freezer-api.conf'
config_files_list = [config_file] if os.path.isfile(config_file) else []
config.parse_args(args=[], default_config_files=config_files_list)
log.setup()
logging.info(_i18n._LI("Freezer API starting"))
logging.info(_i18n._LI("Freezer config file(s) used: %s")
% ', '.join(cfg.CONF.config_file))
try:
db = driver.get_db()
application = get_application(db)
except Exception as err:
message = _i18n._('Unable to start server: %s ') % err
print(message)
logging.fatal(message)
sys.exit(1)
def main():
# quick simple server for testing purposes or simple scenarios
ip, port = '127.0.0.1', 9090
if len(sys.argv) > 1:
ip = sys.argv[1]
if ':' in ip:
ip, port = ip.split(':')
httpd = simple_server.make_server(ip, int(port), application)
message = _i18n._('Server listening on %(ip)s:%(port)s'
% {'ip': ip, 'port': port})
def handle(ex, req, resp, params):
raise falcon.HTTPNotFound(
title=_("Not Found"),
description=ex.message)
def handle(ex, req, resp, params):
raise falcon.HTTPForbidden(
title=_("Access Forbidden"),
description=ex.message)
def handle(ex, req, resp, params):
raise falcon.HTTPInternalServerError(
title=_("Internal Storage Error"),
description=ex.message)
def handle(ex, req, resp, params):
raise falcon.HTTPConflict(
title=_("Document already existing"),
description=ex.message)
def handle(ex, req, resp, params):
raise falcon.HTTPBadRequest(
title=_("Bad request format"),
description=ex.message)
def handle(ex, req, resp, params):
raise falcon.HTTPError(_('500 unknown server error'),
title=_("Internal Server Error"),
description=FreezerAPIException.message)
def __init__(self, message=''):
if message:
self.message = _(str(message))
logging.error(message)
Exception.__init__(self, message)