def search(self,
project_id,
user_id=None,
doc_id=None,
search=None,
offset=0,
limit=10):
search = search or {}
query_dsl = self.get_search_query(project_id=project_id,
user_id=user_id,
doc_id=doc_id,
search=search)
try:
res = self.es.search(index=self.index,
doc_type=self.doc_type,
size=limit,
from_=offset,
body=query_dsl)
except elasticsearch.ConnectionError:
raise freezer_api_exc.StorageEngineError(
message='unable to connect to db server')
except Exception as e:
raise freezer_api_exc.StorageEngineError(
message='search operation failed: {0}'.format(e))
hit_list = res['hits']['hits']
return [x['_source'] for x in hit_list]
def update(self, session_id, session_update_doc):
# remove _version from the document
session_update_doc.pop('_version', 0)
update_doc = {"doc": session_update_doc}
try:
res = self.es.update(index=self.index,
doc_type=self.doc_type,
id=session_id,
body=update_doc)
version = res['_version']
self.es.indices.refresh(index=self.index)
except elasticsearch.TransportError as e:
if e.status_code == 409:
raise freezer_api_exc.DocumentExists(message=e.error)
raise freezer_api_exc.DocumentNotFound(
message=_i18n._('Unable to update session '
'%(id)s %(e)s') % {
'id': session_id,
'e': e
})
except Exception:
raise freezer_api_exc.StorageEngineError(
message=_i18n._('Unable to update session with id %s') %
session_id)
return version
def test_model_query(self):
job_doc1 = copy.deepcopy(self.fake_job_0)
job_doc2 = copy.deepcopy(self.fake_job_0)
self.dbapi.add_job(user_id=self.fake_user_id,
doc=job_doc1)
job_id2 = self.dbapi.add_job(user_id=self.fake_user_id,
doc=job_doc2)
result = self.dbapi.delete_job(user_id=self.fake_user_id,
job_id=job_id2)
session = api.get_db_session()
with session.begin():
try:
query = api.model_query(session, models.Job,
read_deleted='no')
query = query.filter_by(user_id=self.fake_user_id)
result = query.all()
self.assertEqual(len(result), 1)
query = api.model_query(session, models.Job,
read_deleted='only')
query = query.filter_by(user_id=self.fake_user_id)
result = query.all()
self.assertEqual(len(result), 1)
query = api.model_query(session, models.Job,
read_deleted='yes')
query = query.filter_by(user_id=self.fake_user_id)
result = query.all()
self.assertEqual(len(result), 2)
except Exception as e:
raise freezer_api_exc.StorageEngineError(
message='sqlalchemy operation failed {0}'.format(e))
session.close()
def add_tuple(tuple):
session = get_db_session()
with session.begin():
try:
tuple.save(session=session)
except Exception as e:
raise freezer_api_exc.StorageEngineError(
message='Mysql operation failed {0}'.format(e))
def get_search_query(user_id, doc_id, search=None):
search = search or {}
try:
base_filter = TypeManager.get_base_search_filter(user_id, search)
query_filter = {"filter": {"bool": {"must": base_filter}}}
return {'query': {'filtered': query_filter}}
except Exception:
raise freezer_api_exc.StorageEngineError(
message=_i18n._('search operation failed: query not valid'))
def insert(self, doc, doc_id=None):
try:
# remove _version from the document
doc.pop('_version', None)
res = self.es.index(index=self.index, doc_type=self.doc_type,
body=doc, id=doc_id)
created = res['created']
version = res['_version']
self.es.indices.refresh(index=self.index)
except elasticsearch.TransportError as e:
if e.status_code == 409:
raise freezer_api_exc.DocumentExists(message=e.error)
raise freezer_api_exc.StorageEngineError(
message=_i18n._('index operation failed %s') % e)
except Exception as e:
raise freezer_api_exc.StorageEngineError(
message=_i18n._('index operation failed %s') % e)
return (created, version)
def _index(self, doc):
# raises only on severe engine errors
try:
res = self.es.index(index=self.index, doc_type='backups',
body=doc)
except Exception as e:
raise exceptions.StorageEngineError(
message='index operation failed',
resp_body={'engine exception': '{0}'.format(e)})
return res['created']
def _delete_backup(self, user_id, backup_id):
query = '+user_id:{0} +backup_id:{1}'.format(user_id, backup_id)
try:
self.es.delete_by_query(index=self.index,
doc_type='backups',
q=query)
except Exception as e:
raise exceptions.StorageEngineError(
message='search operation failed',
resp_body={'engine exception': '{0}'.format(e)})
def delete(self, user_id, doc_id):
query_dsl = self.get_search_query(user_id, doc_id)
try:
results = self.es.search(index=self.index,
doc_type=self.doc_type,
body=query_dsl)
results = results['hits']['hits']
except Exception as e:
raise freezer_api_exc.StorageEngineError(
message=_i18n._('Scan operation failed: %s') % e)
id = None
for res in results:
id = res.get('_id')
try:
self.es.delete(index=self.index, doc_type=self.doc_type, id=id)
self.es.indices.refresh(index=self.index)
except Exception as e:
raise freezer_api_exc.StorageEngineError(
message=_i18n._('Delete operation failed: %s') % e)
return id
def get_tuple(tablename, user_id, tuple_id, project_id=None):
session = get_db_session()
with session.begin():
try:
query = model_query(session, tablename, project_id=project_id)
query = query.filter_by(user_id=user_id).filter_by(id=tuple_id)
result = query.all()
except Exception as e:
raise freezer_api_exc.StorageEngineError(
message='Mysql operation failed {0}'.format(e))
session.close()
return result
def get_client_byid(user_id, client_id, project_id=None):
session = get_db_session()
with session.begin():
try:
query = model_query(session, models.Client, project_id=project_id)
if client_id:
query = query.filter_by(user_id=user_id).filter_by(
client_id=client_id)
result = query.all()
except Exception as e:
raise freezer_api_exc.StorageEngineError(
message='Mysql operation failed {0}'.format(e))
session.close()
return result
def _get_backup(self, user_id, backup_id=None):
# raises only on severe engine errors
if backup_id:
query = '+user_id:{0} +backup_id:{1}'.format(user_id, backup_id)
else:
query = '+user_id:{0}'.format(user_id)
try:
res = self.es.search(index=self.index, doc_type='backups',
q=query)
except Exception as e:
raise exceptions.StorageEngineError(
message='search operation failed',
resp_body={'engine exception': '{0}'.format(e)})
hit_list = res['hits']['hits']
return [x['_source'] for x in hit_list]
def replace_tuple(tablename, user_id, tuple_id, tuple_values, project_id=None):
session = get_db_session()
with session.begin():
try:
query = model_query(session, tablename, project_id=project_id)
query = query.filter_by(user_id=user_id).filter_by(id=tuple_id)
result = query.update(tuple_values)
if not result:
tuplet = tablename()
tuplet.update(tuple_values)
tuplet.save(session=session)
except Exception as e:
raise freezer_api_exc.StorageEngineError(
message='Mysql operation failed {0}'.format(e))
return tuple_id
def get(self, user_id, doc_id):
try:
res = self.es.get(index=self.index,
doc_type=self.doc_type,
id=doc_id)
doc = res['_source']
except elasticsearch.TransportError:
raise freezer_api_exc.DocumentNotFound(
message=_i18n._('No document found with ID %s') % doc_id)
except Exception as e:
raise freezer_api_exc.StorageEngineError(
message=_i18n._('Get operation failed: %s') % e)
if doc['user_id'] != user_id:
raise freezer_api_exc.AccessForbidden(
_i18n._("Document access forbidden"))
if '_version' in res:
doc['_version'] = res['_version']
return doc
def update(self, job_id, job_update_doc):
# remove _version from the document
job_update_doc.pop('_version', 0)
update_doc = {"doc": job_update_doc}
try:
res = self.es.update(index=self.index, doc_type=self.doc_type,
id=job_id, body=update_doc)
version = res['_version']
self.es.indices.refresh(index=self.index)
except elasticsearch.TransportError as e:
if e.status_code == 409:
raise freezer_api_exc.DocumentExists(message=e.error)
raise freezer_api_exc.DocumentNotFound(
message='Unable to find job to update with id'
' {0} {1}'.format(job_id, e))
except Exception:
raise freezer_api_exc.StorageEngineError(
message='Unable to update job with id {0}'.format(job_id))
return version
def add_backup(self, user_id, user_name, data):
# raises if data is malformed (HTTP_400) or already present (HTTP_409)
backup_metadata_doc = BackupMetadataDoc(user_id, user_name, data)
if not backup_metadata_doc.is_valid():
raise exceptions.BadDataFormat(message='Bad Data Format')
backup_id = backup_metadata_doc.backup_id
existing_data = self._get_backup(user_id, backup_id)
if existing_data:
raise exceptions.DocumentExists(
message='Backup data already existing ({0})'.format(backup_id),
resp_body={'backup_id': backup_id})
if not self._index(backup_metadata_doc.serialize()):
# should never happen
raise exceptions.StorageEngineError(
message='index operation failed',
resp_body={'backup_id': backup_id})
logging.info('Backup metadata indexed, backup_id: {0}'.
format(backup_id))
return backup_id
def delete_tuple(tablename, user_id, tuple_id, project_id=None):
session = get_db_session()
with session.begin():
try:
query = model_query(session, tablename, project_id=project_id)
query = query.filter_by(user_id=user_id).filter_by(id=tuple_id)
result = query.all()
if 1 == len(result):
result[0].delete(session=session)
LOG.info('Tuple delete, Tuple_id: '
'{0} deleted in Table {1}'.format(
tuple_id, tablename))
else:
LOG.info('Tuple delete, Tuple_id: '
'{0} not found in Table {1}'.format(
tuple_id, tablename))
except Exception as e:
raise freezer_api_exc.StorageEngineError(
message='Mysql operation failed {0}'.format(e))
return tuple_id
def update_tuple(tablename, user_id, tuple_id, tuple_values, project_id=None):
session = get_db_session()
with session.begin():
try:
query = model_query(session, tablename, project_id=project_id)
query = query.filter_by(user_id=user_id).filter_by(id=tuple_id)
result = query.update(tuple_values)
except Exception as e:
raise freezer_api_exc.StorageEngineError(
message='Mysql operation failed {0}'.format(e))
if not result:
raise freezer_api_exc.DocumentNotFound(
message='Tuple not registered with ID'
' {0} in Table{1} '.format(tuple_id, tablename))
else:
LOG.info('Tuple updated, tuple_id: {0}'.format(tuple_id))
return tuple_id
def get(self, project_id, doc_id, user_id=None):
try:
res = self.es.get(index=self.index,
doc_type=self.doc_type,
id=doc_id)
doc = res['_source']
except elasticsearch.TransportError:
raise freezer_api_exc.DocumentNotFound(
message='No document found with ID:{0}'.format(doc_id))
except Exception as e:
raise freezer_api_exc.StorageEngineError(
message='Get operation failed: {}'.format(e))
if doc['project_id'] != project_id:
raise freezer_api_exc.AccessForbidden("You are not allowed to"
" access")
if user_id:
if doc['user_id'] != user_id:
raise freezer_api_exc.AccessForbidden(
"Document access forbidden")
if '_version' in res:
doc['_version'] = res['_version']
return doc
def search_tuple(tablename,
user_id,
project_id=None,
offset=0,
limit=100,
search=None):
search = valid_and_get_search_option(search=search)
session = get_db_session()
with session.begin():
try:
# TODO(gecong) search will be implemented in the future
query = model_query(session, tablename, project_id=project_id)
query = query.filter_by(user_id=user_id)
# If search option isn't valid or set, we use limit and offset
# in sqlalchemy level
if len(search) == 0:
query = query.offset(offset)
query = query.limit(limit)
result = query.all()
except Exception as e:
raise freezer_api_exc.StorageEngineError(
message='Mysql operation failed {0}'.format(e))
session.close()
return result, search
def test_StorageEngineError(self):
e = exceptions.StorageEngineError(message='testing')
self.assertRaises(falcon.HTTPInternalServerError, e.handle, self.ex,
self.mock_req, self.mock_req, None)