def get_collection_replicas(scope, name, edge_name, coll_id=None, edge_id=None, session=None):
"""
Get a collection replicas or raise a NoObject exception.
:param scope: The scope of the collection data.
:param name: The name of the collection data.
:param coll_id: Collection id.
:param edge_name: The name of the replicating edge.
:param edge_id: The id of the replicating edge.
:param session: The database session in use.
:raises NoObject: If no edge is founded.
:returns: CollectionReplicas model.
"""
try:
if not coll_id:
coll_id = get_collection_id(scope=scope, name=name, session=session)
if not edge_id:
edge_id = get_edge_id(edge_name=edge_name, session=session)
collection_replicas = session.query(models.CollectionReplicas).filter_by(coll_id=coll_id, edge_id=edge_id).one()
collection_replicas['status'] = collection_replicas.status
return collection_replicas
except sqlalchemy.orm.exc.NoResultFound:
raise exceptions.NoObject('Collection replicas %s:%s(at %s) cannot be found' % (scope, name, edge_name))
def update_collection_replicas(scope, name, edge_name, coll_id=None, edge_id=None, parameters=None, session=None):
"""
update a collection replicas.
:param scope: The scope of the collection data.
:param name: The name of the collection data.
:param coll_id: Collection id.
:param edge_name: The name of the replicating edge.
:param edge_id: The id of the replicating edge.
:param parameters: A dictionary of parameters.
:param session: The database session in use.
:raises NoObject: If no edge is founded.
:raises DatabaseException: If there is a database error.
"""
try:
collection_replicas = get_collection_replicas(scope, name, edge_name, coll_id=coll_id, edge_id=edge_id, session=session)
except sqlalchemy.orm.exc.NoResultFound:
raise exceptions.NoObject('Collection %s:%s cannot be found' % (scope, name))
try:
if 'status' in parameters and \
(isinstance(parameters['status'], str) or isinstance(parameters['status'], unicode)):
parameters['status'] = CollectionReplicasStatus.from_sym(str(parameters['status']))
collection_replicas.update(parameters)
except DatabaseError as error:
raise exceptions.DatabaseException(error.args)
def update_edge(edge_name, parameters, session=None):
"""
update an edge.
:param edge_name: the name of the edge.
:param parameters: A dictionary of parameters.
:param session: The database session in use.
:raises NoObject: If no edge is founded.
:raises DatabaseException: If there is a database error.
:returns: edge id.
"""
try:
if 'edge_type' in parameters and \
(isinstance(parameters['edge_type'], str) or isinstance(parameters['edge_type'], unicode)):
parameters['edge_type'] = EdgeType.from_sym(str(parameters['edge_type']))
if 'status' in parameters and \
(isinstance(parameters['status'], str) or isinstance(parameters['status'], unicode)):
parameters['status'] = EdgeStatus.from_sym(str(parameters['status']))
edge = session.query(models.Edge).filter_by(edge_name=edge_name).one()
except sqlalchemy.orm.exc.NoResultFound:
raise exceptions.NoObject('Edge %s cannot be found' % edge_name)
try:
edge.update(parameters)
except DatabaseError as error:
raise exceptions.DatabaseException(error.args)
return edge.edge_id
def get_requests_by_edge(edge_name, edge_id=None, session=None):
"""
Get requests by edge.
:param edge_name: The name of the edge.
:param edge_id: The id of the edge
:param session: The database session in use.
:raises NoObject: If no request is founded.
:returns: list of Request model.
"""
try:
if not edge_id:
edge_id = get_edge_id(edge_name)
requests = session.query(
models.Request).filter_by(edge_id=edge_id).all()
for request in requests:
request['data_type'] = request.data_type
request['granularity_type'] = request.granularity_type
request['status'] = request.status
return requests
except sqlalchemy.orm.exc.NoResultFound:
raise exceptions.NoObject('No requests at %s' % (edge_name))
def get_edges(status=None, session=None):
"""
Get an Edge or raise a NoObject exception.
:param status: The status of the edge.
:param session: The database session in use.
:raises NoObject: If no edge is founded.
:returns: Edge models.
"""
try:
if status:
if (isinstance(status, str) or isinstance(status, unicode)):
status = EdgeStatus.from_sym(status)
edges = session.query(models.Edge).filter_by(status=status).all()
else:
edges = session.query(models.Edge).all()
for edge in edges:
edge['edge_type'] = edge.edge_type
return edges
except sqlalchemy.orm.exc.NoResultFound:
raise exceptions.NoObject('Cannot find edges with status: %s' % status)
def get_collection(scope, name, coll_id=None, session=None):
"""
Get a collection or raise a NoObject exception.
:param scope: The scope of the collection data.
:param name: The name of the collection data.
:param coll_id: Collection id.
:param session: The database session in use.
:raises NoObject: If no edge is founded.
:returns: Collection model.
"""
try:
if coll_id:
collection = session.query(models.Collection).filter_by(coll_id=coll_id).one()
else:
collection = session.query(models.Collection).filter_by(scope=scope, name=name).one()
collection['collection_type'] = collection.collection_type
collection['global_status'] = collection.global_status
return collection
except sqlalchemy.orm.exc.NoResultFound:
raise exceptions.NoObject('Collection %s:%s cannot be found' % (scope, name))
def get_request(scope=None, name=None, request_id=None, request_meta=None, session=None):
"""
Get a request or raise a NoObject exception.
:param scope: The scope of the request data.
:param name: The name of the request data.
:param request_id: The id of the request.
:param request_meta: The metadata of the request, as Json.
:param session: The database session in use.
:raises NoObject: If no request is founded.
:returns: Request model.
"""
try:
if request_id:
request = session.query(models.Request).filter_by(request_id=request_id).one()
else:
request = session.query(models.Request).filter_by(scope=scope, name=name).filter(request_meta.like(request_meta)).one()
request['data_type'] = request.data_type
request['granularity_type'] = request.granularity_type
request['status'] = request.status
return request
except sqlalchemy.orm.exc.NoResultFound as error:
raise exceptions.NoObject('request %s:%s(id:%s,meta:%s) cannot be found: %s' % (scope, name, request_id, request_meta, error))
def delete_request(request_id, session=None):
"""
Delete a request or raise a NoObject exception.
:param request_id: The request id.
:param session: The database session in use.
:raises NoObject: If no request is founded.
"""
try:
session.query(models.Request).filter_by(request_id=request_id).delete()
except sqlalchemy.orm.exc.NoResultFound:
raise exceptions.NoObject('Request %s cannot be found' % request_id)
def delete_edge(edge_name, session=None):
"""
Delete an Edge or raise a NoObject exception.
:param edge_name: The name of the edge.
:param session: The database session in use.
:raises NoObject: If no edge is founded.
"""
try:
session.query(models.Edge).filter_by(edge_name=edge_name).delete()
except sqlalchemy.orm.exc.NoResultFound:
raise exceptions.NoObject('Edge %s cannot be found' % edge_name)
def get_edge_id(edge_name, session=None):
"""
Get an Edge id or raise a NoObject exception.
:param edge_name: The name of the edge.
:param session: The database session in use.
:raises NoObject: If no edge is founded.
:returns: Edge id.
"""
try:
return session.query(models.Edge.edge_id).filter_by(edge_name=edge_name).one()[0]
except sqlalchemy.orm.exc.NoResultFound:
raise exceptions.NoObject('Edge %s cannot be found' % edge_name)
def get_content_best_match(scope, name, min_id=None, max_id=None, edge_name=None, edge_id=None, status=None, session=None):
"""
Get a collection content or raise a NoObject exception.
:param scope: The scope of the collection data.
:param name: The name of the collection data.
:param min_id: The minimum id of the partial file, related to the whole file.
:param max_id: The maximum id of the partial file, related to the whole file.
:param edge_name: The name of the edge.
:param edge_id: The id of the Edge
:param session: The database session in use.
:raises NoObject: If no edge is founded.
:returns: Content model.
"""
try:
if not edge_id and edge_name:
edge_id = get_edge_id(edge_name=edge_name, session=session)
if status and (isinstance(status, str) or isinstance(status, unicode)):
status = ContentStatus.from_sym(str(status))
query = session.query(models.CollectionContent).filter_by(scope=scope, name=name)
if status:
query = query.filter_by(status=status)
if edge_id:
query = query.filter_by(edge_id=edge_id)
if min_id is not None and max_id is not None:
contents = query.filter(and_(CollectionContent.min_id <= min_id, CollectionContent.max_id >= max_id)).all()
content = None
for row in contents:
if (not content) or (content['max_id'] - content['min_id'] > row['max_id'] - row['min_id']):
content = row
if content is None:
raise sqlalchemy.orm.exc.NoResultFound()
else:
content = query.filter_by(content_type=ContentType.FILE).one()
content['content_type'] = content.content_type
content['status'] = content.status
return content
except sqlalchemy.orm.exc.NoResultFound:
raise exceptions.NoObject('Content %s:%s[%s:%s](at edge %s) cannot be found' % (scope, name, min_id, max_id, edge_id))
def get_contents_by_edge(edge_name, edge_id=None, status=None, coll_id=None, content_type=None,
collection_scope=None, collection_name=None, limit=None, session=None):
"""
Get a collection content or raise a NoObject exception.
:param edge_name: The name of the edge.
:param edge_id: The id of the Edge
:param status: The status of the content.
:param coll_id: The collection id.
:param content_type: The tyep of the content.
:param limit: Number to return limited items.
:param session: The database session in use.
:raises NoObject: If no edge is founded.
:returns: Content model.
"""
try:
if not edge_id:
edge_id = get_edge_id(edge_name=edge_name, session=session)
if not coll_id and (collection_scope and collection_name):
coll_id = get_collection_id(collection_scope, collection_name)
query = session.query(models.CollectionContent).filter_by(edge_id=edge_id)
if status:
if isinstance(status, str) or isinstance(status, unicode):
status = ContentStatus.from_sym(str(status))
query = query.filter_by(status=status)
if coll_id:
query = query.filter_by(coll_id=coll_id)
if content_type:
query = query.filter_by(content_type=content_type)
if limit:
query = query.limit(limit)
contents = query.all()
for content in contents:
content['content_type'] = content.content_type
content['status'] = content.status
return contents
except sqlalchemy.orm.exc.NoResultFound:
raise exceptions.NoObject('No contents at edge %s with status %s' % (edge_name, status))
def delete_collection(scope, name, coll_id=None, session=None):
"""
Delete a collection or raise a NoObject exception.
:param scope: The scope of the collection data.
:param name: The name of the collection data.
:param coll_id: Collection id.
:param session: The database session in use.
:raises NoObject: If no edge is founded.
"""
try:
if coll_id:
session.query(models.Collection).filter_by(coll_id=coll_id).delete()
else:
session.query(models.Collection).filter_by(scope=scope, name=name).delete()
except sqlalchemy.orm.exc.NoResultFound:
raise exceptions.NoObject('Collection %s:%s cannot be found' % (scope, name))
def get_requests(status=None, edge_name=None, edge_id=None, session=None):
"""
Get requests.
:param status: The status of the request data.
:param edge_name: The name of the edge.
:param edge_id: The id of the edge
:param session: The database session in use.
:raises NoObject: If no request is founded.
:returns: Request models.
"""
try:
if edge_name and not edge_id:
edge_id = get_edge_id(edge_name)
if status:
if (isinstance(status, str) or isinstance(status, unicode)):
status = RequestStatus.from_sym(status)
if edge_id:
requests = session.query(models.Request).filter_by(
status=status, edge_id=edge_id).all()
else:
requests = session.query(
models.Request).filter_by(status=status).all()
else:
if edge_id:
requests = session.query(
models.Request).filter_by(edge_id=edge_id).all()
else:
requests = session.query(models.Request).all()
for request in requests:
request['data_type'] = request.data_type
request['granularity_type'] = request.granularity_type
request['status'] = request.status
return requests
except sqlalchemy.orm.exc.NoResultFound:
raise exceptions.NoObject('Cannot find request with status: %s' %
(status))
def get_collection_id(scope, name, session=None):
"""
Get a collection or raise a NoObject exception.
:param scope: The scope of the collection data.
:param name: The name of the collection data.
:param session: The database session in use.
:raises NoObject: If no edge is founded.
:returns: Collection model.
"""
try:
collection_id = session.query(models.Collection.coll_id).filter_by(scope=scope, name=name).one()[0]
return collection_id
except sqlalchemy.orm.exc.NoResultFound:
raise exceptions.NoObject('Collection %s:%s cannot be found' % (scope, name))
def get_contents_statistics(edge_name, edge_id=None, coll_id=None, status=None, content_type=None, session=None):
"""
Get content statistics.
:param edge_name: The name of the edge.
:param edge_id: The id of the Edge
:param coll_id: The collection id.
:param status: The status of the content.
:param content_type: The tyep of the content.
:param session: The database session in use.
:returns: dict.
"""
try:
if not edge_id and edge_name:
edge_id = get_edge_id(edge_name=edge_name, session=session)
query = session.query(models.CollectionContent.edge_id, models.CollectionContent.coll_id,
models.CollectionContent.content_type, models.CollectionContent.status,
func.count(1).label('counter'))
if edge_id:
query = query.filter_by(edge_id=edge_id)
if coll_id:
query = query.filter_by(coll_id=coll_id)
if status:
if isinstance(status, str) or isinstance(status, unicode):
status = ContentStatus.from_sym(str(status))
query = query.filter_by(status=status)
if content_type:
if isinstance(content_type, str) or isinstance(content_type, unicode):
content_type = ContentType.from_sym(str(content_type))
query = query.filter_by(content_type=content_type)
query = query.group_by(models.CollectionContent.edge_id, models.CollectionContent.coll_id,
models.CollectionContent.content_type, models.CollectionContent.status)
statistics = query.all()
return statistics
except sqlalchemy.orm.exc.NoResultFound:
raise exceptions.NoObject('Failed to get statistics for edge %s and collection %s' % (edge_id, coll_id))
def update_request(request_id, parameters, session=None):
"""
update an request.
:param request_id: the request id.
:param parameters: A dictionary of parameters.
:param session: The database session in use.
:raises NoObject: If no request is founded.
:raises DatabaseException: If there is a database error.
:returns: request id.
"""
try:
if 'data_type' in parameters and \
(isinstance(parameters['data_type'], str) or isinstance(parameters['data_type'], unicode)):
parameters['data_type'] = DataType.from_sym(
str(parameters['data_type']))
if 'granularity_type' in parameters and \
(isinstance(parameters['granularity_type'], str) or isinstance(parameters['granularity_type'], unicode)):
parameters['granularity_type'] = GranularityType.from_sym(
str(parameters['granularity_type']))
if 'status' in parameters and \
(isinstance(parameters['status'], str) or isinstance(parameters['status'], unicode)):
parameters['status'] = RequestStatus.from_sym(
str(parameters['status']))
request = session.query(
models.Request).filter_by(request_id=request_id).one()
except sqlalchemy.orm.exc.NoResultFound:
raise exceptions.NoObject('Request %s cannot be found' % request_id)
try:
request.update(parameters)
except DatabaseError as error:
raise exceptions.DatabaseException(error.args)
return request.request_id
def get_contents_by_edge(edge_name, edge_id=None, status=None, coll_id=None, content_type=ContentType.FILE, session=None):
"""
Get a collection content or raise a NoObject exception.
:param edge_name: The name of the edge.
:param edge_id: The id of the Edge
:param status: The status of the content.
:param coll_id: The collection id.
:param session: The database session in use.
:raises NoObject: If no edge is founded.
:returns: Content model.
"""
try:
if not edge_id:
edge_id = get_edge_id(edge_name=edge_name, session=session)
if status:
if isinstance(status, str) or isinstance(status, unicode):
status = ContentStatus.from_sym(str(status))
if coll_id:
contents = session.query(models.CollectionContent).filter_by(edge_id=edge_id, status=status, coll_id=coll_id, content_type=content_type).all()
else:
contents = session.query(models.CollectionContent).filter_by(edge_id=edge_id, status=status, content_type=content_type).all()
else:
if coll_id:
contents = session.query(models.CollectionContent).filter_by(edge_id=edge_id, coll_id=coll_id, content_type=content_type).all()
else:
contents = session.query(models.CollectionContent).filter_by(edge_id=edge_id, content_type=content_type).all()
for content in contents:
content['content_type'] = content.content_type
content['status'] = content.status
return contents
except sqlalchemy.orm.exc.NoResultFound:
raise exceptions.NoObject('No contents at edge %s with status %s' % (edge_name, status))
def delete_content(scope, name, edge_name=None, edge_id=None, content_id=None, session=None):
"""
Delete a collection content or raise a NoObject exception.
:param scope: The scope of the collection data.
:param name: The name of the collection data.
:param edge_name: The name of the edge.
:param edge_id: The id of the Edge
:param session: The database session in use.
:raises NoObject: If no edge is founded.
"""
try:
if content_id:
session.query(models.CollectionContent).filter_by(content_id=content_id).delete()
else:
if not edge_id:
edge_id = get_edge_id(edge_name=edge_name, session=session)
session.query(models.CollectionContent).filter_by(scope=scope, name=name, edge_id=edge_id).delete()
except sqlalchemy.orm.exc.NoResultFound:
raise exceptions.NoObject('Contents %s:%s(at edge %s) cannot be found' % (scope, name, edge_id))
def get_edge(edge_name, edge_id=None, session=None):
"""
Get an Edge or raise a NoObject exception.
:param edge_name: The name of the edge.
:param edge_id: The id of the Edge
:param session: The database session in use.
:raises NoObject: If no edge is founded.
:returns: Edge model.
"""
try:
if edge_id:
edge = session.query(models.Edge).filter_by(edge_id=edge_id).one()
else:
edge = session.query(models.Edge).filter_by(edge_name=edge_name).one()
edge['edge_type'] = edge.edge_type
return edge
except sqlalchemy.orm.exc.NoResultFound:
raise exceptions.NoObject('Edge %s cannot be found' % edge_name)
def update_collection(scope, name, parameters=None, coll_id=None, session=None):
"""
update a collection.
:param scope: The scope of the collection data.
:param name: The name of the collection data.
:param parameters: A dictionary of parameters.
:param coll_id: Collection id.
:param session: The database session in use.
:raises NoObject: If no edge is founded.
:raises DatabaseException: If there is a database error.
:returns: collection id.
"""
try:
if coll_id:
collection = session.query(models.Collection).filter_by(coll_id=coll_id).one()
else:
collection = session.query(models.Collection).filter_by(scope=scope, name=name).one()
except sqlalchemy.orm.exc.NoResultFound:
raise exceptions.NoObject('Collection %s:%s cannot be found' % (scope, name))
try:
if 'collection_type' in parameters and \
(isinstance(parameters['collection_type'], str) or isinstance(parameters['collection_type'], unicode)):
parameters['collection_type'] = CollectionType.from_sym(str(parameters['collection_type']))
if 'global_status' in parameters and \
(isinstance(parameters['global_status'], str) or isinstance(parameters['global_status'], unicode)):
parameters['global_status'] = CollectionStatus.from_sym(str(parameters['global_status']))
collection.update(parameters)
except DatabaseError as error:
raise exceptions.DatabaseException(error.args)
return collection.coll_id
def get_content(scope, name, min_id=None, max_id=None, edge_name=None, edge_id=None, content_id=None, session=None):
"""
Get a collection content or raise a NoObject exception.
:param scope: The scope of the collection data.
:param name: The name of the collection data.
:param min_id: The minimum id of the partial file, related to the whole file.
:param max_id: The maximum id of the partial file, related to the whole file.
:param edge_name: The name of the edge.
:param edge_id: The id of the Edge
:param session: The database session in use.
:raises NoObject: If no edge is founded.
:returns: Content model.
"""
try:
if content_id:
content = session.query(models.CollectionContent).filter_by(content_id=content_id).one()
else:
if not edge_id:
edge_id = get_edge_id(edge_name=edge_name, session=session)
if min_id is not None and max_id is not None:
content = session.query(models.CollectionContent).filter_by(scope=scope, name=name, edge_id=edge_id,
min_id=min_id, max_id=max_id).one()
else:
content_type = ContentType.FILE
content = session.query(models.CollectionContent).filter_by(scope=scope, name=name, content_type=content_type, edge_id=edge_id).one()
content['content_type'] = content.content_type
content['status'] = content.status
return content
except sqlalchemy.orm.exc.NoResultFound:
raise exceptions.NoObject('Content %s:%s[%s:%s](at edge %s) cannot be found' % (scope, name, min_id, max_id, edge_id))
def get_request_response(self, url, type='GET', data=None, headers=None):
"""
Send request to the ESS server and get the response.
:param url: http url to connection.
:param type: request type(GET, PUT, POST, DEL).
:param data: data to be sent to the ESS server.
:param headers: http headers.
:returns: response data as json.
:raises:
"""
result = None
for retry in range(self.retries):
try:
if type == 'GET':
result = self.session.get(url,
timeout=self.timeout,
headers=headers,
verify=False)
elif type == 'PUT':
result = self.session.put(url,
data=json.dumps(data),
timeout=self.timeout,
headers=headers,
verify=False)
elif type == 'POST':
result = self.session.post(url,
data=json.dumps(data),
timeout=self.timeout,
headers=headers,
verify=False)
elif type == 'DEL':
result = self.session.delete(url,
data=json.dumps(data),
timeout=self.timeout,
headers=headers,
verify=False)
else:
return
except requests.exceptions.ConnectionError as error:
logging.warning('ConnectionError: ' + str(error))
if retry >= self.retries - 1:
raise exceptions.ConnectionException('ConnectionError: ' +
str(error))
if result is not None:
if result.status_code == HTTP_STATUS_CODE.OK:
return json.loads(result.text)
elif result.status_code == HTTP_STATUS_CODE.NotFound:
raise exceptions.NoObject("Not found object")
else:
try:
data = json.loads(result.text)
if 'ExceptionClass' in data:
cls = getattr(exceptions, data['ExceptionClass'])
del data['ExceptionClass']
raise cls(**data)
else:
raise exceptions.ESSException(**data)
except AttributeError:
raise exceptions.ESSException(**data)
if result is None:
raise exceptions.ESSException('Response is None')
