def get_requests_by_edge(edge_name, edge_id=None, session=None):
"""
Get requests by edge.
:param edge_name: The name of the edge.
:param edge_id: The id of the edge
:param session: The database session in use.
:raises NoObject: If no request is founded.
:returns: list of Request model.
"""
try:
if not edge_id:
edge_id = get_edge_id(edge_name)
requests = session.query(
models.Request).filter_by(edge_id=edge_id).all()
for request in requests:
request['data_type'] = request.data_type
request['granularity_type'] = request.granularity_type
request['status'] = request.status
return requests
except sqlalchemy.orm.exc.NoResultFound:
raise exceptions.NoObject('No requests at %s' % (edge_name))
def get_collection_replicas(scope, name, edge_name, coll_id=None, edge_id=None, session=None):
"""
Get a collection replicas or raise a NoObject exception.
:param scope: The scope of the collection data.
:param name: The name of the collection data.
:param coll_id: Collection id.
:param edge_name: The name of the replicating edge.
:param edge_id: The id of the replicating edge.
:param session: The database session in use.
:raises NoObject: If no edge is founded.
:returns: CollectionReplicas model.
"""
try:
if not coll_id:
coll_id = get_collection_id(scope=scope, name=name, session=session)
if not edge_id:
edge_id = get_edge_id(edge_name=edge_name, session=session)
collection_replicas = session.query(models.CollectionReplicas).filter_by(coll_id=coll_id, edge_id=edge_id).one()
collection_replicas['status'] = collection_replicas.status
return collection_replicas
except sqlalchemy.orm.exc.NoResultFound:
raise exceptions.NoObject('Collection replicas %s:%s(at %s) cannot be found' % (scope, name, edge_name))
def assign_remote_requests(self):
if not self.head_client:
return
reqs = []
try:
reqs = self.head_client.get_requests(edge_name=self.resource_name,
status=str(
RequestStatus.ASSIGNING))
except NoObject:
pass
except ESSException as error:
self.logger.info(
"Caught exception when get requests from the head service: %s"
% str(error))
for req in reqs:
coll = self.head_client.get_collection(req.scope, req.name)
collection = {
'scope': coll.scope,
'name': coll.name,
'collection_type': coll.collection_type,
'coll_size': coll.coll_size,
'global_status': coll.global_status,
'total_files': coll.total_files,
'num_replicas': coll.num_replicas,
'coll_metadata': coll.coll_metadata
}
try:
add_collection(**collection)
except DuplicatedObject as error:
self.logger.info("Collection is already registered: %s, %s" %
(collection, error))
req.status = RequestStatus.ASSIGNED
processing_meta = req.processing_meta
processing_meta['original_request_id'] = req.request_id
request = {
'scope': req.scope,
'name': req.name,
'data_type': req.data_type,
'granularity_type': req.granularity_type,
'granularity_level': req.granularity_level,
'priority': req.priority,
'edge_id': get_edge_id(self.resource_name),
'status': req.status,
'request_meta': req.request_meta,
'processing_meta': processing_meta,
'errors': req.errors
}
try:
add_request(**request)
except DuplicatedObject as error:
self.logger.info("Request is already registered: %s, %s" %
(request, error))
self.head_client.update_request(req.request_id,
status=str(req.status))
def add_content(scope, name, min_id=None, max_id=None, coll_id=None, content_type=ContentType.FILE,
status=ContentStatus.NEW, priority=0, edge_name=None, edge_id=None, num_success=0,
num_failure=0, last_failed_at=None, pfn_size=0, pfn=None, object_metadata=None, session=None):
"""
Add a collection content.
:param scope: The scope of the collection data.
:param name: The name of the collection data.
:param min_id: The minimum id of the partial file, related to the whole file.
:param max_id: The maximum id of the partial file, related to the whole file.
:param coll_id: The collection it belongs to.
:param content_type: The type of the cotent data.
:param status: The status of the content.
:param priority: The priority as an integer.
:param edge_name: The name of the replicating edge.
:param edge_id: The id of the replicating edge.
:param num_success: The number of successful access.
:param num_failure; The number of failed access
:param last_failed_at: The time of last failure.
:param pfn_size: The size of physical file name.
:param pfn: Physical file name.
:raises DuplicatedObject: If an edge with the same name exists.
:raises DatabaseException: If there is a database error.
:returns: collection content id.
"""
if isinstance(content_type, str) or isinstance(content_type, unicode):
content_type = ContentType.from_sym(str(content_type))
if isinstance(status, str) or isinstance(status, unicode):
status = ContentStatus.from_sym(str(status))
if not edge_id:
edge_id = get_edge_id(edge_name=edge_name, session=session)
new_content = models.CollectionContent(scope=scope, name=name, min_id=min_id, max_id=max_id, coll_id=coll_id,
content_type=content_type, status=status, priority=priority,
edge_id=edge_id, num_success=num_success, num_failure=num_failure,
last_failed_at=last_failed_at, pfn_size=pfn_size, pfn=pfn,
object_metadata=object_metadata)
try:
new_content.save(session=session)
except IntegrityError:
raise exceptions.DuplicatedObject('Content %s:%s[%s:%s](at edge %s) already exists!' % (scope, name, min_id, max_id, edge_name))
except DatabaseError as error:
raise exceptions.DatabaseException(error.args)
return new_content.content_id
def get_content_best_match(scope, name, min_id=None, max_id=None, edge_name=None, edge_id=None, status=None, session=None):
"""
Get a collection content or raise a NoObject exception.
:param scope: The scope of the collection data.
:param name: The name of the collection data.
:param min_id: The minimum id of the partial file, related to the whole file.
:param max_id: The maximum id of the partial file, related to the whole file.
:param edge_name: The name of the edge.
:param edge_id: The id of the Edge
:param session: The database session in use.
:raises NoObject: If no edge is founded.
:returns: Content model.
"""
try:
if not edge_id and edge_name:
edge_id = get_edge_id(edge_name=edge_name, session=session)
if status and (isinstance(status, str) or isinstance(status, unicode)):
status = ContentStatus.from_sym(str(status))
query = session.query(models.CollectionContent).filter_by(scope=scope, name=name)
if status:
query = query.filter_by(status=status)
if edge_id:
query = query.filter_by(edge_id=edge_id)
if min_id is not None and max_id is not None:
contents = query.filter(and_(CollectionContent.min_id <= min_id, CollectionContent.max_id >= max_id)).all()
content = None
for row in contents:
if (not content) or (content['max_id'] - content['min_id'] > row['max_id'] - row['min_id']):
content = row
if content is None:
raise sqlalchemy.orm.exc.NoResultFound()
else:
content = query.filter_by(content_type=ContentType.FILE).one()
content['content_type'] = content.content_type
content['status'] = content.status
return content
except sqlalchemy.orm.exc.NoResultFound:
raise exceptions.NoObject('Content %s:%s[%s:%s](at edge %s) cannot be found' % (scope, name, min_id, max_id, edge_id))
def get_contents_by_edge(edge_name, edge_id=None, status=None, coll_id=None, content_type=None,
collection_scope=None, collection_name=None, limit=None, session=None):
"""
Get a collection content or raise a NoObject exception.
:param edge_name: The name of the edge.
:param edge_id: The id of the Edge
:param status: The status of the content.
:param coll_id: The collection id.
:param content_type: The tyep of the content.
:param limit: Number to return limited items.
:param session: The database session in use.
:raises NoObject: If no edge is founded.
:returns: Content model.
"""
try:
if not edge_id:
edge_id = get_edge_id(edge_name=edge_name, session=session)
if not coll_id and (collection_scope and collection_name):
coll_id = get_collection_id(collection_scope, collection_name)
query = session.query(models.CollectionContent).filter_by(edge_id=edge_id)
if status:
if isinstance(status, str) or isinstance(status, unicode):
status = ContentStatus.from_sym(str(status))
query = query.filter_by(status=status)
if coll_id:
query = query.filter_by(coll_id=coll_id)
if content_type:
query = query.filter_by(content_type=content_type)
if limit:
query = query.limit(limit)
contents = query.all()
for content in contents:
content['content_type'] = content.content_type
content['status'] = content.status
return contents
except sqlalchemy.orm.exc.NoResultFound:
raise exceptions.NoObject('No contents at edge %s with status %s' % (edge_name, status))
def get_requests(status=None, edge_name=None, edge_id=None, session=None):
"""
Get requests.
:param status: The status of the request data.
:param edge_name: The name of the edge.
:param edge_id: The id of the edge
:param session: The database session in use.
:raises NoObject: If no request is founded.
:returns: Request models.
"""
try:
if edge_name and not edge_id:
edge_id = get_edge_id(edge_name)
if status:
if (isinstance(status, str) or isinstance(status, unicode)):
status = RequestStatus.from_sym(status)
if edge_id:
requests = session.query(models.Request).filter_by(
status=status, edge_id=edge_id).all()
else:
requests = session.query(
models.Request).filter_by(status=status).all()
else:
if edge_id:
requests = session.query(
models.Request).filter_by(edge_id=edge_id).all()
else:
requests = session.query(models.Request).all()
for request in requests:
request['data_type'] = request.data_type
request['granularity_type'] = request.granularity_type
request['status'] = request.status
return requests
except sqlalchemy.orm.exc.NoResultFound:
raise exceptions.NoObject('Cannot find request with status: %s' %
(status))
def add_collection_replicas(scope, name, edge_name, coll_id=None, edge_id=None, status=CollectionReplicasStatus.NEW,
transferring_files=0, replicated_files=0, num_active_requests=1, retries=0, session=None):
"""
Add a collection replicas.
:param scope: The scope of the collection data.
:param name: The name of the collection data.
:param edge_name: The name of the replicating edge.
:param edge_id: The id of the replicating edge.
:param coll_id: Collection id.
:param status: The status of the collection replicas.
:param transferring_files: Number of transferring files.
:param replicated_files: Number of replicated files.
:param num_active_requests: Number of active requests.
:param retries: Number of retries.
:param session: The database session in use.
:raises DuplicatedObject: If an edge with the same name exists.
:raises DatabaseException: If there is a database error.
"""
if not coll_id:
coll_id = get_collection_id(scope=scope, name=name, session=session)
if not edge_id:
edge_id = get_edge_id(edge_name=edge_name, session=session)
try:
if isinstance(status, str) or isinstance(status, unicode):
status = CollectionReplicasStatus.from_sym(str(status))
new_collection_replicas = models.CollectionReplicas(coll_id=coll_id, edge_id=edge_id, status=status,
transferring_files=transferring_files,
replicated_files=replicated_files,
num_active_requests=num_active_requests,
retries=retries)
new_collection_replicas.save(session=session)
except IntegrityError as error:
if 'a foreign key constraint fails' in str(error):
raise exceptions.DatabaseException("Failed to add new collection %s:%s(at edge %s): %s" % (scope, name, edge_name, error))
else:
raise exceptions.DuplicatedObject('Collection %s:%s(at edge %s) already exists! (%s)' % (scope, name, edge_name, error))
except DatabaseError as error:
raise exceptions.DatabaseException(error.args)
def get_contents_statistics(edge_name, edge_id=None, coll_id=None, status=None, content_type=None, session=None):
"""
Get content statistics.
:param edge_name: The name of the edge.
:param edge_id: The id of the Edge
:param coll_id: The collection id.
:param status: The status of the content.
:param content_type: The tyep of the content.
:param session: The database session in use.
:returns: dict.
"""
try:
if not edge_id and edge_name:
edge_id = get_edge_id(edge_name=edge_name, session=session)
query = session.query(models.CollectionContent.edge_id, models.CollectionContent.coll_id,
models.CollectionContent.content_type, models.CollectionContent.status,
func.count(1).label('counter'))
if edge_id:
query = query.filter_by(edge_id=edge_id)
if coll_id:
query = query.filter_by(coll_id=coll_id)
if status:
if isinstance(status, str) or isinstance(status, unicode):
status = ContentStatus.from_sym(str(status))
query = query.filter_by(status=status)
if content_type:
if isinstance(content_type, str) or isinstance(content_type, unicode):
content_type = ContentType.from_sym(str(content_type))
query = query.filter_by(content_type=content_type)
query = query.group_by(models.CollectionContent.edge_id, models.CollectionContent.coll_id,
models.CollectionContent.content_type, models.CollectionContent.status)
statistics = query.all()
return statistics
except sqlalchemy.orm.exc.NoResultFound:
raise exceptions.NoObject('Failed to get statistics for edge %s and collection %s' % (edge_id, coll_id))
def get_contents_by_edge(edge_name, edge_id=None, status=None, coll_id=None, content_type=ContentType.FILE, session=None):
"""
Get a collection content or raise a NoObject exception.
:param edge_name: The name of the edge.
:param edge_id: The id of the Edge
:param status: The status of the content.
:param coll_id: The collection id.
:param session: The database session in use.
:raises NoObject: If no edge is founded.
:returns: Content model.
"""
try:
if not edge_id:
edge_id = get_edge_id(edge_name=edge_name, session=session)
if status:
if isinstance(status, str) or isinstance(status, unicode):
status = ContentStatus.from_sym(str(status))
if coll_id:
contents = session.query(models.CollectionContent).filter_by(edge_id=edge_id, status=status, coll_id=coll_id, content_type=content_type).all()
else:
contents = session.query(models.CollectionContent).filter_by(edge_id=edge_id, status=status, content_type=content_type).all()
else:
if coll_id:
contents = session.query(models.CollectionContent).filter_by(edge_id=edge_id, coll_id=coll_id, content_type=content_type).all()
else:
contents = session.query(models.CollectionContent).filter_by(edge_id=edge_id, content_type=content_type).all()
for content in contents:
content['content_type'] = content.content_type
content['status'] = content.status
return contents
except sqlalchemy.orm.exc.NoResultFound:
raise exceptions.NoObject('No contents at edge %s with status %s' % (edge_name, status))
def delete_content(scope, name, edge_name=None, edge_id=None, content_id=None, session=None):
"""
Delete a collection content or raise a NoObject exception.
:param scope: The scope of the collection data.
:param name: The name of the collection data.
:param edge_name: The name of the edge.
:param edge_id: The id of the Edge
:param session: The database session in use.
:raises NoObject: If no edge is founded.
"""
try:
if content_id:
session.query(models.CollectionContent).filter_by(content_id=content_id).delete()
else:
if not edge_id:
edge_id = get_edge_id(edge_name=edge_name, session=session)
session.query(models.CollectionContent).filter_by(scope=scope, name=name, edge_id=edge_id).delete()
except sqlalchemy.orm.exc.NoResultFound:
raise exceptions.NoObject('Contents %s:%s(at edge %s) cannot be found' % (scope, name, edge_id))
def add_contents(collection_scope, collection_name, edge_name, files, session=None):
"""
Add a collection contents.
:param collection_scope: The scope of the collection.
:param collection_name: The name of the collection.
:param edge_name: The edge name.
:param files: list of files.
"""
coll_id = get_collection_id(scope=collection_scope, name=collection_name, session=session)
edge_id = get_edge_id(edge_name=edge_name, session=session)
for file in files:
try:
add_content(scope=file['scope'], name=file['name'], min_id=file['min_id'],
max_id=file['max_id'], coll_id=coll_id, content_type=file['content_type'],
status=file['status'], priority=file['priority'], edge_id=edge_id,
pfn_size=file['pfn_size'] if 'pfn_size' in file else 0,
pfn=file['pfn'] if 'pfn' in file else None,
object_metadata=file['object_metadata'] if 'object_metadata' in file else None,
session=None)
except exceptions.DuplicatedObject:
pass
def get_content(scope, name, min_id=None, max_id=None, edge_name=None, edge_id=None, content_id=None, session=None):
"""
Get a collection content or raise a NoObject exception.
:param scope: The scope of the collection data.
:param name: The name of the collection data.
:param min_id: The minimum id of the partial file, related to the whole file.
:param max_id: The maximum id of the partial file, related to the whole file.
:param edge_name: The name of the edge.
:param edge_id: The id of the Edge
:param session: The database session in use.
:raises NoObject: If no edge is founded.
:returns: Content model.
"""
try:
if content_id:
content = session.query(models.CollectionContent).filter_by(content_id=content_id).one()
else:
if not edge_id:
edge_id = get_edge_id(edge_name=edge_name, session=session)
if min_id is not None and max_id is not None:
content = session.query(models.CollectionContent).filter_by(scope=scope, name=name, edge_id=edge_id,
min_id=min_id, max_id=max_id).one()
else:
content_type = ContentType.FILE
content = session.query(models.CollectionContent).filter_by(scope=scope, name=name, content_type=content_type, edge_id=edge_id).one()
content['content_type'] = content.content_type
content['status'] = content.status
return content
except sqlalchemy.orm.exc.NoResultFound:
raise exceptions.NoObject('Content %s:%s[%s:%s](at edge %s) cannot be found' % (scope, name, min_id, max_id, edge_id))
def test_create_and_check_for_edge_core(self):
""" Edge (CORE): Test the creation, query, and deletion of a Edge """
edge_name = 'test_edge_%s' % str(uuid())
edge_name = edge_name[:29]
properties = {
'edge_type': 'EDGE',
'status': 'ACTIVE',
'is_independent': True,
'continent': 'US',
'country_name': 'US',
'region_code': 'US',
'city': 'Madison',
'longitude': '111111',
'latitude': '2222222',
'total_space': 0,
'used_space': 0,
'num_files': 0
}
register_edge(edge_name, **properties)
edge = get_edge(edge_name)
assert_equal(edge.edge_name, edge_name)
assert_equal(str(edge.edge_type), properties['edge_type'])
assert_equal(str(edge.status), properties['status'])
assert_equal(edge.is_independent, properties['is_independent'])
assert_equal(edge.continent, properties['continent'])
assert_equal(edge.country_name, properties['country_name'])
assert_equal(edge.region_code, properties['region_code'])
assert_equal(edge.city, properties['city'])
assert_equal(edge.longitude, properties['longitude'])
assert_equal(edge.latitude, properties['latitude'])
assert_equal(edge.total_space, properties['total_space'])
assert_equal(edge.used_space, properties['used_space'])
assert_equal(edge.num_files, properties['num_files'])
assert_equal(edge.edge_id, get_edge_id(edge_name))
with assert_raises(exceptions.NoObject):
get_edge_id('not_exist_edge')
with assert_raises(exceptions.NoObject):
get_edge('not_exist_edge')
with assert_raises(exceptions.DuplicatedObject):
register_edge(edge_name)
update_edge(edge_name, parameters={'status': 'LOSTHEARTBEAT'})
edge = get_edge(edge_name)
assert_equal(str(edge.status), 'LOSTHEARTBEAT')
edges = get_edges()
assert_true(len(edges) >= 1)
render_json(**edges[0])
edges[0].to_dict()
edges = get_edges(status='LOSTHEARTBEAT')
assert_true(len(edges) >= 1)
delete_edge(edge_name)
with assert_raises(exceptions.NoObject):
get_edge(edge_name)
