def update_collection_replicas(scope, name, edge_name, coll_id=None, edge_id=None, parameters=None, session=None):
"""
update a collection replicas.
:param scope: The scope of the collection data.
:param name: The name of the collection data.
:param coll_id: Collection id.
:param edge_name: The name of the replicating edge.
:param edge_id: The id of the replicating edge.
:param parameters: A dictionary of parameters.
:param session: The database session in use.
:raises NoObject: If no edge is founded.
:raises DatabaseException: If there is a database error.
"""
try:
collection_replicas = get_collection_replicas(scope, name, edge_name, coll_id=coll_id, edge_id=edge_id, session=session)
except sqlalchemy.orm.exc.NoResultFound:
raise exceptions.NoObject('Collection %s:%s cannot be found' % (scope, name))
try:
if 'status' in parameters and \
(isinstance(parameters['status'], str) or isinstance(parameters['status'], unicode)):
parameters['status'] = CollectionReplicasStatus.from_sym(str(parameters['status']))
collection_replicas.update(parameters)
except DatabaseError as error:
raise exceptions.DatabaseException(error.args)
def update_edge(edge_name, parameters, session=None):
"""
update an edge.
:param edge_name: the name of the edge.
:param parameters: A dictionary of parameters.
:param session: The database session in use.
:raises NoObject: If no edge is founded.
:raises DatabaseException: If there is a database error.
:returns: edge id.
"""
try:
if 'edge_type' in parameters and \
(isinstance(parameters['edge_type'], str) or isinstance(parameters['edge_type'], unicode)):
parameters['edge_type'] = EdgeType.from_sym(str(parameters['edge_type']))
if 'status' in parameters and \
(isinstance(parameters['status'], str) or isinstance(parameters['status'], unicode)):
parameters['status'] = EdgeStatus.from_sym(str(parameters['status']))
edge = session.query(models.Edge).filter_by(edge_name=edge_name).one()
except sqlalchemy.orm.exc.NoResultFound:
raise exceptions.NoObject('Edge %s cannot be found' % edge_name)
try:
edge.update(parameters)
except DatabaseError as error:
raise exceptions.DatabaseException(error.args)
return edge.edge_id
def add_collection_replicas(scope, name, edge_name, coll_id=None, edge_id=None, status=CollectionReplicasStatus.NEW,
transferring_files=0, replicated_files=0, num_active_requests=1, retries=0, session=None):
"""
Add a collection replicas.
:param scope: The scope of the collection data.
:param name: The name of the collection data.
:param edge_name: The name of the replicating edge.
:param edge_id: The id of the replicating edge.
:param coll_id: Collection id.
:param status: The status of the collection replicas.
:param transferring_files: Number of transferring files.
:param replicated_files: Number of replicated files.
:param num_active_requests: Number of active requests.
:param retries: Number of retries.
:param session: The database session in use.
:raises DuplicatedObject: If an edge with the same name exists.
:raises DatabaseException: If there is a database error.
"""
if not coll_id:
coll_id = get_collection_id(scope=scope, name=name, session=session)
if not edge_id:
edge_id = get_edge_id(edge_name=edge_name, session=session)
try:
if isinstance(status, str) or isinstance(status, unicode):
status = CollectionReplicasStatus.from_sym(str(status))
new_collection_replicas = models.CollectionReplicas(coll_id=coll_id, edge_id=edge_id, status=status,
transferring_files=transferring_files,
replicated_files=replicated_files,
num_active_requests=num_active_requests,
retries=retries)
new_collection_replicas.save(session=session)
except IntegrityError as error:
if 'a foreign key constraint fails' in str(error):
raise exceptions.DatabaseException("Failed to add new collection %s:%s(at edge %s): %s" % (scope, name, edge_name, error))
else:
raise exceptions.DuplicatedObject('Collection %s:%s(at edge %s) already exists! (%s)' % (scope, name, edge_name, error))
except DatabaseError as error:
raise exceptions.DatabaseException(error.args)
def add_content(scope, name, min_id=None, max_id=None, coll_id=None, content_type=ContentType.FILE,
status=ContentStatus.NEW, priority=0, edge_name=None, edge_id=None, num_success=0,
num_failure=0, last_failed_at=None, pfn_size=0, pfn=None, object_metadata=None, session=None):
"""
Add a collection content.
:param scope: The scope of the collection data.
:param name: The name of the collection data.
:param min_id: The minimum id of the partial file, related to the whole file.
:param max_id: The maximum id of the partial file, related to the whole file.
:param coll_id: The collection it belongs to.
:param content_type: The type of the cotent data.
:param status: The status of the content.
:param priority: The priority as an integer.
:param edge_name: The name of the replicating edge.
:param edge_id: The id of the replicating edge.
:param num_success: The number of successful access.
:param num_failure; The number of failed access
:param last_failed_at: The time of last failure.
:param pfn_size: The size of physical file name.
:param pfn: Physical file name.
:raises DuplicatedObject: If an edge with the same name exists.
:raises DatabaseException: If there is a database error.
:returns: collection content id.
"""
if isinstance(content_type, str) or isinstance(content_type, unicode):
content_type = ContentType.from_sym(str(content_type))
if isinstance(status, str) or isinstance(status, unicode):
status = ContentStatus.from_sym(str(status))
if not edge_id:
edge_id = get_edge_id(edge_name=edge_name, session=session)
new_content = models.CollectionContent(scope=scope, name=name, min_id=min_id, max_id=max_id, coll_id=coll_id,
content_type=content_type, status=status, priority=priority,
edge_id=edge_id, num_success=num_success, num_failure=num_failure,
last_failed_at=last_failed_at, pfn_size=pfn_size, pfn=pfn,
object_metadata=object_metadata)
try:
new_content.save(session=session)
except IntegrityError:
raise exceptions.DuplicatedObject('Content %s:%s[%s:%s](at edge %s) already exists!' % (scope, name, min_id, max_id, edge_name))
except DatabaseError as error:
raise exceptions.DatabaseException(error.args)
return new_content.content_id
def register_edge(edge_name, edge_type=EdgeType.EDGE, status=EdgeStatus.ACTIVE, is_independent=True,
continent=None, country_name=None, region_code=None, city=None, longitude=None,
latitude=None, total_space=0, used_space=0, reserved_space=0, num_files=0,
session=None):
"""
Register an edge.
:param edge_name: the name of the edge.
:param edge_type: The type of the edge.
:param status: Status of the edge.
:param is_independent: Whether it's using current db or independent db.
:param continent: The continent of the edge.
:param country_name: The country of the edge.
:param region_code: The region code for the edge.
:param city: The city for the edge.
:param latitude: The latitude coordinate of edge.
:param longitude: The longitude coordinate of edge.
:param total_space: Total space of the edge.
:param used_space: Used space of the edge.
:param reserved_space: Reserved space of the edge.
:param num_files: Number of cached files in the edge.
:param session: The database session in use.
:raises DuplicatedObject: If an edge with the same name exists.
:raises DatabaseException: If there is a database error.
:returns: edge id.
"""
if isinstance(edge_type, str) or isinstance(edge_type, unicode):
edge_type = EdgeType.from_sym(str(edge_type))
if isinstance(status, str) or isinstance(status, unicode):
status = EdgeStatus.from_sym(str(status))
new_edge = models.Edge(edge_name=edge_name, edge_type=edge_type, status=status,
is_independent=is_independent, continent=continent, country_name=country_name,
region_code=region_code, city=city, longitude=longitude, latitude=latitude,
total_space=total_space, used_space=used_space, reserved_space=reserved_space,
num_files=num_files)
try:
new_edge.save(session=session)
except IntegrityError:
raise exceptions.DuplicatedObject('Edge %s already exists!' % edge_name)
except DatabaseError as error:
raise exceptions.DatabaseException(error.args)
return new_edge.edge_id
def add_request(scope, name, data_type=DataType.DATASET, granularity_type=GranularityType.FILE,
granularity_level=None, priority=0, edge_id=None, status=RequestStatus.NEW,
request_meta=None, processing_meta=None, errors=None, session=None):
"""
Add a request.
:param scope: The scope of the request data.
:param name: The name of the request data.
:param data_type: The type of the request data.
:param granularity_type: The granularity type, for example File or Partial.
:param granularity_level: The granularity level, for example, number of events.
:param priority: The priority as an integer.
:param edge_id: The id of the assigned edge.
:param status: The status of the request.
:param request_meta: The metadata of the request, as Json.
:param processing_meta: The processing metadata, as Json.
:param errors: The processing errors, as Json.
:param session: The database session in use.
:raises DuplicatedObject: If an request with the same name exists.
:raises DatabaseException: If there is a database error.
:returns: request id.
"""
if isinstance(data_type, str) or isinstance(data_type, unicode):
data_type = DataType.from_sym(str(data_type))
if isinstance(granularity_type, str) or isinstance(granularity_type, unicode):
granularity_type = GranularityType.from_sym(str(granularity_type))
if isinstance(status, str) or isinstance(status, unicode):
status = RequestStatus.from_sym(str(status))
new_request = models.Request(scope=scope, name=name, data_type=data_type, granularity_type=granularity_type,
granularity_level=granularity_level, priority=priority, edge_id=edge_id, status=status,
request_meta=request_meta, processing_meta=processing_meta, errors=errors)
try:
new_request.save(session=session)
except IntegrityError as error:
raise exceptions.DuplicatedObject('Request %s:%s already exists!: %s' % (scope, name, error))
except DatabaseError as error:
raise exceptions.DatabaseException(error.args)
return new_request.request_id
def update_contents(files, session=None):
"""
update a collection content.
:param files: The list of content files.
:param session: The database session in use.
:raises NoObject: If no content is founded.
:raises DatabaseException: If there is a database error.
"""
try:
for file in files:
parameters = {'status': file.status,
'pfn_size': file.pfn_size,
'pfn': file.pfn}
update_content(cope=file.scope, name=file.name, min_id=file.min_id, max_id=file.max_id, edge_id=file.edge_id,
content_id=file.content_id,
parameters=parameters, session=session)
except DatabaseError as error:
raise exceptions.DatabaseException(error.args)
def update_request(request_id, parameters, session=None):
"""
update an request.
:param request_id: the request id.
:param parameters: A dictionary of parameters.
:param session: The database session in use.
:raises NoObject: If no request is founded.
:raises DatabaseException: If there is a database error.
:returns: request id.
"""
try:
if 'data_type' in parameters and \
(isinstance(parameters['data_type'], str) or isinstance(parameters['data_type'], unicode)):
parameters['data_type'] = DataType.from_sym(
str(parameters['data_type']))
if 'granularity_type' in parameters and \
(isinstance(parameters['granularity_type'], str) or isinstance(parameters['granularity_type'], unicode)):
parameters['granularity_type'] = GranularityType.from_sym(
str(parameters['granularity_type']))
if 'status' in parameters and \
(isinstance(parameters['status'], str) or isinstance(parameters['status'], unicode)):
parameters['status'] = RequestStatus.from_sym(
str(parameters['status']))
request = session.query(
models.Request).filter_by(request_id=request_id).one()
except sqlalchemy.orm.exc.NoResultFound:
raise exceptions.NoObject('Request %s cannot be found' % request_id)
try:
request.update(parameters)
except DatabaseError as error:
raise exceptions.DatabaseException(error.args)
return request.request_id
def update_contents_by_id(files, session=None):
"""
update a collection content.
:param files: The list of content files.
:param session: The database session in use.
:raises NoObject: If no content is founded.
:raises DatabaseException: If there is a database error.
"""
try:
for id in files:
parameters = files[id]
if 'status' in parameters and \
(isinstance(parameters['status'], str) or isinstance(parameters['status'], unicode)):
parameters['status'] = ContentStatus.from_sym(str(parameters['status']))
session.query(models.CollectionContent).filter_by(content_id=id).update(parameters)
except DatabaseError as error:
raise exceptions.DatabaseException(error.args)
def update_collection(scope, name, parameters=None, coll_id=None, session=None):
"""
update a collection.
:param scope: The scope of the collection data.
:param name: The name of the collection data.
:param parameters: A dictionary of parameters.
:param coll_id: Collection id.
:param session: The database session in use.
:raises NoObject: If no edge is founded.
:raises DatabaseException: If there is a database error.
:returns: collection id.
"""
try:
if coll_id:
collection = session.query(models.Collection).filter_by(coll_id=coll_id).one()
else:
collection = session.query(models.Collection).filter_by(scope=scope, name=name).one()
except sqlalchemy.orm.exc.NoResultFound:
raise exceptions.NoObject('Collection %s:%s cannot be found' % (scope, name))
try:
if 'collection_type' in parameters and \
(isinstance(parameters['collection_type'], str) or isinstance(parameters['collection_type'], unicode)):
parameters['collection_type'] = CollectionType.from_sym(str(parameters['collection_type']))
if 'global_status' in parameters and \
(isinstance(parameters['global_status'], str) or isinstance(parameters['global_status'], unicode)):
parameters['global_status'] = CollectionStatus.from_sym(str(parameters['global_status']))
collection.update(parameters)
except DatabaseError as error:
raise exceptions.DatabaseException(error.args)
return collection.coll_id
def update_content(scope, name, min_id=None, max_id=None, edge_name=None, edge_id=None, content_id=None, parameters=None, session=None):
"""
update a collection content.
:param scope: The scope of the collection data.
:param name: The name of the collection data.
:param min_id: The minimum id of the partial file, related to the whole file.
:param max_id: The maximum id of the partial file, related to the whole file.
:param edge_name: The name of the edge.
:param edge_id: The id of the Edge
:param session: The database session in use.
:raises NoObject: If no content is founded.
:raises DatabaseException: If there is a database error.
:returns: content id.
"""
content = get_content(scope, name, min_id=min_id, max_id=max_id,
edge_name=edge_name, edge_id=edge_id, session=session)
try:
content = get_content(scope, name, min_id=min_id, max_id=max_id, content_id=content_id,
edge_name=edge_name, edge_id=edge_id, session=session)
if 'content_type' in parameters and \
(isinstance(parameters['content_type'], str) or isinstance(parameters['content_type'], unicode)):
parameters['content_type'] = ContentType.from_sym(str(parameters['content_type']))
if 'status' in parameters and \
(isinstance(parameters['status'], str) or isinstance(parameters['status'], unicode)):
parameters['status'] = ContentStatus.from_sym(str(parameters['status']))
content.update(parameters)
except DatabaseError as error:
raise exceptions.DatabaseException(error.args)
return content.content_id
def add_collection(scope, name, collection_type=CollectionType.DATASET, coll_size=0, global_status=CollectionStatus.NEW,
total_files=0, num_replicas=0, coll_metadata=None, session=None):
"""
Add a collection.
:param scope: The scope of the collection data.
:param name: The name of the collection data.
:param collection_type: The type of the collection data.
:param coll_size: The space size of the collection as an integer.
:param global_status: The status of the collection.
:param total_files: Total number of files of the collection.
:param num_replicas: Number of replicas.
:param coll_metadata: Collection metadata in json.
:param session: The database session in use.
:raises DuplicatedObject: If an edge with the same name exists.
:raises DatabaseException: If there is a database error.
:returns: collection id.
"""
if isinstance(collection_type, str) or isinstance(collection_type, unicode):
collection_type = CollectionType.from_sym(str(collection_type))
if isinstance(global_status, str) or isinstance(global_status, unicode):
global_status = CollectionStatus.from_sym(str(global_status))
new_collection = models.Collection(scope=scope, name=name, collection_type=collection_type, coll_size=coll_size, global_status=global_status,
total_files=total_files, num_replicas=num_replicas, coll_metadata=coll_metadata)
try:
new_collection.save(session=session)
except IntegrityError:
raise exceptions.DuplicatedObject('Collection %s:%s already exists!' % (scope, name))
except DatabaseError as error:
raise exceptions.DatabaseException(error.args)
return new_collection.coll_id