def update_collection_replicas(scope, name, edge_name, coll_id=None, edge_id=None, parameters=None, session=None): """ update a collection replicas. :param scope: The scope of the collection data. :param name: The name of the collection data. :param coll_id: Collection id. :param edge_name: The name of the replicating edge. :param edge_id: The id of the replicating edge. :param parameters: A dictionary of parameters. :param session: The database session in use. :raises NoObject: If no edge is founded. :raises DatabaseException: If there is a database error. """ try: collection_replicas = get_collection_replicas(scope, name, edge_name, coll_id=coll_id, edge_id=edge_id, session=session) except sqlalchemy.orm.exc.NoResultFound: raise exceptions.NoObject('Collection %s:%s cannot be found' % (scope, name)) try: if 'status' in parameters and \ (isinstance(parameters['status'], str) or isinstance(parameters['status'], unicode)): parameters['status'] = CollectionReplicasStatus.from_sym(str(parameters['status'])) collection_replicas.update(parameters) except DatabaseError as error: raise exceptions.DatabaseException(error.args)
def update_edge(edge_name, parameters, session=None): """ update an edge. :param edge_name: the name of the edge. :param parameters: A dictionary of parameters. :param session: The database session in use. :raises NoObject: If no edge is founded. :raises DatabaseException: If there is a database error. :returns: edge id. """ try: if 'edge_type' in parameters and \ (isinstance(parameters['edge_type'], str) or isinstance(parameters['edge_type'], unicode)): parameters['edge_type'] = EdgeType.from_sym(str(parameters['edge_type'])) if 'status' in parameters and \ (isinstance(parameters['status'], str) or isinstance(parameters['status'], unicode)): parameters['status'] = EdgeStatus.from_sym(str(parameters['status'])) edge = session.query(models.Edge).filter_by(edge_name=edge_name).one() except sqlalchemy.orm.exc.NoResultFound: raise exceptions.NoObject('Edge %s cannot be found' % edge_name) try: edge.update(parameters) except DatabaseError as error: raise exceptions.DatabaseException(error.args) return edge.edge_id
def add_collection_replicas(scope, name, edge_name, coll_id=None, edge_id=None, status=CollectionReplicasStatus.NEW, transferring_files=0, replicated_files=0, num_active_requests=1, retries=0, session=None): """ Add a collection replicas. :param scope: The scope of the collection data. :param name: The name of the collection data. :param edge_name: The name of the replicating edge. :param edge_id: The id of the replicating edge. :param coll_id: Collection id. :param status: The status of the collection replicas. :param transferring_files: Number of transferring files. :param replicated_files: Number of replicated files. :param num_active_requests: Number of active requests. :param retries: Number of retries. :param session: The database session in use. :raises DuplicatedObject: If an edge with the same name exists. :raises DatabaseException: If there is a database error. """ if not coll_id: coll_id = get_collection_id(scope=scope, name=name, session=session) if not edge_id: edge_id = get_edge_id(edge_name=edge_name, session=session) try: if isinstance(status, str) or isinstance(status, unicode): status = CollectionReplicasStatus.from_sym(str(status)) new_collection_replicas = models.CollectionReplicas(coll_id=coll_id, edge_id=edge_id, status=status, transferring_files=transferring_files, replicated_files=replicated_files, num_active_requests=num_active_requests, retries=retries) new_collection_replicas.save(session=session) except IntegrityError as error: if 'a foreign key constraint fails' in str(error): raise exceptions.DatabaseException("Failed to add new collection %s:%s(at edge %s): %s" % (scope, name, edge_name, error)) else: raise exceptions.DuplicatedObject('Collection %s:%s(at edge %s) already exists! (%s)' % (scope, name, edge_name, error)) except DatabaseError as error: raise exceptions.DatabaseException(error.args)
def add_content(scope, name, min_id=None, max_id=None, coll_id=None, content_type=ContentType.FILE, status=ContentStatus.NEW, priority=0, edge_name=None, edge_id=None, num_success=0, num_failure=0, last_failed_at=None, pfn_size=0, pfn=None, object_metadata=None, session=None): """ Add a collection content. :param scope: The scope of the collection data. :param name: The name of the collection data. :param min_id: The minimum id of the partial file, related to the whole file. :param max_id: The maximum id of the partial file, related to the whole file. :param coll_id: The collection it belongs to. :param content_type: The type of the cotent data. :param status: The status of the content. :param priority: The priority as an integer. :param edge_name: The name of the replicating edge. :param edge_id: The id of the replicating edge. :param num_success: The number of successful access. :param num_failure; The number of failed access :param last_failed_at: The time of last failure. :param pfn_size: The size of physical file name. :param pfn: Physical file name. :raises DuplicatedObject: If an edge with the same name exists. :raises DatabaseException: If there is a database error. :returns: collection content id. """ if isinstance(content_type, str) or isinstance(content_type, unicode): content_type = ContentType.from_sym(str(content_type)) if isinstance(status, str) or isinstance(status, unicode): status = ContentStatus.from_sym(str(status)) if not edge_id: edge_id = get_edge_id(edge_name=edge_name, session=session) new_content = models.CollectionContent(scope=scope, name=name, min_id=min_id, max_id=max_id, coll_id=coll_id, content_type=content_type, status=status, priority=priority, edge_id=edge_id, num_success=num_success, num_failure=num_failure, last_failed_at=last_failed_at, pfn_size=pfn_size, pfn=pfn, object_metadata=object_metadata) try: new_content.save(session=session) except IntegrityError: raise exceptions.DuplicatedObject('Content %s:%s[%s:%s](at edge %s) already exists!' % (scope, name, min_id, max_id, edge_name)) except DatabaseError as error: raise exceptions.DatabaseException(error.args) return new_content.content_id
def register_edge(edge_name, edge_type=EdgeType.EDGE, status=EdgeStatus.ACTIVE, is_independent=True, continent=None, country_name=None, region_code=None, city=None, longitude=None, latitude=None, total_space=0, used_space=0, reserved_space=0, num_files=0, session=None): """ Register an edge. :param edge_name: the name of the edge. :param edge_type: The type of the edge. :param status: Status of the edge. :param is_independent: Whether it's using current db or independent db. :param continent: The continent of the edge. :param country_name: The country of the edge. :param region_code: The region code for the edge. :param city: The city for the edge. :param latitude: The latitude coordinate of edge. :param longitude: The longitude coordinate of edge. :param total_space: Total space of the edge. :param used_space: Used space of the edge. :param reserved_space: Reserved space of the edge. :param num_files: Number of cached files in the edge. :param session: The database session in use. :raises DuplicatedObject: If an edge with the same name exists. :raises DatabaseException: If there is a database error. :returns: edge id. """ if isinstance(edge_type, str) or isinstance(edge_type, unicode): edge_type = EdgeType.from_sym(str(edge_type)) if isinstance(status, str) or isinstance(status, unicode): status = EdgeStatus.from_sym(str(status)) new_edge = models.Edge(edge_name=edge_name, edge_type=edge_type, status=status, is_independent=is_independent, continent=continent, country_name=country_name, region_code=region_code, city=city, longitude=longitude, latitude=latitude, total_space=total_space, used_space=used_space, reserved_space=reserved_space, num_files=num_files) try: new_edge.save(session=session) except IntegrityError: raise exceptions.DuplicatedObject('Edge %s already exists!' % edge_name) except DatabaseError as error: raise exceptions.DatabaseException(error.args) return new_edge.edge_id
def add_request(scope, name, data_type=DataType.DATASET, granularity_type=GranularityType.FILE, granularity_level=None, priority=0, edge_id=None, status=RequestStatus.NEW, request_meta=None, processing_meta=None, errors=None, session=None): """ Add a request. :param scope: The scope of the request data. :param name: The name of the request data. :param data_type: The type of the request data. :param granularity_type: The granularity type, for example File or Partial. :param granularity_level: The granularity level, for example, number of events. :param priority: The priority as an integer. :param edge_id: The id of the assigned edge. :param status: The status of the request. :param request_meta: The metadata of the request, as Json. :param processing_meta: The processing metadata, as Json. :param errors: The processing errors, as Json. :param session: The database session in use. :raises DuplicatedObject: If an request with the same name exists. :raises DatabaseException: If there is a database error. :returns: request id. """ if isinstance(data_type, str) or isinstance(data_type, unicode): data_type = DataType.from_sym(str(data_type)) if isinstance(granularity_type, str) or isinstance(granularity_type, unicode): granularity_type = GranularityType.from_sym(str(granularity_type)) if isinstance(status, str) or isinstance(status, unicode): status = RequestStatus.from_sym(str(status)) new_request = models.Request(scope=scope, name=name, data_type=data_type, granularity_type=granularity_type, granularity_level=granularity_level, priority=priority, edge_id=edge_id, status=status, request_meta=request_meta, processing_meta=processing_meta, errors=errors) try: new_request.save(session=session) except IntegrityError as error: raise exceptions.DuplicatedObject('Request %s:%s already exists!: %s' % (scope, name, error)) except DatabaseError as error: raise exceptions.DatabaseException(error.args) return new_request.request_id
def update_contents(files, session=None): """ update a collection content. :param files: The list of content files. :param session: The database session in use. :raises NoObject: If no content is founded. :raises DatabaseException: If there is a database error. """ try: for file in files: parameters = {'status': file.status, 'pfn_size': file.pfn_size, 'pfn': file.pfn} update_content(cope=file.scope, name=file.name, min_id=file.min_id, max_id=file.max_id, edge_id=file.edge_id, content_id=file.content_id, parameters=parameters, session=session) except DatabaseError as error: raise exceptions.DatabaseException(error.args)
def update_request(request_id, parameters, session=None): """ update an request. :param request_id: the request id. :param parameters: A dictionary of parameters. :param session: The database session in use. :raises NoObject: If no request is founded. :raises DatabaseException: If there is a database error. :returns: request id. """ try: if 'data_type' in parameters and \ (isinstance(parameters['data_type'], str) or isinstance(parameters['data_type'], unicode)): parameters['data_type'] = DataType.from_sym( str(parameters['data_type'])) if 'granularity_type' in parameters and \ (isinstance(parameters['granularity_type'], str) or isinstance(parameters['granularity_type'], unicode)): parameters['granularity_type'] = GranularityType.from_sym( str(parameters['granularity_type'])) if 'status' in parameters and \ (isinstance(parameters['status'], str) or isinstance(parameters['status'], unicode)): parameters['status'] = RequestStatus.from_sym( str(parameters['status'])) request = session.query( models.Request).filter_by(request_id=request_id).one() except sqlalchemy.orm.exc.NoResultFound: raise exceptions.NoObject('Request %s cannot be found' % request_id) try: request.update(parameters) except DatabaseError as error: raise exceptions.DatabaseException(error.args) return request.request_id
def update_contents_by_id(files, session=None): """ update a collection content. :param files: The list of content files. :param session: The database session in use. :raises NoObject: If no content is founded. :raises DatabaseException: If there is a database error. """ try: for id in files: parameters = files[id] if 'status' in parameters and \ (isinstance(parameters['status'], str) or isinstance(parameters['status'], unicode)): parameters['status'] = ContentStatus.from_sym(str(parameters['status'])) session.query(models.CollectionContent).filter_by(content_id=id).update(parameters) except DatabaseError as error: raise exceptions.DatabaseException(error.args)
def update_collection(scope, name, parameters=None, coll_id=None, session=None): """ update a collection. :param scope: The scope of the collection data. :param name: The name of the collection data. :param parameters: A dictionary of parameters. :param coll_id: Collection id. :param session: The database session in use. :raises NoObject: If no edge is founded. :raises DatabaseException: If there is a database error. :returns: collection id. """ try: if coll_id: collection = session.query(models.Collection).filter_by(coll_id=coll_id).one() else: collection = session.query(models.Collection).filter_by(scope=scope, name=name).one() except sqlalchemy.orm.exc.NoResultFound: raise exceptions.NoObject('Collection %s:%s cannot be found' % (scope, name)) try: if 'collection_type' in parameters and \ (isinstance(parameters['collection_type'], str) or isinstance(parameters['collection_type'], unicode)): parameters['collection_type'] = CollectionType.from_sym(str(parameters['collection_type'])) if 'global_status' in parameters and \ (isinstance(parameters['global_status'], str) or isinstance(parameters['global_status'], unicode)): parameters['global_status'] = CollectionStatus.from_sym(str(parameters['global_status'])) collection.update(parameters) except DatabaseError as error: raise exceptions.DatabaseException(error.args) return collection.coll_id
def update_content(scope, name, min_id=None, max_id=None, edge_name=None, edge_id=None, content_id=None, parameters=None, session=None): """ update a collection content. :param scope: The scope of the collection data. :param name: The name of the collection data. :param min_id: The minimum id of the partial file, related to the whole file. :param max_id: The maximum id of the partial file, related to the whole file. :param edge_name: The name of the edge. :param edge_id: The id of the Edge :param session: The database session in use. :raises NoObject: If no content is founded. :raises DatabaseException: If there is a database error. :returns: content id. """ content = get_content(scope, name, min_id=min_id, max_id=max_id, edge_name=edge_name, edge_id=edge_id, session=session) try: content = get_content(scope, name, min_id=min_id, max_id=max_id, content_id=content_id, edge_name=edge_name, edge_id=edge_id, session=session) if 'content_type' in parameters and \ (isinstance(parameters['content_type'], str) or isinstance(parameters['content_type'], unicode)): parameters['content_type'] = ContentType.from_sym(str(parameters['content_type'])) if 'status' in parameters and \ (isinstance(parameters['status'], str) or isinstance(parameters['status'], unicode)): parameters['status'] = ContentStatus.from_sym(str(parameters['status'])) content.update(parameters) except DatabaseError as error: raise exceptions.DatabaseException(error.args) return content.content_id
def add_collection(scope, name, collection_type=CollectionType.DATASET, coll_size=0, global_status=CollectionStatus.NEW, total_files=0, num_replicas=0, coll_metadata=None, session=None): """ Add a collection. :param scope: The scope of the collection data. :param name: The name of the collection data. :param collection_type: The type of the collection data. :param coll_size: The space size of the collection as an integer. :param global_status: The status of the collection. :param total_files: Total number of files of the collection. :param num_replicas: Number of replicas. :param coll_metadata: Collection metadata in json. :param session: The database session in use. :raises DuplicatedObject: If an edge with the same name exists. :raises DatabaseException: If there is a database error. :returns: collection id. """ if isinstance(collection_type, str) or isinstance(collection_type, unicode): collection_type = CollectionType.from_sym(str(collection_type)) if isinstance(global_status, str) or isinstance(global_status, unicode): global_status = CollectionStatus.from_sym(str(global_status)) new_collection = models.Collection(scope=scope, name=name, collection_type=collection_type, coll_size=coll_size, global_status=global_status, total_files=total_files, num_replicas=num_replicas, coll_metadata=coll_metadata) try: new_collection.save(session=session) except IntegrityError: raise exceptions.DuplicatedObject('Collection %s:%s already exists!' % (scope, name)) except DatabaseError as error: raise exceptions.DatabaseException(error.args) return new_collection.coll_id