def get_requests_by_edge(edge_name, edge_id=None, session=None): """ Get requests by edge. :param edge_name: The name of the edge. :param edge_id: The id of the edge :param session: The database session in use. :raises NoObject: If no request is founded. :returns: list of Request model. """ try: if not edge_id: edge_id = get_edge_id(edge_name) requests = session.query( models.Request).filter_by(edge_id=edge_id).all() for request in requests: request['data_type'] = request.data_type request['granularity_type'] = request.granularity_type request['status'] = request.status return requests except sqlalchemy.orm.exc.NoResultFound: raise exceptions.NoObject('No requests at %s' % (edge_name))
def get_collection_replicas(scope, name, edge_name, coll_id=None, edge_id=None, session=None): """ Get a collection replicas or raise a NoObject exception. :param scope: The scope of the collection data. :param name: The name of the collection data. :param coll_id: Collection id. :param edge_name: The name of the replicating edge. :param edge_id: The id of the replicating edge. :param session: The database session in use. :raises NoObject: If no edge is founded. :returns: CollectionReplicas model. """ try: if not coll_id: coll_id = get_collection_id(scope=scope, name=name, session=session) if not edge_id: edge_id = get_edge_id(edge_name=edge_name, session=session) collection_replicas = session.query(models.CollectionReplicas).filter_by(coll_id=coll_id, edge_id=edge_id).one() collection_replicas['status'] = collection_replicas.status return collection_replicas except sqlalchemy.orm.exc.NoResultFound: raise exceptions.NoObject('Collection replicas %s:%s(at %s) cannot be found' % (scope, name, edge_name))
def assign_remote_requests(self): if not self.head_client: return reqs = [] try: reqs = self.head_client.get_requests(edge_name=self.resource_name, status=str( RequestStatus.ASSIGNING)) except NoObject: pass except ESSException as error: self.logger.info( "Caught exception when get requests from the head service: %s" % str(error)) for req in reqs: coll = self.head_client.get_collection(req.scope, req.name) collection = { 'scope': coll.scope, 'name': coll.name, 'collection_type': coll.collection_type, 'coll_size': coll.coll_size, 'global_status': coll.global_status, 'total_files': coll.total_files, 'num_replicas': coll.num_replicas, 'coll_metadata': coll.coll_metadata } try: add_collection(**collection) except DuplicatedObject as error: self.logger.info("Collection is already registered: %s, %s" % (collection, error)) req.status = RequestStatus.ASSIGNED processing_meta = req.processing_meta processing_meta['original_request_id'] = req.request_id request = { 'scope': req.scope, 'name': req.name, 'data_type': req.data_type, 'granularity_type': req.granularity_type, 'granularity_level': req.granularity_level, 'priority': req.priority, 'edge_id': get_edge_id(self.resource_name), 'status': req.status, 'request_meta': req.request_meta, 'processing_meta': processing_meta, 'errors': req.errors } try: add_request(**request) except DuplicatedObject as error: self.logger.info("Request is already registered: %s, %s" % (request, error)) self.head_client.update_request(req.request_id, status=str(req.status))
def add_content(scope, name, min_id=None, max_id=None, coll_id=None, content_type=ContentType.FILE, status=ContentStatus.NEW, priority=0, edge_name=None, edge_id=None, num_success=0, num_failure=0, last_failed_at=None, pfn_size=0, pfn=None, object_metadata=None, session=None): """ Add a collection content. :param scope: The scope of the collection data. :param name: The name of the collection data. :param min_id: The minimum id of the partial file, related to the whole file. :param max_id: The maximum id of the partial file, related to the whole file. :param coll_id: The collection it belongs to. :param content_type: The type of the cotent data. :param status: The status of the content. :param priority: The priority as an integer. :param edge_name: The name of the replicating edge. :param edge_id: The id of the replicating edge. :param num_success: The number of successful access. :param num_failure; The number of failed access :param last_failed_at: The time of last failure. :param pfn_size: The size of physical file name. :param pfn: Physical file name. :raises DuplicatedObject: If an edge with the same name exists. :raises DatabaseException: If there is a database error. :returns: collection content id. """ if isinstance(content_type, str) or isinstance(content_type, unicode): content_type = ContentType.from_sym(str(content_type)) if isinstance(status, str) or isinstance(status, unicode): status = ContentStatus.from_sym(str(status)) if not edge_id: edge_id = get_edge_id(edge_name=edge_name, session=session) new_content = models.CollectionContent(scope=scope, name=name, min_id=min_id, max_id=max_id, coll_id=coll_id, content_type=content_type, status=status, priority=priority, edge_id=edge_id, num_success=num_success, num_failure=num_failure, last_failed_at=last_failed_at, pfn_size=pfn_size, pfn=pfn, object_metadata=object_metadata) try: new_content.save(session=session) except IntegrityError: raise exceptions.DuplicatedObject('Content %s:%s[%s:%s](at edge %s) already exists!' % (scope, name, min_id, max_id, edge_name)) except DatabaseError as error: raise exceptions.DatabaseException(error.args) return new_content.content_id
def get_content_best_match(scope, name, min_id=None, max_id=None, edge_name=None, edge_id=None, status=None, session=None): """ Get a collection content or raise a NoObject exception. :param scope: The scope of the collection data. :param name: The name of the collection data. :param min_id: The minimum id of the partial file, related to the whole file. :param max_id: The maximum id of the partial file, related to the whole file. :param edge_name: The name of the edge. :param edge_id: The id of the Edge :param session: The database session in use. :raises NoObject: If no edge is founded. :returns: Content model. """ try: if not edge_id and edge_name: edge_id = get_edge_id(edge_name=edge_name, session=session) if status and (isinstance(status, str) or isinstance(status, unicode)): status = ContentStatus.from_sym(str(status)) query = session.query(models.CollectionContent).filter_by(scope=scope, name=name) if status: query = query.filter_by(status=status) if edge_id: query = query.filter_by(edge_id=edge_id) if min_id is not None and max_id is not None: contents = query.filter(and_(CollectionContent.min_id <= min_id, CollectionContent.max_id >= max_id)).all() content = None for row in contents: if (not content) or (content['max_id'] - content['min_id'] > row['max_id'] - row['min_id']): content = row if content is None: raise sqlalchemy.orm.exc.NoResultFound() else: content = query.filter_by(content_type=ContentType.FILE).one() content['content_type'] = content.content_type content['status'] = content.status return content except sqlalchemy.orm.exc.NoResultFound: raise exceptions.NoObject('Content %s:%s[%s:%s](at edge %s) cannot be found' % (scope, name, min_id, max_id, edge_id))
def get_contents_by_edge(edge_name, edge_id=None, status=None, coll_id=None, content_type=None, collection_scope=None, collection_name=None, limit=None, session=None): """ Get a collection content or raise a NoObject exception. :param edge_name: The name of the edge. :param edge_id: The id of the Edge :param status: The status of the content. :param coll_id: The collection id. :param content_type: The tyep of the content. :param limit: Number to return limited items. :param session: The database session in use. :raises NoObject: If no edge is founded. :returns: Content model. """ try: if not edge_id: edge_id = get_edge_id(edge_name=edge_name, session=session) if not coll_id and (collection_scope and collection_name): coll_id = get_collection_id(collection_scope, collection_name) query = session.query(models.CollectionContent).filter_by(edge_id=edge_id) if status: if isinstance(status, str) or isinstance(status, unicode): status = ContentStatus.from_sym(str(status)) query = query.filter_by(status=status) if coll_id: query = query.filter_by(coll_id=coll_id) if content_type: query = query.filter_by(content_type=content_type) if limit: query = query.limit(limit) contents = query.all() for content in contents: content['content_type'] = content.content_type content['status'] = content.status return contents except sqlalchemy.orm.exc.NoResultFound: raise exceptions.NoObject('No contents at edge %s with status %s' % (edge_name, status))
def get_requests(status=None, edge_name=None, edge_id=None, session=None): """ Get requests. :param status: The status of the request data. :param edge_name: The name of the edge. :param edge_id: The id of the edge :param session: The database session in use. :raises NoObject: If no request is founded. :returns: Request models. """ try: if edge_name and not edge_id: edge_id = get_edge_id(edge_name) if status: if (isinstance(status, str) or isinstance(status, unicode)): status = RequestStatus.from_sym(status) if edge_id: requests = session.query(models.Request).filter_by( status=status, edge_id=edge_id).all() else: requests = session.query( models.Request).filter_by(status=status).all() else: if edge_id: requests = session.query( models.Request).filter_by(edge_id=edge_id).all() else: requests = session.query(models.Request).all() for request in requests: request['data_type'] = request.data_type request['granularity_type'] = request.granularity_type request['status'] = request.status return requests except sqlalchemy.orm.exc.NoResultFound: raise exceptions.NoObject('Cannot find request with status: %s' % (status))
def add_collection_replicas(scope, name, edge_name, coll_id=None, edge_id=None, status=CollectionReplicasStatus.NEW, transferring_files=0, replicated_files=0, num_active_requests=1, retries=0, session=None): """ Add a collection replicas. :param scope: The scope of the collection data. :param name: The name of the collection data. :param edge_name: The name of the replicating edge. :param edge_id: The id of the replicating edge. :param coll_id: Collection id. :param status: The status of the collection replicas. :param transferring_files: Number of transferring files. :param replicated_files: Number of replicated files. :param num_active_requests: Number of active requests. :param retries: Number of retries. :param session: The database session in use. :raises DuplicatedObject: If an edge with the same name exists. :raises DatabaseException: If there is a database error. """ if not coll_id: coll_id = get_collection_id(scope=scope, name=name, session=session) if not edge_id: edge_id = get_edge_id(edge_name=edge_name, session=session) try: if isinstance(status, str) or isinstance(status, unicode): status = CollectionReplicasStatus.from_sym(str(status)) new_collection_replicas = models.CollectionReplicas(coll_id=coll_id, edge_id=edge_id, status=status, transferring_files=transferring_files, replicated_files=replicated_files, num_active_requests=num_active_requests, retries=retries) new_collection_replicas.save(session=session) except IntegrityError as error: if 'a foreign key constraint fails' in str(error): raise exceptions.DatabaseException("Failed to add new collection %s:%s(at edge %s): %s" % (scope, name, edge_name, error)) else: raise exceptions.DuplicatedObject('Collection %s:%s(at edge %s) already exists! (%s)' % (scope, name, edge_name, error)) except DatabaseError as error: raise exceptions.DatabaseException(error.args)
def get_contents_statistics(edge_name, edge_id=None, coll_id=None, status=None, content_type=None, session=None): """ Get content statistics. :param edge_name: The name of the edge. :param edge_id: The id of the Edge :param coll_id: The collection id. :param status: The status of the content. :param content_type: The tyep of the content. :param session: The database session in use. :returns: dict. """ try: if not edge_id and edge_name: edge_id = get_edge_id(edge_name=edge_name, session=session) query = session.query(models.CollectionContent.edge_id, models.CollectionContent.coll_id, models.CollectionContent.content_type, models.CollectionContent.status, func.count(1).label('counter')) if edge_id: query = query.filter_by(edge_id=edge_id) if coll_id: query = query.filter_by(coll_id=coll_id) if status: if isinstance(status, str) or isinstance(status, unicode): status = ContentStatus.from_sym(str(status)) query = query.filter_by(status=status) if content_type: if isinstance(content_type, str) or isinstance(content_type, unicode): content_type = ContentType.from_sym(str(content_type)) query = query.filter_by(content_type=content_type) query = query.group_by(models.CollectionContent.edge_id, models.CollectionContent.coll_id, models.CollectionContent.content_type, models.CollectionContent.status) statistics = query.all() return statistics except sqlalchemy.orm.exc.NoResultFound: raise exceptions.NoObject('Failed to get statistics for edge %s and collection %s' % (edge_id, coll_id))
def get_contents_by_edge(edge_name, edge_id=None, status=None, coll_id=None, content_type=ContentType.FILE, session=None): """ Get a collection content or raise a NoObject exception. :param edge_name: The name of the edge. :param edge_id: The id of the Edge :param status: The status of the content. :param coll_id: The collection id. :param session: The database session in use. :raises NoObject: If no edge is founded. :returns: Content model. """ try: if not edge_id: edge_id = get_edge_id(edge_name=edge_name, session=session) if status: if isinstance(status, str) or isinstance(status, unicode): status = ContentStatus.from_sym(str(status)) if coll_id: contents = session.query(models.CollectionContent).filter_by(edge_id=edge_id, status=status, coll_id=coll_id, content_type=content_type).all() else: contents = session.query(models.CollectionContent).filter_by(edge_id=edge_id, status=status, content_type=content_type).all() else: if coll_id: contents = session.query(models.CollectionContent).filter_by(edge_id=edge_id, coll_id=coll_id, content_type=content_type).all() else: contents = session.query(models.CollectionContent).filter_by(edge_id=edge_id, content_type=content_type).all() for content in contents: content['content_type'] = content.content_type content['status'] = content.status return contents except sqlalchemy.orm.exc.NoResultFound: raise exceptions.NoObject('No contents at edge %s with status %s' % (edge_name, status))
def delete_content(scope, name, edge_name=None, edge_id=None, content_id=None, session=None): """ Delete a collection content or raise a NoObject exception. :param scope: The scope of the collection data. :param name: The name of the collection data. :param edge_name: The name of the edge. :param edge_id: The id of the Edge :param session: The database session in use. :raises NoObject: If no edge is founded. """ try: if content_id: session.query(models.CollectionContent).filter_by(content_id=content_id).delete() else: if not edge_id: edge_id = get_edge_id(edge_name=edge_name, session=session) session.query(models.CollectionContent).filter_by(scope=scope, name=name, edge_id=edge_id).delete() except sqlalchemy.orm.exc.NoResultFound: raise exceptions.NoObject('Contents %s:%s(at edge %s) cannot be found' % (scope, name, edge_id))
def add_contents(collection_scope, collection_name, edge_name, files, session=None): """ Add a collection contents. :param collection_scope: The scope of the collection. :param collection_name: The name of the collection. :param edge_name: The edge name. :param files: list of files. """ coll_id = get_collection_id(scope=collection_scope, name=collection_name, session=session) edge_id = get_edge_id(edge_name=edge_name, session=session) for file in files: try: add_content(scope=file['scope'], name=file['name'], min_id=file['min_id'], max_id=file['max_id'], coll_id=coll_id, content_type=file['content_type'], status=file['status'], priority=file['priority'], edge_id=edge_id, pfn_size=file['pfn_size'] if 'pfn_size' in file else 0, pfn=file['pfn'] if 'pfn' in file else None, object_metadata=file['object_metadata'] if 'object_metadata' in file else None, session=None) except exceptions.DuplicatedObject: pass
def get_content(scope, name, min_id=None, max_id=None, edge_name=None, edge_id=None, content_id=None, session=None): """ Get a collection content or raise a NoObject exception. :param scope: The scope of the collection data. :param name: The name of the collection data. :param min_id: The minimum id of the partial file, related to the whole file. :param max_id: The maximum id of the partial file, related to the whole file. :param edge_name: The name of the edge. :param edge_id: The id of the Edge :param session: The database session in use. :raises NoObject: If no edge is founded. :returns: Content model. """ try: if content_id: content = session.query(models.CollectionContent).filter_by(content_id=content_id).one() else: if not edge_id: edge_id = get_edge_id(edge_name=edge_name, session=session) if min_id is not None and max_id is not None: content = session.query(models.CollectionContent).filter_by(scope=scope, name=name, edge_id=edge_id, min_id=min_id, max_id=max_id).one() else: content_type = ContentType.FILE content = session.query(models.CollectionContent).filter_by(scope=scope, name=name, content_type=content_type, edge_id=edge_id).one() content['content_type'] = content.content_type content['status'] = content.status return content except sqlalchemy.orm.exc.NoResultFound: raise exceptions.NoObject('Content %s:%s[%s:%s](at edge %s) cannot be found' % (scope, name, min_id, max_id, edge_id))
def test_create_and_check_for_edge_core(self): """ Edge (CORE): Test the creation, query, and deletion of a Edge """ edge_name = 'test_edge_%s' % str(uuid()) edge_name = edge_name[:29] properties = { 'edge_type': 'EDGE', 'status': 'ACTIVE', 'is_independent': True, 'continent': 'US', 'country_name': 'US', 'region_code': 'US', 'city': 'Madison', 'longitude': '111111', 'latitude': '2222222', 'total_space': 0, 'used_space': 0, 'num_files': 0 } register_edge(edge_name, **properties) edge = get_edge(edge_name) assert_equal(edge.edge_name, edge_name) assert_equal(str(edge.edge_type), properties['edge_type']) assert_equal(str(edge.status), properties['status']) assert_equal(edge.is_independent, properties['is_independent']) assert_equal(edge.continent, properties['continent']) assert_equal(edge.country_name, properties['country_name']) assert_equal(edge.region_code, properties['region_code']) assert_equal(edge.city, properties['city']) assert_equal(edge.longitude, properties['longitude']) assert_equal(edge.latitude, properties['latitude']) assert_equal(edge.total_space, properties['total_space']) assert_equal(edge.used_space, properties['used_space']) assert_equal(edge.num_files, properties['num_files']) assert_equal(edge.edge_id, get_edge_id(edge_name)) with assert_raises(exceptions.NoObject): get_edge_id('not_exist_edge') with assert_raises(exceptions.NoObject): get_edge('not_exist_edge') with assert_raises(exceptions.DuplicatedObject): register_edge(edge_name) update_edge(edge_name, parameters={'status': 'LOSTHEARTBEAT'}) edge = get_edge(edge_name) assert_equal(str(edge.status), 'LOSTHEARTBEAT') edges = get_edges() assert_true(len(edges) >= 1) render_json(**edges[0]) edges[0].to_dict() edges = get_edges(status='LOSTHEARTBEAT') assert_true(len(edges) >= 1) delete_edge(edge_name) with assert_raises(exceptions.NoObject): get_edge(edge_name)