Beispiel #1
0
    def POST(self, edge_name):
        """ Create Edge with given name.
        HTTP Success:
            200 OK
        HTTP Error:
            400 Bad request
            500 Internal Error
        """
        json_data = data()
        args = ['edge_type', 'status', 'is_independent', 'continent', 'country_name', 'region_code', 'city',
                'longitude', 'latitude', 'total_space', 'used_space', 'num_files']

        try:
            parameters = {}
            json_data = json.loads(json_data)
            for key, value in json_data.items():
                if key in args:
                    parameters[key] = value
        except ValueError:
            raise self.generate_http_response(HTTP_STATUS_CODE.BadRequest, exc_cls=exceptions.BadRequest.__name__, exc_msg='Cannot decode json parameter dictionary')

        try:
            edge_id = register_edge(edge_name, **parameters)
        except exceptions.DuplicatedObject as error:
            raise self.generate_http_response(HTTP_STATUS_CODE.Conflict, exc_cls=error.__class__.__name__, exc_msg=error)
        except exceptions.ESSException as error:
            raise self.generate_http_response(HTTP_STATUS_CODE.InternalError, exc_cls=error.__class__.__name__, exc_msg=error)
        except Exception as error:
            print(error)
            print(format_exc())
            raise self.generate_http_response(HTTP_STATUS_CODE.InternalError, exc_cls=exceptions.CoreException.__name__, exc_msg=error)

        raise self.generate_http_response(HTTP_STATUS_CODE.OK, data={'edge_id': edge_id})
Beispiel #2
0
 def finish_tasks(self):
     """
     Finish processing the finished tasks, for example, update db status.
     """
     while not self.finished_tasks.empty():
         task = self.finished_tasks.get()
         self.logger.info("Main thread finishing task: %s" % task)
         try:
             parameters = {
                 'edge_type': self.edge_type,
                 'status': EdgeStatus.ACTIVE,
                 'is_independent': self.is_independent,
                 'continent': self.continent,
                 'country_name': self.country_name,
                 'region_code': self.region_code,
                 'city': self.city,
                 'longitude': self.longitude,
                 'latitude': self.latitude,
                 'total_space': self.total_space,
                 'used_space': self.used_space,
                 'num_files': self.num_files
             }
             self.logger.info("Updating edge %s with parameters: %s" %
                              (self.get_resouce_name(), parameters))
             update_edge(edge_name=self.get_resouce_name(),
                         parameters=parameters)
         except NoObject as error:
             self.logger.info(
                 "Edge %s doesn't exist(%s), will register it" %
                 (self.get_resouce_name(), error))
             register_edge(self.get_resouce_name(),
                           edge_type=self.edge_type,
                           status=EdgeStatus.ACTIVE,
                           is_independent=self.is_independent,
                           continent=self.continent,
                           country_name=self.country_name,
                           region_code=self.region_code,
                           city=self.city,
                           longitude=self.longitude,
                           latitude=self.latitude,
                           total_space=self.total_space,
                           used_space=self.used_space,
                           num_files=self.num_files)
Beispiel #3
0
    def test_create_and_check_for_collection(self):
        """ Catalog (CORE): Test the creation, query, and deletion of a Collection """

        edge_name = 'test_rse_%s' % str(uuid())
        edge_name = edge_name[:29]

        properties_edge = {
            'edge_type': 'EDGE',
            'status': 'ACTIVE',
            'is_independent': True,
            'continent': 'US',
            'country_name': 'US',
            'region_code': 'US',
            'city': 'Madison',
            'longitude': '111111',
            'latitude': '2222222',
            'total_space': 0,
            'used_space': 0,
            'num_files': 0
        }
        register_edge(edge_name, **properties_edge)

        properties = {
            'scope': 'test_scope',
            'name': 'test_name_%s' % str(uuid()),
            'collection_type': 'DATASET',
            'coll_size': 100,
            'global_status': 'NEW',
            'total_files': 100,
            'num_replicas': 10,
            'coll_metadata': {
                'attr1': 'abd',
                'attr2': 1
            }
        }
        collection_id = add_collection(**properties)
        collection = get_collection(properties['scope'], properties['name'])
        assert_equal(collection_id, collection.coll_id)

        json.dumps(collection.to_dict())

        assert_equal(collection.scope, properties['scope'])
        assert_equal(collection.name, properties['name'])
        assert_equal(str(collection.collection_type),
                     properties['collection_type'])
        assert_equal(str(collection.global_status),
                     properties['global_status'])
        assert_equal(collection.coll_size, properties['coll_size'])
        assert_equal(collection.total_files, properties['total_files'])
        assert_equal(collection.num_replicas, properties['num_replicas'])
        # assert_equal(collection.coll_metadata, properties['coll_metadata'])

        with assert_raises(exceptions.DuplicatedObject):
            add_collection(**properties)

        with assert_raises(exceptions.NoObject):
            get_collection(properties['scope'], 'not_exist_name')

        update_collection(scope=properties['scope'],
                          name=properties['name'],
                          parameters={'global_status': 'UNAVAILABLE'})
        collection = get_collection(properties['scope'],
                                    properties['name'],
                                    coll_id=collection.coll_id)
        assert_equal(str(collection.global_status), 'UNAVAILABLE')

        delete_collection(scope=properties['scope'],
                          name=properties['name'],
                          coll_id=collection.coll_id)

        with assert_raises(exceptions.NoObject):
            get_collection(scope=properties['scope'],
                           name=properties['name'],
                           coll_id=collection.coll_id)

        delete_edge(edge_name)
Beispiel #4
0
    def test_create_and_check_for_content(self):
        """ Catalog (CORE): Test the creation, query, and deletion of a Content """
        edge_name = 'test_rse_%s' % str(uuid())
        edge_name = edge_name[:29]

        properties_edge = {
            'edge_type': 'EDGE',
            'status': 'ACTIVE',
            'is_independent': True,
            'continent': 'US',
            'country_name': 'US',
            'region_code': 'US',
            'city': 'Madison',
            'longitude': '111111',
            'latitude': '2222222',
            'total_space': 0,
            'used_space': 0,
            'num_files': 0
        }
        edge_id = register_edge(edge_name, **properties_edge)

        properties_collection = {
            'scope': 'test_scope',
            'name': 'test_name_%s' % str(uuid()),
            'collection_type': 'DATASET',
            'coll_size': 100,
            'global_status': 'NEW',
            'total_files': 100,
            'num_replicas': 10,
            'coll_metadata': {
                'attr1': 'abd',
                'attr2': 1
            },
        }

        collection_id = add_collection(**properties_collection)

        properties = {
            'scope': 'test_scope',
            'name': 'test_name_%s' % str(uuid()),
            'min_id': 0,
            'max_id': 7,
            'coll_id': collection_id,
            'content_type': 'PARTIAL',
            'edge_id': edge_id,
            'status': 'NEW',
            'priority': 100,
            'num_success': 1,
            'num_failure': 0,
            'last_failed_at': None,
            'pfn_size': 1,
            'pfn': 'adss',
            'object_metadata': {
                'size': 1,
                'md5': '1322232',
                'adler32': '12345678'
            }
        }

        content_id = add_content(**properties)
        content = get_content(properties['scope'],
                              properties['name'],
                              properties['min_id'],
                              properties['max_id'],
                              edge_name=edge_name)
        assert_equal(content_id, content.content_id)

        json.dumps(content.to_dict())

        content = get_content(properties['scope'],
                              properties['name'],
                              properties['min_id'],
                              properties['max_id'],
                              edge_id=edge_id)
        assert_equal(content_id, content.content_id)

        assert_equal(content.scope, properties['scope'])
        assert_equal(content.name, properties['name'])
        assert_equal(content.min_id, properties['min_id'])
        assert_equal(content.max_id, properties['max_id'])
        assert_equal(content.coll_id, properties['coll_id'])
        assert_equal(str(content.content_type), properties['content_type'])
        assert_equal(content.edge_id, properties['edge_id'])
        assert_equal(str(content.status), properties['status'])
        assert_equal(content.priority, properties['priority'])
        assert_equal(content.num_success, properties['num_success'])
        assert_equal(content.num_failure, properties['num_failure'])
        assert_equal(content.last_failed_at, properties['last_failed_at'])
        assert_equal(content.pfn_size, properties['pfn_size'])
        assert_equal(content.pfn, properties['pfn'])

        with assert_raises(exceptions.DuplicatedObject):
            add_content(**properties)

        with assert_raises(exceptions.NoObject):
            get_content(properties['scope'],
                        properties['name'],
                        properties['min_id'],
                        properties['max_id'],
                        edge_name='Not_exist_edge')

        with assert_raises(exceptions.NoObject):
            get_content(properties['scope'],
                        properties['name'],
                        properties['min_id'] + 1,
                        properties['max_id'] + 1,
                        edge_name='Not_exist_edge')

        with assert_raises(exceptions.NoObject):
            get_content(properties['scope'],
                        properties['name'],
                        edge_name=edge_name)

        update_content(scope=properties['scope'],
                       name=properties['name'],
                       min_id=properties['min_id'],
                       max_id=properties['max_id'],
                       edge_id=edge_id,
                       parameters={'status': 'UNAVAILABLE'})

        content = get_content(properties['scope'],
                              properties['name'],
                              content_id=content_id)
        assert_equal(str(content.status), 'UNAVAILABLE')

        content = get_content_best_match(properties['scope'],
                                         properties['name'],
                                         properties['min_id'] + 1,
                                         properties['max_id'] - 1,
                                         edge_id=edge_id)
        assert_equal(content_id, content.content_id)

        delete_content(scope=properties['scope'],
                       name=properties['name'],
                       content_id=content_id)

        with assert_raises(exceptions.NoObject):
            get_content(scope=properties['scope'],
                        name=properties['name'],
                        content_id=content_id)

        delete_collection(scope=properties_collection['scope'],
                          name=properties_collection['name'],
                          coll_id=collection_id)
        delete_edge(edge_name)
Beispiel #5
0
    def test_create_and_check_for_collection_replicas(self):
        """ Catalog (CORE): Test the creation, query, and deletion of a Collection replicas"""

        edge_name = 'test_rse_%s' % str(uuid())
        edge_name = edge_name[:29]

        properties_edge = {
            'edge_type': 'EDGE',
            'status': 'ACTIVE',
            'is_independent': True,
            'continent': 'US',
            'country_name': 'US',
            'region_code': 'US',
            'city': 'Madison',
            'longitude': '111111',
            'latitude': '2222222',
            'total_space': 0,
            'used_space': 0,
            'num_files': 0
        }
        edge_id = register_edge(edge_name, **properties_edge)

        properties_coll = {
            'scope': 'test_scope',
            'name': 'test_name_%s' % str(uuid()),
            'collection_type': 'DATASET',
            'coll_size': 100,
            'global_status': 'NEW',
            'total_files': 100,
            'num_replicas': 10,
            'coll_metadata': {
                'attr1': 'abd',
                'attr2': 1
            },
        }
        collection_id = add_collection(**properties_coll)

        properties = {
            'status': 'NEW',
            'transferring_files': 0,
            'replicated_files': 0,
            'num_active_requests': 1,
            'retries': 0
        }

        add_collection_replicas(properties_coll['scope'],
                                properties_coll['name'], edge_name,
                                **properties)
        coll_replicas = get_collection_replicas(properties_coll['scope'],
                                                properties_coll['name'],
                                                edge_name)

        assert_equal(collection_id, coll_replicas.coll_id)
        assert_equal(edge_id, coll_replicas.edge_id)

        assert_equal(str(coll_replicas.status), properties['status'])
        assert_equal(coll_replicas.transferring_files,
                     properties['transferring_files'])
        assert_equal(coll_replicas.replicated_files,
                     properties['replicated_files'])
        assert_equal(coll_replicas.num_active_requests,
                     properties['num_active_requests'])
        assert_equal(coll_replicas.retries, properties['retries'])

        with assert_raises(exceptions.DuplicatedObject):
            add_collection_replicas(properties_coll['scope'],
                                    properties_coll['name'], edge_name,
                                    **properties)

        with assert_raises(exceptions.NoObject):
            get_collection_replicas(properties_coll['scope'], 'not_exist_name',
                                    edge_name)

        coll_replicas1 = get_collection_replicas(properties_coll['scope'],
                                                 properties_coll['name'],
                                                 edge_name,
                                                 edge_id=edge_id)
        assert_equal(coll_replicas1.coll_id, coll_replicas.coll_id)
        assert_equal(coll_replicas1.edge_id, coll_replicas.edge_id)

        coll_replicas1 = get_collection_replicas(properties_coll['scope'],
                                                 properties_coll['name'],
                                                 edge_name,
                                                 coll_id=coll_replicas.coll_id,
                                                 edge_id=edge_id)
        assert_equal(coll_replicas1.coll_id, coll_replicas.coll_id)
        assert_equal(coll_replicas1.edge_id, coll_replicas.edge_id)

        update_collection_replicas(scope=properties_coll['scope'],
                                   name=properties_coll['name'],
                                   edge_name=edge_name,
                                   parameters={'status': 'UNAVAILABLE'})
        coll_replicas = get_collection_replicas(properties_coll['scope'],
                                                properties_coll['name'],
                                                edge_name)
        assert_equal(str(coll_replicas.status), 'UNAVAILABLE')

        delete_collection_replicas(properties_coll['scope'],
                                   properties_coll['name'], edge_name)

        with assert_raises(exceptions.NoObject):
            get_collection_replicas(properties_coll['scope'],
                                    properties_coll['name'], edge_name)

        delete_edge(edge_name)
Beispiel #6
0
    def test_create_and_check_for_edge_core(self):
        """ Edge (CORE): Test the creation, query, and deletion of a Edge """
        edge_name = 'test_edge_%s' % str(uuid())
        edge_name = edge_name[:29]
        properties = {
            'edge_type': 'EDGE',
            'status': 'ACTIVE',
            'is_independent': True,
            'continent': 'US',
            'country_name': 'US',
            'region_code': 'US',
            'city': 'Madison',
            'longitude': '111111',
            'latitude': '2222222',
            'total_space': 0,
            'used_space': 0,
            'num_files': 0
        }
        register_edge(edge_name, **properties)
        edge = get_edge(edge_name)

        assert_equal(edge.edge_name, edge_name)
        assert_equal(str(edge.edge_type), properties['edge_type'])
        assert_equal(str(edge.status), properties['status'])
        assert_equal(edge.is_independent, properties['is_independent'])
        assert_equal(edge.continent, properties['continent'])
        assert_equal(edge.country_name, properties['country_name'])
        assert_equal(edge.region_code, properties['region_code'])
        assert_equal(edge.city, properties['city'])
        assert_equal(edge.longitude, properties['longitude'])
        assert_equal(edge.latitude, properties['latitude'])
        assert_equal(edge.total_space, properties['total_space'])
        assert_equal(edge.used_space, properties['used_space'])
        assert_equal(edge.num_files, properties['num_files'])

        assert_equal(edge.edge_id, get_edge_id(edge_name))

        with assert_raises(exceptions.NoObject):
            get_edge_id('not_exist_edge')

        with assert_raises(exceptions.NoObject):
            get_edge('not_exist_edge')

        with assert_raises(exceptions.DuplicatedObject):
            register_edge(edge_name)

        update_edge(edge_name, parameters={'status': 'LOSTHEARTBEAT'})
        edge = get_edge(edge_name)
        assert_equal(str(edge.status), 'LOSTHEARTBEAT')

        edges = get_edges()
        assert_true(len(edges) >= 1)
        render_json(**edges[0])
        edges[0].to_dict()

        edges = get_edges(status='LOSTHEARTBEAT')
        assert_true(len(edges) >= 1)

        delete_edge(edge_name)

        with assert_raises(exceptions.NoObject):
            get_edge(edge_name)