Esempio n. 1
0
    def get_datasets_at_rse(rse, prefix=None):
        """

        :param rse: The RSE name
        :param prefix: Character(s) to restrict the dataset search
        :return: a dictionary with <dataset name>: <number of files>
        """

        filters = {'scope': 'cms', 'did_type': DIDType.DATASET}
        if prefix:
            filters['name'] = '/' + prefix + '*'

        with monitor.record_timer_block('cms_sync.time_rse_datasets'):
            all_datasets = [
                dataset['name']
                for dataset in list_datasets_per_rse(rse=rse, filters=filters)
            ]

            for dataset in all_datasets:
                datasets = {
                    dataset: ds['available_length']
                    for ds in list_dataset_replicas(
                        scope='cms', name=dataset, deep=True)
                    if ds['rse'] == rse
                }

            # datasets = {dataset['name']: dataset['available_length']
            #             for dataset in list_datasets_per_rse(rse=rse, filters=filters, deep=True)}

        return datasets
Esempio n. 2
0
    def get(self, rse):
        """
        List dataset replicas per RSE.

        .. :quickref: ReplicasRSE; List dataset replicas per RSE.

        :resheader Content-Type: application/x-json-stream
        :status 200: OK.
        :status 401: Invalid auth token.
        :status 406: Not Acceptable.
        :status 500: Internal Error.
        :returns: A dictionary containing all replicas on the RSE.
        """
        try:
            data = ""
            for row in list_datasets_per_rse(rse=rse,
                                             vo=request.environ.get('vo')):
                data += dumps(row, cls=APIEncoder) + '\n'
            return Response(data, content_type='application/x-json-stream')
        except RucioException as error:
            return generate_http_error_flask(500, error.__class__.__name__,
                                             error.args[0])
        except Exception as error:
            print(format_exc())
            return error, 500
Esempio n. 3
0
    def get_datasets_at_rse(rse, prefix=None):
        """

        :param rse: The RSE name
        :param prefix: Character(s) to restrict the dataset search
        :return: a dictionary with <dataset name>: <number of files>
        """

        filters = {'scope': 'cms', 'did_type': DIDType.DATASET}
        if prefix:
            filters['name'] = '/' + prefix + '*'

        account = SYNC_ACCOUNT_FMT % rse.lower()
        rule_filters = {'account': account, 'scope': 'cms', 'did_type': DIDType.DATASET}

        with monitor.record_timer_block('cms_sync.time_rse_datasets'):
            synced_ds = {item['name'] for item in list_replication_rules(filters=rule_filters)
                         if item['expires_at'] is None and (prefix is None or item['name'].startswith('/' + prefix))}

            all_datasets = [dataset['name'] for dataset in list_datasets_per_rse(rse=rse, filters=filters)]

            logging.info('Getting all datasets at %s with prefix %s' % (rse, prefix))

            datasets = {}

            for dataset in all_datasets:
                if dataset in synced_ds:
                    for ds in list_dataset_replicas(scope='cms', name=dataset, deep=True):
                        if ds['rse'] == rse:
                            datasets.update({dataset: ds['available_length']})

        return datasets
Esempio n. 4
0
    def GET(self, rse):
        """
        List dataset replicas replicas.

        HTTP Success:
            200 OK

        HTTP Error:
            401 Unauthorized
            500 InternalError

        :returns: A dictionary containing all replicas on the RSE.
        """
        header('Content-Type', 'application/x-json-stream')
        try:
            for row in list_datasets_per_rse(rse=rse):
                yield dumps(row, cls=APIEncoder) + '\n'
        except RucioException, e:
            raise generate_http_error(500, e.__class__.__name__, e.args[0][0])
Esempio n. 5
0
    def GET(self, rse):
        """
        List dataset replicas replicas.

        HTTP Success:
            200 OK

        HTTP Error:
            401 Unauthorized
            406 Not Acceptable
            500 InternalError

        :returns: A dictionary containing all replicas on the RSE.
        """
        header('Content-Type', 'application/x-json-stream')
        try:
            for row in list_datasets_per_rse(rse=rse, vo=ctx.env.get('vo')):
                yield dumps(row, cls=APIEncoder) + '\n'
        except RucioException as error:
            raise generate_http_error(500, error.__class__.__name__, error.args[0])
        except Exception as error:
            print(format_exc())
            raise InternalError(error)
Esempio n. 6
0
 def generate(vo):
     for row in list_datasets_per_rse(rse=rse, vo=vo):
         yield dumps(row, cls=APIEncoder) + '\n'