示例#1
0
    def add_replication_rule_with_defaults(self,dids, copies, rse_expression, account):

        """
        Add replication rule requires one to send all the values. Add a list of defaults.
        If true options are required, move them into the parameter list.

        :param dids: List of dids (scope/name dictionary)
        :param copies: Number of copies
        :param rse_expression: RSE expression
        :param account: Account for the rule
        :return: None
        """

        (grouping, weight, lifetime, locked, subscription_id, source_replica_expression,  notify,
         purge_replicas, ignore_availability, comment, ask_approval, asynchronous, priority, split_container) = (
            'DATASET', None, None, False, None, None,  None, False, False, None, False, False, 3, False)

        activity = 'Data Consolidation'
        meta = json.dumps({"phedex_group": self.group, "phedex_custodial": self.custodial})

        add_replication_rule(dids=dids, copies=copies, rse_expression=rse_expression, account=account,
                             grouping=grouping, weight=weight, lifetime=lifetime, locked=locked,
                             subscription_id=subscription_id, source_replica_expression=source_replica_expression,
                             activity=activity, notify=notify, purge_replicas=purge_replicas,
                             ignore_availability=ignore_availability, comment=comment, ask_approval=ask_approval,
                             asynchronous=asynchronous, priority=priority, split_container=split_container, meta=meta,
                             issuer=account)
def populateDB():
    listrses = list_rses({'T1': '1'})
    print len(listrses), listrses
    # listrses = list_rses()
    # print len(listrses), listrses
    # sys.exit()
    account = 'root'
    project = 'mc12_8TeV'

    dictDistrib = [{'datatype': 'HITS', 'prodstep': 'merge', 'nbfiles': 302, 'totfilesize': 225394185112, 'nbreplicas': 1}, {'datatype': 'HITS', 'prodstep': 'simul', 'nbfiles': 620, 'totfilesize': 97930909866, 'nbreplicas': 1},
                   {'datatype': 'EVNT', 'prodstep': 'evgen', 'nbfiles': 324, 'totfilesize': 7809298802, 'nbreplicas': 3}, {'datatype': 'AOD', 'prodstep': 'merge', 'nbfiles': 52, 'totfilesize': 106942334943, 'nbreplicas': 4},
                   {'datatype': 'AOD', 'prodstep': 'recon', 'nbfiles': 858, 'totfilesize': 182186965627, 'nbreplicas': 1}]

    for d in dictDistrib:
        for day in xrange(0, 180):
            for i in xrange(0, 30):
                scope = project
                prod_step = d['prodstep']
                datatype = d['datatype']
                nbfiles = int(d['nbfiles'])
                filesize = int(int(d['totfilesize'])/float(nbfiles))
                nbfiles = int(random.gauss(nbfiles, nbfiles/10))
                filesize = int(random.gauss(filesize, filesize/10))
                nbreplicas = int(d['nbreplicas'])
                dataset_meta = {'project': project, 'stream_name': 'dummy', 'prod_step': prod_step, 'datatype': datatype}
                source_rses = []
                if nbreplicas:
                    iter = 0
                    while (len(source_rses) != nbreplicas and iter != 100):
                        rnd_site = random.choice(listrses)
                        iter += 1
                        if rnd_site not in source_rses:
                            source_rses.append(rnd_site)

                    try:
                        dsn = '%s.%s.%s.%i.%i' % (project, prod_step, datatype, day, i)
                        print '%i Creating %s with %i files of size %i located at %i sites' % (i, dsn, nbfiles, filesize, len(source_rses))
                        add_identifier(scope=scope, name=dsn, type='dataset', issuer=account, statuses={'monotonic': True}, meta=dataset_meta)
                        files = ['file_%s' % uuid() for i in xrange(nbfiles)]
                        listfiles = []
                        for file in files:
                            listfiles.append({'scope': scope, 'name': file, 'size': filesize})
                            for source_rse in source_rses:
                                add_file_replica(source_rse, scope, file, filesize, issuer=account)
                        attach_identifier(scope, name=dsn, dids=listfiles, issuer=account)
                        for source_rse in source_rses:
                            try:
                                add_replication_rule(dids=[{'scope': scope, 'name': dsn}], account=account, copies=1, rse_expression=source_rse,
                                                     grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None, issuer='root')
                            except InvalidReplicationRule, e:
                                print e
                    except RucioException, e:
                        print e
示例#3
0
def populateDB():
    listrses = list_rses({'T1': '1'})
    print(len(listrses), listrses)
    # listrses = list_rses()
    # print len(listrses), listrses
    # sys.exit()
    account = 'root'
    project = 'mc12_8TeV'

    dictDistrib = [{'datatype': 'HITS', 'prodstep': 'merge', 'nbfiles': 302, 'totfilesize': 225394185112, 'nbreplicas': 1}, {'datatype': 'HITS', 'prodstep': 'simul', 'nbfiles': 620, 'totfilesize': 97930909866, 'nbreplicas': 1},
                   {'datatype': 'EVNT', 'prodstep': 'evgen', 'nbfiles': 324, 'totfilesize': 7809298802, 'nbreplicas': 3}, {'datatype': 'AOD', 'prodstep': 'merge', 'nbfiles': 52, 'totfilesize': 106942334943, 'nbreplicas': 4},
                   {'datatype': 'AOD', 'prodstep': 'recon', 'nbfiles': 858, 'totfilesize': 182186965627, 'nbreplicas': 1}]

    for d in dictDistrib:
        for day in range(180):
            for i in range(30):
                scope = project
                prod_step = d['prodstep']
                datatype = d['datatype']
                nbfiles = int(d['nbfiles'])
                filesize = int(int(d['totfilesize']) / float(nbfiles))
                nbfiles = int(random.gauss(nbfiles, nbfiles / 10))
                filesize = int(random.gauss(filesize, filesize / 10))
                nbreplicas = int(d['nbreplicas'])
                dataset_meta = {'project': project, 'stream_name': 'dummy', 'prod_step': prod_step, 'datatype': datatype}
                source_rses = []
                if nbreplicas:
                    iter = 0
                    while (len(source_rses) != nbreplicas and iter != 100):
                        rnd_site = random.choice(listrses)
                        iter += 1
                        if rnd_site not in source_rses:
                            source_rses.append(rnd_site)

                    try:
                        dsn = '%s.%s.%s.%i.%i' % (project, prod_step, datatype, day, i)
                        print('%i Creating %s with %i files of size %i located at %i sites' % (i, dsn, nbfiles, filesize, len(source_rses)))
                        add_identifier(scope=scope, name=dsn, type='dataset', issuer=account, statuses={'monotonic': True}, meta=dataset_meta)
                        files = ['file_%s' % uuid() for i in range(nbfiles)]
                        listfiles = []
                        for file in files:
                            listfiles.append({'scope': scope, 'name': file, 'size': filesize})
                            for source_rse in source_rses:
                                add_file_replica(source_rse, scope, file, filesize, issuer=account)
                        attach_identifier(scope, name=dsn, dids=listfiles, issuer=account)
                        for source_rse in source_rses:
                            try:
                                add_replication_rule(dids=[{'scope': scope, 'name': dsn}], account=account, copies=1, rse_expression=source_rse,
                                                     grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None, issuer='root')
                            except InvalidReplicationRule as e:
                                print(e)
                    except RucioException as e:
                        print(e)
示例#4
0
def populateDB(filename=None):
    listrses = list_rses(filters={'deterministic': 1})
    listrses = map(lambda x: x['rse'], listrses)
    account = 'root'

    pdf = generatePDF()

    # Generate 200000 datasets according to the dataset distribution
    for index in range(20000):
        scope_nb = getRandomScope(pdf)
        project = 'user.user%i' % (scope_nb)
        scope = 'user.user%i' % (scope_nb)
        account = 'user%i' % (scope_nb)
        print(scope)
        nbfiles = 53
        filesize = 78000000
        uid = uuid()
        dsn = '%s.%s' % (project, uid)
        rnd_site = random.choice(listrses)
        print('%i Creating %s with %i files of size %i located at %s' %
              (index, dsn, nbfiles, filesize, rnd_site))
        add_identifier(scope=scope,
                       name=dsn,
                       type='dataset',
                       issuer=account,
                       statuses={'monotonic': True})
        monitor.record(timeseries='dbfiller.addnewdataset', delta=1)
        files = ['file_%s' % uuid() for i in range(nbfiles)]
        listfiles = []
        for file in files:
            listfiles.append({'scope': scope, 'name': file, 'size': filesize})
            add_file_replica(rnd_site, scope, file, filesize, issuer=account)
        monitor.record(timeseries='dbfiller.addreplicas', delta=nbfiles)
        attach_identifier(scope, name=dsn, dids=listfiles, issuer=account)
        monitor.record(timeseries='dbfiller.addnewfile', delta=nbfiles)
        try:
            add_replication_rule(dids=[{
                'scope': scope,
                'name': dsn
            }],
                                 account=account,
                                 copies=1,
                                 rse_expression=rnd_site,
                                 grouping='DATASET',
                                 weight=None,
                                 lifetime=None,
                                 locked=False,
                                 subscription_id=None,
                                 issuer=account)
            monitor.record(timeseries='dbfiller.addreplicationrules', delta=1)
        except InvalidReplicationRule as e:
            print(e)
示例#5
0
    def test_api_subscription(self):
        """ SUBSCRIPTION (API): Test external representation of subscriptions """

        sub = 'ext_' + generate_uuid()
        did = 'ext_' + generate_uuid()
        new_acc_name = ''.join(random.choice(string.ascii_lowercase) for x in range(10))
        new_scope_name = ''.join(random.choice(string.ascii_lowercase) for x in range(10))
        add_account(new_acc_name, 'USER', '*****@*****.**', 'root', **self.new_vo)
        add_scope(new_scope_name, new_acc_name, 'root', **self.new_vo)
        api_acc_lim.set_local_account_limit(new_acc_name, self.rse3_name, 10, 'root', **self.new_vo)
        api_acc_lim.set_local_account_limit(new_acc_name, self.rse4_name, 10, 'root', **self.new_vo)
        add_did(new_scope_name, did, 'DATASET', 'root', account=new_acc_name, rse=self.rse3_name, **self.new_vo)

        sub_id = add_subscription(sub, new_acc_name, {'account': [new_acc_name], 'scope': [new_scope_name]},
                                  [{'copies': 1, 'rse_expression': self.rse3_name, 'weight': 0, 'activity': 'User Subscriptions',
                                    'source_replica_expression': self.rse4_name}],
                                  '', False, 0, 0, 3, 'root', **self.new_vo)
        add_replication_rule(dids=[{'scope': new_scope_name, 'name': did}], copies=1, rse_expression=self.rse3_name, weight=None,
                             lifetime=180, grouping='DATASET', account=new_acc_name, locked=False, subscription_id=sub_id,
                             source_replica_expression=self.rse4_name, activity='User Subscriptions', notify=None,
                             purge_replicas=False, ignore_availability=False, comment='', ask_approval=False, asynchronous=False,
                             priority=0, split_container=False, meta='', issuer='root', **self.new_vo)

        out = list_subscriptions(sub, **self.new_vo)
        out = list(out)
        assert_not_equal(0, len(out))
        assert_in(sub_id, [o['id'] for o in out])
        for o in out:
            if o['id'] == sub_id:
                assert_equal(o['account'], new_acc_name)
                rules = loads(o['replication_rules'])[0]
                assert_equal(rules['rse_expression'], self.rse3_name)
                assert_equal(rules['source_replica_expression'], self.rse4_name)
                fil = loads(o['filter'])
                assert_equal(fil['account'], [new_acc_name])
                assert_equal(fil['scope'], [new_scope_name])

        out = list_subscription_rule_states(sub, **self.new_vo)
        out = list(out)
        assert_not_equal(0, len(out))
        for o in out:
            assert_equal(o.account, new_acc_name)

        out = get_subscription_by_id(sub_id, **self.new_vo)
        assert_equal(out['account'], new_acc_name)
        rules = loads(out['replication_rules'])[0]
        assert_equal(rules['rse_expression'], self.rse3_name)
        assert_equal(rules['source_replica_expression'], self.rse4_name)
        fil = loads(out['filter'])
        assert_equal(fil['account'], [new_acc_name])
        assert_equal(fil['scope'], [new_scope_name])
示例#6
0
def populateDB(filename=None):
    listrses = list_rses(filters={'deterministic': 1})
    listrses = map(lambda x: x['rse'], listrses)
    account = 'root'

    pdf = generatePDF()

    # Generate 200000 datasets according to the dataset distribution
    for index in xrange(0, 20000):
        scope_nb = getRandomScope(pdf)
        project = 'user.user%i' % (scope_nb)
        scope = 'user.user%i' % (scope_nb)
        account = 'user%i' % (scope_nb)
        print scope
        nbfiles = 53
        filesize = 78000000
        uid = uuid()
        dsn = '%s.%s' % (project, uid)
        rnd_site = random.choice(listrses)
        print '%i Creating %s with %i files of size %i located at %s' % (index, dsn, nbfiles, filesize, rnd_site)
        add_identifier(scope=scope, name=dsn, type='dataset', issuer=account, statuses={'monotonic': True})
        monitor.record(timeseries='dbfiller.addnewdataset',  delta=1)
        files = ['file_%s' % uuid() for i in xrange(nbfiles)]
        listfiles = []
        for file in files:
            listfiles.append({'scope': scope, 'name': file, 'size': filesize})
            add_file_replica(rnd_site, scope, file, filesize, issuer=account)
        monitor.record(timeseries='dbfiller.addreplicas',  delta=nbfiles)
        attach_identifier(scope, name=dsn, dids=listfiles, issuer=account)
        monitor.record(timeseries='dbfiller.addnewfile',  delta=nbfiles)
        try:
            add_replication_rule(dids=[{'scope': scope, 'name': dsn}], account=account, copies=1, rse_expression=rnd_site,
                                 grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None, issuer=account)
            monitor.record(timeseries='dbfiller.addreplicationrules',  delta=1)
        except InvalidReplicationRule, e:
            print e
def populateDB(filename=None):
    listrses = list_rses(filters={'deterministic': 1})
    print listrses
    listrses = map(lambda x: x['rse'], listrses)
    account = 'root'
    nbDatasets = 0
    list = []
    dictDistrib = {}

    if not filename:
        if os.getenv('RUCIO_HOME'):
            filename = os.getenv(
                'RUCIO_HOME') + '/etc/data12_8TeV_distribution.txt'
        else:
            filename = '/opt/rucio/etc/data12_8TeV_distribution.txt'

    # Get the dataset distribution
    f = open(filename, 'r')
    for line in f:
        if not line.startswith('NBDATASETS'):
            line = line.rstrip('\n')
            strsplit = line.split()
            dictDistrib[(nbDatasets,
                         nbDatasets + int(strsplit[0]))] = strsplit[1:]
            nbDatasets += int(strsplit[0])
            list.append([
                nbDatasets,
            ] + strsplit[1:])
    f.close()

    # Generate 200000 datasets according to the dataset distribution
    for i in xrange(0, 200000):
        rnd = random.random() * nbDatasets
        for lower, upper in dictDistrib:
            if (rnd > lower) and (rnd < upper):
                project = dictDistrib[lower, upper][0]
                scope = project
                run_number = random.randint(0, 1000000)
                tag = random.randint(0, 10000)
                stream_name = dictDistrib[lower, upper][1]
                prod_step = dictDistrib[lower, upper][2]
                datatype = dictDistrib[lower, upper][3]
                provenance = dictDistrib[lower, upper][4]
                group = dictDistrib[lower, upper][5]
                if group == '/atlas/role=production':
                    # account = 'atlasprod'
                    account = 'panda'
                    if provenance == 'T0':
                        group = 'tier0'
                        account = 'tier0'
                    else:
                        group = 'panda'
                else:
                    # account = dictGroups[group]
                    account = 'panda'
                    scope = 'group.%s' % (dictGroups[group])
                    group = dictGroups[group]
                nbfiles = int(dictDistrib[lower, upper][6])
                filesize = int(
                    int(dictDistrib[lower, upper][7]) / float(nbfiles))
                nbreplicas = int(dictDistrib[lower, upper][8])
                if group == 'panda' or group == 'tier0':
                    dataset_meta = {
                        'project': project,
                        'run_number': run_number,
                        'stream_name': stream_name,
                        'prod_step': prod_step,
                        'datatype': datatype,
                        'provenance': provenance,
                        'group': group
                    }
                else:
                    campaign = int(tag / 1000.)
                    dataset_meta = {
                        'project': project,
                        'run_number': run_number,
                        'stream_name': stream_name,
                        'prod_step': prod_step,
                        'datatype': datatype,
                        'provenance': provenance,
                        'group': group,
                        'campaign': '%s_repro_%i' % (group, campaign)
                    }
                source_rses = []
                if nbreplicas:
                    iter = 0
                    while (len(source_rses) != nbreplicas and iter != 100):
                        rnd_site = random.choice(listrses)
                        iter += 1
                        if (rnd_site not in source_rses):
                            source_rses.append(rnd_site)

                    run_number_string = str(run_number)
                    run_number_string = run_number_string.rjust(7, '0')
                    dsn = '%s.%s.%s.%s.%s.%s' % (project, run_number_string,
                                                 stream_name, prod_step,
                                                 datatype, tag)
                    print '%i Creating %s:%s with %i files of size %i located at %i sites' % (
                        i, scope, dsn, nbfiles, filesize, len(source_rses))
                    stime1 = time.time()
                    add_identifier(scope=scope,
                                   name=dsn,
                                   type='dataset',
                                   issuer=account,
                                   statuses={'monotonic': True},
                                   meta=dataset_meta)
                    stime2 = time.time()
                    print 'Time to generate a dataset : %s' % str(stime2 -
                                                                  stime1)
                    monitor.record(timeseries='dbfiller.addnewdataset',
                                   delta=1)
                    files = ['file_%s' % uuid() for i in xrange(nbfiles)]
                    listfiles = []
                    for file in files:
                        listfiles.append({
                            'scope': scope,
                            'name': file,
                            'size': filesize
                        })
                        for source_rse in source_rses:
                            add_file_replica(source_rse,
                                             scope,
                                             file,
                                             filesize,
                                             issuer=account)
                    stime3 = time.time()
                    print 'Time to create replicas : %s' % str(stime3 - stime2)
                    monitor.record(timeseries='dbfiller.addreplicas',
                                   delta=nbfiles * len(source_rses))
                    attach_identifier(scope,
                                      name=dsn,
                                      dids=listfiles,
                                      issuer=account)
                    stime4 = time.time()
                    print 'Time to attach files : %s' % str(stime4 - stime3)
                    monitor.record(timeseries='dbfiller.addnewfile',
                                   delta=nbfiles)
                    for source_rse in source_rses:
                        try:
                            add_replication_rule(dids=[{
                                'scope': scope,
                                'name': dsn
                            }],
                                                 account=account,
                                                 copies=1,
                                                 rse_expression=source_rse,
                                                 grouping='DATASET',
                                                 weight=None,
                                                 lifetime=None,
                                                 locked=False,
                                                 subscription_id=None,
                                                 issuer='root')
                            monitor.record(
                                timeseries='dbfiller.addreplicationrules',
                                delta=1)
                        except InvalidReplicationRule, e:
                            print e
                    stime5 = time.time()
                    print 'Time to attach files : %s' % str(stime5 - stime4)
示例#8
0
    def POST(self):
        """
        Create a new replication rule.

        HTTP Success:
            201 Created

        HTTP Error:
            400 Bad Request
            401 Unauthorized
            404 Not Found
            409 Conflict
            500 Internal Error
        """
        json_data = data()
        try:
            grouping, weight, lifetime, locked, subscription_id, source_replica_expression, activity, notify,\
                purge_replicas, ignore_availability, comment, ask_approval, asynchronous, priority,\
                split_container, meta = 'DATASET', None, None, False, None, None, None, None, False, False, None,\
                False, False, 3, False, None

            params = loads(json_data)
            dids = params['dids']
            account = params['account']
            copies = params['copies']
            rse_expression = params['rse_expression']
            if 'grouping' in params:
                grouping = params['grouping']
            if 'weight' in params:
                weight = params['weight']
            if 'lifetime' in params:
                lifetime = params['lifetime']
            if 'locked' in params:
                locked = params['locked']
            if 'subscription_id' in params:
                subscription_id = params['subscription_id']
            if 'source_replica_expression' in params:
                source_replica_expression = params['source_replica_expression']
            if 'activity' in params:
                activity = params['activity']
            if 'notify' in params:
                notify = params['notify']
            if 'purge_replicas' in params:
                purge_replicas = params['purge_replicas']
            if 'ignore_availability' in params:
                ignore_availability = params['ignore_availability']
            if 'comment' in params:
                comment = params['comment']
            if 'ask_approval' in params:
                ask_approval = params['ask_approval']
            if 'asynchronous' in params:
                asynchronous = params['asynchronous']
            if 'priority' in params:
                priority = params['priority']
            if 'split_container' in params:
                split_container = params['split_container']
            if 'meta' in params:
                meta = params['meta']

        except ValueError:
            raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list')

        try:
            rule_ids = add_replication_rule(dids=dids,
                                            copies=copies,
                                            rse_expression=rse_expression,
                                            weight=weight,
                                            lifetime=lifetime,
                                            grouping=grouping,
                                            account=account,
                                            locked=locked,
                                            subscription_id=subscription_id,
                                            source_replica_expression=source_replica_expression,
                                            activity=activity,
                                            notify=notify,
                                            purge_replicas=purge_replicas,
                                            ignore_availability=ignore_availability,
                                            comment=comment,
                                            ask_approval=ask_approval,
                                            asynchronous=asynchronous,
                                            priority=priority,
                                            split_container=split_container,
                                            meta=meta,
                                            issuer=ctx.env.get('issuer'))
        # TODO: Add all other error cases here
        except InvalidReplicationRule as error:
            raise generate_http_error(409, 'InvalidReplicationRule', error.args[0])
        except DuplicateRule as error:
            raise generate_http_error(409, 'DuplicateRule', error.args[0])
        except InsufficientTargetRSEs as error:
            raise generate_http_error(409, 'InsufficientTargetRSEs', error.args[0])
        except InsufficientAccountLimit as error:
            raise generate_http_error(409, 'InsufficientAccountLimit', error.args[0])
        except InvalidRSEExpression as error:
            raise generate_http_error(409, 'InvalidRSEExpression', error.args[0])
        except DataIdentifierNotFound as error:
            raise generate_http_error(404, 'DataIdentifierNotFound', error.args[0])
        except ReplicationRuleCreationTemporaryFailed as error:
            raise generate_http_error(409, 'ReplicationRuleCreationTemporaryFailed', error.args[0])
        except InvalidRuleWeight as error:
            raise generate_http_error(409, 'InvalidRuleWeight', error.args[0])
        except StagingAreaRuleRequiresLifetime as error:
            raise generate_http_error(409, 'StagingAreaRuleRequiresLifetime', error.args[0])
        except ScratchDiskLifetimeConflict as error:
            raise generate_http_error(409, 'ScratchDiskLifetimeConflict', error.args[0])
        except ManualRuleApprovalBlocked as error:
            raise generate_http_error(409, 'ManualRuleApprovalBlocked', error.args[0])
        except InvalidObject as error:
            raise generate_http_error(409, 'InvalidObject', error.args[0])
        except RucioException as error:
            raise generate_http_error(500, error.__class__.__name__, error.args[0])
        except Exception as error:
            print format_exc()
            raise InternalError(error)

        raise Created(dumps(rule_ids))
示例#9
0
    def post(self):
        """
        Create a new replication rule.

        .. :quickref: AllRule; create new rule

        :<json list dids: List of data identifiers.
        :<json string account: Account issuing the rule.
        :<json int copies: The number of replicas.
        :<json string rse_expression: RSE expression which gets resolved into a list of RSEs.
        :<json string grouping: ALL -  All files will be replicated to the same RSE.
                                       DATASET - All files in the same dataset will be replicated to the same RSE.
                                       NONE - Files will be completely spread over all allowed RSEs without any grouping considerations at all
        :<json int weight: Weighting scheme to be used.
        :<json int lifetime: The lifetime of the replication rule in seconds.
        :<json string locked: If the is locked.
        :<json string subscription_id: The subscription_id, if the rule is created by a subscription.
        :<json string source_replica_expression: Only use replicas as source from these RSEs.
        :<json string activity: Activity to be passed to the conveyor.
        :<json string notify: Notification setting of the rule ('Y', 'N', 'C'; None = 'N').
        :<json bool purge_replicas: Purge setting if a replica should be directly deleted after the rule is deleted.
        :<json bool ignore_availability: Option to ignore the availability of RSEs.
        :<json string comments: Comment about the rule.
        :<json bool ask_approval: Ask for approval for this rule.
        :<json bool asynchronous: Create replication rule asynchronously by the judge-injector.
        :<json int priority: Priority of the rule and the transfers which should be submitted.
        :<json bool split_container: Should a container rule be split into individual dataset rules.
        :<json string meta: Dictionary with metadata from the WFMS.
        :status 201: rule created
        :status 401: Invalid Auth Token
        :status 404: DID not found
        :status 409: Invalid Replication Rule
        :status 409: Duplicate Replication Rule
        :status 409: Insufficient Target RSEs
        :status 409: Insufficient Account Limit
        :status 409: Invalid RSE Expression
        :status 409: Replication Rule Creation Temporary Failed
        :status 409: Invalid Rule Weight
        :status 409: Staging Area Rule Requires Lifetime
        :status 409: Scratch Disk Lifetime Conflict
        :status 409: Manual Rule Approval Blocked
        :status 409: Invalid Object
        :returns: List of ids for created rules
        """
        json_data = request.data
        try:
            grouping, weight, lifetime, locked, subscription_id, source_replica_expression, activity, notify,\
                purge_replicas, ignore_availability, comment, ask_approval, asynchronous, priority,\
                split_container, meta = 'DATASET', None, None, False, None, None, None, None, False, False, None,\
                False, False, 3, False, None

            params = loads(json_data)
            dids = params['dids']
            account = params['account']
            copies = params['copies']
            rse_expression = params['rse_expression']
            if 'grouping' in params:
                grouping = params['grouping']
            if 'weight' in params:
                weight = params['weight']
            if 'lifetime' in params:
                lifetime = params['lifetime']
            if 'locked' in params:
                locked = params['locked']
            if 'subscription_id' in params:
                subscription_id = params['subscription_id']
            if 'source_replica_expression' in params:
                source_replica_expression = params['source_replica_expression']
            if 'activity' in params:
                activity = params['activity']
            if 'notify' in params:
                notify = params['notify']
            if 'purge_replicas' in params:
                purge_replicas = params['purge_replicas']
            if 'ignore_availability' in params:
                ignore_availability = params['ignore_availability']
            if 'comment' in params:
                comment = params['comment']
            if 'ask_approval' in params:
                ask_approval = params['ask_approval']
            if 'asynchronous' in params:
                asynchronous = params['asynchronous']
            if 'priority' in params:
                priority = params['priority']
            if 'split_container' in params:
                split_container = params['split_container']
            if 'meta' in params:
                meta = params['meta']

        except ValueError:
            return generate_http_error_flask(
                400, 'ValueError', 'Cannot decode json parameter list')

        try:
            rule_ids = add_replication_rule(
                dids=dids,
                copies=copies,
                rse_expression=rse_expression,
                weight=weight,
                lifetime=lifetime,
                grouping=grouping,
                account=account,
                locked=locked,
                subscription_id=subscription_id,
                source_replica_expression=source_replica_expression,
                activity=activity,
                notify=notify,
                purge_replicas=purge_replicas,
                ignore_availability=ignore_availability,
                comment=comment,
                ask_approval=ask_approval,
                asynchronous=asynchronous,
                priority=priority,
                split_container=split_container,
                meta=meta,
                issuer=request.environ.get('issuer'))
        # TODO: Add all other error cases here
        except InvalidReplicationRule as error:
            return generate_http_error_flask(409, 'InvalidReplicationRule',
                                             error.args[0])
        except DuplicateRule as error:
            return generate_http_error_flask(409, 'DuplicateRule',
                                             error.args[0])
        except InsufficientTargetRSEs as error:
            return generate_http_error_flask(409, 'InsufficientTargetRSEs',
                                             error.args[0])
        except InsufficientAccountLimit as error:
            return generate_http_error_flask(409, 'InsufficientAccountLimit',
                                             error.args[0])
        except InvalidRSEExpression as error:
            return generate_http_error_flask(409, 'InvalidRSEExpression',
                                             error.args[0])
        except DataIdentifierNotFound as error:
            return generate_http_error_flask(404, 'DataIdentifierNotFound',
                                             error.args[0])
        except ReplicationRuleCreationTemporaryFailed as error:
            return generate_http_error_flask(
                409, 'ReplicationRuleCreationTemporaryFailed', error.args[0])
        except InvalidRuleWeight as error:
            return generate_http_error_flask(409, 'InvalidRuleWeight',
                                             error.args[0])
        except StagingAreaRuleRequiresLifetime as error:
            return generate_http_error_flask(
                409, 'StagingAreaRuleRequiresLifetime', error.args[0])
        except ScratchDiskLifetimeConflict as error:
            return generate_http_error_flask(409,
                                             'ScratchDiskLifetimeConflict',
                                             error.args[0])
        except ManualRuleApprovalBlocked as error:
            return generate_http_error_flask(409, 'ManualRuleApprovalBlocked',
                                             error.args[0])
        except InvalidObject as error:
            return generate_http_error_flask(409, 'InvalidObject',
                                             error.args[0])
        except RucioException as error:
            return generate_http_error_flask(500, error.__class__.__name__,
                                             error.args[0])
        except Exception as error:
            print(error)
            print(format_exc())
            return error, 500

        return Response(dumps(rule_ids), status=201)
示例#10
0
    def post(self):
        """
        Create a new replication rule.

        .. :quickref: AllRule; create new rule

        :<json list dids: List of data identifiers.
        :<json string account: Account issuing the rule.
        :<json int copies: The number of replicas.
        :<json string rse_expression: RSE expression which gets resolved into a list of RSEs.
        :<json string grouping: ALL -  All files will be replicated to the same RSE.
                                       DATASET - All files in the same dataset will be replicated to the same RSE.
                                       NONE - Files will be completely spread over all allowed RSEs without any grouping considerations at all
        :<json int weight: Weighting scheme to be used.
        :<json int lifetime: The lifetime of the replication rule in seconds.
        :<json string locked: If the is locked.
        :<json string subscription_id: The subscription_id, if the rule is created by a subscription.
        :<json string source_replica_expression: Only use replicas as source from these RSEs.
        :<json string activity: Activity to be passed to the conveyor.
        :<json string notify: Notification setting of the rule ('Y', 'N', 'C'; None = 'N').
        :<json bool purge_replicas: Purge setting if a replica should be directly deleted after the rule is deleted.
        :<json bool ignore_availability: Option to ignore the availability of RSEs.
        :<json string comments: Comment about the rule.
        :<json bool ask_approval: Ask for approval for this rule.
        :<json bool asynchronous: Create replication rule asynchronously by the judge-injector.
        :<json int priority: Priority of the rule and the transfers which should be submitted.
        :<json bool split_container: Should a container rule be split into individual dataset rules.
        :<json string meta: Dictionary with metadata from the WFMS.
        :status 201: rule created
        :status 401: Invalid Auth Token
        :status 404: DID not found
        :status 409: Invalid Replication Rule
        :status 409: Duplicate Replication Rule
        :status 409: Insufficient Target RSEs
        :status 409: Insufficient Account Limit
        :status 409: Invalid RSE Expression
        :status 409: Replication Rule Creation Temporary Failed
        :status 409: Invalid Rule Weight
        :status 409: Staging Area Rule Requires Lifetime
        :status 409: Scratch Disk Lifetime Conflict
        :status 409: Manual Rule Approval Blocked
        :status 409: Invalid Object
        :returns: List of ids for created rules
        """
        parameters = json_parameters()
        dids = param_get(parameters, 'dids')
        account = param_get(parameters, 'account')
        copies = param_get(parameters, 'copies')
        rse_expression = param_get(parameters, 'rse_expression')
        try:
            rule_ids = add_replication_rule(
                dids=dids,
                copies=copies,
                rse_expression=rse_expression,
                weight=param_get(parameters, 'weight', default=None),
                lifetime=param_get(parameters, 'lifetime', default=None),
                grouping=param_get(parameters, 'grouping', default='DATASET'),
                account=account,
                locked=param_get(parameters, 'locked', default=False),
                subscription_id=param_get(parameters,
                                          'subscription_id',
                                          default=None),
                source_replica_expression=param_get(
                    parameters, 'source_replica_expression', default=None),
                activity=param_get(parameters, 'activity', default=None),
                notify=param_get(parameters, 'notify', default=None),
                purge_replicas=param_get(parameters,
                                         'purge_replicas',
                                         default=False),
                ignore_availability=param_get(parameters,
                                              'ignore_availability',
                                              default=False),
                comment=param_get(parameters, 'comment', default=None),
                ask_approval=param_get(parameters,
                                       'ask_approval',
                                       default=False),
                asynchronous=param_get(parameters,
                                       'asynchronous',
                                       default=False),
                priority=param_get(parameters, 'priority', default=3),
                split_container=param_get(parameters,
                                          'split_container',
                                          default=False),
                meta=param_get(parameters, 'meta', default=None),
                issuer=request.environ.get('issuer'),
                vo=request.environ.get('vo'),
            )
        except (
                InvalidReplicationRule,
                DuplicateRule,
                InsufficientTargetRSEs,
                InsufficientAccountLimit,
                InvalidRSEExpression,
                ReplicationRuleCreationTemporaryFailed,
                InvalidRuleWeight,
                StagingAreaRuleRequiresLifetime,
                ScratchDiskLifetimeConflict,
                ManualRuleApprovalBlocked,
                InvalidObject,
        ) as error:
            return generate_http_error_flask(409, error)
        except DataIdentifierNotFound as error:
            return generate_http_error_flask(404, error)

        return Response(dumps(rule_ids), status=201)
示例#11
0
    def post(self):
        """
        ---
        summary: Create a new replication rule
        tags:
          - Rule
        requestBody:
          description: Parameters for the new rule.
          content:
            'application/json':
              schema:
                type: object
                required:
                - dids
                - account
                - copies
                - rse_expression
                properties:
                  dids:
                    description: The list of data identifiers.
                    type: array
                    items:
                      type: string
                  account:
                    descipriton: The account of the issuer.
                    type: string
                  copies:
                    description: The number of replicas.
                    type: integer
                  rse_expression:
                    description: The rse expression which gets resolved into a list of RSEs.
                    type: string
                  grouping:
                    description: The grouping of the files to take into account. (ALL, DATASET, NONE)
                    type: string
                  weight:
                     description: Weighting scheme to be used.
                     type: number
                  lifetime:
                     description: The lifetime of the replication rule in seconds.
                     type: integer
                  locked:
                     description: If the rule is locked.
                     type: boolean
                  subscription_id:
                     description: The subscription_id, if the rule is created by a subscription.
                     type: string
                  sourse_replica_expression:
                     description: Only use replicas as source from these RSEs.
                     type: string
                  activity:
                     description: Activity to be passed to the conveyor.
                     type: string
                  notify:
                     description: Notification setting of the rule ('Y', 'N', 'C'; None = 'N').
                     type: string
                  purge_replicas:
                     description: Purge setting if a replica should be directly deleted after the rule is deleted.
                     type: boolean
                  ignore_availability:
                     description: Option to ignore the availability of RSEs.
                     type: boolean
                  comments:
                     description: Comment about the rule.
                     type: string
                  ask_approval:
                     description: Ask for approval for this rule.
                     type: boolean
                  asynchronous:
                     description: Create replication rule asynchronously by the judge-injector.
                     type: boolean
                  priority:
                     description: Priority of the rule and the transfers which should be submitted.
                     type: integer
                  split_container:
                     description: Should a container rule be split into individual dataset rules.
                     type: boolean
                  meta:
                     description: Dictionary with metadata from the WFMS.
                     type: string
        responses:
          201:
            description: Rule created.
            content:
              application/json:
                schema:
                  type: array
                  items:
                    type: string
                    description: Id of each created rule.
          401:
            description: Invalid Auth Token
          404:
            description: No rule found for the given id
          409:
            description: |
              - Invalid Replication Rule
              - Duplicate Replication Rule
              - Insufficient Target RSEs
              - Insufficient Account Limit
              - Invalid RSE Expression
              - Replication Rule Creation Temporary Failed,
              - Invalid Rule Weight
              - Staging Area Rule Requires Lifetime
              - Scratch Disk Lifetime Conflict
              - Manual Rule Approval Blocked
              - Invalid Object
        """
        parameters = json_parameters()
        dids = param_get(parameters, 'dids')
        account = param_get(parameters, 'account')
        copies = param_get(parameters, 'copies')
        rse_expression = param_get(parameters, 'rse_expression')
        try:
            rule_ids = add_replication_rule(
                dids=dids,
                copies=copies,
                rse_expression=rse_expression,
                weight=param_get(parameters, 'weight', default=None),
                lifetime=param_get(parameters, 'lifetime', default=None),
                grouping=param_get(parameters, 'grouping', default='DATASET'),
                account=account,
                locked=param_get(parameters, 'locked', default=False),
                subscription_id=param_get(parameters,
                                          'subscription_id',
                                          default=None),
                source_replica_expression=param_get(
                    parameters, 'source_replica_expression', default=None),
                activity=param_get(parameters, 'activity', default=None),
                notify=param_get(parameters, 'notify', default=None),
                purge_replicas=param_get(parameters,
                                         'purge_replicas',
                                         default=False),
                ignore_availability=param_get(parameters,
                                              'ignore_availability',
                                              default=False),
                comment=param_get(parameters, 'comment', default=None),
                ask_approval=param_get(parameters,
                                       'ask_approval',
                                       default=False),
                asynchronous=param_get(parameters,
                                       'asynchronous',
                                       default=False),
                delay_injection=param_get(parameters,
                                          'delay_injection',
                                          default=None),
                priority=param_get(parameters, 'priority', default=3),
                split_container=param_get(parameters,
                                          'split_container',
                                          default=False),
                meta=param_get(parameters, 'meta', default=None),
                issuer=request.environ.get('issuer'),
                vo=request.environ.get('vo'),
            )
        except (
                InvalidReplicationRule,
                DuplicateRule,
                InsufficientTargetRSEs,
                InsufficientAccountLimit,
                InvalidRSEExpression,
                ReplicationRuleCreationTemporaryFailed,
                InvalidRuleWeight,
                StagingAreaRuleRequiresLifetime,
                ScratchDiskLifetimeConflict,
                ManualRuleApprovalBlocked,
                InvalidObject,
        ) as error:
            return generate_http_error_flask(409, error)
        except DataIdentifierNotFound as error:
            return generate_http_error_flask(404, error)

        return Response(dumps(rule_ids), status=201)
def populateDB(filename=None):
    listrses = list_rses(filters={'deterministic': 1})
    print listrses
    listrses = map(lambda x: x['rse'], listrses)
    account = 'root'
    nbDatasets = 0
    list = []
    dictDistrib = {}

    if not filename:
        if os.getenv('RUCIO_HOME'):
            filename = os.getenv('RUCIO_HOME') + '/etc/data12_8TeV_distribution.txt'
        else:
            filename = '/opt/rucio/etc/data12_8TeV_distribution.txt'

    # Get the dataset distribution
    f = open(filename, 'r')
    for line in f:
        if not line.startswith('NBDATASETS'):
            line = line.rstrip('\n')
            strsplit = line.split()
            dictDistrib[(nbDatasets, nbDatasets + int(strsplit[0]))] = strsplit[1:]
            nbDatasets += int(strsplit[0])
            list.append([nbDatasets, ] + strsplit[1:])
    f.close()

    # Generate 200000 datasets according to the dataset distribution
    for i in xrange(0, 200000):
        rnd = random.random() * nbDatasets
        for lower, upper in dictDistrib:
            if (rnd > lower) and (rnd < upper):
                project = dictDistrib[lower, upper][0]
                scope = project
                run_number = random.randint(0, 1000000)
                tag = random.randint(0, 10000)
                stream_name = dictDistrib[lower, upper][1]
                prod_step = dictDistrib[lower, upper][2]
                datatype = dictDistrib[lower, upper][3]
                provenance = dictDistrib[lower, upper][4]
                group = dictDistrib[lower, upper][5]
                if group == '/atlas/role=production':
                    # account = 'atlasprod'
                    account = 'panda'
                    if provenance == 'T0':
                        group = 'tier0'
                        account = 'tier0'
                    else:
                        group = 'panda'
                else:
                    # account = dictGroups[group]
                    account = 'panda'
                    scope = 'group.%s' % (dictGroups[group])
                    group = dictGroups[group]
                nbfiles = int(dictDistrib[lower, upper][6])
                filesize = int(int(dictDistrib[lower, upper][7])/float(nbfiles))
                nbreplicas = int(dictDistrib[lower, upper][8])
                if group == 'panda' or group == 'tier0':
                    dataset_meta = {'project': project, 'run_number': run_number, 'stream_name': stream_name, 'prod_step': prod_step, 'datatype': datatype, 'provenance': provenance, 'group': group}
                else:
                    campaign = int(tag/1000.)
                    dataset_meta = {'project': project, 'run_number': run_number, 'stream_name': stream_name, 'prod_step': prod_step, 'datatype': datatype, 'provenance': provenance, 'group': group, 'campaign': '%s_repro_%i' % (group, campaign)}
                source_rses = []
                if nbreplicas:
                    iter = 0
                    while (len(source_rses) != nbreplicas and iter != 100):
                        rnd_site = random.choice(listrses)
                        iter += 1
                        if (rnd_site not in source_rses):
                            source_rses.append(rnd_site)

                    run_number_string = str(run_number)
                    run_number_string = run_number_string.rjust(7, '0')
                    dsn = '%s.%s.%s.%s.%s.%s' % (project, run_number_string, stream_name, prod_step, datatype, tag)
                    print '%i Creating %s:%s with %i files of size %i located at %i sites' % (i, scope, dsn, nbfiles, filesize, len(source_rses))
                    stime1 = time.time()
                    add_identifier(scope=scope, name=dsn, type='dataset', issuer=account, statuses={'monotonic': True}, meta=dataset_meta)
                    stime2 = time.time()
                    print 'Time to generate a dataset : %s' % str(stime2 - stime1)
                    monitor.record(timeseries='dbfiller.addnewdataset',  delta=1)
                    files = ['file_%s' % uuid() for i in xrange(nbfiles)]
                    listfiles = []
                    for file in files:
                        listfiles.append({'scope': scope, 'name': file, 'size': filesize})
                        for source_rse in source_rses:
                            add_file_replica(source_rse, scope, file, filesize, issuer=account)
                    stime3 = time.time()
                    print 'Time to create replicas : %s' % str(stime3 - stime2)
                    monitor.record(timeseries='dbfiller.addreplicas',  delta=nbfiles*len(source_rses))
                    attach_identifier(scope, name=dsn, dids=listfiles, issuer=account)
                    stime4 = time.time()
                    print 'Time to attach files : %s' % str(stime4 - stime3)
                    monitor.record(timeseries='dbfiller.addnewfile',  delta=nbfiles)
                    for source_rse in source_rses:
                        try:
                            add_replication_rule(dids=[{'scope': scope, 'name': dsn}], account=account, copies=1, rse_expression=source_rse,
                                                 grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None, issuer='root')
                            monitor.record(timeseries='dbfiller.addreplicationrules',  delta=1)
                        except InvalidReplicationRule, e:
                            print e
                    stime5 = time.time()
                    print 'Time to attach files : %s' % str(stime5 - stime4)
示例#13
0
文件: rule.py 项目: pombredanne/rucio
    def POST(self):
        """
        Create a new replication rule.

        HTTP Success:
            201 Created

        HTTP Error:
            400 Bad Request
            401 Unauthorized
            404 Not Found
            409 Conflict
            500 Internal Error
        """
        json_data = data()
        try:
            grouping, weight, lifetime, locked, subscription_id, source_replica_expression, activity, notify, purge_replicas, ignore_availability = 'DATASET', None, None, False, None, None, None, None, False, False
            params = loads(json_data)
            dids = params['dids']
            account = params['account']
            copies = params['copies']
            rse_expression = params['rse_expression']
            if 'grouping' in params:
                grouping = params['grouping']
            if 'weight' in params:
                weight = params['weight']
            if 'lifetime' in params:
                lifetime = params['lifetime']
            if 'locked' in params:
                locked = params['locked']
            if 'subscription_id' in params:
                subscription_id = params['subscription_id']
            if 'source_replica_expression' in params:
                source_replica_expression = params['source_replica_expression']
            if 'activity' in params:
                activity = params['activity']
            if 'notify' in params:
                notify = params['notify']
            if 'purge_replicas' in params:
                purge_replicas = params['purge_replicas']
            if 'ignore_availability' in params:
                ignore_availability = params['ignore_availability']
        except ValueError:
            raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list')

        try:
            rule_ids = add_replication_rule(dids=dids,
                                            copies=copies,
                                            rse_expression=rse_expression,
                                            weight=weight,
                                            lifetime=lifetime,
                                            grouping=grouping,
                                            account=account,
                                            locked=locked,
                                            subscription_id=subscription_id,
                                            source_replica_expression=source_replica_expression,
                                            activity=activity,
                                            notify=notify,
                                            purge_replicas=purge_replicas,
                                            ignore_availability=ignore_availability,
                                            issuer=ctx.env.get('issuer'))
        # TODO: Add all other error cases here
        except InvalidReplicationRule, e:
            raise generate_http_error(409, 'InvalidReplicationRule', e.args[0][0])