示例#1
0
文件: replica.py 项目: yiiyama/rucio
def delete_replicas(rse, files, issuer, ignore_availability=False):
    """
    Bulk delete file replicas.

    :param rse: The RSE name.
    :param files: The list of files.
    :param issuer: The issuer account.
    :param ignore_availability: Ignore the RSE blacklisting.

    :returns: True is successful, False otherwise
    """
    validate_schema(name='r_dids', obj=files)

    rse_id = get_rse_id(rse=rse)

    kwargs = {'rse': rse, 'rse_id': rse_id}
    if not permission.has_permission(
            issuer=issuer, action='delete_replicas', kwargs=kwargs):
        raise exception.AccessDenied(
            'Account %s can not delete file replicas on %s' % (issuer, rse))
    if not permission.has_permission(
            issuer=issuer, action='skip_availability_check', kwargs=kwargs):
        ignore_availability = False

    for f in files:
        f['scope'] = InternalScope(f['scope'])

    replica.delete_replicas(rse_id=rse_id,
                            files=files,
                            ignore_availability=ignore_availability)
示例#2
0
文件: vo.py 项目: pradeepjasal/rucio
def add_vo(new_vo,
           issuer,
           description=None,
           email=None,
           vo='def',
           session=None):
    '''
    Add a new VO.

    :param new_vo: The name/tag of the VO to add (3 characters).
    :param description: A description of the VO. e.g the full name or a brief description
    :param email: A contact for the VO.
    :param issuer: The user issuing the command.
    :param vo: The vo of the user issuing the command.
    :param session: The database session in use.
    '''

    new_vo = vo_core.map_vo(new_vo)
    validate_schema('vo', new_vo, vo=vo)

    kwargs = {}
    if not has_permission(
            issuer=issuer, action='add_vo', kwargs=kwargs, vo=vo,
            session=session):
        raise exception.AccessDenied(
            'Account {} cannot add a VO'.format(issuer))

    vo_core.add_vo(vo=new_vo,
                   description=description,
                   email=email,
                   session=session)
示例#3
0
文件: replica.py 项目: dciangot/rucio
def add_replicas(rse, files, issuer, ignore_availability=False):
    """
    Bulk add file replicas.

    :param rse: The RSE name.
    :param files: The list of files.
    :param issuer: The issuer account.
    :param ignore_availability: Ignore the RSE blacklisting.

    :returns: True is successful, False otherwise
    """
    validate_schema(name='dids', obj=files)

    kwargs = {'rse': rse}
    if not permission.has_permission(
            issuer=issuer, action='add_replicas', kwargs=kwargs):
        raise exception.AccessDenied(
            'Account %s can not add file replicas on %s' % (issuer, rse))
    if not permission.has_permission(
            issuer=issuer, action='skip_availability_check', kwargs=kwargs):
        ignore_availability = False
    replica.add_replicas(rse=rse,
                         files=files,
                         account=issuer,
                         ignore_availability=ignore_availability)
示例#4
0
文件: replica.py 项目: yiiyama/rucio
def update_replicas_states(rse, files, issuer):
    """
    Update File replica information and state.

    :param rse: The RSE name.
    :param files: The list of files.
    :param issuer: The issuer account.
    """
    for v_file in files:
        v_file.update(
            {"type":
             "FILE"})  # Make sure DIDs are identified as files for checking
    validate_schema(name='dids', obj=files)

    rse_id = get_rse_id(rse=rse)

    kwargs = {'rse': rse, 'rse_id': rse_id}
    if not permission.has_permission(
            issuer=issuer, action='update_replicas_states', kwargs=kwargs):
        raise exception.AccessDenied(
            'Account %s can not update file replicas state on %s' %
            (issuer, rse))
    replicas = []
    for file in files:
        rep = file
        rep['rse_id'] = rse_id
        rep['scope'] = InternalScope(rep['scope'])
        replicas.append(rep)
    replica.update_replicas_states(replicas=replicas)
示例#5
0
def add_subscription(name, account, filter_, replication_rules, comments, lifetime, retroactive, dry_run, priority=None, issuer=None, vo='def'):
    """
    Adds a new subscription which will be verified against every new added file and dataset

    :param account: Account identifier
    :type account:  String
    :param name: Name of the subscription
    :type:  String
    :param filter_: Dictionary of attributes by which the input data should be filtered
                   **Example**: ``{'dsn': 'data11_hi*.express_express.*,data11_hi*physics_MinBiasOverlay*', 'account': 'tzero'}``
    :type filter_:  Dict
    :param replication_rules: Replication rules to be set : Dictionary with keys copies, rse_expression, weight, rse_expression
    :type replication_rules:  Dict
    :param comments: Comments for the subscription
    :type comments:  String
    :param lifetime: Subscription's lifetime (seconds); False if subscription has no lifetime
    :type lifetime:  Integer or False
    :param retroactive: Flag to know if the subscription should be applied on previous data
    :type retroactive:  Boolean
    :param dry_run: Just print the subscriptions actions without actually executing them (Useful if retroactive flag is set)
    :type dry_run:  Boolean
    :param priority: The priority of the subscription
    :type priority: Integer
    :param issuer:  The account issuing this operation.
    :type issuer:  String
    :param vo: The VO to act on.
    :type vo: String
    :returns: subscription_id
    :rtype:   String
    """
    if not has_permission(issuer=issuer, vo=vo, action='add_subscription', kwargs={'account': account}):
        raise AccessDenied('Account %s can not add subscription' % (issuer))
    try:
        if filter_:
            if not isinstance(filter_, dict):
                raise TypeError('filter should be a dict')
            validate_schema(name='subscription_filter', obj=filter_, vo=vo)
        if replication_rules:
            if not isinstance(replication_rules, list):
                raise TypeError('replication_rules should be a list')
            else:
                for rule in replication_rules:
                    validate_schema(name='activity', obj=rule.get('activity', 'default'), vo=vo)
        else:
            raise InvalidObject('You must specify a rule')
    except ValueError as error:
        raise TypeError(error)

    account = InternalAccount(account, vo=vo)

    keys = ['scope', 'account']
    types = [InternalScope, InternalAccount]
    for _key, _type in zip(keys, types):
        if _key in filter_:
            if isinstance(filter_[_key], list):
                filter_[_key] = [_type(val, vo=vo).internal for val in filter_[_key]]
            else:
                filter_[_key] = _type(filter_[_key], vo=vo).internal

    return subscription.add_subscription(name=name, account=account, filter_=dumps(filter_), replication_rules=dumps(replication_rules), comments=comments, lifetime=lifetime, retroactive=retroactive, dry_run=dry_run, priority=priority)
示例#6
0
    def message_handle(self, msg):
        record_counter('daemons.cache.consumer.message_handle.message')

        try:
            if isinstance(msg, dict) and 'operation' in msg.keys():
                if msg['operation'] == 'add_replicas':
                    validate_schema(name='cache_add_replicas', obj=msg)
                    if 'rse' in msg.keys() and 'files' in msg.keys():
                        logging.debug('[%s] %s %s %s' % (self.__broker, msg['operation'], msg['rse'], msg['files']))
                        try:
                            if not self.__rse_volatile.get_volatile(msg['rse']):
                                logging.error("%s volatile is not True, Rucio Cache should not update it." % (msg['rse']))
                            else:
                                cache_add_replicas(rse=msg['rse'], files=msg['files'], account=self.__account, lifetime=msg['lifetime'])
                        except Exception, e:
                            logging.error('[%s] %s %s %s %s with error details: %s' % (self.__broker, msg['operation'], msg['rse'], msg['files'], str(e), str(format_exc())))

                if msg['operation'] == 'delete_replicas':
                    validate_schema(name='cache_delete_replicas', obj=msg)
                    if 'rse' in msg.keys() and 'files' in msg.keys():
                        logging.debug('[%s] %s %s %s' % (self.__broker, msg['operation'], msg['rse'], msg['files']))
                        try:
                            if not self.__rse_volatile.get_volatile(msg['rse']):
                                logging.error("%s volatile is not True, Rucio Cache should not update it." % (msg['rse']))
                            else:
                                cache_delete_replicas(rse=msg['rse'], files=msg['files'], account=self.__account)
                        except Exception, e:
                            logging.error('[%s] %s %s %s %s with error details: %s' % (self.__broker, msg['operation'], msg['rse'], msg['files'], str(e), str(format_exc())))
示例#7
0
文件: did.py 项目: kbg/rucio
def list_dids(scope,
              filters,
              type='collection',
              ignore_case=False,
              limit=None,
              offset=None,
              long=False,
              recursive=False):
    """
    List dids in a scope.

    :param scope: The scope name.
    :param pattern: The wildcard pattern.
    :param type:  The type of the did: all(container, dataset, file), collection(dataset or container), dataset, container
    :param ignore_case: Ignore case distinctions.
    :param limit: The maximum number of DIDs returned.
    :param offset: Offset number.
    :param long: Long format option to display more information for each DID.
    :param recursive: Recursively list DIDs content.
    """
    validate_schema(name='did_filters', obj=filters)
    return did.list_dids(scope=scope,
                         filters=filters,
                         type=type,
                         ignore_case=ignore_case,
                         limit=limit,
                         offset=offset,
                         long=long,
                         recursive=recursive)
示例#8
0
def list_replicas(dids, schemes=None, unavailable=False, request_id=None,
                  ignore_availability=True, all_states=False, rse_expression=None,
                  client_location=None, domain=None, lifetime=None, issuer=None):
    """
    List file replicas for a list of data identifiers.

    :param dids: The list of data identifiers (DIDs).
    :param schemes: A list of schemes to filter the replicas. (e.g. file, http, ...)
    :param unavailable: Also include unavailable replicas in the list.
    :param request_id: ID associated with the request for debugging.
    :param all_states: Return all replicas whatever state they are in. Adds an extra 'states' entry in the result dictionary.
    :param rse_expression: The RSE expression to restrict replicas on a set of RSEs.
    :param client_location: Client location dictionary for PFN modification {'ip', 'fqdn', 'site'}
    :param domain: The network domain for the call, either None, 'wan' or 'lan'. Compatibility fallback: None falls back to 'wan'.
    :param lifetime: If supported, in seconds, restrict the lifetime of the replica PFN.
    :param issuer: The issuer account.
    """
    validate_schema(name='r_dids', obj=dids)

    # Allow selected authenticated users to retrieve signed URLs.
    # Unauthenticated users, or permission-less users will get the raw URL without the signature.
    sign_urls = False
    if permission.has_permission(issuer=issuer, action='get_signed_urls', kwargs={}):
        sign_urls = True

    return replica.list_replicas(dids=dids, schemes=schemes, unavailable=unavailable,
                                 request_id=request_id,
                                 ignore_availability=ignore_availability,
                                 all_states=all_states, rse_expression=rse_expression,
                                 client_location=client_location, domain=domain,
                                 sign_urls=sign_urls, lifetime=lifetime)
示例#9
0
def add_account(account, type_, email, issuer, vo='def', session=None):
    """
    Creates an account with the provided account name, contact information, etc.

    :param account: The account name.
    :param type_: The account type
    :param email: The Email address associated with the account.

    :param issuer: The issuer account_core.
    :param vo: The VO to act on.
    :param session: The database session in use.

    """

    validate_schema(name='account', obj=account, vo=vo)

    kwargs = {'account': account, 'type': type_}
    if not rucio.api.permission.has_permission(issuer=issuer,
                                               vo=vo,
                                               action='add_account',
                                               kwargs=kwargs,
                                               session=session):
        raise rucio.common.exception.AccessDenied(
            'Account %s can not add account' % (issuer))

    account = InternalAccount(account, vo=vo)

    account_core.add_account(account,
                             AccountType[type_.upper()],
                             email,
                             session=session)
示例#10
0
文件: rse.py 项目: nikmagini/rucio
def add_rse(rse,
            issuer,
            vo='def',
            deterministic=True,
            volatile=False,
            city=None,
            region_code=None,
            country_name=None,
            continent=None,
            time_zone=None,
            ISP=None,
            staging_area=False,
            rse_type=None,
            latitude=None,
            longitude=None,
            ASN=None,
            availability=None):
    """
    Creates a new Rucio Storage Element(RSE).

    :param rse: The RSE name.
    :param issuer: The issuer account.
    :param vo: The VO to act on.
    :param deterministic: Boolean to know if the pfn is generated deterministically.
    :param volatile: Boolean for RSE cache.
    :param city: City for the RSE.
    :param region_code: The region code for the RSE.
    :param country_name: The country.
    :param continent: The continent.
    :param time_zone: Timezone.
    :param staging_area: staging area.
    :param ISP: Internet service provider.
    :param rse_type: RSE type.
    :param latitude: Latitude coordinate of RSE.
    :param longitude: Longitude coordinate of RSE.
    :param ASN: Access service network.
    :param availability: Availability.
    """
    validate_schema(name='rse', obj=rse)
    kwargs = {'rse': rse}
    if not permission.has_permission(
            issuer=issuer, vo=vo, action='add_rse', kwargs=kwargs):
        raise exception.AccessDenied('Account %s can not add RSE' % (issuer))

    return rse_module.add_rse(rse,
                              vo=vo,
                              deterministic=deterministic,
                              volatile=volatile,
                              city=city,
                              region_code=region_code,
                              country_name=country_name,
                              staging_area=staging_area,
                              continent=continent,
                              time_zone=time_zone,
                              ISP=ISP,
                              rse_type=rse_type,
                              latitude=latitude,
                              longitude=longitude,
                              ASN=ASN,
                              availability=availability)
示例#11
0
def list_replicas(dids,
                  schemes=None,
                  unavailable=False,
                  request_id=None,
                  ignore_availability=True,
                  all_states=False,
                  rse_expression=None):
    """
    List file replicas for a list of data identifiers.

    :param dids: The list of data identifiers (DIDs).
    :param schemes: A list of schemes to filter the replicas. (e.g. file, http, ...)
    :param unavailable: Also include unavailable replicas in the list.
    :param request_id: ID associated with the request for debugging.
    :param all_states: Return all replicas whatever state they are in. Adds an extra 'states' entry in the result dictionary.
    :param rse_expression: The RSE expression to restrict replicas on a set of RSEs.
    """
    validate_schema(name='r_dids', obj=dids)
    return replica.list_replicas(dids=dids,
                                 schemes=schemes,
                                 unavailable=unavailable,
                                 request_id=request_id,
                                 ignore_availability=ignore_availability,
                                 all_states=all_states,
                                 rse_expression=rse_expression)
示例#12
0
def add_account_attribute(key, value, account, issuer, vo='def', session=None):
    """
    Add an attribute to an account.

    :param key: attribute key.
    :param value: attribute value.
    :param account: The account name.
    :param issuer: The issuer account.
    :param vo: The VO to act on.
    :param session: The database session in use.
    """
    validate_schema(name='account_attribute', obj=key, vo=vo)
    validate_schema(name='account_attribute', obj=value, vo=vo)

    kwargs = {'account': account, 'key': key, 'value': value}
    if not rucio.api.permission.has_permission(issuer=issuer,
                                               vo=vo,
                                               action='add_attribute',
                                               kwargs=kwargs,
                                               session=session):
        raise rucio.common.exception.AccessDenied(
            'Account %s can not add attributes' % (issuer))

    account = InternalAccount(account, vo=vo)

    account_core.add_account_attribute(account, key, value, session=session)
示例#13
0
文件: did.py 项目: purusharths/rucio
def attach_dids_to_dids(attachments, issuer, ignore_duplicate=False):
    """
    Append content to dids.

    :param attachments: The contents.
    :param issuer: The issuer account.
    :param ignore_duplicate: If True, ignore duplicate entries.
    """
    validate_schema(name='attachments', obj=attachments)

    for a in attachments:
        if 'rse' in a:
            rse_id = None
            if a['rse'] is not None:
                rse_id = get_rse_id(rse=a['rse'])
            a['rse_id'] = rse_id

    if not rucio.api.permission.has_permission(
            issuer=issuer,
            action='attach_dids_to_dids',
            kwargs={'attachments': attachments}):
        raise rucio.common.exception.AccessDenied(
            'Account %s can not add data identifiers' % (issuer))

    issuer = InternalAccount(issuer)
    for attachment in attachments:
        attachment['scope'] = InternalScope(attachment['scope'])
        for d in attachment['dids']:
            d['scope'] = InternalScope(d['scope'])

    return did.attach_dids_to_dids(attachments=attachments,
                                   account=issuer,
                                   ignore_duplicate=ignore_duplicate)
示例#14
0
def update_subscription(name, account, metadata=None, issuer=None, vo='def'):
    """
    Updates a subscription

    :param name: Name of the subscription
    :type:  String
    :param account: Account identifier
    :type account:  String
    :param metadata: Dictionary of metadata to update. Supported keys : filter, replication_rules, comments, lifetime, retroactive, dry_run, priority, last_processed
    :type metadata:  Dict
    :param issuer: The account issuing this operation.
    :type issuer: String
    :param vo: The VO to act on.
    :type vo: String
    :raises: SubscriptionNotFound if subscription is not found
    """
    if not has_permission(issuer=issuer,
                          vo=vo,
                          action='update_subscription',
                          kwargs={'account': account}):
        raise AccessDenied('Account %s can not update subscription' % (issuer))
    try:
        if not isinstance(metadata, dict):
            raise TypeError('metadata should be a dict')
        if 'filter' in metadata and metadata['filter']:
            if not isinstance(metadata['filter'], dict):
                raise TypeError('filter should be a dict')
            validate_schema(name='subscription_filter',
                            obj=metadata['filter'],
                            vo=vo)
        if 'replication_rules' in metadata and metadata['replication_rules']:
            if not isinstance(metadata['replication_rules'], list):
                raise TypeError('replication_rules should be a list')
            else:
                for rule in metadata['replication_rules']:
                    validate_schema(name='activity',
                                    obj=rule.get('activity', 'default'),
                                    vo=vo)
    except ValueError as error:
        raise TypeError(error)

    account = InternalAccount(account, vo=vo)

    if 'filter' in metadata and metadata['filter'] is not None:
        filter = metadata['filter']
        keys = ['scope', 'account']
        types = [InternalScope, InternalAccount]

        for _key, _type in zip(keys, types):
            if _key in filter and filter[_key] is not None:
                if isinstance(filter[_key], list):
                    filter[_key] = [
                        _type(val, vo=vo).internal for val in filter[_key]
                    ]
                else:
                    filter[_key] = _type(filter[_key], vo=vo).internal

    return subscription.update_subscription(name=name,
                                            account=account,
                                            metadata=metadata)
示例#15
0
def list_replicas(dids,
                  schemes=None,
                  unavailable=False,
                  request_id=None,
                  ignore_availability=True,
                  all_states=False,
                  rse_expression=None,
                  client_location=None,
                  domain=None):
    """
    List file replicas for a list of data identifiers.

    :param dids: The list of data identifiers (DIDs).
    :param schemes: A list of schemes to filter the replicas. (e.g. file, http, ...)
    :param unavailable: Also include unavailable replicas in the list.
    :param request_id: ID associated with the request for debugging.
    :param all_states: Return all replicas whatever state they are in. Adds an extra 'states' entry in the result dictionary.
    :param rse_expression: The RSE expression to restrict replicas on a set of RSEs.
    :param client_location: Client location dictionary for PFN modification {'ip', 'fqdn', 'site'}
    :param domain: The network domain for the call, either None, 'wan' or 'lan'. Compatibility fallback: None falls back to 'wan'.
    """
    validate_schema(name='r_dids', obj=dids)
    return replica.list_replicas(dids=dids,
                                 schemes=schemes,
                                 unavailable=unavailable,
                                 request_id=request_id,
                                 ignore_availability=ignore_availability,
                                 all_states=all_states,
                                 rse_expression=rse_expression,
                                 client_location=client_location,
                                 domain=domain)
示例#16
0
def update_subscription(name, account, metadata=None, issuer=None):
    """
    Updates a subscription

    :param name: Name of the subscription
    :type:  String
    :param account: Account identifier
    :type account:  String
    :param metadata: Dictionary of metadata to update. Supported keys : filter, replication_rules, comments, lifetime, retroactive, dry_run, priority, last_processed
    :type metadata:  Dict
    :raises: SubscriptionNotFound if subscription is not found
    """
    if not has_permission(issuer=issuer, action='update_subscription', kwargs={'account': account}):
        raise AccessDenied('Account %s can not update subscription' % (issuer))
    try:
        if not isinstance(metadata, dict):
            raise TypeError('metadata should be a dict')
        if 'filter' in metadata and metadata['filter']:
            if not isinstance(metadata['filter'], dict):
                raise TypeError('filter should be a dict')
            validate_schema(name='subscription_filter', obj=metadata['filter'])
        if 'replication_rules' in metadata and metadata['replication_rules']:
            if not isinstance(metadata['replication_rules'], list):
                raise TypeError('replication_rules should be a list')
            else:
                for rule in metadata['replication_rules']:
                    validate_schema(name='activity', obj=rule.get('activity', 'default'))
    except ValueError as error:
        raise TypeError(error)
    return subscription.update_subscription(name=name, account=account, metadata=metadata)
示例#17
0
def list_dataset_replicas_bulk(dids, vo='def'):
    """
    :param dids: The list of did dictionaries with scope and name.
    :param vo: The VO to act on.

    :returns: A list of dict dataset replicas
    """

    validate_schema(name='r_dids', obj=dids, vo=vo)
    names_by_scope = dict()
    for d in dids:
        if d['scope'] in names_by_scope:
            names_by_scope[d['scope']].append(d['name'])
        else:
            names_by_scope[d['scope']] = [
                d['name'],
            ]

    names_by_intscope = dict()
    for scope in names_by_scope:
        internal_scope = InternalScope(scope, vo=vo)
        names_by_intscope[internal_scope] = names_by_scope[scope]

    replicas = replica.list_dataset_replicas_bulk(names_by_intscope)

    for r in replicas:
        yield api_update_return_dict(r)
示例#18
0
文件: did.py 项目: kbg/rucio
def attach_dids(scope, name, attachment, issuer):
    """
    Append content to data did.

    :param attachment: The attachment.
    :param issuer: The issuer account.
    """
    validate_schema(name='attachment', obj=attachment)

    rse_id = None
    if 'rse' in attachment:
        rse_id = get_rse_id(rse=attachment.get('rse'))
        attachment['rse_id'] = rse_id

    kwargs = {'scope': scope, 'name': name, 'attachment': attachment}
    if not rucio.api.permission.has_permission(
            issuer=issuer, action='attach_dids', kwargs=kwargs):
        raise rucio.common.exception.AccessDenied(
            'Account %s can not add data identifiers to %s:%s' %
            (issuer, scope, name))

    if rse_id is not None:
        dids = did.attach_dids(scope=scope,
                               name=name,
                               dids=attachment['dids'],
                               account=attachment.get('account', issuer),
                               rse_id=rse_id)
    else:
        dids = did.attach_dids(scope=scope,
                               name=name,
                               dids=attachment['dids'],
                               account=attachment.get('account', issuer))

    return dids
示例#19
0
def add_replicas(rse, files, issuer, ignore_availability=False, vo='def'):
    """
    Bulk add file replicas.

    :param rse: The RSE name.
    :param files: The list of files.
    :param issuer: The issuer account.
    :param ignore_availability: Ignore blocked RSEs.
    :param vo: The VO to act on.

    :returns: True is successful, False otherwise
    """
    for v_file in files:
        v_file.update({"type": "FILE"})  # Make sure DIDs are identified as files for checking
    validate_schema(name='dids', obj=files, vo=vo)

    rse_id = get_rse_id(rse=rse, vo=vo)

    kwargs = {'rse': rse, 'rse_id': rse_id}
    if not permission.has_permission(issuer=issuer, vo=vo, action='add_replicas', kwargs=kwargs):
        raise exception.AccessDenied('Account %s can not add file replicas on %s' % (issuer, rse))
    if not permission.has_permission(issuer=issuer, vo=vo, action='skip_availability_check', kwargs=kwargs):
        ignore_availability = False

    issuer = InternalAccount(issuer, vo=vo)
    for f in files:
        f['scope'] = InternalScope(f['scope'], vo=vo)
        if 'account' in f:
            f['account'] = InternalAccount(f['account'], vo=vo)

    replica.add_replicas(rse_id=rse_id, files=files, account=issuer, ignore_availability=ignore_availability)
示例#20
0
文件: replica.py 项目: ijjorama/rucio
def update_replicas_states(rse, files, issuer, vo='def'):
    """
    Update File replica information and state.

    :param rse: The RSE name.
    :param files: The list of files.
    :param issuer: The issuer account.
    :param vo: The VO to act on.
    """
    validate_schema(name='dids', obj=files)

    rse_id = get_rse_id(rse=rse, vo=vo)

    kwargs = {'rse': rse, 'rse_id': rse_id}
    if not permission.has_permission(
            issuer=issuer, vo=vo, action='update_replicas_states',
            kwargs=kwargs):
        raise exception.AccessDenied(
            'Account %s can not update file replicas state on %s' %
            (issuer, rse))
    replicas = []
    for file in files:
        rep = file
        rep['rse_id'] = rse_id
        rep['scope'] = InternalScope(rep['scope'], vo=vo)
        replicas.append(rep)
    replica.update_replicas_states(replicas=replicas)
示例#21
0
def list_dids_extended(scope, filters, did_type='collection', ignore_case=False, limit=None, offset=None, long=False, recursive=False, vo='def'):
    """
    List dids in a scope.

    :param scope: The scope name.
    :param pattern: The wildcard pattern.
    :param did_type:  The type of the did: all(container, dataset, file), collection(dataset or container), dataset, container
    :param ignore_case: Ignore case distinctions.
    :param limit: The maximum number of DIDs returned.
    :param offset: Offset number.
    :param long: Long format option to display more information for each DID.
    :param recursive: Recursively list DIDs content.
    """
    validate_schema(name='did_filters', obj=filters, vo=vo)
    scope = InternalScope(scope, vo=vo)

    if 'account' in filters:
        filters['account'] = InternalAccount(filters['account'], vo=vo)
    if 'scope' in filters:
        filters['scope'] = InternalScope(filters['scope'], vo=vo)

    result = did.list_dids_extended(scope=scope, filters=filters, did_type=did_type, ignore_case=ignore_case,
                                    limit=limit, offset=offset, long=long, recursive=recursive)

    for d in result:
        yield api_update_return_dict(d)
示例#22
0
def add_subscription(name,
                     account,
                     filter,
                     replication_rules,
                     comments,
                     lifetime,
                     retroactive,
                     dry_run,
                     priority=None,
                     issuer=None):
    """
    Adds a new subscription which will be verified against every new added file and dataset

    :param account: Account identifier
    :type account:  String
    :param name: Name of the subscription
    :type:  String
    :param filter: Dictionary of attributes by which the input data should be filtered
                   **Example**: ``{'dsn': 'data11_hi*.express_express.*,data11_hi*physics_MinBiasOverlay*', 'account': 'tzero'}``
    :type filter:  Dict
    :param replication_rules: Replication rules to be set : Dictionary with keys copies, rse_expression, weight, rse_expression
    :type replication_rules:  Dict
    :param comments: Comments for the subscription
    :type comments:  String
    :param lifetime: Subscription's lifetime (seconds); False if subscription has no lifetime
    :type lifetime:  Integer or False
    :param retroactive: Flag to know if the subscription should be applied on previous data
    :type retroactive:  Boolean
    :param dry_run: Just print the subscriptions actions without actually executing them (Useful if retroactive flag is set)
    :type dry_run:  Boolean
    :param priority: The priority of the subscription
    :type priority: Integer
    :param issuer:  The account issuing this operation.
    :type comments:  String
    :returns: subscription_id
    :rtype:   String
    """
    if not has_permission(issuer=issuer,
                          action='add_subscription',
                          kwargs={'account': account}):
        raise AccessDenied('Account %s can not add subscription' % (issuer))
    try:
        if filter:
            if type(filter) != dict:
                raise TypeError('filter should be a dict')
            validate_schema(name='subscription_filter', obj=filter)
        if replication_rules:
            if type(replication_rules) != list:
                raise TypeError('replication_rules should be a list')
            else:
                for rule in replication_rules:
                    validate_schema(name='activity',
                                    obj=rule.get('activity', 'default'))
        else:
            raise InvalidObject('You must specify a rule')
    except ValueError, error:
        raise TypeError(error)
示例#23
0
def resurrect(dids, issuer):
    """
    Resurrect DIDs.

    :param dids: A list of dids.
    :param issuer: The issuer account.
    """
    kwargs = {'issuer': issuer}
    if not rucio.api.permission.has_permission(issuer=issuer, action='resurrect', kwargs=kwargs):
        raise rucio.common.exception.AccessDenied('Account %s can not resurrect data identifiers' % (issuer))
    validate_schema(name='dids', obj=dids)
    return did.resurrect(dids=dids)
示例#24
0
def list_replicas(dids, schemes=None, unavailable=False, request_id=None, ignore_availability=True, all_states=False):
    """
    List file replicas for a list of data identifiers.

    :param dids: The list of data identifiers (DIDs).
    :param schemes: A list of schemes to filter the replicas. (e.g. file, http, ...)
    :param unavailable: Also include unavailable replicas in the list.
    :param request_id: ID associated with the request for debugging.
    :param all_states: Return all replicas whatever state they are in. Adds an extra 'states' entry in the result dictionary.
    """
    validate_schema(name='r_dids', obj=dids)
    return replica.list_replicas(dids=dids, schemes=schemes, unavailable=unavailable, request_id=request_id, ignore_availability=ignore_availability, all_states=all_states)
示例#25
0
文件: did.py 项目: pombredanne/rucio
def attach_dids_to_dids(attachments, issuer):
    """
    Append content to dids.

    :param attachments: The contents.
    :param issuer: The issuer account.
    """
    validate_schema(name='attachments', obj=attachments)

    if not rucio.api.permission.has_permission(issuer=issuer, action='attach_dids_to_dids', kwargs={'attachments': attachments}):
        raise rucio.common.exception.AccessDenied('Account %s can not add data identifiers' % (issuer))

    return did.attach_dids_to_dids(attachments=attachments, account=issuer)
示例#26
0
文件: account.py 项目: zzaiin/Rucio
def set_account_status(account, status, issuer='root'):
    """ Set the status of an account_core.

    :param account: Name of the account_core.
    :param status: The status for the account_core.
    """
    validate_schema(name='account', obj=account)
    kwargs = {}
    if not rucio.api.permission.has_permission(
            issuer=issuer, action='set_account_status', kwargs=kwargs):
        raise rucio.common.exception.AccessDenied(
            'Account %s can not change the status of the account' % (issuer))
    return account_core.set_account_status(account, status)
示例#27
0
def get_metadata_bulk(dids, vo='def', session=None):
    """
    Get metadata for a list of dids
    :param dids: A list of dids.
    :param session: The database session in use.
    """

    validate_schema(name='dids', obj=dids, vo=vo)
    for entry in dids:
        entry['scope'] = InternalScope(entry['scope'], vo=vo)
    meta = did.get_metadata_bulk(dids)
    for met in meta:
        yield api_update_return_dict(met)
示例#28
0
def attach_dids(scope, name, attachment, issuer, vo='def', session=None):
    """
    Append content to data did.

    :param attachment: The attachment.
    :param issuer: The issuer account.
    :param vo: The VO to act on.
    :param session: The database session in use.
    """
    validate_schema(name='attachment', obj=attachment, vo=vo)

    rse_id = None
    if 'rse' in attachment:
        if attachment['rse'] is not None:
            rse_id = get_rse_id(rse=attachment['rse'], vo=vo, session=session)
        attachment['rse_id'] = rse_id

    kwargs = {'scope': scope, 'name': name, 'attachment': attachment}
    if not rucio.api.permission.has_permission(issuer=issuer,
                                               vo=vo,
                                               action='attach_dids',
                                               kwargs=kwargs,
                                               session=session):
        raise rucio.common.exception.AccessDenied(
            'Account %s can not add data identifiers to %s:%s' %
            (issuer, scope, name))

    scope = InternalScope(scope, vo=vo)
    issuer = InternalAccount(issuer, vo=vo)
    if 'account' in attachment.keys():
        attachment['account'] = InternalAccount(attachment['account'], vo=vo)
    for d in attachment['dids']:
        d['scope'] = InternalScope(d['scope'], vo=vo)
        if 'account' in d.keys():
            d['account'] = InternalAccount(d['account'], vo=vo)

    if rse_id is not None:
        dids = did.attach_dids(scope=scope,
                               name=name,
                               dids=attachment['dids'],
                               account=attachment.get('account', issuer),
                               rse_id=rse_id,
                               session=session)
    else:
        dids = did.attach_dids(scope=scope,
                               name=name,
                               dids=attachment['dids'],
                               account=attachment.get('account', issuer),
                               session=session)

    return dids
示例#29
0
def import_data(data, issuer, vo='def'):
    """
    Import data to add/update/delete records in Rucio.

    :param data: data to be imported.
    :param issuer: the issuer.
    :param vo: the VO of the issuer.
    """
    kwargs = {'issuer': issuer}
    validate_schema(name='import', obj=data)
    if not permission.has_permission(issuer=issuer, vo=vo, action='import', kwargs=kwargs):
        raise exception.AccessDenied('Account %s can not import data' % issuer)

    return importer.import_data(data, vo=vo)
示例#30
0
def update_account(account, key, value, issuer='root'):
    """ Update a property of an account_core.

    :param account: Name of the account_core.
    :param key: Account property like status.
    :param value: Property value.
    """
    validate_schema(name='account', obj=account)
    kwargs = {}
    if not rucio.api.permission.has_permission(
            issuer=issuer, action='update_account', kwargs=kwargs):
        raise rucio.common.exception.AccessDenied(
            'Account %s can not change %s  of the account' % (issuer, key))
    return account_core.update_account(account, key, value)
示例#31
0
文件: did.py 项目: rcarpa/rucio
def get_metadata_bulk(dids, inherit=False, vo='def'):
    """
    Get metadata for a list of dids
    :param dids:               A list of dids.
    :param inherit:            A boolean. If set to true, the metadata of the parent are concatenated.
    :param vo:                 The VO to act on.
    """

    validate_schema(name='dids', obj=dids, vo=vo)
    for entry in dids:
        entry['scope'] = InternalScope(entry['scope'], vo=vo)
    meta = did.get_metadata_bulk(dids, inherit=inherit)
    for met in meta:
        yield api_update_return_dict(met)
示例#32
0
文件: rule.py 项目: pombredanne/rucio
def add_replication_rule(dids, copies, rse_expression, weight, lifetime, grouping, account, locked, subscription_id, source_replica_expression, activity, notify, purge_replicas, ignore_availability, issuer):
    """
    Adds a replication rule.

    :param dids:                       The data identifier set.
    :param copies:                     The number of replicas.
    :param rse_expression:             Boolean string expression to give the list of RSEs.
    :param weight:                     If the weighting option of the replication rule is used, the choice of RSEs takes their weight into account.
    :param lifetime:                   The lifetime of the replication rules (in seconds).
    :param grouping:                   ALL -  All files will be replicated to the same RSE.
                                       DATASET - All files in the same dataset will be replicated to the same RSE.
                                       NONE - Files will be completely spread over all allowed RSEs without any grouping considerations at all.
    :param account:                    The account owning the rule.
    :param locked:                     If the rule is locked, it cannot be deleted.
    :param subscription_id:            The subscription_id, if the rule is created by a subscription.
    :param source_replica_expression:  Only use replicas from this RSE as sources.
    :param activity:                   Activity to be passed on to the conveyor.
    :param notify:                     Notification setting of the rule.
    :purge purge_replicas:             The purge setting to delete replicas immediately after rule deletion.
    :param ignore_availability:        Option to ignore the availability of RSEs.
    :param issuer:                     The issuing account of this operation.
    :returns:                          List of created replication rules.
    """
    if account is None:
        account = issuer

    kwargs = {'dids': dids, 'copies': copies, 'rse_expression': rse_expression, 'weight': weight, 'lifetime': lifetime,
              'grouping': grouping, 'account': account, 'locked': locked, 'subscription_id': subscription_id,
              'source_replica_expression': source_replica_expression, 'notify': notify, 'activity': activity,
              'purge_replicas': purge_replicas, 'ignore_availability': ignore_availability}

    validate_schema(name='activity', obj=kwargs['activity'])

    if not has_permission(issuer=issuer, action='add_rule', kwargs=kwargs):
        raise AccessDenied('Account %s can not add replication rule' % (issuer))
    return rule.add_rule(account=account,
                         dids=dids,
                         copies=copies,
                         rse_expression=rse_expression,
                         grouping=grouping,
                         weight=weight,
                         lifetime=lifetime,
                         locked=locked,
                         subscription_id=subscription_id,
                         source_replica_expression=source_replica_expression,
                         activity=activity,
                         notify=notify,
                         purge_replicas=purge_replicas,
                         ignore_availability=ignore_availability)
示例#33
0
def attach_dids_to_dids(attachments, issuer, ignore_duplicate=False):
    """
    Append content to dids.

    :param attachments: The contents.
    :param issuer: The issuer account.
    :param ignore_duplicate: If True, ignore duplicate entries.
    """
    validate_schema(name='attachments', obj=attachments)

    if not rucio.api.permission.has_permission(issuer=issuer, action='attach_dids_to_dids', kwargs={'attachments': attachments}):
        raise rucio.common.exception.AccessDenied('Account %s can not add data identifiers' % (issuer))

    return did.attach_dids_to_dids(attachments=attachments, account=issuer,
                                   ignore_duplicate=ignore_duplicate)
示例#34
0
文件: did.py 项目: pombredanne/rucio
def attach_dids(scope, name, attachment, issuer):
    """
    Append content to data did.

    :param attachment: The attachment.
    :param issuer: The issuer account.
    """
    validate_schema(name='attachment', obj=attachment)

    kwargs = {'scope': scope, 'name': name, 'attachment': attachment}
    if not rucio.api.permission.has_permission(issuer=issuer, action='attach_dids', kwargs=kwargs):
        raise rucio.common.exception.AccessDenied('Account %s can not add data identifiers to %s:%s' % (issuer, scope, name))

    return did.attach_dids(scope=scope, name=name, dids=attachment['dids'],
                           account=attachment.get('account', issuer), rse=attachment.get('rse'))
示例#35
0
def add_scope(scope, account, issuer):
    """
    Creates a scope for an account.

    :param account: The account name.
    :param scope: The scope identifier.
    :param issuer: The issuer account.
    """

    validate_schema(name='scope', obj=scope)

    kwargs = {'scope': scope, 'account': account}
    if not rucio.api.permission.has_permission(issuer=issuer, action='add_scope', kwargs=kwargs):
        raise rucio.common.exception.AccessDenied('Account %s can not add scope' % (issuer))

    core_scope.add_scope(scope, account)
示例#36
0
def add_account_attribute(key, value, account, issuer):
    """
    Add an attribute to an account.

    :param key: attribute key.
    :param value: attribute value.
    :param account: The account name.
    :param issuer: The issuer account_core.
    """
    validate_schema(name="account_attribute", obj=key)
    validate_schema(name="account_attribute", obj=value)

    kwargs = {"account": account, "key": key, "value": value}
    if not rucio.api.permission.has_permission(issuer=issuer, action="add_attribute", kwargs=kwargs):
        raise rucio.common.exception.AccessDenied("Account %s can not add attributes" % (issuer))
    account_core.add_account_attribute(account, key, value)
示例#37
0
def add_account(account, type, issuer):
    """
    Creates an account with the provided account name, contact information, etc.

    :param account: The account name.
    :param type: The account type
    :param issuer: The issuer account_core.

    """

    validate_schema(name="account", obj=account)

    kwargs = {"account": account, "type": type}
    if not rucio.api.permission.has_permission(issuer=issuer, action="add_account", kwargs=kwargs):
        raise rucio.common.exception.AccessDenied("Account %s can not add account" % (issuer))

    account_core.add_account(account, AccountType.from_sym(type))
示例#38
0
def resurrect(dids, issuer, vo='def'):
    """
    Resurrect DIDs.

    :param dids: A list of dids.
    :param issuer: The issuer account.
    :param vo: The VO to act on.
    """
    kwargs = {'issuer': issuer}
    if not rucio.api.permission.has_permission(issuer=issuer, vo=vo, action='resurrect', kwargs=kwargs):
        raise rucio.common.exception.AccessDenied('Account %s can not resurrect data identifiers' % (issuer))
    validate_schema(name='dids', obj=dids, vo=vo)

    for d in dids:
        d['scope'] = InternalScope(d['scope'], vo=vo)

    return did.resurrect(dids=dids)
示例#39
0
def delete_replicas(rse, files, issuer, ignore_availability=False):
    """
    Bulk delete file replicas.

    :param rse: The RSE name.
    :param files: The list of files.
    :param issuer: The issuer account.
    :param ignore_availability: Ignore the RSE blacklisting.

    :returns: True is successful, False otherwise
    """
    validate_schema(name='r_dids', obj=files)

    kwargs = {'rse': rse}
    if not permission.has_permission(issuer=issuer, action='delete_replicas', kwargs=kwargs):
        raise exception.AccessDenied('Account %s can not delete file replicas on %s' % (issuer, rse))
    if not permission.has_permission(issuer=issuer, action='skip_availability_check', kwargs=kwargs):
        ignore_availability = False
    replica.delete_replicas(rse=rse, files=files, ignore_availability=ignore_availability)
示例#40
0
def update_replicas_states(rse, files, issuer):

    """
    Update File replica information and state.

    :param rse: The RSE name.
    :param files: The list of files.
    :param issuer: The issuer account.
    """
    validate_schema(name='dids', obj=files)

    kwargs = {'rse': rse}
    if not permission.has_permission(issuer=issuer, action='update_replicas_states', kwargs=kwargs):
        raise exception.AccessDenied('Account %s can not update file replicas state on %s' % (issuer, rse))
    replicas = []
    for file in files:
        rep = file
        rep['rse'] = rse
        replicas.append(rep)
    replica.update_replicas_states(replicas=replicas)
示例#41
0
def add_subscription(name, account, filter, replication_rules, comments, lifetime, retroactive, dry_run):
    """
    Adds a new subscription which will be verified against every new added file and dataset

    :param account: Account identifier
    :type account:  String
    :param name: Name of the subscription
    :type:  String
    :param filter: Dictionary of attributes by which the input data should be filtered
                   **Example**: ``{'dsn': 'data11_hi*.express_express.*,data11_hi*physics_MinBiasOverlay*', 'account': 'tzero'}``
    :type filter:  Dict
    :param replication_rules: Replication rules to be set : Dictionary with keys copies, rse_expression, weight, rse_expression
    :type replication_rules:  Dict
    :param comments: Comments for the subscription
    :type comments:  String
    :param lifetime: Subscription's lifetime (seconds); False if subscription has no lifetime
    :type lifetime:  Integer or False
    :param retroactive: Flag to know if the subscription should be applied on previous data
    :type retroactive:  Boolean
    :param dry_run: Just print the subscriptions actions without actually executing them (Useful if retroactive flag is set)
    :type dry_run:  Boolean
    :returns: subscription_id
    :rtype:   String
    """
    accepted_keys = ['datatype', 'prod_step', 'stream_name', 'project', 'scope', 'pattern', 'type', 'account', 'excluded_pattern', 'grouping', 'group', 'provenance']
    try:
        if filter:
            if type(filter) != dict:
                raise TypeError('filter should be a dict')
            else:
                for key in filter:
                    if key not in accepted_keys:
                        raise InvalidMetadata('Invalid metadata <%s>' % (key))
            validate_schema(name='subscription_filter', obj=filter)
        if replication_rules and type(replication_rules) != list:
            raise TypeError('replication_rules should be a list')
    except ValueError, e:
        raise TypeError(e)
示例#42
0
文件: rse.py 项目: pombredanne/rucio
def add_rse(rse, issuer, deterministic=True, volatile=False, city=None, region_code=None, country_name=None, continent=None, time_zone=None, ISP=None, staging_area=False):
    """
    Creates a new Rucio Storage Element(RSE).

    :param rse: The RSE name.
    :param issuer: The issuer account.
    :param deterministic: Boolean to know if the pfn is generated deterministically.
    :param volatile: Boolean for RSE cache.
    :param city: City for the RSE.
    :param region_code: The region code for the RSE.
    :param country_name: The country.
    :param continent: The continent.
    :param time_zone: Timezone.
    :param staging_area: staging area.
    :param ISP: Internet service provider.
    """
    validate_schema(name='rse', obj=rse)
    kwargs = {'rse': rse}
    if not permission.has_permission(issuer=issuer, action='add_rse', kwargs=kwargs):
        raise exception.AccessDenied('Account %s can not add RSE' % (issuer))

    return rse_module.add_rse(rse, deterministic=deterministic, volatile=volatile, city=city,
                              region_code=region_code, country_name=country_name, staging_area=staging_area,
                              continent=continent, time_zone=time_zone, ISP=ISP)