def get_signed_url(service, operation, url, lifetime=600): """ Get a signed URL for a particular service and operation. The signed URL will be valid for 1 hour. :param service: The service to authorise, currently only 'gsc'. :param operation: The operation to sign, either 'read', 'write', or 'delete'. :param url: The URL to sign. :param lifetime: Lifetime of the signed URL in seconds. :returns: Signed URL as a variable-length string. """ if service not in ['gcs']: raise UnsupportedOperation('Service must be "gcs"') if operation not in ['read', 'write', 'delete']: raise UnsupportedOperation( 'Operation must be "read", "write", or "delete"') if url is None or url == '': raise UnsupportedOperation('URL must not be empty') if not isinstance(lifetime, (int, long)) and lifetime > 0: raise UnsupportedOperation('Lifetime must be greater than 0.') signed_url = None if service == 'gcs': # select the correct operation operations = {'read': 'GET', 'write': 'PUT', 'delete': 'DELETE'} operation = operations[operation] # special case to test signature, force epoch time if lifetime is None: lifetime = 0 else: # GCS is timezone-sensitive, don't use UTC # has to be converted to Unixtime lifetime = datetime.datetime.now() + datetime.timedelta( seconds=lifetime) lifetime = int(time.mktime(lifetime.timetuple())) # sign the path only path = urlparse.urlparse(url).path # assemble message to sign to_sign = "%s\n\n\n%s\n%s" % (operation, lifetime, path) # create URL-capable signature # first character is always a '=', remove it signature = urllib.urlencode( {'': base64.b64encode(CREDS_GCS.sign_blob(to_sign)[1])})[1:] # assemble final signed URL signed_url = 'https://storage.googleapis.com%s?GoogleAccessId=%s&Expires=%s&Signature=%s' % ( path, CREDS_GCS.service_account_email, lifetime, signature) return signed_url
def get_account_limits(self, account, rse_expression, locality): """ Return the correct account limits for the given locality. :param account: The account name. :param rse_expression: Valid RSE expression :param locality: The scope of the account limit. 'local' or 'global'. """ if locality == 'local': return self.get_local_account_limit(account, rse_expression) elif locality == 'global': return self.get_global_account_limit(account, rse_expression) else: from rucio.common.exception import UnsupportedOperation raise UnsupportedOperation('The provided locality (%s) for the account limit was invalid' % locality)
def delete_account_limit(self, account, rse, locality): """ Deletes an account limit for a given limit scope. :param account: The name of the account. :param rse: The rse name. :param locality: The scope of the account limit. 'local' or 'global'. :return: True if quota was created successfully else False. """ if locality == 'local': return self.delete_local_account_limit(account, rse) elif locality == 'global': return self.delete_global_account_limit(account, rse) else: from rucio.common.exception import UnsupportedOperation raise UnsupportedOperation('The provided scope (%s) for the account limit was invalid' % locality)
def __set_transfer_state(external_host, transfer_id, new_state, session=None): """ Update the state of a transfer. Fails silently if the transfer_id does not exist. :param external_host: Selected external host as string in format protocol://fqdn:port :param transfer_id: External transfer job id as a string. :param new_state: New state as string. :param session: Database session to use. """ record_counter('core.request.set_transfer_state') try: rowcount = session.query(models.Request).filter_by(external_id=transfer_id).update({'state': new_state, 'updated_at': datetime.datetime.utcnow()}, synchronize_session=False) except IntegrityError as error: raise RucioException(error.args) if not rowcount: raise UnsupportedOperation("Transfer %s on %s state %s cannot be updated." % (transfer_id, external_host, new_state))
def set_transfer_update_time(external_host, transfer_id, update_time=datetime.datetime.utcnow(), session=None): """ Update the state of a request. Fails silently if the transfer_id does not exist. :param external_host: Selected external host as string in format protocol://fqdn:port :param transfer_id: External transfer job id as a string. :param update_time: Time stamp. :param session: Database session to use. """ record_counter('core.request.set_transfer_update_time') try: rowcount = session.query(models.Request).filter_by(external_id=transfer_id, state=RequestState.SUBMITTED).update({'updated_at': update_time}, synchronize_session=False) except IntegrityError as error: raise RucioException(error.args) if not rowcount: raise UnsupportedOperation("Transfer %s doesn't exist or its status is not submitted." % (transfer_id))
def add_files(lfns, account, ignore_availability, session=None): """ Bulk add files : - Create the file and replica. - If doesn't exist create the dataset containing the file as well as a rule on the dataset on ANY sites. - Create all the ascendants of the dataset if they do not exist :param lfns: List of lfn (dictionary {'lfn': <lfn>, 'rse': <rse>, 'bytes': <bytes>, 'adler32': <adler32>, 'guid': <guid>, 'pfn': <pfn>} :param issuer: The issuer account. :param ignore_availability: A boolean to ignore blocklisted sites. :session: The session used """ attachments = [] # The list of scopes is necessary for the extract_scope scopes = list_scopes(session=session) scopes = [scope.external for scope in scopes] exist_lfn = [] for lfn in lfns: # First check if the file exists filename = lfn['lfn'] lfn_scope, _ = extract_scope(filename, scopes) lfn_scope = InternalScope(lfn_scope) exists, did_type = _exists(lfn_scope, filename) if exists: continue # Get all the ascendants of the file lfn_split = filename.split('/') lpns = ["/".join(lfn_split[:idx]) for idx in range(2, len(lfn_split))] lpns.reverse() print(lpns) # The parent must be a dataset. Register it as well as the rule dsn_name = lpns[0] dsn_scope, _ = extract_scope(dsn_name, scopes) dsn_scope = InternalScope(dsn_scope) exists, did_type = _exists(dsn_scope, dsn_name) if exists and did_type == DIDType.CONTAINER: raise UnsupportedOperation('Cannot create %s as dataset' % dsn_name) if (dsn_name not in exist_lfn) and not exists: print('Will create %s' % dsn_name) add_did(scope=dsn_scope, name=dsn_name, type=DIDType.DATASET, account=InternalAccount(account), statuses=None, meta=None, rules=[{ 'copies': 1, 'rse_expression': 'ANY=true', 'weight': None, 'account': InternalAccount(account), 'lifetime': None, 'grouping': 'NONE' }], lifetime=None, dids=None, rse_id=None, session=session) exist_lfn.append(dsn_name) parent_name = lpns[1] parent_scope, _ = extract_scope(parent_name, scopes) parent_scope = InternalScope(parent_scope) attachments.append({ 'scope': parent_scope, 'name': parent_name, 'dids': [{ 'scope': dsn_scope, 'name': dsn_name }] }) # Register the file rse_id = lfn.get('rse_id', None) if not rse_id: raise InvalidType('Missing rse_id') bytes = lfn.get('bytes', None) guid = lfn.get('guid', None) adler32 = lfn.get('adler32', None) pfn = lfn.get('pfn', None) files = { 'scope': lfn_scope, 'name': filename, 'bytes': bytes, 'adler32': adler32 } if pfn: files['pfn'] = str(pfn) if guid: files['meta'] = {'guid': guid} add_replicas(rse_id=rse_id, files=[files], dataset_meta=None, account=InternalAccount(account), ignore_availability=ignore_availability, session=session) add_rule(dids=[{ 'scope': lfn_scope, 'name': filename }], account=InternalAccount(account), copies=1, rse_expression=lfn['rse'], grouping=None, weight=None, lifetime=86400, locked=None, subscription_id=None, session=session) attachments.append({ 'scope': dsn_scope, 'name': dsn_name, 'dids': [{ 'scope': lfn_scope, 'name': filename }] }) # Now loop over the ascendants of the dataset and created them for lpn in lpns[1:]: child_scope, _ = extract_scope(lpn, scopes) child_scope = InternalScope(child_scope) exists, did_type = _exists(child_scope, lpn) if exists and did_type == DIDType.DATASET: raise UnsupportedOperation('Cannot create %s as container' % lpn) if (lpn not in exist_lfn) and not exists: print('Will create %s' % lpn) add_did(scope=child_scope, name=lpn, type=DIDType.CONTAINER, account=InternalAccount(account), statuses=None, meta=None, rules=None, lifetime=None, dids=None, rse_id=None, session=session) exist_lfn.append(lpn) parent_name = lpns[lpns.index(lpn) + 1] parent_scope, _ = extract_scope(parent_name, scopes) parent_scope = InternalScope(parent_scope) attachments.append({ 'scope': parent_scope, 'name': parent_name, 'dids': [{ 'scope': child_scope, 'name': lpn }] }) # Finally attach everything attach_dids_to_dids(attachments, account=InternalAccount(account), ignore_duplicate=True, session=session)
def get_signed_url(rse_id, service, operation, url, lifetime=600): """ Get a signed URL for a particular service and operation. The signed URL will be valid for 1 hour but can be overriden. :param rse_id: The ID of the RSE that the URL points to. :param service: The service to authorise, either 'gcs', 's3' or 'swift'. :param operation: The operation to sign, either 'read', 'write', or 'delete'. :param url: The URL to sign. :param lifetime: Lifetime of the signed URL in seconds. :returns: Signed URL as a variable-length string. """ global CREDS_GCS if service not in ['gcs', 's3', 'swift']: raise UnsupportedOperation('Service must be "gcs", "s3" or "swift"') if operation not in ['read', 'write', 'delete']: raise UnsupportedOperation( 'Operation must be "read", "write", or "delete"') if url is None or url == '': raise UnsupportedOperation('URL must not be empty') if lifetime: if not isinstance(lifetime, integer_types): try: lifetime = int(lifetime) except: raise UnsupportedOperation( 'Lifetime must be convertible to numeric.') signed_url = None if service == 'gcs': if not CREDS_GCS: CREDS_GCS = ServiceAccountCredentials.from_json_keyfile_name( config_get( 'credentials', 'gcs', raise_exception=False, default='/opt/rucio/etc/google-cloud-storage-test.json')) components = urlparse(url) host = components.netloc # select the correct operation operations = {'read': 'GET', 'write': 'PUT', 'delete': 'DELETE'} operation = operations[operation] # special case to test signature, force epoch time if lifetime is None: lifetime = 0 else: # GCS is timezone-sensitive, don't use UTC # has to be converted to Unixtime lifetime = datetime.datetime.now() + datetime.timedelta( seconds=lifetime) lifetime = int(time.mktime(lifetime.timetuple())) # sign the path only path = components.path # assemble message to sign to_sign = "%s\n\n\n%s\n%s" % (operation, lifetime, path) # create URL-capable signature # first character is always a '=', remove it signature = urlencode( {'': base64.b64encode(CREDS_GCS.sign_blob(to_sign)[1])})[1:] # assemble final signed URL signed_url = 'https://%s%s?GoogleAccessId=%s&Expires=%s&Signature=%s' % ( host, path, CREDS_GCS.service_account_email, lifetime, signature) elif service == 's3': # split URL to get hostname, bucket and key components = urlparse(url) host = components.netloc pathcomponents = components.path.split('/') if len(pathcomponents) < 3: raise UnsupportedOperation('Not a valid S3 URL') bucket = pathcomponents[1] key = '/'.join(pathcomponents[2:]) # remove port number from host if present colon = host.find(':') port = '443' if colon >= 0: port = host[colon + 1:] host = host[:colon] # look up in RSE account configuration by RSE ID cred_name = rse_id cred = REGION.get('s3-%s' % cred_name) if cred is NO_VALUE: rse_cred = get_rse_credentials() cred = rse_cred.get(cred_name) REGION.set('s3-%s' % cred_name, cred) access_key = cred['access_key'] secret_key = cred['secret_key'] signature_version = cred['signature_version'] region_name = cred['region'] if operation == 'read': s3op = 'get_object' elif operation == 'write': s3op = 'put_object' else: s3op = 'delete_object' with record_timer_block('credential.signs3'): s3 = boto3.client('s3', endpoint_url='https://' + host + ':' + port, aws_access_key_id=access_key, aws_secret_access_key=secret_key, config=Config( signature_version=signature_version, region_name=region_name)) signed_url = s3.generate_presigned_url(s3op, Params={ 'Bucket': bucket, 'Key': key }, ExpiresIn=lifetime) elif service == 'swift': # split URL to get hostname and path components = urlparse(url) host = components.netloc # remove port number from host if present colon = host.find(':') if colon >= 0: host = host[:colon] # use RSE ID to look up key cred_name = rse_id # look up tempurl signing key cred = REGION.get('swift-%s' % cred_name) if cred is NO_VALUE: rse_cred = get_rse_credentials() cred = rse_cred.get(cred_name) REGION.set('swift-%s' % cred_name, cred) tempurl_key = cred['tempurl_key'] if operation == 'read': swiftop = 'GET' elif operation == 'write': swiftop = 'PUT' else: swiftop = 'DELETE' expires = int(time.time() + lifetime) # create signed URL with record_timer_block('credential.signswift'): hmac_body = u'%s\n%s\n%s' % (swiftop, expires, components.path) # Python 3 hmac only accepts bytes or bytearray sig = hmac.new(bytearray(tempurl_key, 'utf-8'), bytearray(hmac_body, 'utf-8'), sha1).hexdigest() signed_url = 'https://' + host + components.path + '?temp_url_sig=' + sig + '&temp_url_expires=' + str( expires) return signed_url
def add_files(lfns, account, ignore_availability, vo='def', session=None): """ Bulk add files : - Create the file and replica. - If doesn't exist create the dataset containing the file as well as a rule on the dataset on ANY sites. - Create all the ascendants of the dataset if they do not exist :param lfns: List of lfn (dictionary {'lfn': <lfn>, 'rse': <rse>, 'bytes': <bytes>, 'adler32': <adler32>, 'guid': <guid>, 'pfn': <pfn>} :param issuer: The issuer account. :param ignore_availability: A boolean to ignore blocklisted sites. :param vo: The VO to act on :param session: The session used """ rule_extension_list = [] attachments = [] # The list of scopes is necessary for the extract_scope filter_ = {'scope': InternalScope(scope='*', vo=vo)} scopes = list_scopes(filter_=filter_, session=session) scopes = [scope.external for scope in scopes] exist_lfn = [] try: lifetime_dict = config_get(section='dirac', option='lifetime', session=session) lifetime_dict = loads(lifetime_dict) except ConfigNotFound: lifetime_dict = {} except JSONDecodeError as err: raise InvalidType('Problem parsing lifetime option in dirac section : %s' % str(err)) except Exception as err: raise RucioException(str(err)) for lfn in lfns: # First check if the file exists filename = lfn['lfn'] lfn_scope, _ = extract_scope(filename, scopes) lfn_scope = InternalScope(lfn_scope, vo=vo) exists, did_type = _exists(lfn_scope, filename) if exists: continue # Get all the ascendants of the file lfn_split = filename.split('/') lpns = ["/".join(lfn_split[:idx]) for idx in range(2, len(lfn_split))] lpns.reverse() print(lpns) # The parent must be a dataset. Register it as well as the rule dsn_name = lpns[0] dsn_scope, _ = extract_scope(dsn_name, scopes) dsn_scope = InternalScope(dsn_scope, vo=vo) # Compute lifetime lifetime = None if dsn_scope in lifetime_dict: lifetime = lifetime_dict[dsn_scope] else: for pattern in lifetime_dict: if re.match(pattern, dsn_scope): lifetime = lifetime_dict[pattern] break exists, did_type = _exists(dsn_scope, dsn_name) if exists and did_type == DIDType.CONTAINER: raise UnsupportedOperation('Cannot create %s as dataset' % dsn_name) if (dsn_name not in exist_lfn) and not exists: print('Will create %s' % dsn_name) # to maintain a compatibility between master and LTS-1.26 branches remove keywords for first 3 arguments add_did(dsn_scope, dsn_name, DIDType.DATASET, account=InternalAccount(account, vo=vo), statuses=None, meta=None, rules=[{'copies': 1, 'rse_expression': 'ANY=true', 'weight': None, 'account': InternalAccount(account, vo=vo), 'lifetime': None, 'grouping': 'NONE'}], lifetime=None, dids=None, rse_id=None, session=session) exist_lfn.append(dsn_name) parent_name = lpns[1] parent_scope, _ = extract_scope(parent_name, scopes) parent_scope = InternalScope(parent_scope, vo=vo) attachments.append({'scope': parent_scope, 'name': parent_name, 'dids': [{'scope': dsn_scope, 'name': dsn_name}]}) rule_extension_list.append((dsn_scope, dsn_name)) if lifetime and (dsn_scope, dsn_name) not in rule_extension_list: # Reset the lifetime of the rule to the configured value rule = [rul for rul in list_rules({'scope': dsn_scope, 'name': dsn_name, 'account': InternalAccount(account, vo=vo)}, session=session) if rul['rse_expression'] == 'ANY=true'] if rule: update_rule(rule[0]['id'], options={'lifetime': lifetime}, session=session) rule_extension_list.append((dsn_scope, dsn_name)) # Register the file rse_id = lfn.get('rse_id', None) if not rse_id: raise InvalidType('Missing rse_id') bytes_ = lfn.get('bytes', None) guid = lfn.get('guid', None) adler32 = lfn.get('adler32', None) pfn = lfn.get('pfn', None) files = {'scope': lfn_scope, 'name': filename, 'bytes': bytes_, 'adler32': adler32} if pfn: files['pfn'] = str(pfn) if guid: files['meta'] = {'guid': guid} add_replicas(rse_id=rse_id, files=[files], dataset_meta=None, account=InternalAccount(account, vo=vo), ignore_availability=ignore_availability, session=session) add_rule(dids=[{'scope': lfn_scope, 'name': filename}], account=InternalAccount(account, vo=vo), copies=1, rse_expression=lfn['rse'], grouping=None, weight=None, lifetime=86400, locked=None, subscription_id=None, session=session) attachments.append({'scope': dsn_scope, 'name': dsn_name, 'dids': [{'scope': lfn_scope, 'name': filename}]}) # Now loop over the ascendants of the dataset and created them for lpn in lpns[1:]: child_scope, _ = extract_scope(lpn, scopes) child_scope = InternalScope(child_scope, vo=vo) exists, did_type = _exists(child_scope, lpn) if exists and did_type == DIDType.DATASET: raise UnsupportedOperation('Cannot create %s as container' % lpn) if (lpn not in exist_lfn) and not exists: print('Will create %s' % lpn) add_did(child_scope, lpn, DIDType.CONTAINER, account=InternalAccount(account, vo=vo), statuses=None, meta=None, rules=None, lifetime=None, dids=None, rse_id=None, session=session) exist_lfn.append(lpn) parent_name = lpns[lpns.index(lpn) + 1] parent_scope, _ = extract_scope(parent_name, scopes) parent_scope = InternalScope(parent_scope, vo=vo) attachments.append({'scope': parent_scope, 'name': parent_name, 'dids': [{'scope': child_scope, 'name': lpn}]}) # Finally attach everything attach_dids_to_dids(attachments, account=InternalAccount(account, vo=vo), ignore_duplicate=True, session=session)
def add_exception(dids, account, pattern, comments, expires_at, session=None): """ Add exceptions to Lifetime Model. :param dids: The list of dids :param account: The account of the requester. :param pattern: The account. :param comments: The comments associated to the exception. :param expires_at: The expiration date of the exception. :param session: The database session in use. returns: A dictionary with id of the exceptions split by scope, datatype. """ from rucio.core.did import get_metadata_bulk result = dict() result['exceptions'] = dict() try: max_extension = config_get('lifetime_model', 'max_extension', default=None, session=session) if max_extension: if not expires_at: expires_at = datetime.utcnow() + timedelta(days=max_extension) else: if isinstance(expires_at, string_types): expires_at = str_to_date(expires_at) if expires_at > datetime.utcnow() + timedelta( days=max_extension): expires_at = datetime.utcnow() + timedelta( days=max_extension) except (ConfigNotFound, ValueError, NoSectionError): max_extension = None try: cutoff_date = config_get('lifetime_model', 'cutoff_date', default=None, session=session) except (ConfigNotFound, NoSectionError): raise UnsupportedOperation('Cannot submit exception at that date.') try: cutoff_date = datetime.strptime(cutoff_date, '%Y-%m-%d') except ValueError: raise UnsupportedOperation('Cannot submit exception at that date.') if cutoff_date < datetime.utcnow(): raise UnsupportedOperation('Cannot submit exception at that date.') did_group = dict() not_affected = list() list_dids = [(did['scope'], did['name']) for did in dids] metadata = [meta for meta in get_metadata_bulk(dids=dids, session=session)] for did in metadata: scope, name, did_type = did['scope'], did['name'], did['did_type'] if (scope, name) in list_dids: list_dids.remove((scope, name)) datatype = did.get('datatype', '') eol_at = did.get('eol_at', None) if eol_at and eol_at < cutoff_date: if (scope, datatype) not in did_group: did_group[(scope, datatype)] = [list(), 0] did_group[(scope, datatype)][0].append({ 'scope': scope, 'name': name, 'did_type': did_type }) did_group[(scope, datatype)][1] += did['bytes'] or 0 else: not_affected.append((scope, name, did_type)) for entry in did_group: exception_id = __add_exception(did_group[entry][0], account=account, pattern=pattern, comments=comments, expires_at=expires_at, estimated_volume=did_group[entry][1], session=session) result['exceptions'][exception_id] = did_group[entry][0] result['unknown'] = [{ 'scope': did[0], 'name': did[1], 'did_type': DIDType.DATASET } for did in list_dids] result['not_affected'] = [{ 'scope': did[0], 'name': did[1], 'did_type': did[2] } for did in not_affected] return result