def get_signed_url(service, operation, url, lifetime=600):
"""
Get a signed URL for a particular service and operation.
The signed URL will be valid for 1 hour.
:param service: The service to authorise, currently only 'gsc'.
:param operation: The operation to sign, either 'read', 'write', or 'delete'.
:param url: The URL to sign.
:param lifetime: Lifetime of the signed URL in seconds.
:returns: Signed URL as a variable-length string.
"""
if service not in ['gcs']:
raise UnsupportedOperation('Service must be "gcs"')
if operation not in ['read', 'write', 'delete']:
raise UnsupportedOperation(
'Operation must be "read", "write", or "delete"')
if url is None or url == '':
raise UnsupportedOperation('URL must not be empty')
if not isinstance(lifetime, (int, long)) and lifetime > 0:
raise UnsupportedOperation('Lifetime must be greater than 0.')
signed_url = None
if service == 'gcs':
# select the correct operation
operations = {'read': 'GET', 'write': 'PUT', 'delete': 'DELETE'}
operation = operations[operation]
# special case to test signature, force epoch time
if lifetime is None:
lifetime = 0
else:
# GCS is timezone-sensitive, don't use UTC
# has to be converted to Unixtime
lifetime = datetime.datetime.now() + datetime.timedelta(
seconds=lifetime)
lifetime = int(time.mktime(lifetime.timetuple()))
# sign the path only
path = urlparse.urlparse(url).path
# assemble message to sign
to_sign = "%s\n\n\n%s\n%s" % (operation, lifetime, path)
# create URL-capable signature
# first character is always a '=', remove it
signature = urllib.urlencode(
{'': base64.b64encode(CREDS_GCS.sign_blob(to_sign)[1])})[1:]
# assemble final signed URL
signed_url = 'https://storage.googleapis.com%s?GoogleAccessId=%s&Expires=%s&Signature=%s' % (
path, CREDS_GCS.service_account_email, lifetime, signature)
return signed_url
def get_account_limits(self, account, rse_expression, locality):
"""
Return the correct account limits for the given locality.
:param account: The account name.
:param rse_expression: Valid RSE expression
:param locality: The scope of the account limit. 'local' or 'global'.
"""
if locality == 'local':
return self.get_local_account_limit(account, rse_expression)
elif locality == 'global':
return self.get_global_account_limit(account, rse_expression)
else:
from rucio.common.exception import UnsupportedOperation
raise UnsupportedOperation('The provided locality (%s) for the account limit was invalid' % locality)
def delete_account_limit(self, account, rse, locality):
"""
Deletes an account limit for a given limit scope.
:param account: The name of the account.
:param rse: The rse name.
:param locality: The scope of the account limit. 'local' or 'global'.
:return: True if quota was created successfully else False.
"""
if locality == 'local':
return self.delete_local_account_limit(account, rse)
elif locality == 'global':
return self.delete_global_account_limit(account, rse)
else:
from rucio.common.exception import UnsupportedOperation
raise UnsupportedOperation('The provided scope (%s) for the account limit was invalid' % locality)
def __set_transfer_state(external_host, transfer_id, new_state, session=None):
"""
Update the state of a transfer. Fails silently if the transfer_id does not exist.
:param external_host: Selected external host as string in format protocol://fqdn:port
:param transfer_id: External transfer job id as a string.
:param new_state: New state as string.
:param session: Database session to use.
"""
record_counter('core.request.set_transfer_state')
try:
rowcount = session.query(models.Request).filter_by(external_id=transfer_id).update({'state': new_state, 'updated_at': datetime.datetime.utcnow()}, synchronize_session=False)
except IntegrityError as error:
raise RucioException(error.args)
if not rowcount:
raise UnsupportedOperation("Transfer %s on %s state %s cannot be updated." % (transfer_id, external_host, new_state))
def set_transfer_update_time(external_host, transfer_id, update_time=datetime.datetime.utcnow(), session=None):
"""
Update the state of a request. Fails silently if the transfer_id does not exist.
:param external_host: Selected external host as string in format protocol://fqdn:port
:param transfer_id: External transfer job id as a string.
:param update_time: Time stamp.
:param session: Database session to use.
"""
record_counter('core.request.set_transfer_update_time')
try:
rowcount = session.query(models.Request).filter_by(external_id=transfer_id, state=RequestState.SUBMITTED).update({'updated_at': update_time}, synchronize_session=False)
except IntegrityError as error:
raise RucioException(error.args)
if not rowcount:
raise UnsupportedOperation("Transfer %s doesn't exist or its status is not submitted." % (transfer_id))
def add_files(lfns, account, ignore_availability, session=None):
"""
Bulk add files :
- Create the file and replica.
- If doesn't exist create the dataset containing the file as well as a rule on the dataset on ANY sites.
- Create all the ascendants of the dataset if they do not exist
:param lfns: List of lfn (dictionary {'lfn': <lfn>, 'rse': <rse>, 'bytes': <bytes>, 'adler32': <adler32>, 'guid': <guid>, 'pfn': <pfn>}
:param issuer: The issuer account.
:param ignore_availability: A boolean to ignore blocklisted sites.
:session: The session used
"""
attachments = []
# The list of scopes is necessary for the extract_scope
scopes = list_scopes(session=session)
scopes = [scope.external for scope in scopes]
exist_lfn = []
for lfn in lfns:
# First check if the file exists
filename = lfn['lfn']
lfn_scope, _ = extract_scope(filename, scopes)
lfn_scope = InternalScope(lfn_scope)
exists, did_type = _exists(lfn_scope, filename)
if exists:
continue
# Get all the ascendants of the file
lfn_split = filename.split('/')
lpns = ["/".join(lfn_split[:idx]) for idx in range(2, len(lfn_split))]
lpns.reverse()
print(lpns)
# The parent must be a dataset. Register it as well as the rule
dsn_name = lpns[0]
dsn_scope, _ = extract_scope(dsn_name, scopes)
dsn_scope = InternalScope(dsn_scope)
exists, did_type = _exists(dsn_scope, dsn_name)
if exists and did_type == DIDType.CONTAINER:
raise UnsupportedOperation('Cannot create %s as dataset' %
dsn_name)
if (dsn_name not in exist_lfn) and not exists:
print('Will create %s' % dsn_name)
add_did(scope=dsn_scope,
name=dsn_name,
type=DIDType.DATASET,
account=InternalAccount(account),
statuses=None,
meta=None,
rules=[{
'copies': 1,
'rse_expression': 'ANY=true',
'weight': None,
'account': InternalAccount(account),
'lifetime': None,
'grouping': 'NONE'
}],
lifetime=None,
dids=None,
rse_id=None,
session=session)
exist_lfn.append(dsn_name)
parent_name = lpns[1]
parent_scope, _ = extract_scope(parent_name, scopes)
parent_scope = InternalScope(parent_scope)
attachments.append({
'scope': parent_scope,
'name': parent_name,
'dids': [{
'scope': dsn_scope,
'name': dsn_name
}]
})
# Register the file
rse_id = lfn.get('rse_id', None)
if not rse_id:
raise InvalidType('Missing rse_id')
bytes = lfn.get('bytes', None)
guid = lfn.get('guid', None)
adler32 = lfn.get('adler32', None)
pfn = lfn.get('pfn', None)
files = {
'scope': lfn_scope,
'name': filename,
'bytes': bytes,
'adler32': adler32
}
if pfn:
files['pfn'] = str(pfn)
if guid:
files['meta'] = {'guid': guid}
add_replicas(rse_id=rse_id,
files=[files],
dataset_meta=None,
account=InternalAccount(account),
ignore_availability=ignore_availability,
session=session)
add_rule(dids=[{
'scope': lfn_scope,
'name': filename
}],
account=InternalAccount(account),
copies=1,
rse_expression=lfn['rse'],
grouping=None,
weight=None,
lifetime=86400,
locked=None,
subscription_id=None,
session=session)
attachments.append({
'scope': dsn_scope,
'name': dsn_name,
'dids': [{
'scope': lfn_scope,
'name': filename
}]
})
# Now loop over the ascendants of the dataset and created them
for lpn in lpns[1:]:
child_scope, _ = extract_scope(lpn, scopes)
child_scope = InternalScope(child_scope)
exists, did_type = _exists(child_scope, lpn)
if exists and did_type == DIDType.DATASET:
raise UnsupportedOperation('Cannot create %s as container' %
lpn)
if (lpn not in exist_lfn) and not exists:
print('Will create %s' % lpn)
add_did(scope=child_scope,
name=lpn,
type=DIDType.CONTAINER,
account=InternalAccount(account),
statuses=None,
meta=None,
rules=None,
lifetime=None,
dids=None,
rse_id=None,
session=session)
exist_lfn.append(lpn)
parent_name = lpns[lpns.index(lpn) + 1]
parent_scope, _ = extract_scope(parent_name, scopes)
parent_scope = InternalScope(parent_scope)
attachments.append({
'scope':
parent_scope,
'name':
parent_name,
'dids': [{
'scope': child_scope,
'name': lpn
}]
})
# Finally attach everything
attach_dids_to_dids(attachments,
account=InternalAccount(account),
ignore_duplicate=True,
session=session)
def get_signed_url(rse_id, service, operation, url, lifetime=600):
"""
Get a signed URL for a particular service and operation.
The signed URL will be valid for 1 hour but can be overriden.
:param rse_id: The ID of the RSE that the URL points to.
:param service: The service to authorise, either 'gcs', 's3' or 'swift'.
:param operation: The operation to sign, either 'read', 'write', or 'delete'.
:param url: The URL to sign.
:param lifetime: Lifetime of the signed URL in seconds.
:returns: Signed URL as a variable-length string.
"""
global CREDS_GCS
if service not in ['gcs', 's3', 'swift']:
raise UnsupportedOperation('Service must be "gcs", "s3" or "swift"')
if operation not in ['read', 'write', 'delete']:
raise UnsupportedOperation(
'Operation must be "read", "write", or "delete"')
if url is None or url == '':
raise UnsupportedOperation('URL must not be empty')
if lifetime:
if not isinstance(lifetime, integer_types):
try:
lifetime = int(lifetime)
except:
raise UnsupportedOperation(
'Lifetime must be convertible to numeric.')
signed_url = None
if service == 'gcs':
if not CREDS_GCS:
CREDS_GCS = ServiceAccountCredentials.from_json_keyfile_name(
config_get(
'credentials',
'gcs',
raise_exception=False,
default='/opt/rucio/etc/google-cloud-storage-test.json'))
components = urlparse(url)
host = components.netloc
# select the correct operation
operations = {'read': 'GET', 'write': 'PUT', 'delete': 'DELETE'}
operation = operations[operation]
# special case to test signature, force epoch time
if lifetime is None:
lifetime = 0
else:
# GCS is timezone-sensitive, don't use UTC
# has to be converted to Unixtime
lifetime = datetime.datetime.now() + datetime.timedelta(
seconds=lifetime)
lifetime = int(time.mktime(lifetime.timetuple()))
# sign the path only
path = components.path
# assemble message to sign
to_sign = "%s\n\n\n%s\n%s" % (operation, lifetime, path)
# create URL-capable signature
# first character is always a '=', remove it
signature = urlencode(
{'': base64.b64encode(CREDS_GCS.sign_blob(to_sign)[1])})[1:]
# assemble final signed URL
signed_url = 'https://%s%s?GoogleAccessId=%s&Expires=%s&Signature=%s' % (
host, path, CREDS_GCS.service_account_email, lifetime, signature)
elif service == 's3':
# split URL to get hostname, bucket and key
components = urlparse(url)
host = components.netloc
pathcomponents = components.path.split('/')
if len(pathcomponents) < 3:
raise UnsupportedOperation('Not a valid S3 URL')
bucket = pathcomponents[1]
key = '/'.join(pathcomponents[2:])
# remove port number from host if present
colon = host.find(':')
port = '443'
if colon >= 0:
port = host[colon + 1:]
host = host[:colon]
# look up in RSE account configuration by RSE ID
cred_name = rse_id
cred = REGION.get('s3-%s' % cred_name)
if cred is NO_VALUE:
rse_cred = get_rse_credentials()
cred = rse_cred.get(cred_name)
REGION.set('s3-%s' % cred_name, cred)
access_key = cred['access_key']
secret_key = cred['secret_key']
signature_version = cred['signature_version']
region_name = cred['region']
if operation == 'read':
s3op = 'get_object'
elif operation == 'write':
s3op = 'put_object'
else:
s3op = 'delete_object'
with record_timer_block('credential.signs3'):
s3 = boto3.client('s3',
endpoint_url='https://' + host + ':' + port,
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
config=Config(
signature_version=signature_version,
region_name=region_name))
signed_url = s3.generate_presigned_url(s3op,
Params={
'Bucket': bucket,
'Key': key
},
ExpiresIn=lifetime)
elif service == 'swift':
# split URL to get hostname and path
components = urlparse(url)
host = components.netloc
# remove port number from host if present
colon = host.find(':')
if colon >= 0:
host = host[:colon]
# use RSE ID to look up key
cred_name = rse_id
# look up tempurl signing key
cred = REGION.get('swift-%s' % cred_name)
if cred is NO_VALUE:
rse_cred = get_rse_credentials()
cred = rse_cred.get(cred_name)
REGION.set('swift-%s' % cred_name, cred)
tempurl_key = cred['tempurl_key']
if operation == 'read':
swiftop = 'GET'
elif operation == 'write':
swiftop = 'PUT'
else:
swiftop = 'DELETE'
expires = int(time.time() + lifetime)
# create signed URL
with record_timer_block('credential.signswift'):
hmac_body = u'%s\n%s\n%s' % (swiftop, expires, components.path)
# Python 3 hmac only accepts bytes or bytearray
sig = hmac.new(bytearray(tempurl_key, 'utf-8'),
bytearray(hmac_body, 'utf-8'), sha1).hexdigest()
signed_url = 'https://' + host + components.path + '?temp_url_sig=' + sig + '&temp_url_expires=' + str(
expires)
return signed_url
def add_files(lfns, account, ignore_availability, vo='def', session=None):
"""
Bulk add files :
- Create the file and replica.
- If doesn't exist create the dataset containing the file as well as a rule on the dataset on ANY sites.
- Create all the ascendants of the dataset if they do not exist
:param lfns: List of lfn (dictionary {'lfn': <lfn>, 'rse': <rse>, 'bytes': <bytes>, 'adler32': <adler32>, 'guid': <guid>, 'pfn': <pfn>}
:param issuer: The issuer account.
:param ignore_availability: A boolean to ignore blocklisted sites.
:param vo: The VO to act on
:param session: The session used
"""
rule_extension_list = []
attachments = []
# The list of scopes is necessary for the extract_scope
filter_ = {'scope': InternalScope(scope='*', vo=vo)}
scopes = list_scopes(filter_=filter_, session=session)
scopes = [scope.external for scope in scopes]
exist_lfn = []
try:
lifetime_dict = config_get(section='dirac', option='lifetime', session=session)
lifetime_dict = loads(lifetime_dict)
except ConfigNotFound:
lifetime_dict = {}
except JSONDecodeError as err:
raise InvalidType('Problem parsing lifetime option in dirac section : %s' % str(err))
except Exception as err:
raise RucioException(str(err))
for lfn in lfns:
# First check if the file exists
filename = lfn['lfn']
lfn_scope, _ = extract_scope(filename, scopes)
lfn_scope = InternalScope(lfn_scope, vo=vo)
exists, did_type = _exists(lfn_scope, filename)
if exists:
continue
# Get all the ascendants of the file
lfn_split = filename.split('/')
lpns = ["/".join(lfn_split[:idx]) for idx in range(2, len(lfn_split))]
lpns.reverse()
print(lpns)
# The parent must be a dataset. Register it as well as the rule
dsn_name = lpns[0]
dsn_scope, _ = extract_scope(dsn_name, scopes)
dsn_scope = InternalScope(dsn_scope, vo=vo)
# Compute lifetime
lifetime = None
if dsn_scope in lifetime_dict:
lifetime = lifetime_dict[dsn_scope]
else:
for pattern in lifetime_dict:
if re.match(pattern, dsn_scope):
lifetime = lifetime_dict[pattern]
break
exists, did_type = _exists(dsn_scope, dsn_name)
if exists and did_type == DIDType.CONTAINER:
raise UnsupportedOperation('Cannot create %s as dataset' % dsn_name)
if (dsn_name not in exist_lfn) and not exists:
print('Will create %s' % dsn_name)
# to maintain a compatibility between master and LTS-1.26 branches remove keywords for first 3 arguments
add_did(dsn_scope,
dsn_name,
DIDType.DATASET,
account=InternalAccount(account, vo=vo),
statuses=None,
meta=None,
rules=[{'copies': 1, 'rse_expression': 'ANY=true', 'weight': None, 'account': InternalAccount(account, vo=vo), 'lifetime': None, 'grouping': 'NONE'}],
lifetime=None,
dids=None,
rse_id=None,
session=session)
exist_lfn.append(dsn_name)
parent_name = lpns[1]
parent_scope, _ = extract_scope(parent_name, scopes)
parent_scope = InternalScope(parent_scope, vo=vo)
attachments.append({'scope': parent_scope, 'name': parent_name, 'dids': [{'scope': dsn_scope, 'name': dsn_name}]})
rule_extension_list.append((dsn_scope, dsn_name))
if lifetime and (dsn_scope, dsn_name) not in rule_extension_list:
# Reset the lifetime of the rule to the configured value
rule = [rul for rul in list_rules({'scope': dsn_scope, 'name': dsn_name, 'account': InternalAccount(account, vo=vo)}, session=session) if rul['rse_expression'] == 'ANY=true']
if rule:
update_rule(rule[0]['id'], options={'lifetime': lifetime}, session=session)
rule_extension_list.append((dsn_scope, dsn_name))
# Register the file
rse_id = lfn.get('rse_id', None)
if not rse_id:
raise InvalidType('Missing rse_id')
bytes_ = lfn.get('bytes', None)
guid = lfn.get('guid', None)
adler32 = lfn.get('adler32', None)
pfn = lfn.get('pfn', None)
files = {'scope': lfn_scope, 'name': filename, 'bytes': bytes_, 'adler32': adler32}
if pfn:
files['pfn'] = str(pfn)
if guid:
files['meta'] = {'guid': guid}
add_replicas(rse_id=rse_id,
files=[files],
dataset_meta=None,
account=InternalAccount(account, vo=vo),
ignore_availability=ignore_availability,
session=session)
add_rule(dids=[{'scope': lfn_scope, 'name': filename}],
account=InternalAccount(account, vo=vo),
copies=1,
rse_expression=lfn['rse'],
grouping=None,
weight=None,
lifetime=86400,
locked=None,
subscription_id=None,
session=session)
attachments.append({'scope': dsn_scope, 'name': dsn_name, 'dids': [{'scope': lfn_scope, 'name': filename}]})
# Now loop over the ascendants of the dataset and created them
for lpn in lpns[1:]:
child_scope, _ = extract_scope(lpn, scopes)
child_scope = InternalScope(child_scope, vo=vo)
exists, did_type = _exists(child_scope, lpn)
if exists and did_type == DIDType.DATASET:
raise UnsupportedOperation('Cannot create %s as container' % lpn)
if (lpn not in exist_lfn) and not exists:
print('Will create %s' % lpn)
add_did(child_scope,
lpn,
DIDType.CONTAINER,
account=InternalAccount(account, vo=vo),
statuses=None,
meta=None,
rules=None,
lifetime=None,
dids=None,
rse_id=None,
session=session)
exist_lfn.append(lpn)
parent_name = lpns[lpns.index(lpn) + 1]
parent_scope, _ = extract_scope(parent_name, scopes)
parent_scope = InternalScope(parent_scope, vo=vo)
attachments.append({'scope': parent_scope, 'name': parent_name, 'dids': [{'scope': child_scope, 'name': lpn}]})
# Finally attach everything
attach_dids_to_dids(attachments,
account=InternalAccount(account, vo=vo),
ignore_duplicate=True,
session=session)
def add_exception(dids, account, pattern, comments, expires_at, session=None):
"""
Add exceptions to Lifetime Model.
:param dids: The list of dids
:param account: The account of the requester.
:param pattern: The account.
:param comments: The comments associated to the exception.
:param expires_at: The expiration date of the exception.
:param session: The database session in use.
returns: A dictionary with id of the exceptions split by scope, datatype.
"""
from rucio.core.did import get_metadata_bulk
result = dict()
result['exceptions'] = dict()
try:
max_extension = config_get('lifetime_model',
'max_extension',
default=None,
session=session)
if max_extension:
if not expires_at:
expires_at = datetime.utcnow() + timedelta(days=max_extension)
else:
if isinstance(expires_at, string_types):
expires_at = str_to_date(expires_at)
if expires_at > datetime.utcnow() + timedelta(
days=max_extension):
expires_at = datetime.utcnow() + timedelta(
days=max_extension)
except (ConfigNotFound, ValueError, NoSectionError):
max_extension = None
try:
cutoff_date = config_get('lifetime_model',
'cutoff_date',
default=None,
session=session)
except (ConfigNotFound, NoSectionError):
raise UnsupportedOperation('Cannot submit exception at that date.')
try:
cutoff_date = datetime.strptime(cutoff_date, '%Y-%m-%d')
except ValueError:
raise UnsupportedOperation('Cannot submit exception at that date.')
if cutoff_date < datetime.utcnow():
raise UnsupportedOperation('Cannot submit exception at that date.')
did_group = dict()
not_affected = list()
list_dids = [(did['scope'], did['name']) for did in dids]
metadata = [meta for meta in get_metadata_bulk(dids=dids, session=session)]
for did in metadata:
scope, name, did_type = did['scope'], did['name'], did['did_type']
if (scope, name) in list_dids:
list_dids.remove((scope, name))
datatype = did.get('datatype', '')
eol_at = did.get('eol_at', None)
if eol_at and eol_at < cutoff_date:
if (scope, datatype) not in did_group:
did_group[(scope, datatype)] = [list(), 0]
did_group[(scope, datatype)][0].append({
'scope': scope,
'name': name,
'did_type': did_type
})
did_group[(scope, datatype)][1] += did['bytes'] or 0
else:
not_affected.append((scope, name, did_type))
for entry in did_group:
exception_id = __add_exception(did_group[entry][0],
account=account,
pattern=pattern,
comments=comments,
expires_at=expires_at,
estimated_volume=did_group[entry][1],
session=session)
result['exceptions'][exception_id] = did_group[entry][0]
result['unknown'] = [{
'scope': did[0],
'name': did[1],
'did_type': DIDType.DATASET
} for did in list_dids]
result['not_affected'] = [{
'scope': did[0],
'name': did[1],
'did_type': did[2]
} for did in not_affected]
return result
