def _get_file_guid(self, file):
"""
Get the guid of a file, trying different strategies
(This function is meant to be used as class internal only)
:param file: dictionary describing the file
:returns: the guid
"""
guid = file.get('guid')
if not guid and 'pool.root' in file['basename'].lower(
) and not file.get('no_register'):
status, output, err = execute('pool_extractFileIdentifier %s' %
file['path'])
if status != 0:
msg = 'Trying to upload ROOT files but pool_extractFileIdentifier tool can not be found.\n'
msg += 'Setup your ATHENA environment and try again.'
raise RucioException(msg)
try:
guid = output.splitlines()[-1].split()[0].replace('-',
'').lower()
except Exception:
raise RucioException(
'Error extracting GUID from ouput of pool_extractFileIdentifier'
)
elif guid:
guid = guid.replace('-', '')
else:
guid = generate_uuid()
return guid
def mysql_convert_decimal_to_float(pymysql=False):
"""
The default datatype returned by mysql-python for numerics is decimal.Decimal.
This type cannot be serialised to JSON, therefore we need to autoconvert to floats.
Even worse, there's two types of decimals created by the MySQLdb driver, so we must
override both.
:return converter: Converter object
"""
def pymysql_converter():
from pymysql.constants import FIELD_TYPE
from pymysql.converters import conversions as conv
converter = conv.copy()
converter[FIELD_TYPE.DECIMAL] = float
converter[FIELD_TYPE.NEWDECIMAL] = float
return converter
if pymysql:
if not EXTRA_MODULES['pymysql']:
raise RucioException('Trying to use pymysql without having it installed!')
else:
converter = pymysql_converter()
elif EXTRA_MODULES['MySQLdb']:
import MySQLdb.converters # pylint: disable=import-error
from MySQLdb.constants import FIELD_TYPE # pylint: disable=import-error
converter = MySQLdb.converters.conversions.copy()
converter[FIELD_TYPE.DECIMAL] = float
converter[FIELD_TYPE.NEWDECIMAL] = float
elif EXTRA_MODULES['pymysql']:
converter = pymysql_converter()
else:
raise RucioException('Trying to use MySQL without mysql-python or pymysql installed!')
return converter
def add_scope(scope, account, session=None):
""" add a scope for the given account name.
:param scope: the name for the new scope.
:param account: the account to add the scope to.
:param session: The database session in use.
"""
if not vo_exists(vo=scope.vo, session=session):
raise VONotFound('VO {} not found'.format(scope.vo))
result = session.query(models.Account).filter_by(
account=account, status=AccountStatus.ACTIVE).first()
if result is None:
raise AccountNotFound('Account ID \'%s\' does not exist' % account)
new_scope = models.Scope(scope=scope,
account=account,
status=ScopeStatus.OPEN)
try:
new_scope.save(session=session)
except IntegrityError as e:
if match('.*IntegrityError.*ORA-00001: unique constraint.*SCOPES_PK.*violated.*', e.args[0]) \
or match('.*IntegrityError.*Duplicate entry.*for key.*', e.args[0]) \
or match('.*IntegrityError.*UNIQUE constraint failed: scopes.scope.*', e.args[0]) \
or match('.*IntegrityError.*duplicate key value violates unique constraint.*', e.args[0]) \
or match('.*UniqueViolation.*duplicate key value violates unique constraint.*', e.args[0]) \
or match('.*IntegrityError.*columns? .*not unique.*', e.args[0]):
raise Duplicate('Scope \'%s\' already exists!' % scope)
else:
raise RucioException(e)
except:
raise RucioException(str(format_exc()))
def set_transfers_state(transfers, submitted_at, session=None):
"""
Update the transfer info of a request.
:param transfers: Dictionary containing request transfer info.
:param session: Database session to use.
"""
try:
for request_id in transfers:
rowcount = session.query(models.Request)\
.filter_by(id=request_id)\
.filter(models.Request.state == RequestState.SUBMITTING)\
.update({'state': transfers[request_id]['state'],
'external_id': transfers[request_id]['external_id'],
'external_host': transfers[request_id]['external_host'],
'source_rse_id': transfers[request_id]['src_rse_id'],
'submitted_at': submitted_at},
synchronize_session=False)
if rowcount == 0:
raise RucioException("Failed to set requests %s tansfer %s: request doesn't exist or is not in SUBMITTING state" % (request_id, transfers[request_id]))
request_type = transfers[request_id].get('request_type', None)
msg = {'request-id': request_id,
'request-type': str(request_type).lower() if request_type else request_type,
'scope': transfers[request_id]['scope'],
'name': transfers[request_id]['name'],
'src-rse-id': transfers[request_id]['metadata'].get('src_rse_id', None),
'src-rse': transfers[request_id]['metadata'].get('src_rse', None),
'dst-rse-id': transfers[request_id]['metadata'].get('dst_rse_id', None),
'dst-rse': transfers[request_id]['metadata'].get('dst_rse', None),
'state': str(transfers[request_id]['state']),
'activity': transfers[request_id]['metadata'].get('activity', None),
'file-size': transfers[request_id]['metadata'].get('filesize', None),
'bytes': transfers[request_id]['metadata'].get('filesize', None),
'checksum-md5': transfers[request_id]['metadata'].get('md5', None),
'checksum-adler': transfers[request_id]['metadata'].get('adler32', None),
'external-id': transfers[request_id]['external_id'],
'external-host': transfers[request_id]['external_host'],
'queued_at': str(submitted_at)}
if msg['request-type']:
transfer_status = '%s-%s' % (msg['request-type'], msg['state'])
else:
transfer_status = 'transfer-%s' % msg['state']
transfer_status = transfer_status.lower()
message_core.add_message(transfer_status, msg, session=session)
except IntegrityError as error:
raise RucioException(error.args)
def add_value(key, value, session=None):
"""
Adds a new value to a key.
:param key: the name for the key.
:param value: the value.
:param session: The database session in use.
"""
new_value = models.DIDKeyValueAssociation(key=key, value=value)
try:
new_value.save(session=session)
except IntegrityError as error:
if match('.*IntegrityError.*columns? key.*value.*not unique.*', error.args[0]):
raise Duplicate('key-value \'%(key)s-%(value)s\' already exists!' % locals())
if match('.*IntegrityError.*foreign key constraints? failed.*', error.args[0]):
raise KeyNotFound("key '%(key)s' does not exist!" % locals())
if match('.*IntegrityError.*ORA-02291: integrity constraint.*DID_MAP_KEYS_FK.*violated.*', error.args[0]):
raise KeyNotFound("key '%(key)s' does not exist!" % locals())
if error.args[0] == "(IntegrityError) (1452, 'Cannot add or update a child row: a foreign key constraint fails (`rucio`.`did_key_map`, CONSTRAINT `DID_MAP_KEYS_FK` FOREIGN KEY (`key`) REFERENCES `did_keys` (`key`))')":
raise KeyNotFound("key '%(key)s' does not exist!" % locals())
raise RucioException(error.args)
k = session.query(models.DIDKey).filter_by(key=key).one()
# Check value against regexp, if defined
if k.value_regexp and not match(k.value_regexp, value):
raise InvalidValueForKey("The value '%s' for the key '%s' does not match the regular expression '%s'" % (value, key, k.value_regexp))
# Check value type, if defined
type_map = dict([(str(t), t) for t in AUTHORIZED_VALUE_TYPES])
if k.value_type and not isinstance(value, type_map.get(k.value_type)):
raise InvalidValueForKey("The value '%s' for the key '%s' does not match the required type '%s'" % (value, key, k.value_type))
def new_funct(*args, **kwargs):
if isgeneratorfunction(function):
raise RucioException(
'read_session decorator should not be used with generator. Use stream_session instead.'
)
if not kwargs.get('session'):
session = get_session()
try:
kwargs['session'] = session
return function(*args, **kwargs)
except TimeoutError as error:
session.rollback() # pylint: disable=maybe-no-member
raise DatabaseException(str(error))
except DatabaseError as error:
session.rollback() # pylint: disable=maybe-no-member
raise DatabaseException(str(error))
except:
session.rollback() # pylint: disable=maybe-no-member
raise
finally:
session.remove()
try:
return function(*args, **kwargs)
except Exception:
raise
def update_messages_services(messages, services, session=None):
"""
Update the services for all messages with the given IDs.
:param messages: The messages to delete as a list of dictionaries.
:param services: A coma separated string containing the list of services to report to.
:param session: The database session to use.
"""
message_condition = []
for message in messages:
message_condition.append(Message.id == message['id'])
if len(message['payload']) > 4000:
message['payload_nolimit'] = message.pop('payload')
try:
if message_condition:
stmt = update(Message).\
prefix_with("/*+ index(messages MESSAGES_ID_PK) */", dialect='oracle').\
where(or_(*message_condition)).\
execution_options(synchronize_session=False).\
values(services=services)
session.execute(stmt)
except IntegrityError as err:
raise RucioException(err.args)
def mysql_convert_decimal_to_float():
"""
The default datatype returned by mysql-python for numerics is decimal.Decimal.
This type cannot be serialised to JSON, therefore we need to autoconvert to floats.
Even worse, there's two types of decimals created by the MySQLdb driver, so we must
override both.
:return converter: Converter object
"""
converter = None
try:
import MySQLdb.converters # pylint: disable=import-error
from MySQLdb.constants import FIELD_TYPE # pylint: disable=import-error
converter = MySQLdb.converters.conversions.copy()
converter[FIELD_TYPE.DECIMAL] = float
converter[FIELD_TYPE.NEWDECIMAL] = float
except ImportError:
try:
from pymysql.constants import FIELD_TYPE
from pymysql.converters import conversions as conv
converter = conv.copy()
converter[FIELD_TYPE.DECIMAL] = float
converter[FIELD_TYPE.NEWDECIMAL] = float
except ImportError:
raise RucioException(
'Trying to use MySQL without mysql-python or pymysql installed!'
)
return converter
def download_dids(self, items, num_threads=2, trace_custom_fields={}):
"""
Download items with given DIDs. This function can also download datasets and wildcarded DIDs.
:param items: List of dictionaries. Each dictionary describing an item to download. Keys:
did - DID string of this file (e.g. 'scope:file.name'). Wildcards are not allowed
rse - Optional: rse name (e.g. 'CERN-PROD_DATADISK') or rse expression from where to download
force_scheme - Optional: force a specific scheme to download this item. (Default: None)
base_dir - Optional: base directory where the downloaded files will be stored. (Default: '.')
no_subdir - Optional: If true, files are written directly into base_dir and existing files are overwritten. (Default: False)
nrandom - Optional: if the DID addresses a dataset, nrandom files will be randomly choosen for download from the dataset
ignore_checksum - Optional: If true, skips the checksum validation between the downloaded file and the rucio catalouge. (Default: False)
transfer_timeout - Optional: Timeout time for the download protocols. (Default: None)
:param num_threads: Suggestion of number of threads to use for the download. It will be lowered if it's too high.
:param trace_custom_fields: Custom key value pairs to send with the traces
:returns: a list of dictionaries with an entry for each file, containing the input options, the did, and the clientState
:raises InputValidationError: if one of the input items is in the wrong format
:raises NoFilesDownloaded: if no files could be downloaded
:raises NotAllFilesDownloaded: if not all files could be downloaded
:raises RucioException: if something unexpected went wrong during the download
"""
trace_custom_fields['uuid'] = generate_uuid()
input_items = self._prepare_items_for_download(items)
num_files_in = len(input_items)
output_items = self._download_multithreaded(input_items, num_threads, trace_custom_fields)
num_files_out = len(output_items)
if num_files_in != num_files_out:
raise RucioException('%d items were in the input queue but only %d are in the output queue' % (num_files_in, num_files_out))
return self._check_output(output_items)
def new_funct(*args, **kwargs):
if not isgeneratorfunction(function):
raise RucioException(
'stream_session decorator should be used only with generator. Use read_session instead.'
)
if not kwargs.get('session'):
session = get_session()
try:
kwargs['session'] = session
for row in function(*args, **kwargs):
yield row
except TimeoutError as error:
session.rollback() # pylint: disable=maybe-no-member
raise DatabaseException(str(error))
except DatabaseError as error:
session.rollback() # pylint: disable=maybe-no-member
raise DatabaseException(str(error))
except:
session.rollback() # pylint: disable=maybe-no-member
raise
finally:
session.remove()
else:
try:
for row in function(*args, **kwargs):
yield row
except:
raise
def get_file_guid(self, file):
guid = file.get('guid')
if not guid and 'pool.root' in file['basename'].lower() and not file.get('no_register'):
status, output, err = execute('pool_extractFileIdentifier %s' % file['path'])
if status != 0:
msg = 'Trying to upload ROOT files but pool_extractFileIdentifier tool can not be found.\n'
msg += 'Setup your ATHENA environment and try again.'
raise RucioException(msg)
try:
guid = output.splitlines()[-1].split()[0].replace('-', '').lower()
except Exception:
raise RucioException('Error extracting GUID from ouput of pool_extractFileIdentifier')
elif guid:
guid = guid.replace('-', '')
else:
guid = generate_uuid()
return guid
def update_subscription(name, account, filter=None, replication_rules=None, comments=None, lifetime=None, retroactive=None, dry_run=None, state=None, priority=None, session=None):
"""
Updates a subscription
:param name: Name of the subscription
:type: String
:param account: Account identifier
:type account: String
:param filter: Dictionary of attributes by which the input data should be filtered
**Example**: ``{'dsn': 'data11_hi*.express_express.*,data11_hi*physics_MinBiasOverlay*', 'account': 'tzero'}``
:type filter: Dict
:param replication_rules: Replication rules to be set : Dictionary with keys copies, rse_expression, weight, rse_expression
:type replication_rules: Dict
:param transfer_requests: Transfer requests to be issued. List of tuples holding count, RSE-tag, group; If the group flag is set to ``true``, this transfer_request will resolve to the same RSE for all files in the same dataset
**Example**: ``[(1, 'T1-DATADISKS', True), (2, 'T2-DATADISKS', False)]``
:type transfer_requests: List
:param comments: Comments for the subscription
:type comments: String
:param lifetime: Subscription's lifetime (days)
:type lifetime: Integer or None
:param retroactive: Flag to know if the subscription should be applied on previous data
:type retroactive: Boolean
:param dry_run: Just print the subscriptions actions without actually executing them (Useful if retroactive flag is set)
:type dry_run: Boolean
:param state: The state of the subscription
:param priority: The priority of the subscription
:type priority: Integer
:param session: The database session in use.
:raises: exception.NotFound if subscription is not found
"""
values = {'state': SubscriptionState.UPDATED}
if filter:
values['filter'] = filter
if replication_rules:
values['replication_rules'] = replication_rules
if lifetime:
values['lifetime'] = datetime.datetime.utcnow() + datetime.timedelta(days=lifetime)
if retroactive:
values['retroactive'] = retroactive
if dry_run:
values['dry_run'] = dry_run
if comments:
values['comments'] = comments
if priority:
values['policyid'] = priority
if state and state == SubscriptionState.INACTIVE:
values['state'] = SubscriptionState.INACTIVE
values['expired_at'] = datetime.datetime.utcnow()
try:
rowcount = session.query(models.Subscription).filter_by(account=account, name=name).update(values)
if rowcount == 0:
raise SubscriptionNotFound("Subscription for account '%(account)s' named '%(name)s' not found" % locals())
except IntegrityError as error:
raise RucioException(error.args)
def truncate_messages(session=None):
"""
Delete all stored messages. This is for internal purposes only.
:param session: The database session to use.
"""
try:
session.query(Message).delete(synchronize_session=False)
except IntegrityError, e:
raise RucioException(e.args)
def get_dataset_by_guid(guid, vo='def'):
"""
Get the parent datasets for a given GUID.
:param guid: The GUID.
:param vo: The VO to act on.
:returns: A did
"""
dids = did.get_dataset_by_guid(guid=guid)
for d in dids:
if d['scope'].vo != vo:
raise RucioException('GUID unavailable on VO {}'.format(vo))
yield api_update_return_dict(d)
def add_subscription(name, account, filter, replication_rules, comments, lifetime, retroactive, dry_run, priority=3, session=None):
"""
Adds a new subscription which will be verified against every new added file and dataset
:param account: Account identifier
:type account: String
:param name: Name of the subscription
:type: String
:param filter: Dictionary of attributes by which the input data should be filtered
**Example**: ``{'dsn': 'data11_hi*.express_express.*,data11_hi*physics_MinBiasOverlay*', 'account': 'tzero'}``
:type filter: Dict
:param replication_rules: Replication rules to be set : Dictionary with keys copies, rse_expression, weight, rse_expression
:type replication_rules: Dict
:param comments: Comments for the subscription
:type comments: String
:param lifetime: Subscription's lifetime (days)
:type lifetime: Integer or None
:param retroactive: Flag to know if the subscription should be applied on previous data
:type retroactive: Boolean
:param dry_run: Just print the subscriptions actions without actually executing them (Useful if retroactive flag is set)
:type dry_run: Boolean
:param priority: The priority of the subscription
:type priority: Integer
:param session: The database session in use.
:returns: The subscriptionid
"""
retroactive = bool(retroactive) # Force boolean type, necessary for strict SQL
state = SubscriptionState.ACTIVE
lifetime = None
if retroactive:
state = SubscriptionState.NEW
if lifetime:
lifetime = datetime.datetime.utcnow() + datetime.timedelta(days=lifetime)
new_subscription = models.Subscription(name=name, filter=filter, account=account, replication_rules=replication_rules, state=state, lifetime=lifetime,
retroactive=retroactive, policyid=priority, comments=comments)
try:
new_subscription.save(session=session)
except IntegrityError as error:
if re.match('.*IntegrityError.*ORA-00001: unique constraint.*SUBSCRIPTIONS_PK.*violated.*', error.args[0])\
or re.match(".*IntegrityError.*UNIQUE constraint failed: subscriptions.name, subscriptions.account.*", error.args[0])\
or re.match(".*columns name, account are not unique.*", error.args[0])\
or re.match('.*IntegrityError.*ORA-00001: unique constraint.*SUBSCRIPTIONS_NAME_ACCOUNT_UQ.*violated.*', error.args[0])\
or re.match('.*IntegrityError.*1062.*Duplicate entry.*', error.args[0]) \
or re.match('.*IntegrityError.*duplicate key value violates unique constraint.*', error.args[0]):
raise SubscriptionDuplicate('Subscription \'%s\' owned by \'%s\' already exists!' % (name, account))
raise RucioException(error.args)
return new_subscription.id
def get_replica_locks_for_rule_id(rule_id, vo='def'):
"""
Get the replica locks for a rule_id.
:param rule_id: Rule ID.
:param vo: The VO to act on.
:return: List of dicts.
"""
locks = lock.get_replica_locks_for_rule_id(rule_id=rule_id)
for l in locks:
if l['scope'].vo != vo:
raise RucioException('Cannot get rule on VO {}'.format(vo))
yield api_update_return_dict(l)
def add_naming_convention(scope, regexp, convention_type, session=None):
"""
add a naming convention for a given scope
:param scope: the name for the scope.
:param regexp: the regular expression to validate the name.
:param convention_type: the did_type on which the regexp should apply.
:param session: The database session in use.
"""
# validate the regular expression
try:
compile(regexp)
except error:
raise RucioException('Invalid regular expression %s!' % regexp)
new_convention = models.NamingConvention(scope=scope,
regexp=regexp,
convention_type=convention_type)
try:
new_convention.save(session=session)
except IntegrityError:
raise Duplicate('Naming convention already exists!')
except:
raise RucioException(str(format_exc()))
def extract_scope(self, did):
# Try to extract the scope from the DSN
if did.find(':') > -1:
if len(did.split(':')) > 2:
raise RucioException('Too many colons. Cannot extract scope and name')
scope, name = did.split(':')[0], did.split(':')[1]
if name.endswith('/'):
name = name[:-1]
return scope, name
else:
scope = did.split('.')[0]
if did.startswith('user') or did.startswith('group'):
scope = ".".join(did.split('.')[0:2])
if did.endswith('/'):
did = did[:-1]
return scope, did
def list_subscriptions(name=None,
account=None,
state=None,
session=None,
logger=logging.log):
"""
Returns a dictionary with the subscription information :
Examples: ``{'status': 'INACTIVE/ACTIVE/BROKEN', 'last_modified_date': ...}``
:param name: Name of the subscription
:type name: String
:param account: Account identifier
:type account: String
:param session: The database session in use.
:param logger: Optional decorated logger that can be passed from the calling daemons or servers.
:returns: Dictionary containing subscription parameter
:rtype: Dict
:raises: exception.NotFound if subscription is not found
"""
query = session.query(models.Subscription)
try:
if name:
query = query.filter_by(name=name)
if account:
if '*' in account.internal:
account_str = account.internal.replace('*', '%')
query = query.filter(
models.Subscription.account.like(account_str))
else:
query = query.filter_by(account=account)
if state:
query = query.filter_by(state=state)
except IntegrityError as error:
logger(logging.ERROR, str(error))
raise RucioException(error.args)
result = {}
for row in query:
result = {}
for column in row.__table__.columns:
result[column.name] = getattr(row, column.name)
yield result
if result == {}:
raise SubscriptionNotFound(
"Subscription for account '%(account)s' named '%(name)s' not found"
% locals())
def psql_convert_decimal_to_float(dbapi_conn, connection_rec):
"""
The default datatype returned by psycopg2 for numerics is decimal.Decimal.
This type cannot be serialised to JSON, therefore we need to autoconvert to floats.
:param dbapi_conn: DBAPI connection
:param connection_rec: connection record
"""
try:
import psycopg2.extensions # pylint: disable=import-error
except:
raise RucioException('Trying to use PostgreSQL without psycopg2 or psycopg2-binary installed!')
DEC2FLOAT = psycopg2.extensions.new_type(psycopg2.extensions.DECIMAL.values,
'DEC2FLOAT',
lambda value, curs: float(value) if value is not None else None)
psycopg2.extensions.register_type(DEC2FLOAT)
def touch(self, scope, name, rse=None):
"""
Sends a touch trace for a given file or dataset.
:param scope: the scope of the file/dataset to update.
:param name: the name of file/dataset to update.
:param rse: optional parameter if a specific replica should be touched.
:raises DataIdentifierNotFound: if given dids does not exist.
:raises RSENotFound: if rse is not None and given rse does not exist.
:raises UnsupportedDIDType: if type of the given DID is not FILE or DATASET.
:raises RucioException: if trace could not be sent successfully.
"""
trace = {}
trace['eventType'] = 'touch'
trace['clientState'] = 'DONE'
trace['account'] = self.account
if self.vo != 'def':
trace['vo'] = self.vo
if rse:
self.get_rse(rse) # pylint: disable=no-member
trace['localSite'] = trace['remoteSite'] = rse
info = self.get_did(scope, name) # pylint: disable=no-member
if info['type'] == 'CONTAINER':
raise UnsupportedDIDType("%s:%s is a container." % (scope, name))
if info['type'] == 'FILE':
trace['scope'] = scope
trace['filename'] = name
elif info['type'] == 'DATASET':
trace['datasetScope'] = scope
trace['dataset'] = name
url = '%s/%s/' % (choice(self.list_hosts), self.TRACES_BASEURL)
try:
post(url, verify=False, data=dumps(trace))
except Exception as error:
raise RucioException("Could not send trace. " + str(error))
def update_subscription(name, account, metadata=None, session=None):
"""
Updates a subscription
:param name: Name of the subscription
:type: String
:param account: Account identifier
:type account: String
:param metadata: Dictionary of metadata to update. Supported keys : filter, replication_rules, comments, lifetime, retroactive, dry_run, priority, last_processed
:type metadata: Dict
:param session: The database session in use.
:raises: SubscriptionNotFound if subscription is not found
"""
values = {'state': SubscriptionState.UPDATED}
if 'filter' in metadata and metadata['filter']:
values['filter'] = dumps(metadata['filter'])
if 'replication_rules' in metadata and metadata['replication_rules']:
values['replication_rules'] = dumps(metadata['replication_rules'])
if 'lifetime' in metadata and metadata['lifetime']:
values['lifetime'] = datetime.datetime.utcnow() + datetime.timedelta(
days=float(metadata['lifetime']))
if 'retroactive' in metadata and metadata['retroactive']:
values['retroactive'] = metadata['retroactive']
if 'dry_run' in metadata and metadata['dry_run']:
values['dry_run'] = metadata['dry_run']
if 'comments' in metadata and metadata['comments']:
values['comments'] = metadata['comments']
if 'priority' in metadata and metadata['priority']:
values['policyid'] = metadata['priority']
if 'last_processed' in metadata and metadata['last_processed']:
values['last_processed'] = metadata['last_processed']
if 'state' in metadata and metadata['state'] == SubscriptionState.INACTIVE:
values['state'] = SubscriptionState.INACTIVE
values['expired_at'] = datetime.datetime.utcnow()
try:
rowcount = session.query(models.Subscription).filter_by(
account=account, name=name).update(values)
if rowcount == 0:
raise SubscriptionNotFound(
"Subscription for account '%(account)s' named '%(name)s' not found"
% locals())
except IntegrityError as error:
raise RucioException(error.args)
def delete_expired_tokens(total_workers,
worker_number,
limit=1000,
session=None):
"""
Delete expired tokens.
:param total_workers: Number of total workers.
:param worker_number: id of the executing worker.
:param limit: Maximum number of tokens to delete.
:param session: Database session in use.
:returns: number of deleted rows
"""
# get expired tokens
try:
# delete all expired tokens except tokens which have refresh token that is still valid
query = session.query(models.Token.token).filter(and_(models.Token.expired_at <= datetime.datetime.utcnow()))\
.filter(or_(models.Token.refresh_expired_at.__eq__(None),
models.Token.refresh_expired_at <= datetime.datetime.utcnow()))\
.order_by(models.Token.expired_at)
query = filter_thread_work(session=session,
query=query,
total_threads=total_workers,
thread_id=worker_number,
hash_variable='token')
# limiting the number of tokens deleted at once
filtered_tokens_query = query.limit(limit)
# remove expired tokens
deleted_tokens = 0
filtered_bunches = query_bunches(filtered_tokens_query, 10)
for items in filtered_bunches:
deleted_tokens += session.query(models.Token.token)\
.filter(models.Token.token.in_(items))\
.with_for_update(skip_locked=True)\
.delete(synchronize_session='fetch')
except Exception as error:
raise RucioException(error.args)
return deleted_tokens
def get_tokens_for_refresh(total_workers,
worker_number,
refreshrate=3600,
limit=1000,
session=None):
"""
Get tokens which expired or will expire before (now + refreshrate)
next run of this function and which have valid refresh token.
:param total_workers: Number of total workers.
:param worker_number: id of the executing worker.
:param limit: Maximum number of tokens to refresh per call.
:param session: Database session in use.
:return: filtered_tokens, list of tokens eligible for refresh. Throws an Exception otherwise.
"""
try:
# get tokens for refresh that expire in the next <refreshrate> seconds
expiration_future = datetime.datetime.utcnow() + datetime.timedelta(
seconds=refreshrate)
query = session.query(models.Token.token).filter(and_(models.Token.refresh == true(),
models.Token.refresh_expired_at > datetime.datetime.utcnow(),
models.Token.expired_at < expiration_future))\
.order_by(models.Token.expired_at)
query = filter_thread_work(session=session,
query=query,
total_threads=total_workers,
thread_id=worker_number,
hash_variable='token')
# limiting the number of tokens for refresh
filtered_tokens_query = query.limit(limit)
filtered_tokens = []
filtered_bunches = query_bunches(filtered_tokens_query, 10)
for items in filtered_bunches:
filtered_tokens += session.query(models.Token).filter(
models.Token.token.in_(items)).with_for_update(
skip_locked=True).all()
except Exception as error:
raise RucioException(error.args)
return filtered_tokens
def set_transfer_update_time(external_host, transfer_id, update_time=datetime.datetime.utcnow(), session=None):
"""
Update the state of a request. Fails silently if the transfer_id does not exist.
:param external_host: Selected external host as string in format protocol://fqdn:port
:param transfer_id: External transfer job id as a string.
:param update_time: Time stamp.
:param session: Database session to use.
"""
record_counter('core.request.set_transfer_update_time')
try:
rowcount = session.query(models.Request).filter_by(external_id=transfer_id, state=RequestState.SUBMITTED).update({'updated_at': update_time}, synchronize_session=False)
except IntegrityError as error:
raise RucioException(error.args)
if not rowcount:
raise UnsupportedOperation("Transfer %s doesn't exist or its status is not submitted." % (transfer_id))
def __set_transfer_state(external_host, transfer_id, new_state, session=None):
"""
Update the state of a transfer. Fails silently if the transfer_id does not exist.
:param external_host: Selected external host as string in format protocol://fqdn:port
:param transfer_id: External transfer job id as a string.
:param new_state: New state as string.
:param session: Database session to use.
"""
record_counter('core.request.set_transfer_state')
try:
rowcount = session.query(models.Request).filter_by(external_id=transfer_id).update({'state': new_state, 'updated_at': datetime.datetime.utcnow()}, synchronize_session=False)
except IntegrityError as error:
raise RucioException(error.args)
if not rowcount:
raise UnsupportedOperation("Transfer %s on %s state %s cannot be updated." % (transfer_id, external_host, new_state))
def prepare_sources_for_transfers(transfers, session=None):
"""
Prepare the sources for transfers.
:param transfers: Dictionary containing request transfer info.
:param session: Database session to use.
"""
try:
for request_id in transfers:
rowcount = session.query(models.Request)\
.filter_by(id=request_id)\
.filter(models.Request.state == RequestState.QUEUED)\
.update({'state': transfers[request_id]['state'],
'external_id': transfers[request_id]['external_id'],
'external_host': transfers[request_id]['external_host'],
'dest_url': transfers[request_id]['dest_url'],
'submitted_at': datetime.datetime.utcnow()},
synchronize_session=False)
if rowcount == 0:
raise RequestNotFound("Failed to prepare transfer: request %s does not exist or is not in queued state" % (request_id))
if 'file' in transfers[request_id]:
file = transfers[request_id]['file']
for src_rse, src_url, src_rse_id, rank in file['sources']:
src_rowcount = session.query(models.Source)\
.filter_by(request_id=request_id)\
.filter(models.Source.rse_id == src_rse_id)\
.update({'is_using': True}, synchronize_session=False)
if src_rowcount == 0:
models.Source(request_id=file['metadata']['request_id'],
scope=file['metadata']['scope'],
name=file['metadata']['name'],
rse_id=src_rse_id,
dest_rse_id=file['metadata']['dest_rse_id'],
ranking=rank if rank else 0,
bytes=file['metadata']['filesize'],
url=src_url,
is_using=True).\
save(session=session, flush=False)
except IntegrityError as error:
raise RucioException(error.args)
def _parse_list_replica_metalink(self, metalink_str):
root = ElementTree.fromstring(metalink_str)
files = []
# metalink namespace
ns = '{urn:ietf:params:xml:ns:metalink}'
# loop over all <file> tags of the metalink string
for file_ml in root.findall(ns + 'file'):
# search for identity-tag
cur_did = file_ml.find(ns + 'identity')
if not ElementTree.iselement(cur_did):
raise RucioException('Failed to locate identity-tag inside %s' % ElementTree.tostring(file_ml))
# try extracting scope,name
scope, name = self._split_did_str(cur_did.text)
cur_file = {'scope': scope,
'name': name,
'bytes': None,
'adler32': None,
'md5': None,
'sources': []}
size = file_ml.find(ns + 'size')
if ElementTree.iselement(size):
cur_file['bytes'] = int(size.text)
for cur_hash in file_ml.findall(ns + 'hash'):
hash_type = cur_hash.get('type')
if hash_type:
cur_file[hash_type] = cur_hash.text
for rse_ml in file_ml.findall(ns + 'url'):
# check if location attrib (rse name) is given
rse = rse_ml.get('location')
pfn = rse_ml.text
cur_file['sources'].append({'pfn': pfn, 'rse': rse})
files.append(cur_file)
return files
def delete_messages(messages, session=None):
"""
Delete all messages with the given IDs, and archive them to the history.
:param messages: The messages to delete as a list of dictionaries.
"""
message_condition = []
for message in messages:
message_condition.append(Message.id == message['id'])
try:
if message_condition:
session.query(Message).\
with_hint(Message, "index(messages MESSAGES_ID_PK)", 'oracle').\
filter(or_(*message_condition)).\
delete(synchronize_session=False)
session.bulk_insert_mappings(MessageHistory, messages)
except IntegrityError, e:
raise RucioException(e.args)
def get_subscription_by_id(subscription_id, session=None):
"""
Get a specific subscription by id.
:param subscription_id: The subscription_id to select.
:param session: The database session in use.
:raises: SubscriptionNotFound if no Subscription can be found.
"""
try:
subscription = session.query(models.Subscription).filter_by(id=subscription_id).one()
result = {}
for column in subscription.__table__.columns:
result[column.name] = getattr(subscription, column.name)
return result
except NoResultFound:
raise SubscriptionNotFound('No subscription with the id %s found' % (subscription_id))
except StatementError:
raise RucioException('Badly formatted subscription id (%s)' % (subscription_id))
