def create_scope_name_temp_table(name, session): """ Create a temporary table with columns 'scope' and 'name' """ columns = [ Column("scope", InternalScopeString(get_schema_value('SCOPE_LENGTH'))), Column("name", String(get_schema_value('NAME_LENGTH'))), ] return create_temp_table( name, *columns, primary_key=columns, session=session, )
def upgrade(): ''' Upgrade the database to this revision ''' if context.get_context().dialect.name in ['oracle', 'mysql', 'postgresql']: schema = context.get_context().version_table_schema if context.get_context().version_table_schema else '' add_column('dids', sa.Column('closed_at', sa.DateTime), schema=schema) add_column('contents_history', sa.Column('deleted_at', sa.DateTime), schema=schema) create_table('naming_conventions', sa.Column('scope', sa.String(get_schema_value('SCOPE_LENGTH'))), sa.Column('regexp', sa.String(255)), sa.Column('convention_type', sa.Enum(KeyType, name='CVT_TYPE_CHK', create_constraint=True, values_callable=lambda obj: [e.value for e in obj])), sa.Column('updated_at', sa.DateTime, default=datetime.datetime.utcnow), sa.Column('created_at', sa.DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow)) create_primary_key('NAMING_CONVENTIONS_PK', 'naming_conventions', ['scope']) create_foreign_key('NAMING_CONVENTIONS_SCOPE_FK', 'naming_conventions', 'scopes', ['scope'], ['scope']) create_check_constraint('NAMING_CONVENTIONS_CREATED_NN', 'naming_conventions', 'created_at is not null') create_check_constraint('NAMING_CONVENTIONS_UPDATED_NN', 'naming_conventions', 'updated_at is not null')
def run(once=False, group_bulk=1, group_policy='rule', mock=False, rses=None, include_rses=None, exclude_rses=None, vos=None, bulk=100, source_strategy=None, activities=None, exclude_activities=None, sleep_time=600, max_sources=4, retry_other_fts=False, total_threads=1): """ Starts up the conveyer threads. """ if mock: logging.info('mock source replicas: enabled') multi_vo = config_get_bool('common', 'multi_vo', raise_exception=False, default=False) working_rses = None if rses or include_rses or exclude_rses: working_rses = get_conveyor_rses(rses, include_rses, exclude_rses, vos) logging.info("RSE selection: RSEs: %s, Include: %s, Exclude: %s", rses, include_rses, exclude_rses) elif multi_vo: working_rses = get_conveyor_rses(rses, include_rses, exclude_rses, vos) logging.info("RSE selection: automatic for relevant VOs") else: logging.info("RSE selection: automatic") logging.info('starting submitter threads') if exclude_activities: if not activities: if not multi_vo: vos = ['def'] if vos and len(vos) == 1: activities = get_schema_value('ACTIVITY', vos[0]) elif vos and len(vos) > 1: logging.warning('Cannot get activity list from schema when multiple VOs given, either provide `activities` argument or run on a single VO') activities = [None] else: logging.warning('Cannot get activity list from schema when no VO given, either provide `activities` argument or `vos` with a single entry') activities = [None] for activity in exclude_activities: if activity in activities: activities.remove(activity) threads = [threading.Thread(target=submitter, kwargs={'once': once, 'rses': working_rses, 'bulk': bulk, 'group_bulk': group_bulk, 'group_policy': group_policy, 'activities': activities, 'mock': mock, 'sleep_time': sleep_time, 'max_sources': max_sources, 'source_strategy': source_strategy, 'retry_other_fts': retry_other_fts}) for _ in range(0, total_threads)] [thread.start() for thread in threads] logging.info('waiting for interrupts') # Interruptible joins require a timeout. while threads: threads = [thread.join(timeout=3.14) for thread in threads if thread and thread.isAlive()]
def upgrade(): ''' Upgrade the database to this revision ''' if context.get_context().dialect.name in ['oracle', 'mysql', 'postgresql']: create_table('virtual_placements', sa.Column('scope', sa.String(get_schema_value('SCOPE_LENGTH'))), sa.Column('name', sa.String(get_schema_value('NAME_LENGTH'))), sa.Column('placements', JSON()), sa.Column('created_at', sa.DateTime), sa.Column('updated_at', sa.DateTime) ) create_primary_key('VP_PK', 'virtual_placements', ['scope', 'name']) create_foreign_key('VP_FK', 'virtual_placements', 'dids', ['scope', 'name'], ['scope', 'name'])
def upgrade(): ''' Upgrade the database to this revision ''' if context.get_context().dialect.name in ['oracle', 'mysql', 'postgresql']: create_table( 'quarantined_replicas', sa.Column('rse_id', GUID()), sa.Column('path', sa.String(1024)), sa.Column('bytes', sa.BigInteger), sa.Column('md5', sa.String(32)), sa.Column('adler32', sa.String(8)), sa.Column('scope', sa.String(get_schema_value('SCOPE_LENGTH'))), sa.Column('name', sa.String(get_schema_value('NAME_LENGTH'))), sa.Column('created_at', sa.DateTime, default=datetime.datetime.utcnow), sa.Column('updated_at', sa.DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow)) create_table( 'quarantined_replicas_history', sa.Column('rse_id', GUID()), sa.Column('path', sa.String(1024)), sa.Column('bytes', sa.BigInteger), sa.Column('md5', sa.String(32)), sa.Column('adler32', sa.String(8)), sa.Column('scope', sa.String(get_schema_value('SCOPE_LENGTH'))), sa.Column('name', sa.String(get_schema_value('NAME_LENGTH'))), sa.Column('created_at', sa.DateTime), sa.Column('updated_at', sa.DateTime), sa.Column('deleted_at', sa.DateTime, default=datetime.datetime.utcnow)) create_primary_key('QURD_REPLICAS_STATE_PK', 'quarantined_replicas', ['rse_id', 'path']) create_check_constraint('QURD_REPLICAS_CREATED_NN', 'quarantined_replicas', 'created_at is not null') create_check_constraint('QURD_REPLICAS_UPDATED_NN', 'quarantined_replicas', 'updated_at is not null') create_foreign_key('QURD_REPLICAS_RSE_ID_FK', 'quarantined_replicas', 'rses', ['rse_id'], ['id'])
def parse_scope_name(scope_name): """ Parses the given scope_name according to the schema's SCOPE_NAME_REGEXP and returns a (scope, name) tuple. :param scope_name: the scope_name string to be parsed. :raises ValueError: when scope_name could not be parsed. :returns: a (scope, name) tuple. """ # why again does that regex start with a slash? scope_name = re.match(get_schema_value('SCOPE_NAME_REGEXP'), '/' + scope_name) if scope_name is None: raise ValueError('cannot parse scope and name') return scope_name.group(1, 2)
def run(once=False, group_bulk=1, group_policy='rule', mock=False, rses=None, include_rses=None, exclude_rses=None, bulk=100, source_strategy=None, activities=None, exclude_activities=None, sleep_time=600, max_sources=4, retry_other_fts=False, total_threads=1): """ Starts up the conveyer threads. """ if mock: logging.info('mock source replicas: enabled') working_rses = None if rses or include_rses or exclude_rses: working_rses = get_conveyor_rses(rses, include_rses, exclude_rses) logging.info("RSE selection: RSEs: %s, Include: %s, Exclude: %s", rses, include_rses, exclude_rses) else: logging.info("RSE selection: automatic") logging.info('starting submitter threads') if exclude_activities: if not activities: activities = get_schema_value('ACTIVITY') for activity in exclude_activities: if activity in activities: activities.remove(activity) threads = [threading.Thread(target=submitter, kwargs={'once': once, 'rses': working_rses, 'bulk': bulk, 'group_bulk': group_bulk, 'group_policy': group_policy, 'activities': activities, 'mock': mock, 'sleep_time': sleep_time, 'max_sources': max_sources, 'source_strategy': source_strategy, 'retry_other_fts': retry_other_fts}) for _ in range(0, total_threads)] [thread.start() for thread in threads] logging.info('waiting for interrupts') # Interruptible joins require a timeout. while threads: threads = [thread.join(timeout=3.14) for thread in threads if thread and thread.isAlive()]
except ImportError: from urllib.parse import parse_qs from web import application, ctx, header, InternalError, loadhook from rucio.api.lock import get_dataset_locks_by_rse, get_dataset_locks from rucio.common.exception import RucioException, RSENotFound from rucio.common.schema import get_schema_value from rucio.common.utils import generate_http_error, render_json from rucio.web.rest.common import rucio_loadhook, check_accept_header_wrapper LOGGER = getLogger("rucio.lock") SH = StreamHandler() SH.setLevel(DEBUG) LOGGER.addHandler(SH) URLS = ('%s' % get_schema_value('SCOPE_NAME_REGEXP'), 'LockByScopeName', '/(.*)', 'LockByRSE') class LockByRSE(object): """ REST APIs for dataset locks. """ @check_accept_header_wrapper(['application/x-json-stream']) def GET(self, rse): """ get locks for a given rse. HTTP Success: 200 OK HTTP Error: 404 Not Found 406 Not Acceptable
from rucio.common.replica_sorter import sort_random, sort_geoip, sort_closeness, sort_dynamic, sort_ranking from rucio.common.schema import get_schema_value from rucio.common.utils import generate_http_error, parse_response, APIEncoder, render_json_list from rucio.common.constants import SUPPORTED_PROTOCOLS from rucio.web.rest.common import rucio_loadhook, rucio_unloadhook, RucioController, check_accept_header_wrapper URLS = ('/list/?$', 'ListReplicas', '/?$', 'Replicas', '/suspicious/?$', 'SuspiciousReplicas', '/bad/states/?$', 'BadReplicasStates', '/bad/summary/?$', 'BadReplicasSummary', '/bad/pfns/?$', 'BadPFNs', '/rse/(.*)/?$', 'ReplicasRSE', '/bad/?$', 'BadReplicas', '/dids/?$', 'ReplicasDIDs', '%s/datasets$' % get_schema_value('SCOPE_NAME_REGEXP'), 'DatasetReplicas', '/datasets_bulk/?$', 'DatasetReplicasBulk', '%s/datasets_vp$' % get_schema_value('SCOPE_NAME_REGEXP'), 'DatasetReplicasVP', '%s/?$' % get_schema_value('SCOPE_NAME_REGEXP'), 'Replicas', '/tombstone/?$', 'Tombstone') class Replicas(RucioController): @check_accept_header_wrapper(['application/x-json-stream', 'application/metalink4+xml']) def GET(self, scope, name): """ List all replicas for data identifiers. HTTP Success: 200 OK
try: from urlparse import parse_qs except ImportError: from urllib.parse import parse_qs URLS = ( '/(.*)/$', 'Scope', '/(.*)/guid', 'GUIDLookup', '/(.*)/dids/search', 'Search', '/(.*)/dids/search_extended', 'SearchExtended', '%s/files' % get_schema_value('SCOPE_NAME_REGEXP'), 'Files', '%s/dids/history' % get_schema_value('SCOPE_NAME_REGEXP'), 'AttachmentHistory', '%s/dids' % get_schema_value('SCOPE_NAME_REGEXP'), 'Attachment', '%s/meta/(.*)' % get_schema_value('SCOPE_NAME_REGEXP'), 'Meta', '%s/meta' % get_schema_value('SCOPE_NAME_REGEXP'), 'Meta', '%s/status' % get_schema_value('SCOPE_NAME_REGEXP'), 'DIDs', '%s/rules' % get_schema_value('SCOPE_NAME_REGEXP'), 'Rules', '%s/parents' % get_schema_value('SCOPE_NAME_REGEXP'), 'Parents',
import abc import re from dogpile.cache import make_region from dogpile.cache.api import NoValue from hashlib import sha256 from six import add_metaclass from rucio.common import schema from rucio.common.config import config_get from rucio.common.exception import InvalidRSEExpression, RSEBlacklisted from rucio.core.rse import list_rses, get_rses_with_attribute, get_rse_attribute from rucio.db.sqla.session import transactional_session DEFAULT_RSE_ATTRIBUTE = schema.get_schema_value( 'DEFAULT_RSE_ATTRIBUTE')['pattern'] RSE_ATTRIBUTE = schema.get_schema_value('RSE_ATTRIBUTE')['pattern'] PRIMITIVE = r'(\(*(%s|%s|%s)\)*)' % (RSE_ATTRIBUTE, DEFAULT_RSE_ATTRIBUTE, r'\*') UNION = r'(\|%s)' % (PRIMITIVE) INTERSECTION = r'(\&%s)' % (PRIMITIVE) COMPLEMENT = r'(\\%s)' % (PRIMITIVE) PATTERN = r'^%s(%s|%s|%s)*' % (PRIMITIVE, UNION, INTERSECTION, COMPLEMENT) REGION = make_region().configure('dogpile.cache.memcached', expiration_time=600, arguments={ 'url': config_get('cache', 'url', False, '127.0.0.1:11211'),
DataIdentifierAlreadyExists, DuplicateContent, AccessDenied, KeyNotFound, DatabaseException, Duplicate, InvalidValueForKey, UnsupportedStatus, UnsupportedOperation, RSENotFound, RucioException, RuleNotFound, InvalidMetadata) from rucio.common.schema import get_schema_value from rucio.common.utils import generate_http_error, render_json, APIEncoder, parse_response from rucio.web.rest.common import rucio_loadhook, RucioController, check_accept_header_wrapper URLS = ( '/(.*)/$', 'Scope', '/(.*)/guid', 'GUIDLookup', '/(.*)/dids/search', 'Search', '/(.*)/dids/search_extended', 'SearchExtended', '%s/files' % get_schema_value('SCOPE_NAME_REGEXP'), 'Files', '%s/dids/history' % get_schema_value('SCOPE_NAME_REGEXP'), 'AttachmentHistory', '%s/dids' % get_schema_value('SCOPE_NAME_REGEXP'), 'Attachment', '%s/meta/(.*)' % get_schema_value('SCOPE_NAME_REGEXP'), 'Meta', '%s/meta' % get_schema_value('SCOPE_NAME_REGEXP'), 'Meta', '%s/status' % get_schema_value('SCOPE_NAME_REGEXP'), 'DIDs', '%s/rules' % get_schema_value('SCOPE_NAME_REGEXP'), 'Rules', '%s/parents' % get_schema_value('SCOPE_NAME_REGEXP'), 'Parents', '%s/associated_rules' % get_schema_value('SCOPE_NAME_REGEXP'), 'AssociatedRules', '%s/did_meta' % get_schema_value('SCOPE_NAME_REGEXP'), 'DidMeta', '/(.*)/(.*)/(.*)/(.*)/(.*)/sample', 'Sample', '%s' % get_schema_value('SCOPE_NAME_REGEXP'), 'DIDs', '', 'BulkDIDS', '/attachments', 'Attachments', '/new', 'NewDIDs', '/resurrect', 'Resurrect',
from rucio.common.utils import generate_http_error, render_json, APIEncoder from rucio.web.rest.common import rucio_loadhook, check_accept_header_wrapper LOGGER = getLogger("rucio.rule") SH = StreamHandler() SH.setLevel(DEBUG) LOGGER.addHandler(SH) URLS = ( '/(.+)/locks', 'ReplicaLocks', '/(.+)/reduce', 'ReduceRule', '/(.+)/move', 'MoveRule', '%s/history' % get_schema_value('SCOPE_NAME_REGEXP'), 'RuleHistoryFull', '/(.+)/history', 'RuleHistory', '/(.+)/analysis', 'RuleAnalysis', '/', 'AllRule', '/(.+)', 'Rule', ) class Rule: """ REST APIs for replication rules. """ @check_accept_header_wrapper(['application/json'])
from __future__ import print_function from json import dumps from logging import getLogger, StreamHandler, DEBUG from traceback import format_exc from web import application, ctx, loadhook, header, InternalError from rucio.api.did import list_archive_content from rucio.common.schema import get_schema_value from rucio.web.rest.common import rucio_loadhook, RucioController, check_accept_header_wrapper LOGGER, SH = getLogger("rucio.meta"), StreamHandler() SH.setLevel(DEBUG) LOGGER.addHandler(SH) URLS = ('%s/files' % get_schema_value('SCOPE_NAME_REGEXP'), 'Archive') class Archive(RucioController): """ REST APIs for archive. """ @check_accept_header_wrapper(['application/x-json-stream']) def GET(self, scope, name): """ List archive content keys. HTTP Success: 200 Success HTTP Error: 406 Not Acceptable """ header('Content-Type', 'application/x-json-stream')
ReplicationRuleCreationTemporaryFailed, InvalidRuleWeight, StagingAreaRuleRequiresLifetime, DuplicateRule, InvalidObject, AccountNotFound, RuleReplaceFailed, ScratchDiskLifetimeConflict, ManualRuleApprovalBlocked, UnsupportedOperation) from rucio.common.schema import get_schema_value from rucio.common.utils import generate_http_error, render_json, APIEncoder from rucio.web.rest.common import rucio_loadhook, check_accept_header_wrapper LOGGER = getLogger("rucio.rule") SH = StreamHandler() SH.setLevel(DEBUG) LOGGER.addHandler(SH) URLS = ('/(.+)/locks', 'ReplicaLocks', '/(.+)/reduce', 'ReduceRule', '/(.+)/move', 'MoveRule', '%s/history' % get_schema_value('SCOPE_NAME_REGEXP'), 'RuleHistoryFull', '/(.+)/history', 'RuleHistory', '/(.+)/analysis', 'RuleAnalysis', '/', 'AllRule', '/(.+)', 'Rule',) class Rule: """ REST APIs for replication rules. """ @check_accept_header_wrapper(['application/json']) def GET(self, rule_id): """ get rule information for given rule id. HTTP Success: 200 OK
from web import application, ctx, loadhook, header from rucio.api import request from rucio.db.sqla.constants import RequestState from rucio.core.rse import get_rses_with_attribute_value, get_rse_name from rucio.common.schema import get_schema_value from rucio.common.utils import generate_http_error, render_json from rucio.web.rest.common import rucio_loadhook, RucioController, exception_wrapper, check_accept_header_wrapper LOGGER = getLogger("rucio.request") SH = StreamHandler() SH.setLevel(DEBUG) LOGGER.addHandler(SH) URLS = ('%s/(.+)' % get_schema_value('SCOPE_NAME_REGEXP'), 'RequestGet', '/list', 'RequestsGet') class RequestGet(RucioController): """ REST API to get requests. """ @exception_wrapper @check_accept_header_wrapper(['application/json']) def GET(self, scope, name, rse): """ List request for given DID to a destination RSE. HTTP Success: 200 OK HTTP Error:
'/accounts/(.+)/usage/', 'ALocalUsage', '/accounts/(.+)/usage/(.+)', 'ALocalUsage', '/accounts/(.+)', 'AAccountParameter', '/accounts/?$', 'AAccount' ] URLS += [ '/accountlimits/local/(.+)/(.+)', 'ALLocalAccountLimit', '/accountlimits/global/(.+)/(.+)', 'ALGlobalAccountLimit', '/accountlimits/(.+)/(.+)', 'ALLocalAccountLimit', ] URLS += [ '/archives%s/files' % get_schema_value('SCOPE_NAME_REGEXP'), 'AVArchive' ] URLS += [ '/config/(.+)/(.+)/(.*)', 'COptionSet', '/config/(.+)/(.+)', 'COptionGetDel', '/config/(.+)', 'CSection', '/config', 'CConfig' ] URLS += [ '/dids/(.*)/$', 'DScope', '/dids/(.*)/guid', 'DGUIDLookup', '/dids/(.*)/dids/search', 'DSearch', '/dids%s/files' % get_schema_value('SCOPE_NAME_REGEXP'),
from rucio.common.replica_sorter import sort_random, sort_geoip, sort_closeness, sort_ranking, sort_dynamic, site_selector from rucio.common.schema import get_schema_value from rucio.web.rest.common import RucioController, check_accept_header_wrapper from rucio.web.rest.utils import generate_http_error try: from urlparse import parse_qs except ImportError: from urllib.parse import parse_qs LOGGER = getLogger("rucio.rucio") SH = StreamHandler() SH.setLevel(DEBUG) LOGGER.addHandler(SH) URLS = ('%s/metalink?$' % get_schema_value('SCOPE_NAME_REGEXP'), 'MetaLinkRedirector', '%s/?$' % get_schema_value('SCOPE_NAME_REGEXP'), 'HeaderRedirector') class MetaLinkRedirector(RucioController): @check_accept_header_wrapper(['application/metalink4+xml']) def GET(self, scope, name): """ Metalink redirect HTTP Success: 200 OK HTTP Error: 401 Unauthorized