def create_scope_name_temp_table(name, session):
"""
Create a temporary table with columns 'scope' and 'name'
"""
columns = [
Column("scope", InternalScopeString(get_schema_value('SCOPE_LENGTH'))),
Column("name", String(get_schema_value('NAME_LENGTH'))),
]
return create_temp_table(
name,
*columns,
primary_key=columns,
session=session,
)
def upgrade():
'''
Upgrade the database to this revision
'''
if context.get_context().dialect.name in ['oracle', 'mysql', 'postgresql']:
schema = context.get_context().version_table_schema if context.get_context().version_table_schema else ''
add_column('dids', sa.Column('closed_at', sa.DateTime), schema=schema)
add_column('contents_history', sa.Column('deleted_at', sa.DateTime), schema=schema)
create_table('naming_conventions',
sa.Column('scope', sa.String(get_schema_value('SCOPE_LENGTH'))),
sa.Column('regexp', sa.String(255)),
sa.Column('convention_type', sa.Enum(KeyType,
name='CVT_TYPE_CHK',
create_constraint=True,
values_callable=lambda obj: [e.value for e in obj])),
sa.Column('updated_at', sa.DateTime, default=datetime.datetime.utcnow),
sa.Column('created_at', sa.DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow))
create_primary_key('NAMING_CONVENTIONS_PK', 'naming_conventions', ['scope'])
create_foreign_key('NAMING_CONVENTIONS_SCOPE_FK', 'naming_conventions',
'scopes', ['scope'], ['scope'])
create_check_constraint('NAMING_CONVENTIONS_CREATED_NN', 'naming_conventions',
'created_at is not null')
create_check_constraint('NAMING_CONVENTIONS_UPDATED_NN', 'naming_conventions',
'updated_at is not null')
def run(once=False, group_bulk=1, group_policy='rule', mock=False,
rses=None, include_rses=None, exclude_rses=None, vos=None, bulk=100, source_strategy=None,
activities=None, exclude_activities=None, sleep_time=600, max_sources=4, retry_other_fts=False, total_threads=1):
"""
Starts up the conveyer threads.
"""
if mock:
logging.info('mock source replicas: enabled')
multi_vo = config_get_bool('common', 'multi_vo', raise_exception=False, default=False)
working_rses = None
if rses or include_rses or exclude_rses:
working_rses = get_conveyor_rses(rses, include_rses, exclude_rses, vos)
logging.info("RSE selection: RSEs: %s, Include: %s, Exclude: %s", rses, include_rses, exclude_rses)
elif multi_vo:
working_rses = get_conveyor_rses(rses, include_rses, exclude_rses, vos)
logging.info("RSE selection: automatic for relevant VOs")
else:
logging.info("RSE selection: automatic")
logging.info('starting submitter threads')
if exclude_activities:
if not activities:
if not multi_vo:
vos = ['def']
if vos and len(vos) == 1:
activities = get_schema_value('ACTIVITY', vos[0])
elif vos and len(vos) > 1:
logging.warning('Cannot get activity list from schema when multiple VOs given, either provide `activities` argument or run on a single VO')
activities = [None]
else:
logging.warning('Cannot get activity list from schema when no VO given, either provide `activities` argument or `vos` with a single entry')
activities = [None]
for activity in exclude_activities:
if activity in activities:
activities.remove(activity)
threads = [threading.Thread(target=submitter, kwargs={'once': once,
'rses': working_rses,
'bulk': bulk,
'group_bulk': group_bulk,
'group_policy': group_policy,
'activities': activities,
'mock': mock,
'sleep_time': sleep_time,
'max_sources': max_sources,
'source_strategy': source_strategy,
'retry_other_fts': retry_other_fts}) for _ in range(0, total_threads)]
[thread.start() for thread in threads]
logging.info('waiting for interrupts')
# Interruptible joins require a timeout.
while threads:
threads = [thread.join(timeout=3.14) for thread in threads if thread and thread.isAlive()]
def upgrade():
'''
Upgrade the database to this revision
'''
if context.get_context().dialect.name in ['oracle', 'mysql', 'postgresql']:
create_table('virtual_placements',
sa.Column('scope', sa.String(get_schema_value('SCOPE_LENGTH'))),
sa.Column('name', sa.String(get_schema_value('NAME_LENGTH'))),
sa.Column('placements', JSON()),
sa.Column('created_at', sa.DateTime),
sa.Column('updated_at', sa.DateTime)
)
create_primary_key('VP_PK', 'virtual_placements', ['scope', 'name'])
create_foreign_key('VP_FK', 'virtual_placements', 'dids',
['scope', 'name'], ['scope', 'name'])
def upgrade():
'''
Upgrade the database to this revision
'''
if context.get_context().dialect.name in ['oracle', 'mysql', 'postgresql']:
create_table(
'quarantined_replicas', sa.Column('rse_id', GUID()),
sa.Column('path', sa.String(1024)),
sa.Column('bytes', sa.BigInteger), sa.Column('md5', sa.String(32)),
sa.Column('adler32', sa.String(8)),
sa.Column('scope', sa.String(get_schema_value('SCOPE_LENGTH'))),
sa.Column('name', sa.String(get_schema_value('NAME_LENGTH'))),
sa.Column('created_at',
sa.DateTime,
default=datetime.datetime.utcnow),
sa.Column('updated_at',
sa.DateTime,
default=datetime.datetime.utcnow,
onupdate=datetime.datetime.utcnow))
create_table(
'quarantined_replicas_history', sa.Column('rse_id', GUID()),
sa.Column('path', sa.String(1024)),
sa.Column('bytes', sa.BigInteger), sa.Column('md5', sa.String(32)),
sa.Column('adler32', sa.String(8)),
sa.Column('scope', sa.String(get_schema_value('SCOPE_LENGTH'))),
sa.Column('name', sa.String(get_schema_value('NAME_LENGTH'))),
sa.Column('created_at', sa.DateTime),
sa.Column('updated_at', sa.DateTime),
sa.Column('deleted_at',
sa.DateTime,
default=datetime.datetime.utcnow))
create_primary_key('QURD_REPLICAS_STATE_PK', 'quarantined_replicas',
['rse_id', 'path'])
create_check_constraint('QURD_REPLICAS_CREATED_NN',
'quarantined_replicas',
'created_at is not null')
create_check_constraint('QURD_REPLICAS_UPDATED_NN',
'quarantined_replicas',
'updated_at is not null')
create_foreign_key('QURD_REPLICAS_RSE_ID_FK', 'quarantined_replicas',
'rses', ['rse_id'], ['id'])
def parse_scope_name(scope_name):
"""
Parses the given scope_name according to the schema's
SCOPE_NAME_REGEXP and returns a (scope, name) tuple.
:param scope_name: the scope_name string to be parsed.
:raises ValueError: when scope_name could not be parsed.
:returns: a (scope, name) tuple.
"""
# why again does that regex start with a slash?
scope_name = re.match(get_schema_value('SCOPE_NAME_REGEXP'), '/' + scope_name)
if scope_name is None:
raise ValueError('cannot parse scope and name')
return scope_name.group(1, 2)
def run(once=False, group_bulk=1, group_policy='rule',
mock=False, rses=None, include_rses=None, exclude_rses=None, bulk=100, source_strategy=None,
activities=None, exclude_activities=None, sleep_time=600, max_sources=4, retry_other_fts=False, total_threads=1):
"""
Starts up the conveyer threads.
"""
if mock:
logging.info('mock source replicas: enabled')
working_rses = None
if rses or include_rses or exclude_rses:
working_rses = get_conveyor_rses(rses, include_rses, exclude_rses)
logging.info("RSE selection: RSEs: %s, Include: %s, Exclude: %s", rses, include_rses, exclude_rses)
else:
logging.info("RSE selection: automatic")
logging.info('starting submitter threads')
if exclude_activities:
if not activities:
activities = get_schema_value('ACTIVITY')
for activity in exclude_activities:
if activity in activities:
activities.remove(activity)
threads = [threading.Thread(target=submitter, kwargs={'once': once,
'rses': working_rses,
'bulk': bulk,
'group_bulk': group_bulk,
'group_policy': group_policy,
'activities': activities,
'mock': mock,
'sleep_time': sleep_time,
'max_sources': max_sources,
'source_strategy': source_strategy,
'retry_other_fts': retry_other_fts}) for _ in range(0, total_threads)]
[thread.start() for thread in threads]
logging.info('waiting for interrupts')
# Interruptible joins require a timeout.
while threads:
threads = [thread.join(timeout=3.14) for thread in threads if thread and thread.isAlive()]
except ImportError:
from urllib.parse import parse_qs
from web import application, ctx, header, InternalError, loadhook
from rucio.api.lock import get_dataset_locks_by_rse, get_dataset_locks
from rucio.common.exception import RucioException, RSENotFound
from rucio.common.schema import get_schema_value
from rucio.common.utils import generate_http_error, render_json
from rucio.web.rest.common import rucio_loadhook, check_accept_header_wrapper
LOGGER = getLogger("rucio.lock")
SH = StreamHandler()
SH.setLevel(DEBUG)
LOGGER.addHandler(SH)
URLS = ('%s' % get_schema_value('SCOPE_NAME_REGEXP'), 'LockByScopeName',
'/(.*)', 'LockByRSE')
class LockByRSE(object):
""" REST APIs for dataset locks. """
@check_accept_header_wrapper(['application/x-json-stream'])
def GET(self, rse):
""" get locks for a given rse.
HTTP Success:
200 OK
HTTP Error:
404 Not Found
406 Not Acceptable
from rucio.common.replica_sorter import sort_random, sort_geoip, sort_closeness, sort_dynamic, sort_ranking
from rucio.common.schema import get_schema_value
from rucio.common.utils import generate_http_error, parse_response, APIEncoder, render_json_list
from rucio.common.constants import SUPPORTED_PROTOCOLS
from rucio.web.rest.common import rucio_loadhook, rucio_unloadhook, RucioController, check_accept_header_wrapper
URLS = ('/list/?$', 'ListReplicas',
'/?$', 'Replicas',
'/suspicious/?$', 'SuspiciousReplicas',
'/bad/states/?$', 'BadReplicasStates',
'/bad/summary/?$', 'BadReplicasSummary',
'/bad/pfns/?$', 'BadPFNs',
'/rse/(.*)/?$', 'ReplicasRSE',
'/bad/?$', 'BadReplicas',
'/dids/?$', 'ReplicasDIDs',
'%s/datasets$' % get_schema_value('SCOPE_NAME_REGEXP'), 'DatasetReplicas',
'/datasets_bulk/?$', 'DatasetReplicasBulk',
'%s/datasets_vp$' % get_schema_value('SCOPE_NAME_REGEXP'), 'DatasetReplicasVP',
'%s/?$' % get_schema_value('SCOPE_NAME_REGEXP'), 'Replicas',
'/tombstone/?$', 'Tombstone')
class Replicas(RucioController):
@check_accept_header_wrapper(['application/x-json-stream', 'application/metalink4+xml'])
def GET(self, scope, name):
"""
List all replicas for data identifiers.
HTTP Success:
200 OK
try:
from urlparse import parse_qs
except ImportError:
from urllib.parse import parse_qs
URLS = (
'/(.*)/$',
'Scope',
'/(.*)/guid',
'GUIDLookup',
'/(.*)/dids/search',
'Search',
'/(.*)/dids/search_extended',
'SearchExtended',
'%s/files' % get_schema_value('SCOPE_NAME_REGEXP'),
'Files',
'%s/dids/history' % get_schema_value('SCOPE_NAME_REGEXP'),
'AttachmentHistory',
'%s/dids' % get_schema_value('SCOPE_NAME_REGEXP'),
'Attachment',
'%s/meta/(.*)' % get_schema_value('SCOPE_NAME_REGEXP'),
'Meta',
'%s/meta' % get_schema_value('SCOPE_NAME_REGEXP'),
'Meta',
'%s/status' % get_schema_value('SCOPE_NAME_REGEXP'),
'DIDs',
'%s/rules' % get_schema_value('SCOPE_NAME_REGEXP'),
'Rules',
'%s/parents' % get_schema_value('SCOPE_NAME_REGEXP'),
'Parents',
import abc
import re
from dogpile.cache import make_region
from dogpile.cache.api import NoValue
from hashlib import sha256
from six import add_metaclass
from rucio.common import schema
from rucio.common.config import config_get
from rucio.common.exception import InvalidRSEExpression, RSEBlacklisted
from rucio.core.rse import list_rses, get_rses_with_attribute, get_rse_attribute
from rucio.db.sqla.session import transactional_session
DEFAULT_RSE_ATTRIBUTE = schema.get_schema_value(
'DEFAULT_RSE_ATTRIBUTE')['pattern']
RSE_ATTRIBUTE = schema.get_schema_value('RSE_ATTRIBUTE')['pattern']
PRIMITIVE = r'(\(*(%s|%s|%s)\)*)' % (RSE_ATTRIBUTE, DEFAULT_RSE_ATTRIBUTE,
r'\*')
UNION = r'(\|%s)' % (PRIMITIVE)
INTERSECTION = r'(\&%s)' % (PRIMITIVE)
COMPLEMENT = r'(\\%s)' % (PRIMITIVE)
PATTERN = r'^%s(%s|%s|%s)*' % (PRIMITIVE, UNION, INTERSECTION, COMPLEMENT)
REGION = make_region().configure('dogpile.cache.memcached',
expiration_time=600,
arguments={
'url':
config_get('cache', 'url', False,
'127.0.0.1:11211'),
DataIdentifierAlreadyExists, DuplicateContent,
AccessDenied, KeyNotFound, DatabaseException,
Duplicate, InvalidValueForKey,
UnsupportedStatus, UnsupportedOperation,
RSENotFound, RucioException, RuleNotFound,
InvalidMetadata)
from rucio.common.schema import get_schema_value
from rucio.common.utils import generate_http_error, render_json, APIEncoder, parse_response
from rucio.web.rest.common import rucio_loadhook, RucioController, check_accept_header_wrapper
URLS = (
'/(.*)/$', 'Scope',
'/(.*)/guid', 'GUIDLookup',
'/(.*)/dids/search', 'Search',
'/(.*)/dids/search_extended', 'SearchExtended',
'%s/files' % get_schema_value('SCOPE_NAME_REGEXP'), 'Files',
'%s/dids/history' % get_schema_value('SCOPE_NAME_REGEXP'), 'AttachmentHistory',
'%s/dids' % get_schema_value('SCOPE_NAME_REGEXP'), 'Attachment',
'%s/meta/(.*)' % get_schema_value('SCOPE_NAME_REGEXP'), 'Meta',
'%s/meta' % get_schema_value('SCOPE_NAME_REGEXP'), 'Meta',
'%s/status' % get_schema_value('SCOPE_NAME_REGEXP'), 'DIDs',
'%s/rules' % get_schema_value('SCOPE_NAME_REGEXP'), 'Rules',
'%s/parents' % get_schema_value('SCOPE_NAME_REGEXP'), 'Parents',
'%s/associated_rules' % get_schema_value('SCOPE_NAME_REGEXP'), 'AssociatedRules',
'%s/did_meta' % get_schema_value('SCOPE_NAME_REGEXP'), 'DidMeta',
'/(.*)/(.*)/(.*)/(.*)/(.*)/sample', 'Sample',
'%s' % get_schema_value('SCOPE_NAME_REGEXP'), 'DIDs',
'', 'BulkDIDS',
'/attachments', 'Attachments',
'/new', 'NewDIDs',
'/resurrect', 'Resurrect',
from rucio.common.utils import generate_http_error, render_json, APIEncoder
from rucio.web.rest.common import rucio_loadhook, check_accept_header_wrapper
LOGGER = getLogger("rucio.rule")
SH = StreamHandler()
SH.setLevel(DEBUG)
LOGGER.addHandler(SH)
URLS = (
'/(.+)/locks',
'ReplicaLocks',
'/(.+)/reduce',
'ReduceRule',
'/(.+)/move',
'MoveRule',
'%s/history' % get_schema_value('SCOPE_NAME_REGEXP'),
'RuleHistoryFull',
'/(.+)/history',
'RuleHistory',
'/(.+)/analysis',
'RuleAnalysis',
'/',
'AllRule',
'/(.+)',
'Rule',
)
class Rule:
""" REST APIs for replication rules. """
@check_accept_header_wrapper(['application/json'])
from __future__ import print_function
from json import dumps
from logging import getLogger, StreamHandler, DEBUG
from traceback import format_exc
from web import application, ctx, loadhook, header, InternalError
from rucio.api.did import list_archive_content
from rucio.common.schema import get_schema_value
from rucio.web.rest.common import rucio_loadhook, RucioController, check_accept_header_wrapper
LOGGER, SH = getLogger("rucio.meta"), StreamHandler()
SH.setLevel(DEBUG)
LOGGER.addHandler(SH)
URLS = ('%s/files' % get_schema_value('SCOPE_NAME_REGEXP'), 'Archive')
class Archive(RucioController):
""" REST APIs for archive. """
@check_accept_header_wrapper(['application/x-json-stream'])
def GET(self, scope, name):
"""
List archive content keys.
HTTP Success:
200 Success
HTTP Error:
406 Not Acceptable
"""
header('Content-Type', 'application/x-json-stream')
ReplicationRuleCreationTemporaryFailed, InvalidRuleWeight, StagingAreaRuleRequiresLifetime,
DuplicateRule, InvalidObject, AccountNotFound, RuleReplaceFailed, ScratchDiskLifetimeConflict,
ManualRuleApprovalBlocked, UnsupportedOperation)
from rucio.common.schema import get_schema_value
from rucio.common.utils import generate_http_error, render_json, APIEncoder
from rucio.web.rest.common import rucio_loadhook, check_accept_header_wrapper
LOGGER = getLogger("rucio.rule")
SH = StreamHandler()
SH.setLevel(DEBUG)
LOGGER.addHandler(SH)
URLS = ('/(.+)/locks', 'ReplicaLocks',
'/(.+)/reduce', 'ReduceRule',
'/(.+)/move', 'MoveRule',
'%s/history' % get_schema_value('SCOPE_NAME_REGEXP'), 'RuleHistoryFull',
'/(.+)/history', 'RuleHistory',
'/(.+)/analysis', 'RuleAnalysis',
'/', 'AllRule',
'/(.+)', 'Rule',)
class Rule:
""" REST APIs for replication rules. """
@check_accept_header_wrapper(['application/json'])
def GET(self, rule_id):
""" get rule information for given rule id.
HTTP Success:
200 OK
from web import application, ctx, loadhook, header
from rucio.api import request
from rucio.db.sqla.constants import RequestState
from rucio.core.rse import get_rses_with_attribute_value, get_rse_name
from rucio.common.schema import get_schema_value
from rucio.common.utils import generate_http_error, render_json
from rucio.web.rest.common import rucio_loadhook, RucioController, exception_wrapper, check_accept_header_wrapper
LOGGER = getLogger("rucio.request")
SH = StreamHandler()
SH.setLevel(DEBUG)
LOGGER.addHandler(SH)
URLS = ('%s/(.+)' % get_schema_value('SCOPE_NAME_REGEXP'), 'RequestGet',
'/list', 'RequestsGet')
class RequestGet(RucioController):
""" REST API to get requests. """
@exception_wrapper
@check_accept_header_wrapper(['application/json'])
def GET(self, scope, name, rse):
"""
List request for given DID to a destination RSE.
HTTP Success:
200 OK
HTTP Error:
'/accounts/(.+)/usage/', 'ALocalUsage', '/accounts/(.+)/usage/(.+)',
'ALocalUsage', '/accounts/(.+)', 'AAccountParameter', '/accounts/?$',
'AAccount'
]
URLS += [
'/accountlimits/local/(.+)/(.+)',
'ALLocalAccountLimit',
'/accountlimits/global/(.+)/(.+)',
'ALGlobalAccountLimit',
'/accountlimits/(.+)/(.+)',
'ALLocalAccountLimit',
]
URLS += [
'/archives%s/files' % get_schema_value('SCOPE_NAME_REGEXP'), 'AVArchive'
]
URLS += [
'/config/(.+)/(.+)/(.*)', 'COptionSet', '/config/(.+)/(.+)',
'COptionGetDel', '/config/(.+)', 'CSection', '/config', 'CConfig'
]
URLS += [
'/dids/(.*)/$',
'DScope',
'/dids/(.*)/guid',
'DGUIDLookup',
'/dids/(.*)/dids/search',
'DSearch',
'/dids%s/files' % get_schema_value('SCOPE_NAME_REGEXP'),
from rucio.common.replica_sorter import sort_random, sort_geoip, sort_closeness, sort_ranking, sort_dynamic, site_selector
from rucio.common.schema import get_schema_value
from rucio.web.rest.common import RucioController, check_accept_header_wrapper
from rucio.web.rest.utils import generate_http_error
try:
from urlparse import parse_qs
except ImportError:
from urllib.parse import parse_qs
LOGGER = getLogger("rucio.rucio")
SH = StreamHandler()
SH.setLevel(DEBUG)
LOGGER.addHandler(SH)
URLS = ('%s/metalink?$' % get_schema_value('SCOPE_NAME_REGEXP'),
'MetaLinkRedirector', '%s/?$' % get_schema_value('SCOPE_NAME_REGEXP'),
'HeaderRedirector')
class MetaLinkRedirector(RucioController):
@check_accept_header_wrapper(['application/metalink4+xml'])
def GET(self, scope, name):
"""
Metalink redirect
HTTP Success:
200 OK
HTTP Error:
401 Unauthorized
