def validateSiteLists(arguments):
whiteList = arguments.get("SiteWhitelist", [])
blackList = arguments.get("SiteBlacklist", [])
whiteList = makeList(whiteList)
blackList = makeList(blackList)
res = (set(whiteList) & set(blackList))
if len(res):
msg = "Validation failed: The same site cannot be white and blacklisted: %s" % list(res)
raise WMSpecFactoryException(msg)
# store the properly formatted values (list instead of string)
arguments["SiteWhitelist"] = whiteList
arguments["SiteBlacklist"] = blackList
return
def validateSiteLists(arguments):
whiteList = arguments.get("SiteWhitelist", [])
blackList = arguments.get("SiteBlacklist", [])
whiteList = makeList(whiteList)
blackList = makeList(blackList)
res = (set(whiteList) & set(blackList))
if len(res):
msg = "Validation failed: The same site cannot be white and blacklisted: %s" % list(res)
raise WMSpecFactoryException(msg)
# store the properly formatted values (list instead of string)
arguments["SiteWhitelist"] = whiteList
arguments["SiteBlacklist"] = blackList
return
def testMakeNonEmptyList(self):
"""
Test the makeNonEmptyList function.
It has exactly the same behaviour as makeList, but throws an exception
for empty list or string.
"""
self.assertEqual(makeList(['123']), makeNonEmptyList(['123']))
self.assertItemsEqual(makeList(['123', 456, '789']), makeNonEmptyList(['123', 456, '789']))
self.assertItemsEqual(makeList(u'123,456'), makeNonEmptyList(u'123, 456'))
self.assertItemsEqual(makeList('["aa","bb","cc"]'), makeNonEmptyList('["aa", "bb", "cc"]'))
self.assertRaises(ValueError, makeNonEmptyList, [])
self.assertRaises(ValueError, makeNonEmptyList, "")
def testMakeNonEmptyList(self):
"""
Test the makeNonEmptyList function.
It has exactly the same behaviour as makeList, but throws an exception
for empty list or string.
"""
self.assertEqual(makeList(['123']), makeNonEmptyList(['123']))
self.assertListEqual(makeList(['123', 456, '789']), makeNonEmptyList(['123', 456, '789']))
self.assertListEqual(makeList(u'123,456'), makeNonEmptyList(u'123, 456'))
self.assertListEqual(makeList('["aa","bb","cc"]'), makeNonEmptyList('["aa", "bb", "cc"]'))
self.assertRaises(ValueError, makeNonEmptyList, [])
self.assertRaises(ValueError, makeNonEmptyList, "")
def testMakeList(self):
"""
Test the makeList function
"""
self.assertEqual(makeList([]), [])
self.assertEqual(makeList(""), [])
self.assertEqual(makeList(['123']), ['123'])
self.assertEqual(makeList([456]), [456])
self.assertItemsEqual(makeList(['123', 456, '789']),
['123', 456, '789'])
self.assertEqual(makeList('123'), ['123'])
self.assertEqual(makeList(u'123'), [u'123'])
self.assertItemsEqual(makeList('123,456'), ['123', '456'])
self.assertItemsEqual(makeList(u'123,456'), [u'123', u'456'])
self.assertItemsEqual(makeList('["aa","bb","cc"]'), ['aa', 'bb', 'cc'])
self.assertItemsEqual(makeList(u' ["aa", "bb", "cc"] '),
['aa', 'bb', 'cc'])
self.assertRaises(ValueError, makeList, 123)
self.assertRaises(ValueError, makeList, 123.456)
self.assertRaises(ValueError, makeList, {1: 123})
def testMakeList(self):
"""
Test the makeList function
"""
self.assertEqual(makeList([]), [])
self.assertEqual(makeList(""), [])
self.assertEqual(makeList(['123']), ['123'])
self.assertEqual(makeList([456]), [456])
self.assertItemsEqual(makeList(['123', 456, '789']), ['123', 456, '789'])
self.assertEqual(makeList('123'), ['123'])
self.assertEqual(makeList(u'123'), [u'123'])
self.assertItemsEqual(makeList('123,456'), ['123', '456'])
self.assertItemsEqual(makeList(u'123,456'), [u'123', u'456'])
self.assertItemsEqual(makeList('["aa","bb","cc"]'), ['aa', 'bb', 'cc'])
self.assertItemsEqual(makeList(u' ["aa", "bb", "cc"] '), ['aa', 'bb', 'cc'])
self.assertRaises(ValueError, makeList, 123)
self.assertRaises(ValueError, makeList, 123.456)
self.assertRaises(ValueError, makeList, {1: 123})
def post(self, **kwargs):
"""This is used by ASO/CRAB3 Operator to acquire Transfers/Publication or update/retry/kill them.
Find a description next to each call"""
subresource = kwargs['subresource']
binds = {}
timeNow = int(time.time())
binds['last_update'] = [timeNow]
binds['asoworker'] = [kwargs['asoworker']]
if subresource == 'acquireTransfers':
###############################################
# acquireTransfers API
# ---------------------------------------------
# Description:
# ASO calls this view and CRABServer will mark all docs, which do not
# have asoworker assigned from this transfer_state to new_transfer_state
# ---------------------------------------------
# Always required variables:
# (str) asoworker: ASO Worker name for which acquire Transfers.
###############################################
binds['limit'] = [1000]
binds['transfer_state'] = [TRANSFERDB_STATUSES['NEW']]
binds['new_transfer_state'] = [TRANSFERDB_STATUSES['ACQUIRED']]
binds['username'] = [kwargs['username']]
return self.api.modifynocheck(self.transferDB.AcquireTransfers_sql, **binds)
elif subresource == 'acquirePublication':
###############################################
# acquirePublication API
# ---------------------------------------------
# Description:
# ASO calls this view and CRABServer will mark all docs, which file transfer
# status is done, will update asoworker to new asoworker status. also will
# look for this specific publication state and will change it to new_publication_state
# :transfer_done AND publish = :publish_new
# ---------------------------------------------
# Always required variables:
# (str) asoworker: ASO Worker name for which acquire Publication. It will select entries in
# transfersdb which have this asoworker AND all those where asoworker is null and in those
# set it to the value passed in arg
###############################################
# TODO: Maybe we want to acquire publication per ASO machine
# by meaning if we have 2 ASO concurrent running, each of them can acquire multiple files
# from same task. Maybe is good to enforce acquire only for tasks which do not have
# asoworker defined. This would enforce to group files per and lower a bit load on dbs.
binds['publish_flag'] = [1]
binds['transfer_state'] = [TRANSFERDB_STATUSES['DONE']]
binds['publication_state'] = [PUBLICATIONDB_STATUSES['NEW']]
binds['new_publication_state'] = [PUBLICATIONDB_STATUSES['ACQUIRED']]
return self.api.modifynocheck(self.transferDB.AcquirePublication_sql, **binds)
elif subresource == 'updateTransfers':
###############################################
# updateTransfers API
# ---------------------------------------------
# Description:
# ASO calls this view and CRABServer will select entries in filetransferdb via their tm_id
# and will make all posted modification.
# If there are passed N (2 or 4) items, it will do corresponding action in update
# :::::::
# 2 items, Required keys are:
# ([str, str]) list_of_ids: list of ids which are in database.
# ([str, str]) list_of_transfer_state: transfer_state which is one of: ['FAILED', 'DONE', 'RETRY', 'SUBMITTED', 'KILLED']
# optional: ([str, str]) list_of_failure_reason
# :::::::
# 4 items, Required keys are:
# ([str, str]) list_of_ids: list of ids which are in database.
# ([str, str]) list_of_transfer_state: transfer_state which is one of: ['FAILED', 'DONE', 'RETRY', 'SUBMITTED', 'KILLED']
# ([str, str]) list_of_fts_instance: list of fts instance to which transfer was submitted.
# ([str, str]) list_of_fts_id: list of fts id for this specific document transfer
# ---------------------------------------------
# Always required variables:
# (str) asoworker: ASO Worker name to be set for transfers which were acquired
###############################################
binds['last_update'] = [timeNow]
# TODO: fix case: if 'fts_instance' in kwargs
if kwargs['list_of_fts_instance']:
#del errorMsg
ids = makeList(kwargs['list_of_ids'])
states = makeList(kwargs['list_of_transfer_state'])
retry = [0 for x in states]
reasons = ["" for x in states]
instances = makeList(kwargs['list_of_fts_instance'])
fts_id = makeList(kwargs['list_of_fts_id'])
for num in range(len(ids)):
binds['id'] = [ids[num]]
binds['transfer_state'] = [TRANSFERDB_STATUSES[states[num]]]
binds['fts_instance'] = [str(instances[num])]
binds['fts_id'] = [str(fts_id[num])]
binds['fail_reason'] = [reasons[num]]
binds['retry_value'] = [int(retry[num])]
self.api.modifynocheck(self.transferDB.UpdateTransfers_sql, **binds)
else:
ids = makeList(kwargs['list_of_ids'])
states = makeList(kwargs['list_of_transfer_state'])
retry = [0 for x in states]
reasons = ["" for x in states]
if kwargs['list_of_retry_value'] is not None:
reasons = makeList(kwargs['list_of_failure_reason'])
retry = makeList(kwargs['list_of_retry_value'])
for num in range(len(ids)):
binds['id'] = [ids[num]]
binds['transfer_state'] = [TRANSFERDB_STATUSES[states[num]]]
binds['fts_instance'] = [None]
binds['fts_id'] = [None]
binds['fail_reason'] = [reasons[num]]
binds['retry_value'] = [int(retry[num])]
self.api.modifynocheck(self.transferDB.UpdateTransfers_sql, **binds)
elif subresource == 'updatePublication':
###############################################
# updatePublication API
# ---------------------------------------------
# Description:
# ASO calls this view and CRABServer will mark all docs, which have asoworker
# in row and will make publication status for all list of ids new status
# list of ids are keys which need to be updated.
# 2 items are required in list_of_ids. Keys:
# (str) id: Document id which is in database.
# (str) publication_state: publication_state which is one of: ['FAILED', 'DONE', 'RETRY']
# ---------------------------------------------
# Always required variables:
# (str) asoworker: ASO Worker name for which acquire Publication.
###############################################
binds['last_update'] = [timeNow]
#compareOut, errorMsg = self.compareLen(kwargs, ['list_of_ids', 'list_of_publication_state'])
#if compareOut:
# del errorMsg
ids = makeList(kwargs['list_of_ids'])
states = makeList(kwargs['list_of_publication_state'])
retry = [0 for x in states]
reasons = ["" for x in states]
if kwargs['list_of_retry_value'] is not None:
reasons = makeList(kwargs['list_of_failure_reason'])
retry = makeList(kwargs['list_of_retry_value'])
for num in range(len(ids)):
binds['publication_state'] = [PUBLICATIONDB_STATUSES[states[num]]]
binds['id'] = [ids[num]]
binds['fail_reason'] = [reasons[num]]
binds['retry_value'] = [int(retry[num])]
binds['publish'] = [kwargs["publish_flag"] or -1]
self.api.modify(self.transferDB.UpdatePublication_sql, **binds)
elif subresource == 'retryPublication':
###############################################
# retryPublication API
# ---------------------------------------------
# Description:
# ASO calls this view and CRABServer will mark all docs with publication_state NEW, which have asoworker
# assigned and it did not reached max retries count and last_update is lower
# than specified time_to.
# ---------------------------------------------
# Always required variables:
# (str) asoworker: ASO Worker name for which acquire Publication.
# (str) time_to: timespan until which time
###############################################
if 'time_to' not in kwargs:
raise InvalidParameter()
binds['time_to'] = kwargs['time_to']
binds['new_publication_state'] = PUBLICATIONDB_STATUSES['NEW']
binds['publication_state'] = PUBLICATIONDB_STATUSES['RETRY']
self.api.modify(self.transferDB.RetryPublication_sql, **binds)
elif subresource == 'retryTransfers':
###############################################
# retryTransfers API
# ---------------------------------------------
# Description:
# ASO calls this view and CRABServer will mark all docs with transfer_state NEW, which have asoworker
# assigned and it did not reached max retries count and last_update is lower
# than specified time_to.
# ---------------------------------------------
# Always required variables:
# (str) asoworker: ASO Worker name for which retry transfers.
# (str) time_to: timespan until which time
###############################################
if 'time_to' not in kwargs:
raise InvalidParameter()
binds['time_to'] = [kwargs['time_to']]
binds['new_transfer_state'] = [TRANSFERDB_STATUSES['NEW']]
binds['transfer_state'] = [TRANSFERDB_STATUSES['RETRY']]
return self.api.modifynocheck(self.transferDB.RetryTransfers_sql, **binds)
elif subresource == 'killTransfers':
###############################################
# killTransfers API
# ---------------------------------------------
# Description:
# ASO calls this view and CRABServer will mark all docs with transfer_state KILL for specific list_of_ids
# which have asoworker assigned and their TRANSFER_STATE is NEW
# than specified time_to.
# list_of_ids is a list which contains string of ids
# ---------------------------------------------
# Always required variables:
# (str) asoworker: ASO Worker name for which acquire Publication.
###############################################
for oneId in kwargs['list_of_ids']:
binds['id'] = [oneId]
binds['transfer_state'] = [TRANSFERDB_STATUSES['NEW']]
binds['new_transfer_state'] = [TRANSFERDB_STATUSES['KILL']]
self.api.modify(self.transferDB.KillTransfers_sql, **binds)