def validateSiteLists(arguments): whiteList = arguments.get("SiteWhitelist", []) blackList = arguments.get("SiteBlacklist", []) whiteList = makeList(whiteList) blackList = makeList(blackList) res = (set(whiteList) & set(blackList)) if len(res): msg = "Validation failed: The same site cannot be white and blacklisted: %s" % list(res) raise WMSpecFactoryException(msg) # store the properly formatted values (list instead of string) arguments["SiteWhitelist"] = whiteList arguments["SiteBlacklist"] = blackList return
def validateSiteLists(arguments): whiteList = arguments.get("SiteWhitelist", []) blackList = arguments.get("SiteBlacklist", []) whiteList = makeList(whiteList) blackList = makeList(blackList) res = (set(whiteList) & set(blackList)) if len(res): msg = "Validation failed: The same site cannot be white and blacklisted: %s" % list(res) raise WMSpecFactoryException(msg) # store the properly formatted values (list instead of string) arguments["SiteWhitelist"] = whiteList arguments["SiteBlacklist"] = blackList return
def testMakeNonEmptyList(self): """ Test the makeNonEmptyList function. It has exactly the same behaviour as makeList, but throws an exception for empty list or string. """ self.assertEqual(makeList(['123']), makeNonEmptyList(['123'])) self.assertItemsEqual(makeList(['123', 456, '789']), makeNonEmptyList(['123', 456, '789'])) self.assertItemsEqual(makeList(u'123,456'), makeNonEmptyList(u'123, 456')) self.assertItemsEqual(makeList('["aa","bb","cc"]'), makeNonEmptyList('["aa", "bb", "cc"]')) self.assertRaises(ValueError, makeNonEmptyList, []) self.assertRaises(ValueError, makeNonEmptyList, "")
def testMakeNonEmptyList(self): """ Test the makeNonEmptyList function. It has exactly the same behaviour as makeList, but throws an exception for empty list or string. """ self.assertEqual(makeList(['123']), makeNonEmptyList(['123'])) self.assertListEqual(makeList(['123', 456, '789']), makeNonEmptyList(['123', 456, '789'])) self.assertListEqual(makeList(u'123,456'), makeNonEmptyList(u'123, 456')) self.assertListEqual(makeList('["aa","bb","cc"]'), makeNonEmptyList('["aa", "bb", "cc"]')) self.assertRaises(ValueError, makeNonEmptyList, []) self.assertRaises(ValueError, makeNonEmptyList, "")
def testMakeList(self): """ Test the makeList function """ self.assertEqual(makeList([]), []) self.assertEqual(makeList(""), []) self.assertEqual(makeList(['123']), ['123']) self.assertEqual(makeList([456]), [456]) self.assertItemsEqual(makeList(['123', 456, '789']), ['123', 456, '789']) self.assertEqual(makeList('123'), ['123']) self.assertEqual(makeList(u'123'), [u'123']) self.assertItemsEqual(makeList('123,456'), ['123', '456']) self.assertItemsEqual(makeList(u'123,456'), [u'123', u'456']) self.assertItemsEqual(makeList('["aa","bb","cc"]'), ['aa', 'bb', 'cc']) self.assertItemsEqual(makeList(u' ["aa", "bb", "cc"] '), ['aa', 'bb', 'cc']) self.assertRaises(ValueError, makeList, 123) self.assertRaises(ValueError, makeList, 123.456) self.assertRaises(ValueError, makeList, {1: 123})
def testMakeList(self): """ Test the makeList function """ self.assertEqual(makeList([]), []) self.assertEqual(makeList(""), []) self.assertEqual(makeList(['123']), ['123']) self.assertEqual(makeList([456]), [456]) self.assertItemsEqual(makeList(['123', 456, '789']), ['123', 456, '789']) self.assertEqual(makeList('123'), ['123']) self.assertEqual(makeList(u'123'), [u'123']) self.assertItemsEqual(makeList('123,456'), ['123', '456']) self.assertItemsEqual(makeList(u'123,456'), [u'123', u'456']) self.assertItemsEqual(makeList('["aa","bb","cc"]'), ['aa', 'bb', 'cc']) self.assertItemsEqual(makeList(u' ["aa", "bb", "cc"] '), ['aa', 'bb', 'cc']) self.assertRaises(ValueError, makeList, 123) self.assertRaises(ValueError, makeList, 123.456) self.assertRaises(ValueError, makeList, {1: 123})
def post(self, **kwargs): """This is used by ASO/CRAB3 Operator to acquire Transfers/Publication or update/retry/kill them. Find a description next to each call""" subresource = kwargs['subresource'] binds = {} timeNow = int(time.time()) binds['last_update'] = [timeNow] binds['asoworker'] = [kwargs['asoworker']] if subresource == 'acquireTransfers': ############################################### # acquireTransfers API # --------------------------------------------- # Description: # ASO calls this view and CRABServer will mark all docs, which do not # have asoworker assigned from this transfer_state to new_transfer_state # --------------------------------------------- # Always required variables: # (str) asoworker: ASO Worker name for which acquire Transfers. ############################################### binds['limit'] = [1000] binds['transfer_state'] = [TRANSFERDB_STATUSES['NEW']] binds['new_transfer_state'] = [TRANSFERDB_STATUSES['ACQUIRED']] binds['username'] = [kwargs['username']] return self.api.modifynocheck(self.transferDB.AcquireTransfers_sql, **binds) elif subresource == 'acquirePublication': ############################################### # acquirePublication API # --------------------------------------------- # Description: # ASO calls this view and CRABServer will mark all docs, which file transfer # status is done, will update asoworker to new asoworker status. also will # look for this specific publication state and will change it to new_publication_state # :transfer_done AND publish = :publish_new # --------------------------------------------- # Always required variables: # (str) asoworker: ASO Worker name for which acquire Publication. It will select entries in # transfersdb which have this asoworker AND all those where asoworker is null and in those # set it to the value passed in arg ############################################### # TODO: Maybe we want to acquire publication per ASO machine # by meaning if we have 2 ASO concurrent running, each of them can acquire multiple files # from same task. Maybe is good to enforce acquire only for tasks which do not have # asoworker defined. This would enforce to group files per and lower a bit load on dbs. binds['publish_flag'] = [1] binds['transfer_state'] = [TRANSFERDB_STATUSES['DONE']] binds['publication_state'] = [PUBLICATIONDB_STATUSES['NEW']] binds['new_publication_state'] = [PUBLICATIONDB_STATUSES['ACQUIRED']] return self.api.modifynocheck(self.transferDB.AcquirePublication_sql, **binds) elif subresource == 'updateTransfers': ############################################### # updateTransfers API # --------------------------------------------- # Description: # ASO calls this view and CRABServer will select entries in filetransferdb via their tm_id # and will make all posted modification. # If there are passed N (2 or 4) items, it will do corresponding action in update # ::::::: # 2 items, Required keys are: # ([str, str]) list_of_ids: list of ids which are in database. # ([str, str]) list_of_transfer_state: transfer_state which is one of: ['FAILED', 'DONE', 'RETRY', 'SUBMITTED', 'KILLED'] # optional: ([str, str]) list_of_failure_reason # ::::::: # 4 items, Required keys are: # ([str, str]) list_of_ids: list of ids which are in database. # ([str, str]) list_of_transfer_state: transfer_state which is one of: ['FAILED', 'DONE', 'RETRY', 'SUBMITTED', 'KILLED'] # ([str, str]) list_of_fts_instance: list of fts instance to which transfer was submitted. # ([str, str]) list_of_fts_id: list of fts id for this specific document transfer # --------------------------------------------- # Always required variables: # (str) asoworker: ASO Worker name to be set for transfers which were acquired ############################################### binds['last_update'] = [timeNow] # TODO: fix case: if 'fts_instance' in kwargs if kwargs['list_of_fts_instance']: #del errorMsg ids = makeList(kwargs['list_of_ids']) states = makeList(kwargs['list_of_transfer_state']) retry = [0 for x in states] reasons = ["" for x in states] instances = makeList(kwargs['list_of_fts_instance']) fts_id = makeList(kwargs['list_of_fts_id']) for num in range(len(ids)): binds['id'] = [ids[num]] binds['transfer_state'] = [TRANSFERDB_STATUSES[states[num]]] binds['fts_instance'] = [str(instances[num])] binds['fts_id'] = [str(fts_id[num])] binds['fail_reason'] = [reasons[num]] binds['retry_value'] = [int(retry[num])] self.api.modifynocheck(self.transferDB.UpdateTransfers_sql, **binds) else: ids = makeList(kwargs['list_of_ids']) states = makeList(kwargs['list_of_transfer_state']) retry = [0 for x in states] reasons = ["" for x in states] if kwargs['list_of_retry_value'] is not None: reasons = makeList(kwargs['list_of_failure_reason']) retry = makeList(kwargs['list_of_retry_value']) for num in range(len(ids)): binds['id'] = [ids[num]] binds['transfer_state'] = [TRANSFERDB_STATUSES[states[num]]] binds['fts_instance'] = [None] binds['fts_id'] = [None] binds['fail_reason'] = [reasons[num]] binds['retry_value'] = [int(retry[num])] self.api.modifynocheck(self.transferDB.UpdateTransfers_sql, **binds) elif subresource == 'updatePublication': ############################################### # updatePublication API # --------------------------------------------- # Description: # ASO calls this view and CRABServer will mark all docs, which have asoworker # in row and will make publication status for all list of ids new status # list of ids are keys which need to be updated. # 2 items are required in list_of_ids. Keys: # (str) id: Document id which is in database. # (str) publication_state: publication_state which is one of: ['FAILED', 'DONE', 'RETRY'] # --------------------------------------------- # Always required variables: # (str) asoworker: ASO Worker name for which acquire Publication. ############################################### binds['last_update'] = [timeNow] #compareOut, errorMsg = self.compareLen(kwargs, ['list_of_ids', 'list_of_publication_state']) #if compareOut: # del errorMsg ids = makeList(kwargs['list_of_ids']) states = makeList(kwargs['list_of_publication_state']) retry = [0 for x in states] reasons = ["" for x in states] if kwargs['list_of_retry_value'] is not None: reasons = makeList(kwargs['list_of_failure_reason']) retry = makeList(kwargs['list_of_retry_value']) for num in range(len(ids)): binds['publication_state'] = [PUBLICATIONDB_STATUSES[states[num]]] binds['id'] = [ids[num]] binds['fail_reason'] = [reasons[num]] binds['retry_value'] = [int(retry[num])] binds['publish'] = [kwargs["publish_flag"] or -1] self.api.modify(self.transferDB.UpdatePublication_sql, **binds) elif subresource == 'retryPublication': ############################################### # retryPublication API # --------------------------------------------- # Description: # ASO calls this view and CRABServer will mark all docs with publication_state NEW, which have asoworker # assigned and it did not reached max retries count and last_update is lower # than specified time_to. # --------------------------------------------- # Always required variables: # (str) asoworker: ASO Worker name for which acquire Publication. # (str) time_to: timespan until which time ############################################### if 'time_to' not in kwargs: raise InvalidParameter() binds['time_to'] = kwargs['time_to'] binds['new_publication_state'] = PUBLICATIONDB_STATUSES['NEW'] binds['publication_state'] = PUBLICATIONDB_STATUSES['RETRY'] self.api.modify(self.transferDB.RetryPublication_sql, **binds) elif subresource == 'retryTransfers': ############################################### # retryTransfers API # --------------------------------------------- # Description: # ASO calls this view and CRABServer will mark all docs with transfer_state NEW, which have asoworker # assigned and it did not reached max retries count and last_update is lower # than specified time_to. # --------------------------------------------- # Always required variables: # (str) asoworker: ASO Worker name for which retry transfers. # (str) time_to: timespan until which time ############################################### if 'time_to' not in kwargs: raise InvalidParameter() binds['time_to'] = [kwargs['time_to']] binds['new_transfer_state'] = [TRANSFERDB_STATUSES['NEW']] binds['transfer_state'] = [TRANSFERDB_STATUSES['RETRY']] return self.api.modifynocheck(self.transferDB.RetryTransfers_sql, **binds) elif subresource == 'killTransfers': ############################################### # killTransfers API # --------------------------------------------- # Description: # ASO calls this view and CRABServer will mark all docs with transfer_state KILL for specific list_of_ids # which have asoworker assigned and their TRANSFER_STATE is NEW # than specified time_to. # list_of_ids is a list which contains string of ids # --------------------------------------------- # Always required variables: # (str) asoworker: ASO Worker name for which acquire Publication. ############################################### for oneId in kwargs['list_of_ids']: binds['id'] = [oneId] binds['transfer_state'] = [TRANSFERDB_STATUSES['NEW']] binds['new_transfer_state'] = [TRANSFERDB_STATUSES['KILL']] self.api.modify(self.transferDB.KillTransfers_sql, **binds)