def validate(self, apiobj, method, api, param, safe):
"""Validating all the input parameter as enforced by the WMCore.REST module"""
authz_login_valid()
if method in ['PUT']:
validate_str("campaign", param, safe, RX_CAMPAIGN, optional=False)
validate_strlist("workflow", param, safe, RX_TASKNAME)
elif method in ['POST']:
validate_str("campaign", param, safe, RX_CAMPAIGN, optional=False)
elif method in ['GET']:
validate_str("campaign", param, safe, RX_CAMPAIGN, optional=True)
validate_num('age', param, safe, optional=True)
validate_str('subresource',
param,
safe,
RX_SUBRESTAT,
optional=True)
validate_num('limit', param, safe, optional=True)
if not safe.kwargs['campaign'] and not safe.kwargs['age']:
raise InvalidParameter("Invalid input parameters")
if not safe.kwargs['campaign'] and safe.kwargs['subresource']:
raise InvalidParameter("Invalid input parameters")
elif method in ['DELETE']:
validate_str("campaign", param, safe, RX_CAMPAIGN, optional=False)
validate_num("force", param, safe, optional=True)
def getGroupedPublicationQuery(self, grouping, username, taskname,
asoworker):
"""Get query and bind parameters for query"""
binds = {}
if grouping == 0:
# Return grouped status of publication for each asoworker
return self.transferDB.GetGroupedPublicationStatistics0_sql, binds
elif grouping == 1:
# Return grouped status of publication for user
if not username:
raise InvalidParameter('Username is not defined')
binds['username'] = username
if asoworker:
binds['asoworker'] = asoworker
return self.transferDB.GetGroupedPublicationStatistics1_sql, binds
else:
return self.transferDB.GetGroupedPublicationStatistics1a_sql, binds
elif grouping == 2:
# Return grouped status of publication for user and taskname
if not username or not taskname:
raise InvalidParameter('Required variable is not defined')
binds['username'] = username
binds['taskname'] = taskname
if asoworker:
binds['asoworker'] = asoworker
return self.transferDB.GetGroupedPublicationStatistics2_sql, binds
else:
return self.transferDB.GetGroupedPublicationStatistics2a_sql, binds
def _checkOutLFN(self, kwargs):
"""Check the lfn parameter: it must start with '/store/user/username/' or '/store/group/groupname[/subgroupname]*/username/',
where username is the one registered in SiteDB (i.e. the one used in the CERN primary account).
If lfn is not there, default to '/store/user/username/'.
Handle old clients lfnprefix parameter.
"""
## Is this the username registered in SiteDB?
username = cherrypy.request.user['login']
if kwargs['lfnprefix']:
kwargs['lfn'] = '/store/user/%s/%s' % (username,
kwargs["lfnprefix"])
elif not kwargs['lfn']:
## Default to '/store/user/username/' in case the user did not specify the lfn parameter.
kwargs['lfn'] = '/store/user/%s/' % (username)
else:
msg = "The parameter Data.outLFN in the CRAB configuration file must start with either"
msg += " '/store/user/<username>/' or '/store/group/<groupname>[/<subgroupname>]*/<username>/'"
msg += " (or '/store/local/' if publication is off),"
msg += " where username is your username as registered in SiteDB"
msg += " (i.e. the username of your CERN primary account)."
if kwargs['lfn'].startswith('/store/group/'):
if len(kwargs['lfn'].split('/')) < 5 or kwargs['lfn'].split(
'/'
)[3] == username or username not in kwargs['lfn'].split('/'):
raise InvalidParameter(msg)
elif kwargs['lfn'].startswith('/store/user/'):
if len(kwargs['lfn'].split('/')) < 4 or kwargs['lfn'].split(
'/')[3] != username:
raise InvalidParameter(msg)
elif kwargs['lfn'].startswith('/store/local/'):
if kwargs['publication']:
raise InvalidParameter(msg)
else:
raise InvalidParameter(msg)
def post(self, workflow, status, subresource, jobset, failure, resubmittedjobs, getstatus, workername, limit, runs, lumis):
""" Updates task information """
if failure is not None:
try:
failure = b64decode(failure)
except TypeError:
raise InvalidParameter("Failure message is not in the accepted format")
methodmap = {"state": {"args": (SetStatusTask.sql,), "method": self.api.modify, "kwargs": {"status": [status],
"taskname": [workflow]}},
"start": {"args": (SetReadyTasks.sql,), "method": self.api.modify, "kwargs": {"tm_task_status": [status],
"tm_taskname": [workflow]}},
"failure": {"args": (SetFailedTasks.sql,), "method": self.api.modify, "kwargs": {"tm_task_status": [status],
"failure": [failure],
"tm_taskname": [workflow]}},
"success": {"args": (SetInjectedTasks.sql,), "method": self.api.modify, "kwargs": {"tm_task_status": [status],
"panda_jobset_id": [jobset],
"tm_taskname": [workflow],
"resubmitted_jobs": [str(resubmittedjobs)]}},
"process": {"args": (UpdateWorker.sql,), "method": self.api.modifynocheck, "kwargs": {"tw_name": [workername],
"get_status": [getstatus],
"limit": [limit],
"set_status": [status]}},
"lumimask": {"args": (runs, lumis,), "method": self.setLumiMask, "kwargs": {"taskname": [workflow],}},
}
if subresource is None:
subresource = 'state'
if not subresource in methodmap.keys():
raise InvalidParameter("Subresource of workflowdb has not been found")
methodmap[subresource]['method'](*methodmap[subresource]['args'], **methodmap[subresource]['kwargs'])
return []
def wrapped_func(*args, **kwargs):
logger = logging.getLogger("CRABLogger.Utils")
myproxyserver = "myproxy.cern.ch"
userdn = kwargs['userdn']
defaultDelegation = {'logger': logger,
'proxyValidity' : '192:00',
'min_time_left' : 36000,
'server_key': serverKey,
'server_cert': serverCert,}
timeleftthreshold = 60 * 60 * 24
mypclient = SimpleMyProxy(defaultDelegation)
userproxy = None
userhash = sha1(kwargs['userdn']).hexdigest()
if serverDN:
try:
userproxy = mypclient.logonRenewMyProxy(username=userhash, myproxyserver=myproxyserver, myproxyport=7512)
except MyProxyException, me:
# Unsure if this works in standalone mode...
cherrypy.log(str(me))
cherrypy.log(str(serverKey))
cherrypy.log(str(serverCert))
invalidp = InvalidParameter("Impossible to retrieve proxy from %s for %s and hash %s" %
(myproxyserver, kwargs['userdn'], userhash))
setattr(invalidp, 'trace', str(me))
raise invalidp
else:
if not re.match(RX_CERT, userproxy):
raise InvalidParameter("Retrieved malformed proxy from %s for %s and hash %s" %
(myproxyserver, kwargs['userdn'], userhash))
def deletewarnings(self, **kwargs):
""" Deleet warnings from the warning column in the database. Can be tested with:
curl -X POST https://mmascher-poc.cern.ch/crabserver/dev/task -k --key /tmp/x509up_u8440 --cert /tmp/x509up_u8440 \
-d 'subresource=deletewarnings&workflow=140710_233424_crab3test-5:mmascher_crab_HCprivate12' -v
"""
#check if the parameter is there
if 'workflow' not in kwargs or not kwargs['workflow']:
raise InvalidParameter("Task name not found in the input parameters")
#decoding and setting the parameters
workflow = kwargs['workflow']
authz_owner_match(self.api, [workflow], self.Task) #check that I am modifying my own workflow
# rows = self.api.query(None, None, "SELECT tm_task_warnings FROM tasks WHERE tm_taskname = :workflow", workflow=workflow)#self.Task.TASKSUMMARY_sql)
rows = self.api.query(None, None, self.Task.ID_sql, taskname=workflow)#self.Task.TASKSUMMARY_sql)
rows = list(rows) #from generator to list
if len(rows)==0:
raise InvalidParameter("Task %s not found in the task database" % workflow)
row = self.Task.ID_tuple(*rows[0])
warnings = literal_eval(row.task_warnings.read() if row.task_warnings else '[]')
if len(warnings)<1:
raise ExecutionError("No warnings to remove.")
self.api.modify(self.Task.DeleteWarnings_sql, workflow=[workflow])
return []
def addwarning(self, **kwargs):
""" Add a warning to the wraning column in the database. Can be tested with:
curl -X POST https://mmascher-poc.cern.ch/crabserver/dev/task -k --key /tmp/x509up_u8440 --cert /tmp/x509up_u8440 \
-d 'subresource=addwarning&workflow=140710_233424_crab3test-5:mmascher_crab_HCprivate12&warning=blahblah' -v
"""
#check if the parameters are there
if 'warning' not in kwargs or not kwargs['warning']:
raise InvalidParameter("Warning message not found in the input parameters")
if 'workflow' not in kwargs or not kwargs['workflow']:
raise InvalidParameter("Task name not found in the input parameters")
#decoding and setting the parameters
workflow = kwargs['workflow']
authz_owner_match(self.api, [workflow], self.Task) #check that I am modifying my own workflow
try:
warning = b64decode(kwargs['warning'])
except TypeError:
raise InvalidParameter("Failure message is not in the accepted format")
# rows = self.api.query(None, None, "SELECT tm_task_warnings FROM tasks WHERE tm_taskname = :workflow", workflow=workflow)#self.Task.TASKSUMMARY_sql)
rows = self.api.query(None, None, self.Task.ID_sql, taskname=workflow)#self.Task.TASKSUMMARY_sql)
rows = list(rows) #from generator to list
if len(rows)==0:
raise InvalidParameter("Task %s not found in the task database" % workflow)
row = self.Task.ID_tuple(*rows[0])
warnings = literal_eval(row.task_warnings.read() if row.task_warnings else '[]')
if len(warnings)>10:
raise ExecutionError("You cannot add more than 10 warnings to a task")
warnings.append(warning)
self.api.modify(self.Task.SetWarnings_sql, warnings=[str(warnings)], workflow=[workflow])
return []
def addoutputdatasets(self, **kwargs):
if 'outputdatasets' not in kwargs or not kwargs['outputdatasets']:
raise InvalidParameter(
"Output datasets not found in the input parameters")
if 'workflow' not in kwargs or not kwargs['workflow']:
raise InvalidParameter(
"Task name not found in the input parameters")
workflow = kwargs['workflow']
authz_owner_match(
self.api, [workflow],
self.Task) #check that I am modifying my own workflow
row = self.Task.ID_tuple(*self.api.query(
None, None, self.Task.ID_sql, taskname=workflow).next())
outputdatasets = literal_eval(
row.output_dataset.read() if row.output_dataset else '[]')
outputdatasets = str(
list(set(outputdatasets + kwargs['outputdatasets'])))
self.api.modify(self.Task.SetUpdateOutDataset_sql,
tm_output_dataset=[outputdatasets],
tm_taskname=[workflow])
return []
def _checkPublishDataName(self, kwargs, outlfn, requestname, username):
"""
Validate the (user specified part of the) output dataset name for publication
using the WMCore.Lexicon method userprocdataset(), which does the same
validation as DBS (see discussion with Yuyi Guo in following github CRABClient
issue: https://github.com/dmwm/CRABClient/issues/4257).
This function will get deprecated in the 1509 release. We are keeping it around
for backward compatibility but I would like to remove it at some point
"""
if 'publishname' not in kwargs:
## This now can be undefined since new clients use publishname2
msg = "Server parameter 'publishname' is not defined."
msg += " It is possible that old client is being used."
self.logger.info(msg)
return
## The client defines kwargs['publishname'] = <Data.outputDatasetTag>-<isbchecksum>
## if the user defines Data.outputDatasetTag, and kwargs['publishname'] = <isbchecksum> otherwise.
## (The PostJob replaces then the isbchecksum by the psethash.)
## Here we add the requestname if the user did not specify the publishname
if kwargs['publishname'].find('-') == -1:
outputDatasetTagToCheck = "%s-%s" % (requestname.replace(
':', '_'), kwargs['publishname'])
else:
outputDatasetTagToCheck = "%s" % (kwargs['publishname'])
kwargs[
'publishname'] = outputDatasetTagToCheck #that's what the version earlier than 1509 were putting in the DB
if 'publishgroupname' in kwargs and int(
kwargs['publishgroupname']
): #the first half of the if is for backward compatibility
if not (outlfn.startswith('/store/group/')
and outlfn.split('/')[3]):
msg = "Parameter 'publishgroupname' is True,"
msg += " but parameter 'lfn' does not start with '/store/group/<groupname>'."
raise InvalidParameter(msg)
group_user_prefix = outlfn.split('/')[3]
else:
group_user_prefix = username
outputDatasetTagToCheck = "%s-%s" % (group_user_prefix,
outputDatasetTagToCheck)
try:
userprocdataset(outputDatasetTagToCheck)
except AssertionError:
## The messages below are more descriptive than if we would use
## the message from AssertionError exception.
if kwargs['publishname'].find('-') == -1:
param = 'General.requestName'
extrastr = 'crab_'
else:
param = 'Data.outputDatasetTag'
extrastr = ''
msg = "Invalid CRAB configuration parameter %s." % (param)
msg += " The combined string '%s-%s<%s>' should not have more than 166 characters" % (
group_user_prefix, extrastr, param)
msg += " and should match the regular expression %s" % (
userProcDSParts['publishdataname'])
raise InvalidParameter(msg)
def _check_ip(argname, val):
if not isinstance(val, str) or not RXIP.match(val):
raise InvalidParameter("Incorrect '%s' parameter" % argname)
try:
val = IPAddress(val)
except:
raise InvalidParameter("Incorrect '%s' parameter" % argname)
return val
def get(self, subresource, id, username, taskname):
""" Retrieve all columns for a specified task or
"""
binds = {}
if subresource == 'getById':
###############################################
# getById API
# ---------------------------------------------
# Description:
# Users & Also PostJob can query this API to get info about specific document
# in database.
# ---------------------------------------------
# Always required variables:
# id: id
###############################################
binds['id'] = id
return self.api.query(None, None, self.transferDB.GetById_sql,
**binds)
if not username:
raise InvalidParameter('Username is not defined')
if not taskname:
raise InvalidParameter('TaskName is not defined')
binds['username'] = username
binds['taskname'] = taskname
print(binds)
if subresource == 'getTransferStatus':
###############################################
# getTransferStatus API
# ---------------------------------------------
# Description:
# Get Transfer status for specific task which belongs to user
# ---------------------------------------------
# Always required variables:
# username: username
# taskname: taskname
###############################################
return self.api.query(
None, None, self.transferDB.GetTaskStatusForTransfers_sql,
**binds)
elif subresource == 'getPublicationStatus':
###############################################
# getPublicationStatus API
# ---------------------------------------------
# Description:
# Get Transfer status for specific task which belongs to user
# ---------------------------------------------
# Always required variables:
# username: username
# taskname: taskname
###############################################
return self.api.query(
None, None, self.transferDB.GetTaskStatusForPublication_sql,
**binds)
return {}
def validate(self, apiobj, method, api, param, safe):
if not param.args:
raise InvalidParameter("Path required")
for name in param.args:
if not RX_PATH.match(name):
raise InvalidParameter("Invalid path")
item = self._prefix + tuple(param.args)
if not self.api.scraper.exists(item, lambda v: bool(v.data)):
raise MissingObject("No such image")
safe.kwargs["item"] = item
param.args[:] = []
def _checkPublishDataName2(kwargs, outlfn, requestname, username):
"""
Validate the (user specified part of the) output dataset name for publication
using the WMCore.Lexicon method userprocdataset(), which does the same
validation as DBS (see discussion with Yuyi Guo in following github CRABClient
issue: https://github.com/dmwm/CRABClient/issues/4257).
"""
if 'publishname2' not in kwargs or not kwargs['publishname2']:
## provide the default publication name if it was not specified in the client
outputDatasetTagToCheck = requestname.replace(':', '_')
else:
outputDatasetTagToCheck = kwargs['publishname2']
## Add the isbchecksum placeholder
outputDatasetTagToCheck += "-" + "0" * 32
#saves that in kwargs since it's what we want
kwargs['publishname2'] = outputDatasetTagToCheck
##Determine if it's a dataset that will go into a group space and therefore the (group)username prefix it will be used
if 'publishgroupname' in kwargs and int(
kwargs['publishgroupname']
): #the first half of the if is for backward compatibility
if not (outlfn.startswith('/store/group/')
and outlfn.split('/')[3]):
msg = "Parameter 'publishgroupname' is True,"
msg += " but parameter 'lfn' does not start with '/store/group/<groupname>'."
raise InvalidParameter(msg)
group_user_prefix = outlfn.split('/')[3]
else:
group_user_prefix = username
outputDatasetTagToCheck = "%s-%s" % (group_user_prefix,
outputDatasetTagToCheck)
try:
userprocdataset(outputDatasetTagToCheck)
except AssertionError:
## The messages below are more descriptive than if we would use
## the message from AssertionError exception.
if not kwargs['publishname2']:
param = 'General.requestName'
extrastr = 'crab_'
else:
param = 'Data.outputDatasetTag'
extrastr = ''
msg = "Invalid CRAB configuration parameter %s." % (param)
msg += " The combined string '%s-%s<%s>' should not have more than 166 characters" % (
group_user_prefix, extrastr, param)
msg += " and should match the regular expression %s" % (
userProcDSParts['publishdataname'])
raise InvalidParameter(msg)
def _checkReleases(self, jobarch, jobsw):
""" Check if the software needed by the user is available in the tag collector
Uses allScramArchsAndVersions from WMCore. If an IOError is raised report an error message.
If the list of releases is empty (reason may be an ExpatError) then report an error message
If the asked released is not there then report an error message
"""
msg = False
goodReleases = {}
try:
goodReleases = self.tagCollector.releases_by_architecture()
except: #pylint: disable=bare-except
msg = "Error connecting to %s (params: %s) and determining the list of available releases. " % \
(self.tagCollector['endpoint'], self.tagCollector.tcArgs) + "Skipping the check of the releases"
else:
if goodReleases == {}:
msg = "The list of releases at %s (params: %s) is empty. " % \
(self.tagCollector['endpoint'], self.tagCollector.tcArgs) + "Skipping the check of the releases"
elif jobarch not in goodReleases or jobsw not in goodReleases[jobarch]:
msg = "ERROR: %s on %s is not among supported releases" % (jobsw, jobarch)
msg += "\nUse config.JobType.allowUndistributedCMSSW = True if you are sure of what you are doing"
excasync = "ERROR: %s on %s is not among supported releases or an error occurred" % (jobsw, jobarch)
invalidp = InvalidParameter(msg, errobj=excasync)
setattr(invalidp, 'trace', '')
raise invalidp
if msg:
#Need to log the message in the db for the users
self.logger.warning(msg)
def wrapped_func(*args, **kwargs):
logger = logging.getLogger("CRABLogger.Utils")
myproxyserver = "myproxy.cern.ch"
userdn = kwargs['userdn']
defaultDelegation = {
'logger': logger,
'proxyValidity': '192:00',
'min_time_left': 36000,
'server_key': serverKey,
'server_cert': serverCert,
}
timeleftthreshold = 60 * 60 * 24
mypclient = SimpleMyProxy(defaultDelegation)
userproxy = None
try:
userproxy = mypclient.logonRenewMyProxy(
username=sha1(kwargs['userdn']).hexdigest(),
myproxyserver=myproxyserver,
myproxyport=7512)
except MyProxyException, me:
import cherrypy
cherrypy.log(str(me))
cherrypy.log(str(serverKey))
cherrypy.log(str(serverCert))
invalidp = InvalidParameter(
"Impossible to retrieve proxy from %s for %s." %
(myproxyserver, kwargs['userdn']))
setattr(invalidp, 'trace', str(me))
raise invalidp
def validate(self, apiobj, method, api, param, safe):
if not param.args:
raise InvalidParameter("Missing tile")
tile = param.args.pop(0)
m = RX_TILE.match(tile)
if not param.args and m:
safe.kwargs["tile"] = m.groups()
def _checkSite(self, site, pnn = False):
sites = self.allPNNNames.sites if pnn else self.allCMSNames.sites
if site not in sites:
excasync = ValueError("A site name you specified is not valid")
invalidp = InvalidParameter("The parameter %s is not in the list of known CMS sites %s" % (site, sites), errobj = excasync)
setattr(invalidp, 'trace', '')
raise invalidp
def _expandSites(self, sites, pnn=False):
"""Check if there are sites cotaining the '*' wildcard and convert them in the corresponding list
Raise exception if any wildcard site does expand to an empty list
"""
res = set()
for site in sites:
if '*' in site:
sitere = re.compile(site.replace('*', '.*'))
expanded = [
str(s) for s in
(self.allPNNNames.sites if pnn else self.allCMSNames.sites)
if sitere.match(s)
]
self.logger.debug("Site %s expanded to %s during validate",
site, expanded)
if not expanded:
excasync = ValueError("Remote output data site not valid")
invalidp = InvalidParameter(
"Cannot expand site %s to anything" % site,
errobj=excasync)
setattr(invalidp, 'trace', '')
raise invalidp
res = res.union(expanded)
else:
self._checkSite(site, pnn)
res.add(site)
return list(res)
def search(self, **kwargs):
"""Retrieves all the columns of a task in the task table (select * from task ...)
The API is (only?) used in the monitor for operator.
curl -X GET 'https://mmascher-dev6.cern.ch/crabserver/dev/task?subresource=search&workflow=150224_230633:mmascher_crab_testecmmascher-dev6_3' \
-k --key /tmp/x509up_u8440 --cert /tmp/x509up_u8440 -v"""
if 'workflow' not in kwargs or not kwargs['workflow']:
raise InvalidParameter(
"Task name not found in the input parameters")
try:
row = next(
self.api.query(None,
None,
self.Task.QuickSearch_sql,
taskname=kwargs["workflow"]))
except StopIteration:
raise ExecutionError(
"Impossible to find task %s in the database." %
kwargs["workflow"])
def getval(col):
""" Some columns in oracle can be CLOB and we need to call read on them.
"""
#TODO move the function in ServerUtils and use it when required (e.g.: mysql LONGTEXT does not need read())
try:
return str(col)
except:
return col.read()
return [getval(col) for col in row]
def _check_tarfile(argname, val, hashkey, newchecksum):
"""Check that `argname` `val` is a tar file and that provided 'hashkey`
matches with the hashkey calculated on the `val`.
:arg str argname: name of the argument
:arg file val: the file object
:arg str hashkey: the sha256 hexdigest of the file, calculated over the tuple
(name, size, mtime, uname) of all the tarball members
:return: the val if the validation passes."""
# checking that is a valid file or an input string
# note: the input string is generated on client side just when the input file is empty
_check_file(argname, val)
digest = None
try:
#This newchecksum param and the if/else branch is there for backward compatibility.
#The parameter, older exclusion and checksum functions should be removed in the future.
if newchecksum == 2:
digest = calculateChecksum(val.file,
exclude=NEW_USER_SANDBOX_EXCLUSIONS)
elif newchecksum == 1:
digest = calculateChecksum(val.file,
exclude=USER_SANDBOX_EXCLUSIONS)
else:
tar = tarfile.open(fileobj=val.file, mode='r')
lsl = [(x.name, int(x.size), int(x.mtime), x.uname)
for x in tar.getmembers()]
hasher = hashlib.sha256(str(lsl))
digest = hasher.hexdigest()
except tarfile.ReadError:
raise InvalidParameter('File is not a .tgz file.')
if not digest or hashkey != digest:
raise ChecksumFailed("Checksums do not match")
return val
def _checkReleases(self, jobarch, jobsw):
""" Check if the software needed by the user is available in the tag collector
Uses allScramArchsAndVersions from WMCore. If an IOError is raised report an error message.
If the list of releases is empty (reason may be an ExpatError) then report an error message
If the asked released is not there then report an error message
"""
msg = False
goodReleases = {}
try:
goodReleases = allScramArchsAndVersions()
except IOError:
msg = "Error connecting to %s and determine the list of available releases. " % TAG_COLLECTOR_URL +\
"Skipping the check of the releases"
else:
if goodReleases == {}:
msg = "The list of releases at %s is empty. " % TAG_COLLECTOR_URL +\
"Skipping the check of the releases"
elif jobarch not in goodReleases or jobsw not in goodReleases[
jobarch]:
msg = "ERROR: %s on %s is not among supported releases" % (
jobsw, jobarch)
msg += "\nUse config.JobType.allowNonProductionCMSSW = True if you are sure of what you are doing"
excasync = "ERROR: %s on %s is not among supported releases or an error occurred" % (
jobsw, jobarch)
invalidp = InvalidParameter(msg, errobj=excasync)
setattr(invalidp, 'trace', '')
raise invalidp
if msg:
#Need to log the message in the db for the users
self.logger.warning(msg)
def updateschedd(self, **kwargs):
""" Change scheduler for task submission.
curl -X POST https://balcas-crab.cern.ch/crabserver/dev/task -ks --key $X509_USER_PROXY --cert $X509_USER_PROXY --cacert $X509_USER_PROXY \
-d 'subresource=updateschedd&workflow=150316_221646:jbalcas_crab_test_submit-5-274334&scheddname=vocms095.asdadasdasdacern.ch' -v
"""
if 'scheddname' not in kwargs or not kwargs['scheddname']:
raise InvalidParameter("Schedd name not found in the input parameters")
if 'workflow' not in kwargs or not kwargs['workflow']:
raise InvalidParameter("Task name not found in the input parameters")
workflow = kwargs['workflow']
authz_owner_match(self.api, [workflow], self.Task) #check that I am modifying my own workflow
self.api.modify(self.Task.UpdateSchedd_sql, scheddname=[kwargs['scheddname']], workflow=[workflow])
return []
def addddmreqid(self, **kwargs):
""" Add DDM request ID to DDM_reqid column in the database. Can be tested with:
curl -X POST https://balcas-crab.cern.ch/crabserver/dev/task -k --key $X509_USER_PROXY --cert $X509_USER_PROXY \
-d 'subresource=addddmreqid&workflow=?&taskstatus=TAPERECALL&ddmreqid=d2b715f526e14f91b0c299abb560d5d7' -v
"""
#check if the parameters are there
if 'ddmreqid' not in kwargs or not kwargs['ddmreqid']:
raise InvalidParameter("DDM request ID not found in the input parameters")
if 'workflow' not in kwargs or not kwargs['workflow']:
raise InvalidParameter("Task name not found in the input parameters")
workflow = kwargs['workflow']
authz_owner_match(self.api, [workflow], self.Task) #check that I am modifying my own workflow
self.api.modify(self.Task.UpdateDDMReqId_sql, taskstatus=[kwargs['taskstatus']], ddmreqid=[kwargs['ddmreqid']], workflow=[workflow])
return []
def addwebdir(self, **kwargs):
""" Add web directory to web_dir column in the database. Can be tested with:
curl -X POST https://balcas-crab.cern.ch/crabserver/dev/task -k --key $X509_USER_PROXY --cert $X509_USER_PROXY \
-d 'subresource=addwebdir&workflow=140710_233424_crab3test-5:mmascher_crab_HCprivate12&webdirurl=http://cmsweb.cern.ch/crabserver/testtask' -v
"""
#check if the parameters are there
if 'webdirurl' not in kwargs or not kwargs['webdirurl']:
raise InvalidParameter("Web directory url not found in the input parameters")
if 'workflow' not in kwargs or not kwargs['workflow']:
raise InvalidParameter("Task name not found in the input parameters")
workflow = kwargs['workflow']
authz_owner_match(self.api, [workflow], self.Task) #check that I am modifying my own workflow
self.api.modify(self.Task.UpdateWebUrl_sql, webdirurl=[kwargs['webdirurl']], workflow=[workflow])
return []
def webdir(self, **kwargs):
if 'workflow' not in kwargs or not kwargs['workflow']:
raise InvalidParameter(
"Task name not found in the input parameters")
workflow = kwargs['workflow']
row = self.Task.ID_tuple(*self.api.query(
None, None, self.Task.ID_sql, taskname=workflow).next())
yield row.user_webdir
def getpublishurl(self, **kwargs):
if 'workflow' not in kwargs or not kwargs['workflow']:
raise InvalidParameter("Task name not found in the input parameters")
try:
row = next(self.api.query(None, None, self.Task.GetPublishUrl_sql, taskname=kwargs['workflow']))
except StopIteration:
raise ExecutionError("Impossible to find task %s in the database." % kwargs['workflow'])
yield row
def validate(self, apiobj, method, api, param, safe):
DataServiceEntity.validate(self, apiobj, method, api, param, safe)
if not param.args or not isinstance(param.args[0], basestring):
raise InvalidParameter("Missing date argument")
try:
day = param.args.pop(0)
if day == "today":
safe.kwargs["mode"] = "H"
safe.kwargs["day"] = int(time.time() / 86400) * 86400
elif day[0] in ("H", "D") and day[1:].isdigit():
safe.kwargs["mode"] = day[0]
safe.kwargs["day"] = calendar.timegm(time.strptime(day[1:], "%Y%m%d"))
else:
raise InvalidParameter("Invalid date parameter")
except ValueError, e:
raise InvalidParameter("Invalid date parameter", errobj = e,
trace = traceback.format_exc())
def _checkSite(self, site):
if site not in self.allCMSNames.sites:
excasync = ValueError("Remote output data site not valid")
invalidp = InvalidParameter(
"The parameter %s is not in the list of known CMS sites %s" %
(site, self.allCMSNames.sites),
errobj=excasync)
setattr(invalidp, 'trace', '')
raise invalidp
def webdir(self, **kwargs):
if 'workflow' not in kwargs or not kwargs['workflow']:
raise InvalidParameter("Task name not found in the input parameters")
workflow = kwargs['workflow']
try:
row = self.Task.ID_tuple(*next(self.api.query(None, None, self.Task.ID_sql, taskname=workflow)))
except StopIteration:
raise ExecutionError("Impossible to find task %s in the database." % kwargs["workflow"])
yield row.user_webdir
def _checkASODestination(self, site):
self._checkSite(site)
if site in self.centralcfg.centralconfig.get('banned-out-destinations',
[]):
excasync = ValueError("Remote output data site is banned")
invalidp = InvalidParameter("The output site you specified in the config.Site.storageSite parameter (%s) is blacklisted (banned sites: %s)" %\
(site, self.centralcfg.centralconfig['banned-out-destinations']), errobj = excasync)
setattr(invalidp, 'trace', '')
raise invalidp
