def _checkPublishDataName(self, kwargs, outlfn, requestname, username):
"""
Validate the (user specified part of the) output dataset name for publication
using the WMCore.Lexicon method userprocdataset(), which does the same
validation as DBS (see discussion with Yuyi Guo in following github CRABClient
issue: https://github.com/dmwm/CRABClient/issues/4257).
This function will get deprecated in the 1509 release. We are keeping it around
for backward compatibility but I would like to remove it at some point
"""
if "publishname" not in kwargs:
## This now can be undefined since new clients use publishname2
msg = "Server parameter 'publishname' is not defined."
msg += " It is possible that old client is being used."
self.logger.info(msg)
return
## The client defines kwargs['publishname'] = <Data.outputDatasetTag>-<isbchecksum>
## if the user defines Data.outputDatasetTag, and kwargs['publishname'] = <isbchecksum> otherwise.
## (The PostJob replaces then the isbchecksum by the psethash.)
## Here we add the requestname if the user did not specify the publishname
if kwargs["publishname"].find("-") == -1:
outputDatasetTagToCheck = "%s-%s" % (requestname.replace(":", "_"), kwargs["publishname"])
else:
outputDatasetTagToCheck = "%s" % (kwargs["publishname"])
kwargs[
"publishname"
] = outputDatasetTagToCheck # that's what the version earlier than 1509 were putting in the DB
if "publishgroupname" in kwargs and int(
kwargs["publishgroupname"]
): # the first half of the if is for backward compatibility
if not (outlfn.startswith("/store/group/") and outlfn.split("/")[3]):
msg = "Parameter 'publishgroupname' is True,"
msg += " but parameter 'lfn' does not start with '/store/group/<groupname>'."
raise InvalidParameter(msg)
group_user_prefix = outlfn.split("/")[3]
else:
group_user_prefix = username
outputDatasetTagToCheck = "%s-%s" % (group_user_prefix, outputDatasetTagToCheck)
try:
userprocdataset(outputDatasetTagToCheck)
except AssertionError:
## The messages below are more descriptive than if we would use
## the message from AssertionError exception.
if kwargs["publishname"].find("-") == -1:
param = "General.requestName"
extrastr = "crab_"
else:
param = "Data.outputDatasetTag"
extrastr = ""
msg = "Invalid CRAB configuration parameter %s." % (param)
msg += " The combined string '%s-%s<%s>' should not have more than 166 characters" % (
group_user_prefix,
extrastr,
param,
)
msg += " and should match the regular expression %s" % (userProcDSParts["publishdataname"])
raise InvalidParameter(msg)
def _checkPublishDataName(self, kwargs, outlfn, requestname, username):
"""
Validate the (user specified part of the) output dataset name for publication
using the WMCore.Lexicon method userprocdataset(), which does the same
validation as DBS (see discussion with Yuyi Guo in following github CRABClient
issue: https://github.com/dmwm/CRABClient/issues/4257).
This function will get deprecated in the 1509 release. We are keeping it around
for backward compatibility but I would like to remove it at some point
"""
if 'publishname' not in kwargs:
## This now can be undefined since new clients use publishname2
msg = "Server parameter 'publishname' is not defined."
msg += " It is possible that old client is being used."
self.logger.info(msg)
return
## The client defines kwargs['publishname'] = <Data.outputDatasetTag>-<isbchecksum>
## if the user defines Data.outputDatasetTag, and kwargs['publishname'] = <isbchecksum> otherwise.
## (The PostJob replaces then the isbchecksum by the psethash.)
## Here we add the requestname if the user did not specify the publishname
if kwargs['publishname'].find('-') == -1:
outputDatasetTagToCheck = "%s-%s" % (requestname.replace(
':', '_'), kwargs['publishname'])
else:
outputDatasetTagToCheck = "%s" % (kwargs['publishname'])
kwargs[
'publishname'] = outputDatasetTagToCheck #that's what the version earlier than 1509 were putting in the DB
if 'publishgroupname' in kwargs and int(
kwargs['publishgroupname']
): #the first half of the if is for backward compatibility
if not (outlfn.startswith('/store/group/')
and outlfn.split('/')[3]):
msg = "Parameter 'publishgroupname' is True,"
msg += " but parameter 'lfn' does not start with '/store/group/<groupname>'."
raise InvalidParameter(msg)
group_user_prefix = outlfn.split('/')[3]
else:
group_user_prefix = username
outputDatasetTagToCheck = "%s-%s" % (group_user_prefix,
outputDatasetTagToCheck)
try:
userprocdataset(outputDatasetTagToCheck)
except AssertionError:
## The messages below are more descriptive than if we would use
## the message from AssertionError exception.
if kwargs['publishname'].find('-') == -1:
param = 'General.requestName'
extrastr = 'crab_'
else:
param = 'Data.outputDatasetTag'
extrastr = ''
msg = "Invalid CRAB configuration parameter %s." % (param)
msg += " The combined string '%s-%s<%s>' should not have more than 166 characters" % (
group_user_prefix, extrastr, param)
msg += " and should match the regular expression %s" % (
userProcDSParts['publishdataname'])
raise InvalidParameter(msg)
def _checkPublishDataName(self, kwargs, outlfn):
"""
Validate the (user specified part of the) output dataset name for publication
using the WMCore.Lexicon method userprocdataset(), which does the same
validation as DBS (see discussion with Yuyi Guo in following github CRABClient
issue: https://github.com/dmwm/CRABClient/issues/4257).
"""
## These if statements should actually never evaluate to True, because
## the client always defines kwargs['publishname'] and kwargs['workflow'].
if 'publishname' not in kwargs:
msg = "Server parameter 'publishname' is not defined."
msg += " Unable to validate the publication dataset name."
raise InvalidParameter(msg)
if kwargs['publishname'].find('-') == -1 and 'workflow' not in kwargs:
msg = "Server parameter 'workflow' is not defined."
msg += " Unable to validate the publication dataset name."
raise InvalidParameter(msg)
## The client defines kwargs['publishname'] = <Data.publishDataName>-<isbchecksum>
## if the user defines Data.publishDataName, and kwargs['publishname'] = <isbchecksum> otherwise.
## (The PostJob replaces then the isbchecksum by the psethash.)
## In DataWorkflow.py, the publish_name column of the TaskDB is defined
## in this way (without "username-"), and that's what is used later by
## PostJob to define the output dataset name.
if kwargs['publishname'].find('-') == -1:
publishDataNameToCheck = "%s-%s" % (kwargs['workflow'].replace(':','_'), kwargs['publishname'])
else:
publishDataNameToCheck = "%s" % (kwargs['publishname'])
if 'publishgroupname' in kwargs and int(kwargs['publishgroupname']): #the first half of the if is for backward compatibility
if not (outlfn.startswith('/store/group/') and outlfn.split('/')[3]):
msg = "Invalid CRAB configuration."
msg += " Parameter Data.publishWithGroupName is True,"
msg += " but Data.outLFNDirBase does not start with '/store/group/<groupname>'."
raise InvalidParameter(msg)
group_user_prefix = outlfn.split('/')[3]
else:
username = cherrypy.request.user['login']
group_user_prefix = username
publishDataNameToCheck = "%s-%s" % (group_user_prefix, publishDataNameToCheck)
try:
userprocdataset(publishDataNameToCheck)
except AssertionError:
## The messages below are more descriptive than if we would use
## the message from AssertionError exception.
if kwargs['publishname'].find('-') == -1:
param = 'General.requestName'
extrastr = 'crab_'
else:
param = 'Data.publishDataName'
extrastr = ''
msg = "Invalid CRAB configuration parameter %s." % (param)
msg += " The combined string '%s-%s<%s>' should not have more than 166 characters" % (group_user_prefix, extrastr, param)
msg += " and should match the regular expression %s" % (userProcDSParts['publishdataname'])
raise InvalidParameter(msg)
def _checkPublishDataName2(self, kwargs, outlfn, requestname, username):
"""
Validate the (user specified part of the) output dataset name for publication
using the WMCore.Lexicon method userprocdataset(), which does the same
validation as DBS (see discussion with Yuyi Guo in following github CRABClient
issue: https://github.com/dmwm/CRABClient/issues/4257).
"""
if "publishname2" not in kwargs or not kwargs["publishname2"]:
## provide the default publication name if it was not specified in the client
outputDatasetTagToCheck = requestname.replace(":", "_")
else:
outputDatasetTagToCheck = kwargs["publishname2"]
## Add the isbchecksum placeholder
outputDatasetTagToCheck += "-" + "0" * 32
# saves that in kwargs since it's what we want
kwargs["publishname2"] = outputDatasetTagToCheck
##Determine if it's a dataset that will go into a group space and therefore the (group)username prefix it will be used
if "publishgroupname" in kwargs and int(
kwargs["publishgroupname"]
): # the first half of the if is for backward compatibility
if not (outlfn.startswith("/store/group/") and outlfn.split("/")[3]):
msg = "Parameter 'publishgroupname' is True,"
msg += " but parameter 'lfn' does not start with '/store/group/<groupname>'."
raise InvalidParameter(msg)
group_user_prefix = outlfn.split("/")[3]
else:
group_user_prefix = username
outputDatasetTagToCheck = "%s-%s" % (group_user_prefix, outputDatasetTagToCheck)
try:
userprocdataset(outputDatasetTagToCheck)
except AssertionError:
## The messages below are more descriptive than if we would use
## the message from AssertionError exception.
if not kwargs["publishname2"]:
param = "General.requestName"
extrastr = "crab_"
else:
param = "Data.outputDatasetTag"
extrastr = ""
msg = "Invalid CRAB configuration parameter %s." % (param)
msg += " The combined string '%s-%s<%s>' should not have more than 166 characters" % (
group_user_prefix,
extrastr,
param,
)
msg += " and should match the regular expression %s" % (userProcDSParts["publishdataname"])
raise InvalidParameter(msg)
def _checkPublishDataName2(kwargs, outlfn, requestname, username):
"""
Validate the (user specified part of the) output dataset name for publication
using the WMCore.Lexicon method userprocdataset(), which does the same
validation as DBS (see discussion with Yuyi Guo in following github CRABClient
issue: https://github.com/dmwm/CRABClient/issues/4257).
"""
if 'publishname2' not in kwargs or not kwargs['publishname2']:
## provide the default publication name if it was not specified in the client
outputDatasetTagToCheck = requestname.replace(':', '_')
else:
outputDatasetTagToCheck = kwargs['publishname2']
## Add the isbchecksum placeholder
outputDatasetTagToCheck += "-" + "0" * 32
#saves that in kwargs since it's what we want
kwargs['publishname2'] = outputDatasetTagToCheck
##Determine if it's a dataset that will go into a group space and therefore the (group)username prefix it will be used
if 'publishgroupname' in kwargs and int(
kwargs['publishgroupname']
): #the first half of the if is for backward compatibility
if not (outlfn.startswith('/store/group/')
and outlfn.split('/')[3]):
msg = "Parameter 'publishgroupname' is True,"
msg += " but parameter 'lfn' does not start with '/store/group/<groupname>'."
raise InvalidParameter(msg)
group_user_prefix = outlfn.split('/')[3]
else:
group_user_prefix = username
outputDatasetTagToCheck = "%s-%s" % (group_user_prefix,
outputDatasetTagToCheck)
try:
userprocdataset(outputDatasetTagToCheck)
except AssertionError:
## The messages below are more descriptive than if we would use
## the message from AssertionError exception.
if not kwargs['publishname2']:
param = 'General.requestName'
extrastr = 'crab_'
else:
param = 'Data.outputDatasetTag'
extrastr = ''
msg = "Invalid CRAB configuration parameter %s." % (param)
msg += " The combined string '%s-%s<%s>' should not have more than 166 characters" % (
group_user_prefix, extrastr, param)
msg += " and should match the regular expression %s" % (
userProcDSParts['publishdataname'])
raise InvalidParameter(msg)
def _checkPublishDataName(self, kwargs):
"""
Validate the (user specified part of the) output dataset name for publication
using the WMCore.Lexicon method userprocdataset(), which does the same
validation as DBS (see discussion with Yuyi Guo in following github CRABClient
issue: https://github.com/dmwm/CRABClient/issues/4257).
"""
## These if statements should actually never evaluate to True, because
## the client always defines kwargs['publishname'] and kwargs['workflow'].
if 'publishname' not in kwargs:
msg = "Server parameter 'publishname' is not defined."
msg += " Unable to validate the publication dataset name."
raise InvalidParameter(msg)
if kwargs['publishname'].find('-') == -1 and 'workflow' not in kwargs:
msg = "Server parameter 'workflow' is not defined."
msg += " Unable to validate the publication dataset name."
raise InvalidParameter(msg)
## The client defines kwargs['publishname'] = <Data.publishDataName>-<isbchecksum>
## if the user defines Data.publishDataName, and kwargs['publishname'] = <isbchecksum> otherwise.
## (The PostJob replaces then the isbchecksum by the psethash.)
## In DataWorkflow.py, the publish_name column of the TaskDB is defined
## in this way (without "username-"), and that's what is used later by
## PostJob to define the output dataset name.
if kwargs['publishname'].find('-') == -1:
publishDataNameToCheck = "username-%s-%s" % (kwargs['workflow'].replace(':','_'), kwargs['publishname'])
else:
publishDataNameToCheck = "username-%s" % (kwargs['publishname'])
try:
userprocdataset(publishDataNameToCheck)
except AssertionError:
## The messages below are more descriptive than if we would use
## the message from AssertionError exception.
if kwargs['publishname'].find('-') == -1:
msg = "Invalid CRAB configuration parameter General.requestName."
msg += " The parameter should not have more than 152 characters"
msg += " and should match the regular expression %s" % (userProcDSParts['publishdataname'])
else:
msg = "Invalid CRAB configuration parameter Data.publishDataName."
msg += " The parameter should not have more than 157 characters"
msg += " and should match the regular expression %s" % (userProcDSParts['publishdataname'])
raise InvalidParameter(msg)
