def getDataFromURL(url, proxyfilename = None):
"""
Read the content of a URL and return it as a string.
Type of content should not matter, it can be a json file or a tarball for example.
url: the link you would like to retrieve
proxyfilename: the x509 proxy certificate to be used in case auth is required
Returns binary data encoded as a string, which can be later processed
according to what kind of content it represents.
"""
# Get rid of unicode which may cause problems in pycurl
stringUrl = url.encode('ascii')
reqHandler = RequestHandler()
_, data = reqHandler.request(url=stringUrl, params={}, ckey=proxyfilename,
cert=proxyfilename,
capath=HTTPRequests.getCACertPath())
return data
def getDataFromURL(url, proxyfilename = None):
"""
Read the content of a URL and return it as a string.
Type of content should not matter, it can be a json file or a tarball for example.
url: the link you would like to retrieve
proxyfilename: the x509 proxy certificate to be used in case auth is required
Returns binary data encoded as a string, which can be later processed
according to what kind of content it represents.
"""
# Get rid of unicode which may cause problems in pycurl
stringUrl = url.encode('ascii')
reqHandler = RequestHandler()
_, data = reqHandler.request(url=stringUrl, params={}, ckey=proxyfilename,
cert=proxyfilename,
capath=HTTPRequests.getCACertPath())
return data
def __call__(self):
valid = False
configmsg = 'Default'
self.logger.debug("Started submission")
# Get some debug parameters
oneEventMode = hasattr(self.configuration, 'Debug') and \
getattr(self.configuration.Debug, 'oneEventMode')
######### Check if the user provided unexpected parameters ########
#init the dictionary with all the known parameters
SpellChecker.DICTIONARY = SpellChecker.train( [ val['config'] for _, val in self.requestmapper.iteritems() if val['config'] ] + \
[ x for x in self.otherConfigParams ] )
#iterate on the parameters provided by the user
for section in self.configuration.listSections_():
for attr in getattr(self.configuration, section).listSections_():
par = (section + '.' + attr)
#if the parameter is not know exit, but try to correct it before
if not SpellChecker.is_correct( par ):
msg = 'The parameter %s is not known.' % par
msg += '' if SpellChecker.correct(par) == par else ' Did you mean %s?' % SpellChecker.correct(par)
raise ConfigurationException(msg)
#usertarball and cmsswconfig use this parameter and we should set it up in a correct way
self.configuration.General.serverUrl = self.serverurl
uniquerequestname = None
self.logger.debug("Working on %s" % str(self.requestarea))
configreq = {}
for param in self.requestmapper:
mustbetype = getattr(types, self.requestmapper[param]['type'])
if self.requestmapper[param]['config']:
attrs = self.requestmapper[param]['config'].split('.')
temp = self.configuration
for attr in attrs:
temp = getattr(temp, attr, None)
if temp is None:
break
if temp is not None:
if mustbetype == type(temp):
configreq[param] = temp
else:
raise ConfigurationException("Invalid type " + str(type(temp)) + " for parameter " + self.requestmapper[param]['config'] \
+ ". It is needed a " + str(mustbetype) + ".")
elif self.requestmapper[param]['default'] is not None:
configreq[param] = self.requestmapper[param]['default']
temp = self.requestmapper[param]['default']
elif self.requestmapper[param]['required']:
raise ConfigurationException("Missing parameter " + self.requestmapper[param]['config'] + " from the configuration.")
else:
## parameter not strictly required
pass
if param == "workflow":
if mustbetype == type(self.requestname):
configreq["workflow"] = self.requestname
elif param in ['savelogsflag','publication','nonprodsw','ignorelocality','saveoutput']:#TODO use clientmappig to do this
configreq[param] = 1 if temp else 0
elif param in ['dbsurl','publishdbsurl']:
if param == 'dbsurl':
dbstype = 'reader'
elif param == 'publishdbsurl':
dbstype = 'writer'
alloweddbsurls = DBSURLS[dbstype].values()
alloweddbsurlsaliases = DBSURLS[dbstype].keys()
if configreq[param] in alloweddbsurlsaliases:
configreq[param] = DBSURLS[dbstype][configreq[param]]
else:
if configreq[param].rstrip('/') in alloweddbsurls:
configreq[param] = configreq[param].rstrip('/')
else:
raise ConfigurationException("Invalid argument " + configreq[param] + " for parameter " + self.requestmapper[param]['config'] + " in the configuration.")
if (configreq['saveoutput'] or configreq['savelogsflag']) and 'asyncdest' not in configreq:
raise ConfigurationException("Missing parameter " + self.requestmapper['asyncdest']['config'] + " from the configuration.")
# Add debug parameters to the configreq dict
configreq['oneEventMode'] = int(oneEventMode)
jobconfig = {}
self.configuration.JobType.proxyfilename = self.proxyfilename
self.configuration.JobType.capath = HTTPRequests.getCACertPath()
#get the backend URLs from the server external configuration
serverBackendURLs = server_info('backendurls', self.serverurl, self.proxyfilename, self.getUrl(self.instance, resource='info'))
#if cacheSSL is specified in the server external configuration we will use it to upload the sandbox (baseURL will be ignored)
self.configuration.JobType.filecacheurl = serverBackendURLs['cacheSSL'] if 'cacheSSL' in serverBackendURLs else None
pluginParams = [ self.configuration, self.logger, os.path.join(self.requestarea, 'inputs') ]
if getattr(self.configuration.JobType, 'pluginName', None) is not None:
jobtypes = getJobTypes()
plugjobtype = jobtypes[upper(self.configuration.JobType.pluginName)](*pluginParams)
inputfiles, jobconfig, isbchecksum = plugjobtype.run(configreq)
else:
fullname = self.configuration.JobType.externalPluginFile
basename = os.path.basename(fullname).split('.')[0]
plugin = addPlugin(fullname)[basename]
pluginInst = plugin(*pluginParams)
inputfiles, jobconfig, isbchecksum = pluginInst.run(configreq)
if not configreq['publishname']:
configreq['publishname'] = isbchecksum
else:
configreq['publishname'] = "%s-%s" %(configreq['publishname'], isbchecksum)
configreq.update(jobconfig)
server = HTTPRequests(self.serverurl, self.proxyfilename, self.proxyfilename, version=__version__)
self.logger.info("Sending the request to the server")
self.logger.debug("Submitting %s " % str( configreq ) )
dictresult, status, reason = server.put( self.uri, data = self._encodeRequest(configreq) )
self.logger.debug("Result: %s" % dictresult)
if status != 200:
msg = "Problem sending the request:\ninput:%s\noutput:%s\nreason:%s" % (str(configreq), str(dictresult), str(reason))
raise RESTCommunicationException(msg)
elif dictresult.has_key("result"):
uniquerequestname = dictresult["result"][0]["RequestName"]
else:
msg = "Problem during submission, no request ID returned:\ninput:%s\noutput:%s\nreason:%s" \
% (str(configreq), str(dictresult), str(reason))
raise RESTCommunicationException(msg)
tmpsplit = self.serverurl.split(':')
createCache(self.requestarea, tmpsplit[0], tmpsplit[1] if len(tmpsplit)>1 else '', uniquerequestname,
voRole=self.voRole, voGroup=self.voGroup, instance=self.instance,
originalConfig = self.configuration)
self.logger.info("%sSuccess%s: Your task has been delivered to the CRAB3 server." %(colors.GREEN, colors.NORMAL))
if not self.options.wait:
self.logger.info("Please use 'crab status' to check how the submission process proceed")
self.logger.debug("Request ID: %s " % uniquerequestname)
if self.options.wait:
self.checkStatusLoop(server,uniquerequestname)
return uniquerequestname
def __call__(self):
valid = False
configmsg = 'Default'
if not os.path.isfile(self.options.config):
raise MissingOptionException("Configuration file '%s' not found" % self.options.config)
self.logger.debug("Started submission")
######### Check if the user provided unexpected parameters ########
#init the dictionary with all the known parameters
SpellChecker.DICTIONARY = SpellChecker.train( [ val['config'] for _, val in self.requestmapper.iteritems() if val['config'] ] + \
[ x for x in self.otherConfigParams ] )
#iterate on the parameters provided by the user
for section in self.configuration.listSections_():
for attr in getattr(self.configuration, section).listSections_():
par = (section + '.' + attr)
#if the parameter is not know exit, but try to correct it before
if not SpellChecker.is_correct( par ):
msg = 'The parameter %s is not known.' % par
msg += '' if SpellChecker.correct(par) == par else ' Did you mean %s?' % SpellChecker.correct(par)
raise ConfigurationException(msg)
#usertarball and cmsswconfig use this parameter and we should set it up in a correct way
self.configuration.General.serverUrl = self.serverurl
uniquerequestname = None
self.logger.debug("Working on %s" % str(self.requestarea))
configreq = {}
for param in self.requestmapper:
mustbetype = getattr(types, self.requestmapper[param]['type'])
if self.requestmapper[param]['config']:
attrs = self.requestmapper[param]['config'].split('.')
temp = self.configuration
for attr in attrs:
temp = getattr(temp, attr, None)
if temp is None:
break
if temp:
if mustbetype == type(temp):
configreq[param] = temp
else:
raise ConfigurationException("Invalid type " + str(type(temp)) + " for parameter " + self.requestmapper[param]['config'] \
+ ". It is needed a " + str(mustbetype) + ".")
elif self.requestmapper[param]['default'] is not None:
configreq[param] = self.requestmapper[param]['default']
elif self.requestmapper[param]['required']:
raise ConfigurationException("Missing parameter " + self.requestmapper[param]['config'] + " from the configuration.")
else:
## parameter not strictly required
pass
if param == "workflow":
if mustbetype == type(self.requestname):
configreq["workflow"] = self.requestname
elif param == "savelogsflag":
configreq["savelogsflag"] = 1 if temp else 0
elif param == "publication":
configreq["publication"] = 1 if temp else 0
elif param == "blacklistT1":
blacklistT1 = self.voRole != 't1access'
#if the user choose to remove the automatic T1 blacklisting and has not the t1acces role
if getattr (self.configuration.Site, 'removeT1Blacklisting', False) and blacklistT1:
self.logger.info("WARNING: You disabled the T1 automatic blacklisting without having the t1access role")
blacklistT1 = False
configreq["blacklistT1"] = 1 if blacklistT1 else 0
jobconfig = {}
self.configuration.JobType.proxyfilename = self.proxyfilename
self.configuration.JobType.capath = HTTPRequests.getCACertPath()
#get the backend URLs from the server external configuration
serverBackendURLs = server_info('backendurls', self.serverurl, self.proxyfilename, self.getUrl(self.instance, resource='info'))
#if cacheSSL is specified in the server external configuration we will use it to upload the sandbox (baseURL will be ignored)
self.configuration.JobType.filecacheurl = serverBackendURLs['cacheSSL'] if 'cacheSSL' in serverBackendURLs else None
#otherwise we will contact the baseurl to get the cache hostname
self.configuration.JobType.baseurl = serverBackendURLs['baseURL']
pluginParams = [ self.configuration, self.logger, os.path.join(self.requestarea, 'inputs') ]
if getattr(self.configuration.JobType, 'pluginName', None) is not None:
jobtypes = getJobTypes()
plugjobtype = jobtypes[upper(self.configuration.JobType.pluginName)](*pluginParams)
inputfiles, jobconfig, isbchecksum = plugjobtype.run(configreq)
else:
fullname = self.configuration.JobType.externalPluginFile
basename = os.path.basename(fullname).split('.')[0]
plugin = addPlugin(fullname)[basename]
pluginInst = plugin(*pluginParams)
inputfiles, jobconfig, isbchecksum = pluginInst.run(configreq)
if not configreq['publishname']:
configreq['publishname'] = isbchecksum
else:
configreq['publishname'] = "%s-%s" %(configreq['publishname'], isbchecksum)
configreq.update(jobconfig)
server = HTTPRequests(self.serverurl, self.proxyfilename, self.proxyfilename, version=__version__)
self.logger.info("Sending the request to the server")
self.logger.debug("Submitting %s " % str( configreq ) )
dictresult, status, reason = server.put( self.uri, data = self._encodeRequest(configreq) )
self.logger.debug("Result: %s" % dictresult)
if status != 200:
msg = "Problem sending the request:\ninput:%s\noutput:%s\nreason:%s" % (str(configreq), str(dictresult), str(reason))
raise RESTCommunicationException(msg)
elif dictresult.has_key("result"):
uniquerequestname = dictresult["result"][0]["RequestName"]
else:
msg = "Problem during submission, no request ID returned:\ninput:%s\noutput:%s\nreason:%s" \
% (str(configreq), str(dictresult), str(reason))
raise RESTCommunicationException(msg)
tmpsplit = self.serverurl.split(':')
createCache(self.requestarea, tmpsplit[0], tmpsplit[1] if len(tmpsplit)>1 else '', uniquerequestname,
voRole=self.voRole, voGroup=self.voGroup, instance=self.instance)
self.logger.info("Submission completed")
self.logger.debug("Request ID: %s " % uniquerequestname)
self.logger.debug("Ended submission")
return uniquerequestname
