def preInitialization(self):
if not hasattr(self.config.PilotMonitorComponent, "pilotMonitorHandler"):
self.config.PilotMonitorComponent.pilotMonitorHandler = \
'PilotMonitor.Handler.PilotMonitorHandler'
factory = WMFactory("PilotMonitor");
self.messages["MonitorPilots"] = factory.loadObject(\
self.config.PilotMonitorComponent.pilotMonitorHandler, self)
tqconfig = Configuration.loadConfigurationFile( \
self.config.PilotMonitorComponent.TQConfig )
myThread = threading.currentThread()
self.logger = myThread.logger
self.logger.debug( tqconfig)
self.tqStateApi = TQStateApi(self.logger, tqconfig, None)
def bootstrap():
print("Entering TaskManagerBootstrap with args: %s" % sys.argv)
command = sys.argv[1]
if command == "POSTJOB":
return PostJob.PostJob().execute(*sys.argv[2:])
elif command == "PREJOB":
return PreJob.PreJob().execute(*sys.argv[2:])
elif command == "PREDAG":
return PreDAG.PreDAG().execute(*sys.argv[2:])
infile, outfile = sys.argv[2:]
adfile = os.environ["_CONDOR_JOB_AD"]
print("Parsing classad")
with open(adfile, "r") as fd:
ad = classad.parseOld(fd)
print("..done")
in_args = []
if infile != "None":
with open(infile, "r") as fd:
in_args = pickle.load(fd)
config = Configuration.Configuration()
config.section_("Services")
config.Services.DBSUrl = 'https://cmsweb.cern.ch/dbs/prod/phys03/DBSWriter/'
ad['tm_taskname'] = ad.eval("CRAB_Workflow")
ad['tm_split_algo'] = ad.eval("CRAB_SplitAlgo")
ad['tm_dbs_url'] = ad.eval("CRAB_DBSURL")
ad['tm_input_dataset'] = ad.eval("DESIRED_CMSDataset")
ad['tm_outfiles'] = HTCondorUtils.unquote(
ad.eval("CRAB_AdditionalOutputFiles"))
ad['tm_tfile_outfiles'] = HTCondorUtils.unquote(
ad.eval("CRAB_TFileOutputFiles"))
ad['tm_edm_outfiles'] = HTCondorUtils.unquote(
ad.eval("CRAB_EDMOutputFiles"))
ad['tm_site_whitelist'] = HTCondorUtils.unquote(
ad.eval("CRAB_SiteWhitelist"))
ad['tm_site_blacklist'] = HTCondorUtils.unquote(
ad.eval("CRAB_SiteBlacklist"))
ad['tm_job_type'] = 'Analysis'
print("TaskManager got this raw ad")
print(ad)
pure_ad = {}
for key in ad:
try:
pure_ad[key] = ad.eval(key)
if isinstance(pure_ad[key], classad.Value):
del pure_ad[key]
if isinstance(pure_ad[key], list):
pure_ad[key] = [i.eval() for i in pure_ad[key]]
except:
pass
ad = pure_ad
ad['CRAB_AlgoArgs'] = json.loads(ad["CRAB_AlgoArgs"])
ad['tm_split_args'] = ad["CRAB_AlgoArgs"]
ad['tarball_location'] = os.environ.get('CRAB_TARBALL_LOCATION', '')
print("TaskManagerBootstrap got this ad:")
pprint.pprint(ad)
results = task.execute(in_args, task=ad).result
print(results)
with open(outfile, "w") as fd:
pickle.dump(results, fd)
return 0
import WMCore.Configuration as Configuration
config = Configuration.Configuration()
config.section_("Services")
config.Services.DBSUrl = 'http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet'
config.section_("Sites")
config.Sites.available = ["T2_US_Nebraska"]
def bootstrap():
print "Entering TaskManagerBootstrap with args: %s" % sys.argv
command = sys.argv[1]
if command == "POSTJOB":
return PostJob.PostJob().execute(*sys.argv[2:])
elif command == "PREJOB":
return PreJob.PreJob().execute(*sys.argv[2:])
elif command == "FINAL":
return Final.Final().execute(*sys.argv[2:])
elif command == "ASO":
return ASO.async_stageout(*sys.argv[2:])
infile, outfile = sys.argv[2:]
adfile = os.environ["_CONDOR_JOB_AD"]
print "Parsing classad"
with open(adfile, "r") as fd:
ad = classad.parseOld(fd)
print "..done"
in_args = []
if infile != "None":
with open(infile, "r") as fd:
in_args = pickle.load(fd)
config = Configuration.Configuration()
config.section_("Services")
config.Services.DBSUrl = 'https://cmsweb.cern.ch/dbs/prod/phys03/DBSWriter/'
ad['tm_taskname'] = ad.eval("CRAB_Workflow")
ad['tm_split_algo'] = ad.eval("CRAB_SplitAlgo")
ad['tm_dbs_url'] = ad.eval("CRAB_DBSUrl")
ad['tm_input_dataset'] = ad.eval("CRAB_InputData")
ad['tm_outfiles'] = HTCondorUtils.unquote(
ad.eval("CRAB_AdditionalOutputFiles"))
ad['tm_tfile_outfiles'] = HTCondorUtils.unquote(
ad.eval("CRAB_TFileOutputFiles"))
ad['tm_edm_outfiles'] = HTCondorUtils.unquote(
ad.eval("CRAB_EDMOutputFiles"))
ad['tm_site_whitelist'] = HTCondorUtils.unquote(
ad.eval("CRAB_SiteWhitelist"))
ad['tm_site_blacklist'] = HTCondorUtils.unquote(
ad.eval("CRAB_SiteBlacklist"))
ad['tm_job_type'] = 'Analysis'
print "TaskManager got this raw ad"
print ad
pure_ad = {}
for key in ad:
try:
pure_ad[key] = ad.eval(key)
if isinstance(pure_ad[key], classad.Value):
del pure_ad[key]
if isinstance(pure_ad[key], types.ListType):
pure_ad[key] = [i.eval() for i in pure_ad[key]]
except:
pass
ad = pure_ad
ad['CRAB_AlgoArgs'] = json.loads(ad["CRAB_AlgoArgs"])
ad['tm_split_args'] = ad["CRAB_AlgoArgs"]
ad['tarball_location'] = os.environ.get('CRAB_TARBALL_LOCATION', '')
print "TaskManagerBootstrap got this ad:"
pprint.pprint(ad)
if command == "DBS":
task = DBSDataDiscovery.DBSDataDiscovery(config)
elif command == "SPLIT":
task = Splitter.Splitter(config)
print "Got this result from the splitter"
pprint.pprint(task)
results = task.execute(in_args, task=ad).result
if command == "SPLIT":
results = DagmanCreator.create_subdag(results, task=ad)
print results
with open(outfile, "w") as fd:
pickle.dump(results, fd)
return 0
siteThr, siteValue, taskPacks )
return result
except:
self.logger.debug(sys.exc_info()[0])
self.logger.debug(sys.exc_info()[1])
return {'Error':'ERROR'}
registerMonitor(PilotBlSimpleMonitor, PilotBlSimpleMonitor.__name__)
#for testing purpose
if __name__ == '__main__':
args = {'cpCmd':'rfcp','rfioSer':''}
from WMCore import Configuration
tqconfig = Configuration.loadConfigurationFile( \
"/data/khawar/antonio/tqcode/extras/WMCore-conf.py" )
from TQComp.Apis.TQStateApi import TQStateApi
myThread = threading.currentThread()
logger = logging.getLogger()
#self.logger.debug( tqconfig)
tqStateApi = TQStateApi(logger, tqconfig, None)
pbsched = PilotBlSimpleMonitor()
pbsched.monitorAll(tqStateApi)
#pbsched('srm-cms.cern.ch')
#poller = PABossLitePoll()
#poller('abc.it.fr')
#pbsched.getGroups(None)