random.choice(string.ascii_uppercase + string.digits) for _ in range(assFileSpec.fsize))) oFile.close() fileSpec.add_associated_file(assFileSpec) jobSpec = JobSpec() jobSpec.jobParams = { 'outFiles': fileSpec.lfn + ',log', 'scopeOut': 'panda', 'scopeLog': 'panda', 'logFile': 'log', 'realDatasets': 'panda.' + fileSpec.lfn, 'ddmEndPointOut': 'BNL-OSG2_DATADISK', } jobSpec.computingSite = queueName jobSpec.PandaID = job_id jobSpec.add_out_file(fileSpec) print "file to transfer - {}".format(assFileSpec.path) print "dump(jobSpec)" #dump(jobSpec) pluginFactory = PluginFactory() # get stage-out plugin stagerCore = pluginFactory.get_plugin(queueConfig.stager) print "plugin={0}".format(stagerCore.__class__.__name__) print "testing zip" tmpStat, tmpOut = stagerCore.zip_output(jobSpec) if tmpStat:
unlocked = stagerCore.dbInterface.release_object_lock('dummy_id_for_out_0') if unlocked : tmpLog.debug('unlocked db') else: tmpLog.debug(' Could not unlock db') # loop over the job id's creating various JobSpecs jobSpec_list = [] for job_id in range(begin_job_id,end_job_id+1): jobSpec = JobSpec() jobSpec.jobParams = { 'scopeLog': 'panda', 'logFile': 'log', } jobSpec.computingSite = queueName jobSpec.PandaID = job_id jobSpec.modificationTime = datetime.datetime.now() realDataset = 'panda.sgotest.' + uuid.uuid4().hex ddmEndPointOut = 'BNL-OSG2_DATADISK' outFiles_scope_str = '' outFiles_str = '' realDatasets_str = '' ddmEndPointOut_str = '' # create up 5 files for output for index in range(random.randint(1, 5)): fileSpec = FileSpec() assFileSpec = FileSpec() fileSpec.fileType = 'es_output' assFileSpec.lfn = 'panda.sgotest.' + uuid.uuid4().hex fileSpec.lfn = assFileSpec.lfn + '.gz' fileSpec.scope = 'panda'
os.remove(harvester_config.db.database_filename) except Exception: pass for loggerName, loggerObj in iteritems(logging.Logger.manager.loggerDict): if loggerName.startswith('panda.log'): if len(loggerObj.handlers) == 0: continue if loggerName.split('.')[-1] in ['db_proxy']: continue stdoutHandler = logging.StreamHandler(sys.stdout) stdoutHandler.setFormatter(loggerObj.handlers[0].formatter) loggerObj.addHandler(stdoutHandler) queueConfigMapper = QueueConfigMapper() proxy = DBProxy() proxy.make_tables(queueConfigMapper) job = JobSpec() job.PandaID = 1 job.modificationTime = datetime.datetime.now() proxy.insert_jobs([job]) newJob = proxy.get_job(1) a = CommunicatorPool() a.get_jobs('siteName', 'nodeName', 'prodSourceLabel', 'computingElement', 1, {})
'TXT.19772875._044894.tar.gz.1', 'scopeIn': 'mc15_13TeV', 'fsize': '658906675', 'GUID': '7e3776f9bb0af341b03e59d3de895a13', 'checksum': 'ad:3734bdd9', 'ddmEndPointIn': 'BNL-OSG2_DATADISK', 'realDatasetsIn': 'mc15_13TeV.363638.MGPy8EG_N30NLO_Wmunu_Ht500_700_BFilter.merge.DAOD_STDM4.e4944_s2726_r7772_r7676_p2842_tid09596175_00', } jobSpec.computingSite = queueName jobSpec.PandaID = '11111' from pandaharvester.harvestercore.plugin_factory import PluginFactory pluginFactory = PluginFactory() # get plugin preparatorCore = pluginFactory.get_plugin(queueConfig.preparator) print("plugin={0}".format(preparatorCore.__class__.__name__)) print(jobSpec) print("testing stagein:") print("BasePath from preparator configuration: %s " % preparatorCore.basePath) preparatorCore.basePath = preparatorCore.basePath + "/testdata/" print("basePath redifuned for test data: %s " % preparatorCore.basePath)
except Exception: pass for loggerName, loggerObj in iteritems(logging.Logger.manager.loggerDict): if loggerName.startswith('panda.log'): if len(loggerObj.handlers) == 0: continue if loggerName.split('.')[-1] in ['db_proxy']: continue stdoutHandler = logging.StreamHandler(sys.stdout) stdoutHandler.setFormatter(loggerObj.handlers[0].formatter) loggerObj.addHandler(stdoutHandler) queueConfigMapper = QueueConfigMapper() proxy = DBProxy() proxy.make_tables(queueConfigMapper) job = JobSpec() job.PandaID = 1 job.modificationTime = datetime.datetime.now() proxy.insert_jobs([job]) newJob = proxy.get_job(1) a = CommunicatorPool() a.get_jobs('siteName', 'nodeName', 'prodSourceLabel', 'computingElement', 1, {})
import sys import uuid from pandaharvester.harvestercore.job_spec import JobSpec from pandaharvester.harvestercore.file_spec import FileSpec from pandaharvester.harvestercore.event_spec import EventSpec from pandaharvester.harvestercore.communicator_pool import CommunicatorPool rID = sys.argv[1] taskid = rID.split('-')[0] pandaid = long(rID.split('-')[1]) job = JobSpec() job.PandaID = pandaid event = EventSpec() file = FileSpec() file.status = 'finished' file.objstoreID = 9575 file.pathConvention = 1000 file.lfn = str(uuid.uuid4().hex) + '.zip' file.fsize = 555 file.chksum = '0d2a9dc9' event.eventRangeID = rID event.eventStatus = 'finished' job.zipEventMap = {1: {'events':[event], 'zip':file}} a = CommunicatorPool() a.update_jobs([job])