def renew_credential(self): # make logger mainLog = self.make_logger(_logger, method_name='renew_credential') # make communication channel to PanDA com = CommunicatorPool() proxy, msg = com.get_proxy(self.voms, (self.inCertFile, self.inCertFile)) if proxy is not None: pFile = open(self.outCertFile, 'w') pFile.write(proxy) pFile.close() else: mainLog.error( 'failed to renew credential with a server message : {0}'. format(msg)) return proxy is not None, msg
def __init__(self, single_mode=False, stop_event=None, daemon_mode=True): # initialize database and config self.singleMode = single_mode self.stopEvent = stop_event self.daemonMode = daemon_mode from pandaharvester.harvestercore.communicator_pool import CommunicatorPool self.communicatorPool = CommunicatorPool() from pandaharvester.harvestercore.queue_config_mapper import QueueConfigMapper self.queueConfigMapper = QueueConfigMapper() from pandaharvester.harvestercore.db_proxy_pool import DBProxyPool as DBProxy dbProxy = DBProxy() dbProxy.make_tables(self.queueConfigMapper)
print('value for jobType not valid, defaulted to {0}'.format( jobType)) # resourceType should be 'SCORE', 'SCORE_HIMEM', 'MCORE', 'MCORE_HIMEM'. If not specified defaults to single core if sys.argv[3] in ('SCORE', 'SCORE_HIMEM', 'MCORE', 'MCORE_HIMEM'): resourceType = sys.argv[3] else: print('value for resourceType not valid, defaulted to {0}'.format( resourceType)) print('Running with queueName:{0}, jobType:{1}, resourceType:{2}'.format( queueName, jobType, resourceType)) pluginFactory = PluginFactory() com = CommunicatorPool() # get job jobSpecList = [] if queueConfig.mapType != WorkSpec.MT_NoJob: jobs, errStr = com.get_jobs(queueConfig.queueName, 'nodeName', queueConfig.prodSourceLabel, 'computingElement', 1, None) if len(jobs) == 0: print("Failed to get jobs at {0} due to {1}".format( queueConfig.queueName, errStr)) sys.exit(0) jobSpec = JobSpec() jobSpec.convert_job_json(jobs[0])
stdoutHandler.setFormatter(loggerObj.handlers[0].formatter) loggerObj.addHandler(stdoutHandler) msgStr = "plugin={0}".format(preparatorCore.__class__.__name__) tmpLog.debug(msgStr) msgStr = "Initial queueConfig.preparator = {}".format( initial_queueConfig_preparator) tmpLog.debug(msgStr) msgStr = "Modified queueConfig.preparator = {}".format( modified_queueConfig_preparator) tmpLog.debug(msgStr) scope = 'panda' proxy = DBProxy() communicator = CommunicatorPool() cacher = Cacher(communicator, single_mode=True) cacher.run() tmpLog.debug("plugin={0}".format(preparatorCore.__class__.__name__)) tmpLog.debug("BasePath from preparator configuration: %s " % preparatorCore.basePath) # get all jobs in table in a preparing substate tmpLog.debug('try to get all jobs in a preparing substate') jobSpec_list = proxy.get_jobs_in_sub_status('preparing', 2000, None, None, None, None, None, None) tmpLog.debug('got {0} jobs'.format(len(jobSpec_list))) # loop over all found jobs if len(jobSpec_list) > 0: for jobSpec in jobSpec_list:
os.remove(harvester_config.db.database_filename) except Exception: pass for loggerName, loggerObj in iteritems(logging.Logger.manager.loggerDict): if loggerName.startswith('panda.log'): if len(loggerObj.handlers) == 0: continue if loggerName.split('.')[-1] in ['db_proxy']: continue stdoutHandler = logging.StreamHandler(sys.stdout) stdoutHandler.setFormatter(loggerObj.handlers[0].formatter) loggerObj.addHandler(stdoutHandler) queueConfigMapper = QueueConfigMapper() proxy = DBProxy() proxy.make_tables(queueConfigMapper) job = JobSpec() job.PandaID = 1 job.modificationTime = datetime.datetime.now() proxy.insert_jobs([job]) newJob = proxy.get_job(1) a = CommunicatorPool() a.get_jobs('siteName', 'nodeName', 'prodSourceLabel', 'computingElement', 1, {})
def cacher_refresh(arguments): from pandaharvester.harvestercore.communicator_pool import CommunicatorPool from pandaharvester.harvesterbody.cacher import Cacher communicatorPool = CommunicatorPool() cacher = Cacher(communicatorPool) cacher.execute(force_update=True, skip_lock=True, n_thread=4)
import os import sys from pandaharvester.harvestercore.communicator_pool import CommunicatorPool pandaid = int(sys.argv[1]) jobsetid = int(sys.argv[2]) taskid = int(sys.argv[3]) try: n = int(sys.argv[4]) except Exception: n = 1 data = {pandaid: {'pandaID': pandaid, 'taskID': taskid, 'jobsetID': jobsetid, 'nRanges': n} } a = CommunicatorPool() o = a.get_event_ranges(data, False, os.getcwd())
except Exception: pass for loggerName, loggerObj in iteritems(logging.Logger.manager.loggerDict): if loggerName.startswith('panda.log'): if len(loggerObj.handlers) == 0: continue if loggerName.split('.')[-1] in ['db_proxy']: continue stdoutHandler = logging.StreamHandler(sys.stdout) stdoutHandler.setFormatter(loggerObj.handlers[0].formatter) loggerObj.addHandler(stdoutHandler) queueConfigMapper = QueueConfigMapper() proxy = DBProxy() proxy.make_tables(queueConfigMapper) job = JobSpec() job.PandaID = 1 job.modificationTime = datetime.datetime.now() proxy.insert_jobs([job]) newJob = proxy.get_job(1) a = CommunicatorPool() a.get_jobs('siteName', 'nodeName', 'prodSourceLabel', 'computingElement', 1, {})
import sys from pandaharvester.harvestercore.db_proxy_pool import DBProxyPool as DBProxy from pandaharvester.harvestercore.communicator_pool import CommunicatorPool from pandaharvester.harvestermessenger import shared_file_messenger workerID = int(sys.argv[1]) proxy = DBProxy() workSpec = proxy.get_worker_with_id(workerID) jobSpec = proxy.get_jobs_with_worker_id(workerID, None)[0] accessPoint = workSpec.get_access_point() try: os.makedirs(accessPoint) except: pass node = {} node['pandaID'] = jobSpec.PandaID node['jobsetID'] = jobSpec.jobParams['jobsetID'] node['taskID'] = jobSpec.taskID a = CommunicatorPool() tmpStat, tmpVal = a.getEventRanges(node) mess = shared_file_messenger.SharedFileMessenger() mess.feed_events(workSpec, tmpVal)
import sys from pandaharvester.harvestercore.communicator_pool import CommunicatorPool pandaid = int(sys.argv[1]) jobsetid = int(sys.argv[2]) taskid = int(sys.argv[3]) try: n = int(sys.argv[4]) except Exception: n = 1 data = {pandaid: {'pandaID': pandaid, 'taskID': taskid, 'jobsetID': jobsetid, 'nRanges': n} } a = CommunicatorPool() o = a.get_event_ranges(data)
import sys from pandaharvester.harvestercore.communicator_pool import CommunicatorPool pandaid = int(sys.argv[1]) jobsetid = int(sys.argv[2]) taskid = int(sys.argv[3]) try: n = int(sys.argv[4]) except Exception: n = 1 data = { pandaid: { 'pandaID': pandaid, 'taskID': taskid, 'jobsetID': jobsetid, 'nRanges': n } } a = CommunicatorPool() o = a.get_event_ranges(data)
from pandaharvester.harvestercore.communicator_pool import CommunicatorPool from pandaharvester.harvestercore.job_spec import JobSpec from pandaharvester.harvestermisc import signal_utils fork_child_pid = os.fork() if fork_child_pid != 0: signal_utils.set_suicide_handler(None) os.wait() else: queueName = sys.argv[1] queueConfigMapper = QueueConfigMapper() queueConfig = queueConfigMapper.get_queue(queueName) pluginFactory = PluginFactory() com = CommunicatorPool() # get job jobSpecList = [] if queueConfig.mapType != WorkSpec.MT_NoJob: jobs, errStr = com.get_jobs(queueConfig.queueName, 'nodeName', queueConfig.prodSourceLabel, 'computingElement', 1, None) if len(jobs) == 0: print ("Failed to get jobs at {0} due to {1}".format(queueConfig.queueName, errStr)) sys.exit(0) jobSpec = JobSpec() jobSpec.convert_job_json(jobs[0]) # set input file paths inFiles = jobSpec.get_input_file_attributes()
import sys import uuid from pandaharvester.harvestercore.job_spec import JobSpec from pandaharvester.harvestercore.file_spec import FileSpec from pandaharvester.harvestercore.event_spec import EventSpec from pandaharvester.harvestercore.communicator_pool import CommunicatorPool rID = sys.argv[1] taskid = rID.split('-')[0] pandaid = long(rID.split('-')[1]) job = JobSpec() job.PandaID = pandaid event = EventSpec() file = FileSpec() file.status = 'finished' file.objstoreID = 9575 file.pathConvention = 1000 file.lfn = str(uuid.uuid4().hex) + '.zip' file.fsize = 555 file.chksum = '0d2a9dc9' event.eventRangeID = rID event.eventStatus = 'finished' job.zipEventMap = {1: {'events':[event], 'zip':file}} a = CommunicatorPool() a.update_jobs([job])
import sys import logging from future.utils import iteritems from pandaharvester.harvestercore.communicator_pool import CommunicatorPool for loggerName, loggerObj in iteritems(logging.Logger.manager.loggerDict): if loggerName.startswith('panda.log'): if len(loggerObj.handlers) == 0: continue if loggerName.split('.')[-1] in ['db_proxy']: continue stdoutHandler = logging.StreamHandler(sys.stdout) stdoutHandler.setFormatter(loggerObj.handlers[0].formatter) loggerObj.addHandler(stdoutHandler) a = CommunicatorPool() return_object = a.check_panda() print (return_object)