from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.plugin_base import PluginBase from pandaharvester.harvestercloud.googlecloud import compute, ZONE, PROJECT from pandaharvester.harvestercore.queue_config_mapper import QueueConfigMapper import googleapiclient base_logger = core_utils.setup_logger('google_sweeper') class GoogleSweeper(PluginBase): """ Sweeper with kill/clean-up functions for Google Compute Engine """ def __init__(self, **kwarg): PluginBase.__init__(self, **kwarg) self.queue_config_mapper = QueueConfigMapper() def kill_worker(self, work_spec): """ Sends the command to Google to destroy a VM :param work_spec: worker specification :type work_spec: WorkSpec :return: A tuple of return code (True for success, False otherwise) and error dialog :rtype: (bool, string) """ try: vm_name = work_spec.batchID queue_config = self.queue_config_mapper.get_queue(work_spec.computingSite) try:
if len(sys.argv) > 4: globus_sleep_time = int(sys.argv[4]) queueConfigMapper = QueueConfigMapper() queueConfig = queueConfigMapper.get_queue(queueName) initial_queueConfig_preparator = queueConfig.preparator queueConfig.preparator['module'] = 'pandaharvester.harvesterpreparator.go_bulk_preparator' queueConfig.preparator['name'] = 'GlobusBulkPreparator' modified_queueConfig_preparator = queueConfig.preparator pluginFactory = PluginFactory() # get stage-out plugin preparatorCore = pluginFactory.get_plugin(queueConfig.preparator) # logger _logger = core_utils.setup_logger('stageInTest_go_bulk_preparator') tmpLog = core_utils.make_logger(_logger, method_name='stageInTest_go_bulk_preparator') tmpLog.debug('start') for loggerName, loggerObj in logging.Logger.manager.loggerDict.iteritems(): #print "loggerName - {}".format(loggerName) if loggerName.startswith('panda.log'): if len(loggerObj.handlers) == 0: continue if loggerName.split('.')[-1] in ['db_proxy']: continue stdoutHandler = logging.StreamHandler(sys.stdout) stdoutHandler.setFormatter(loggerObj.handlers[0].formatter) loggerObj.addHandler(stdoutHandler) msgStr = "plugin={0}".format(preparatorCore.__class__.__name__)
from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.plugin_base import PluginBase from pandaharvester.harvestercore.work_spec import WorkSpec as ws try: import subprocess32 as subprocess except Exception: import subprocess import datetime # logger baseLogger = core_utils.setup_logger('titan_utils') class TitanUtils(PluginBase): # constructor def __init__(self, **kwarg): PluginBase.__init__(self, **kwarg) tmpLog = self.make_logger(baseLogger, method_name='__init__') tmpLog.info("Titan utils initiated") def get_batchjob_info(self, batchid): """ Collect job info from scheduler :param batchid: :return res - dictonary with job state and some timing: """ """ :param batchid: :return:
import subprocess32 as subprocess except Exception: import subprocess from email.mime.text import MIMEText from pandaharvester.harvesterconfig import harvester_config from pandaharvester.harvestercore import core_utils from pandaharvester.harvesterbody.agent_base import AgentBase from pandalogger import logger_config logDir = logger_config.daemon['logdir'] lockFileName = os.path.join(logDir, 'watcher.lock') # logger _logger = core_utils.setup_logger('watcher') # watching the system class Watcher(AgentBase): # constructor def __init__(self, single_mode=False): AgentBase.__init__(self, single_mode) self.startTime = datetime.datetime.utcnow() # main loop def run(self): while True: # execute self.execute() # check if being terminated
import datetime import uuid from pandaharvester.harvestercore.plugin_base import PluginBase from pandaharvester.harvestercore import core_utils # dummy transfer identifier dummy_transfer_id = 'dummy_id_for_out' # logger baseLogger = core_utils.setup_logger('dummy_bulk_stager') # dummy plugin for stager with bulk transfers. For JobSpec and DBInterface methods, see # https://github.com/PanDAWMS/panda-harvester/wiki/Utilities#file-grouping-for-file-transfers class DummyBulkStager(PluginBase): # constructor def __init__(self, **kwarg): PluginBase.__init__(self, **kwarg) # check status def check_status(self, jobspec): # make logger tmpLog = core_utils.make_logger(baseLogger, 'PandaID={0}'.format(jobspec.PandaID), method_name='check_status') tmpLog.debug('start') # get transfer groups groups = jobspec.get_groups_of_output_files() # lock if the dummy transfer ID is used to avoid submitting duplicated transfer requests if dummy_transfer_id in groups:
from pandaharvester.harvesterconfig import harvester_config from pandaharvester.harvestermover import mover_utils from pandaharvester.harvestercore.queue_config_mapper import QueueConfigMapper from pandaharvester.harvesterstager.base_stager import BaseStager from .base_stager import BaseStager def dump(tmpLog,obj): for attr in dir(obj): if hasattr( obj, attr ): tmpLog.debug( "obj.%s = %s" % (attr, getattr(obj, attr))) # logger baseLogger = core_utils.setup_logger('yoda_rucio_rse_direct_stager') class Error(EnvironmentError): pass class SpecialFileError(EnvironmentError): """Raised when trying to do a kind of operation (e.g. copying) which is not supported on a special file (e.g. a named pipe)""" class ExecError(EnvironmentError): """Raised when a command could not be executed""" # stager plugin with RSE + local site move behaviour for Yoda zip files class YodaRucioRseDirectStager(BaseStager):
from pandaharvester.harvesterconfig import harvester_config from pandaharvester.harvestermover import mover_utils from pandaharvester.harvestercore.queue_config_mapper import QueueConfigMapper from pandaharvester.harvesterstager.base_stager import BaseStager from .base_stager import BaseStager def dump(tmpLog,obj): for attr in dir(obj): if hasattr( obj, attr ): tmpLog.debug( "obj.%s = %s" % (attr, getattr(obj, attr))) # logger baseLogger = core_utils.setup_logger('yoda_rucio_rse_direct_stager') class Error(EnvironmentError): pass class SpecialFileError(EnvironmentError): """Raised when trying to do a kind of operation (e.g. copying) which is not supported on a special file (e.g. a named pipe)""" class ExecError(EnvironmentError): """Raised when a command could not be executed""" # stager plugin with RSE + local site move behaviour for Yoda zip files class YodaRucioRseDirectStager(BaseStager):
import datetime from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.db_proxy_pool import DBProxyPool as DBProxy from pandaharvester.harvestercore.plugin_base import PluginBase # logger baseLogger = core_utils.setup_logger('simple_throttler') # simple throttler class SimpleThrottler(PluginBase): # constructor def __init__(self, **kwarg): # logic type : AND: throttled if all rules are satisfied, OR: throttled if one rule is satisfied self.logicType = 'OR' PluginBase.__init__(self, **kwarg) self.dbProxy = DBProxy() # check if to be throttled def to_be_throttled(self, queue_config): tmpLog = self.make_logger(baseLogger, 'computingSite={0}'.format(queue_config.queueName), method_name='to_be_throttled') tmpLog.debug('start') # set default return vale if self.logicType == 'OR': retVal = False, "no rule was satisfied" else: retVal = True, "all rules were satisfied" # loop over all rules criteriaList = []
import os import shutil from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.plugin_base import PluginBase from act.common.aCTConfig import aCTConfigARC from act.atlas.aCTDBPanda import aCTDBPanda # logger baseLogger = core_utils.setup_logger('act_sweeper') # plugin for aCT sweeper class ACTSweeper(PluginBase): # constructor def __init__(self, **kwarg): PluginBase.__init__(self, **kwarg) self.log = core_utils.make_logger(baseLogger, 'aCT sweeper', method_name='__init__') self.actDB = aCTDBPanda(self.log) # kill a worker def kill_worker(self, workspec): """ Mark aCT job as tobekilled. :param workspec: worker specification :type workspec: WorkSpec :return: A tuple of return code (True for success, False otherwise) and error dialog :rtype: (bool, string)
import os.path import os from future.utils import iteritems from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.plugin_base import PluginBase from pandaharvester.harvestermover import mover_utils from pilot.api import data # logger baseLogger = core_utils.setup_logger('pilotmover_preparator') # plugin for preparator based on Pilot2.0 Data API # Pilot 2.0 should be deployed as library # default self.basePath came from preparator section of configuration file class PilotmoverPreparator(PluginBase): """ Praparator bring files from remote ATLAS/Rucio storage to local facility. """ # constructor def __init__(self, **kwarg): PluginBase.__init__(self, **kwarg) # check status def check_status(self, jobspec): return True, ''
#if len(sys.argv) > 4: # globus_sleep_time = int(sys.argv[4]) queueConfigMapper = QueueConfigMapper() queueConfig = queueConfigMapper.get_queue(queueName) initial_queueConfig_preparator = queueConfig.preparator queueConfig.preparator['module'] = 'pandaharvester.harvesterpreparator.go_bulk_preparator' queueConfig.preparator['name'] = 'GlobusBulkPreparator' modified_queueConfig_preparator = queueConfig.preparator pluginFactory = PluginFactory() # get stage-out plugin preparatorCore = pluginFactory.get_plugin(queueConfig.preparator) # logger _logger = core_utils.setup_logger('further_testing_go_bulk_preparator') tmpLog = core_utils.make_logger(_logger, method_name='further_testing_go_bulk_preparator') tmpLog.debug('start') for loggerName, loggerObj in logging.Logger.manager.loggerDict.iteritems(): #print "loggerName - {}".format(loggerName) if loggerName.startswith('panda.log'): if len(loggerObj.handlers) == 0: continue if loggerName.split('.')[-1] in ['db_proxy']: continue stdoutHandler = logging.StreamHandler(sys.stdout) stdoutHandler.setFormatter(loggerObj.handlers[0].formatter) loggerObj.addHandler(stdoutHandler) msgStr = "plugin={0}".format(preparatorCore.__class__.__name__)
import tempfile try: import subprocess32 as subprocess except: import subprocess import os import stat from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.plugin_base import PluginBase # logger baseLogger = core_utils.setup_logger('cobalt_submitter') # submitter for Cobalt batch system class CobaltSubmitter(PluginBase): # constructor def __init__(self, **kwarg): self.uploadLog = False self.logBaseURL = None PluginBase.__init__(self, **kwarg) # template for batch script tmpFile = open(self.templateFile) self.template = tmpFile.read() tmpFile.close() # submit workers def submit_workers(self, workspec_list): retList = [] retStrList = []
except: import subprocess import random import uuid from concurrent.futures import ThreadPoolExecutor import re from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.plugin_base import PluginBase from pandaharvester.harvestermisc.cloud_openstack_utils import OS_SimpleClient # setup base logger baseLogger = core_utils.setup_logger('cloud_openstack_submitter') def _init_script_replace(string, **kwarg): new_string = string macro_map = { '\$\(workerID\)': str(kwarg['workerID']), '\$\(batchID\)': str(kwarg['batchID']), '\$\(accessPoint\)': str(kwarg['accessPoint']), } for k, v in macro_map.items(): new_string = re.sub(k, v, new_string) return new_string # make cloud initialization script
import six from concurrent.futures import ThreadPoolExecutor as Pool from pandaharvester.harvestercore import core_utils from pandaharvester.harvesterconfig import harvester_config from pandaharvester.harvestercore.work_spec import WorkSpec from pandaharvester.harvestercore.worker_errors import WorkerErrors from pandaharvester.harvestercore.plugin_base import PluginBase from pandaharvester.harvestermisc.htcondor_utils import condor_job_id_from_workspec, get_host_batchid_map from pandaharvester.harvestermisc.htcondor_utils import CondorJobQuery, CondorJobManage # logger baseLogger = core_utils.setup_logger('htcondor_monitor') # Native HTCondor job status map CONDOR_JOB_STATUS_MAP = { '1': 'idle', '2': 'running', '3': 'removed', '4': 'completed', '5': 'held', '6': 'transferring_output', '7': 'suspended', } # Check one worker
import json import os from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.work_spec import WorkSpec from pandaharvester.harvestercore.worker_errors import WorkerErrors from pandaharvester.harvestercore.file_spec import FileSpec from pandaharvester.harvesterconfig import harvester_config from .base_stager import BaseStager from act.atlas.aCTDBPanda import aCTDBPanda # logger baseLogger = core_utils.setup_logger('act_stager') # json for job report jsonJobReport = harvester_config.payload_interaction.jobReportFile # aCT stager plugin class ACTStager(BaseStager): # constructor def __init__(self, **kwarg): BaseStager.__init__(self, **kwarg) # Set up aCT DB connection self.log = core_utils.make_logger(baseLogger, 'aCT stager', method_name='__init__') try: self.actDB = aCTDBPanda(self.log) except Exception as e: self.log.error('Could not connect to aCT database: {0}'.format(str(e))) self.actDB = None
import sys import threading import traceback from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.work_spec import WorkSpec from pandaharvester.harvestercore.plugin_base import PluginBase from pandaharvester.harvestersubmitter.apfgrid_submitter import APFGridSubmitter try: from autopyfactory import condorlib except ImportError: logging.error("Unable to import htcondor/condorlib. sys.path=%s" % sys.path) # setup base logger baseLogger = core_utils.setup_logger() class APFGridMonitorSingleton(type): def __init__(self, *args, **kwargs): super(APFGridMonitorSingleton, self).__init__(*args, **kwargs) self.__instance = None def __call__(self, *args, **kwargs): if self.__instance is None: self.__instance = super(APFGridMonitorSingleton, self).__call__(*args, **kwargs) return self.__instance class APFGridMonitor(object): '''
globus_sleep_time = int(sys.argv[3]) queueConfigMapper = QueueConfigMapper() queueConfig = queueConfigMapper.get_queue(queueName) initial_queueConfig_preparator = queueConfig.preparator queueConfig.preparator[ 'module'] = 'pandaharvester.harvesterpreparator.go_preparator' queueConfig.preparator['name'] = 'GoPreparator' modified_queueConfig_preparator = queueConfig.preparator pluginFactory = PluginFactory() # get stage-out plugin preparatorCore = pluginFactory.get_plugin(queueConfig.preparator) # logger _logger = core_utils.setup_logger('stageInTest_go_preparator') tmpLog = core_utils.make_logger(_logger, method_name='stageInTest_go_preparator') tmpLog.debug('start') for loggerName, loggerObj in iteritems(logging.Logger.manager.loggerDict): #print "loggerName - {}".format(loggerName) if loggerName.startswith('panda.log'): if len(loggerObj.handlers) == 0: continue if loggerName.split('.')[-1] in ['db_proxy']: continue stdoutHandler = logging.StreamHandler(sys.stdout) stdoutHandler.setFormatter(loggerObj.handlers[0].formatter) loggerObj.addHandler(stdoutHandler)
""" Base class to communicate with WMS """ import abc from future.utils import with_metaclass from pandaharvester.harvestercore import core_utils # logger _logger = core_utils.setup_logger('communicator') # base class for communication with WMS class BaseCommunicator(with_metaclass(abc.ABCMeta, object)): # constructor def __init__(self): pass # make logger def make_logger(self, tag=None, method_name=None): return core_utils.make_logger(_logger, token=tag, method_name=method_name) # get jobs def get_jobs(self, site_name, node_name, prod_source_label, computing_element, n_jobs, additional_criteria): return [], '' # update jobs def update_jobs(self, jobspec_list, id):
import socket import datetime from future.utils import iteritems from pandaharvester.harvesterconfig import harvester_config from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.db_proxy_pool import DBProxyPool as DBProxy from pandaharvester.harvesterbody.agent_base import AgentBase from pandaharvester.harvestercore.command_spec import CommandSpec from pandaharvester import commit_timestamp from pandaharvester import panda_pkg_info # logger _logger = core_utils.setup_logger('command_manager') # class to retrieve commands from panda server class CommandManager(AgentBase): # constructor def __init__(self, communicator, queue_config_mapper, single_mode=False): AgentBase.__init__(self, single_mode) self.db_proxy = DBProxy() self.communicator = communicator self.queueConfigMapper = queue_config_mapper self.nodeName = socket.gethostname() self.lastHeartbeat = None # set single mode def set_single_mode(self, single_mode): self.singleMode = single_mode def convert_to_command_specs(self, commands):
try: killWorkerFile = harvester_config.payload_interaction.killWorkerFile except Exception: killWorkerFile = 'kill_worker.json' # json for heartbeats from the worker try: heartbeatFile = harvester_config.payload_interaction.heartbeatFile except Exception: heartbeatFile = 'worker_heartbeat.json' # suffix to read json suffixReadJson = '.read' # logger _logger = core_utils.setup_logger('shared_file_messenger') def set_logger(master_logger): global _logger _logger = master_logger # filter for log.tgz def filter_log_tgz(extra=None): patt = ['*.log', '*.txt', '*.xml', '*.json', 'log*'] if extra is not None: patt += extra return '-o '.join(['-name "{0}" '.format(i) for i in patt])
import re try: import subprocess32 as subprocess except ImportError: import subprocess import json import os from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.work_spec import WorkSpec from pandaharvester.harvestercore.plugin_base import PluginBase # logger baseLogger = core_utils.setup_logger('slurm_squeue_monitor') # monitor for SLURM batch system with squeue class SlurmSqueueMonitor(PluginBase): _HARVESTER_POSTMORTEM_FILENAME = "FINISHED" # constructor def __init__(self, **kwarg): PluginBase.__init__(self, **kwarg) # check workers def check_workers(self, workspec_list): retList = [] for workSpec in workspec_list: # make logger tmpLog = self.make_logger(baseLogger,
import os from pandaharvester.harvestercore import core_utils from pandaharvester.harvestersweeper.base_sweeper import BaseSweeper from pandaharvester.harvestermisc.k8s_utils import k8s_Client # logger baseLogger = core_utils.setup_logger('k8s_sweeper') # sweeper for K8S class K8sSweeper(BaseSweeper): # constructor def __init__(self, **kwarg): BaseSweeper.__init__(self, **kwarg) self.k8s_client = k8s_Client(namespace=self.k8s_namespace, config_file=self.k8s_config_file) self._all_pods_list = [] # # kill a worker # def kill_worker(self, workspec): # tmpLog = self.make_logger(baseLogger, 'workerID={0}'.format(workspec.workerID), # method_name='kill_worker') # # tmpRetVal = (None, 'Nothing done') # # job_id = workspec.batchID # try: # self.k8s_client.delete_job(job_id)
import datetime from pandaharvester.harvesterconfig import harvester_config from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.db_proxy_pool import DBProxyPool as DBProxy from pandaharvester.harvestercore.plugin_factory import PluginFactory from pandaharvester.harvesterbody.agent_base import AgentBase from pandaharvester.harvestercore.pilot_errors import PilotErrors from pandaharvester.harvestercore.job_spec import JobSpec from pandaharvester.harvestercore.file_spec import FileSpec # logger _logger = core_utils.setup_logger('preparator') # class to prepare jobs class Preparator(AgentBase): # constructor def __init__(self, communicator, queue_config_mapper, single_mode=False): AgentBase.__init__(self, single_mode) self.dbProxy = DBProxy() self.communicator = communicator self.queueConfigMapper = queue_config_mapper self.pluginFactory = PluginFactory() # main loop def run(self): lockedBy = 'preparator-{0}'.format(self.get_pid()) while True: sw = core_utils.get_stopwatch() mainLog = self.make_logger(_logger,
from pandaharvester.harvestercore.work_spec import WorkSpec from pandaharvester.harvestercore.plugin_base import PluginBase from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.queue_config_mapper import QueueConfigMapper from pandaharvester.harvestercloud.googlecloud import compute, ZONE, PROJECT base_logger = core_utils.setup_logger('google_monitor') class GoogleMonitor(PluginBase): def __init__(self, **kwarg): PluginBase.__init__(self, **kwarg) self.queue_config_mapper = QueueConfigMapper() # States taken from: https://cloud.google.com/compute/docs/instances/checking-instance-status self.vm_to_worker_status = { 'RUNNING': WorkSpec.ST_running, 'TERMINATED': WorkSpec.ST_running, # the VM is stopped, but has to be fully deleted 'STOPPING': WorkSpec.ST_finished, 'PROVISIONING': WorkSpec.ST_submitted, 'STAGING': WorkSpec.ST_submitted } def list_vms(self, zone): """ List the status of the running VMs :return: """ try: result = compute.instances().list(project=PROJECT, zone=zone).execute()
import os.path import os from future.utils import iteritems from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.plugin_base import PluginBase from pandaharvester.harvestermover import mover_utils from pilot.api import data # logger baseLogger = core_utils.setup_logger('pilotmover_preparator') # plugin for preparator based on Pilot2.0 Data API # Pilot 2.0 should be deployed as library # default self.basePath came from preparator section of configuration file class PilotmoverPreparator(PluginBase): """ Praparator bring files from remote ATLAS/Rucio storage to local facility. """ # constructor def __init__(self, **kwarg): PluginBase.__init__(self, **kwarg) # check status def check_stage_in_status(self, jobspec): return True, '' # trigger preparation
import random from pandaharvester.harvestercore.work_spec import WorkSpec from pandaharvester.harvestercore import core_utils from pandaharvester.harvestermisc.info_utils import PandaQueuesDict from pandaharvester.harvestercore.resource_type_mapper import ResourceTypeMapper from .base_worker_maker import BaseWorkerMaker import datetime # logger _logger = core_utils.setup_logger('simple_worker_maker') # simple maker class SimpleWorkerMaker(BaseWorkerMaker): # constructor def __init__(self, **kwarg): self.jobAttributesToUse = ['nCore', 'minRamCount', 'maxDiskCount', 'maxWalltime', 'ioIntensity'] BaseWorkerMaker.__init__(self, **kwarg) self.rt_mapper = ResourceTypeMapper() try: self.pilotTypeRandomWeightsPermille except AttributeError: self.pilotTypeRandomWeightsPermille = {} finally: # randomize pilot type with weighting weight_rc = self.pilotTypeRandomWeightsPermille.get('RC', 0) weight_alrb = self.pilotTypeRandomWeightsPermille.get('ALRB', 0) weight_pt = self.pilotTypeRandomWeightsPermille.get('PT', 0) weight_tmp_sum = weight_rc + weight_alrb + weight_pt if weight_tmp_sum > 1000:
import json from pandaharvester.harvestercore import core_utils from pandaharvester.harvestermessenger import http_server_messenger from pandaharvester.harvestermisc.frontend_utils import HarvesterToken # logger _logger = core_utils.setup_logger('apache_messenger') http_server_messenger.set_logger(_logger) # handler class ApacheHandler(http_server_messenger.HttpHandler): def __init__(self, *args, **kwargs): self.responseCode = None self.form = dict() self.message = None self.headerList = [('Content-Type', 'text/plain')] http_server_messenger.HttpHandler.__init__(self, *args, **kwargs) def setup(self): pass def handle(self): pass def finish(self): pass def send_response(self, code, message=None): self.responseCode = code
try: import subprocess32 as subprocess except: import subprocess from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.plugin_base import PluginBase # logger _logger = core_utils.setup_logger('grid_cred_manager') # credential manager using grid-proxy class GridCredManager(PluginBase): # constructor def __init__(self, **kwarg): PluginBase.__init__(self, **kwarg) # check proxy def check_credential(self): # make logger mainLog = self.make_logger(_logger, method_name='check_credential') comStr = "grid-proxy-info -exists -hours 72 -file {0}".format(self.outCertFile) mainLog.debug(comStr) try: p = subprocess.Popen(comStr.split(), shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdOut, stdErr = p.communicate() retCode = p.returncode
if len(sys.argv) > 3: end_job_id = int(sys.argv[3]) queueConfigMapper = QueueConfigMapper() queueConfig = queueConfigMapper.get_queue(queueName) initial_queueConfig_stager = queueConfig.stager queueConfig.stager['module'] = 'pandaharvester.harvesterstager.go_bulk_stager' queueConfig.stager['name'] = 'GlobusBulkStager' modified_queueConfig_stager = queueConfig.stager pluginFactory = PluginFactory() # get stage-out plugin stagerCore = pluginFactory.get_plugin(queueConfig.stager) # logger _logger = core_utils.setup_logger('stageOutTest_go_bulk_stager') tmpLog = core_utils.make_logger(_logger, method_name='stageOutTest_go_bulk_stager') tmpLog.debug('start') for loggerName, loggerObj in logging.Logger.manager.loggerDict.iteritems(): #print "loggerName - {}".format(loggerName) if loggerName.startswith('panda.log'): if len(loggerObj.handlers) == 0: continue if loggerName.split('.')[-1] in ['db_proxy']: continue stdoutHandler = logging.StreamHandler(sys.stdout) stdoutHandler.setFormatter(loggerObj.handlers[0].formatter) loggerObj.addHandler(stdoutHandler) msgStr = "plugin={0}".format(stagerCore.__class__.__name__)
import radical.utils import saga import os import shutil from pandaharvester.harvestercore.plugin_base import PluginBase from pandaharvester.harvestercore import core_utils from pandaharvester.harvestersubmitter.saga_submitter import SAGASubmitter # logger baseLogger = core_utils.setup_logger('saga_sweeper') # dummy plugin for sweeper class SAGASweeper(PluginBase): # constructor def __init__(self, **kwarg): PluginBase.__init__(self, **kwarg) tmpLog = core_utils.make_logger(baseLogger, method_name='__init__') tmpLog.info("[{0}] SAGA adaptor will be used".format(self.adaptor)) # kill a worker def kill_worker(self, workspec): """Kill a worker in a scheduling system like batch systems and computing elements. :param workspec: worker specification :type workspec: WorkSpec :return: A tuple of return code (True for success, False otherwise) and error dialog :rtype: (bool, string) """ job_service = saga.job.Service(self.adaptor)
from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.db_proxy_pool import DBProxyPool as DBProxy from pandaharvester.harvestercore.plugin_factory import PluginFactory # logger _logger = core_utils.setup_logger('worker_maker') # class to make worker class WorkerMaker: # constructor def __init__(self): self.pluginFactory = PluginFactory() self.dbProxy = DBProxy() # make workers def make_workers(self, jobchunk_list, queue_config, n_ready, resource_type): tmpLog = core_utils.make_logger(_logger, 'queue={0}'.format( queue_config.queueName), method_name='make_workers') tmpLog.debug('start') try: # get plugin maker = self.pluginFactory.get_plugin(queue_config.workerMaker) if maker is None: # not found tmpLog.error('plugin for {0} not found'.format( queue_config.queueName)) return [], jobchunk_list
if len(sys.argv) > 3: end_job_id = int(sys.argv[3]) queueConfigMapper = QueueConfigMapper() queueConfig = queueConfigMapper.get_queue(queueName) initial_queueConfig_stager = queueConfig.stager queueConfig.stager['module'] = 'pandaharvester.harvesterstager.go_bulk_stager' queueConfig.stager['name'] = 'GlobusBulkStager' modified_queueConfig_stager = queueConfig.stager pluginFactory = PluginFactory() # get stage-out plugin stagerCore = pluginFactory.get_plugin(queueConfig.stager) # logger _logger = core_utils.setup_logger('stageOutTest_go_bulk_stager') tmpLog = core_utils.make_logger(_logger, method_name='stageOutTest_go_bulk_stager') tmpLog.debug('start') for loggerName, loggerObj in logging.Logger.manager.loggerDict.iteritems(): #print "loggerName - {}".format(loggerName) if loggerName.startswith('panda.log'): if len(loggerObj.handlers) == 0: continue if loggerName.split('.')[-1] in ['db_proxy']: continue stdoutHandler = logging.StreamHandler(sys.stdout) stdoutHandler.setFormatter(loggerObj.handlers[0].formatter) loggerObj.addHandler(stdoutHandler)
import random from concurrent.futures import ThreadPoolExecutor import re from math import sqrt, log1p from pandaharvester.harvesterconfig import harvester_config from pandaharvester.harvestercore.queue_config_mapper import QueueConfigMapper from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.plugin_base import PluginBase from pandaharvester.harvestermisc.info_utils import PandaQueuesDict from pandaharvester.harvestermisc.htcondor_utils import get_job_id_tuple_from_batchid from pandaharvester.harvestermisc.htcondor_utils import CondorJobSubmit # logger baseLogger = core_utils.setup_logger('htcondor_submitter') # Integer division round up def _div_round_up(a, b): return a // b + int(a % b > 0) # Compute weight of each CE according to worker stat, return tuple(dict, total weight score) def _get_ce_weighting(ce_endpoint_list=[], worker_ce_all_tuple=None): multiplier = 1000. n_ce = len(ce_endpoint_list) worker_limits_dict, worker_ce_stats_dict, worker_ce_backend_throughput_dict, time_window, n_new_workers = worker_ce_all_tuple N = float(n_ce) Q = float(worker_limits_dict['nQueueLimitWorker']) W = float(worker_limits_dict['maxWorkers'])
import json import os import os.path import threading from queue import Queue from http.server import HTTPServer, BaseHTTPRequestHandler from socketserver import ThreadingMixIn from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.db_proxy_pool import DBProxyPool as DBProxy from pandaharvester.harvesterconfig import harvester_config from pandaharvester.harvestermessenger import shared_file_messenger # logger _logger = core_utils.setup_logger('http_server_messenger') shared_file_messenger.set_logger(_logger) def set_logger(master_logger): global _logger _logger = master_logger shared_file_messenger.set_logger(master_logger) # handler for http front-end class HttpHandler(BaseHTTPRequestHandler): def __init__(self, *args, **kwargs): self.dbProxy = DBProxy() self.tmpLog = None BaseHTTPRequestHandler.__init__(self, *args, **kwargs)
import re import subprocess import json import os.path from pprint import pprint from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.work_spec import WorkSpec from pandaharvester.harvestercore.plugin_base import PluginBase # logger baseLogger = core_utils.setup_logger('cobalt_monitor') # qstat output # JobID User WallTime Nodes State Location # =================================================== # 77734 fcurtis 06:00:00 64 queued None # monitor for HTCONDOR batch system class CobaltMonitor(PluginBase): # constructor def __init__(self, **kwarg): PluginBase.__init__(self, **kwarg) # check workers def check_workers(self, workspec_list): retList = [] for workSpec in workspec_list: # print "pprint(dir(workSpec))" # pprint(dir(workSpec))
import os import sys import shutil import os.path import uuid from future.utils import iteritems from pandaharvester.harvestercore import core_utils from .base_stager import BaseStager from pandaharvester.harvestermover import mover_utils from rucio.client import Client as RucioClient from rucio.common.exception import RuleNotFound # logger baseLogger = core_utils.setup_logger('rucio_stager') # plugin for stage-out with Rucio class RucioStager(BaseStager): # constructor def __init__(self, **kwarg): BaseStager.__init__(self, **kwarg) if not hasattr(self, 'scopeForTmp'): self.scopeForTmp = 'panda' # check status def check_status(self, jobspec): # make logger tmpLog = self.make_logger(baseLogger, 'PandaID={0}'.format(jobspec.PandaID),
import arc import json import urllib from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.plugin_base import PluginBase from pandaharvester.harvesterconfig import harvester_config from act.common.aCTConfig import aCTConfigARC from act.common.aCTProxy import aCTProxy from act.atlas.aCTDBPanda import aCTDBPanda # logger baseLogger = core_utils.setup_logger('act_submitter') # submitter for aCT class ACTSubmitter(PluginBase): # constructor def __init__(self, **kwarg): PluginBase.__init__(self, **kwarg) # Set up aCT DB connection self.log = core_utils.make_logger(baseLogger, 'aCT submitter', method_name='__init__') self.conf = aCTConfigARC() self.actDB = aCTDBPanda(self.log, self.conf.get(["db", "file"])) # Credential dictionary role: proxy file self.certs = dict( zip([
from pandaharvester.harvestercore import core_utils from .base_stager import BaseStager # logger baseLogger = core_utils.setup_logger('rse_direct_stager') # stager plugin with RSE + no data motion class RseDirectStager(BaseStager): """In the workflow for RseDirectStager, workers directly upload output files to RSE and thus there is no data motion in Harvester.""" # constructor def __init__(self, **kwarg): BaseStager.__init__(self, **kwarg) # check status def check_status(self, jobspec): for fileSpec in jobspec.get_output_file_specs(skip_done=True): fileSpec.status = 'finished' return True, '' # trigger stage out def trigger_stage_out(self, jobspec): return True, '' # zip output files def zip_output(self, jobspec): # make logger tmpLog = self.make_logger(baseLogger, 'PandaID={0}'.format(jobspec.PandaID),
import copy from future.utils import iteritems from pandaharvester.harvesterconfig import harvester_config from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.db_proxy_pool import DBProxyPool as DBProxy from pandaharvester.harvestercore.plugin_factory import PluginFactory from pandaharvester.harvestermisc.apfmon import Apfmon # logger _logger = core_utils.setup_logger('worker_adjuster') # class to define number of workers to submit class WorkerAdjuster(object): # constructor def __init__(self, queue_config_mapper): self.queueConfigMapper = queue_config_mapper self.pluginFactory = PluginFactory() self.dbProxy = DBProxy() self.throttlerMap = dict() self.apf_mon = Apfmon(self.queueConfigMapper) try: self.maxNewWorkers = harvester_config.submitter.maxNewWorkers except AttributeError: self.maxNewWorkers = None # define number of workers to submit based on various information def define_num_workers(self, static_num_workers, site_name): tmpLog = core_utils.make_logger(_logger, 'site={0}'.format(site_name),
try: import subprocess32 as subprocess except: import subprocess from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.plugin_base import PluginBase # logger _logger = core_utils.setup_logger('grid_cred_manager') # credential manager using grid-proxy class GridCredManager(PluginBase): # constructor def __init__(self, **kwarg): PluginBase.__init__(self, **kwarg) # check proxy def check_credential(self): # make logger mainLog = self.make_logger(_logger, method_name='check_credential') comStr = "grid-proxy-info -exists -hours 72 -file {0}".format( self.outCertFile) mainLog.debug(comStr) try: p = subprocess.Popen(comStr.split(), shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdOut, stdErr = p.communicate()
from pandaharvester.harvestercore.fifos import SpecialFIFOBase # condor python or command api try: import htcondor except ImportError: CONDOR_API = 'command' else: CONDOR_API = 'python' #=============================================================== #=== Definitions =============================================== # logger baseLogger = core_utils.setup_logger('htcondor_utils') # module level lock moduleLock = threading.Lock() # List of job ads required CONDOR_JOB_ADS_LIST = [ 'ClusterId', 'ProcId', 'JobStatus', 'LastJobStatus', 'JobStartDate', 'EnteredCurrentStatus', 'ExitCode', 'HoldReason', 'LastHoldReason',
from pandaharvester.harvestercore import core_utils from .base_stager import BaseStager from pandaharvester.harvestercore.queue_config_mapper import QueueConfigMapper from pandaharvester.harvestercore.db_proxy_pool import DBProxyPool as DBProxy import uuid import shutil import os # logger baseLogger = core_utils.setup_logger('cp_compass_stager_hpc') class cpCompasStagerHPC(BaseStager): # constructor def __init__(self, **kwarg): BaseStager.__init__(self, **kwarg) self.queue_config_mapper = QueueConfigMapper() # check status def check_stage_out_status(self, jobspec): """Check the status of stage-out procedure. If staging-out is done synchronously in trigger_stage_out this method should always return True. Output files are available through jobspec.get_outfile_specs(skip_done=False) which gives a list of FileSpecs not yet done. FileSpec.attemptNr shows how many times the transfer was checked for the file. If the file was successfully transferred, status should be set to 'finished'. Or 'failed', if the file failed to be transferred. Once files are set to 'finished' or 'failed', jobspec.get_outfile_specs(skip_done=False) ignores them. :param jobspec: job specifications
from pandaharvester.harvestercore.fifos import SpecialFIFOBase # condor python or command api try: import htcondor except ImportError: CONDOR_API = 'command' else: CONDOR_API = 'python' #=============================================================== #=== Definitions =============================================== # logger baseLogger = core_utils.setup_logger('htcondor_utils') # module level lock moduleLock = threading.Lock() # List of job ads required CONDOR_JOB_ADS_LIST = [ 'ClusterId', 'ProcId', 'JobStatus', 'LastJobStatus', 'JobStartDate', 'EnteredCurrentStatus', 'ExitCode', 'HoldReason', 'LastHoldReason', 'RemoveReason', 'harvesterWorkerID', ]
from thread import get_ident from pandaharvester.harvesterconfig import harvester_config from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.plugin_factory import PluginFactory from pandaharvester.harvestercore.db_proxy_pool import DBProxyPool as DBProxy from pandaharvester.harvestercore.db_interface import DBInterface # attribute list _attribute_list = ['id', 'item', 'score'] # fifo object spec FifoObject = collections.namedtuple('FifoObject', _attribute_list, verbose=False, rename=False) # logger _logger = core_utils.setup_logger('fifos') # base class of fifo message queue class FIFOBase(object): # constructor def __init__(self, **kwarg): for tmpKey, tmpVal in iteritems(kwarg): setattr(self, tmpKey, tmpVal) self.hostname = socket.gethostname() self.os_pid = os.getpid() self.dbProxy = DBProxy() self.dbInterface = DBInterface() # get process identifier def get_pid(self):
xmlOutputsBaseFileName = harvester_config.payload_interaction.eventStatusDumpXmlFile # json for event request jsonEventsRequestFileName = harvester_config.payload_interaction.eventRequestFile # json to feed events jsonEventsFeedFileName = harvester_config.payload_interaction.eventRangesFile # json to update events jsonEventsUpdateFileName = harvester_config.payload_interaction.updateEventsFile # suffix to read json suffixReadJson = '.read' # logger baselogger = core_utils.setup_logger() class ARCMessenger(BaseMessenger): '''Mechanism for passing information about completed jobs back to harvester.''' def __init__(self, **kwarg): self.jobSpecFileFormat = 'json' BaseMessenger.__init__(self, **kwarg) self.schedulerid = harvester_config.master.harvester_id self.tmpdir = '/tmp' # TODO configurable or common function # Credential dictionary role: proxy file self.certs = dict( zip([ r.split('=')[1] for r in list(harvester_config.credmanager.voms)
import tempfile try: import subprocess32 as subprocess except: import subprocess import os import stat from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.plugin_base import PluginBase # logger baseLogger = core_utils.setup_logger('cobalt_submitter') # submitter for Cobalt batch system class CobaltSubmitter(PluginBase): # constructor def __init__(self, **kwarg): self.uploadLog = False self.logBaseURL = None PluginBase.__init__(self, **kwarg) # template for batch script tmpFile = open(self.templateFile) self.template = tmpFile.read() tmpFile.close() # submit workers def submit_workers(self, workspec_list): retList = [] retStrList = []
from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.work_spec import WorkSpec from pandaharvester.harvestercore.plugin_base import PluginBase from pandaharvester.harvesterconfig import harvester_config # json for job report jsonJobReport = harvester_config.payload_interaction.jobReportFile # json for outputs jsonOutputsFileName = harvester_config.payload_interaction.eventStatusDumpJsonFile # xml for outputs xmlOutputsBaseFileName = harvester_config.payload_interaction.eventStatusDumpXmlFile # logger baseLogger = core_utils.setup_logger('act_messenger') class ACTMessenger(PluginBase): '''Mechanism for passing information about completed jobs back to harvester.''' def __init__(self, **kwarg): PluginBase.__init__(self, **kwarg) # get access point def get_access_point(self, workspec, panda_id): if workspec.mapType == WorkSpec.MT_MultiJobs: accessPoint = os.path.join(workspec.get_access_point(), str(panda_id)) else: accessPoint = workspec.get_access_point() return accessPoint
import re try: import subprocess32 as subprocess except: import subprocess from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.work_spec import WorkSpec from pandaharvester.harvestercore.plugin_base import PluginBase # logger baseLogger = core_utils.setup_logger('slurm_monitor') # monitor for SLURM batch system class SlurmMonitor(PluginBase): # constructor def __init__(self, **kwarg): PluginBase.__init__(self, **kwarg) # check workers def check_workers(self, workspec_list): retList = [] for workSpec in workspec_list: # make logger tmpLog = self.make_logger(baseLogger, 'workerID={0}'.format(workSpec.workerID), method_name='check_workers') # command comStr = "sacct --jobs={0}".format(workSpec.batchID) # check tmpLog.debug('check with {0}'.format(comStr))
from __future__ import division import math import random from pandaharvester.harvestercore.work_spec import WorkSpec from pandaharvester.harvestercore.job_spec import JobSpec from pandaharvester.harvestercore import core_utils from pandaharvester.harvestermisc.info_utils import PandaQueuesDict from pandaharvester.harvestercore.resource_type_mapper import ResourceTypeMapper from .base_worker_maker import BaseWorkerMaker import datetime # logger _logger = core_utils.setup_logger('simple_worker_maker') # simple maker class SimpleWorkerMaker(BaseWorkerMaker): # constructor def __init__(self, **kwarg): self.jobAttributesToUse = ['nCore', 'minRamCount', 'maxDiskCount', 'maxWalltime', 'ioIntensity'] BaseWorkerMaker.__init__(self, **kwarg) self.rt_mapper = ResourceTypeMapper() def get_job_core_and_memory(self, queue_dict, job_spec): job_memory = job_spec.jobParams.get('minRamCount', 0) or 0 job_corecount = job_spec.jobParams.get('coreCount', 1) or 1 unified_queue = queue_dict.get('capability', '') == 'ucore'
import os import json from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.work_spec import WorkSpec from pandaharvester.harvestercore.plugin_base import PluginBase from pandaharvester.harvesterconfig import harvester_config from act.common.aCTConfig import aCTConfigARC from act.atlas.aCTDBPanda import aCTDBPanda # json for job report jsonJobReport = harvester_config.payload_interaction.jobReportFile # logger baseLogger = core_utils.setup_logger('act_monitor') # monitor for aCT plugin class ACTMonitor(PluginBase): # constructor def __init__(self, **kwarg): PluginBase.__init__(self, **kwarg) # Set up aCT DB connection self.log = core_utils.make_logger(baseLogger, 'aCT submitter', method_name='__init__') self.actDB = aCTDBPanda(self.log) # get access point def get_access_point(self, workspec, panda_id): if workspec.mapType == WorkSpec.MT_MultiJobs:
from future.utils import iteritems try: import pprofile except Exception: pass from pandalogger import logger_config from pandaharvester import commit_timestamp from pandaharvester import panda_pkg_info from pandaharvester.harvesterconfig import harvester_config from pandaharvester.harvestercore import core_utils from pandaharvester.harvestermisc.apfmon import Apfmon # logger _logger = core_utils.setup_logger('master') # for singleton master_instance = False master_lock = threading.Lock() # the master class which runs the main process class Master(object): # constructor def __init__(self, single_mode=False, stop_event=None, daemon_mode=True): # initialize database and config self.singleMode = single_mode self.stopEvent = stop_event self.daemonMode = daemon_mode
import uuid import os from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.plugin_base import PluginBase from pandaharvester.harvestercore.work_spec import WorkSpec # setup base logger baseLogger = core_utils.setup_logger('dummy_submitter') # dummy submitter class DummySubmitter(PluginBase): # constructor def __init__(self, **kwarg): self.logBaseURL = 'http://localhost/test' PluginBase.__init__(self, **kwarg) # submit workers def submit_workers(self, workspec_list): """Submit workers to a scheduling system like batch systems and computing elements. This method takes a list of WorkSpecs as input argument, and returns a list of tuples. Each tuple is composed of a return code and a dialog message. The return code could be True (for success), False (for permanent failures), or None (for temporary failures). If the return code is None, submission is retried maxSubmissionAttempts times at most which is defined for each queue in queue_config.json. Nth tuple in the returned list corresponds to submission status and dialog message for Nth worker in the given WorkSpec list. A unique identifier is set to WorkSpec.batchID when submission is successful, so that they can be identified in the scheduling system. It would be useful to set other attributes like queueName (batch queue name), computingElement (CE's host name), and nodeID (identifier of the node where the worker is running).
import time import datetime from pandaharvester.harvesterconfig import harvester_config from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.db_proxy_pool import DBProxyPool as DBProxy from pandaharvester.harvestercore.command_spec import CommandSpec from pandaharvester.harvesterbody.agent_base import AgentBase from pandaharvester.harvestercore.pilot_errors import PilotErrors # logger _logger = core_utils.setup_logger('propagator') STATS_PERIOD = 300 METRICS_PERIOD = 300 # propagate important checkpoints to panda class Propagator(AgentBase): # constructor def __init__(self, communicator, queue_config_mapper, single_mode=False): AgentBase.__init__(self, single_mode) self.dbProxy = DBProxy() self.communicator = communicator self.queueConfigMapper = queue_config_mapper self._last_stats_update = None self._last_metrics_update = None # main loop def run(self): while True: sw_main = core_utils.get_stopwatch()
""" Based on: https://cloud.google.com/compute/docs/tutorials/python-guide#before-you-begin """ import time from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.plugin_base import PluginBase # from requests.exceptions import SSLError from pandaharvester.harvestercloud.googlecloud import compute, GoogleVM, ZONE, PROJECT from pandaharvester.harvestercore.queue_config_mapper import QueueConfigMapper # setup base logger base_logger = core_utils.setup_logger('google_submitter') def wait_for_operation(project, zone, operation_name): """ Waits for an operation to complete. TODO: decide whether we want to block or just move on and list the instance status later :param project: :param zone: :param operation_name: :return: """ tmp_log = core_utils.make_logger(base_logger, method_name='wait_for_operation') tmp_log.debug('Waiting for operation to finish...') while True: result = compute.zoneOperations().get(
import os.path from pandaharvester.harvestercore.work_spec import WorkSpec from pandaharvester.harvestercore.plugin_base import PluginBase from pandaharvester.harvestercore import core_utils # logger baseLogger = core_utils.setup_logger('dummy_monitor') # dummy monitor class DummyMonitor(PluginBase): # constructor def __init__(self, **kwarg): PluginBase.__init__(self, **kwarg) # check workers def check_workers(self, workspec_list): """Check status of workers. This method takes a list of WorkSpecs as input argument and returns a list of worker's statuses. Nth element if the return list corresponds to the status of Nth WorkSpec in the given list. Worker's status is one of WorkSpec.ST_finished, WorkSpec.ST_failed, WorkSpec.ST_cancelled, WorkSpec.ST_running, WorkSpec.ST_submitted. nativeExitCode and nativeStatus of WorkSpec can be arbitrary strings to help understanding behaviour of the resource and/or batch scheduler. :param workspec_list: a list of work specs instances :return: A tuple of return code (True for success, False otherwise) and a list of worker's statuses. :rtype: (bool, [string,]) """ retList = [] for workSpec in workspec_list: # make logger
import os import shutil try: import subprocess32 as subprocess except ImportError: import subprocess from pandaharvester.harvestercore import core_utils from pandaharvester.harvestersweeper.base_sweeper import BaseSweeper # logger baseLogger = core_utils.setup_logger('slurm_sweeper') # plugin for sweeper with SLURM class SlurmSweeper(BaseSweeper): # constructor def __init__(self, **kwarg): BaseSweeper.__init__(self, **kwarg) # kill a worker def kill_worker(self, workspec): """Kill a worker in a scheduling system like batch systems and computing elements. :param workspec: worker specification :type workspec: WorkSpec :return: A tuple of return code (True for success, False otherwise) and error dialog :rtype: (bool, string) """ # make logger tmpLog = self.make_logger(baseLogger,
import threading import uuid import socket import six import pexpect from pandaharvester.harvestercore import core_utils if six.PY2: pexpect_spawn = pexpect.spawn else: pexpect_spawn = pexpect.spawnu # logger baseLogger = core_utils.setup_logger('ssh_tunnel_pool') # Pool of SSH tunnels class SshTunnelPool(object): # constructor def __init__(self): self.lock = threading.Lock() self.pool = dict() self.params = dict() # make a dict key def make_dict_key(self, host, port): return '{0}:{1}'.format(host, port)
import time import datetime from pandaharvester.harvesterconfig import harvester_config from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.db_proxy_pool import DBProxyPool as DBProxy from pandaharvester.harvestercore.command_spec import CommandSpec from pandaharvester.harvesterbody.agent_base import AgentBase from pandaharvester.harvestercore.pilot_errors import PilotErrors # logger _logger = core_utils.setup_logger('propagator') STATS_PERIOD = 300 METRICS_PERIOD = 300 # propagate important checkpoints to panda class Propagator(AgentBase): # constructor def __init__(self, communicator, queue_config_mapper, single_mode=False): AgentBase.__init__(self, single_mode) self.dbProxy = DBProxy() self.communicator = communicator self.queueConfigMapper = queue_config_mapper self._last_stats_update = None self._last_metrics_update = None # main loop def run(self): while True:
import socket import datetime from future.utils import iteritems from pandaharvester.harvesterconfig import harvester_config from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.job_spec import JobSpec from pandaharvester.harvestercore.file_spec import FileSpec from pandaharvester.harvestercore.db_proxy_pool import DBProxyPool as DBProxy from pandaharvester.harvesterbody.agent_base import AgentBase # logger _logger = core_utils.setup_logger('job_fetcher') # class to fetch jobs class JobFetcher(AgentBase): # constructor def __init__(self, communicator, queue_config_mapper, single_mode=False): AgentBase.__init__(self, single_mode) self.dbProxy = DBProxy() self.communicator = communicator self.nodeName = socket.gethostname() self.queueConfigMapper = queue_config_mapper # main loop def run(self): while True: mainLog = self.make_logger(_logger, 'id={0}'.format(self.get_pid()), method_name='run') mainLog.debug('getting number of jobs to be fetched') # get number of jobs to be fetched
import tempfile try: import subprocess32 as subprocess except: import subprocess from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.plugin_base import PluginBase # logger baseLogger = core_utils.setup_logger('slurm_submitter') # submitter for SLURM batch system class SlurmSubmitter(PluginBase): # constructor def __init__(self, **kwarg): self.uploadLog = False self.logBaseURL = None PluginBase.__init__(self, **kwarg) # template for batch script tmpFile = open(self.templateFile) self.template = tmpFile.read() tmpFile.close() # submit workers def submit_workers(self, workspec_list): retList = [] retStrList = [] for workSpec in workspec_list: # make logger