from pandaharvester.harvestercore import core_utils
from pandaharvester.harvestercore.plugin_base import PluginBase
from pandaharvester.harvestercloud.googlecloud import compute, ZONE, PROJECT
from pandaharvester.harvestercore.queue_config_mapper import QueueConfigMapper
import googleapiclient
base_logger = core_utils.setup_logger('google_sweeper')
class GoogleSweeper(PluginBase):
"""
Sweeper with kill/clean-up functions for Google Compute Engine
"""
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
self.queue_config_mapper = QueueConfigMapper()
def kill_worker(self, work_spec):
"""
Sends the command to Google to destroy a VM
:param work_spec: worker specification
:type work_spec: WorkSpec
:return: A tuple of return code (True for success, False otherwise) and error dialog
:rtype: (bool, string)
"""
try:
vm_name = work_spec.batchID
queue_config = self.queue_config_mapper.get_queue(work_spec.computingSite)
try:
if len(sys.argv) > 4:
globus_sleep_time = int(sys.argv[4])
queueConfigMapper = QueueConfigMapper()
queueConfig = queueConfigMapper.get_queue(queueName)
initial_queueConfig_preparator = queueConfig.preparator
queueConfig.preparator['module'] = 'pandaharvester.harvesterpreparator.go_bulk_preparator'
queueConfig.preparator['name'] = 'GlobusBulkPreparator'
modified_queueConfig_preparator = queueConfig.preparator
pluginFactory = PluginFactory()
# get stage-out plugin
preparatorCore = pluginFactory.get_plugin(queueConfig.preparator)
# logger
_logger = core_utils.setup_logger('stageInTest_go_bulk_preparator')
tmpLog = core_utils.make_logger(_logger, method_name='stageInTest_go_bulk_preparator')
tmpLog.debug('start')
for loggerName, loggerObj in logging.Logger.manager.loggerDict.iteritems():
#print "loggerName - {}".format(loggerName)
if loggerName.startswith('panda.log'):
if len(loggerObj.handlers) == 0:
continue
if loggerName.split('.')[-1] in ['db_proxy']:
continue
stdoutHandler = logging.StreamHandler(sys.stdout)
stdoutHandler.setFormatter(loggerObj.handlers[0].formatter)
loggerObj.addHandler(stdoutHandler)
msgStr = "plugin={0}".format(preparatorCore.__class__.__name__)
from pandaharvester.harvestercore import core_utils
from pandaharvester.harvestercore.plugin_base import PluginBase
from pandaharvester.harvestercore.work_spec import WorkSpec as ws
try:
import subprocess32 as subprocess
except Exception:
import subprocess
import datetime
# logger
baseLogger = core_utils.setup_logger('titan_utils')
class TitanUtils(PluginBase):
# constructor
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
tmpLog = self.make_logger(baseLogger, method_name='__init__')
tmpLog.info("Titan utils initiated")
def get_batchjob_info(self, batchid):
"""
Collect job info from scheduler
:param batchid:
:return res - dictonary with job state and some timing:
"""
"""
:param batchid:
:return:
import subprocess32 as subprocess
except Exception:
import subprocess
from email.mime.text import MIMEText
from pandaharvester.harvesterconfig import harvester_config
from pandaharvester.harvestercore import core_utils
from pandaharvester.harvesterbody.agent_base import AgentBase
from pandalogger import logger_config
logDir = logger_config.daemon['logdir']
lockFileName = os.path.join(logDir, 'watcher.lock')
# logger
_logger = core_utils.setup_logger('watcher')
# watching the system
class Watcher(AgentBase):
# constructor
def __init__(self, single_mode=False):
AgentBase.__init__(self, single_mode)
self.startTime = datetime.datetime.utcnow()
# main loop
def run(self):
while True:
# execute
self.execute()
# check if being terminated
import datetime
import uuid
from pandaharvester.harvestercore.plugin_base import PluginBase
from pandaharvester.harvestercore import core_utils
# dummy transfer identifier
dummy_transfer_id = 'dummy_id_for_out'
# logger
baseLogger = core_utils.setup_logger('dummy_bulk_stager')
# dummy plugin for stager with bulk transfers. For JobSpec and DBInterface methods, see
# https://github.com/PanDAWMS/panda-harvester/wiki/Utilities#file-grouping-for-file-transfers
class DummyBulkStager(PluginBase):
# constructor
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
# check status
def check_status(self, jobspec):
# make logger
tmpLog = core_utils.make_logger(baseLogger,
'PandaID={0}'.format(jobspec.PandaID),
method_name='check_status')
tmpLog.debug('start')
# get transfer groups
groups = jobspec.get_groups_of_output_files()
# lock if the dummy transfer ID is used to avoid submitting duplicated transfer requests
if dummy_transfer_id in groups:
from pandaharvester.harvesterconfig import harvester_config
from pandaharvester.harvestermover import mover_utils
from pandaharvester.harvestercore.queue_config_mapper import QueueConfigMapper
from pandaharvester.harvesterstager.base_stager import BaseStager
from .base_stager import BaseStager
def dump(tmpLog,obj):
for attr in dir(obj):
if hasattr( obj, attr ):
tmpLog.debug( "obj.%s = %s" % (attr, getattr(obj, attr)))
# logger
baseLogger = core_utils.setup_logger('yoda_rucio_rse_direct_stager')
class Error(EnvironmentError):
pass
class SpecialFileError(EnvironmentError):
"""Raised when trying to do a kind of operation (e.g. copying) which is
not supported on a special file (e.g. a named pipe)"""
class ExecError(EnvironmentError):
"""Raised when a command could not be executed"""
# stager plugin with RSE + local site move behaviour for Yoda zip files
class YodaRucioRseDirectStager(BaseStager):
from pandaharvester.harvesterconfig import harvester_config
from pandaharvester.harvestermover import mover_utils
from pandaharvester.harvestercore.queue_config_mapper import QueueConfigMapper
from pandaharvester.harvesterstager.base_stager import BaseStager
from .base_stager import BaseStager
def dump(tmpLog,obj):
for attr in dir(obj):
if hasattr( obj, attr ):
tmpLog.debug( "obj.%s = %s" % (attr, getattr(obj, attr)))
# logger
baseLogger = core_utils.setup_logger('yoda_rucio_rse_direct_stager')
class Error(EnvironmentError):
pass
class SpecialFileError(EnvironmentError):
"""Raised when trying to do a kind of operation (e.g. copying) which is
not supported on a special file (e.g. a named pipe)"""
class ExecError(EnvironmentError):
"""Raised when a command could not be executed"""
# stager plugin with RSE + local site move behaviour for Yoda zip files
class YodaRucioRseDirectStager(BaseStager):
import datetime
from pandaharvester.harvestercore import core_utils
from pandaharvester.harvestercore.db_proxy_pool import DBProxyPool as DBProxy
from pandaharvester.harvestercore.plugin_base import PluginBase
# logger
baseLogger = core_utils.setup_logger('simple_throttler')
# simple throttler
class SimpleThrottler(PluginBase):
# constructor
def __init__(self, **kwarg):
# logic type : AND: throttled if all rules are satisfied, OR: throttled if one rule is satisfied
self.logicType = 'OR'
PluginBase.__init__(self, **kwarg)
self.dbProxy = DBProxy()
# check if to be throttled
def to_be_throttled(self, queue_config):
tmpLog = self.make_logger(baseLogger, 'computingSite={0}'.format(queue_config.queueName),
method_name='to_be_throttled')
tmpLog.debug('start')
# set default return vale
if self.logicType == 'OR':
retVal = False, "no rule was satisfied"
else:
retVal = True, "all rules were satisfied"
# loop over all rules
criteriaList = []
import os
import shutil
from pandaharvester.harvestercore import core_utils
from pandaharvester.harvestercore.plugin_base import PluginBase
from act.common.aCTConfig import aCTConfigARC
from act.atlas.aCTDBPanda import aCTDBPanda
# logger
baseLogger = core_utils.setup_logger('act_sweeper')
# plugin for aCT sweeper
class ACTSweeper(PluginBase):
# constructor
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
self.log = core_utils.make_logger(baseLogger, 'aCT sweeper', method_name='__init__')
self.actDB = aCTDBPanda(self.log)
# kill a worker
def kill_worker(self, workspec):
""" Mark aCT job as tobekilled.
:param workspec: worker specification
:type workspec: WorkSpec
:return: A tuple of return code (True for success, False otherwise) and error dialog
:rtype: (bool, string)
import os.path
import os
from future.utils import iteritems
from pandaharvester.harvestercore import core_utils
from pandaharvester.harvestercore.plugin_base import PluginBase
from pandaharvester.harvestermover import mover_utils
from pilot.api import data
# logger
baseLogger = core_utils.setup_logger('pilotmover_preparator')
# plugin for preparator based on Pilot2.0 Data API
# Pilot 2.0 should be deployed as library
# default self.basePath came from preparator section of configuration file
class PilotmoverPreparator(PluginBase):
"""
Praparator bring files from remote ATLAS/Rucio storage to local facility.
"""
# constructor
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
# check status
def check_status(self, jobspec):
return True, ''
#if len(sys.argv) > 4:
# globus_sleep_time = int(sys.argv[4])
queueConfigMapper = QueueConfigMapper()
queueConfig = queueConfigMapper.get_queue(queueName)
initial_queueConfig_preparator = queueConfig.preparator
queueConfig.preparator['module'] = 'pandaharvester.harvesterpreparator.go_bulk_preparator'
queueConfig.preparator['name'] = 'GlobusBulkPreparator'
modified_queueConfig_preparator = queueConfig.preparator
pluginFactory = PluginFactory()
# get stage-out plugin
preparatorCore = pluginFactory.get_plugin(queueConfig.preparator)
# logger
_logger = core_utils.setup_logger('further_testing_go_bulk_preparator')
tmpLog = core_utils.make_logger(_logger, method_name='further_testing_go_bulk_preparator')
tmpLog.debug('start')
for loggerName, loggerObj in logging.Logger.manager.loggerDict.iteritems():
#print "loggerName - {}".format(loggerName)
if loggerName.startswith('panda.log'):
if len(loggerObj.handlers) == 0:
continue
if loggerName.split('.')[-1] in ['db_proxy']:
continue
stdoutHandler = logging.StreamHandler(sys.stdout)
stdoutHandler.setFormatter(loggerObj.handlers[0].formatter)
loggerObj.addHandler(stdoutHandler)
msgStr = "plugin={0}".format(preparatorCore.__class__.__name__)
import tempfile
try:
import subprocess32 as subprocess
except:
import subprocess
import os
import stat
from pandaharvester.harvestercore import core_utils
from pandaharvester.harvestercore.plugin_base import PluginBase
# logger
baseLogger = core_utils.setup_logger('cobalt_submitter')
# submitter for Cobalt batch system
class CobaltSubmitter(PluginBase):
# constructor
def __init__(self, **kwarg):
self.uploadLog = False
self.logBaseURL = None
PluginBase.__init__(self, **kwarg)
# template for batch script
tmpFile = open(self.templateFile)
self.template = tmpFile.read()
tmpFile.close()
# submit workers
def submit_workers(self, workspec_list):
retList = []
retStrList = []
except:
import subprocess
import random
import uuid
from concurrent.futures import ThreadPoolExecutor
import re
from pandaharvester.harvestercore import core_utils
from pandaharvester.harvestercore.plugin_base import PluginBase
from pandaharvester.harvestermisc.cloud_openstack_utils import OS_SimpleClient
# setup base logger
baseLogger = core_utils.setup_logger('cloud_openstack_submitter')
def _init_script_replace(string, **kwarg):
new_string = string
macro_map = {
'\$\(workerID\)': str(kwarg['workerID']),
'\$\(batchID\)': str(kwarg['batchID']),
'\$\(accessPoint\)': str(kwarg['accessPoint']),
}
for k, v in macro_map.items():
new_string = re.sub(k, v, new_string)
return new_string
# make cloud initialization script
import six
from concurrent.futures import ThreadPoolExecutor as Pool
from pandaharvester.harvestercore import core_utils
from pandaharvester.harvesterconfig import harvester_config
from pandaharvester.harvestercore.work_spec import WorkSpec
from pandaharvester.harvestercore.worker_errors import WorkerErrors
from pandaharvester.harvestercore.plugin_base import PluginBase
from pandaharvester.harvestermisc.htcondor_utils import condor_job_id_from_workspec, get_host_batchid_map
from pandaharvester.harvestermisc.htcondor_utils import CondorJobQuery, CondorJobManage
# logger
baseLogger = core_utils.setup_logger('htcondor_monitor')
# Native HTCondor job status map
CONDOR_JOB_STATUS_MAP = {
'1': 'idle',
'2': 'running',
'3': 'removed',
'4': 'completed',
'5': 'held',
'6': 'transferring_output',
'7': 'suspended',
}
# Check one worker
import json
import os
from pandaharvester.harvestercore import core_utils
from pandaharvester.harvestercore.work_spec import WorkSpec
from pandaharvester.harvestercore.worker_errors import WorkerErrors
from pandaharvester.harvestercore.file_spec import FileSpec
from pandaharvester.harvesterconfig import harvester_config
from .base_stager import BaseStager
from act.atlas.aCTDBPanda import aCTDBPanda
# logger
baseLogger = core_utils.setup_logger('act_stager')
# json for job report
jsonJobReport = harvester_config.payload_interaction.jobReportFile
# aCT stager plugin
class ACTStager(BaseStager):
# constructor
def __init__(self, **kwarg):
BaseStager.__init__(self, **kwarg)
# Set up aCT DB connection
self.log = core_utils.make_logger(baseLogger, 'aCT stager', method_name='__init__')
try:
self.actDB = aCTDBPanda(self.log)
except Exception as e:
self.log.error('Could not connect to aCT database: {0}'.format(str(e)))
self.actDB = None
import sys
import threading
import traceback
from pandaharvester.harvestercore import core_utils
from pandaharvester.harvestercore.work_spec import WorkSpec
from pandaharvester.harvestercore.plugin_base import PluginBase
from pandaharvester.harvestersubmitter.apfgrid_submitter import APFGridSubmitter
try:
from autopyfactory import condorlib
except ImportError:
logging.error("Unable to import htcondor/condorlib. sys.path=%s" % sys.path)
# setup base logger
baseLogger = core_utils.setup_logger()
class APFGridMonitorSingleton(type):
def __init__(self, *args, **kwargs):
super(APFGridMonitorSingleton, self).__init__(*args, **kwargs)
self.__instance = None
def __call__(self, *args, **kwargs):
if self.__instance is None:
self.__instance = super(APFGridMonitorSingleton, self).__call__(*args, **kwargs)
return self.__instance
class APFGridMonitor(object):
'''
globus_sleep_time = int(sys.argv[3])
queueConfigMapper = QueueConfigMapper()
queueConfig = queueConfigMapper.get_queue(queueName)
initial_queueConfig_preparator = queueConfig.preparator
queueConfig.preparator[
'module'] = 'pandaharvester.harvesterpreparator.go_preparator'
queueConfig.preparator['name'] = 'GoPreparator'
modified_queueConfig_preparator = queueConfig.preparator
pluginFactory = PluginFactory()
# get stage-out plugin
preparatorCore = pluginFactory.get_plugin(queueConfig.preparator)
# logger
_logger = core_utils.setup_logger('stageInTest_go_preparator')
tmpLog = core_utils.make_logger(_logger,
method_name='stageInTest_go_preparator')
tmpLog.debug('start')
for loggerName, loggerObj in iteritems(logging.Logger.manager.loggerDict):
#print "loggerName - {}".format(loggerName)
if loggerName.startswith('panda.log'):
if len(loggerObj.handlers) == 0:
continue
if loggerName.split('.')[-1] in ['db_proxy']:
continue
stdoutHandler = logging.StreamHandler(sys.stdout)
stdoutHandler.setFormatter(loggerObj.handlers[0].formatter)
loggerObj.addHandler(stdoutHandler)
"""
Base class to communicate with WMS
"""
import abc
from future.utils import with_metaclass
from pandaharvester.harvestercore import core_utils
# logger
_logger = core_utils.setup_logger('communicator')
# base class for communication with WMS
class BaseCommunicator(with_metaclass(abc.ABCMeta, object)):
# constructor
def __init__(self):
pass
# make logger
def make_logger(self, tag=None, method_name=None):
return core_utils.make_logger(_logger, token=tag, method_name=method_name)
# get jobs
def get_jobs(self, site_name, node_name, prod_source_label, computing_element, n_jobs,
additional_criteria):
return [], ''
# update jobs
def update_jobs(self, jobspec_list, id):
import socket
import datetime
from future.utils import iteritems
from pandaharvester.harvesterconfig import harvester_config
from pandaharvester.harvestercore import core_utils
from pandaharvester.harvestercore.db_proxy_pool import DBProxyPool as DBProxy
from pandaharvester.harvesterbody.agent_base import AgentBase
from pandaharvester.harvestercore.command_spec import CommandSpec
from pandaharvester import commit_timestamp
from pandaharvester import panda_pkg_info
# logger
_logger = core_utils.setup_logger('command_manager')
# class to retrieve commands from panda server
class CommandManager(AgentBase):
# constructor
def __init__(self, communicator, queue_config_mapper, single_mode=False):
AgentBase.__init__(self, single_mode)
self.db_proxy = DBProxy()
self.communicator = communicator
self.queueConfigMapper = queue_config_mapper
self.nodeName = socket.gethostname()
self.lastHeartbeat = None
# set single mode
def set_single_mode(self, single_mode):
self.singleMode = single_mode
def convert_to_command_specs(self, commands):
try:
killWorkerFile = harvester_config.payload_interaction.killWorkerFile
except Exception:
killWorkerFile = 'kill_worker.json'
# json for heartbeats from the worker
try:
heartbeatFile = harvester_config.payload_interaction.heartbeatFile
except Exception:
heartbeatFile = 'worker_heartbeat.json'
# suffix to read json
suffixReadJson = '.read'
# logger
_logger = core_utils.setup_logger('shared_file_messenger')
def set_logger(master_logger):
global _logger
_logger = master_logger
# filter for log.tgz
def filter_log_tgz(extra=None):
patt = ['*.log', '*.txt', '*.xml', '*.json', 'log*']
if extra is not None:
patt += extra
return '-o '.join(['-name "{0}" '.format(i) for i in patt])
import re
try:
import subprocess32 as subprocess
except ImportError:
import subprocess
import json
import os
from pandaharvester.harvestercore import core_utils
from pandaharvester.harvestercore.work_spec import WorkSpec
from pandaharvester.harvestercore.plugin_base import PluginBase
# logger
baseLogger = core_utils.setup_logger('slurm_squeue_monitor')
# monitor for SLURM batch system with squeue
class SlurmSqueueMonitor(PluginBase):
_HARVESTER_POSTMORTEM_FILENAME = "FINISHED"
# constructor
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
# check workers
def check_workers(self, workspec_list):
retList = []
for workSpec in workspec_list:
# make logger
tmpLog = self.make_logger(baseLogger,
import os
from pandaharvester.harvestercore import core_utils
from pandaharvester.harvestersweeper.base_sweeper import BaseSweeper
from pandaharvester.harvestermisc.k8s_utils import k8s_Client
# logger
baseLogger = core_utils.setup_logger('k8s_sweeper')
# sweeper for K8S
class K8sSweeper(BaseSweeper):
# constructor
def __init__(self, **kwarg):
BaseSweeper.__init__(self, **kwarg)
self.k8s_client = k8s_Client(namespace=self.k8s_namespace, config_file=self.k8s_config_file)
self._all_pods_list = []
# # kill a worker
# def kill_worker(self, workspec):
# tmpLog = self.make_logger(baseLogger, 'workerID={0}'.format(workspec.workerID),
# method_name='kill_worker')
#
# tmpRetVal = (None, 'Nothing done')
#
# job_id = workspec.batchID
# try:
# self.k8s_client.delete_job(job_id)
import datetime
from pandaharvester.harvesterconfig import harvester_config
from pandaharvester.harvestercore import core_utils
from pandaharvester.harvestercore.db_proxy_pool import DBProxyPool as DBProxy
from pandaharvester.harvestercore.plugin_factory import PluginFactory
from pandaharvester.harvesterbody.agent_base import AgentBase
from pandaharvester.harvestercore.pilot_errors import PilotErrors
from pandaharvester.harvestercore.job_spec import JobSpec
from pandaharvester.harvestercore.file_spec import FileSpec
# logger
_logger = core_utils.setup_logger('preparator')
# class to prepare jobs
class Preparator(AgentBase):
# constructor
def __init__(self, communicator, queue_config_mapper, single_mode=False):
AgentBase.__init__(self, single_mode)
self.dbProxy = DBProxy()
self.communicator = communicator
self.queueConfigMapper = queue_config_mapper
self.pluginFactory = PluginFactory()
# main loop
def run(self):
lockedBy = 'preparator-{0}'.format(self.get_pid())
while True:
sw = core_utils.get_stopwatch()
mainLog = self.make_logger(_logger,
from pandaharvester.harvestercore.work_spec import WorkSpec
from pandaharvester.harvestercore.plugin_base import PluginBase
from pandaharvester.harvestercore import core_utils
from pandaharvester.harvestercore.queue_config_mapper import QueueConfigMapper
from pandaharvester.harvestercloud.googlecloud import compute, ZONE, PROJECT
base_logger = core_utils.setup_logger('google_monitor')
class GoogleMonitor(PluginBase):
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
self.queue_config_mapper = QueueConfigMapper()
# States taken from: https://cloud.google.com/compute/docs/instances/checking-instance-status
self.vm_to_worker_status = {
'RUNNING': WorkSpec.ST_running,
'TERMINATED': WorkSpec.ST_running, # the VM is stopped, but has to be fully deleted
'STOPPING': WorkSpec.ST_finished,
'PROVISIONING': WorkSpec.ST_submitted,
'STAGING': WorkSpec.ST_submitted
}
def list_vms(self, zone):
"""
List the status of the running VMs
:return:
"""
try:
result = compute.instances().list(project=PROJECT, zone=zone).execute()
import os.path
import os
from future.utils import iteritems
from pandaharvester.harvestercore import core_utils
from pandaharvester.harvestercore.plugin_base import PluginBase
from pandaharvester.harvestermover import mover_utils
from pilot.api import data
# logger
baseLogger = core_utils.setup_logger('pilotmover_preparator')
# plugin for preparator based on Pilot2.0 Data API
# Pilot 2.0 should be deployed as library
# default self.basePath came from preparator section of configuration file
class PilotmoverPreparator(PluginBase):
"""
Praparator bring files from remote ATLAS/Rucio storage to local facility.
"""
# constructor
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
# check status
def check_stage_in_status(self, jobspec):
return True, ''
# trigger preparation
import random
from pandaharvester.harvestercore.work_spec import WorkSpec
from pandaharvester.harvestercore import core_utils
from pandaharvester.harvestermisc.info_utils import PandaQueuesDict
from pandaharvester.harvestercore.resource_type_mapper import ResourceTypeMapper
from .base_worker_maker import BaseWorkerMaker
import datetime
# logger
_logger = core_utils.setup_logger('simple_worker_maker')
# simple maker
class SimpleWorkerMaker(BaseWorkerMaker):
# constructor
def __init__(self, **kwarg):
self.jobAttributesToUse = ['nCore', 'minRamCount', 'maxDiskCount', 'maxWalltime', 'ioIntensity']
BaseWorkerMaker.__init__(self, **kwarg)
self.rt_mapper = ResourceTypeMapper()
try:
self.pilotTypeRandomWeightsPermille
except AttributeError:
self.pilotTypeRandomWeightsPermille = {}
finally:
# randomize pilot type with weighting
weight_rc = self.pilotTypeRandomWeightsPermille.get('RC', 0)
weight_alrb = self.pilotTypeRandomWeightsPermille.get('ALRB', 0)
weight_pt = self.pilotTypeRandomWeightsPermille.get('PT', 0)
weight_tmp_sum = weight_rc + weight_alrb + weight_pt
if weight_tmp_sum > 1000:
import json
from pandaharvester.harvestercore import core_utils
from pandaharvester.harvestermessenger import http_server_messenger
from pandaharvester.harvestermisc.frontend_utils import HarvesterToken
# logger
_logger = core_utils.setup_logger('apache_messenger')
http_server_messenger.set_logger(_logger)
# handler
class ApacheHandler(http_server_messenger.HttpHandler):
def __init__(self, *args, **kwargs):
self.responseCode = None
self.form = dict()
self.message = None
self.headerList = [('Content-Type', 'text/plain')]
http_server_messenger.HttpHandler.__init__(self, *args, **kwargs)
def setup(self):
pass
def handle(self):
pass
def finish(self):
pass
def send_response(self, code, message=None):
self.responseCode = code
try:
import subprocess32 as subprocess
except:
import subprocess
from pandaharvester.harvestercore import core_utils
from pandaharvester.harvestercore.plugin_base import PluginBase
# logger
_logger = core_utils.setup_logger('grid_cred_manager')
# credential manager using grid-proxy
class GridCredManager(PluginBase):
# constructor
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
# check proxy
def check_credential(self):
# make logger
mainLog = self.make_logger(_logger, method_name='check_credential')
comStr = "grid-proxy-info -exists -hours 72 -file {0}".format(self.outCertFile)
mainLog.debug(comStr)
try:
p = subprocess.Popen(comStr.split(),
shell=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdOut, stdErr = p.communicate()
retCode = p.returncode
if len(sys.argv) > 3:
end_job_id = int(sys.argv[3])
queueConfigMapper = QueueConfigMapper()
queueConfig = queueConfigMapper.get_queue(queueName)
initial_queueConfig_stager = queueConfig.stager
queueConfig.stager['module'] = 'pandaharvester.harvesterstager.go_bulk_stager'
queueConfig.stager['name'] = 'GlobusBulkStager'
modified_queueConfig_stager = queueConfig.stager
pluginFactory = PluginFactory()
# get stage-out plugin
stagerCore = pluginFactory.get_plugin(queueConfig.stager)
# logger
_logger = core_utils.setup_logger('stageOutTest_go_bulk_stager')
tmpLog = core_utils.make_logger(_logger, method_name='stageOutTest_go_bulk_stager')
tmpLog.debug('start')
for loggerName, loggerObj in logging.Logger.manager.loggerDict.iteritems():
#print "loggerName - {}".format(loggerName)
if loggerName.startswith('panda.log'):
if len(loggerObj.handlers) == 0:
continue
if loggerName.split('.')[-1] in ['db_proxy']:
continue
stdoutHandler = logging.StreamHandler(sys.stdout)
stdoutHandler.setFormatter(loggerObj.handlers[0].formatter)
loggerObj.addHandler(stdoutHandler)
msgStr = "plugin={0}".format(stagerCore.__class__.__name__)
import radical.utils
import saga
import os
import shutil
from pandaharvester.harvestercore.plugin_base import PluginBase
from pandaharvester.harvestercore import core_utils
from pandaharvester.harvestersubmitter.saga_submitter import SAGASubmitter
# logger
baseLogger = core_utils.setup_logger('saga_sweeper')
# dummy plugin for sweeper
class SAGASweeper(PluginBase):
# constructor
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
tmpLog = core_utils.make_logger(baseLogger, method_name='__init__')
tmpLog.info("[{0}] SAGA adaptor will be used".format(self.adaptor))
# kill a worker
def kill_worker(self, workspec):
"""Kill a worker in a scheduling system like batch systems and computing elements.
:param workspec: worker specification
:type workspec: WorkSpec
:return: A tuple of return code (True for success, False otherwise) and error dialog
:rtype: (bool, string)
"""
job_service = saga.job.Service(self.adaptor)
from pandaharvester.harvestercore import core_utils
from pandaharvester.harvestercore.db_proxy_pool import DBProxyPool as DBProxy
from pandaharvester.harvestercore.plugin_factory import PluginFactory
# logger
_logger = core_utils.setup_logger('worker_maker')
# class to make worker
class WorkerMaker:
# constructor
def __init__(self):
self.pluginFactory = PluginFactory()
self.dbProxy = DBProxy()
# make workers
def make_workers(self, jobchunk_list, queue_config, n_ready,
resource_type):
tmpLog = core_utils.make_logger(_logger,
'queue={0}'.format(
queue_config.queueName),
method_name='make_workers')
tmpLog.debug('start')
try:
# get plugin
maker = self.pluginFactory.get_plugin(queue_config.workerMaker)
if maker is None:
# not found
tmpLog.error('plugin for {0} not found'.format(
queue_config.queueName))
return [], jobchunk_list
if len(sys.argv) > 3:
end_job_id = int(sys.argv[3])
queueConfigMapper = QueueConfigMapper()
queueConfig = queueConfigMapper.get_queue(queueName)
initial_queueConfig_stager = queueConfig.stager
queueConfig.stager['module'] = 'pandaharvester.harvesterstager.go_bulk_stager'
queueConfig.stager['name'] = 'GlobusBulkStager'
modified_queueConfig_stager = queueConfig.stager
pluginFactory = PluginFactory()
# get stage-out plugin
stagerCore = pluginFactory.get_plugin(queueConfig.stager)
# logger
_logger = core_utils.setup_logger('stageOutTest_go_bulk_stager')
tmpLog = core_utils.make_logger(_logger,
method_name='stageOutTest_go_bulk_stager')
tmpLog.debug('start')
for loggerName, loggerObj in logging.Logger.manager.loggerDict.iteritems():
#print "loggerName - {}".format(loggerName)
if loggerName.startswith('panda.log'):
if len(loggerObj.handlers) == 0:
continue
if loggerName.split('.')[-1] in ['db_proxy']:
continue
stdoutHandler = logging.StreamHandler(sys.stdout)
stdoutHandler.setFormatter(loggerObj.handlers[0].formatter)
loggerObj.addHandler(stdoutHandler)
import random
from concurrent.futures import ThreadPoolExecutor
import re
from math import sqrt, log1p
from pandaharvester.harvesterconfig import harvester_config
from pandaharvester.harvestercore.queue_config_mapper import QueueConfigMapper
from pandaharvester.harvestercore import core_utils
from pandaharvester.harvestercore.plugin_base import PluginBase
from pandaharvester.harvestermisc.info_utils import PandaQueuesDict
from pandaharvester.harvestermisc.htcondor_utils import get_job_id_tuple_from_batchid
from pandaharvester.harvestermisc.htcondor_utils import CondorJobSubmit
# logger
baseLogger = core_utils.setup_logger('htcondor_submitter')
# Integer division round up
def _div_round_up(a, b):
return a // b + int(a % b > 0)
# Compute weight of each CE according to worker stat, return tuple(dict, total weight score)
def _get_ce_weighting(ce_endpoint_list=[], worker_ce_all_tuple=None):
multiplier = 1000.
n_ce = len(ce_endpoint_list)
worker_limits_dict, worker_ce_stats_dict, worker_ce_backend_throughput_dict, time_window, n_new_workers = worker_ce_all_tuple
N = float(n_ce)
Q = float(worker_limits_dict['nQueueLimitWorker'])
W = float(worker_limits_dict['maxWorkers'])
import json
import os
import os.path
import threading
from queue import Queue
from http.server import HTTPServer, BaseHTTPRequestHandler
from socketserver import ThreadingMixIn
from pandaharvester.harvestercore import core_utils
from pandaharvester.harvestercore.db_proxy_pool import DBProxyPool as DBProxy
from pandaharvester.harvesterconfig import harvester_config
from pandaharvester.harvestermessenger import shared_file_messenger
# logger
_logger = core_utils.setup_logger('http_server_messenger')
shared_file_messenger.set_logger(_logger)
def set_logger(master_logger):
global _logger
_logger = master_logger
shared_file_messenger.set_logger(master_logger)
# handler for http front-end
class HttpHandler(BaseHTTPRequestHandler):
def __init__(self, *args, **kwargs):
self.dbProxy = DBProxy()
self.tmpLog = None
BaseHTTPRequestHandler.__init__(self, *args, **kwargs)
import re
import subprocess
import json
import os.path
from pprint import pprint
from pandaharvester.harvestercore import core_utils
from pandaharvester.harvestercore.work_spec import WorkSpec
from pandaharvester.harvestercore.plugin_base import PluginBase
# logger
baseLogger = core_utils.setup_logger('cobalt_monitor')
# qstat output
# JobID User WallTime Nodes State Location
# ===================================================
# 77734 fcurtis 06:00:00 64 queued None
# monitor for HTCONDOR batch system
class CobaltMonitor(PluginBase):
# constructor
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
# check workers
def check_workers(self, workspec_list):
retList = []
for workSpec in workspec_list:
# print "pprint(dir(workSpec))"
# pprint(dir(workSpec))
import os
import sys
import shutil
import os.path
import uuid
from future.utils import iteritems
from pandaharvester.harvestercore import core_utils
from .base_stager import BaseStager
from pandaharvester.harvestermover import mover_utils
from rucio.client import Client as RucioClient
from rucio.common.exception import RuleNotFound
# logger
baseLogger = core_utils.setup_logger('rucio_stager')
# plugin for stage-out with Rucio
class RucioStager(BaseStager):
# constructor
def __init__(self, **kwarg):
BaseStager.__init__(self, **kwarg)
if not hasattr(self, 'scopeForTmp'):
self.scopeForTmp = 'panda'
# check status
def check_status(self, jobspec):
# make logger
tmpLog = self.make_logger(baseLogger,
'PandaID={0}'.format(jobspec.PandaID),
import arc
import json
import urllib
from pandaharvester.harvestercore import core_utils
from pandaharvester.harvestercore.plugin_base import PluginBase
from pandaharvester.harvesterconfig import harvester_config
from act.common.aCTConfig import aCTConfigARC
from act.common.aCTProxy import aCTProxy
from act.atlas.aCTDBPanda import aCTDBPanda
# logger
baseLogger = core_utils.setup_logger('act_submitter')
# submitter for aCT
class ACTSubmitter(PluginBase):
# constructor
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
# Set up aCT DB connection
self.log = core_utils.make_logger(baseLogger,
'aCT submitter',
method_name='__init__')
self.conf = aCTConfigARC()
self.actDB = aCTDBPanda(self.log, self.conf.get(["db", "file"]))
# Credential dictionary role: proxy file
self.certs = dict(
zip([
from pandaharvester.harvestercore import core_utils
from .base_stager import BaseStager
# logger
baseLogger = core_utils.setup_logger('rse_direct_stager')
# stager plugin with RSE + no data motion
class RseDirectStager(BaseStager):
"""In the workflow for RseDirectStager, workers directly upload output files to RSE
and thus there is no data motion in Harvester."""
# constructor
def __init__(self, **kwarg):
BaseStager.__init__(self, **kwarg)
# check status
def check_status(self, jobspec):
for fileSpec in jobspec.get_output_file_specs(skip_done=True):
fileSpec.status = 'finished'
return True, ''
# trigger stage out
def trigger_stage_out(self, jobspec):
return True, ''
# zip output files
def zip_output(self, jobspec):
# make logger
tmpLog = self.make_logger(baseLogger,
'PandaID={0}'.format(jobspec.PandaID),
import copy
from future.utils import iteritems
from pandaharvester.harvesterconfig import harvester_config
from pandaharvester.harvestercore import core_utils
from pandaharvester.harvestercore.db_proxy_pool import DBProxyPool as DBProxy
from pandaharvester.harvestercore.plugin_factory import PluginFactory
from pandaharvester.harvestermisc.apfmon import Apfmon
# logger
_logger = core_utils.setup_logger('worker_adjuster')
# class to define number of workers to submit
class WorkerAdjuster(object):
# constructor
def __init__(self, queue_config_mapper):
self.queueConfigMapper = queue_config_mapper
self.pluginFactory = PluginFactory()
self.dbProxy = DBProxy()
self.throttlerMap = dict()
self.apf_mon = Apfmon(self.queueConfigMapper)
try:
self.maxNewWorkers = harvester_config.submitter.maxNewWorkers
except AttributeError:
self.maxNewWorkers = None
# define number of workers to submit based on various information
def define_num_workers(self, static_num_workers, site_name):
tmpLog = core_utils.make_logger(_logger,
'site={0}'.format(site_name),
try:
import subprocess32 as subprocess
except:
import subprocess
from pandaharvester.harvestercore import core_utils
from pandaharvester.harvestercore.plugin_base import PluginBase
# logger
_logger = core_utils.setup_logger('grid_cred_manager')
# credential manager using grid-proxy
class GridCredManager(PluginBase):
# constructor
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
# check proxy
def check_credential(self):
# make logger
mainLog = self.make_logger(_logger, method_name='check_credential')
comStr = "grid-proxy-info -exists -hours 72 -file {0}".format(
self.outCertFile)
mainLog.debug(comStr)
try:
p = subprocess.Popen(comStr.split(),
shell=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdOut, stdErr = p.communicate()
from pandaharvester.harvestercore.fifos import SpecialFIFOBase
# condor python or command api
try:
import htcondor
except ImportError:
CONDOR_API = 'command'
else:
CONDOR_API = 'python'
#===============================================================
#=== Definitions ===============================================
# logger
baseLogger = core_utils.setup_logger('htcondor_utils')
# module level lock
moduleLock = threading.Lock()
# List of job ads required
CONDOR_JOB_ADS_LIST = [
'ClusterId',
'ProcId',
'JobStatus',
'LastJobStatus',
'JobStartDate',
'EnteredCurrentStatus',
'ExitCode',
'HoldReason',
'LastHoldReason',
from pandaharvester.harvestercore import core_utils
from .base_stager import BaseStager
from pandaharvester.harvestercore.queue_config_mapper import QueueConfigMapper
from pandaharvester.harvestercore.db_proxy_pool import DBProxyPool as DBProxy
import uuid
import shutil
import os
# logger
baseLogger = core_utils.setup_logger('cp_compass_stager_hpc')
class cpCompasStagerHPC(BaseStager):
# constructor
def __init__(self, **kwarg):
BaseStager.__init__(self, **kwarg)
self.queue_config_mapper = QueueConfigMapper()
# check status
def check_stage_out_status(self, jobspec):
"""Check the status of stage-out procedure. If staging-out is done synchronously in trigger_stage_out
this method should always return True.
Output files are available through jobspec.get_outfile_specs(skip_done=False) which gives
a list of FileSpecs not yet done.
FileSpec.attemptNr shows how many times the transfer was checked for the file.
If the file was successfully transferred, status should be set to 'finished'.
Or 'failed', if the file failed to be transferred. Once files are set to 'finished' or 'failed',
jobspec.get_outfile_specs(skip_done=False) ignores them.
:param jobspec: job specifications
from pandaharvester.harvestercore.fifos import SpecialFIFOBase
# condor python or command api
try:
import htcondor
except ImportError:
CONDOR_API = 'command'
else:
CONDOR_API = 'python'
#===============================================================
#=== Definitions ===============================================
# logger
baseLogger = core_utils.setup_logger('htcondor_utils')
# module level lock
moduleLock = threading.Lock()
# List of job ads required
CONDOR_JOB_ADS_LIST = [
'ClusterId', 'ProcId', 'JobStatus', 'LastJobStatus',
'JobStartDate', 'EnteredCurrentStatus', 'ExitCode',
'HoldReason', 'LastHoldReason', 'RemoveReason',
'harvesterWorkerID',
]
from thread import get_ident
from pandaharvester.harvesterconfig import harvester_config
from pandaharvester.harvestercore import core_utils
from pandaharvester.harvestercore.plugin_factory import PluginFactory
from pandaharvester.harvestercore.db_proxy_pool import DBProxyPool as DBProxy
from pandaharvester.harvestercore.db_interface import DBInterface
# attribute list
_attribute_list = ['id', 'item', 'score']
# fifo object spec
FifoObject = collections.namedtuple('FifoObject', _attribute_list, verbose=False, rename=False)
# logger
_logger = core_utils.setup_logger('fifos')
# base class of fifo message queue
class FIFOBase(object):
# constructor
def __init__(self, **kwarg):
for tmpKey, tmpVal in iteritems(kwarg):
setattr(self, tmpKey, tmpVal)
self.hostname = socket.gethostname()
self.os_pid = os.getpid()
self.dbProxy = DBProxy()
self.dbInterface = DBInterface()
# get process identifier
def get_pid(self):
xmlOutputsBaseFileName = harvester_config.payload_interaction.eventStatusDumpXmlFile
# json for event request
jsonEventsRequestFileName = harvester_config.payload_interaction.eventRequestFile
# json to feed events
jsonEventsFeedFileName = harvester_config.payload_interaction.eventRangesFile
# json to update events
jsonEventsUpdateFileName = harvester_config.payload_interaction.updateEventsFile
# suffix to read json
suffixReadJson = '.read'
# logger
baselogger = core_utils.setup_logger()
class ARCMessenger(BaseMessenger):
'''Mechanism for passing information about completed jobs back to harvester.'''
def __init__(self, **kwarg):
self.jobSpecFileFormat = 'json'
BaseMessenger.__init__(self, **kwarg)
self.schedulerid = harvester_config.master.harvester_id
self.tmpdir = '/tmp' # TODO configurable or common function
# Credential dictionary role: proxy file
self.certs = dict(
zip([
r.split('=')[1]
for r in list(harvester_config.credmanager.voms)
import tempfile
try:
import subprocess32 as subprocess
except:
import subprocess
import os
import stat
from pandaharvester.harvestercore import core_utils
from pandaharvester.harvestercore.plugin_base import PluginBase
# logger
baseLogger = core_utils.setup_logger('cobalt_submitter')
# submitter for Cobalt batch system
class CobaltSubmitter(PluginBase):
# constructor
def __init__(self, **kwarg):
self.uploadLog = False
self.logBaseURL = None
PluginBase.__init__(self, **kwarg)
# template for batch script
tmpFile = open(self.templateFile)
self.template = tmpFile.read()
tmpFile.close()
# submit workers
def submit_workers(self, workspec_list):
retList = []
retStrList = []
from pandaharvester.harvestercore import core_utils
from pandaharvester.harvestercore.work_spec import WorkSpec
from pandaharvester.harvestercore.plugin_base import PluginBase
from pandaharvester.harvesterconfig import harvester_config
# json for job report
jsonJobReport = harvester_config.payload_interaction.jobReportFile
# json for outputs
jsonOutputsFileName = harvester_config.payload_interaction.eventStatusDumpJsonFile
# xml for outputs
xmlOutputsBaseFileName = harvester_config.payload_interaction.eventStatusDumpXmlFile
# logger
baseLogger = core_utils.setup_logger('act_messenger')
class ACTMessenger(PluginBase):
'''Mechanism for passing information about completed jobs back to harvester.'''
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
# get access point
def get_access_point(self, workspec, panda_id):
if workspec.mapType == WorkSpec.MT_MultiJobs:
accessPoint = os.path.join(workspec.get_access_point(), str(panda_id))
else:
accessPoint = workspec.get_access_point()
return accessPoint
import re
try:
import subprocess32 as subprocess
except:
import subprocess
from pandaharvester.harvestercore import core_utils
from pandaharvester.harvestercore.work_spec import WorkSpec
from pandaharvester.harvestercore.plugin_base import PluginBase
# logger
baseLogger = core_utils.setup_logger('slurm_monitor')
# monitor for SLURM batch system
class SlurmMonitor(PluginBase):
# constructor
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
# check workers
def check_workers(self, workspec_list):
retList = []
for workSpec in workspec_list:
# make logger
tmpLog = self.make_logger(baseLogger, 'workerID={0}'.format(workSpec.workerID),
method_name='check_workers')
# command
comStr = "sacct --jobs={0}".format(workSpec.batchID)
# check
tmpLog.debug('check with {0}'.format(comStr))
from __future__ import division
import math
import random
from pandaharvester.harvestercore.work_spec import WorkSpec
from pandaharvester.harvestercore.job_spec import JobSpec
from pandaharvester.harvestercore import core_utils
from pandaharvester.harvestermisc.info_utils import PandaQueuesDict
from pandaharvester.harvestercore.resource_type_mapper import ResourceTypeMapper
from .base_worker_maker import BaseWorkerMaker
import datetime
# logger
_logger = core_utils.setup_logger('simple_worker_maker')
# simple maker
class SimpleWorkerMaker(BaseWorkerMaker):
# constructor
def __init__(self, **kwarg):
self.jobAttributesToUse = ['nCore', 'minRamCount', 'maxDiskCount', 'maxWalltime', 'ioIntensity']
BaseWorkerMaker.__init__(self, **kwarg)
self.rt_mapper = ResourceTypeMapper()
def get_job_core_and_memory(self, queue_dict, job_spec):
job_memory = job_spec.jobParams.get('minRamCount', 0) or 0
job_corecount = job_spec.jobParams.get('coreCount', 1) or 1
unified_queue = queue_dict.get('capability', '') == 'ucore'
import os
import json
from pandaharvester.harvestercore import core_utils
from pandaharvester.harvestercore.work_spec import WorkSpec
from pandaharvester.harvestercore.plugin_base import PluginBase
from pandaharvester.harvesterconfig import harvester_config
from act.common.aCTConfig import aCTConfigARC
from act.atlas.aCTDBPanda import aCTDBPanda
# json for job report
jsonJobReport = harvester_config.payload_interaction.jobReportFile
# logger
baseLogger = core_utils.setup_logger('act_monitor')
# monitor for aCT plugin
class ACTMonitor(PluginBase):
# constructor
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
# Set up aCT DB connection
self.log = core_utils.make_logger(baseLogger, 'aCT submitter', method_name='__init__')
self.actDB = aCTDBPanda(self.log)
# get access point
def get_access_point(self, workspec, panda_id):
if workspec.mapType == WorkSpec.MT_MultiJobs:
from future.utils import iteritems
try:
import pprofile
except Exception:
pass
from pandalogger import logger_config
from pandaharvester import commit_timestamp
from pandaharvester import panda_pkg_info
from pandaharvester.harvesterconfig import harvester_config
from pandaharvester.harvestercore import core_utils
from pandaharvester.harvestermisc.apfmon import Apfmon
# logger
_logger = core_utils.setup_logger('master')
# for singleton
master_instance = False
master_lock = threading.Lock()
# the master class which runs the main process
class Master(object):
# constructor
def __init__(self, single_mode=False, stop_event=None, daemon_mode=True):
# initialize database and config
self.singleMode = single_mode
self.stopEvent = stop_event
self.daemonMode = daemon_mode
import uuid
import os
from pandaharvester.harvestercore import core_utils
from pandaharvester.harvestercore.plugin_base import PluginBase
from pandaharvester.harvestercore.work_spec import WorkSpec
# setup base logger
baseLogger = core_utils.setup_logger('dummy_submitter')
# dummy submitter
class DummySubmitter(PluginBase):
# constructor
def __init__(self, **kwarg):
self.logBaseURL = 'http://localhost/test'
PluginBase.__init__(self, **kwarg)
# submit workers
def submit_workers(self, workspec_list):
"""Submit workers to a scheduling system like batch systems and computing elements.
This method takes a list of WorkSpecs as input argument, and returns a list of tuples.
Each tuple is composed of a return code and a dialog message.
The return code could be True (for success), False (for permanent failures), or None (for temporary failures).
If the return code is None, submission is retried maxSubmissionAttempts times at most which is defined
for each queue in queue_config.json.
Nth tuple in the returned list corresponds to submission status and dialog message for Nth worker
in the given WorkSpec list.
A unique identifier is set to WorkSpec.batchID when submission is successful,
so that they can be identified in the scheduling system. It would be useful to set other attributes
like queueName (batch queue name), computingElement (CE's host name), and nodeID (identifier of the node
where the worker is running).
import time
import datetime
from pandaharvester.harvesterconfig import harvester_config
from pandaharvester.harvestercore import core_utils
from pandaharvester.harvestercore.db_proxy_pool import DBProxyPool as DBProxy
from pandaharvester.harvestercore.command_spec import CommandSpec
from pandaharvester.harvesterbody.agent_base import AgentBase
from pandaharvester.harvestercore.pilot_errors import PilotErrors
# logger
_logger = core_utils.setup_logger('propagator')
STATS_PERIOD = 300
METRICS_PERIOD = 300
# propagate important checkpoints to panda
class Propagator(AgentBase):
# constructor
def __init__(self, communicator, queue_config_mapper, single_mode=False):
AgentBase.__init__(self, single_mode)
self.dbProxy = DBProxy()
self.communicator = communicator
self.queueConfigMapper = queue_config_mapper
self._last_stats_update = None
self._last_metrics_update = None
# main loop
def run(self):
while True:
sw_main = core_utils.get_stopwatch()
"""
Based on: https://cloud.google.com/compute/docs/tutorials/python-guide#before-you-begin
"""
import time
from pandaharvester.harvestercore import core_utils
from pandaharvester.harvestercore.plugin_base import PluginBase
# from requests.exceptions import SSLError
from pandaharvester.harvestercloud.googlecloud import compute, GoogleVM, ZONE, PROJECT
from pandaharvester.harvestercore.queue_config_mapper import QueueConfigMapper
# setup base logger
base_logger = core_utils.setup_logger('google_submitter')
def wait_for_operation(project, zone, operation_name):
"""
Waits for an operation to complete.
TODO: decide whether we want to block or just move on and list the instance status later
:param project:
:param zone:
:param operation_name:
:return:
"""
tmp_log = core_utils.make_logger(base_logger,
method_name='wait_for_operation')
tmp_log.debug('Waiting for operation to finish...')
while True:
result = compute.zoneOperations().get(
import os.path
from pandaharvester.harvestercore.work_spec import WorkSpec
from pandaharvester.harvestercore.plugin_base import PluginBase
from pandaharvester.harvestercore import core_utils
# logger
baseLogger = core_utils.setup_logger('dummy_monitor')
# dummy monitor
class DummyMonitor(PluginBase):
# constructor
def __init__(self, **kwarg):
PluginBase.__init__(self, **kwarg)
# check workers
def check_workers(self, workspec_list):
"""Check status of workers. This method takes a list of WorkSpecs as input argument
and returns a list of worker's statuses.
Nth element if the return list corresponds to the status of Nth WorkSpec in the given list. Worker's
status is one of WorkSpec.ST_finished, WorkSpec.ST_failed, WorkSpec.ST_cancelled, WorkSpec.ST_running,
WorkSpec.ST_submitted. nativeExitCode and nativeStatus of WorkSpec can be arbitrary strings to help
understanding behaviour of the resource and/or batch scheduler.
:param workspec_list: a list of work specs instances
:return: A tuple of return code (True for success, False otherwise) and a list of worker's statuses.
:rtype: (bool, [string,])
"""
retList = []
for workSpec in workspec_list:
# make logger
import os
import shutil
try:
import subprocess32 as subprocess
except ImportError:
import subprocess
from pandaharvester.harvestercore import core_utils
from pandaharvester.harvestersweeper.base_sweeper import BaseSweeper
# logger
baseLogger = core_utils.setup_logger('slurm_sweeper')
# plugin for sweeper with SLURM
class SlurmSweeper(BaseSweeper):
# constructor
def __init__(self, **kwarg):
BaseSweeper.__init__(self, **kwarg)
# kill a worker
def kill_worker(self, workspec):
"""Kill a worker in a scheduling system like batch systems and computing elements.
:param workspec: worker specification
:type workspec: WorkSpec
:return: A tuple of return code (True for success, False otherwise) and error dialog
:rtype: (bool, string)
"""
# make logger
tmpLog = self.make_logger(baseLogger,
import threading
import uuid
import socket
import six
import pexpect
from pandaharvester.harvestercore import core_utils
if six.PY2:
pexpect_spawn = pexpect.spawn
else:
pexpect_spawn = pexpect.spawnu
# logger
baseLogger = core_utils.setup_logger('ssh_tunnel_pool')
# Pool of SSH tunnels
class SshTunnelPool(object):
# constructor
def __init__(self):
self.lock = threading.Lock()
self.pool = dict()
self.params = dict()
# make a dict key
def make_dict_key(self, host, port):
return '{0}:{1}'.format(host, port)
import time
import datetime
from pandaharvester.harvesterconfig import harvester_config
from pandaharvester.harvestercore import core_utils
from pandaharvester.harvestercore.db_proxy_pool import DBProxyPool as DBProxy
from pandaharvester.harvestercore.command_spec import CommandSpec
from pandaharvester.harvesterbody.agent_base import AgentBase
from pandaharvester.harvestercore.pilot_errors import PilotErrors
# logger
_logger = core_utils.setup_logger('propagator')
STATS_PERIOD = 300
METRICS_PERIOD = 300
# propagate important checkpoints to panda
class Propagator(AgentBase):
# constructor
def __init__(self, communicator, queue_config_mapper, single_mode=False):
AgentBase.__init__(self, single_mode)
self.dbProxy = DBProxy()
self.communicator = communicator
self.queueConfigMapper = queue_config_mapper
self._last_stats_update = None
self._last_metrics_update = None
# main loop
def run(self):
while True:
import socket
import datetime
from future.utils import iteritems
from pandaharvester.harvesterconfig import harvester_config
from pandaharvester.harvestercore import core_utils
from pandaharvester.harvestercore.job_spec import JobSpec
from pandaharvester.harvestercore.file_spec import FileSpec
from pandaharvester.harvestercore.db_proxy_pool import DBProxyPool as DBProxy
from pandaharvester.harvesterbody.agent_base import AgentBase
# logger
_logger = core_utils.setup_logger('job_fetcher')
# class to fetch jobs
class JobFetcher(AgentBase):
# constructor
def __init__(self, communicator, queue_config_mapper, single_mode=False):
AgentBase.__init__(self, single_mode)
self.dbProxy = DBProxy()
self.communicator = communicator
self.nodeName = socket.gethostname()
self.queueConfigMapper = queue_config_mapper
# main loop
def run(self):
while True:
mainLog = self.make_logger(_logger, 'id={0}'.format(self.get_pid()), method_name='run')
mainLog.debug('getting number of jobs to be fetched')
# get number of jobs to be fetched
import tempfile
try:
import subprocess32 as subprocess
except:
import subprocess
from pandaharvester.harvestercore import core_utils
from pandaharvester.harvestercore.plugin_base import PluginBase
# logger
baseLogger = core_utils.setup_logger('slurm_submitter')
# submitter for SLURM batch system
class SlurmSubmitter(PluginBase):
# constructor
def __init__(self, **kwarg):
self.uploadLog = False
self.logBaseURL = None
PluginBase.__init__(self, **kwarg)
# template for batch script
tmpFile = open(self.templateFile)
self.template = tmpFile.read()
tmpFile.close()
# submit workers
def submit_workers(self, workspec_list):
retList = []
retStrList = []
for workSpec in workspec_list:
# make logger
