def __init__(self,submitterName):
#dictionary to store timestamps of various objects
self._timestamp = {}
self._start_time = time.time()
self.killswitch = os.path.join(DATA_PATH, 'kill_' + str(os.getpid())) #used to terminate the process (for example when running on cron)
Submitter = importName(submitterName, submitterName)
if Submitter is None:
Submitter = importName('diane.submitters.' + submitterName, submitterName)
if Submitter is None:
logger.warning('Can\'t find %s submitter.', submitterName)
return
self.submitter=Submitter()
self.submitter.parser.description = self.submitter.parser.description
#update the help of --diane-worker-number option as now it has a slightly different meaning
self.submitter.parser.get_option('--diane-worker-number').help = 'the minimum number of worker nodes to keep alive'
self.submitter.parser.add_option('--repeat-interval', dest='REPEAT', type='int', default=0, help='time in secons before re-checking the number of workers. If not specified (e.g. for use with cron) the operation will be performed only once')
self.submitter.parser.add_option('--run-time', dest='RUNTIME', type='int', default=0, help='time in seconds to run the script. After RUNTIME hours have passed, the script will terminate. This option is indented to be used with acrontab')
self.submitter.parser.add_option('--diane-max-pending', dest='MAX_PENDING', type='int', default=20, help='the maximum number of workers which are waiting to be run. If the number of waiting workers exceeds MAX_PENDING no more workers will be submitted until the already submitted ones are run')
self.submitter.parser.add_option('--pending-timeout', dest='TIMEOUT', type='int', default=43200, help='maximum time in seconds to keep pending jobs. When the job gets too old it is removed and new one is submitted instead. This is to avoid jobs beign stuck in the queue, etc.')
self.submitter.parser.add_option('--purge-timeout', dest='PURGE_TIMEOUT', type='int', default=86400, help='maximum time in seconds to keep the information about old (finished) jobs. After timeout is exceeded, the job is removed from Ganga repository')
self.submitter.parser.add_option('--lockfile_path', dest='LOCKFILE_DIR', type='string', default=DATA_PATH, help='specify where to keep the lockfile')
self.submitter.parser.add_option('--kill', action='store_true', dest='KILL', default=False, help='kills the running instance of the directory service')
self.submitter.parser.add_option('--square-fitness', action='store_true', dest='SQUARE_FITNESS', default=False, help='squares the fitness to increase the penalty factor')
self.submitter.initialize()
if not self.submitter.options.N_WORKERS > 0:
logger.warning('You must specify the minimum number of workers to keep alive. Run with --help to see possible options.')
return
#check for request to kill the process
if self.submitter.options.KILL:
logger.info('Kill command received; attempting to kill the agent_factory process')
files = []
if os.path.exists(self.submitter.options.LOCKFILE_DIR):
files = os.listdir(self.submitter.options.LOCKFILE_DIR)
for f in files:
if f.startswith('lockfile_'):
os.popen('touch ' + os.path.join(self.submitter.options.LOCKFILE_DIR, f.replace('lockfile', 'kill')))
sys.exit(0)
def create_application_proxy(boot_msg, app_init, agent, **kwds):
boot = streamer.loads(boot_msg)
import os
if boot.darname:
agent.ftc.download(boot.darname)
dar = tarfile.open(boot.darname, "r:gz")
try:
dar.extractall("_python")
except AttributeError: # python < 2.5
os.system("mkdir -p _python")
os.system("cd _python; tar xfzv ../%s" % boot.darname)
import sys
app_python_path = os.path.abspath("_python")
sys.path.insert(0, app_python_path)
diane.config.restore_config(boot.config)
logger.info("application boot and run data received")
boot.log()
diane.config.log_configuration(title="updated configuration")
boot.agent = agent
c = diane.config.getConfig("WorkerAgent")
boot.application_shell_command = c.APPLICATION_SHELL
boot.application_shell_pre_process = ""
boot.application_shell_post_process = ""
# perform a setup action of the application
setup_application = importName(boot.name, "setup_application")
if setup_application:
try:
r = setup_application(streamer.loads(app_init), agent)
if not r is None:
boot.application_shell_pre_process, boot.application_shell_post_process = r
except Exception, x:
handleApplicationFailure(x)
task_result = self.app.do_work(task_data)
return streamer.dumps(task_result)
except Exception, x:
handleApplicationFailure(x)
def create_proxy(boot, **kwds):
logger.debug("InprocessApplicationProxy.create_proxy()")
symbols = {}
try:
__import__(boot.name)
except Exception, x:
handleApplicationFailure(x) #'cannot import application package "%s": %s',boot.name, str(x))
# FIXME: setup environment from boot data
workerClass = importName(boot.name, boot.workerClassName)
workerObj = workerClass()
workerObj._agent = boot.agent
return InprocessApplicationProxy(workerObj)
create_proxy = staticmethod(create_proxy)
def print_obj(obj):
for x in dir(obj):
print x, getattr(obj, x)
import diane.config
from diane.diane_exceptions import DianeException