def __init__(self, appname):
parser = self.parser = argparse.ArgumentParser(prog='work_generator')
parser.add_argument('--cushion',
nargs=1,
default=[2000],
type=int,
help='number of unsent jobs to keep')
parser.add_argument(
'--max_jobs_create',
nargs=1,
default=[50],
type=int,
help=
'maximum number jobs to create in one pass, before waiting for transitioner'
)
parser.add_argument('--sleep_interval',
nargs=1,
default=[5],
type=int,
help='how many seconds to sleep between passes')
parser.add_argument('--debug',
action='store_true',
default=False,
help='print out debug messages')
self.add_args(parser)
self.args = vars(parser.parse_args())
self.appname = appname
self.log = SchedMessages()
self.log.set_debug_level(DEBUG if self.args['debug'] else NORMAL)
def __init__(self,appname):
parser = self.parser = argparse.ArgumentParser(prog='work_generator')
parser.add_argument('--cushion',nargs=1,default=[2000],type=int,help='number of unsent jobs to keep')
parser.add_argument('--sleep_interval',nargs=1,default=[5],type=int,help='how many seconds to sleep between passes')
parser.add_argument('--debug',action='store_true',default=False,help='print out debug messages')
self.add_args(parser)
self.args = vars(parser.parse_args())
self.appname = appname
self.log = SchedMessages()
self.log.set_debug_level(DEBUG if self.args['debug'] else NORMAL)
class WorkGenerator(object):
def __init__(self, appname):
parser = self.parser = argparse.ArgumentParser(prog='work_generator')
parser.add_argument('--cushion',
nargs=1,
default=[2000],
type=int,
help='number of unsent jobs to keep')
parser.add_argument('--sleep_interval',
nargs=1,
default=[5],
type=int,
help='how many seconds to sleep between passes')
parser.add_argument('--debug',
action='store_true',
default=False,
help='print out debug messages')
self.add_args(parser)
self.args = vars(parser.parse_args())
self.appname = appname
self.log = SchedMessages()
self.log.set_debug_level(DEBUG if self.args['debug'] else NORMAL)
def add_args(self, parser):
"""
Override to add own custom arguments.
"""
pass
def create_work(self, create_work_args, input_files):
return create_work(self.appname, create_work_args, input_files)
def run(self):
database.connect()
dbconnection = database.get_dbconnection()
cursor = dbconnection.cursor()
while True:
if osp.exists('../stop_daemons'):
self.log.printf(NORMAL, "Stop deamons file found.\n")
sys.exit()
try:
dbconnection.commit()
app = database.Apps.find1(name=self.appname)
num_unsent = database.Results.count(
app=app, server_state=RESULT_SERVER_STATE_UNSENT)
if num_unsent < self.args['cushion'][0]:
num_create = self.args['cushion'][0] - num_unsent
self.log.printf(NORMAL,
"%i unsent results. Creating %i more.\n",
num_unsent, num_create)
self.make_jobs(num=num_create)
# wait for transitioner to create jobs
now = int(time())
while True:
self.log.printf(DEBUG, "Waiting for transitioner...\n")
dbconnection.commit()
cursor.execute(
"select min(transition_time) as t from workunit")
if cursor.fetchone()['t'] > now: break
sleep(1)
self.log.printf(DEBUG, "Created.\n")
continue
else:
self.log.printf(DEBUG, "%i unsent results.\n", num_unsent)
except CheckOutputError:
pass
except Exception as e:
self.log.printf(CRITICAL, "Error: %s\n", str(e))
traceback.print_exception(type(e), e,
sys.exc_info()[2], None, sys.stderr)
sleep(int(self.args['sleep_interval'][0]))
import boinc_path_config
from sched_messages import SchedMessages, CRITICAL
from subprocess import CalledProcessError, check_output as _check_output, STDOUT
from itertools import chain
import os, os.path as osp
from uuid import uuid4
from time import time
import argparse
import re
log = SchedMessages()
class CheckOutputError(Exception):
pass
projdir = osp.realpath(osp.join(osp.dirname(__file__), '..', '..'))
def _get_create_work_args():
try:
_check_output([osp.join(projdir, 'bin', 'create_work')], stderr=STDOUT)
except CalledProcessError as e:
doc = e.output
matches = [
g.groups()
for g in [re.search('--(.*?) (.*?) ', l) for l in doc.splitlines()]
if g
]
args = {k: {'n': int, 'x': float}.get(v, str) for k, v in matches}
class WorkGenerator(object):
def __init__(self, appname):
parser = self.parser = argparse.ArgumentParser(prog='work_generator')
parser.add_argument('--cushion',
nargs=1,
default=[2000],
type=int,
help='number of unsent jobs to keep')
parser.add_argument('--sleep_interval',
nargs=1,
default=[5],
type=int,
help='how many seconds to sleep between passes')
parser.add_argument('--debug',
action='store_true',
default=False,
help='print out debug messages')
self.add_args(parser)
self.args = vars(parser.parse_args())
self.appname = appname
self.log = SchedMessages()
self.log.set_debug_level(DEBUG if self.args['debug'] else NORMAL)
def add_args(self, parser):
pass
def check_output(self, cmd, *args, **kwargs):
"""
Wraps subprocess.check_output and logs errors to BOINC
"""
try:
return _check_output(cmd, stderr=STDOUT, *args, **kwargs)
except CalledProcessError as e:
self.log.printf(CRITICAL, "Error calling %s:\n%s\n", str(cmd),
e.output)
raise CheckOutputError
except Exception as e:
self.log.printf(CRITICAL, "Error calling %s:\n%s\n", str(cmd),
str(e))
raise CheckOutputError
def stage_file(self, name, contents, perm=None):
base, ext = osp.splitext(name)
fullname = base + '_' + md5(str(contents) +
str(time())).hexdigest() + ext
download_path = self.check_output(['../bin/dir_hier_path',
fullname]).strip()
with open(download_path, 'w') as f:
f.write(contents)
if perm: os.chmod(download_path, perm)
return fullname
def create_work(self, create_work_args, input_files):
"""
Creates and stages input files based on a list of (name,contents) in input_files,
and calls bin/create_work with extra args specified by create_work_args
"""
self.check_output((['bin/create_work', '--appname', self.appname] +
self.args['create_work_args'][0].split() +
[self.stage_file(*i) for i in input_files]),
cwd='..')
def run(self):
database.connect()
dbconnection = database.get_dbconnection()
cursor = dbconnection.cursor()
while True:
if osp.exists('../stop_daemons'):
self.log.printf(NORMAL, "Stop deamons file found.\n")
sys.exit()
try:
dbconnection.commit()
app = database.Apps.find1(name=self.appname)
num_unsent = database.Results.count(
app=app, server_state=RESULT_SERVER_STATE_UNSENT)
if num_unsent < self.args['cushion'][0]:
num_create = self.args['cushion'][0] - num_unsent
self.log.printf(NORMAL,
"%i unsent results. Creating %i more.\n",
num_unsent, num_create)
self.make_jobs(num=num_create)
# wait for transitioner to create jobs
now = int(time())
while True:
self.log.printf(DEBUG, "Waiting for transitioner...\n")
dbconnection.commit()
cursor.execute(
"select min(transition_time) as t from workunit")
if cursor.fetchone()['t'] > now: break
sleep(1)
self.log.printf(DEBUG, "Created.\n")
continue
else:
self.log.printf(DEBUG, "%i unsent results.\n", num_unsent)
except CheckOutputError:
pass
except Exception as e:
self.log.printf(CRITICAL, "Error: %s\n", str(e))
traceback.print_exception(type(e), e,
sys.exc_info()[2], None, sys.stderr)
sleep(int(self.args['sleep_interval'][0]))
class WorkGenerator(object):
def __init__(self,appname):
parser = self.parser = argparse.ArgumentParser(prog='work_generator')
parser.add_argument('--cushion',nargs=1,default=[2000],type=int,help='number of unsent jobs to keep')
parser.add_argument('--sleep_interval',nargs=1,default=[5],type=int,help='how many seconds to sleep between passes')
parser.add_argument('--debug',action='store_true',default=False,help='print out debug messages')
self.add_args(parser)
self.args = vars(parser.parse_args())
self.appname = appname
self.log = SchedMessages()
self.log.set_debug_level(DEBUG if self.args['debug'] else NORMAL)
def add_args(self,parser):
"""
Override to add own custom arguments.
"""
pass
def create_work(self,create_work_args,input_files):
return create_work(self.appname,create_work_args,input_files)
def run(self):
database.connect()
dbconnection = database.get_dbconnection()
cursor = dbconnection.cursor()
while True:
if osp.exists('../stop_daemons'):
self.log.printf(NORMAL,"Stop deamons file found.\n")
sys.exit()
try:
dbconnection.commit()
app = database.Apps.find1(name=self.appname)
num_unsent = database.Results.count(app=app, server_state=RESULT_SERVER_STATE_UNSENT)
if num_unsent<self.args['cushion'][0]:
num_create = self.args['cushion'][0]-num_unsent
self.log.printf(NORMAL,"%i unsent results. Creating %i more.\n",num_unsent,num_create)
self.make_jobs(num=num_create)
# wait for transitioner to create jobs
now = int(time())
while True:
self.log.printf(DEBUG,"Waiting for transitioner...\n")
dbconnection.commit()
cursor.execute("select min(transition_time) as t from workunit")
if cursor.fetchone()['t']>now: break
sleep(1)
self.log.printf(DEBUG,"Created.\n")
continue
else:
self.log.printf(DEBUG,"%i unsent results.\n",num_unsent)
except CheckOutputError:
pass
except Exception as e:
self.log.printf(CRITICAL,"Error: %s\n",str(e))
traceback.print_exception(type(e), e, sys.exc_info()[2], None, sys.stderr)
sleep(int(self.args['sleep_interval'][0]))