def __init__(self, appname): parser = self.parser = argparse.ArgumentParser(prog='work_generator') parser.add_argument('--cushion', nargs=1, default=[2000], type=int, help='number of unsent jobs to keep') parser.add_argument( '--max_jobs_create', nargs=1, default=[50], type=int, help= 'maximum number jobs to create in one pass, before waiting for transitioner' ) parser.add_argument('--sleep_interval', nargs=1, default=[5], type=int, help='how many seconds to sleep between passes') parser.add_argument('--debug', action='store_true', default=False, help='print out debug messages') self.add_args(parser) self.args = vars(parser.parse_args()) self.appname = appname self.log = SchedMessages() self.log.set_debug_level(DEBUG if self.args['debug'] else NORMAL)
def __init__(self,appname): parser = self.parser = argparse.ArgumentParser(prog='work_generator') parser.add_argument('--cushion',nargs=1,default=[2000],type=int,help='number of unsent jobs to keep') parser.add_argument('--sleep_interval',nargs=1,default=[5],type=int,help='how many seconds to sleep between passes') parser.add_argument('--debug',action='store_true',default=False,help='print out debug messages') self.add_args(parser) self.args = vars(parser.parse_args()) self.appname = appname self.log = SchedMessages() self.log.set_debug_level(DEBUG if self.args['debug'] else NORMAL)
class WorkGenerator(object): def __init__(self, appname): parser = self.parser = argparse.ArgumentParser(prog='work_generator') parser.add_argument('--cushion', nargs=1, default=[2000], type=int, help='number of unsent jobs to keep') parser.add_argument('--sleep_interval', nargs=1, default=[5], type=int, help='how many seconds to sleep between passes') parser.add_argument('--debug', action='store_true', default=False, help='print out debug messages') self.add_args(parser) self.args = vars(parser.parse_args()) self.appname = appname self.log = SchedMessages() self.log.set_debug_level(DEBUG if self.args['debug'] else NORMAL) def add_args(self, parser): """ Override to add own custom arguments. """ pass def create_work(self, create_work_args, input_files): return create_work(self.appname, create_work_args, input_files) def run(self): database.connect() dbconnection = database.get_dbconnection() cursor = dbconnection.cursor() while True: if osp.exists('../stop_daemons'): self.log.printf(NORMAL, "Stop deamons file found.\n") sys.exit() try: dbconnection.commit() app = database.Apps.find1(name=self.appname) num_unsent = database.Results.count( app=app, server_state=RESULT_SERVER_STATE_UNSENT) if num_unsent < self.args['cushion'][0]: num_create = self.args['cushion'][0] - num_unsent self.log.printf(NORMAL, "%i unsent results. Creating %i more.\n", num_unsent, num_create) self.make_jobs(num=num_create) # wait for transitioner to create jobs now = int(time()) while True: self.log.printf(DEBUG, "Waiting for transitioner...\n") dbconnection.commit() cursor.execute( "select min(transition_time) as t from workunit") if cursor.fetchone()['t'] > now: break sleep(1) self.log.printf(DEBUG, "Created.\n") continue else: self.log.printf(DEBUG, "%i unsent results.\n", num_unsent) except CheckOutputError: pass except Exception as e: self.log.printf(CRITICAL, "Error: %s\n", str(e)) traceback.print_exception(type(e), e, sys.exc_info()[2], None, sys.stderr) sleep(int(self.args['sleep_interval'][0]))
import boinc_path_config from sched_messages import SchedMessages, CRITICAL from subprocess import CalledProcessError, check_output as _check_output, STDOUT from itertools import chain import os, os.path as osp from uuid import uuid4 from time import time import argparse import re log = SchedMessages() class CheckOutputError(Exception): pass projdir = osp.realpath(osp.join(osp.dirname(__file__), '..', '..')) def _get_create_work_args(): try: _check_output([osp.join(projdir, 'bin', 'create_work')], stderr=STDOUT) except CalledProcessError as e: doc = e.output matches = [ g.groups() for g in [re.search('--(.*?) (.*?) ', l) for l in doc.splitlines()] if g ] args = {k: {'n': int, 'x': float}.get(v, str) for k, v in matches}
class WorkGenerator(object): def __init__(self, appname): parser = self.parser = argparse.ArgumentParser(prog='work_generator') parser.add_argument('--cushion', nargs=1, default=[2000], type=int, help='number of unsent jobs to keep') parser.add_argument('--sleep_interval', nargs=1, default=[5], type=int, help='how many seconds to sleep between passes') parser.add_argument('--debug', action='store_true', default=False, help='print out debug messages') self.add_args(parser) self.args = vars(parser.parse_args()) self.appname = appname self.log = SchedMessages() self.log.set_debug_level(DEBUG if self.args['debug'] else NORMAL) def add_args(self, parser): pass def check_output(self, cmd, *args, **kwargs): """ Wraps subprocess.check_output and logs errors to BOINC """ try: return _check_output(cmd, stderr=STDOUT, *args, **kwargs) except CalledProcessError as e: self.log.printf(CRITICAL, "Error calling %s:\n%s\n", str(cmd), e.output) raise CheckOutputError except Exception as e: self.log.printf(CRITICAL, "Error calling %s:\n%s\n", str(cmd), str(e)) raise CheckOutputError def stage_file(self, name, contents, perm=None): base, ext = osp.splitext(name) fullname = base + '_' + md5(str(contents) + str(time())).hexdigest() + ext download_path = self.check_output(['../bin/dir_hier_path', fullname]).strip() with open(download_path, 'w') as f: f.write(contents) if perm: os.chmod(download_path, perm) return fullname def create_work(self, create_work_args, input_files): """ Creates and stages input files based on a list of (name,contents) in input_files, and calls bin/create_work with extra args specified by create_work_args """ self.check_output((['bin/create_work', '--appname', self.appname] + self.args['create_work_args'][0].split() + [self.stage_file(*i) for i in input_files]), cwd='..') def run(self): database.connect() dbconnection = database.get_dbconnection() cursor = dbconnection.cursor() while True: if osp.exists('../stop_daemons'): self.log.printf(NORMAL, "Stop deamons file found.\n") sys.exit() try: dbconnection.commit() app = database.Apps.find1(name=self.appname) num_unsent = database.Results.count( app=app, server_state=RESULT_SERVER_STATE_UNSENT) if num_unsent < self.args['cushion'][0]: num_create = self.args['cushion'][0] - num_unsent self.log.printf(NORMAL, "%i unsent results. Creating %i more.\n", num_unsent, num_create) self.make_jobs(num=num_create) # wait for transitioner to create jobs now = int(time()) while True: self.log.printf(DEBUG, "Waiting for transitioner...\n") dbconnection.commit() cursor.execute( "select min(transition_time) as t from workunit") if cursor.fetchone()['t'] > now: break sleep(1) self.log.printf(DEBUG, "Created.\n") continue else: self.log.printf(DEBUG, "%i unsent results.\n", num_unsent) except CheckOutputError: pass except Exception as e: self.log.printf(CRITICAL, "Error: %s\n", str(e)) traceback.print_exception(type(e), e, sys.exc_info()[2], None, sys.stderr) sleep(int(self.args['sleep_interval'][0]))
class WorkGenerator(object): def __init__(self,appname): parser = self.parser = argparse.ArgumentParser(prog='work_generator') parser.add_argument('--cushion',nargs=1,default=[2000],type=int,help='number of unsent jobs to keep') parser.add_argument('--sleep_interval',nargs=1,default=[5],type=int,help='how many seconds to sleep between passes') parser.add_argument('--debug',action='store_true',default=False,help='print out debug messages') self.add_args(parser) self.args = vars(parser.parse_args()) self.appname = appname self.log = SchedMessages() self.log.set_debug_level(DEBUG if self.args['debug'] else NORMAL) def add_args(self,parser): """ Override to add own custom arguments. """ pass def create_work(self,create_work_args,input_files): return create_work(self.appname,create_work_args,input_files) def run(self): database.connect() dbconnection = database.get_dbconnection() cursor = dbconnection.cursor() while True: if osp.exists('../stop_daemons'): self.log.printf(NORMAL,"Stop deamons file found.\n") sys.exit() try: dbconnection.commit() app = database.Apps.find1(name=self.appname) num_unsent = database.Results.count(app=app, server_state=RESULT_SERVER_STATE_UNSENT) if num_unsent<self.args['cushion'][0]: num_create = self.args['cushion'][0]-num_unsent self.log.printf(NORMAL,"%i unsent results. Creating %i more.\n",num_unsent,num_create) self.make_jobs(num=num_create) # wait for transitioner to create jobs now = int(time()) while True: self.log.printf(DEBUG,"Waiting for transitioner...\n") dbconnection.commit() cursor.execute("select min(transition_time) as t from workunit") if cursor.fetchone()['t']>now: break sleep(1) self.log.printf(DEBUG,"Created.\n") continue else: self.log.printf(DEBUG,"%i unsent results.\n",num_unsent) except CheckOutputError: pass except Exception as e: self.log.printf(CRITICAL,"Error: %s\n",str(e)) traceback.print_exception(type(e), e, sys.exc_info()[2], None, sys.stderr) sleep(int(self.args['sleep_interval'][0]))