def test_schedule_domain(engine, cleanup): reset_ops(engine) from . import task_testenv from . import task_prodenv api.freeze_operations(engine, domain='test') api.freeze_operations(engine, domain='production') api.freeze_operations(engine, domain='production', hostid='192.168.122.42') with pytest.raises(ValueError) as err: api.schedule(engine, 'foo') assert err.value.args[0] == 'Ambiguous operation selection' api.schedule(engine, 'foo', domain='test') # there two of them but .schedule will by default pick the one # matching the *current* host api.schedule(engine, 'foo', domain='production') api.schedule(engine, 'foo', domain='production', hostid='192.168.122.42') api.schedule(engine, 'foo', domain='production', hostid=host()) hosts = [ host for host, in engine.execute( 'select host from rework.task as t, rework.operation as op ' 'where t.operation = op.id').fetchall() ] assert hosts.count(host()) == 3 assert hosts.count('192.168.122.42') == 1 with pytest.raises(Exception): api.schedule(engine, 'foo', domain='production', hostid='172.16.0.1') with pytest.raises(Exception): api.schedule(engine, 'foo', domain='bogusdomain')
def freeze_operations(engine, domain=None, domain_map=None, hostid=None): values = [] if hostid is None: hostid = host() if domain_map: domain = domain_map.get(domain, domain) for (fdomain, fname), (func, timeout) in __task_registry__.items(): if domain_map: fdomain = domain_map.get(fdomain, fdomain) if domain is not None and domain != fdomain: continue if timeout is not None: timeout = delta_isoformat(timeout) funcmod = func.__module__ module = sys.modules[funcmod] modpath = module.__file__ # python2 if modpath.endswith('pyc'): modpath = modpath[:-1] modpath = str(Path(modpath).resolve()) values.append({ 'host': hostid, 'name': fname, 'path': modpath, 'domain': fdomain, 'timeout': timeout }) recorded = [] alreadyknown = [] for value in values: with engine.begin() as cn: try: q = insert( 'rework.operation' ).values( **value ) q.do(cn) recorded.append(value) except IntegrityError: alreadyknown.append(value) return recorded, alreadyknown
def view(db_uri): """monitor and control workers and tasks""" ipaddr = helper.host() port = 5679 server = Thread(name='reworkui.webapp', target=startapp, kwargs={ 'host': ipaddr, 'port': port, 'dburi': db_uri }) server.daemon = True server.start() browser = webbrowser.open('http://{ipaddr}:{port}'.format(ipaddr=ipaddr, port=port)) if not browser: # no DISPLAY print('You can point your browser to http://{ipaddr}:{port}'.format( ipaddr=ipaddr, port=port)) input()
def reap_dead_workers(self): sql = ('select id, pid from rework.worker ' 'where host = %(host)s and running = true ' 'and domain = %(domain)s') deadlist = [] for wid, pid in self.engine.execute(sql, { 'host': host(), 'domain': self.domain }).fetchall(): try: cmd = ' '.join(psutil.Process(pid).cmdline()) if 'new-worker' not in cmd and str(self.engine.url) not in cmd: print('pid {} was probably recycled'.format(pid)) deadlist.append(wid) except psutil.NoSuchProcess: deadlist.append(wid) if deadlist: with self.engine.begin() as cn: mark_dead_workers(cn, deadlist, 'Unaccounted death (hard crash)') return deadlist
def __init__(self, engine, domain='default', minworkers=None, maxworkers=2, maxruns=0, maxmem=0, debug=False, debugfile=None): self.engine = engine self.domain = domain self.maxworkers = maxworkers self.minworkers = minworkers if minworkers is not None else maxworkers assert 0 <= self.minworkers <= self.maxworkers self.maxruns = maxruns self.maxmem = maxmem self.debugport = 6666 if debug else 0 self.workers = {} self.host = host() self.debugfile = None self.monid = None if debugfile: self.debugfile = Path(debugfile).open('wb') signal.signal(signal.SIGTERM, self.sigterm)
def new_worker(self): with self.engine.begin() as cn: q = insert('rework.worker').values(host=host(), domain=self.domain) return q.do(cn).scalar()
def schedule(engine, opname, inputdata=None, rawinputdata=None, hostid=None, module=None, domain=None, metadata=None): """schedule an operation to be run by a worker It returns a `Task` object. The operation name is the only mandatory parameter. The `domain` can be specified to avoid an ambiguity if an operation is defined within several domains. An `inputdata` object can be given. It can be any picklable python object. It will be available through `task.input`. Alternatively `rawinputdata` can be provided. It must be a byte string. It can be useful to transmit file contents and avoid the pickling overhead. It will be available through `task.rawinput`. Lastly, `metadata` can be provided as a json-serializable python dictionary. It can contain anything. """ if metadata: assert isinstance(metadata, dict) if inputdata is not None: rawinputdata = dumps(inputdata, protocol=2) q = select('id').table('rework.operation').where( name=opname ) if hostid is not None: q.where(host=hostid) if module is not None: q.where(modname=module) if domain is not None: q.where(domain=domain) with engine.begin() as cn: opids = q.do(cn).fetchall() if len(opids) > 1: if hostid is None: return schedule( engine, opname, rawinputdata=rawinputdata, hostid=host(), module=module, domain=domain, metadata=metadata ) raise ValueError('Ambiguous operation selection') if not len(opids): raise Exception('No operation was found for these parameters') opid = opids[0][0] q = insert( 'rework.task' ).values( operation=opid, input=rawinputdata, status='queued', metadata=json.dumps(metadata) ) tid = q.do(cn).scalar() return Task(engine, tid, opid)