def test_schedule_domain(engine, cleanup):
reset_ops(engine)
from . import task_testenv
from . import task_prodenv
api.freeze_operations(engine, domain='test')
api.freeze_operations(engine, domain='production')
api.freeze_operations(engine, domain='production', hostid='192.168.122.42')
with pytest.raises(ValueError) as err:
api.schedule(engine, 'foo')
assert err.value.args[0] == 'Ambiguous operation selection'
api.schedule(engine, 'foo', domain='test')
# there two of them but .schedule will by default pick the one
# matching the *current* host
api.schedule(engine, 'foo', domain='production')
api.schedule(engine, 'foo', domain='production', hostid='192.168.122.42')
api.schedule(engine, 'foo', domain='production', hostid=host())
hosts = [
host for host, in engine.execute(
'select host from rework.task as t, rework.operation as op '
'where t.operation = op.id').fetchall()
]
assert hosts.count(host()) == 3
assert hosts.count('192.168.122.42') == 1
with pytest.raises(Exception):
api.schedule(engine, 'foo', domain='production', hostid='172.16.0.1')
with pytest.raises(Exception):
api.schedule(engine, 'foo', domain='bogusdomain')
def freeze_operations(engine, domain=None, domain_map=None,
hostid=None):
values = []
if hostid is None:
hostid = host()
if domain_map:
domain = domain_map.get(domain, domain)
for (fdomain, fname), (func, timeout) in __task_registry__.items():
if domain_map:
fdomain = domain_map.get(fdomain, fdomain)
if domain is not None and domain != fdomain:
continue
if timeout is not None:
timeout = delta_isoformat(timeout)
funcmod = func.__module__
module = sys.modules[funcmod]
modpath = module.__file__
# python2
if modpath.endswith('pyc'):
modpath = modpath[:-1]
modpath = str(Path(modpath).resolve())
values.append({
'host': hostid,
'name': fname,
'path': modpath,
'domain': fdomain,
'timeout': timeout
})
recorded = []
alreadyknown = []
for value in values:
with engine.begin() as cn:
try:
q = insert(
'rework.operation'
).values(
**value
)
q.do(cn)
recorded.append(value)
except IntegrityError:
alreadyknown.append(value)
return recorded, alreadyknown
def view(db_uri):
"""monitor and control workers and tasks"""
ipaddr = helper.host()
port = 5679
server = Thread(name='reworkui.webapp',
target=startapp,
kwargs={
'host': ipaddr,
'port': port,
'dburi': db_uri
})
server.daemon = True
server.start()
browser = webbrowser.open('http://{ipaddr}:{port}'.format(ipaddr=ipaddr,
port=port))
if not browser: # no DISPLAY
print('You can point your browser to http://{ipaddr}:{port}'.format(
ipaddr=ipaddr, port=port))
input()
def reap_dead_workers(self):
sql = ('select id, pid from rework.worker '
'where host = %(host)s and running = true '
'and domain = %(domain)s')
deadlist = []
for wid, pid in self.engine.execute(sql, {
'host': host(),
'domain': self.domain
}).fetchall():
try:
cmd = ' '.join(psutil.Process(pid).cmdline())
if 'new-worker' not in cmd and str(self.engine.url) not in cmd:
print('pid {} was probably recycled'.format(pid))
deadlist.append(wid)
except psutil.NoSuchProcess:
deadlist.append(wid)
if deadlist:
with self.engine.begin() as cn:
mark_dead_workers(cn, deadlist,
'Unaccounted death (hard crash)')
return deadlist
def __init__(self,
engine,
domain='default',
minworkers=None,
maxworkers=2,
maxruns=0,
maxmem=0,
debug=False,
debugfile=None):
self.engine = engine
self.domain = domain
self.maxworkers = maxworkers
self.minworkers = minworkers if minworkers is not None else maxworkers
assert 0 <= self.minworkers <= self.maxworkers
self.maxruns = maxruns
self.maxmem = maxmem
self.debugport = 6666 if debug else 0
self.workers = {}
self.host = host()
self.debugfile = None
self.monid = None
if debugfile:
self.debugfile = Path(debugfile).open('wb')
signal.signal(signal.SIGTERM, self.sigterm)
def new_worker(self):
with self.engine.begin() as cn:
q = insert('rework.worker').values(host=host(), domain=self.domain)
return q.do(cn).scalar()
def schedule(engine,
opname,
inputdata=None,
rawinputdata=None,
hostid=None,
module=None,
domain=None,
metadata=None):
"""schedule an operation to be run by a worker
It returns a `Task` object.
The operation name is the only mandatory parameter.
The `domain` can be specified to avoid an ambiguity if an
operation is defined within several domains.
An `inputdata` object can be given. It can be any picklable python
object. It will be available through `task.input`.
Alternatively `rawinputdata` can be provided. It must be a byte
string. It can be useful to transmit file contents and avoid the
pickling overhead. It will be available through `task.rawinput`.
Lastly, `metadata` can be provided as a json-serializable python
dictionary. It can contain anything.
"""
if metadata:
assert isinstance(metadata, dict)
if inputdata is not None:
rawinputdata = dumps(inputdata, protocol=2)
q = select('id').table('rework.operation').where(
name=opname
)
if hostid is not None:
q.where(host=hostid)
if module is not None:
q.where(modname=module)
if domain is not None:
q.where(domain=domain)
with engine.begin() as cn:
opids = q.do(cn).fetchall()
if len(opids) > 1:
if hostid is None:
return schedule(
engine,
opname,
rawinputdata=rawinputdata,
hostid=host(),
module=module,
domain=domain,
metadata=metadata
)
raise ValueError('Ambiguous operation selection')
if not len(opids):
raise Exception('No operation was found for these parameters')
opid = opids[0][0]
q = insert(
'rework.task'
).values(
operation=opid,
input=rawinputdata,
status='queued',
metadata=json.dumps(metadata)
)
tid = q.do(cn).scalar()
return Task(engine, tid, opid)