def test_task_life_cycle(engine, client, refresh):
with workers(engine):
tasks = []
for user in ('Babar', 'Babar', 'Celeste'):
res = client.put('/schedule-task/good_job?user={}'.format(user),
upload_files=[('input_file', 'input.xml', b'the file', 'text/xml')]
)
tid = int(res.body)
t1 = Task.byid(engine, tid)
t1.join()
assert t1.raw_output == b'Well done !'
res = client.get('/job_results/{}'.format(t1.tid))
assert res.headers['Content-Type'] == 'application/zip'
assert res.body == b'Well done !'
tasks.append(t1)
res = client.put('/schedule-task/bad_job?user=Celeste',
upload_files=[('input_file', 'input.xml', b'the file', 'text/xml')]
)
tid = int(res.body)
t2 = Task.byid(engine, tid)
t2.join()
tasks.append(t2)
res = client.get('/job_results/{}'.format(t2.tid))
assert res.headers['Content-Type'] == 'text/plain; charset=utf-8'
assert res.body.startswith(b'Traceback')
assert 'I am a little crasher.' in res.text
results = []
for t in tasks:
results.append(client.get('/delete-task/{}'.format(t.tid)))
assert all(res.body == b'true' for res in results)
res = client.get('/services-table')
ipaddr = re.compile('^\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}$')
def edit(elt):
if elt.text:
if 'test_rui' in elt.text:
elt.text = Path(elt.text).name
elif ipaddr.match(elt.text):
elt.text = 'HOSTNAME'
if 'value' in elt.attrib and ipaddr.match(elt.attrib['value']):
elt.attrib['value'] = 'HOSTNAME'
return elt
html = edittag(('td', 'input'), edit, res.text)
refpath = DATADIR / 'services.html'
if refresh:
refpath.write_bytes(html)
assert html == refpath.read_bytes()
def test_relaunch(engine, client):
with workers(engine) as mon:
res = client.put('/schedule-task/good_job?user=Babar',
upload_files=[('input_file', 'input.xml', b'the file', 'text/xml')])
tid = int(res.body)
t = Task.byid(engine, tid)
t.join()
res = client.put(f'/relaunch-task/{tid}')
newtid = int(res.body)
t2 = Task.byid(engine, newtid)
t2.join()
def list_tasks(dburi, tracebacks=False, logcount=False):
init()
engine = create_engine(find_dburi(dburi))
opmap = dict(
engine.execute('select id, name from rework.operation').fetchall())
sql = ('select id from rework.task order by id')
for tid, in engine.execute(sql):
task = Task.byid(engine, tid)
stat = task.state
print(Style.RESET_ALL + str(tid),
Fore.GREEN + opmap[task.operation],
status_color[stat] + stat,
end=' ')
if logcount:
sql = 'select count(*) from rework.log where task = %(tid)s'
count = engine.execute(sql, {'tid': task.tid}).scalar()
print(Style.RESET_ALL + '{} log lines'.format(count), end=' ')
finished = task._propvalue('finished')
started = task._propvalue('started')
print(Fore.WHITE + '[{}]'.format(
task._propvalue('queued').strftime('%Y-%m-%d %H:%M:%S.%f%Z')),
end=' ')
if started:
print(Fore.WHITE +
'→ [{}]'.format(started.strftime('%Y-%m-%d %H:%M:%S.%f%Z')),
end=' ')
if finished:
print(Fore.WHITE +
'→ [{}]'.format(finished.strftime('%Y-%m-%d %H:%M:%S.%f%Z')),
end=' ')
if tracebacks and task.traceback:
print(Fore.YELLOW + task.traceback, end='')
print()
def main_loop(self):
runs = 0
while True:
self.heartbeat()
task = Task.fromqueue(self.engine, self.wid, self.domain)
while task:
task.run()
# run count
runs += 1
if self.maxruns and runs >= self.maxruns:
return
self.heartbeat()
task = Task.fromqueue(self.engine, self.wid, self.domain)
time.sleep(1)
def test_basic_task_operations(engine):
api.schedule(engine,
'print_sleep_and_go_away',
21,
metadata={'user': '******'})
known = [(name, Path(path).name) for name, path in engine.execute(
'select name, path from rework.operation order by name').fetchall()]
assert [
('allocate_and_leak_mbytes', 'tasks.py'),
('capture_logs', 'tasks.py'),
('flush_captured_stdout', 'tasks.py'),
('infinite_loop', 'tasks.py'),
('infinite_loop_long_timeout', 'tasks.py'),
('infinite_loop_timeout', 'tasks.py'),
('log_swarm', 'tasks.py'),
('normal_exception', 'tasks.py'),
('print_sleep_and_go_away', 'tasks.py'),
('raw_input', 'tasks.py'),
('run_in_non_default_domain', 'tasks.py'),
('stderr_swarm', 'tasks.py'),
('unstopable_death', 'tasks.py'),
] == known
mon = Monitor(engine)
wid = mon.new_worker()
t = Task.fromqueue(engine, wid)
t.run()
assert t.output == 42
assert t.metadata == {'user': '******'}
cdate = t._propvalue('queued')
now = datetime.now()
assert now.year == cdate.year
assert now.month == cdate.month
t2 = Task.byid(engine, t.tid)
assert (t2.tid, t2.operation) == (t.tid, t.operation)
t3 = Task.byid(engine, 42000)
assert t3 is None
with pytest.raises(Exception) as err:
api.schedule(engine, 'no_such_task')
assert err.value.args[0] == 'No operation was found for these parameters'
def track_timeouts(self):
if not self.workers:
return
sql = ('select task.id, task.started, timeout '
'from rework.operation as op, '
' rework.task as task '
'where '
' task.operation = op.id and '
' timeout is not null and '
' task.worker in ({})').format(','.join(
str(wid) for wid in self.wids))
with self.engine.begin() as cn:
for tid, start_time, timeout in cn.execute(sql).fetchall():
start_time = start_time.astimezone(pytz.utc)
delta = parse_delta(timeout)
now = utcnow()
if (now - start_time) > delta:
Task.byid(self.engine, tid).abort()
def abort_task(dburi, taskid):
""" immediately abort the given task
This will be done by doing a preemptive kill on
its associated worker.
"""
engine = create_engine(find_dburi(dburi))
task = Task.byid(engine, taskid)
task.abort()
def test_abort(engine, client):
with workers(engine) as mon:
res = client.put('/schedule-task/abortme?user=Babar',
upload_files=[('input_file', 'input.xml', b'the file', 'text/xml')])
tid = int(res.body)
t = Task.byid(engine, tid)
assert not t.aborted
res = client.get(f'/abort-task/{tid}')
mon.preemptive_kill()
assert t.aborted
def abort_task(tid):
if not has_permission('abort'):
abort(403, 'You cannoy do that.')
return
t = Task.byid(engine, tid)
if t is None:
abort(404, 'NO SUCH JOB')
if t.aborted:
return json.dumps(False)
t.abort()
return json.dumps(True)
def relaunch_task(tid):
if not has_permission('relaunch'):
return json.dumps(0)
t = Task.byid(engine, tid)
if t is None:
return json.dumps(0)
op = select('name', 'host', 'domain').table('rework.operation').join(
'rework.task as task on (task.operation = operation.id)').where(
'task.id = %(tid)s', tid=t.tid).do(engine).fetchone()
newtask = api.schedule(engine,
op.name,
rawinputdata=t.raw_input,
domain=op.domain,
hostid=op.host,
metadata=t.metadata)
return json.dumps(newtask.tid)
def buildrow(id, domain, status):
job = Task.byid(engine, id)
if job is None: # deleted
return
try:
tid = job.tid
operation = job.operation
traceback = job.traceback
queued = job._propvalue('queued')
started = job._propvalue('started')
finished = job._propvalue('finished')
meta = job.metadata
worker = job._propvalue('worker')
state = job.state
deathinfo = job.deathinfo
except Exception as e:
print(e)
return
rows.append(
(tid, domain, status, operation, traceback,
queued, started, finished,
meta, worker, state, deathinfo)
)
def log_task(dburi, taskid, fromid=None, watch=False):
engine = create_engine(find_dburi(dburi))
task = Task.byid(engine, taskid)
def watchlogs(fromid):
lid = fromid
for lid, line in task.logs(fromid=fromid):
print(Fore.GREEN + line.strip())
return lid
if not watch:
watchlogs(fromid)
if task.traceback:
print(task.traceback)
return
while True:
fromid = watchlogs(fromid)
sleep(1)
if task.status == 'done':
break
print(Style.RESET_ALL)
def schedule(engine,
opname,
inputdata=None,
rawinputdata=None,
hostid=None,
module=None,
domain=None,
metadata=None):
"""schedule an operation to be run by a worker
It returns a `Task` object.
The operation name is the only mandatory parameter.
The `domain` can be specified to avoid an ambiguity if an
operation is defined within several domains.
An `inputdata` object can be given. It can be any picklable python
object. It will be available through `task.input`.
Alternatively `rawinputdata` can be provided. It must be a byte
string. It can be useful to transmit file contents and avoid the
pickling overhead. It will be available through `task.rawinput`.
Lastly, `metadata` can be provided as a json-serializable python
dictionary. It can contain anything.
"""
if metadata:
assert isinstance(metadata, dict)
if inputdata is not None:
rawinputdata = dumps(inputdata, protocol=2)
q = select('id').table('rework.operation').where(
name=opname
)
if hostid is not None:
q.where(host=hostid)
if module is not None:
q.where(modname=module)
if domain is not None:
q.where(domain=domain)
with engine.begin() as cn:
opids = q.do(cn).fetchall()
if len(opids) > 1:
if hostid is None:
return schedule(
engine,
opname,
rawinputdata=rawinputdata,
hostid=host(),
module=module,
domain=domain,
metadata=metadata
)
raise ValueError('Ambiguous operation selection')
if not len(opids):
raise Exception('No operation was found for these parameters')
opid = opids[0][0]
q = insert(
'rework.task'
).values(
operation=opid,
input=rawinputdata,
status='queued',
metadata=json.dumps(metadata)
)
tid = q.do(cn).scalar()
return Task(engine, tid, opid)
def join():
Task.byid(t.engine, t.tid).join(timeout=2)
def getjob(engine, jobid):
try:
return Task.byid(engine, int(jobid))
except:
return None