def test_more_unstarted(engine):
with engine.begin() as cn:
cn.execute('delete from rework.task')
cn.execute('delete from rework.worker')
with workers(engine) as mon:
nworkers = engine.execute(
'select count(*) from rework.worker').scalar()
assert nworkers == 1
with engine.begin() as cn:
insert('rework.worker').values(
host='127.0.0.1', domain='default').do(cn) # unborn worker
nworkers = engine.execute(
'select count(*) from rework.worker').scalar()
assert nworkers == 2
t1 = api.schedule(engine, 'raw_input', b'foo')
t2 = api.schedule(engine, 'raw_input', b'foo')
t1.join()
t2.join()
assert engine.execute('select count(*) from rework.task').scalar() == 2
nworkers = engine.execute(
'select count(*) from rework.worker').scalar()
assert nworkers == 2
mon.cleanup_unstarted()
nworkers = engine.execute(
'select count(*) from rework.worker').scalar()
assert nworkers == 1
assert engine.execute('select count(*) from rework.task').scalar() == 2
def test_cleanup_unstarted(engine):
with engine.begin() as cn:
cn.execute('delete from rework.task')
cn.execute('delete from rework.worker')
mon = Monitor(engine, 'default', None, 1, 1, 0, False)
mon.register()
mon.ensure_workers()
with engine.begin() as cn:
insert('rework.worker').values(host='127.0.0.1', domain='default').do(
cn) # unborn worker
nworkers = engine.execute('select count(*) from rework.worker').scalar()
assert nworkers == 2
t = api.schedule(engine, 'raw_input', b'foo')
t.join()
mon.killall(msg=None)
deleted = mon.cleanup_unstarted()
assert deleted == 1
assert engine.execute('select count(*) from rework.worker').scalar() == 1
assert engine.execute('select count(*) from rework.task').scalar() == 1
deleted = mon.cleanup_unstarted()
assert deleted == 0
def test_list_workers(engine, cli):
with workers(engine):
r = cli('list-workers', engine.url)
assert ('<X> <X>@<X>.<X>.<X>.<X> <X> Mb [running (idle)] '
'[<X>-<X>-<X> <X>:<X>:<X>.<X>+<X>] '
'→ [<X>-<X>-<X> <X>:<X>:<X>.<X>+<X>]') == scrub(r.output)
with engine.begin() as cn:
insert('rework.worker').values(host='12345', domain='default').do(cn)
r = cli('list-workers', engine.url)
assert ('<X> <X>@<X>.<X>.<X>.<X> <X> Mb [dead] '
'[<X>-<X>-<X> <X>:<X>:<X>.<X>+<X>] '
'→ [<X>-<X>-<X> <X>:<X>:<X>.<X>+<X>] '
'→ [<X>-<X>-<X> <X>:<X>:<X>.<X>+<X>] '
'Forcefully killed by the monitor. \n'
'<X> <nopid>@<X> <X> Mb [unstarted] '
'[<X>-<X>-<X> <X>:<X>:<X>.<X>+<X>]') == scrub(r.output)
def sqlInsert(params,extraParams=None,dbType='postgres'):
'''
Give the SQL INSERT statement
@param params: dict keyed by field name of values
@param extraParams: any extra fields that you have created beyond the normal ais message fields
@rtype: sqlhelp.insert
@return: insert class instance
@todo: allow optional type checking of params?
@warning: this will take invalid keys happily and do what???
'''
import sqlhelp
i = sqlhelp.insert('bsreport',dbType=dbType)
if dbType=='postgres':
finished = []
for key in params:
if key in finished:
continue
if key not in toPgFields and key not in fromPgFields:
if type(params[key])==Decimal: i.add(key,float(params[key]))
else: i.add(key,params[key])
else:
if key in fromPgFields:
val = params[key]
# Had better be a WKT type like POINT(-88.1 30.321)
i.addPostGIS(key,val)
finished.append(key)
else:
# Need to construct the type.
pgName = toPgFields[key]
#valStr='GeomFromText(\''+pgTypes[pgName]+'('
valStr=pgTypes[pgName]+'('
vals = []
for nonPgKey in fromPgFields[pgName]:
vals.append(str(params[nonPgKey]))
finished.append(nonPgKey)
valStr+=' '.join(vals)+')'
i.addPostGIS(pgName,valStr)
else:
for key in params:
if type(params[key])==Decimal: i.add(key,float(params[key]))
else: i.add(key,params[key])
if None != extraParams:
for key in extraParams:
i.add(key,extraParams[key])
return i
def freeze_operations(engine, domain=None, domain_map=None,
hostid=None):
values = []
if hostid is None:
hostid = host()
if domain_map:
domain = domain_map.get(domain, domain)
for (fdomain, fname), (func, timeout) in __task_registry__.items():
if domain_map:
fdomain = domain_map.get(fdomain, fdomain)
if domain is not None and domain != fdomain:
continue
if timeout is not None:
timeout = delta_isoformat(timeout)
funcmod = func.__module__
module = sys.modules[funcmod]
modpath = module.__file__
# python2
if modpath.endswith('pyc'):
modpath = modpath[:-1]
modpath = str(Path(modpath).resolve())
values.append({
'host': hostid,
'name': fname,
'path': modpath,
'domain': fdomain,
'timeout': timeout
})
recorded = []
alreadyknown = []
for value in values:
with engine.begin() as cn:
try:
q = insert(
'rework.operation'
).values(
**value
)
q.do(cn)
recorded.append(value)
except IntegrityError:
alreadyknown.append(value)
return recorded, alreadyknown
def register(self):
# register in db
with self.engine.begin() as cn:
q = insert('rework.monitor').values(domain=self.domain,
options=json.dumps({
'maxworkers':
self.maxworkers,
'minworkers':
self.minworkers,
'maxruns':
self.maxruns,
'maxmem':
self.maxmem,
'debugport':
self.debugport
}))
self.monid = q.do(cn).scalar()
def refresh_tasks(engine, inithash, domain):
taskstates = tasks_info(engine, domain)
thash = md5(str(taskstates).encode('ascii')).hexdigest()
if thash != inithash:
htmltable = generate_tasks_table(engine, taskstates)
q = insert('rework.taskstable').values(
hash=thash,
domain=domain,
content=htmltable
)
with engine.begin() as cn:
q.do(cn)
inithash = thash
# cleanup old tables
sql = ('delete from rework.taskstable '
'where hash != %(hash)s '
'and domain = %(domain)s')
with engine.begin() as cn:
cn.execute(sql, hash=thash, domain=domain)
return inithash
def sqlInsert(params,extraParams=None,dbType='postgres'):
'''
Give the SQL INSERT statement
@param params: dict keyed by field name of values
@param extraParams: any extra fields that you have created beyond the normal ais message fields
@rtype: sqlhelp.insert
@return: insert class instance
@todo: allow optional type checking of params?
@warning: this will take invalid keys happily and do what???
'''
import sqlhelp
i = sqlhelp.insert('b_staticdata',dbType=dbType)
for key in params:
if type(params[key])==Decimal: i.add(key,float(params[key]))
else: i.add(key,params[key])
if None != extraParams:
for key in extraParams:
i.add(key,extraParams[key])
return i
def _new_revision(self, cn, name, head, tsstart, tsend, author,
insertion_date, metadata):
tablename = self._series_to_tablename(cn, name)
if insertion_date is not None:
assert insertion_date.tzinfo is not None
idate = pd.Timestamp(insertion_date)
else:
idate = pd.Timestamp(datetime.utcnow(), tz='UTC')
latest_idate = self.latest_insertion_date(cn, name)
if latest_idate:
assert idate > latest_idate
if metadata:
metadata = json.dumps(metadata)
q = insert(f'"{self.namespace}.revision"."{tablename}" ').values(
snapshot=head,
tsstart=tsstart,
tsend=tsend,
author=author,
insertion_date=idate,
metadata=metadata)
q.do(cn)
def new_worker(self):
with self.engine.begin() as cn:
q = insert('rework.worker').values(host=host(), domain=self.domain)
return q.do(cn).scalar()
def schedule(engine,
opname,
inputdata=None,
rawinputdata=None,
hostid=None,
module=None,
domain=None,
metadata=None):
"""schedule an operation to be run by a worker
It returns a `Task` object.
The operation name is the only mandatory parameter.
The `domain` can be specified to avoid an ambiguity if an
operation is defined within several domains.
An `inputdata` object can be given. It can be any picklable python
object. It will be available through `task.input`.
Alternatively `rawinputdata` can be provided. It must be a byte
string. It can be useful to transmit file contents and avoid the
pickling overhead. It will be available through `task.rawinput`.
Lastly, `metadata` can be provided as a json-serializable python
dictionary. It can contain anything.
"""
if metadata:
assert isinstance(metadata, dict)
if inputdata is not None:
rawinputdata = dumps(inputdata, protocol=2)
q = select('id').table('rework.operation').where(
name=opname
)
if hostid is not None:
q.where(host=hostid)
if module is not None:
q.where(modname=module)
if domain is not None:
q.where(domain=domain)
with engine.begin() as cn:
opids = q.do(cn).fetchall()
if len(opids) > 1:
if hostid is None:
return schedule(
engine,
opname,
rawinputdata=rawinputdata,
hostid=host(),
module=module,
domain=domain,
metadata=metadata
)
raise ValueError('Ambiguous operation selection')
if not len(opids):
raise Exception('No operation was found for these parameters')
opid = opids[0][0]
q = insert(
'rework.task'
).values(
operation=opid,
input=rawinputdata,
status='queued',
metadata=json.dumps(metadata)
)
tid = q.do(cn).scalar()
return Task(engine, tid, opid)