def reset(runid: int, tn: str, tskn, alg) -> None:
conn = dawgie.db.post._conn()
cur = dawgie.db.post._cur(conn)
cur.execute('SELECT * from Target WHERE name = %s;', [tn])
tn_ID = _fetchone(cur, 'Dataset: Could not find target ID')
cur.execute('SELECT * from TASK WHERE name = %s;', [tskn])
task_ID = _fetchone(cur, 'Dataset load: Could not find task ID')
cur.execute('SELECT pk FROM Algorithm WHERE name = %s AND task_ID = %s;',
[alg.name(), task_ID])
alg_ID = list(set([pk[0] for pk in cur.fetchall()]))
algv = set()
for sv in alg.state_vectors():
cur.execute(
'SELECT pk FROM StateVector WHERE name = %s AND ' +
'alg_ID = ANY(%s);', [sv.name(), alg_ID])
sv_ID = list(set([pk[0] for pk in cur.fetchall()]))
cur.execute(
'SELECT alg_ID, sv_ID FROM Prime WHERE run_ID = %s AND ' +
' tn_ID = %s AND task_ID = %s AND alg_ID = ANY(%s) AND ' +
'sv_ID = ANY(%s);', [str(runid), tn_ID, task_ID, alg_ID, sv_ID])
ids = cur.fetchall()
algv.update(set([fk[0] for fk in ids]))
svv = set([fk[1] for fk in ids])
if len(svv) == 1:
cur.execute(
'SELECT design,implementation,bugfix ' +
'FROM StateVector WHERE PK = %s;', [svv.pop()])
v = cur.fetchone()
sv._set_ver(dawgie.VERSION(v[0], v[1], v[2]))
else:
log.critical(
'Dataset load: The postgres db is corrupt ' +
'because found %d IDs for the specific state ' + 'vector %s',
len(svv),
'.'.join([str(runid), tn, tskn,
alg.name(),
sv.name()]))
pass
if len(algv) == 1:
cur.execute(
'SELECT design,implementation,bugfix ' +
'FROM Algorithm WHERE PK = %s;', [algv.pop()])
v = cur.fetchone()
alg._set_ver(dawgie.VERSION(v[0], v[1], v[2]))
else:
log.critical(
'Dataset load: The postgres db is corrupt ' +
'because found %d IDs for the specific algorithm %s', len(svv),
'.'.join([str(runid), tn, tskn, alg.name()]))
conn.commit()
cur.close()
conn.close()
return
def __init__(self):
dawgie.Algorithm.__init__(self)
self.__base = ae.network.bot.Engine()
self.__dirty = ae.StateVector()
self.__noise = ae.network.bot.Analyzer()
self._version_ = dawgie.VERSION(1,0,0)
return
def __init__(self):
dawgie.Algorithm.__init__(self)
self.__cmd = Command()
self.__err = Sensor()
self.__sum = StateVector('total', {'voltage': ae.Value(10, 6)})
self._version_ = dawgie.VERSION(1, 0, 0)
return
def __init__(self):
dawgie.Algorithm.__init__(self)
self.__base = SummingNode()
self.__pid = StateVector('law', {
'P': ae.Value(0.5, 2),
'I': ae.Value(0.3, 2),
'D': ae.Value(0, 2)
})
self.__response = StateVector('response', {
'accum': ae.Value(0, 13),
'voltage': ae.Value(0, 3)
})
self._version_ = dawgie.VERSION(1, 0, 0)
return
def __init__(self, db, task):
dawgie.StateVector.__init__(self)
self._version_ = dawgie.VERSION(1, 1, 1)
self['db_input'] = MetricValue(db.input)
self['db_memory'] = MetricValue(db.mem)
self['db_output'] = MetricValue(db.output)
self['db_pages'] = MetricValue(db.pages)
self['db_system'] = MetricValue(db.sys)
self['db_user'] = MetricValue(db.user)
self['task_input'] = MetricValue(task.input)
self['task_memory'] = MetricValue(task.mem)
self['task_output'] = MetricValue(task.output)
self['task_pages'] = MetricValue(task.pages)
self['task_system'] = MetricValue(task.sys)
self['task_user'] = MetricValue(task.user)
return
def metrics() -> '[dawgie.db.METRIC_DATA]':
if dawgie.db.shelf._db is None:
raise RuntimeError('called metrics before open')
result = []
log.info('metrics() - starting')
keys = [k for k in _prime_keys()]
log.info('metrics() - total prime keys %d', len(keys))
keys = [
k for k in sorted(
filter(lambda s: s.split('.')[4] == '__metric__', keys))
]
log.info('metrics() - total __metric__ in prime keys %d', len(keys))
for m in keys:
log.info('metrics() - working on %s', m)
runid, target, task, algn, _svn, vn = m.split('.')
if not result or any([
result[-1].run_id != runid, result[-1].target != target,
result[-1].task != task, result[-1].alg_name != algn
]):
log.info('metrics() - make new reuslt')
msv = dawgie.util.MetricStateVector(
dawgie.METRIC(-2, -2, -2, -2, -2, -2),
dawgie.METRIC(-2, -2, -2, -2, -2, -2))
result.append(
dawgie.db.METRIC_DATA(alg_name=algn,
alg_ver=dawgie.VERSION(-1, -1, -1),
run_id=runid,
sv=msv,
target=target,
task=task))
log.info('metrics() - result length %d', len(result))
pass
try:
log.info('metrics() - reading data and decoding')
msv[vn] = dawgie.db.util.decode(dawgie.db.shelf._db.primary[m])
except FileNotFoundError:
log.exception('missing metric data for %s', m)
pass
return result
def _fill_item(self, l1k, l2k, l3k):
if isinstance(self.__span['table'][l1k][l2k][l3k], ENTRY):
conn = dawgie.db.post._conn()
cur = dawgie.db.post._cur(conn)
cur.execute(
'SELECT blob_name from Prime WHERE run_ID = %s and ' +
'tn_ID = %s and task_ID = %s and alg_ID = %s and ' +
'sv_ID = %s and val_ID = %s;',
self.__span['table'][l1k][l2k][l3k])
value = dawgie.db.util.decode(*cur.fetchone())
cur.execute(
'SELECT design,implementation,bugfix ' +
'FROM StateVector WHERE pk= %s;',
[self.__span['table'][l1k][l2k][l3k].sv_ID])
value._set_ver(dawgie.VERSION(*cur.fetchone()))
self.__span['table'][l1k][l2k][l3k] = value
conn.commit()
cur.close()
conn.close()
else:
value = self.__span['table'][l1k][l2k][l3k]
return value
def metrics() -> '[dawgie.db.METRIC_DATA]':
result = []
svs = {}
conn = dawgie.db.post._conn()
cur = dawgie.db.post._cur(conn)
cur.execute('SELECT PK from StateVector WHERE name = %s;',
('__metric__', ))
sv_IDs = [t[0] for t in cur.fetchall()]
cur.execute('SELECT * from Prime WHERE sv_ID = ANY(%s);', (sv_IDs, ))
for row in cur.fetchall():
key = (row[1], row[2], row[3], row[4])
msv = svs[key] if key in svs else \
dawgie.util.MetricStateVector(dawgie.METRIC(-2,-2,-2,-2,-2,-2),
dawgie.METRIC(-2,-2,-2,-2,-2,-2))
cur.execute('SELECT name FROM Value WHERE PK = %s;', (row[6], ))
vn = cur.fetchone()[0]
msv[vn] = dawgie.db.util.decode(row[7])
svs[key] = msv
pass
for key, msv in svs.items():
cur.execute('SELECT name FROM Target where PK = %s;', (key[2], ))
target = cur.fetchone()[0]
cur.execute('SELECT name FROM Task where PK = %s;', (key[1], ))
task = cur.fetchone()[0]
cur.execute(
'SELECT name,design,implementation,bugfix FROM Algorithm ' +
'where PK = %s;', (key[3], ))
alg = cur.fetchone()
result.append(
dawgie.db.METRIC_DATA(alg_name=alg[0],
alg_ver=dawgie.VERSION(*alg[1:]),
run_id=key[0],
sv=msv,
target=target,
task=task))
pass
cur.close()
conn.close()
return result
def __init__(self):
dawgie.Analyzer.__init__(self)
self.__noise = ae.StateVector()
self._version_ = dawgie.VERSION(1,0,0)
return
def __init__(self, array: numpy.ndarray = None, uid: int = 0):
dawgie.Value.__init__(self)
self.__array = array
self.__uid = uid
self._version_ = dawgie.VERSION(1, 0, 0)
return
def __init__(self):
dawgie.StateVector.__init__(self)
self['image'] = Value(None)
self._version_ = dawgie.VERSION(1, 0, 0)
return
def __init__(self):
dawgie.Regression.__init__(self)
self.__base = ae.network.bot.Analyzer()
self.__data = ae.StateVector()
self._version_ = dawgie.VERSION(1, 0, 0)
return
def __init__(self):
dawgie.Algorithm.__init__(self)
self.__base = ae.disk.bot.Engine()
self.__clean = ae.StateVector()
self._version_ = dawgie.VERSION(1, 0, 0)
return
def __init__(self):
dawgie.Algorithm.__init__(self)
self.__base = Model()
self.__output = StateVector('actual', {'voltage': ae.Value(10, 7)})
self._version_ = dawgie.VERSION(1, 0, 0)
return
def __init__ (self):
dawgie.StateVector.__init__(self)
self['oops'] = aV()
self['snafu'] = aV()
self._version_ = dawgie.VERSION(1,1,1)
return
def _load(self, algref=None, err=True, ver=None):
# Load state vectors with data from db into algorithm
# Take highest run number row from primary table for that task,
# algorithm, state vector if current run does not exist in
# Primary table.
# pylint: disable=too-many-locals,too-many-statements
if self._alg().abort(): raise dawgie.AbortAEError()
log.info("In Interface load")
conn = dawgie.db.post._conn()
cur = dawgie.db.post._cur(conn)
if algref:
ft = dawgie.Factories.resolve(algref)
tn = self._tn()
if ft == dawgie.Factories.analysis:
args = (dawgie.util.task_name(algref.factory),
self._bot()._ps_hint(), self._bot()._runid())
tn = '__all__'
elif ft == dawgie.Factories.regress:
args = (dawgie.util.task_name(algref.factory),
self._bot()._ps_hint(), self._tn())
elif ft == dawgie.Factories.task:
args = (dawgie.util.task_name(algref.factory),
self._bot()._ps_hint(), self._bot()._runid(),
self._tn())
else:
raise KeyError('Unknown factory type {}'.format(
algref.factory.__name__))
child = connect(algref.impl, algref.factory(*args), tn)
child.load(err=err, ver=ver)
else:
# get the target
tn_ID = self.__tn_id(conn, cur)
# Get task id that matches task name
cur.execute('SELECT * from TASK WHERE name = %s;', [self._task()])
task_ID = _fetchone(cur, 'Dataset load: Could not find task ID')
cur.execute(
'SELECT pk FROM Algorithm WHERE name = %s AND ' +
'task_ID = %s;', [self._alg().name(), task_ID])
alg_ID = list(set([pk[0] for pk in cur.fetchall()]))
msv = dawgie.util.MetricStateVector(
dawgie.METRIC(-1, -1, -1, -1, -1, -1),
dawgie.METRIC(-1, -1, -1, -1, -1, -1))
for sv in self._alg().state_vectors() + [msv]:
cur.execute(
'SELECT pk FROM StateVector WHERE name = %s AND ' +
'alg_ID = ANY(%s);', [sv.name(), alg_ID])
sv_ID = list(set([pk[0] for pk in cur.fetchall()]))
cur.execute(
'SELECT run_ID FROM Prime WHERE tn_ID = %s AND ' +
'task_ID = %s AND alg_ID = ANY(%s) AND ' +
'sv_ID = ANY(%s);', [tn_ID, task_ID, alg_ID, sv_ID])
run_ID = set([pk[0] for pk in cur.fetchall()])
if not run_ID:
log.info('Dataset load: Could not find any runs that ' +
'match given the algorithm and state vector')
continue
else:
run_ID = self._runid() if self._runid() in run_ID \
else max(run_ID)
cur.execute(
'SELECT alg_ID,sv_ID FROM Prime WHERE ' +
'run_ID = %s AND tn_ID = %s AND task_ID = %s ' +
' AND alg_ID = ANY(%s) and sv_ID = ANY(%s);',
[run_ID, tn_ID, task_ID, alg_ID, sv_ID])
narrowed = set(cur.fetchall())
if len(narrowed) != 1:
log.critical(
'Dataset load: The postgres db is corrupt ' +
'because found %d IDs', len(narrowed))
pass
na_ID, nsv_ID = narrowed.pop()
self.__fill(cur, sv, na_ID, run_ID, task_ID, tn_ID, nsv_ID)
cur.execute(
'SELECT design,implementation,bugfix ' +
'FROM Algorithm WHERE pk = %s;', [na_ID])
av = cur.fetchone()
cur.execute(
'SELECT design,implementation,bugfix ' +
'FROM StateVector WHERE pk= %s;', [nsv_ID])
svv = cur.fetchone()
self._alg()._version_seal_ = dawgie.VERSION(*av)
sv._version_seal_ = dawgie.VERSION(*svv)
pass
self.msv = msv
pass
conn.commit()
cur.close()
conn.close()
return
def __init__(self, content=None):
dawgie.Value.__init__(self)
self.__content = content
self._version_ = dawgie.VERSION(1, 1, 0)
return
def __init__(self):
dawgie.Analyzer.__init__(self)
self._version_ = dawgie.VERSION(1, 0, 0)
return
def __init__(self, ff=True):
dawgie.Algorithm.__init__(self)
self.__base = Control() if ff else None
self.__model = StateVector('voltage', {'value': ae.Value(0, 4)})
self._version_ = dawgie.VERSION(1, 0, 0)
return
def __init__(self):
dawgie.Algorithm.__init__(self)
self.__command = StateVector('request', {'voltage': ae.Value(0, 1)})
self._version_ = dawgie.VERSION(1, 0, 0)
return
def __init__(self, name: str, start: {str: dawgie.Value}):
dawgie.StateVector.__init__(self)
self.__name = name
self.update(start)
self._version_ = dawgie.VERSION(1, 0, 0)
return
def __init__(self):
dawgie.Algorithm.__init__(self)
self.__base = Analyzer()
self.__image = ae.StateVector()
self._version_ = dawgie.VERSION(1,0,0)
return
def __init__(self):
dawgie.Analyzer.__init__(self)
self._product = aSV()
self._version_ = dawgie.VERSION(1,1,1)
return
def __init__(self, design, impl, bf):
dawgie.Version.__init__(self)
self._version_ = dawgie.VERSION(design, impl, bf)
return
def __init__(self):
dawgie.Value.__init__(self)
self._version_ = dawgie.VERSION(1,1,1)
return
def __init__(self):
dawgie.Algorithm.__init__(self)
self.__model = Model(False)
self.__sensed = StateVector('measured', {'voltage': ae.Value(0, 5)})
self._version_ = dawgie.VERSION(1, 0, 0)
return
