def get_data_by_uuid_impl(self, datauuid):
sql = f'''
select
X,Y,Z,P,U,V,W,Q,R,S,l,m,T,C,cols,nested,des
from
data
left join dataset on dataset=dsid
left join hist on hist=hid
left join attr on attr=aid
where
did=?'''
self.query(sql, [datauuid])
row = self.get_one()
if row is None:
return None
# Recover requested matrices/vectors.
# TODO: surely there is duplicated code to be refactored in this file!
dic = {'name': row['des'], 'history': zlibext_unpack(row['nested'])}
fields = [
Data.from_alias[k] for k, v in row.items()
if len(k) == 1 and v is not None
]
for field in fields:
mid = row[field]
if mid is not None:
self.query(f'select val,w,h from mat where mid=?', [mid])
rone = self.get_one()
dic[field] = unpack_data(rone['val'], rone['w'], rone['h'])
return Data(columns=zlibext_unpack(row['cols']), **dic)
def get_finished_names_by_mark_impl(self, mark):
"""
Finished means nonfailed and unlocked results.
The original datasets will not be returned.
original dataset = stored with no history of transformations.
All results are returned (apply & use, as many as there are),
so the names come along with their history,
:param mark:
:return: [dicts]
"""
self.query(
f"""
select
des, nested
from
res join data on dtr=did
join dataset on dataset=dsid
join hist on hist=hid
where
end!='0000-00-00 00:00:00' and
fail=0 and mark=?
""", [mark])
rows = self.get_all()
if rows is None:
return None
else:
for row in rows:
row['name'] = row.pop('des')
row['history'] = zlibext_unpack(row.pop('nested'))
return rows
def get_data_by_name_impl(self, name, fields=None, history=None):
"""
To just recover the original dataset you can pass history=None.
Specify fields if you want to reduce traffic, otherwise all available
fields will be fetched.
ps. 1: Obviously, when getting prediction data (i.e., results),
the history which led to the predictions should be provided.
:param name:
:param fields: None=get full Data; case insensitive; e.g. 'X,y,Z'
:param history: nested tuples
:param just_check_exists:
:return:
"""
hist_uuid = uuid(zlibext_pack(history))
sql = f'''
select
X,Y,Z,P,U,V,W,Q,R,S,l,m,T,C,cols,des
from
data
left join dataset on dataset=dsid
left join attr on attr=aid
where
des=? and hist=?'''
self.query(sql, [name, hist_uuid])
row = self.get_one()
if row is None:
return None
# Recover requested matrices/vectors.
dic = {'name': name, 'history': history}
if fields is None:
flst = [k for k, v in row.items() if len(k) == 1 and v is not None]
else:
flst = fields.split(',')
for field in flst:
mid = row[field]
if mid is not None:
self.query(f'select val,w,h from mat where mid=?', [mid])
rone = self.get_one()
dic[field] = unpack_data(rone['val'], rone['w'], rone['h'])
return Data(columns=zlibext_unpack(row['cols']), **dic)
def get_result_impl(self, component: Component, input_data):
"""
Look for a result in database. Download only affected matrices/vectors.
ps.: put a model inside component requested
:return: Resulting Data
"""
if component.failed or component.locked_by_others:
return None, True, component.failed is not None
fields = [Data.from_alias[f] for f in component.modifies(component.op)]
if self._dump:
raise Exception('Are we really starting to store dump of '
'components?')
self.query(
f'''
select
des, spent, fail, end, host, nested as history, cols
{',' + ','.join(fields) if len(fields) > 0 else ''}
{', dump' if self._dump else ''}
from
res
left join data on dout = did
left join dataset on dataset = dsid
left join hist on hist = hid
left join attr on attr = aid
{'left join inst on inst=iid' if self._dump else ''}
where
config=? and op=? and dtr=? and din=?''', [
component.uuid, component.op,
component.train_data_uuid__mutable(),
input_data.uuid()
])
result = self.get_one()
if result is None:
return None, False, False
if result['des'] is not None:
# sanity check
if result['des'] != input_data.name:
raise Exception('Resulting data name differs from input data',
f"{result['des']}!={input_data.name}")
# Recover relevant matrices/vectors.
dic = {'X': None}
for field in fields:
mid = result[field]
if mid is not None:
self.query(f'select val,w,h from mat where mid=?', [mid])
rone = self.get_one()
if rone is not None:
dic[field] = \
unpack_data(rone['val'], rone['w'], rone['h'])
# Create Data.
history = zlibext_unpack(result['history'])
columns = zlibext_unpack(result['cols'])
data = Data(name=result['des'],
history=history,
columns=columns,
**dic)
# Join untouched matrices/vectors.
output_data = input_data.merged(data)
else:
output_data = None
component.model = result['dump'].model if 'dump' in result else None
component.time_spent = result['spent']
component.failed = result['fail'] and result['fail'] == 1
component.locked_by_others = result['end'] == '0000-00-00 00:00:00'
component.host = result['host']
ended = component.failed is not None
return output_data, True, ended