def start_from_file(self, fileref):
#logmessage("Starting from file " + str(fileref))
existing_entry = db.session.execute(select(MachineLearning).filter_by(group_id=self.group_id)).first()
if existing_entry is not None:
return
file_info = get_info_from_file_reference(fileref, folder='sources')
if 'fullpath' not in file_info or file_info['fullpath'] is None or not os.path.exists(file_info['fullpath']):
return
#raise Exception("File reference " + str(fileref) + " is invalid")
with open(file_info['fullpath'], 'r', encoding='utf-8') as fp:
content = fp.read()
if 'mimetype' in file_info and file_info['mimetype'] == 'application/json':
aref = json.loads(content)
elif 'extension' in file_info and file_info['extension'].lower() in ['yaml', 'yml']:
aref = yaml.load(content, Loader=yaml.FullLoader)
if isinstance(aref, dict) and hasattr(self, 'group_id'):
the_group_id = re.sub(r'.*:', '', self.group_id)
if the_group_id in aref:
aref = aref[the_group_id]
if isinstance(aref, list):
nowtime = datetime.datetime.utcnow()
for entry in aref:
if 'independent' in entry:
depend = entry.get('dependent', None)
if depend is not None:
new_entry = MachineLearning(group_id=self.group_id, independent=codecs.encode(pickle.dumps(entry['independent']), 'base64').decode(), dependent=codecs.encode(pickle.dumps(depend), 'base64').decode(), modtime=nowtime, create_time=nowtime, active=True, key=entry.get('key', None), info=codecs.encode(pickle.dumps(entry['info']), 'base64').decode() if entry.get('info', None) is not None else None)
else:
new_entry = MachineLearning(group_id=self.group_id, independent=codecs.encode(pickle.dumps(entry['independent']), 'base64').decode(), modtime=nowtime, create_time=nowtime, active=False, key=entry.get('key', None), info=codecs.encode(pickle.dumps(entry['info']), 'base64').decode() if entry.get('info', None) is not None else None)
db.session.add(new_entry)
db.session.commit()
def add_to_training_set(self, independent, dependent, key=None, info=None):
self._initialize()
nowtime = datetime.datetime.utcnow()
if dependent is not None:
new_entry = MachineLearning(group_id=self.group_id, independent=codecs.encode(pickle.dumps(independent), 'base64').decode(), dependent=codecs.encode(pickle.dumps(dependent), 'base64').decode(), info=codecs.encode(pickle.dumps(info), 'base64').decode() if info is not None else None, create_time=nowtime, modtime=nowtime, active=True, key=key)
else:
new_entry = MachineLearning(group_id=self.group_id, independent=codecs.encode(pickle.dumps(independent), 'base64').decode(), info=codecs.encode(pickle.dumps(info), 'base64').decode() if info is not None else None, create_time=nowtime, modtime=nowtime, active=False, key=key)
db.session.add(new_entry)
db.session.commit()
return new_entry.id
def _save_entry(self, **kwargs):
self._initialize()
the_id = kwargs.get('id', None)
need_to_reset = False
if the_id is None:
the_entry = MachineLearning(group_id=self.group_id)
existing = False
else:
the_entry = db.session.execute(select(MachineLearning).filter_by(group_id=self.group_id, id=the_id)).scalar()
existing = True
if the_entry is None:
raise Exception("There was no entry in the database for id " + str(the_id) + " with group id " + str(self.group_id))
if 'dependent' in kwargs:
if existing and the_entry.dependent is not None and the_entry.dependent != kwargs['dependent']:
need_to_reset = True
the_entry.dependent = codecs.encode(pickle.dumps(kwargs['dependent']), 'base64').decode()
the_entry.active = True
if 'independent' in kwargs:
if existing and the_entry.independent is not None and the_entry.independent != kwargs['independent']:
need_to_reset = True
the_entry.independent = codecs.encode(pickle.dumps(kwargs['independent']), 'base64').decode()
if 'key' in kwargs:
the_entry.key = kwargs['key']
if 'info' in kwargs:
the_entry.info = codecs.encode(pickle.dumps(kwargs['info']), 'base64').decode()
the_entry.modtime = datetime.datetime.utcnow()
if not existing:
db.session.add(the_entry)
db.session.commit()
if need_to_reset:
self.reset()
def save_for_classification(self, indep, key=None, info=None):
self._initialize()
if key is None:
existing_entry = db.session.execute(
select(MachineLearning).filter_by(
group_id=self.group_id,
dependent=None,
independent=codecs.encode(pickle.dumps(indep),
'base64').decode())).scalar()
else:
existing_entry = db.session.execute(
select(MachineLearning).filter_by(
group_id=self.group_id,
key=key,
independent=codecs.encode(pickle.dumps(indep),
'base64').decode())).scalar()
if existing_entry is not None:
#logmessage("entry is already there")
return existing_entry.id
new_entry = MachineLearning(
group_id=self.group_id,
independent=codecs.encode(pickle.dumps(indep), 'base64').decode(),
create_time=datetime.datetime.utcnow(),
active=False,
key=key,
info=codecs.encode(pickle.dumps(info), 'base64').decode()
if info is not None else None)
db.session.add(new_entry)
db.session.commit()
return new_entry.id
def _save_entry(self, **kwargs):
self._initialize()
the_id = kwargs.get('id', None)
need_to_reset = False
if the_id is None:
the_entry = MachineLearning(group_id=self.group_id)
existing = False
else:
the_entry = MachineLearning.query.filter_by(group_id=self.group_id, id=the_id).first()
existing = True
if the_entry is None:
raise Exception("There was no entry in the database for id " + str(the_id) + " with group id " + str(self.group_id))
if 'dependent' in kwargs:
if existing and the_entry.dependent is not None and the_entry.dependent != kwargs['dependent']:
need_to_reset = True
the_entry.dependent = codecs.encode(pickle.dumps(kwargs['dependent']), 'base64').decode()
the_entry.active = True
if 'independent' in kwargs:
if existing and the_entry.independent is not None and the_entry.independent != kwargs['independent']:
need_to_reset = True
the_entry.independent = codecs.encode(pickle.dumps(kwargs['independent']), 'base64').decode()
if 'key' in kwargs:
the_entry.key = kwargs['key']
if 'info' in kwargs:
the_entry.info = codecs.encode(pickle.dumps(kwargs['info']), 'base64').decode()
the_entry.modtime = datetime.datetime.utcnow()
if not existing:
db.session.add(the_entry)
db.session.commit()
if need_to_reset:
self.reset()
def save_for_classification(self, text, key=None):
self._initialize()
if key is None:
existing_entry = MachineLearning.query.filter_by(group_id=self.group_id, dependent=None, independent=codecs.encode(pickle.dumps(text), 'base64').decode()).first()
else:
existing_entry = MachineLearning.query.filter_by(group_id=self.group_id, key=key, independent=codecs.encode(pickle.dumps(text), 'base64').decode()).first()
if existing_entry is not None:
logmessage("entry is already there")
return existing_entry.id
new_entry = MachineLearning(group_id=self.group_id, independent=codecs.encode(pickle.dumps(text), 'base64').decode(), create_time=datetime.datetime.utcnow(), active=False, key=key)
db.session.add(new_entry)
db.session.commit()
return new_entry.id
def start_from_file(self, fileref):
#logmessage("Starting from file " + str(fileref))
existing_entry = MachineLearning.query.filter_by(
group_id=self.group_id).first()
if existing_entry is not None:
return
file_info = get_info_from_file_reference(fileref, folder='sources')
if 'fullpath' not in file_info or file_info[
'fullpath'] is None or not os.path.exists(
file_info['fullpath']):
raise Exception("File reference " + str(fileref) + " is invalid")
with open(file_info['fullpath'], 'rU') as fp:
content = fp.read().decode('utf8')
if 'mimetype' in file_info and file_info[
'mimetype'] == 'application/json':
aref = json.loads(content)
elif 'extension' in file_info and file_info['extension'].lower() in [
'yaml', 'yml'
]:
aref = yaml.load(content)
if type(aref) is list:
nowtime = datetime.datetime.utcnow()
for entry in aref:
if 'independent' in entry:
new_entry = MachineLearning(
group_id=self.group_id,
independent=codecs.encode(
pickle.dumps(entry['independent']),
'base64').decode(),
dependent=codecs.encode(
pickle.dumps(entry.get('dependent', None)),
'base64').decode(),
modtime=nowtime,
create_time=nowtime,
active=True,
key=entry.get('key', None),
info=codecs.encode(pickle.dumps(
entry['info']), 'base64').decode() if entry.get(
'info', None) is not None else None)
db.session.add(new_entry)
db.session.commit()
