def train(self, intervals):
finish = False
prepare = Preparer(intervals, **self.config)
while not finish:
# get events
rows, flag = prepare.get_data_from_db()
batch_states = []
batch_newstates = []
batch_actions = []
for row in rows:
# get info about event
time_event = None
tag_id = None
user_id = None
time_delta = None
# init features
state = self.model.get_features(user_id, tag_id, time_event)
next_state = self.model.get_features(user_id, tag_id,
time_event + time_delta)
action = 1
batch_states.append(state)
batch_newstates.append(next_state)
batch_actions.append(action)
if len(batch_states) > 0:
self.model_dqnn.train(batch_states, batch_newstates,
batch_actions)
if not flag:
finish = prepare.next_iteration()
def __init__(self):
self.client = docker.from_env(timeout=86400)
self.preparer = Preparer()
self.searcher = Searcher()
self.trainer = Trainer()
self.interactor = Interactor()
self.generate_save_tag = lambda tag, save_id: hashlib.sha256(
(tag + save_id).encode()).hexdigest()
def prepare_backup(self, host, backup_type, backup_dir, prepare_dir, logger):
prepare_obj = Preparer(host=host,
backup_type=backup_type,
backup_dir=backup_dir,
prepare_dir=prepare_dir,
logger=logger)
prepare_obj.setup()
return prepare_obj.prepare()
def prepare_backup(self, host, backup_type, backup_dir, prepare_dir,
logger):
prepare_obj = Preparer(host=host,
backup_type=backup_type,
backup_dir=backup_dir,
prepare_dir=prepare_dir,
logger=logger)
prepare_obj.setup()
return prepare_obj.prepare()
def prepare_backup(self, host, backup_type, backup_dir, prepare_dir, logger):
prepare_obj = Preparer(host=host,
backup_type=backup_type,
backup_dir=backup_dir,
prepare_dir=prepare_dir,
logger=logger)
backup_lock.acquire()
try:
prepare_obj.setup()
finally:
backup_lock.release()
return prepare_obj.prepare()
def prepare_backup(self, host, backup_type, backup_dir, prepare_dir,
logger):
prepare_obj = Preparer(host=host,
backup_type=backup_type,
backup_dir=backup_dir,
prepare_dir=prepare_dir,
logger=logger)
backup_lock.acquire()
try:
prepare_obj.setup()
finally:
backup_lock.release()
return prepare_obj.prepare()
class Manager:
def __init__(self):
self.client = docker.from_env(timeout=86400)
self.preparer = Preparer()
self.searcher = Searcher()
self.trainer = Trainer()
self.interactor = Interactor()
self.generate_save_tag = lambda tag, save_id: hashlib.sha256(
(tag + save_id).encode()).hexdigest()
def set_preparer_config(self, preparer_config):
self.preparer.set_config(preparer_config)
def set_searcher_config(self, searcher_config):
self.searcher.set_config(searcher_config)
def set_trainer_config(self, trainer_config):
self.trainer.set_config(trainer_config)
def set_interactor_config(self, interactor_config):
self.interactor.set_config(interactor_config)
def prepare(self, preparer_config=None):
if preparer_config:
self.set_preparer_config(preparer_config)
self.preparer.prepare(self.client, COLLECTION_PATH_GUEST,
self.generate_save_tag)
def search(self, searcher_config=None):
if searcher_config:
self.set_searcher_config(searcher_config)
self.searcher.search(self.client, OUTPUT_PATH_GUEST, TOPIC_PATH_HOST,
TOPIC_PATH_GUEST, TEST_SPLIT_PATH_GUEST,
self.generate_save_tag)
def train(self, trainer_config=None):
if trainer_config:
self.set_trainer_config(trainer_config)
self.trainer.train(self.client, TOPIC_PATH_GUEST,
TEST_SPLIT_PATH_GUEST, VALIDATION_SPLIT_PATH_GUEST,
self.generate_save_tag)
def interact(self, interactor_config=None):
if interactor_config:
self.set_interactor_config(interactor_config)
self.interactor.interact(self.client, self.generate_save_tag)
def predict(self, intervals):
finish = False
prepare = Preparer(intervals, **self.config)
while not finish:
# get events
rows, flag = prepare.get_data_from_db()
for row in rows:
# get info about event
time_event = None
tag_id = None
user_id = None
time_delta = None
# init features
state = self.model.get_features(user_id, tag_id, time_event)
predict = self.model_dqnn.predict(state)
def predict(self, model_dqnn, intervals):
finish = False
estimator = Estimator()
prepare = Preparer(intervals, **self.config)
while not finish:
# get events
rows, flag = prepare.get_data_from_db()
for row in rows:
# get info about event
time_event = None
user_id = None
time_delta = None
time_state = time_event - datetime.timedelta(seconds=1)
time_next_state = time_event + time_delta
categories = self.model.get_read_categories(
user_id, time_state, time_next_state, self.all_categories)
result = {}
for tag_id in categories:
action = categories[tag_id]
state = self.model.get_features(user_id, tag_id,
time_state)
predict = model_dqnn.predict(state)
def train(self, model_dqnn, intervals):
finish = False
prepare = Preparer(intervals, **self.config)
prepare.generate_category_features()
self.all_categories = [
categorie[0] for categorie in prepare.list_categories
]
while not finish:
# get events
rows, flag = prepare.get_encode_data_to_db()
batch_states = []
batch_newstates = []
batch_actions = []
for row in rows:
# get info about event
id_event = row['id']
event = self.model._dictionary.get_coment(id_event)
time_event = event['time']
user_id = event['username_id']
time_delta = datetime.timedelta(hours=1)
# init features
time_state = time_event - datetime.timedelta(seconds=1)
time_next_state = time_event + time_delta
categories = self.model.get_read_categories(
user_id, time_state, time_next_state, self.all_categories)
for tag_id in categories:
action = categories[tag_id]
state = self.model.get_features(user_id, tag_id,
time_state)
if state is None:
continue
next_state = self.model.get_features(
user_id, tag_id, time_next_state)
batch_states.append(state)
batch_newstates.append(next_state)
batch_actions.append(action)
if len(batch_states) > 0:
model_dqnn.train(batch_states, batch_newstates, batch_actions)
if not flag:
finish = prepare.next_iteration()