def remove(self, partial_id):
entry = self.get_from_db(partial_id)
if not self.dry:
self.db.remove_by_partial_id(partial_id)
utils.remove_file_or_dir(os.path.join(self.location, entry.id))
return entry
def restore(self):
"""Restores the game directory using the backed up files and downloaded files."""
# Check some stuff
if not hasattr(self, "game_dir") or self.game_dir is None:
print("Please select a game directory")
return
if not self.backup_dir.exists():
print("Backup directory doesn't exist")
return
if len(os.listdir(self.backup_dir.absolute())) == 0:
print("No backup stored")
return
backup_file_list = list(os.listdir(self.backup_dir.absolute()))
download_file_list = list(os.listdir(self.download_dir.absolute()))
# Remove added files from the path
print("Removing patched files")
for file in download_file_list:
utils.remove_file_or_dir(self.game_dir, file)
print("Finished removing patched files")
# Copy backed up files to game path again
print("Restoring backup")
for file in backup_file_list:
utils.copy_file_or_dir(self.backup_dir, self.game_dir, file)
print("Finished restoring backup")
print("DONE!")
def get_agent_file(agent_name):
directory_name = os.environ.get("MODELS_PATH") + "export/"
file_name = agent_name + ".json"
file_path = directory_name + file_name
remove_file_or_dir(file_path)
dic = AgentsService.get_instance().create_agent_file(agent_name)
create_folder(directory_name)
with open(file_path, "w+") as f:
json.dump(dic,f)
return(send_from_directory(directory = directory_name, filename = "./" + file_name, as_attachment = True))
def delete_agent(self, agent_name):
""" Remove entries in database """
self.cleanup_agent(agent_name)
""" Remove persisted models """
model_loader = get_loader(os.environ.get("MODEL_LOADER"))
model_loader.remove_all(agent_name)
""" Remove model file """
remove_file_or_dir(os.environ.get("MODELS_PATH") + agent_name)
self.delete_agent_from_memory(agent_name)
def __enter__(self):
self.start = datetime.now()
all_trained_agents = list(
AgentsService.get_instance().get_trained_agents())
logger.info("Agents found for Loading: {0}".format(all_trained_agents))
for agent in all_trained_agents:
try:
logger.info("Start loading Agent: {0}".format(
agent.get("name")))
AgentsService.get_instance().load_agent(
agent.get("name"), agent.get("last_version"))
except Exception as e:
logger.error(
"Exception when loading agent {0} with version {1} inside Cron Loader. {2}"
.format(agent.get("name"), agent.get("last_version"), e),
exc_info=True)
remove_file_or_dir(modelName)
def load_agent(self, agent_name, model_name):
model_loader = get_loader(os.environ.get("MODEL_LOADER"))
download_path = model_loader.retrieve(agent_name, model_name)
self.bots[agent_name] = Agent(model=download_path)
self.agents_repository.update_model(agent_name, model_name)
logger.info("Agent {0} loaded with version {1}, ".format(
agent_name, model_name))
try:
downloadDir = os.environ.get("MODELS_PATH") + agent_name + "/"
for root, dirs, files in os.walk(downloadDir):
for file in files:
if not file == model_name + ".tar.gz":
remove_file_or_dir(os.path.join(root, file))
except Exception as e:
logger.error(
"Exception when deleting models for agent {0}, after loading model {1}. {2}"
.format(agent_name, model_name, e),
exc_info=True)
def __init__(self, **kwargs):
model_path = kwargs.get("model", None)
try:
if model_path is not None:
logger.info("Loading model " + model_path)
if os.path.isdir(model_path):
self.interpreter = Interpreter.load(model_path)
elif model_path.endswith(".tar.gz"):
model = unpack_model(model_path)
if os.path.isdir(model + "/nlu"):
self.interpreter = Interpreter.load(model + "/nlu")
else:
self.interpreter = Interpreter.load(model)
self.interpreter.parse("ok")
"""Cleanup tmp files and directories"""
try :
if model is not None:
remove_file_or_dir(model)
tmp_dir = "/tmp/"
for root, dirs, files in os.walk(tmp_dir):
for file in files:
if file.startswith("tmp") and file.endswith(".py"):
remove_file_or_dir(tmp_dir+file)
except Exception as e:
logger.error("Exception when cleanup tmp files and directories.", exc_info=True)
else:
self.interpreter = None
except Exception as e:
logger.error("Error when loading model {0}, exception {1}".format(model_path, e), exc_info=True)
remove_file_or_dir(model_path)
raise
def train_agent(self, agent_name):
try:
train_data = {}
train_data["rasa_nlu_data"] = {}
training = self._get_agent_training_data(agent_name)
train_data["rasa_nlu_data"]["common_examples"] = training
if training in [[], None]:
logger.warn(
"Agent {0} has no training data".format(agent_name))
return ("Could not found training data for Agent {0}".format(
agent_name))
if self._has_at_list_two_intents(training):
logger.warn(
"Agent {0} has only one intent, while a minimum of two intents are needed"
.format(agent_name))
return ("Could not train Agent {0}, only found 1 intent".
format(agent_name))
else:
try:
os.makedirs("./models/" + agent_name)
except FileExistsError:
"""directory already exists"""
pass
train_data["rasa_nlu_data"][
"lookup_tables"] = self._get_agent_lookups(agent_name)
train_data["rasa_nlu_data"][
"entity_synonyms"] = self._get_agent_synonyms(agent_name)
config = self.agents_repository.get_agent_config(
agent_name).get("config")
training_data_file = os.environ.get(
"MODELS_PATH") + agent_name + ".json"
config_file = os.environ.get(
"MODELS_PATH") + agent_name + "_config.json"
create_file(training_data_file, json.dumps(train_data))
create_file(config_file, json.dumps(config))
agent = Agent(agentName=agent_name,
botconfig=config_file,
data=training_data_file)
model_recorder = get_recorder(os.environ.get("MODEL_RECORDER"))
model_recorder.save(agent_name, agent.model_path,
agent.model_name, agent.model_version)
versions = model_recorder.list_versions(agent_name)
now = int(datetime.timestamp(datetime.now()))
self.agents_repository.update_trained_agent(
agent_name, agent, versions, now)
"""Clean Up"""
remove_file_or_dir(training_data_file)
remove_file_or_dir(config_file)
remove_file_or_dir(os.environ.get("MODELS_PATH") + agent_name)
logger.info(
"Bot {0}, successfully trained, and model {1} persisted. ".
format(agent_name, agent.model_name))
except Exception as e:
logger.error("Exception when training agent {0}. {1}".format(
agent_name, e),
exc_info=True)
file_name = file_list[i]
src_path = constant.APK_SOURCES_DIR + file_name
file_md5 = utils.get_file_md5(src_path)
file_size = os.path.getsize(src_path)
file_size /= float(1024 * 1024)
file_size = round(file_size, 2)
print('1. file_md5: ' + file_md5)
print('2. file_name: ' + file_name)
print('3. file_size: ' + str(file_size) + 'MB')
dst_path = constant.OUTCOMES_DIR + file_md5 + '/'
try:
utils.make_dir(dst_path)
except OSError:
print('3. The Num.' + str(i) + ' apk workspace has existed')
else:
print('>> The Num.' + str(i) +
' workspace is been created in outcomes successfully')
decompiler.decompile(src_path)
utils.remove_file_or_dir('apktool_unzip', dst_path)
utils.remove_file_or_dir('apk_enjarify.jar', dst_path)
utils.remove_file_or_dir('procyon_decompile_java/', dst_path)
print('7. Scanner: Begin static analysis...')
src_path = constant.APK_SOURCES_DIR + file_name
dst_path = constant.OUTCOMES_DIR + file_md5 + '/'
# scanner.do_static_scan(src_path, dst_path)
# scanner.do_dynamic_scan(src_path, dst_path)
# print('------The Num.' + str(i) + ' apk scanning finishes------\n')