def removeModel(model_deploy_id):
working_dir_path = getCurrentWorkingDir();
storage_dir = "{}/{}".format(working_dir_path, configuration.storage_folder_name);
deploy_dir = "{}/model{}".format(storage_dir, model_deploy_id);
with rhythmicDB(configuration.db_name, configuration.db_file_name) as db:
db.execute(
"""
DELETE FROM models_table WHERE id = '{}';
""".format(
model_deploy_id
)
);
deployment_files = scanModelFolder(deploy_dir);
deployment_folders = [deploy_dir];
for the_file in deployment_files:
if deployment_files[the_file]["is_dir"]:
deployment_folders.append(the_file);
else:
remove(the_file);
if len(deployment_folders) > 0:
for the_folder in sorted(deployment_folders, key = lambda folder_name: len(folder_name), reverse = True):
rmDir(the_folder);
return "Success";
def main():
with rhythmicDB(configuration.db_name, configuration.db_file_name) as db:
db.runScript(general_table + models_table)
print(
"Database is created, tables are created. DB file: \"{}\", {}".format(
configuration.db_file_name, str(datetime.now())))
def removeModel(data):
"""
var data_for_helper =
{
"model_id": the_id,
"model_path": model_path
}
"""
with rhythmicDB(configuration.db_name, configuration.db_file_name) as db:
db.execute("PRAGMA foreign_keys = ON;")
deploy_destination_request_result = \
db.execute(
"""
SELECT deploy_destination, deploy_id FROM models_table WHERE id = '{}'
""".format(
data["model_id"]
)
)
db.execute("""
DELETE FROM models_table WHERE id = '{}';
""".format(data["model_id"]))
model_storage_path = data["model_path"] + "/{}".format(
configuration.storage_folder_name)
versions_packages = scanFolder(model_storage_path, look_level_above=False)
for version_package in versions_packages:
remove(version_package)
rmDir(model_storage_path)
removal_operation_status = "Success"
if deploy_destination_request_result:
deploy_destination = deploy_destination_request_result[0][0]
model_deploy_id = deploy_destination_request_result[0][1]
print("{}/helpers/remove_deployment/{}".format(deploy_destination,
model_deploy_id))
try:
deploy_server_request_result = requests.post(
"{}/helpers/remove_deployment/{}".format(
deploy_destination, model_deploy_id))
if deploy_server_request_result.status_code != 200:
removal_operation_status = "Probably the deployment has to be removed manually from the server side: {}".\
format(deploy_server_request_result.status_code)
except Exception as what_is_wrong:
removal_operation_status = "Probably the deployment has to be removed manually from the server side: {}".\
format(what_is_wrong)
print(removal_operation_status)
return removal_operation_status
def getModelsList():
the_models_list = [];
with rhythmicDB(db_name = "SQLite", db_filename = configuration.db_file_name) as db:
deployed_models = db.execute("SELECT * FROM models_table WHERE 1");
if len(deployed_models) > 0:
for the_model_data_row in deployed_models:
the_models_list.append(modelPropertiesDictionary(the_model_data_row));
return the_models_list;
def uniqueName(the_name):
unique_name = the_name;
probe = [None]
i = 0;
with rhythmicDB(configuration.db_name, configuration.db_file_name) as db:
while len(probe) !=0:
probe = db.execute("SELECT model_name FROM models_table WHERE model_name = '{}'".format(unique_name));
if len(probe) !=0:
i += 1;
unique_name = "{}_{}".format(the_name, str(i));
return unique_name;
def getModelsList():
with rhythmicDB(configuration.db_name, configuration.db_file_name) as db:
models_table = db.execute(
"""
SELECT * FROM models_table WHERE 1 ORDER BY last_version_timestamp DESC;
""");
models = [];
for stored_model in models_table:
the_model = modelPropertiesDictionary(stored_model);
the_model["deploy_status"] = getModelDeployStatus(the_model);
models.append(the_model);
return models;
def restoreFile(data):
"""
var data =
{
"file_absolute_path": absolute_path,
"model_path": window.model_path,
"model_id": window.the_model_id,
"version_number": window.active_version_number,
"version_id": window.active_version_id
}
"""
model_version_archive_file_name = "{}/{}/model_{}_ver{}.zip".\
format(
data["model_path"],
configuration.storage_folder_name,
data["model_id"],
data["version_number"]
);
unpackSingleFile(model_version_archive_file_name, data["file_absolute_path"], data["model_path"]);
#after unpacking we have to update last_modified_time for UI would not mark that file as modified
base_index = data["file_absolute_path"].rfind("/");
file_containing_folder = data["file_absolute_path"][: base_index + 1]
folder_contents = scanFolder(file_containing_folder);
if data["file_absolute_path"] in folder_contents:
with rhythmicDB(configuration.db_name, configuration.db_file_name) as db:
db.execute(
"""
UPDATE files_table SET last_modified_time ='{}' WHERE model_version_id = '{}' AND absolute_path = '{}';
""".format(
folder_contents[ data["file_absolute_path"] ]["last_modified_time"],
data["version_id"],
data["file_absolute_path"]
)
);
return "Success";
def updateMetadataAndDeployables(data):
with rhythmicDB(configuration.db_name, configuration.db_file_name) as db:
if data["metadata_changed"]:
db.execute(
"UPDATE versions_table SET metadata = '{}' WHERE id='{}';".
format(data["actual_metadata"], data["active_version_id"]))
if data["deployables_changed"]:
for item in data["changed_items"]:
#SQLite has no boolean types:
if data["changed_items"][item]:
is_deployed = 1
else:
is_deployed = 0
db.execute(
"UPDATE files_table SET is_deployed = '{}' WHERE id = '{}';"
.format(is_deployed, item))
return "Success"
def deployModel(deploy_files):
"""
deploy_files is a dictionary of werkzeug `FileStorage`s
data passed with file:
var data =
{
"deploy_id": window.the_model_deploy_id,
"model_name": window.the_model_name,
"model_id": window.the_model_id,
"version_number": window.active_version_number,
"model_metadata": encodeURI(window.actual_metadata)
}
"""
deploy_result = {}
timestamp = datetime.now()
working_dir_path = getCurrentWorkingDir()
storage_dir = "{}/{}".format(working_dir_path,
configuration.storage_folder_name)
checkDir(storage_dir)
for file_name in deploy_files:
if file_name.endswith(".json"):
deploy_data_file_name = file_name
else:
artifacts_archive_name = file_name
if deploy_data_file_name:
metadata_encoded_json = deploy_files[deploy_data_file_name].read()
else:
return "Fault: no metadata file received"
metadata_json = metadata_encoded_json.decode()
metadata = json.loads(metadata_json)
# got the metadata here, putting data to db:
with rhythmicDB(configuration.db_name, configuration.db_file_name) as db:
if metadata["deploy_id"] != 0:
deploy_id_ok = db.execute("""
SELECT id FROM models_table WHERE id = '{}';
""".format(metadata["deploy_id"]))
if deploy_id_ok:
passed_deploy_id = metadata["deploy_id"]
else:
passed_deploy_id = 0
else:
passed_deploy_id = 0
if passed_deploy_id == 0:
model_deploy_id = db.execute("""
INSERT INTO models_table
(
model_name,
last_deploy_timestamp,
active_version,
build_id
)
VALUES
('{}', '{}', '{}', '{}');
""".format(metadata["model_name"], timestamp,
metadata["version_number"], metadata["model_id"]))
else:
model_deploy_id = passed_deploy_id
db.execute("""
UPDATE models_table SET
last_deploy_timestamp = '{}',
active_version = '{}'
WHERE id = '{}';
""".format(timestamp, metadata["version_number"],
model_deploy_id))
# finished with db here, let's create a folder and unpack artifacts there
deploy_dir = "{}/model{}".format(storage_dir, model_deploy_id)
checkDir(deploy_dir)
model_files_dir = "{}/files".format(deploy_dir)
checkDir(model_files_dir)
new_archive_name = "deploy_{}_{}.zip".format(metadata["model_id"],
model_deploy_id)
new_data_file_name = "deploy_{}_{}.json".format(metadata["model_id"],
model_deploy_id)
deploy_files[artifacts_archive_name].save("{}/{}".format(
deploy_dir, new_archive_name))
deploy_files[deploy_data_file_name].save("{}/{}".format(
deploy_dir, new_data_file_name))
archive_path = "{}/{}".format(deploy_dir, new_archive_name)
with ZipFile(archive_path) as deploy_package:
deploy_package.extractall(model_files_dir)
#################################
#Unpacking done. call runModel (got ot write it first);
#################################
deploy_result["model_deploy_id"] = model_deploy_id
deploy_result["status"] = "Success"
return deploy_result
def deployModel(data):
"""
var data =
{
"deploy_url": window.actual_deploy_destination,
"deploy_id": window.the_model_deploy_id,
"model_path": window.model_path,
"model_name": window.the_model_name,
"model_id": window.the_model_id,
"version_number": window.active_version_number,
"files_data": window.active_version_files_data,
"model_metadata": encodeURI(window.actual_metadata)
}
"""
deploy_url = "{}/deploy".format(data["deploy_url"]);
deploy_data = {};
excluded_properties = \
[
"deploy_url",
"model_path",
"files_data"
];
for the_property in data:
if the_property not in excluded_properties:
deploy_data[the_property] = data[the_property];
deploy_data_json = json.dumps(deploy_data);
deployed_files = {};
for the_file in data["files_data"]:
if data["files_data"][the_file]["is_deployed"]:
deployed_files[the_file] = data["files_data"][the_file];
package_name = "deploy_{}_{}.zip".format(data["model_id"], data["deploy_id"]);
data_file_name = "deploy_{}_{}.json".format(data["model_id"], data["deploy_id"]);
package_path = "{}/{}/{}".format(data["model_path"], configuration.storage_folder_name, package_name);
data_file_path = "{}/{}/{}".format(data["model_path"], configuration.storage_folder_name, data_file_name);
with open(data_file_path, "w+") as deploy_data_file:
deploy_data_file.write(deploy_data_json);
packFiles(data["model_path"], package_path, deployed_files);
package_file = open(package_path, "rb");
data_file = open(data_file_path, "rb");
deploy_files = {data_file_name: data_file, package_name: package_file};
try:
files_deploy_result = requests.post(deploy_url, files = deploy_files);
except Exception as e:
return "Deploy failed: {}".format(e);
package_file.close();
data_file.close();
deploy_execution_data_json = files_deploy_result.content;
deploy_execution_data = json.loads(deploy_execution_data_json);
# ***** deploy_result["model_deploy_id"] = model_deploy_id;
# ***** deploy_result["status"] = "Success";
deploy_id = deploy_execution_data["model_deploy_id"];
deploy_status = deploy_execution_data["status"];
if deploy_id != data["deploy_id"]:
with rhythmicDB(configuration.db_name, configuration.db_file_name) as db:
db.execute(
"""
UPDATE models_table SET
deploy_id = '{}'
WHERE id = '{}';
""".format(
deploy_id,
data["model_id"]
)
);
return deploy_status;
def setModelVersionActive(data):
"""
var data =
{
"model_id": window.the_model_id,
"model_path": window.model_path,
"desired_version_id": desired_version_id,
"desired_version_number": desired_version_number
}
"""
model_version_archive_file_name = "{}/{}/model_{}_ver{}.zip".\
format(
data["model_path"],
configuration.storage_folder_name,
data["model_id"],
data["desired_version_number"]
)
if not exists(model_version_archive_file_name):
return "No stored version {} package found. Activation cancelled.".format(
data["desired_version_number"])
files_to_purge = scanModelFolder(data["model_path"])
for filename in files_to_purge:
remove(filename)
unpackVersion(model_version_archive_file_name, data["model_path"])
# we have now to udate last_modified timestamps so system would not identify freshly unpacked files as modified.
# to minimize db queries, we'll just rewrite existinf rows in files_table.
# but first we'll set proper active_version for the model to perform al db operations in one ocntext.
with rhythmicDB(configuration.db_name, configuration.db_file_name) as db:
db.execute("""
UPDATE models_table SET active_version = '{}' WHERE id = '{}';
""".format(data["desired_version_number"], data["model_id"]))
existing_files_records = db.execute("""
SELECT * FROM files_table WHERE model_version_id = '{}';
""".format(data["desired_version_id"]))
db.execute("""
DELETE FROM files_table WHERE model_version_id = '{}';
""".format(data["desired_version_id"]))
files_record_request = \
"""
INSERT INTO files_table
(
model_version_id,
absolute_path,
file_commit_state,
last_modified_time,
is_deployed
)
VALUES
"""
active_version_files = scanModelFolder(data["model_path"])
files_record_values = ""
for the_record in existing_files_records:
version_file_properties = filePropertiesDictionary(the_record)
file_absolute_path = version_file_properties["absolute_path"]
if (version_file_properties["is_deployed"]):
is_deployed = 1
else:
is_deployed = 0
if file_absolute_path in active_version_files:
last_modified_time = active_version_files[file_absolute_path][
"last_modified_time"]
else:
last_modified_time = version_file_properties[
"last_modified_time"]
files_record_values += \
"('{}', '{}', '{}', '{}', '{}'), \n".format(
data["desired_version_id"],
file_absolute_path,
version_file_properties["file_commit_state"],
last_modified_time,
is_deployed
)
files_record_request += files_record_values[:len(files_record_values) -
3] + ";"
#truncating `, \n` from the end of request and adding `;`.
db.execute(files_record_request)
return "Success"
def createNewVersion(data):
timestamp = str(datetime.now())
#=================starting DB work =====================================
with rhythmicDB(configuration.db_name, configuration.db_file_name) as db:
update_model_parameters_request = \
"""
UPDATE models_table SET
last_version_timestamp = '{}',
last_version = '{}',
active_version = '{}'
WHERE id = '{}';
""".format(
timestamp,
data["new_version_number"],
data["new_version_number"],
data["model_id"]
)
db.execute(update_model_parameters_request)
add_new_version_request = \
"""
INSERT INTO versions_table
(
model_id,
version,
metadata,
commit_comment,
created_timestamp
)
VALUES ('{}', '{}', '{}', '{}', '{}');
""".format(
data["model_id"],
data["new_version_number"],
data["metadata"],
"Derives from version " + data["the_active_version"] + ".",
timestamp
)
new_version_id = db.execute(add_new_version_request)
record_files_data_request = \
"""
INSERT INTO files_table
(
model_version_id,
absolute_path,
file_commit_state,
last_modified_time,
is_deployed
)
VALUES
"""
if len(data["files_tracker"]) > 0:
files_record_values = ""
for the_file in data["files_tracker"]:
file_properties = data["files_tracker"][the_file]
last_modified_time = file_properties["last_modified_time"]
if (the_file in data["modified_files"]):
file_commit_state = "modified"
last_modified_time = data["modified_files"][the_file]
elif file_properties["file_commit_state"] == "emerged":
file_commit_state = "new"
else:
file_commit_state = "same"
#SQLite has no boolean type, casting int:
if file_properties["is_deployed"]:
is_deployed = 1
else:
is_deployed = 0
files_record_values += "('{}', '{}', '{}', '{}', '{}'), \n".format(new_version_id, the_file, file_commit_state,\
last_modified_time, is_deployed)
record_files_data_request += files_record_values[:len(
files_record_values) - 3] + ";"
#truncating `, \n` from the end of request and adding `;`.
db.execute(record_files_data_request)
#================= finished DB work =====================================
model_storage_path = data["model_path"] + "/{}".format(
configuration.storage_folder_name)
if ((not exists(model_storage_path)) or (not isDir(model_storage_path))):
makeDir(model_storage_path)
#================= Starting building ver0 .zip in storage =====================================
archive_name = model_storage_path + "/model_{}_ver{}.zip".format(
data["model_id"], data["new_version_number"])
packFiles(data["model_path"], archive_name, data["files_tracker"])
#================= Finished building ver0 .zip in storage =====================================
return "Success"
from flask import Flask, request, render_template as renderTemplate;
from . import helpers;
from functools import wraps;
import json;
from rhythmic import rhythmicDB, Logger;
app = Flask(__name__);
#==========================================================================
#==================== INITIALIZATION =========================================
#==========================================================================
deploy_storage = helpers.DeployMemoryStorage();
with rhythmicDB(db_name = "SQLite", db_filename = helpers.configuration.db_file_name) as db:
predeployed_ids = db.execute("SELECT id FROM models_table WHERE 1");
if len(predeployed_ids) > 0:
for predeployed_id in predeployed_ids:
deploy_storage.deployCell(predeployed_id[0]);
deploy_logger = Logger();
deploy_logger.writeDown("Starting RhythmicML Deploy API Server.");
#==========================================================================
#==================== DECORATORS =========================================
#==========================================================================
def checkPost(entry_point):
@wraps(entry_point)
def wrapper(*args, **kwargs):
if request.method == "POST":
def modelAllStaticData(model_id):
all_model_info = \
{
"properties": {},
"model_versions": {}
};
with rhythmicDB(configuration.db_name, configuration.db_file_name) as db:
model_properties = db.execute(
"""
SELECT * FROM models_table WHERE id = '{}';
""".format(model_id)
);
all_model_info["properties"] = modelPropertiesDictionary(model_properties[0]);
all_model_info["properties"]["deploy_status"] = getModelDeployStatus(all_model_info["properties"]);
model_versions = db.execute(
"""
SELECT * FROM versions_table WHERE model_id = '{}' ORDER BY version DESC;
""".format(model_id)
);
for model_version in model_versions:
version_properties = versionPropertiesDictionary(model_version);
version_files = {};
version_tracked_files = db.execute(
"""
SELECT * FROM files_table WHERE model_version_id = '{}' ORDER BY absolute_path ASC;
""".format(version_properties["id"])
);
for version_tracked_file in version_tracked_files:
version_tracked_file_data = filePropertiesDictionary(version_tracked_file);
version_files_key = version_tracked_file_data["absolute_path"];
version_files[ version_files_key ] = version_tracked_file_data;
model_versions_key = version_properties["version"];
all_model_info["model_versions"][model_versions_key] = \
{
"version_properties": version_properties,
"version_files": version_files
};
if model_versions_key == all_model_info["properties"]["active_version"]:
#let us check for changes and prepare them for display, for active modelversion only
actual_folder_contents = scanModelFolder(all_model_info["properties"]["path"]);
absent_files, changed_folders, modified_files = differences(version_files, actual_folder_contents);
all_model_info["model_versions"][model_versions_key].update(
{
"changed_folders": changed_folders,
"absent_files": absent_files,
"modified_files": modified_files
});
return all_model_info;
def setNewModelDeployDestination(model_id, new_deploy_destionation):
with rhythmicDB(configuration.db_name, configuration.db_file_name) as db:
db.execute("UPDATE models_table SET deploy_destination = '{}' WHERE id = '{}';".format(new_deploy_destionation, model_id));
return "Success";
def addNewModel(model_name=None, model_dir=None):
"""
addNewModel(model_name = None, model_dir = None, db_file_name = ".rhml_db.sqlite3")
0. The path is checked to be already present in the database. If true, the according status returned, workflow stops.
1. Model wrapper script template copied to the model's directory.
2. The record containing model name, model path is added to `models_table`.
3. The record of version 0 is added to `versions_table`.
4. The model folder is scanned recursiveliy, adding all the files found to the `files_table` (absolute paths).
5. The `.rhml_storage` folder created within specified model directory.
6. The initial, 0-version archive is created.
"""
if model_dir == "~":
model_path = expandUser(model_dir)
else:
model_path = model_dir
if ((not model_name) or (not model_path)):
return "Can't add the model: model name or model path is missing."
timestamp = str(datetime.now())
#####################################################33##########3#####
templateSource = configuration.model_wrapper_class_file_name
templateDestination = "{}/{}".format(
model_path, configuration.model_wrapper_class_file_name)
if not exists(templateDestination):
copyFile(templateSource, templateDestination)
#######################################################################33
#=================starting DB work =====================================
with rhythmicDB(configuration.db_name, configuration.db_file_name) as db:
probe = db.execute(
"SELECT model_name FROM models_table WHERE model_path = '{}'".
format(model_path))
if len(probe) > 0:
return "The model [ {} ] stored in [ {} ] is already in the base.".format(
probe[0][0], model_path)
new_model_id = db.execute("""
INSERT INTO models_table
(
model_name,
model_path,
last_version_timestamp
)
VALUES
(
'{}', '{}', '{}'
);
""".format(model_name, model_path, timestamp))
new_model_version_id = db.execute("""
INSERT INTO versions_table
(
model_id,
created_timestamp
)
VALUES
(
'{}', '{}'
);
""".format(new_model_id, timestamp))
files_record_request = \
"""
INSERT INTO files_table
(
model_version_id,
absolute_path,
last_modified_time
)
VALUES
"""
new_model_files = scanModelFolder(model_path)
if len(new_model_files) > 0:
files_record_values = ""
for item_path in new_model_files:
item = new_model_files[item_path]
files_record_values += "('{}', '{}', '{}'), \n".format(
new_model_version_id, item_path,
item["last_modified_time"])
files_record_request += files_record_values[:len(
files_record_values) - 3] + ";"
#truncating `, \n` from the end of request and adding `;`.
db.execute(files_record_request)
#================= finished DB work =====================================
model_storage_path = model_path + "/{}".format(
configuration.storage_folder_name)
if ((not exists(model_storage_path)) or (not isDir(model_storage_path))):
makeDir(model_storage_path)
#================= Starting building ver0 .zip in storage =====================================
archive_name = model_storage_path + "/model_{}_ver0.zip".format(
new_model_id)
packFiles(model_path, archive_name, new_model_files)
#================= Finished building ver0 .zip in storage =====================================
return "Success"
