def get_hyperparams(request):
try:
username = request.data.get("username")
user = User(username=username, password=None)
user = user.find()
store_obj = Store(user)
project_id = request.data.get("project_id")
if not store_obj.exist(project_id):
raise Exception("No such project exists")
project_dir = store_obj.path + os.sep + project_id
with open(project_dir + os.sep + "hyperparams.json", "r") as f:
hyperparams = json.load(f)
status, success, message = 200, True, "hyperparams fetched"
except Exception as e:
status, success, message, hyperparams = (
500,
False,
"Could not fetch hyperparams ",
{},
)
return JsonResponse(
{"success": success, "message": message, "hyperparams": hyperparams},
status=status,
)
def delete_project(request):
try:
username = request.data.get("username")
user = User(username=username, password=None)
user = user.find()
project_id = request.data.get("project_id")
store_obj = Store(user)
err, exception = store_obj.delete(project_id)
if err:
raise Exception(exception)
if project_id.startswith("shared_"):
project_dir = os.path.join(
store_obj.rootpath,
request.data.get("owner"),
project_id.replace("shared_", ""),
)
with open(project_dir + os.sep + "meta.json", "r") as f:
metadata = json.load(f)
if metadata.get("shared_with", None):
if username in metadata.get("shared_with"):
metadata["shared_with"].remove(username)
with open(project_dir + os.sep + "meta.json", "w") as f:
json.dump(metadata, f)
status, success, message = 200, True, "Project Deleted Successfully"
except Exception as e:
status, success, message = 500, False, "Project could not be deleted"
return JsonResponse({"success": success, "message": message}, status=status)
def save_layers(request):
try:
username = request.data.get("username")
user = User(username=username, password=None)
user = user.find()
store_obj = Store(user)
project_id = request.data.get("project_id")
layers = request.data.get("layer_json")
components = request.data.get("component_array")
if not store_obj.exist(project_id):
raise Exception("No such project exists")
project_dir = store_obj.path + os.sep + project_id
with open(project_dir + os.sep + "layers.json", "w") as f:
json.dump(layers, f)
with open(project_dir + os.sep + "components.json", "w") as f:
json.dump(components, f)
status, success, message = 200, True, "Layers saved successfully"
except Exception as e:
status, success, message = 500, False, "Could not save layers"
return JsonResponse({"success": success, "message": message}, status=status)
def download_code(request):
"""
Endpoint for dowloading generated code.
Inputs:
-------
request:
Requires: username & project_id in request data
token in request header
Returns:
--------
response: HttpResponse (Required File)
(Yes, not JsonResponse like other views)
"""
try:
username = request.data.get("username")
user = User(username=username, password=None)
user = user.find()
store_obj = Store(user)
project_id = request.data.get("project_id")
if not store_obj.exist(project_id):
raise Exception("No such project exists")
project_dir = store_obj.path + os.sep + project_id
f = open(project_dir + os.sep + "test.py", "r")
response = HttpResponse(f, content_type="application/x-python-code")
response["Content-Disposition"] = "attachment; filename=test.py"
except Exception as e:
response = HttpResponse("<h1>File not found</h1>")
return response
def get_preprocessing_params(request):
try:
username = request.data.get("username")
user = User(username=username, password=None)
user = user.find()
store_obj = Store(user)
project_id = request.data.get("project_id")
if not store_obj.exist(project_id):
raise Exception("No such project exists")
project_dir = store_obj.path + os.sep + project_id
with open(project_dir + os.sep + "preprocessing.json", "r") as f:
preprocessing = json.load(f)
status, success, message = 200, True, "preprocessing params fetched"
except Exception as e:
status, success, message, preprocessing = 500, False, str(e), {}
return JsonResponse(
{
"success": success,
"message": message,
"preprocessing": preprocessing
},
status=status,
)
def save_preprocessing_params(request):
try:
username = request.data.get("username")
user = User(username=username, password=None)
user = user.find()
store_obj = Store(user)
project_id = request.data.get("project_id")
preprocessing = request.data.get("preprocessing_params")
if not store_obj.exist(project_id):
raise Exception("No such project exists")
project_dir = store_obj.path + os.sep + project_id
with open(project_dir + os.sep + "preprocessing.json", "w") as f:
json.dump(preprocessing, f)
status, success, message = 200, True, "Preprocessing params saved successfully"
except Exception as e:
status, success, message = 500, False, str(e)
return JsonResponse({
"success": success,
"message": message
},
status=status)
def get_project(request):
try:
username = request.data.get("username")
user = User(username=username, password=None)
user = user.find()
store_obj = Store(user)
project_id = request.data.get("project_id")
if not store_obj.exist(project_id):
raise Exception("No such project exists")
project_dir = store_obj.path + os.sep + project_id
with open(project_dir + os.sep + "layers.json", "r") as f:
layers = json.load(f)
b2f_json = layers
status, success, message = 200, True, "Layers Fetched"
except Exception as e:
status, success, message, b2f_json = (
500,
False,
"Layers could not be fetched",
{},
)
return JsonResponse(
{"success": success, "message": message, "b2f_json": b2f_json}, status=status
)
def get_all_projects(request):
try:
username = request.data.get("username")
user = User(username=username, password=None)
user = user.find()
store_obj = Store(user)
delete_broken_symlinks(store_obj.path)
project_ids = store_obj.enlist()
projects = []
for id in project_ids:
path = store_obj.path + os.sep + id + os.sep + "meta.json"
with open(path, "r") as f:
metadata = json.load(f)
list_item = {id: metadata}
projects.append(list_item)
status, success, message = 200, True, "Projects Fetched"
except Exception as e:
status, success, message, projects = (
500,
False,
"Projects could not be fetched",
[],
)
return JsonResponse(
{"success": success, "message": message, "projects": projects}, status=status
)
def generate(request):
training_params = request.data.get("training_params")
username = request.data.get("username")
user = User(username=username, password=None)
user = user.find()
project_id = request.data.get("project_id")
store_obj = Store(user)
if not store_obj.exist(project_id):
raise Exception("No such project exists")
project_dir = store_obj.path + os.sep + project_id
with open(project_dir + os.sep + "meta.json", "r") as f:
metadata = json.load(f)
with open(project_dir + os.sep + "preprocessing.json", "r") as f:
preprocessing = json.load(f)
lib = metadata.get("lib", "keras").lower()
lang = metadata.get("lang", "python").lower()
meta_params = {}
meta_params["lib"] = lib
meta_params["lang"] = lang
meta_params["dataset-path"] = metadata.get("data_dir", ".")
meta_params["save_plots"] = training_params["plot"]
meta_params.update(preprocessing)
with open(project_dir + os.sep + "layers.json", "r") as f:
layers = json.load(f)
with open(project_dir + os.sep + "components.json", "r") as f:
components = json.load(f)
training_params.update(meta_params)
training_params.update(layers)
inputs = json_to_dict.MakeDict(training_params).parse()
parser_path = "dlmml.parser." + lang + "." + lib + ".main"
parser = importlib.import_module(parser_path)
# TODO: move to javascript in the next version
user_data = UserData(metadata, components, layers, inputs, preprocessing)
status, error = parser.generate_code(inputs)
if status:
print("Error", error)
msg = "Could not generate code"
path = ""
else:
print("File generated")
msg = "File Generated Successfully"
path = "file:///" + os.getcwd() + os.sep + "test.py"
format_code("test.py")
copy_file(project_dir)
return JsonResponse({"message": msg, "path": path})
def edit_project(request):
try:
username = request.data.get("username")
user = User(username=username, password=None)
user = user.find()
project_id = request.data.get("project_id")
project_name = request.data.get("project_name")
project_description = request.data.get("project_description")
data_dir = request.data.get("data_dir")
output_file_name = request.data.get("output_file_name")
owner = request.data.get("owner")
store_obj = Store(user)
if owner and not owner == username:
project_id = "shared_" + project_id
if not store_obj.exist(project_id):
raise Exception("No such project exists")
project_dir = store_obj.path + os.sep + project_id
with open(project_dir + os.sep + "meta.json", "r") as f:
metadata = json.load(f)
metadata["project_name"] = (
project_name if project_name is not None else metadata["project_name"]
)
metadata["project_description"] = (
project_description
if project_description is not None
else metadata["project_description"]
)
metadata["data_dir"] = (
data_dir if data_dir is not None else metadata["data_dir"]
)
metadata["output_file_name"] = (
output_file_name
if output_file_name is not None
else metadata["output_file_name"]
)
with open(project_dir + os.sep + "meta.json", "w") as f:
json.dump(metadata, f)
status, success, message = 200, True, "Project Updated Successfully"
except Exception as e:
status, success, message = 500, False, "Could not update the Project"
return JsonResponse({"success": success, "message": message}, status=status)
def create_project(request):
try:
username = request.data.get("username")
user = User(username=username, password=None)
user = user.find()
project_id = generate_uid()
project_name = request.data.get("project_name")
lang = request.data.get("language")
lib = request.data.get("library")
task = request.data.get("task")
data_dir = request.data.get("path")
output_file_name = request.data.get("output_file_name")
store_obj = Store(user)
if store_obj.exist(project_id):
raise Exception("Project Already Exists!")
project_dir = store_obj.create(project_id)
metadata = {
"project_id": project_id,
"project_name": project_name,
"lib": lib,
"lang": lang,
"task": task,
"data_dir": data_dir,
"output_file_name": output_file_name,
"username": username,
}
with open(project_dir + os.sep + "meta.json", "w") as f:
json.dump(metadata, f)
status, success, message = 200, True, "Project Created Successfully"
except Exception as e:
status, success, message = 500, False, str(e)
return JsonResponse({
"success": success,
"message": message
},
status=status)
def delete_project(request):
try:
username = request.data.get("username")
user = User(username=username, password=None)
user = user.find()
project_id = request.data.get("project_id")
store_obj = Store(user)
err, exception = store_obj.delete(project_id)
if err:
raise Exception(exception)
status, success, message = 200, True, "Project Deleted Successfully"
except Exception as e:
status, success, message = 500, False, str(e)
return JsonResponse({
"success": success,
"message": message
},
status=status)
def test_exists():
user = MockUser().mock_user()
store = Store(user)
store.rootpath = os.path.expanduser(tempdir + "/.autodl/")
store.path = posixpath.join(store.rootpath, store.user.get("username"))
project_path = store.create(project="test_project_dir")
project_path_exists = store.exist(project="test_project_dir")
assert project_path_exists == True
project_path_none = store.exist()
assert project_path_none == True
def test_delete():
user = MockUser().mock_user()
store = Store(user)
store.rootpath = os.path.expanduser(tempdir + "/.autodl/")
store.path = posixpath.join(store.rootpath, store.user.get("username"))
project_path = store.create(project="test_project_dir")
state, result = store.delete(project="test_project_dir")
assert os.path.exists(project_path) == False
shared_with_user = MockUser().mock_user()
shared_with = shared_with_user.get("username")
test_path = store.rootpath + shared_with + "/shared_test_project.json"
file = "shared_test_project.json"
with open(posixpath.join(store.path, file), "w") as fp:
pass
state, result = store.delete(project="shared_test_project.json")
assert os.path.exists(test_path) == False
project_path_none = store.create(project=None)
state, result = store.delete(project=None)
assert os.path.exists(project_path_none) == False
def test_create():
user = MockUser().mock_user()
store = Store(user)
store.rootpath = os.path.expanduser(tempdir + "/.autodl/")
store.path = posixpath.join(store.rootpath, store.user.get("username"))
test_path = store.path + "/test_project_dir"
project_path = store.create(project="test_project_dir")
assert test_path == project_path
path_none = store.create()
assert store.path == path_none
def test_enlist():
user = MockUser().mock_user()
store = Store(user)
store.rootpath = os.path.expanduser(tempdir + "/.autodl/")
store.path = posixpath.join(store.rootpath, store.user.get("username"))
project_path_1 = store.create(project="test_project_dir_0")
project_path_2 = store.create(project="test_project_dir_1")
project_path_3 = store.create(project="test_project_dir_2")
test_projects = store.enlist()
if (
"test_project_dir_0" in test_projects
and "test_project_dir_1" in test_projects
and "test_project_dir_2" in test_projects
):
assert True
def publish_on_github(request):
"""
Endpoint to push the code to github
Inputs:
-------
request:
Requires: username,project_id, details (commit_message,filename,reponame) in request data
Returns:
--------
response: JsonResponse describing success or failure
"""
username = request.data.get("username")
details = request.data.get("details")
project_id = details.get("project_id")
commit_message = details.get("git_commit_message")
filename = details.get("git_file_name")
repo_name = details.get("git_repo_name")
make_private = details.get("make_private")
user = User(username=username, password=None)
this_user = user.find()
store_obj = Store(this_user)
project_dir = store_obj.path + os.sep + project_id
git_access_token = this_user.get("GitAccessToken")
try:
git_access_token = decrypt(git_access_token)
assert git_access_token is not None
push_status, message, repo_full_name = push_to_github(
git_access_token,
repo_name,
filename,
commit_message,
make_private,
project_dir,
)
if push_status == 200:
status, success, message, repo_full_name = (
200,
True,
"Successfully Published",
repo_full_name,
)
else:
status, success, message, repo_full_name = (
500,
False,
"Something went wrong",
"",
)
except:
status, success, message, repo_full_name = (
500,
False,
"Something went wrong",
"",
)
return JsonResponse(
{"success": success, "message": message, "repo_full_name": repo_full_name},
status=status,
)
def share_project(request):
"""
Endpoint to share projects.
Inputs:
-------
request:
Requires: user to share with, user sharing the project, project_id in request data
token in request header
Returns:
--------
response: JsonResponse describing failure or success
"""
try:
share_with = request.data.get("share_with")
username = request.data.get("username")
user = User(username=username, password=None)
user = user.find()
project_id = request.data.get("project_id")
store_obj = Store(user)
project_dir = store_obj.path + os.sep + project_id
with open(project_dir + os.sep + "meta.json", "r") as f:
metadata = json.load(f)
dst = os.path.join(
store_obj.rootpath,
share_with,
"shared_" + str(project_id),
)
os.symlink(project_dir, dst)
if not metadata.get("shared_with", None):
metadata["shared_with"] = []
metadata["shared_with"].append(share_with)
with open(project_dir + os.sep + "meta.json", "w") as f:
json.dump(metadata, f)
status, success, message = 200, True, "Shared Successfully"
except FileExistsError:
status, success, message = (
500,
False,
"Project is already being shared",
)
except OSError as e:
if e.winerror == 1314:
status, success, message = (
500,
False,
"System administrator privileges are required to share projects.",
)
else:
status, success, message = (
500,
False,
"Could not share project",
)
except:
status, success, message = 500, False, "Failed"
return JsonResponse({"success": success, "message": message}, status=status)
def clone_project(request):
try:
username = request.data.get("username")
model_layers = request.data.get("model_layers")
preprocessing_parameters = request.data.get("preprocessing_parameters")
hyper_parameters = request.data.get("hyper_parameters")
user = User(username=username, password=None)
user = user.find()
project_id = request.data.get("project_id")
clone_id = generate_uid()
project_name = request.data.get("project_name")
project_description = request.data.get("project_description")
lang = request.data.get("language")
lib = request.data.get("library")
task = request.data.get("task")
data_dir = request.data.get("path")
output_file_name = request.data.get("output_file_name")
store_obj = Store(user)
if store_obj.exist(clone_id):
raise Exception("Project Already Exists!")
clone_dir = store_obj.create(clone_id)
metadata = {
"project_id": clone_id,
"project_name": project_name,
"project_description": project_description,
"lib": lib,
"lang": lang,
"task": task,
"data_dir": data_dir,
"output_file_name": output_file_name,
"username": username,
}
with open(clone_dir + os.sep + "meta.json", "w") as f:
json.dump(metadata, f)
project_dir = store_obj.path + os.sep + project_id
project_json = project_dir + os.sep + "layers.json"
if model_layers and os.path.isfile(project_json):
with open(model_layers) as f:
layers = json.load(f)
with open(clone_dir + os.sep + "layers.json", "w") as f:
json.dump(layers, f)
with open(project_dir + os.sep + "components.json", "r") as f:
components = json.load(f)
with open(clone_dir + os.sep + "components.json", "w") as f:
json.dump(components, f)
project_preprocessing = project_dir + os.sep + "preprocessing.json"
if preprocessing_parameters and os.path.isfile(project_preprocessing):
with open(project_preprocessing, "r") as f:
preprocessing = json.load(f)
with open(clone_dir + os.sep + "preprocessing.json", "w") as f:
json.dump(preprocessing, f)
project_hyperparameters = project_dir + os.sep + "hyperparams.json"
if hyper_parameters and os.path.isfile(project_hyperparameters):
with open(project_hyperparameters, "r") as f:
hyperparams = json.load(f)
with open(clone_dir + os.sep + "hyperparams.json", "w") as f:
json.dump(hyperparams, f)
status, success, message = 200, True, "Project Cloned Successfully"
except Exception as e:
status, success, message = 500, False, "Project could not be cloned"
return JsonResponse({"success": success, "message": message}, status=status)
def local_deploy(request):
"""
Endpoint for locally deploying user model.
Inputs:
-------
request:
Requires: username, project_id, model_categories & model_download_type in request data
token in request header
Returns:
--------
response: HttpResponse (Required Download)
(Yes, not JsonResponse like other views)
"""
username = request.data.get("username")
user = User(username=username, password=None)
user = user.find()
store_obj = Store(user)
try:
project_id = request.data.get("project_id")
if not store_obj.exist(project_id):
raise ProjectNotFound(project_id)
project_dir = store_obj.path + os.sep + project_id
deployment_dir = os.path.join(project_dir, "deployment")
remove_dir(deployment_dir)
model_categories = request.data.get("model_categories")
model_download_type = request.data.get("model_download_type")
deployment = Deployment(project_dir, deployment_dir, model_categories,
model_download_type)
deployment.clone_flask_app()
logger.debug("Cloned Flask App to be deployed")
deployment.edit_flask_app()
logger.debug("Modified Flask App to be deployed")
response = deployment.download_app()
except ProjectNotFound:
logger.error("Unable to find the Project")
raise
except CloneGenerationFailed:
logger.error("Unable to clone the Flask App")
raise
except AppUpsertionFailed:
logger.error("Unable to modify the Flask App")
raise
except AppDownloadFailed:
logger.error("Unable to download the Flask App")
raise
else:
logger.info("Local Deployment successful")
return response
def cloud_deploy(request):
"""
Endpoint for deploying user model to user-preferred cloud environment (AWS / GCP).
Inputs:
-------
request:
Requires: username, project_id, model_categories & pickle file details in request data
token in request header
Returns:
--------
response: JsonResponse for Cloud Deployment describing Success or Failure
"""
username = request.data.get("username")
user = User(username=username, password=None)
user = user.find()
store_obj = Store(user)
try:
project_id = request.data.get("project_id")
if not store_obj.exist(project_id):
raise Exception("No such project exists")
project_dir = store_obj.path + os.sep + project_id
# Accessing pickle uploaded in chunks
current_chunk = request.data.get("current_chunk")
total_chunks = request.data.get("total_chunks")
pkl_file_bytes = request.data.get("pkl_file_bytes")
pkl_file_content = bytearray(pkl_file_bytes.values())
pkl_dir = os.path.join(project_dir, "pickle")
pkl_path = os.path.join(pkl_dir, f"model.pkl")
# Appending pickle chunks
append_pkl_contents(pkl_dir, pkl_path, pkl_file_content)
if current_chunk == total_chunks:
logger.info("Pickle file uploaded successfully")
model_categories = request.data.get("model_categories")
deployment_dir = os.path.join(project_dir, "deployment")
remove_dir(deployment_dir)
deployment = Deployment(project_dir, deployment_dir,
model_categories)
deployment.clone_flask_app()
logger.debug("Cloned Flask App to be deployed")
deployment.copy_pkl_file(pkl_path)
remove_dir(pkl_dir)
deployment.edit_flask_app()
logger.debug("Modified Flask App to be deployed")
status, success, message = 200, True, "Cloud Deployment Successful"
else:
status, success, message = 204, True, "Cloud Deployment Underway"
except ProjectNotFound:
logger.error("Unable to find the Project")
raise
except PickleAppendFailed:
logger.error("Unable to append Pickle file contents")
remove_dir(pkl_dir)
raise
except CloneGenerationFailed:
logger.error("Unable to clone the Flask App")
remove_dir(pkl_dir)
raise
except PickleCopyFailed:
logger.error("Unable to copy pickle file into deployment directory")
remove_dir(deployment_dir)
remove_dir(pkl_dir)
raise
except AppUpsertionFailed:
logger.error("Unable to modify the Flask App")
remove_dir(deployment_dir)
remove_dir(pkl_dir)
raise
else:
logger.info("Cloud Deployment Successful")
return JsonResponse({
"success": success,
"message": message
},
status=status)
