def form_valid(self, form):
for field in INGESTOR_OPTIONS:
if not getattr(form.instance, field):
setattr(form.instance, field, INGESTOR_OPTIONS[field])
form.instance.deployment = Deployment.get_by_designator(
self.kwargs.pop('deployment_designator', None))
return super(IngestionCreateView, self).form_valid(form)
def form_valid(self, form):
csv_file = form.cleaned_data.get('csv_file', None)
try:
self.new_deployment = Deployment.create_from_csv_file(csv_file)
self.new_deployment.log_action(self.request.user, "Created.")
except Deployment.AlreadyExists, e:
msg = "%s %s." % (e.message,
e.html_link_to_object("View the deployment"))
messages.warning(self.request, msg)
return super(DeploymentCreateView, self).form_invalid(form)
def associate_deployment(req):
p = _createParams(req)
if req.method == 'GET':
p['form'] = AssociateDeploymentForm()
return render_to_response(TEMPLATE_PATH + 'associate.html', p,
context_instance=RequestContext(req))
elif req.method == 'POST':
form = AssociateDeploymentForm(req.POST)
if form.is_valid():
d = Deployment()
d.owner = req.user
_save_associate_deployment(form, d)
messages.add_message(req, messages.INFO,
'%s successfully associated with %s.' % (d.project,
d.provider))
return HttpResponseRedirect(reverse('deployments'))
else:
p['form'] = form
return render_to_response(TEMPLATE_PATH + 'associate.html', p,
context_instance=RequestContext(req))
def get_object(self):
try:
object = Ingestion.objects.get(
**{
'deployment':
Deployment.get_by_designator(
self.kwargs.pop('deployment_designator', None)),
'index':
self.kwargs.pop('index', None),
})
return object
except:
raise Http404(u"No ingestion found.")
def create(self, validated_data):
# Get Data
project_name = validated_data.get('project_name', None)
deploy_by = validated_data.get('deploy_by', '')
environment = validated_data.get('environment', '')
version = validated_data.get('version', '')
changelog = validated_data.get('changelog', '')
deployment_time = validated_data.get('deployment_time', '')
project_obj, created = Project.objects.get_or_create(name=project_name)
deploy = Deployment(
project=project_obj,
deploy_by=deploy_by,
environment=environment,
version=version,
changelog=changelog,
deployment_time=deployment_time
)
deploy.save()
return deploy
def post(self, request, *args, **kwargs):
if "_import" in request.POST:
repository = [
o for o in Github(GITHUB_TOKEN).get_user().get_orgs()
if o.login == 'ooi-integration'
][0].get_repo('ingestion-csvs')
def get_csvs(repo, filepath, _csv_files={}):
for item in repo.get_dir_contents(filepath):
if "#" in item.path:
continue
if item.type == "dir":
print item.path
_csv_files = get_csvs(repo, item.path)
elif item.path.endswith(".csv"):
print item.path
_csv_files[item.path] = item.decoded_content
return _csv_files
csv_files = get_csvs(repository, ".")
for f in csv_files:
csv_file = csv_files[f]
new_deployment, created = Deployment.create_or_update_from_content(
f, csv_file)
print new_deployment.designator, created
new_deployment.process_csv()
print "processed"
if created:
new_deployment.log_action(self.request.user, "Created.")
else:
new_deployment.log_action(self.request.user,
"Updated from Github.")
new_deployment.log_action(
self.request.user,
"Processed CSV and created %d new data groups." %
new_deployment.data_groups.count())
return super(DeploymentGithubImportView,
self).dispatch(request, *args, **kwargs)
def cloud_deploy(request):
"""
Endpoint for deploying user model to user-preferred cloud environment (AWS / GCP).
Inputs:
-------
request:
Requires: username, project_id, model_categories & pickle file details in request data
token in request header
Returns:
--------
response: JsonResponse for Cloud Deployment describing Success or Failure
"""
username = request.data.get("username")
user = User(username=username, password=None)
user = user.find()
store_obj = Store(user)
try:
project_id = request.data.get("project_id")
if not store_obj.exist(project_id):
raise Exception("No such project exists")
project_dir = store_obj.path + os.sep + project_id
# Accessing pickle uploaded in chunks
current_chunk = request.data.get("current_chunk")
total_chunks = request.data.get("total_chunks")
pkl_file_bytes = request.data.get("pkl_file_bytes")
pkl_file_content = bytearray(pkl_file_bytes.values())
pkl_dir = os.path.join(project_dir, "pickle")
pkl_path = os.path.join(pkl_dir, f"model.pkl")
# Appending pickle chunks
append_pkl_contents(pkl_dir, pkl_path, pkl_file_content)
if current_chunk == total_chunks:
logger.info("Pickle file uploaded successfully")
model_categories = request.data.get("model_categories")
deployment_dir = os.path.join(project_dir, "deployment")
remove_dir(deployment_dir)
deployment = Deployment(project_dir, deployment_dir,
model_categories)
deployment.clone_flask_app()
logger.debug("Cloned Flask App to be deployed")
deployment.copy_pkl_file(pkl_path)
remove_dir(pkl_dir)
deployment.edit_flask_app()
logger.debug("Modified Flask App to be deployed")
status, success, message = 200, True, "Cloud Deployment Successful"
else:
status, success, message = 204, True, "Cloud Deployment Underway"
except ProjectNotFound:
logger.error("Unable to find the Project")
raise
except PickleAppendFailed:
logger.error("Unable to append Pickle file contents")
remove_dir(pkl_dir)
raise
except CloneGenerationFailed:
logger.error("Unable to clone the Flask App")
remove_dir(pkl_dir)
raise
except PickleCopyFailed:
logger.error("Unable to copy pickle file into deployment directory")
remove_dir(deployment_dir)
remove_dir(pkl_dir)
raise
except AppUpsertionFailed:
logger.error("Unable to modify the Flask App")
remove_dir(deployment_dir)
remove_dir(pkl_dir)
raise
else:
logger.info("Cloud Deployment Successful")
return JsonResponse({
"success": success,
"message": message
},
status=status)
def local_deploy(request):
"""
Endpoint for locally deploying user model.
Inputs:
-------
request:
Requires: username, project_id, model_categories & model_download_type in request data
token in request header
Returns:
--------
response: HttpResponse (Required Download)
(Yes, not JsonResponse like other views)
"""
username = request.data.get("username")
user = User(username=username, password=None)
user = user.find()
store_obj = Store(user)
try:
project_id = request.data.get("project_id")
if not store_obj.exist(project_id):
raise ProjectNotFound(project_id)
project_dir = store_obj.path + os.sep + project_id
deployment_dir = os.path.join(project_dir, "deployment")
remove_dir(deployment_dir)
model_categories = request.data.get("model_categories")
model_download_type = request.data.get("model_download_type")
deployment = Deployment(project_dir, deployment_dir, model_categories,
model_download_type)
deployment.clone_flask_app()
logger.debug("Cloned Flask App to be deployed")
deployment.edit_flask_app()
logger.debug("Modified Flask App to be deployed")
response = deployment.download_app()
except ProjectNotFound:
logger.error("Unable to find the Project")
raise
except CloneGenerationFailed:
logger.error("Unable to clone the Flask App")
raise
except AppUpsertionFailed:
logger.error("Unable to modify the Flask App")
raise
except AppDownloadFailed:
logger.error("Unable to download the Flask App")
raise
else:
logger.info("Local Deployment successful")
return response
def get_object(self):
try:
return Deployment.get_by_designator(self.kwargs.pop('slug', None))
except:
raise Http404(u"No deployment found.")
