def deleteNotebook(notebookId: str): """ Service to run notebook job """ res = ApiResponse(message="Error in cloning notebook") response = Zeppelin.deleteNotebook(notebookId) if response: res.update(True, "Notebook deleted successfully", None) return res
def deleteNotebookJob(notebookId: int): """ Service to update crontab of an existing NotebookJob :param notebookId: ID of the Notebook for which to delete """ res = ApiResponse() NotebookJob.objects.filter(name=notebookId).delete() res.update(True, "NotebookJob deleted successfully", None) return res
def updateTriggerWorkflow(workflowId: int, triggerWorkflowId: int, triggerWorkflowStatus: int): """Update given workflow's trigger workflow""" res = ApiResponse(message="Error in updating trigger workflow") updateStatus = Workflow.objects.filter(id=workflowId).update( triggerWorkflow_id=triggerWorkflowId, triggerWorkflowStatus=triggerWorkflowStatus) res.update(True, "Trigger workflow updated successfully", updateStatus) return res
def cloneNotebook(notebookId: str, payload: dict): """ Service to run notebook job """ res = ApiResponse(message="Error in cloning notebook") response = Zeppelin.cloneNotebook(notebookId, json.dumps(payload)) if response: res.update(True, "Notebook cloned successfully", None) return res
def getSchedules(): """ Service to get all CrontabSchedule objects """ res = ApiResponse() crontabSchedules = CrontabSchedule.objects.all() data = CrontabScheduleSerializer(crontabSchedules, many=True).data res.update(True, "Schedules fetched successfully", data) return res
def unarchiveNotebook(notebookId: str, notebookName: str): """ Service to unarchive notebook """ res = ApiResponse(message="Error in archiving notebook") response = Zeppelin.renameNotebook(notebookId, notebookName) if response: res.update(True, "Notebook unarchived successfully", None) return res
def getAllAccountSettings(): """ Service to get keys and values of all AccountSettings """ res = ApiResponse() accountSettings = AccountSetting.objects.all() data = AccountSettingSerializer(accountSettings, many=True).data res.update(True, "Fetched account settings successfully", data) return res
def clearNotebookResults(notebookId: str): """ Service to run notebook job """ res = ApiResponse(message="Error in clearing notebook") response = Zeppelin.clearNotebookResults(notebookId) if response: res.update(True, "Notebook cleared successfully", None) return res
def deleteWorkflow(workflowId: int): """ Delete workflow :param workflowId: id of Workflows.Workflow """ res = ApiResponse(message="Error in deleting workflow logs") Workflow.objects.filter(id=workflowId).delete() res.update(True, "Workflow deleted successfully") return res
def removeConnection(connection_id): res = ApiResponse() connection = Connection.objects.get(id=connection_id) if connection.notebookobject_set.count() == 0: Connection.objects.get(id=connection_id).delete() res.update(True, "Connection deleted successfully") else: res.update(False, "Cannot delete connection because of dependent notebook") return res
def stopNotebookJob(notebookId: str): """ Service to run notebook job """ res = ApiResponse(message="Error in stopping notebook") # Updating runStatus that the task was aborted response = Zeppelin.stopNotebookJob(notebookId) if response: res.update(True, "Notebook stopped successfully", None) return res
def getNotebookObject(notebookObjId: int): """ Service to fetch specified NotebookObject :param notebookObjId: ID of the notebook object """ res = ApiResponse(message="Error retrieving specified Notebook Object") notebookObj = NotebookObject.objects.get(id=notebookObjId) data = NotebookObjectSerializer(notebookObj).data res.update(True, "NotebookObject retrieved successfully", data) return res
def archivedNotebooks(): """ Get archived notebooks """ res = ApiResponse(message="Error retrieving archived notebooks") notebooks = Zeppelin.getAllNotebooks("~Trash") if notebooks: res.update(True, "Archived notebooks retrieved successfully", notebooks) return res
def getSingleSchedule(scheduleId: int): """ Service to get singleSchedule :param scheduleId: int """ res = ApiResponse() schedules = Schedule.objects.filter(crontabschedule_ptr_id=scheduleId) data = ScheduleSerializer(schedules, many=True).data res.update(True, "Schedules fetched successfully", data) return res
def toggleNotebookJob(notebookId: int, enabled: bool): """ Service to update crontab of an existing NotebookJob :param notebookId: ID of the NotebookJob for which to update crontab :param crontabScheduleId: ID of CrontabSchedule """ res = ApiResponse() NotebookJob.objects.filter(notebookId=notebookId).update( enabled=enabled) res.update(True, "NotebookJob updated successfully", None) return res
def deleteNotebook(notebookId: str): """ Service to run notebook job """ res = ApiResponse(message="Error in deleting notebook") response = Zeppelin.deleteNotebook(notebookId) if response: NotebookObject.objects.filter( notebookZeppelinId=notebookId).delete() res.update(True, "Notebook deleted successfully", None) return res
def getAccountSetting(key: str): """ Service to get the value of AccountSetting with specified key :param key: Key of the Account Setting """ res = ApiResponse() try: value = AccountSetting.objects.get(key=key).value res.update(True, "Fetched account setting value successfully", value) except: res.update(False, "Error in fetching account setting value") return res
def updateNotebookJob(notebookJobId: int, crontabScheduleId: int): """ Service to update crontab of an existing NotebookJob :param notebookId: ID of the NotebookJob for which to update crontab :param crontabScheduleId: ID of CrontabSchedule """ res = ApiResponse() crontabScheduleObj = CrontabSchedule.objects.get(id=crontabScheduleId) NotebookJob.objects.filter(id=notebookJobId).update( crontab=crontabScheduleObj) res.update(True, "NotebookJob updated successfully", None) return res
def updateWorkflow( id: int, name: str, scheduleId: int, triggerWorkflowId: int, triggerWorkflowStatus: str, notebookIds: List[int], ): """ Updates workflow :param name: name of new workflow :param scheduleId: crontab id :param triggerWorkflowId: id of workflow which triggers this workflow :param triggerWorkflowStatus: ["success", "failure", "always"] required status of triggerWorkflow to trigger this workflow :param notebookIds: notebookIds for workflow """ res = ApiResponse(message="Error in updating workflow") workflow = Workflow.objects.get(id=id) if not workflow: return res workflow.name = name workflow.triggerWorkflow_id = triggerWorkflowId workflow.triggerWorkflowStatus = triggerWorkflowStatus workflow.save() if scheduleId: if workflow.periodictask: workflow.periodictask.crontab_id = scheduleId workflow.periodictask.save() else: periodictask = PeriodicTask.objects.create( crontab_id=scheduleId, name=name, task="workflows.tasks.runWorkflowJob", args=str([workflow.id])) workflow.periodictask = periodictask workflow.save() else: if workflow.periodictask: PeriodicTask.objects.get(id=workflow.periodictask).delete() workflow.periodictask = None workflow.save() WorkflowNotebookMap.objects.filter(workflow_id=id).delete() notebookJobs = [ WorkflowNotebookMap(workflow_id=id, notebookId=notebookId) for notebookId in notebookIds ] WorkflowNotebookMap.objects.bulk_create(notebookJobs) res.update(True, "Workflow updated successfully", None) return res
def updateAccountSettings(settings: list): """ Service to set specified value of AccountSetting with specified key :param key: Key of the Account Setting :param value: Value to be set in the Account Setting """ res = ApiResponse() for setting in settings: accountSetting = AccountSetting.objects.get(key=setting["key"]) accountSetting.value = setting["value"] accountSetting.save() res.update(True, "Updated account setting successfully") return res
def addConnection(payload): res = ApiResponse() connectionType = ConnectionType.objects.get(id=payload["connectionType_id"]) connection = Connection.objects.create( name=payload["name"], description=payload["description"], connectionType=connectionType ) for param in payload["params"]: cp = ConnectionParam.objects.get(name=param, connectionType=connectionType) ConnectionParamValue.objects.create( connectionParam=cp, value=payload["params"][param], connection=connection ) res.update(True, "Connection added successfully") return res
def runNotebookJob(notebookId: str): """ Service to run notebook job """ res = ApiResponse("Error in running notebook") notebookRunLogs = NotebookRunLogs.objects.create( notebookId=notebookId, status=NOTEBOOK_STATUS_QUEUED, runType="Manual") runNotebookJobTask.delay(notebookId=notebookId, notebookRunLogsId=notebookRunLogs.id, runType="Manual") res.update(True, "Notebook triggered successfully", None) return res
def stopNotebookJob(notebookId: str): """ Service to stop notebook job """ res = ApiResponse(message="Error in stopping notebook") # Updating NotebookRunLogs that the task is being aborted notebookNotebookRunLogs = NotebookRunLogs.objects.filter( notebookId=notebookId).order_by("-startTimestamp").first() if (notebookNotebookRunLogs.status == NOTEBOOK_STATUS_RUNNING): notebookNotebookRunLogs.status = NOTEBOOK_STATUS_ABORT notebookNotebookRunLogs.save() zeppelin = ZeppelinAPI(notebookNotebookRunLogs.zeppelinServerId) thread = threading.Thread(target=zeppelin.stopNotebookJob, args=[notebookId]) thread.start() res.update(True, "Aborting notebook job", None) return res
def updateSchedule(id, cron, timezone, name): """ Service to update Schedule param id: int param cron: Crontab in string format param timezone: Timezone in string format param name: String """ res = ApiResponse() schedules = Schedule.objects.get(crontabschedule_ptr_id=id) schedules.cron = cron schedules.timezone = timezone schedules.name = name schedules.save() res.update(True, "Schedules updated successfully", []) return res
def getWorkflowRunLogs(workflowRunLogsId: int): """ Service to fetch logs related to given workflowRun :param workflowRunLogsId: if of model workflows.workflowRun """ res = ApiResponse(message="Error in retrieving workflow logs") workflowRun = WorkflowRunLogs.objects.get(id=workflowRunLogsId) total = [] res.update( True, "WorkflowRuns retrieved successfully", { "total": total, "workflowRunLogs": [] }, ) return res
def addNotebookJob(notebookId: str, crontabScheduleId: int): """ Service to add a new NotebookJob :param notebookId: ID of the notebook for which to create job :param crontabScheduleId: ID of CrontabSchedule """ res = ApiResponse() crontabScheduleObj = CrontabSchedule.objects.get(id=crontabScheduleId) NotebookJob.objects.update_or_create(name=notebookId, notebookId=notebookId, defaults={ "crontab": crontabScheduleObj, "task": CELERY_TASK_NAME, "args": f'["{notebookId}"]' }) res.update(True, "NotebookJob added successfully", None) return res
def updateAccountSettings(settings: list): """ Service to set specified value of AccountSetting with specified key :param key: Key of the Account Setting :param value: Value to be set in the Account Setting """ res = ApiResponse() for setting in settings: accountSetting = AccountSetting.objects.get(key=setting["key"]) if len(accountSetting.accountsettingvalue_set.all()): accountSettingValue = accountSetting.accountsettingvalue_set.all( )[0] accountSettingValue.value = setting["value"] accountSettingValue.save() else: AccountSettingValue.objects.create( accountSetting=accountSetting, value=setting["value"]) res.update(True, "Updated account setting successfully") return res
def getNotebookJobDetails(notebookId: int, runStatusOffset: int = 0): """ Service to fetch run details and logs of the selected NotebookJob :param notebookId: ID of the NotebookJob :param runStatusOffset: Offset for fetching NotebookJob run statuses """ res = ApiResponse() notebookJobData = {} runStatuses = RunStatus.objects.filter(notebookId=notebookId).order_by( "-startTimestamp")[runStatusOffset:runStatusOffset + RUN_STATUS_LIMIT] notebookRunCount = RunStatus.objects.filter( notebookId=notebookId).count() notebookJobData["runStatuses"] = RunStatusSerializer(runStatuses, many=True).data notebookJobData["count"] = notebookRunCount res.update(True, "NotebookJobs retrieved successfully", notebookJobData) return res
def addSchedule(cron: str, timezone: str = None): """ Service to add CrontabSchedule :param cron: Crontab in string format :param timezone: Timezone string for which to configure CrontabSchedule """ res = ApiResponse() cronElements = cron.split() if len(cronElements) != 5: res.update(False, "Crontab must contain five elements") return res timezone = timezone if timezone else "UTC" CrontabSchedule.objects.create(minute=cronElements[0], hour=cronElements[1], day_of_month=cronElements[2], month_of_year=cronElements[3], day_of_week=cronElements[4], timezone=timezone) res.update(True, "Schedule added successfully", None) return res
def getWorkflows(offset: int = 0): """ Service to fetch and serialize Workflows :param offset: Offset for fetching NotebookJob objects """ LIMIT = 10 res = ApiResponse(message="Error retrieving workflows") workflows = Workflow.objects.filter(enabled=True).order_by("-id") total = workflows.count() data = WorkflowSerializer(workflows[offset:LIMIT], many=True).data res.update( True, "Workflows retrieved successfully", { "total": total, "workflows": data }, ) return res