def deleteNotebook(notebookId: str):
"""
Service to run notebook job
"""
res = ApiResponse(message="Error in cloning notebook")
response = Zeppelin.deleteNotebook(notebookId)
if response:
res.update(True, "Notebook deleted successfully", None)
return res
def deleteNotebookJob(notebookId: int):
"""
Service to update crontab of an existing NotebookJob
:param notebookId: ID of the Notebook for which to delete
"""
res = ApiResponse()
NotebookJob.objects.filter(name=notebookId).delete()
res.update(True, "NotebookJob deleted successfully", None)
return res
def updateTriggerWorkflow(workflowId: int, triggerWorkflowId: int,
triggerWorkflowStatus: int):
"""Update given workflow's trigger workflow"""
res = ApiResponse(message="Error in updating trigger workflow")
updateStatus = Workflow.objects.filter(id=workflowId).update(
triggerWorkflow_id=triggerWorkflowId,
triggerWorkflowStatus=triggerWorkflowStatus)
res.update(True, "Trigger workflow updated successfully", updateStatus)
return res
def cloneNotebook(notebookId: str, payload: dict):
"""
Service to run notebook job
"""
res = ApiResponse(message="Error in cloning notebook")
response = Zeppelin.cloneNotebook(notebookId, json.dumps(payload))
if response:
res.update(True, "Notebook cloned successfully", None)
return res
def getSchedules():
"""
Service to get all CrontabSchedule objects
"""
res = ApiResponse()
crontabSchedules = CrontabSchedule.objects.all()
data = CrontabScheduleSerializer(crontabSchedules, many=True).data
res.update(True, "Schedules fetched successfully", data)
return res
def unarchiveNotebook(notebookId: str, notebookName: str):
"""
Service to unarchive notebook
"""
res = ApiResponse(message="Error in archiving notebook")
response = Zeppelin.renameNotebook(notebookId, notebookName)
if response:
res.update(True, "Notebook unarchived successfully", None)
return res
def getAllAccountSettings():
"""
Service to get keys and values of all AccountSettings
"""
res = ApiResponse()
accountSettings = AccountSetting.objects.all()
data = AccountSettingSerializer(accountSettings, many=True).data
res.update(True, "Fetched account settings successfully", data)
return res
def clearNotebookResults(notebookId: str):
"""
Service to run notebook job
"""
res = ApiResponse(message="Error in clearing notebook")
response = Zeppelin.clearNotebookResults(notebookId)
if response:
res.update(True, "Notebook cleared successfully", None)
return res
def deleteWorkflow(workflowId: int):
"""
Delete workflow
:param workflowId: id of Workflows.Workflow
"""
res = ApiResponse(message="Error in deleting workflow logs")
Workflow.objects.filter(id=workflowId).delete()
res.update(True, "Workflow deleted successfully")
return res
def removeConnection(connection_id):
res = ApiResponse()
connection = Connection.objects.get(id=connection_id)
if connection.notebookobject_set.count() == 0:
Connection.objects.get(id=connection_id).delete()
res.update(True, "Connection deleted successfully")
else:
res.update(False, "Cannot delete connection because of dependent notebook")
return res
def stopNotebookJob(notebookId: str):
"""
Service to run notebook job
"""
res = ApiResponse(message="Error in stopping notebook")
# Updating runStatus that the task was aborted
response = Zeppelin.stopNotebookJob(notebookId)
if response:
res.update(True, "Notebook stopped successfully", None)
return res
def getNotebookObject(notebookObjId: int):
"""
Service to fetch specified NotebookObject
:param notebookObjId: ID of the notebook object
"""
res = ApiResponse(message="Error retrieving specified Notebook Object")
notebookObj = NotebookObject.objects.get(id=notebookObjId)
data = NotebookObjectSerializer(notebookObj).data
res.update(True, "NotebookObject retrieved successfully", data)
return res
def archivedNotebooks():
"""
Get archived notebooks
"""
res = ApiResponse(message="Error retrieving archived notebooks")
notebooks = Zeppelin.getAllNotebooks("~Trash")
if notebooks:
res.update(True, "Archived notebooks retrieved successfully",
notebooks)
return res
def getSingleSchedule(scheduleId: int):
"""
Service to get singleSchedule
:param scheduleId: int
"""
res = ApiResponse()
schedules = Schedule.objects.filter(crontabschedule_ptr_id=scheduleId)
data = ScheduleSerializer(schedules, many=True).data
res.update(True, "Schedules fetched successfully", data)
return res
def toggleNotebookJob(notebookId: int, enabled: bool):
"""
Service to update crontab of an existing NotebookJob
:param notebookId: ID of the NotebookJob for which to update crontab
:param crontabScheduleId: ID of CrontabSchedule
"""
res = ApiResponse()
NotebookJob.objects.filter(notebookId=notebookId).update(
enabled=enabled)
res.update(True, "NotebookJob updated successfully", None)
return res
def deleteNotebook(notebookId: str):
"""
Service to run notebook job
"""
res = ApiResponse(message="Error in deleting notebook")
response = Zeppelin.deleteNotebook(notebookId)
if response:
NotebookObject.objects.filter(
notebookZeppelinId=notebookId).delete()
res.update(True, "Notebook deleted successfully", None)
return res
def getAccountSetting(key: str):
"""
Service to get the value of AccountSetting with specified key
:param key: Key of the Account Setting
"""
res = ApiResponse()
try:
value = AccountSetting.objects.get(key=key).value
res.update(True, "Fetched account setting value successfully", value)
except:
res.update(False, "Error in fetching account setting value")
return res
def updateNotebookJob(notebookJobId: int, crontabScheduleId: int):
"""
Service to update crontab of an existing NotebookJob
:param notebookId: ID of the NotebookJob for which to update crontab
:param crontabScheduleId: ID of CrontabSchedule
"""
res = ApiResponse()
crontabScheduleObj = CrontabSchedule.objects.get(id=crontabScheduleId)
NotebookJob.objects.filter(id=notebookJobId).update(
crontab=crontabScheduleObj)
res.update(True, "NotebookJob updated successfully", None)
return res
def updateWorkflow(
id: int,
name: str,
scheduleId: int,
triggerWorkflowId: int,
triggerWorkflowStatus: str,
notebookIds: List[int],
):
"""
Updates workflow
:param name: name of new workflow
:param scheduleId: crontab id
:param triggerWorkflowId: id of workflow which triggers this workflow
:param triggerWorkflowStatus: ["success", "failure", "always"] required
status of triggerWorkflow to trigger this workflow
:param notebookIds: notebookIds for workflow
"""
res = ApiResponse(message="Error in updating workflow")
workflow = Workflow.objects.get(id=id)
if not workflow:
return res
workflow.name = name
workflow.triggerWorkflow_id = triggerWorkflowId
workflow.triggerWorkflowStatus = triggerWorkflowStatus
workflow.save()
if scheduleId:
if workflow.periodictask:
workflow.periodictask.crontab_id = scheduleId
workflow.periodictask.save()
else:
periodictask = PeriodicTask.objects.create(
crontab_id=scheduleId,
name=name,
task="workflows.tasks.runWorkflowJob",
args=str([workflow.id]))
workflow.periodictask = periodictask
workflow.save()
else:
if workflow.periodictask:
PeriodicTask.objects.get(id=workflow.periodictask).delete()
workflow.periodictask = None
workflow.save()
WorkflowNotebookMap.objects.filter(workflow_id=id).delete()
notebookJobs = [
WorkflowNotebookMap(workflow_id=id, notebookId=notebookId)
for notebookId in notebookIds
]
WorkflowNotebookMap.objects.bulk_create(notebookJobs)
res.update(True, "Workflow updated successfully", None)
return res
def updateAccountSettings(settings: list):
"""
Service to set specified value of AccountSetting with specified key
:param key: Key of the Account Setting
:param value: Value to be set in the Account Setting
"""
res = ApiResponse()
for setting in settings:
accountSetting = AccountSetting.objects.get(key=setting["key"])
accountSetting.value = setting["value"]
accountSetting.save()
res.update(True, "Updated account setting successfully")
return res
def addConnection(payload):
res = ApiResponse()
connectionType = ConnectionType.objects.get(id=payload["connectionType_id"])
connection = Connection.objects.create(
name=payload["name"], description=payload["description"], connectionType=connectionType
)
for param in payload["params"]:
cp = ConnectionParam.objects.get(name=param, connectionType=connectionType)
ConnectionParamValue.objects.create(
connectionParam=cp, value=payload["params"][param], connection=connection
)
res.update(True, "Connection added successfully")
return res
def runNotebookJob(notebookId: str):
"""
Service to run notebook job
"""
res = ApiResponse("Error in running notebook")
notebookRunLogs = NotebookRunLogs.objects.create(
notebookId=notebookId,
status=NOTEBOOK_STATUS_QUEUED,
runType="Manual")
runNotebookJobTask.delay(notebookId=notebookId,
notebookRunLogsId=notebookRunLogs.id,
runType="Manual")
res.update(True, "Notebook triggered successfully", None)
return res
def stopNotebookJob(notebookId: str):
"""
Service to stop notebook job
"""
res = ApiResponse(message="Error in stopping notebook")
# Updating NotebookRunLogs that the task is being aborted
notebookNotebookRunLogs = NotebookRunLogs.objects.filter(
notebookId=notebookId).order_by("-startTimestamp").first()
if (notebookNotebookRunLogs.status == NOTEBOOK_STATUS_RUNNING):
notebookNotebookRunLogs.status = NOTEBOOK_STATUS_ABORT
notebookNotebookRunLogs.save()
zeppelin = ZeppelinAPI(notebookNotebookRunLogs.zeppelinServerId)
thread = threading.Thread(target=zeppelin.stopNotebookJob,
args=[notebookId])
thread.start()
res.update(True, "Aborting notebook job", None)
return res
def updateSchedule(id, cron, timezone, name):
"""
Service to update Schedule
param id: int
param cron: Crontab in string format
param timezone: Timezone in string format
param name: String
"""
res = ApiResponse()
schedules = Schedule.objects.get(crontabschedule_ptr_id=id)
schedules.cron = cron
schedules.timezone = timezone
schedules.name = name
schedules.save()
res.update(True, "Schedules updated successfully", [])
return res
def getWorkflowRunLogs(workflowRunLogsId: int):
"""
Service to fetch logs related to given workflowRun
:param workflowRunLogsId: if of model workflows.workflowRun
"""
res = ApiResponse(message="Error in retrieving workflow logs")
workflowRun = WorkflowRunLogs.objects.get(id=workflowRunLogsId)
total = []
res.update(
True,
"WorkflowRuns retrieved successfully",
{
"total": total,
"workflowRunLogs": []
},
)
return res
def addNotebookJob(notebookId: str, crontabScheduleId: int):
"""
Service to add a new NotebookJob
:param notebookId: ID of the notebook for which to create job
:param crontabScheduleId: ID of CrontabSchedule
"""
res = ApiResponse()
crontabScheduleObj = CrontabSchedule.objects.get(id=crontabScheduleId)
NotebookJob.objects.update_or_create(name=notebookId,
notebookId=notebookId,
defaults={
"crontab": crontabScheduleObj,
"task": CELERY_TASK_NAME,
"args": f'["{notebookId}"]'
})
res.update(True, "NotebookJob added successfully", None)
return res
def updateAccountSettings(settings: list):
"""
Service to set specified value of AccountSetting with specified key
:param key: Key of the Account Setting
:param value: Value to be set in the Account Setting
"""
res = ApiResponse()
for setting in settings:
accountSetting = AccountSetting.objects.get(key=setting["key"])
if len(accountSetting.accountsettingvalue_set.all()):
accountSettingValue = accountSetting.accountsettingvalue_set.all(
)[0]
accountSettingValue.value = setting["value"]
accountSettingValue.save()
else:
AccountSettingValue.objects.create(
accountSetting=accountSetting, value=setting["value"])
res.update(True, "Updated account setting successfully")
return res
def getNotebookJobDetails(notebookId: int, runStatusOffset: int = 0):
"""
Service to fetch run details and logs of the selected NotebookJob
:param notebookId: ID of the NotebookJob
:param runStatusOffset: Offset for fetching NotebookJob run statuses
"""
res = ApiResponse()
notebookJobData = {}
runStatuses = RunStatus.objects.filter(notebookId=notebookId).order_by(
"-startTimestamp")[runStatusOffset:runStatusOffset +
RUN_STATUS_LIMIT]
notebookRunCount = RunStatus.objects.filter(
notebookId=notebookId).count()
notebookJobData["runStatuses"] = RunStatusSerializer(runStatuses,
many=True).data
notebookJobData["count"] = notebookRunCount
res.update(True, "NotebookJobs retrieved successfully",
notebookJobData)
return res
def addSchedule(cron: str, timezone: str = None):
"""
Service to add CrontabSchedule
:param cron: Crontab in string format
:param timezone: Timezone string for which to configure CrontabSchedule
"""
res = ApiResponse()
cronElements = cron.split()
if len(cronElements) != 5:
res.update(False, "Crontab must contain five elements")
return res
timezone = timezone if timezone else "UTC"
CrontabSchedule.objects.create(minute=cronElements[0],
hour=cronElements[1],
day_of_month=cronElements[2],
month_of_year=cronElements[3],
day_of_week=cronElements[4],
timezone=timezone)
res.update(True, "Schedule added successfully", None)
return res
def getWorkflows(offset: int = 0):
"""
Service to fetch and serialize Workflows
:param offset: Offset for fetching NotebookJob objects
"""
LIMIT = 10
res = ApiResponse(message="Error retrieving workflows")
workflows = Workflow.objects.filter(enabled=True).order_by("-id")
total = workflows.count()
data = WorkflowSerializer(workflows[offset:LIMIT], many=True).data
res.update(
True,
"Workflows retrieved successfully",
{
"total": total,
"workflows": data
},
)
return res
