Exemplo n.º 1
0
    def get(self, request, workflow_id):
        """
        Export workflow to zip file
        """
        jms = JMS(user=request.user)
        workflow = jms.GetWorkflow(workflow_id)
        serializer = WorkflowDetailSerializer(workflow)

        json_path = "/tmp/workflow_%s_%s.json" % (str(workflow_id),
                                                  request.user.username)
        with open(json_path, 'w') as f:
            f.write(JSONRenderer().render(serializer.data))

        zip_path = '/tmp/workflow_%s_%s.zip' % (str(workflow_id),
                                                request.user.username)
        workflow_dir = '%s/workflows/%s/' % (jms.base_dir, str(workflow_id))

        with ZipFile(zip_path, 'w') as myzip:
            myzip.write(json_path, "workflow.json")
            for root, dirs, files in os.walk(workflow_dir):
                for f in files:
                    myzip.write(os.path.join(root, f), "scripts/%s" % f)

        wrapper = FileWrapper(file(zip_path))

        response = HttpResponse(wrapper,
                                content_type='application/force-download')
        response['Content-Length'] = os.path.getsize(zip_path)
        response[
            'Content-Disposition'] = 'attachment; filename=workflow_%s_%s.zip' % (
                str(workflow_id), request.user.username)

        return response
Exemplo n.º 2
0
 def delete(self, request, job_id, user_id):
     '''
     Remove user's access to job
     '''
     jms = JMS(user=request.user)
     jms.DeleteUserJobAccessRight(job_id, user_id)
     return Response()
Exemplo n.º 3
0
    def get(self, request, job_id, download_type, type_id):
        """
        Fetch a job file
        """
        jms = JMS(user=request.user)

        filename = ''
        if download_type.lower() == "parameters":
            param = JobStageParameter.objects.get(
                Parameter__ParameterID=type_id, JobStage__Job__JobID=job_id)
            filename = param.Value
        elif download_type.lower() == "outputs":
            output = ExpectedOutput.objects.get(pk=type_id)
            filename = output.ExpectedOutputFileName

        path = "%s/%s" % (job_id, filename)

        job = jms.GetJob(job_id)
        filepath = os.path.join(jms.users_dir,
                                job.User.username + "/jobs/" + path)
        wrapper = FileWrapper(file(filepath))

        response = HttpResponse(wrapper,
                                content_type='application/force-download')
        response['Content-Length'] = os.path.getsize(filepath)
        response['Content-Disposition'] = 'attachment; filename=%s' % filename
        return response
Exemplo n.º 4
0
 def delete(self, request, job_id, group_id):
     '''
     Remove groups's access to job
     '''
     jms = JMS(user=request.user)
     jms.DeleteGroupJobAccessRight(job_id, group_id)
     return Response()
Exemplo n.º 5
0
    def post(self, request, upload_type, type_id):
        """
        Uploads files to the server and returns the list of files for a job or workflow
        """
        jms = JMS(user=request.user)

        if upload_type == 'jobs':
            rootpath = os.path.join(
                jms.users_dir,
                request.user.username + '/jobs/' + type_id + '/')
        elif upload_type == 'workflows':
            rootpath = os.path.join(jms.base_dir, 'workflows/' + type_id + '/')
        else:
            return Response(status=404)

        jms.createJobDir(rootpath)

        for k, v in request.FILES.iteritems():
            for f in request.FILES.getlist(k):
                with open(os.path.join(rootpath, f.name),
                          'wb+') as destination:
                    for chunk in f.chunks():
                        destination.write(chunk)
                os.chmod(os.path.join(rootpath, f.name), 0777)

        return Response(os.listdir(rootpath), status=200)
Exemplo n.º 6
0
 def delete(self, request, workflow_id, group_id):
     '''
     Remove access to workflow for group
     '''
     jms = JMS(user=request.user)
     jms.DeleteGroupWorkflowAccessRight(workflow_id, group_id)
     return Response()
Exemplo n.º 7
0
 def put(self, request, batch_job_id):
     """
     Start batch job
     """
     jms = JMS(user=request.user)
     jms.StartBatchJob(batch_job_id)
     return Response()
Exemplo n.º 8
0
    def run(self):
        count = 1
        while True:
            with open("/tmp/queue-daemon.txt", "w") as f:
                print >> f, str(count)
                count += 1

                try:
                    process = subprocess.Popen("qstat -x",
                                               shell=True,
                                               stdout=subprocess.PIPE)
                    out, err = process.communicate()

                    data = objectify.fromstring(out)

                    jms = JMS()
                    for job in data.Job:
                        print >> f, job.Job_Id
                        try:
                            jms.AddUpdateClusterJob(job)
                        except Exception, err:
                            print >> f, ''.join(
                                traceback.format_exception(*sys.exc_info()))

                    # Reset database connection to avoid "MySQL has gone away" error after daemon
                    # has been running for a long time
                    db.close_connection()

                except Exception, err:
                    print >> f, "Error: " + str(err)
Exemplo n.º 9
0
 def delete(self, request, job_id):
     """
     Delete a job from history
     """
     jms = JMS(user=request.user)
     jms.DeleteJob(job_id)
     return Response(status=200)
Exemplo n.º 10
0
 def delete(self, request, batch_job_id):
     """
     Delete batch job - deletes all individual jobs as well
     """
     jms = JMS(user=request.user)
     jms.DeleteBatchJob(batch_job_id)
     return Response()
Exemplo n.º 11
0
    def post(self, request):
        """
        Submit a new job
        """
        job = lambda: None
        job.__dict__ = json.loads(request.body)

        name = job.JobName
        wokflow_id = job.WorkflowID
        user = request.user
        description = job.Description

        stages = []
        for s in job.Stages:
            stage = objects.JobStageInput(s["StageID"], s["StageName"],
                                          s["Parameters"], s["RequiresEdit"],
                                          s["Queue"], s["Nodes"],
                                          s["MaxCores"], s["Memory"],
                                          s["Walltime"])
            stages.append(stage)

        jms = JMS(user=request.user)
        job_id = jms.CreateWorkflowJob(name, wokflow_id, description, stages)

        return Response(job_id, status=200)
Exemplo n.º 12
0
    def put(self, request, profile_id):
        """
        Update an input profile
        """
        input_profile = lambda: None
        input_profile.__dict__ = json.loads(request.body)

        with transaction.atomic():
            jms = JMS(user=request.user)
            profile = jms.UpdateInputProfile(
                InputProfileID=profile_id,
                InputProfileName=input_profile.InputProfileName,
                Description=input_profile.Description)

            #delete current profile parameters
            for ip in profile.InputProfileParameters.all():
                ip.delete()

            #replace deleted params with new ones
            for ip in input_profile.InputProfileParameters:
                profile_param = jms.CreateInputProfileParameter(
                    InputProfileID=profile.InputProfileID,
                    ParameterID=ip["ParameterID"],
                    Value=ip["Value"])

        return Response()
Exemplo n.º 13
0
 def delete(self, request, profile_id):
     """
     Delete an input profile
     """
     jms = JMS(user=request.user)
     jms.DeleteInputProfile(profile_id)
     return Response()
Exemplo n.º 14
0
 def delete(self, request, workflow_id):
     """
     Delete workflow by WorkflowID
     """
     jms = JMS(user=request.user)
     jms.DeleteWorkflow(workflow_id)
     return Response()
Exemplo n.º 15
0
    def post(self, request, batch_job_id, file_type):
        """
        Uploads batch files
        """
        jms = JMS(user=request.user)

        if file_type == 'batch':
            rootpath = os.path.join(
                jms.users_dir,
                jms.user.username + '/jobs/batch_jobs/' + batch_job_id + '/')
        elif file_type == 'input':
            rootpath = os.path.join(
                jms.users_dir, jms.user.username + '/jobs/batch_jobs/' +
                batch_job_id + '/files')
        else:
            return Response(status=404)

        jms.createJobDir(rootpath)

        for k, v in request.FILES.iteritems():
            for f in request.FILES.getlist(k):
                with open(os.path.join(rootpath, f.name),
                          'wb+') as destination:
                    for chunk in f.chunks():
                        destination.write(chunk)
                os.chmod(os.path.join(rootpath, f.name), 0777)

        return Response(os.listdir(rootpath), status=200)
Exemplo n.º 16
0
 def delete(self, request, workflow_id, user_id):
     '''
     Remove access to workflow for user
     '''
     jms = JMS(user=request.user)
     jms.DeleteUserWorkflowAccessRight(workflow_id, user_id)
     return Response()
Exemplo n.º 17
0
 def get(self, request):
     """
     Get nodes
     """
     jms = JMS(user=request.user)
     nodes = jms.GetNodes()
     return Response(json.dumps(nodes, default=lambda o: o.__dict__))
Exemplo n.º 18
0
 def get(self, request):
     """
     Get server settings
     """
     jms = JMS(user=request.user)
     settings = jms.GetServerSettings()
     return Response(json.dumps(settings, default=lambda o: o.__dict__))
Exemplo n.º 19
0
 def get(self, request, workflow_id):
     """
     Fetch workflow by WorkflowID
     """
     jms = JMS(user=request.user)
     data = jms.GetWorkflow(workflow_id)
     serializer = WorkflowDetailSerializer(data)
     return Response(serializer.data)
Exemplo n.º 20
0
 def get(self, request, batch_job_id):
     """
     Fetch batch job details
     """
     jms = JMS(user=request.user)
     job = jms.GetBatchJob(batch_job_id)
     serializer = BatchJobSerializer(job)
     return Response(serializer.data)
Exemplo n.º 21
0
 def get(self, request, profile_id):
     """
     Fetch an input profile
     """
     jms = JMS(user=request.user)
     profile = jms.GetInputProfile(profile_id)
     serializer = InputProfileDetailSerializer(profile)
     return Response(serializer.data)
Exemplo n.º 22
0
    def delete(self, request, job_id, comment_id):
        """
        Delete a comment - must be your comment unless you have admin privileges for the job
        """
        jms = JMS(user=request.user)
        response_code = jms.DeleteComment(comment_id, request.user)

        return Response(status=response_code)
Exemplo n.º 23
0
 def get(self, request):
     """
     Get all batch jobs
     """
     jms = JMS(user=request.user)
     jobs = jms.GetBatchJobs()
     serializer = BatchJobSerializer(jobs, many=True)
     return Response(serializer.data)
Exemplo n.º 24
0
 def get(self, request, job_id):
     """
     Fetch a job based on its ID
     """
     jms = JMS(user=request.user)
     response = jms.GetJob(job_id)
     serializer = JobDetailSerializer(response)
     return Response(serializer.data)
Exemplo n.º 25
0
    def delete(self, request, cluster_id):
        """
        Stop a job running on the cluster
        """
        jms = JMS(user=request.user)
        code = jms.StopClusterJob(cluster_id)

        return Response(status=200)
Exemplo n.º 26
0
    def put(self, request, job_id):
        """
        Start a created job
        """
        jms = JMS(user=request.user)
        jms.StartJob(job_id)

        return Response()
Exemplo n.º 27
0
 def get(self, request):
     """
     Fetch all jobs for user
     """
     jms = JMS(user=request.user)
     jobs = jms.GetJobs()
     serializer = JobSerializer(jobs, many=True)
     return Response(serializer.data)
Exemplo n.º 28
0
 def get(self, request):
     """
     Get queue and node usage statistics for all nodes in the cluster
     """
     jms = JMS(user=request.user)
     dashboard = jms.GetDashboard()
     return Response(
         json.dumps(dashboard, default=lambda o: o.__dict__,
                    sort_keys=True))
Exemplo n.º 29
0
    def put(self, request, job_stage_id):
        '''
        Continue a job that has been put in a held state
        '''
        jms = JMS(user=request.user)
        jobstage = jms.GetJobStage(job_stage_id)
        jms.ContinueStage(jobstage)

        return Response(status=status.HTTP_200_OK)
Exemplo n.º 30
0
    def get(self, request, cluster_id):
        """
        Get details of a job running on the cluster
        """
        jms = JMS(user=request.user)
        job = jms.GetClusterJob(job_id=cluster_id)

        serializer = ClusterJobSerializer(job)
        return Response(serializer.data)
Exemplo n.º 31
0
 def put(self, request, upload_type, type_id, file_name):
     """
     Update a job or workflow file
     """
     jms = JMS(user=request.user)
     
     if upload_type == 'jobs':
         rootpath = os.path.join('/jobs/', type_id)
     elif upload_type == 'workflows':
         rootpath = os.path.join(jms.base_dir, 'workflows/' + type_id)
     else:
         return Response(status=404)
      
     jms.createJobDir(rootpath)
     path = os.path.join(rootpath, file_name);
     
     jms.CreateFile(path, request.body)
     
     return Response(os.listdir(rootpath), status=200)
Exemplo n.º 32
0
 def post(self, request, batch_job_id, file_type):
     """
     Uploads batch files
     """        
     jms = JMS(user=request.user)
     
     if file_type == 'batch':
         rootpath = os.path.join(jms.users_dir, jms.user.username + '/jobs/batch_jobs/' + batch_job_id + '/')
     elif file_type == 'input':
         rootpath = os.path.join(jms.users_dir, jms.user.username + '/jobs/batch_jobs/' + batch_job_id + '/files')
     else:
         return Response(status=404)
          
     jms.createJobDir(rootpath)
     
     for k, v in request.FILES.iteritems():
         for f in request.FILES.getlist(k):
             with open(os.path.join(rootpath, f.name), 'wb+') as destination:
                 for chunk in f.chunks():
                     destination.write(chunk)
             os.chmod(os.path.join(rootpath, f.name), 0777)       
             
     return Response(os.listdir(rootpath), status=200)      
Exemplo n.º 33
0
 def post(self, request, upload_type, type_id):
     """
     Uploads files to the server and returns the list of files for a job or workflow
     """        
     jms = JMS(user=request.user)
     
     if upload_type == 'jobs':
         rootpath = os.path.join(jms.users_dir, request.user.username + '/jobs/' + type_id + '/')
     elif upload_type == 'workflows':
         rootpath = os.path.join(jms.base_dir, 'workflows/' + type_id + '/')
     else:
         return Response(status=404)
          
     jms.createJobDir(rootpath)
     
     for k, v in request.FILES.iteritems():
         for f in request.FILES.getlist(k):
             with open(os.path.join(rootpath, f.name), 'wb+') as destination:
                 for chunk in f.chunks():
                     destination.write(chunk)
             os.chmod(os.path.join(rootpath, f.name), 0777)       
             
     return Response(os.listdir(rootpath), status=200)
Exemplo n.º 34
0
 def post(self, request):
     #parse POST json
     workflow = lambda:None
     workflow.__dict__ = json.loads(request.body)
     
     jms = JMS(user=request.user)
     
     workflowID = workflow.WorkflowID
     
     with transaction.atomic():
         
         #if the workflow ID is <= 0, this is a new workflow that we must create
         if workflow.WorkflowID <= 0:
             
             #Create workflow    
             workflowID = jms.CreateWorkflow(WorkflowName=workflow.WorkflowName, Description=workflow.Description)
             
             #create workflow stages
             map_ids = {}
             map_param_ids = {}
             for stage in workflow.Stages:
                 CreateStage(stage, workflowID, map_ids, map_param_ids, jms)
         
         #if the workflow ID is > 0, this workflow already exists in the database and we must update it
         elif workflow.WorkflowID > 0:
             old_workflow = jms.GetWorkflow(workflow.WorkflowID)
             
             #Update workflow
             jms.UpdateWorkflow(WorkflowID=workflow.WorkflowID, WorkflowName=workflow.WorkflowName, Description=workflow.Description)
             
             stages = jms.GetStages(workflow.WorkflowID)
                 
             #delete removed stages
             for old_stage in stages:
                 include = False
                 for current_stage in workflow.Stages:
                     if current_stage["StageID"] == old_stage.StageID:
                         include = True
                         break
                 
                 if not include:
                     jms.DeleteStage(StageID=old_stage.StageID)
             
             map_ids = {}
             map_param_ids = {}
             #update existing stages and create new stages
             for current_stage in workflow.Stages:
                 
                 stage_exists = False
                 for s in old_workflow.Stages.all():
                     if s.StageID == current_stage["StageID"]:
                         stage_exists = True
                         break
                                         
                 if stage_exists:
                     #update stage                
                     command = current_stage["Command"]
                     
                     jms.UpdateStage(StageID=current_stage["StageID"], StageName=current_stage["StageName"], StageTypeID=current_stage["StageType"], 
                         WorkflowID=workflow.WorkflowID, Command=command, StageIndex=current_stage["StageIndex"], Queue=current_stage["Queue"], 
                         MaxCores=current_stage["MaxCores"], Nodes=current_stage["Nodes"], Memory=current_stage["Memory"], Walltime=current_stage["Walltime"])
                     
                     map_ids[current_stage["StageID"]] = current_stage["StageID"]
                                 
                     deps = jms.GetStageDependencies(current_stage["StageID"])
                     
                     #delete dependencies
                     for old_dep in deps:
                         jms.DeleteStageDependency(StageDependencyID=old_dep.StageDependencyID)
                             
                     #add dependencies
                     for current_dep in current_stage["StageDependencies"]:
                         jms.CreateStageDependency(StageID=current_stage["StageID"], DependantOnID=map_ids[current_dep["StageID"]], ConditionID=current_dep["ConditionID"], ExitCodeValue=current_dep["Value"])                            
                     
                     params = jms.GetParameters(current_stage["StageID"])
                     
                     #delete removed parameters
                     for old_param in params:
                         #if old_param doesn't exist in the new set of parameters, delete it from the DB
                         Include = RecursiveFindParameter(current_stage["Parameters"], old_param)                                
                         if not Include:
                             jms.DeleteParameter(ParameterID=old_param.ParameterID)
                     
                     #loop through parameters, updating existing parameters and creating new ones
                     for current_param in current_stage["Parameters"]:
                         
                         param_exists = True
                         try:
                             jms.GetParameter(ParameterID=current_param["ParameterID"])
                         except Exception, e:
                             param_exists = False
                         
                         val = current_param["Value"]
                         
                         if not current_param["Type"]:
                             current_param["Type"] = 1
                         elif current_param["Type"] == 6:
                             val = map_param_ids[int(val)]
                         
                         if param_exists:
                             #recursively update parameters
                             RecursiveUpdateParameters(current_param, map_param_ids, current_stage["StageID"], jms)
                         else:                                        
                             #create parameter    
                             RecursiveCreateParameters(current_param, map_param_ids, current_stage["StageID"], jms)                        
                     
                     outputs = jms.GetExpectedOutputs(current_stage["StageID"])
                     
                     #delete removed outputs
                     for old_out in outputs:
                         Include = False
                         for current_out in current_stage["ExpectedOutputs"]:
                             if current_out["ExpectedOutputID"] == old_out.ExpectedOutputID:
                                 Include = True
                                 break
                         
                         if not Include:
                             jms.DeleteExpectedOutput(ExpectedOutputID=old_out.ExpectedOutputID)
                     
                     #update outputs
                     for current_output in current_stage["ExpectedOutputs"]:                                
                             
                         if current_output["ExpectedOutputID"] > 0:
                             #update output
                             jms.UpdateExpectedOutput(ExpectedOutputID=current_output["ExpectedOutputID"], ExpectedOutputFileName=current_output["ExpectedOutputFileName"])
                         else:
                             #create output
                             output_id = jms.CreateExpectedOutput(ExpectedOutputFileName=current_output["ExpectedOutputFileName"], StageID=current_stage["StageID"])
                                                                 
                 else:
                     #stage doesn't exist so create it
                     CreateStage(current_stage, workflowID, map_ids, map_param_ids, jms)
         
         #create the directory where the workflow scripts will be stored
         jms.createJobDir(jms.base_dir + "/workflows/" + str(workflowID))