Exemple #1
0
def execute(job_id):
    """Executes a job through the plugin system

    Parameters
    ----------
    job_id : str
        The id of the job to execute
    """
    # Create the new job
    job = ProcessingJob(job_id)
    job_dir = join(get_work_base_dir(), job.id)
    software = job.command.software
    plugin_start_script = software.start_script
    plugin_env_script = software.environment_script

    # Get the command to start the plugin
    cmd = '%s "%s" "%s" "%s" "%s" "%s"' % (
        qiita_config.plugin_launcher, plugin_env_script, plugin_start_script,
        qiita_config.base_url, job.id, job_dir)

    # Start the plugin
    std_out, std_err, return_value = system_call(cmd)
    if return_value != 0:
        # Something wrong happened during the plugin start procedure
        job.status = 'error'
        log = LogEntry.create(
            'Runtime',
            "Error starting plugin '%s':\nStd output:%s\nStd error:%s"
            % (software.name, std_out, std_err))
        job.log = log
Exemple #2
0
def artifact_post_req(user, artifact_id):
    """Deletes the artifact

    Parameters
    ----------
    user : qiita_db.user.User
        The user requesting the action
    artifact_id : int
        Id of the artifact being deleted
    """
    artifact_id = int(artifact_id)
    artifact = Artifact(artifact_id)
    check_artifact_access(user, artifact)

    analysis = artifact.analysis

    if analysis:
        # Do something when deleting in the analysis part to keep track of it
        redis_key = "analysis_%s" % analysis.id
    else:
        pt_id = artifact.prep_templates[0].id
        redis_key = PREP_TEMPLATE_KEY_FORMAT % pt_id

    qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
    cmd = qiita_plugin.get_command('delete_artifact')
    params = Parameters.load(cmd, values_dict={'artifact': artifact_id})
    job = ProcessingJob.create(user, params, True)

    r_client.set(redis_key, dumps({'job_id': job.id, 'is_qiita_job': True}))

    job.submit()

    return {'job': job.id}
Exemple #3
0
def _submit(ctx, user, parameters):
    """Submit a plugin job to a cluster

    Parameters
    ----------
    ctx : qiita_db.ware.Dispatch
        A Dispatch object to submit through
    user : qiita_db.user.User
        The user doing the submission
    parameters : qiita_db.software.Parameters
        The parameters of the job

    Returns
    -------
    str
        The job id
    """
    job = ProcessingJob.create(user, parameters)
    redis_deets = {
        'job_id': job.id,
        'pubsub': user.id,
        'messages': user.id + ':messages'
    }
    ctx.submit_async(_redis_wrap, execute, redis_deets, job.id)
    return job.id
Exemple #4
0
def job_ajax_get_req(job_id):
    """Returns the job information

    Parameters
    ----------
    job_id : str
        The job id

    Returns
    -------
    dict of objects
        A dictionary containing the job information
        {'status': str,
         'message': str,
         'job_id': str,
         'job_status': str,
         'job_step': str,
         'job_parameters': dict of {str: str}}
    """
    job = ProcessingJob(job_id)
    return {
        'status': 'success',
        'message': '',
        'job_id': job.id,
        'job_status': job.status,
        'job_step': job.step,
        'job_parameters': job.parameters.values
    }
Exemple #5
0
def study_delete_req(study_id, user_id):
    """Delete a given study

    Parameters
    ----------
    study_id : int
        Study id to delete
    user_id : str
        User requesting the deletion

    Returns
    -------
    dict
        Status of deletion, in the format
        {status: status,
         message: message}
    """
    access_error = check_access(study_id, user_id)
    if access_error:
        return access_error

    qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
    cmd = qiita_plugin.get_command('delete_study')
    params = Parameters.load(cmd, values_dict={'study': study_id})
    job = ProcessingJob.create(User(user_id), params, True)
    # Store the job id attaching it to the sample template id
    r_client.set(STUDY_KEY_FORMAT % study_id, dumps({'job_id': job.id}))

    job.submit()

    return {'status': 'success', 'message': ''}
Exemple #6
0
    def post(self, preprocessed_data_id):
        user = self.current_user
        # make sure user is admin and can therefore actually submit to EBI
        if user.level != 'admin':
            raise HTTPError(403, reason="User %s cannot submit to EBI!" %
                            user.id)
        submission_type = self.get_argument('submission_type')

        if submission_type not in ['ADD', 'MODIFY']:
            raise HTTPError(403, reason="User: %s, %s is not a recognized "
                            "submission type" % (user.id, submission_type))

        study = Artifact(preprocessed_data_id).study
        state = study.ebi_submission_status
        if state == 'submitting':
            message = "Cannot resubmit! Current state is: %s" % state
            self.display_template(preprocessed_data_id, message, 'danger')
        else:
            qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
            cmd = qiita_plugin.get_command('submit_to_EBI')
            params = Parameters.load(
                cmd, values_dict={'artifact': preprocessed_data_id,
                                  'submission_type': submission_type})
            job = ProcessingJob.create(user, params, True)

            r_client.set('ebi_submission_%s' % preprocessed_data_id,
                         dumps({'job_id': job.id, 'is_qiita_job': True}))
            job.submit()

            level = 'success'
            message = 'EBI submission started. Job id: %s' % job.id

            self.redirect("%s/study/description/%d?level=%s&message=%s" % (
                qiita_config.portal_dir, study.id, level, url_escape(message)))
Exemple #7
0
    def post(self, preprocessed_data_id):
        user = self.current_user
        # make sure user is admin and can therefore actually submit to EBI
        if user.level != 'admin':
            raise HTTPError(403, reason="User %s cannot submit to EBI!" %
                            user.id)
        submission_type = self.get_argument('submission_type')

        if submission_type not in ['ADD', 'MODIFY']:
            raise HTTPError(403, reason="User: %s, %s is not a recognized "
                            "submission type" % (user.id, submission_type))

        study = Artifact(preprocessed_data_id).study
        state = study.ebi_submission_status
        if state == 'submitting':
            message = "Cannot resubmit! Current state is: %s" % state
            self.display_template(preprocessed_data_id, message, 'danger')
        else:
            qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
            cmd = qiita_plugin.get_command('submit_to_EBI')
            params = Parameters.load(
                cmd, values_dict={'artifact': preprocessed_data_id,
                                  'submission_type': submission_type})
            job = ProcessingJob.create(user, params, True)

            r_client.set('ebi_submission_%s' % preprocessed_data_id,
                         dumps({'job_id': job.id, 'is_qiita_job': True}))
            job.submit()

            level = 'success'
            message = 'EBI submission started. Job id: %s' % job.id

            self.redirect("%s/study/description/%d?level=%s&message=%s" % (
                qiita_config.portal_dir, study.id, level, url_escape(message)))
Exemple #8
0
def prep_template_jobs_get_req(prep_id, user_id):
    """Returns graph of all artifacts created from the prep base artifact

    Parameters
    ----------
    prep_id : int
        Prep template ID to get graph for
    user_id : str
        User making the request

    Returns
    -------
    dict with the jobs information

    Notes
    -----
    Nodes are identified by the corresponding Artifact ID.
    """
    prep = PrepTemplate(int(prep_id))
    access_error = check_access(prep.study_id, user_id)
    if access_error:
        return access_error

    job_info = r_client.get(PREP_TEMPLATE_KEY_FORMAT % prep_id)
    result = {}
    if job_info:
        job_info = defaultdict(lambda: '', loads(job_info))
        job_id = job_info['job_id']
        job = ProcessingJob(job_id)
        result[job.id] = {'status': job.status, 'step': job.step,
                          'error': job.log.msg if job.log else ""}

    return result
Exemple #9
0
    def post(self, preprocessed_data_id):
        user = self.current_user
        # make sure user is admin and can therefore actually submit to VAMPS
        if user.level != 'admin':
            raise HTTPError(403, "User %s cannot submit to VAMPS!" %
                            user.id)
        msg = ''
        msg_level = 'success'

        plugin = Software.from_name_and_version('Qiita', 'alpha')
        cmd = plugin.get_command('submit_to_VAMPS')
        artifact = Artifact(preprocessed_data_id)

        # Check if the artifact is already being submitted to VAMPS
        is_being_submitted = any(
            [j.status in ('queued', 'running')
             for j in artifact.jobs(cmd=cmd)])

        if is_being_submitted == 'submitting':
            msg = "Cannot resubmit! Data is already being submitted"
            msg_level = 'danger'
            self.display_template(preprocessed_data_id, msg, msg_level)
        else:
            params = Parameters.load(
                cmd, values_dict={'artifact': preprocessed_data_id})
            job = ProcessingJob.create(user, params, True)
            job.submit()
            self.redirect('/study/description/%s' % artifact.study.study_id)
Exemple #10
0
def study_delete_req(study_id, user_id):
    """Delete a given study

    Parameters
    ----------
    study_id : int
        Study id to delete
    user_id : str
        User requesting the deletion

    Returns
    -------
    dict
        Status of deletion, in the format
        {status: status,
         message: message}
    """
    access_error = check_access(study_id, user_id)
    if access_error:
        return access_error

    qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
    cmd = qiita_plugin.get_command('delete_study')
    params = Parameters.load(cmd, values_dict={'study': study_id})
    job = ProcessingJob.create(User(user_id), params, True)
    # Store the job id attaching it to the sample template id
    r_client.set(STUDY_KEY_FORMAT % study_id,
                 dumps({'job_id': job.id}))

    job.submit()

    return {'status': 'success', 'message': ''}
Exemple #11
0
def artifact_post_req(user, artifact_id):
    """Deletes the artifact

    Parameters
    ----------
    user : qiita_db.user.User
        The user requesting the action
    artifact_id : int
        Id of the artifact being deleted
    """
    artifact_id = int(artifact_id)
    artifact = Artifact(artifact_id)
    check_artifact_access(user, artifact)

    analysis = artifact.analysis

    if analysis:
        # Do something when deleting in the analysis part to keep track of it
        redis_key = "analysis_%s" % analysis.id
    else:
        pt_id = artifact.prep_templates[0].id
        redis_key = PREP_TEMPLATE_KEY_FORMAT % pt_id

    qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
    cmd = qiita_plugin.get_command('delete_artifact')
    params = Parameters.load(cmd, values_dict={'artifact': artifact_id})
    job = ProcessingJob.create(user, params)

    r_client.set(redis_key, dumps({'job_id': job.id, 'is_qiita_job': True}))

    job.submit()
Exemple #12
0
    def post(self):
        """Retrieves the archiving information

        Returns
        -------
        dict
            The contents of the analysis keyed by sample id

        Notes
        -----
            Argument "path" must be the Qiita job_id which is used to infer
            the merging scheme.
            Argument "features" is a list of feature identifier,
            e.g. Deblur sequences.

            Feature identifiers not found in the archive won't be included in
            the return dictionary.
        """
        job_id = self.get_argument('job_id')
        features = self.request.arguments['features']

        ms = Archive.get_merging_scheme_from_job(ProcessingJob(job_id))
        response = Archive.retrieve_feature_values(archive_merging_scheme=ms,
                                                   features=features)

        self.write(response)
Exemple #13
0
    def post(self, preprocessed_data_id):
        user = self.current_user
        # make sure user is admin and can therefore actually submit to VAMPS
        if user.level != 'admin':
            raise HTTPError(403, "User %s cannot submit to VAMPS!" % user.id)
        msg = ''
        msg_level = 'success'

        plugin = Software.from_name_and_version('Qiita', 'alpha')
        cmd = plugin.get_command('submit_to_VAMPS')
        artifact = Artifact(preprocessed_data_id)

        # Check if the artifact is already being submitted to VAMPS
        is_being_submitted = any([
            j.status in ('queued', 'running') for j in artifact.jobs(cmd=cmd)
        ])

        if is_being_submitted == 'submitting':
            msg = "Cannot resubmit! Data is already being submitted"
            msg_level = 'danger'
            self.display_template(preprocessed_data_id, msg, msg_level)
        else:
            params = Parameters.load(
                cmd, values_dict={'artifact': preprocessed_data_id})
            job = ProcessingJob.create(user, params)
            job.submit()
            self.redirect('/study/description/%s' % artifact.study.study_id)
 def _create_job(self, cmd_name, values_dict):
     self.user = User('*****@*****.**')
     qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
     cmd = qiita_plugin.get_command(cmd_name)
     params = Parameters.load(cmd, values_dict=values_dict)
     job = ProcessingJob.create(self.user, params, True)
     job._set_status('queued')
     return job
 def _create_job(self, cmd_name, values_dict):
     self.user = User('*****@*****.**')
     qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
     cmd = qiita_plugin.get_command(cmd_name)
     params = Parameters.load(cmd, values_dict=values_dict)
     job = ProcessingJob.create(self.user, params, True)
     job._set_status('queued')
     return job
Exemple #16
0
def sample_template_put_req(study_id, user_id, sample_template):
    """Updates a sample template using the given file

    Parameters
    ----------
    study_id : int
        The current study object id
    user_id : str
        The current user object id
    sample_template : str
        filename to use for updating

    Returns
    -------
    dict
        results dictonary in the format
        {'status': status,
         'message': msg,
         'file': sample_template}

    status can be success, warning, or error depending on result
    message has the warnings or errors
    file has the file name
    """
    exists = _check_sample_template_exists(int(study_id))
    if exists['status'] != 'success':
        return exists
    access_error = check_access(int(study_id), user_id)
    if access_error:
        return access_error

    fp_rsp = check_fp(study_id, sample_template)
    if fp_rsp['status'] != 'success':
        # Unknown filepath, so return the error message
        return fp_rsp
    fp_rsp = fp_rsp['file']

    msg = ''
    status = 'success'

    # Offload the update of the sample template to the cluster
    qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
    cmd = qiita_plugin.get_command('update_sample_template')
    params = Parameters.load(cmd,
                             values_dict={
                                 'study': int(study_id),
                                 'template_fp': fp_rsp
                             })
    job = ProcessingJob.create(User(user_id), params)

    # Store the job id attaching it to the sample template id
    r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id,
                 dumps({'job_id': job.id}))

    job.submit()

    return {'status': status, 'message': msg, 'file': sample_template}
Exemple #17
0
def sample_template_handler_post_request(study_id,
                                         user,
                                         filepath,
                                         data_type=None):
    """Creates a new sample template

    Parameters
    ----------
    study_id: int
        The study to add the sample information
    user: qiita_db.user import User
        The user performing the request
    filepath: str
        The path to the sample template file
    data_type: str, optional
        If filepath is a QIIME mapping file, the data type of the prep
        information file

    Returns
    -------
    dict of {'job': str}
        job: the id of the job adding the sample information to the study

    Raises
    ------
    HTTPError
        404 if the filepath doesn't exist
    """
    # Check if the current user has access to the study
    sample_template_checks(study_id, user)

    # Check if the file exists
    fp_rsp = check_fp(study_id, filepath)
    if fp_rsp['status'] != 'success':
        raise HTTPError(404, 'Filepath not found')
    filepath = fp_rsp['file']

    is_mapping_file = looks_like_qiime_mapping_file(filepath)
    if is_mapping_file and not data_type:
        raise HTTPError(
            400, 'Please, choose a data type if uploading a '
            'QIIME mapping file')

    qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
    cmd = qiita_plugin.get_command('create_sample_template')
    params = Parameters.load(cmd,
                             values_dict={
                                 'fp': filepath,
                                 'study_id': study_id,
                                 'is_mapping_file': is_mapping_file,
                                 'data_type': data_type
                             })
    job = ProcessingJob.create(user, params, True)
    r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id,
                 dumps({'job_id': job.id}))
    job.submit()
    return {'job': job.id}
Exemple #18
0
def sample_template_handler_post_request(study_id, user, filepath,
                                         data_type=None, direct_upload=False):
    """Creates a new sample template

    Parameters
    ----------
    study_id: int
        The study to add the sample information
    user: qiita_db.user import User
        The user performing the request
    filepath: str
        The path to the sample template file
    data_type: str, optional
        If filepath is a QIIME mapping file, the data type of the prep
        information file
    direct_upload: boolean, optional
        If filepath is a direct upload; if False we need to process the
        filepath as part of the study upload folder

    Returns
    -------
    dict of {'job': str}
        job: the id of the job adding the sample information to the study

    Raises
    ------
    HTTPError
        404 if the filepath doesn't exist
    """
    # Check if the current user has access to the study
    sample_template_checks(study_id, user)

    # Check if the file exists
    if not direct_upload:
        fp_rsp = check_fp(study_id, filepath)
        if fp_rsp['status'] != 'success':
            raise HTTPError(404, reason='Filepath not found')
        filepath = fp_rsp['file']

    is_mapping_file = looks_like_qiime_mapping_file(filepath)
    if is_mapping_file and not data_type:
        raise HTTPError(400, reason='Please, choose a data type if uploading '
                        'a QIIME mapping file')

    qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
    cmd = qiita_plugin.get_command('create_sample_template')
    params = Parameters.load(
        cmd, values_dict={'fp': filepath, 'study_id': study_id,
                          'is_mapping_file': is_mapping_file,
                          'data_type': data_type})
    job = ProcessingJob.create(user, params, True)
    r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id,
                 dumps({'job_id': job.id}))
    job.submit()
    return {'job': job.id}
Exemple #19
0
    def post(self, study_id):
        method = self.get_argument('remote-request-type')
        url = self.get_argument('inputURL')
        ssh_key = self.request.files['ssh-key'][0]['body']
        status = 'success'
        message = ''

        try:
            study = Study(int(study_id))
        except QiitaDBUnknownIDError:
            raise HTTPError(404, reason="Study %s does not exist" % study_id)
        check_access(self.current_user,
                     study,
                     no_public=True,
                     raise_error=True)

        _, upload_folder = get_mountpoint("uploads")[0]
        upload_folder = join(upload_folder, study_id)
        ssh_key_fp = join(upload_folder, '.key.txt')

        create_nested_path(upload_folder)

        with open(ssh_key_fp, 'wb') as f:
            f.write(ssh_key)
        chmod(ssh_key_fp, 0o600)

        qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
        if method == 'list':
            cmd = qiita_plugin.get_command('list_remote_files')
            params = Parameters.load(cmd,
                                     values_dict={
                                         'url': url,
                                         'private_key': ssh_key_fp,
                                         'study_id': study_id
                                     })
        elif method == 'transfer':
            cmd = qiita_plugin.get_command('download_remote_files')
            params = Parameters.load(cmd,
                                     values_dict={
                                         'url': url,
                                         'private_key': ssh_key_fp,
                                         'destination': upload_folder
                                     })
        else:
            status = 'error'
            message = 'Not a valid method'

        if status == 'success':
            job = ProcessingJob.create(self.current_user, params, True)
            job.submit()
            r_client.set(UPLOAD_STUDY_FORMAT % study_id,
                         dumps({'job_id': job.id}))

        self.write({'status': status, 'message': message})
    def test_artifact_post_request(self):
        # No access
        with self.assertRaises(QiitaHTTPError):
            artifact_post_req(User('*****@*****.**'), 1)

        artifact_post_req(User('*****@*****.**'), 2)
        # Wait until the job is completed
        wait_for_prep_information_job(1)
        # Check that the delete function has been actually called
        job = ProcessingJob(loads(r_client.get('prep_template_1'))['job_id'])
        self.assertEqual(job.status, 'error')
        self.assertIn('Cannot delete artifact 2', job.log.msg)
Exemple #21
0
def wait_for_processing_job(job_id):
    """Waits until a processing job is completed

    Parameters
    ----------
    job_id : str
        Job id
    """
    job = ProcessingJob(job_id)
    while job.status not in ('success', 'error'):
        sleep(0.05)
    sleep(0.05)
Exemple #22
0
    def test_artifact_summary_post_request(self):
        # No access
        with self.assertRaises(QiitaHTTPError):
            artifact_summary_post_request(User('*****@*****.**'), 1)

        # Returns already existing job
        job = ProcessingJob.create(
            User('*****@*****.**'),
            Parameters.load(Command(7), values_dict={'input_data': 2}))
        job._set_status('queued')
        obs = artifact_summary_post_request(User('*****@*****.**'), 2)
        exp = {'job': [job.id, 'queued', None]}
        self.assertEqual(obs, exp)
    def test_artifact_summary_post_request(self):
        # No access
        with self.assertRaises(QiitaHTTPError):
            artifact_summary_post_request(User('*****@*****.**'), 1)

        # Returns already existing job
        job = ProcessingJob.create(
            User('*****@*****.**'),
            Parameters.load(Command(7), values_dict={'input_data': 2})
        )
        job._set_status('queued')
        obs = artifact_summary_post_request(User('*****@*****.**'), 2)
        exp = {'job': [job.id, 'queued', None]}
        self.assertEqual(obs, exp)
Exemple #24
0
    def display_template(self, study_id, msg):
        """Simple function to avoid duplication of code"""
        study_id = int(study_id)
        study = Study(study_id)
        user = self.current_user
        level = 'info'
        message = ''
        remote_url = ''
        remote_files = []
        check_access(user, study, no_public=True, raise_error=True)

        job_info = r_client.get(UPLOAD_STUDY_FORMAT % study_id)
        if job_info:
            job_info = defaultdict(lambda: '', loads(job_info))
            job_id = job_info['job_id']
            job = ProcessingJob(job_id)
            job_status = job.status
            processing = job_status not in ('success', 'error')
            url = job.parameters.values['url']
            if processing:
                if job.command.name == 'list_remote_files':
                    message = 'Retrieving remote files: listing %s' % url
                else:
                    message = 'Retrieving remote files: download %s' % url
            elif job_status == 'error':
                level = 'danger'
                message = job.log.msg.replace('\n', '</br>')
                # making errors nicer for users
                if 'No such file' in message:
                    message = 'URL not valid: <i>%s</i>, please review.' % url
            else:
                remote_url = job_info['url']
                remote_files = job_info['files']
                level = job_info['alert_type']
                message = job_info['alert_msg'].replace('\n', '</br>')

        # getting the ontologies
        self.render('upload.html',
                    study_title=study.title,
                    study_info=study.info,
                    study_id=study_id,
                    is_admin=user.level == 'admin',
                    extensions=','.join(qiita_config.valid_upload_extension),
                    max_upload_size=qiita_config.max_upload_size,
                    level=level,
                    message=message,
                    remote_url=remote_url,
                    remote_files=remote_files,
                    files=get_files_from_uploads_folders(str(study_id)))
Exemple #25
0
def wait_for_processing_job(job_id):
    """Waits until a processing job is completed

    Parameters
    ----------
    job_id : str
        Job id
    """
    job = ProcessingJob(job_id)
    while job.status not in ('success', 'error'):
        sleep(0.8)
    # this print is useful for debugging
    if job.status == 'error':
        print("==> %s: %s" % (job.id, job.log.msg))
    sleep(0.8)
Exemple #26
0
    def post(self):
        analysis_id = int(self.get_argument('analysis_id'))

        user = self.current_user
        check_analysis_access(user, Analysis(analysis_id))

        qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
        cmd = qiita_plugin.get_command('delete_analysis')
        params = Parameters.load(cmd, values_dict={'analysis_id': analysis_id})
        job = ProcessingJob.create(user, params, True)
        # Store the job id attaching it to the sample template id
        r_client.set('analysis_delete_%d' % analysis_id,
                     dumps({'job_id': job.id}))
        job.submit()

        self.redirect("%s/analysis/list/" % (qiita_config.portal_dir))
Exemple #27
0
    def patch(self):
        """Updates / stores feature information in the archive.

        Notes
        -----
            Argument "path" must be the Qiita job_id which is used to infer
            the merging scheme.
            Argument "value" is a json string, i.e. result of a json.dump(obj)
            of a dictionary, keyed with feature identifiers.
        """
        req_path = self.get_argument('path')
        req_value = self.get_argument('value')

        ms = Archive.get_merging_scheme_from_job(ProcessingJob(req_path))

        self.write(Archive.insert_features(ms, loads(req_value)))
Exemple #28
0
def artifact_summary_post_request(user_id, artifact_id):
    """Launches the HTML summary generation and returns the job information

    Parameters
    ----------
    user_id : str
        The user making the request
    artifact_id : int or str
        The artifact id

    Returns
    -------
    dict of objects
        A dictionary containing the artifact summary information
        {'status': str,
         'message': str,
         'job': list of [str, str, str]}
    """
    artifact_id = int(artifact_id)
    artifact = Artifact(artifact_id)

    access_error = check_access(artifact.study.id, user_id)
    if access_error:
        return access_error

    # Check if the summary is being generated or has been already generated
    command = Command.get_html_generator(artifact.artifact_type)
    jobs = artifact.jobs(cmd=command)
    jobs = [j for j in jobs if j.status in ['queued', 'running', 'success']]
    if jobs:
        # The HTML summary is either being generated or already generated.
        # Return the information of that job so we only generate the HTML
        # once
        job = jobs[0]
    else:
        # Create a new job to generate the HTML summary and return the newly
        # created job information
        job = ProcessingJob.create(
            User(user_id),
            Parameters.load(command, values_dict={'input_data': artifact_id}))
        job.submit()

    return {
        'status': 'success',
        'message': '',
        'job': [job.id, job.status, job.step]
    }
Exemple #29
0
    def test_patch(self):
        # Create a new job - through a workflow since that is the only way
        # of creating jobs in the interface
        exp_command = Command(1)
        json_str = (
            '{"input_data": 1, "max_barcode_errors": 1.5, '
            '"barcode_type": "golay_12", "max_bad_run_length": 3, '
            '"rev_comp": false, "phred_quality_threshold": 3, '
            '"rev_comp_barcode": false, "rev_comp_mapping_barcodes": false, '
            '"min_per_read_length_fraction": 0.75, "sequence_max_n": 0}')
        exp_params = Parameters.load(exp_command, json_str=json_str)
        exp_user = User('*****@*****.**')
        name = "Test processing workflow"

        # tests success
        wf = ProcessingWorkflow.from_scratch(exp_user,
                                             exp_params,
                                             name=name,
                                             force=True)

        graph = wf.graph
        nodes = graph.nodes()
        job_id = nodes[0].id

        response = self.patch('/study/process/job/', {
            'op': 'remove',
            'path': job_id
        })
        self.assertEqual(response.code, 200)
        exp = {
            'status':
            'error',
            'message':
            "Can't delete job %s. It is 'in_construction' "
            "status. Please use /study/process/workflow/" % job_id
        }
        self.assertEqual(loads(response.body), exp)

        # Test success
        ProcessingJob(job_id)._set_error('Killed for testing')
        response = self.patch('/study/process/job/', {
            'op': 'remove',
            'path': job_id
        })
        self.assertEqual(response.code, 200)
        exp = {'status': 'success', 'message': ''}
        self.assertEqual(loads(response.body), exp)
Exemple #30
0
def artifact_summary_post_request(user_id, artifact_id):
    """Launches the HTML summary generation and returns the job information

    Parameters
    ----------
    user_id : str
        The user making the request
    artifact_id : int or str
        The artifact id

    Returns
    -------
    dict of objects
        A dictionary containing the artifact summary information
        {'status': str,
         'message': str,
         'job': list of [str, str, str]}
    """
    artifact_id = int(artifact_id)
    artifact = Artifact(artifact_id)

    access_error = check_access(artifact.study.id, user_id)
    if access_error:
        return access_error

    # Check if the summary is being generated or has been already generated
    command = Command.get_html_generator(artifact.artifact_type)
    jobs = artifact.jobs(cmd=command)
    jobs = [j for j in jobs if j.status in ['queued', 'running', 'success']]
    if jobs:
        # The HTML summary is either being generated or already generated.
        # Return the information of that job so we only generate the HTML
        # once
        job = jobs[0]
    else:
        # Create a new job to generate the HTML summary and return the newly
        # created job information
        job = ProcessingJob.create(
            User(user_id),
            Parameters.load(command, values_dict={'input_data': artifact_id}))
        job.submit()

    return {'status': 'success',
            'message': '',
            'job': [job.id, job.status, job.step]}
Exemple #31
0
    def test_artifact_summary_post_request(self):
        # No access
        obs = artifact_summary_post_request('*****@*****.**', 1)
        exp = {'status': 'error',
               'message': 'User does not have access to study'}
        self.assertEqual(obs, exp)

        # Returns already existing job
        job = ProcessingJob.create(
            User('*****@*****.**'),
            Parameters.load(Command(7), values_dict={'input_data': 2})
        )
        job._set_status('queued')
        obs = artifact_summary_post_request('*****@*****.**', 2)
        exp = {'status': 'success',
               'message': '',
               'job': [job.id, 'queued', None]}
        self.assertEqual(obs, exp)
Exemple #32
0
    def post(self, study_id):
        method = self.get_argument('remote-request-type')
        url = self.get_argument('inputURL')
        ssh_key = self.request.files['ssh-key'][0]['body']
        status = 'success'
        message = ''

        try:
            study = Study(int(study_id))
        except QiitaDBUnknownIDError:
            raise HTTPError(404, reason="Study %s does not exist" % study_id)
        check_access(
            self.current_user, study, no_public=True, raise_error=True)

        _, upload_folder = get_mountpoint("uploads")[0]
        upload_folder = join(upload_folder, study_id)
        ssh_key_fp = join(upload_folder, '.key.txt')

        create_nested_path(upload_folder)

        with open(ssh_key_fp, 'w') as f:
            f.write(ssh_key)

        qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
        if method == 'list':
            cmd = qiita_plugin.get_command('list_remote_files')
            params = Parameters.load(cmd, values_dict={
                'url': url, 'private_key': ssh_key_fp, 'study_id': study_id})
        elif method == 'transfer':
            cmd = qiita_plugin.get_command('download_remote_files')
            params = Parameters.load(cmd, values_dict={
                'url': url, 'private_key': ssh_key_fp,
                'destination': upload_folder})
        else:
            status = 'error'
            message = 'Not a valid method'

        if status == 'success':
            job = ProcessingJob.create(self.current_user, params, True)
            job.submit()
            r_client.set(
                UPLOAD_STUDY_FORMAT % study_id, dumps({'job_id': job.id}))

        self.write({'status': status, 'message': message})
Exemple #33
0
    def get(self):
        user = self.current_user
        is_local_request = is_localhost(self.request.headers['host'])

        uanalyses = user.shared_analyses | user.private_analyses
        user_analysis_ids = set([a.id for a in uanalyses])

        panalyses = Analysis.get_by_status('public')
        public_analysis_ids = set([a.id for a in panalyses])
        public_analysis_ids = public_analysis_ids - user_analysis_ids

        user_analyses = generate_analysis_list(user_analysis_ids)
        public_analyses = generate_analysis_list(public_analysis_ids, True)

        dlop = partial(download_link_or_path, is_local_request)

        messages = {'info': '', 'danger': ''}
        for analysis_id in user_analysis_ids:
            job_info = r_client.get('analysis_delete_%d' % analysis_id)
            if job_info:
                job_info = defaultdict(lambda: '', loads(job_info))
                job_id = job_info['job_id']
                job = ProcessingJob(job_id)
                job_status = job.status
                processing = job_status not in ('success', 'error')
                if processing:
                    messages['info'] += ('Analysis %s is being deleted<br/>' %
                                         analysis_id)
                elif job_status == 'error':
                    messages['danger'] += (job.log.msg.replace('\n', '<br/>') +
                                           '<br/>')
                else:
                    if job_info['alert_type'] not in messages:
                        messages[job_info['alert_type']] = []
                    messages[job_info['alert_type']] += (
                        job.log.msg.replace('\n', '<br/>') + '<br/>')

        self.render("list_analyses.html",
                    user_analyses=user_analyses,
                    public_analyses=public_analyses,
                    messages=messages,
                    dlop=dlop)
Exemple #34
0
def artifact_summary_post_request(user, artifact_id):
    """Launches the HTML summary generation and returns the job information

    Parameters
    ----------
    user : qiita_db.user.User
        The user making the request
    artifact_id : int or str
        The artifact id

    Returns
    -------
    dict of objects
        A dictionary containing the job summary information
        {'job': [str, str, str]}
    """
    artifact_id = int(artifact_id)
    artifact = Artifact(artifact_id)

    check_artifact_access(user, artifact)

    # Check if the summary is being generated or has been already generated
    command = Command.get_html_generator(artifact.artifact_type)
    jobs = artifact.jobs(cmd=command)
    jobs = [j for j in jobs if j.status in ['queued', 'running', 'success']]
    if jobs:
        # The HTML summary is either being generated or already generated.
        # Return the information of that job so we only generate the HTML
        # once - Magic number 0 -> we are ensuring that there is only one
        # job generating the summary, so we can use the index 0 to access to
        # that job
        job = jobs[0]
    else:
        # Create a new job to generate the HTML summary and return the newly
        # created job information
        job = ProcessingJob.create(
            user,
            Parameters.load(command, values_dict={'input_data': artifact_id}),
            True)
        job.submit()

    return {'job': [job.id, job.status, job.step]}
Exemple #35
0
    def test_artifact_summary_post_request(self):
        # No access
        obs = artifact_summary_post_request('*****@*****.**', 1)
        exp = {
            'status': 'error',
            'message': 'User does not have access to study'
        }
        self.assertEqual(obs, exp)

        # Returns already existing job
        job = ProcessingJob.create(
            User('*****@*****.**'),
            Parameters.load(Command(7), values_dict={'input_data': 2}))
        job._set_status('queued')
        obs = artifact_summary_post_request('*****@*****.**', 2)
        exp = {
            'status': 'success',
            'message': '',
            'job': [job.id, 'queued', None]
        }
        self.assertEqual(obs, exp)
Exemple #36
0
def artifact_summary_post_request(user, artifact_id):
    """Launches the HTML summary generation and returns the job information

    Parameters
    ----------
    user : qiita_db.user.User
        The user making the request
    artifact_id : int or str
        The artifact id

    Returns
    -------
    dict of objects
        A dictionary containing the job summary information
        {'job': [str, str, str]}
    """
    artifact_id = int(artifact_id)
    artifact = Artifact(artifact_id)

    check_artifact_access(user, artifact)

    # Check if the summary is being generated or has been already generated
    command = Command.get_html_generator(artifact.artifact_type)
    jobs = artifact.jobs(cmd=command)
    jobs = [j for j in jobs if j.status in ['queued', 'running', 'success']]
    if jobs:
        # The HTML summary is either being generated or already generated.
        # Return the information of that job so we only generate the HTML
        # once - Magic number 0 -> we are ensuring that there is only one
        # job generating the summary, so we can use the index 0 to access to
        # that job
        job = jobs[0]
    else:
        # Create a new job to generate the HTML summary and return the newly
        # created job information
        job = ProcessingJob.create(user, Parameters.load(
            command, values_dict={'input_data': artifact_id}), True)
        job.submit()

    return {'job': [job.id, job.status, job.step]}
Exemple #37
0
def _submit(ctx, user, parameters):
    """Submit a plugin job to a cluster

    Parameters
    ----------
    ctx : qiita_db.ware.Dispatch
        A Dispatch object to submit through
    user : qiita_db.user.User
        The user doing the submission
    parameters : qiita_db.software.Parameters
        The parameters of the job

    Returns
    -------
    str
        The job id
    """
    job = ProcessingJob.create(user, parameters)
    redis_deets = {'job_id': job.id, 'pubsub': user.id,
                   'messages': user.id + ':messages'}
    ctx.submit_async(_redis_wrap, execute, redis_deets, job.id)
    return job.id
Exemple #38
0
def sample_template_delete_req(study_id, user_id):
    """Deletes the sample template attached to the study

    Parameters
    ----------
    study_id : int
        The current study object id
    user_id : str
        The current user object id

    Returns
    -------
    dict
        results dictonary in the format
        {'status': status,
         'message': msg}

    status can be success, warning, or error depending on result
    message has the warnings or errors
    """
    exists = _check_sample_template_exists(int(study_id))
    if exists['status'] != 'success':
        return exists
    access_error = check_access(int(study_id), user_id)
    if access_error:
        return access_error

    qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
    cmd = qiita_plugin.get_command('delete_sample_template')
    params = Parameters.load(cmd, values_dict={'study': int(study_id)})
    job = ProcessingJob.create(User(user_id), params)

    # Store the job id attaching it to the sample template id
    r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id,
                 dumps({'job_id': job.id}))

    job.submit()

    return {'status': 'success', 'message': ''}
Exemple #39
0
def job_ajax_get_req(job_id):
    """Returns the job information

    Parameters
    ----------
    job_id : str
        The job id

    Returns
    -------
    dict of objects
        A dictionary containing the job information
        {'status': str,
         'message': str,
         'job_id': str,
         'job_status': str,
         'job_step': str,
         'job_parameters': dict of {str: str}}
    """
    job = ProcessingJob(job_id)
    cmd = job.command
    sw = cmd.software
    job_status = job.status
    job_error = job.log.msg if job.log is not None else None
    return {
        'status': 'success',
        'message': '',
        'job_id': job.id,
        'job_external_id': job.external_id,
        'job_status': job_status,
        'job_step': job.step,
        'job_parameters': job.parameters.values,
        'job_error': job_error,
        'command': cmd.name,
        'command_description': cmd.description,
        'software': sw.name,
        'software_version': sw.version
    }
Exemple #40
0
def get_sample_template_processing_status(st_id):
    # Initialize variables here
    processing = False
    alert_type = ''
    alert_msg = ''
    job_info = r_client.get(SAMPLE_TEMPLATE_KEY_FORMAT % st_id)
    if job_info:
        job_info = defaultdict(lambda: '', loads(job_info))
        job_id = job_info['job_id']
        job = ProcessingJob(job_id)
        job_status = job.status
        processing = job_status not in ('success', 'error')
        if processing:
            alert_type = 'info'
            alert_msg = 'This sample template is currently being processed'
        elif job_status == 'error':
            alert_type = 'danger'
            alert_msg = job.log.msg.replace('\n', '</br>')
        else:
            alert_type = job_info['alert_type']
            alert_msg = job_info['alert_msg'].replace('\n', '</br>')

    return processing, alert_type, alert_msg
Exemple #41
0
def sample_template_handler_delete_request(study_id, user):
    """Deletes the sample template

    Parameters
    ----------
    study_id: int
        The study to delete the sample information
    user: qiita_db.user
        The user performing the request

    Returns
    -------
    dict of {'job': str}
        job: the id of the job deleting the sample information to the study

    Raises
    ------
    HTTPError
        404 If the sample template doesn't exist
    """
    # Check if the current user has access to the study and if the sample
    # template exists
    sample_template_checks(study_id, user, check_exists=True)

    qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
    cmd = qiita_plugin.get_command('delete_sample_template')
    params = Parameters.load(cmd, values_dict={'study': int(study_id)})
    job = ProcessingJob.create(user, params, True)

    # Store the job if deleteing the sample template
    r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id,
                 dumps({'job_id': job.id}))

    job.submit()

    return {'job': job.id}
Exemple #42
0
def sample_template_handler_delete_request(study_id, user):
    """Deletes the sample template

    Parameters
    ----------
    study_id: int
        The study to delete the sample information
    user: qiita_db.user
        The user performing the request

    Returns
    -------
    dict of {'job': str}
        job: the id of the job deleting the sample information to the study

    Raises
    ------
    HTTPError
        404 If the sample template doesn't exist
    """
    # Check if the current user has access to the study and if the sample
    # template exists
    sample_template_checks(study_id, user, check_exists=True)

    qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
    cmd = qiita_plugin.get_command('delete_sample_template')
    params = Parameters.load(cmd, values_dict={'study': int(study_id)})
    job = ProcessingJob.create(user, params, True)

    # Store the job if deleteing the sample template
    r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id,
                 dumps({'job_id': job.id}))

    job.submit()

    return {'job': job.id}
    def test_artifact_summary_get_request(self):
        user = User('*****@*****.**')
        # Artifact w/o summary
        obs = artifact_summary_get_request(user, 1)
        exp_files = [
            (1L, '1_s_G1_L001_sequences.fastq.gz (raw forward seqs)'),
            (2L, '1_s_G1_L001_sequences_barcodes.fastq.gz (raw barcodes)')]
        exp = {'name': 'Raw data 1',
               'artifact_id': 1,
               'artifact_type': 'FASTQ',
               'artifact_timestamp': '2012-10-01 09:10',
               'visibility': 'private',
               'editable': True,
               'buttons': ('<button onclick="if (confirm(\'Are you sure you '
                           'want to make public artifact id: 1?\')) { '
                           'set_artifact_visibility(\'public\', 1) }" '
                           'class="btn btn-primary btn-sm">Make public'
                           '</button> <button onclick="if (confirm(\'Are you '
                           'sure you want to revert to sandbox artifact id: '
                           '1?\')) { set_artifact_visibility(\'sandbox\', 1) '
                           '}" class="btn btn-primary btn-sm">Revert to '
                           'sandbox</button>'),
               'processing_info': {},
               'files': exp_files,
               'is_from_analysis': False,
               'summary': None,
               'job': None,
               'errored_summary_jobs': []}
        self.assertEqual(obs, exp)

        # Artifact with summary being generated
        job = ProcessingJob.create(
            User('*****@*****.**'),
            Parameters.load(Command(7), values_dict={'input_data': 1})
        )
        job._set_status('queued')
        obs = artifact_summary_get_request(user, 1)
        exp = {'name': 'Raw data 1',
               'artifact_id': 1,
               'artifact_type': 'FASTQ',
               'artifact_timestamp': '2012-10-01 09:10',
               'visibility': 'private',
               'editable': True,
               'buttons': ('<button onclick="if (confirm(\'Are you sure you '
                           'want to make public artifact id: 1?\')) { '
                           'set_artifact_visibility(\'public\', 1) }" '
                           'class="btn btn-primary btn-sm">Make public'
                           '</button> <button onclick="if (confirm(\'Are you '
                           'sure you want to revert to sandbox artifact id: '
                           '1?\')) { set_artifact_visibility(\'sandbox\', 1) '
                           '}" class="btn btn-primary btn-sm">Revert to '
                           'sandbox</button>'),
               'processing_info': {},
               'files': exp_files,
               'is_from_analysis': False,
               'summary': None,
               'job': [job.id, 'queued', None],
               'errored_summary_jobs': []}
        self.assertEqual(obs, exp)

        # Artifact with summary
        fd, fp = mkstemp(suffix=".html")
        close(fd)
        with open(fp, 'w') as f:
            f.write('<b>HTML TEST - not important</b>\n')
        a = Artifact(1)
        a.set_html_summary(fp)
        self._files_to_remove.extend([fp, a.html_summary_fp[1]])
        exp_files.append(
            (a.html_summary_fp[0],
             '%s (html summary)' % basename(a.html_summary_fp[1])))
        exp_summary_path = relpath(
            a.html_summary_fp[1], qiita_config.base_data_dir)
        obs = artifact_summary_get_request(user, 1)
        exp = {'name': 'Raw data 1',
               'artifact_id': 1,
               'artifact_type': 'FASTQ',
               'artifact_timestamp': '2012-10-01 09:10',
               'visibility': 'private',
               'editable': True,
               'buttons': ('<button onclick="if (confirm(\'Are you sure you '
                           'want to make public artifact id: 1?\')) { '
                           'set_artifact_visibility(\'public\', 1) }" '
                           'class="btn btn-primary btn-sm">Make public'
                           '</button> <button onclick="if (confirm(\'Are you '
                           'sure you want to revert to sandbox artifact id: '
                           '1?\')) { set_artifact_visibility(\'sandbox\', 1) '
                           '}" class="btn btn-primary btn-sm">Revert to '
                           'sandbox</button>'),
               'processing_info': {},
               'files': exp_files,
               'is_from_analysis': False,
               'summary': exp_summary_path,
               'job': None,
               'errored_summary_jobs': []}
        self.assertEqual(obs, exp)

        # No access
        demo_u = User('*****@*****.**')
        with self.assertRaises(QiitaHTTPError):
            obs = artifact_summary_get_request(demo_u, 1)

        # A non-owner/share user can't see the files
        a.visibility = 'public'
        obs = artifact_summary_get_request(demo_u, 1)
        exp = {'name': 'Raw data 1',
               'artifact_id': 1,
               'artifact_type': 'FASTQ',
               'artifact_timestamp': '2012-10-01 09:10',
               'visibility': 'public',
               'editable': False,
               'buttons': '',
               'processing_info': {},
               'files': [],
               'is_from_analysis': False,
               'summary': exp_summary_path,
               'job': None,
               'errored_summary_jobs': []}
        self.assertEqual(obs, exp)

        # returnig to private
        a.visibility = 'private'

        # admin gets buttons
        obs = artifact_summary_get_request(User('*****@*****.**'), 2)
        exp_files = [
            (3L, '1_seqs.fna (preprocessed fasta)'),
            (4L, '1_seqs.qual (preprocessed fastq)'),
            (5L, '1_seqs.demux (preprocessed demux)')]
        exp = {'name': 'Demultiplexed 1',
               'artifact_id': 2,
               'artifact_type': 'Demultiplexed',
               'artifact_timestamp': '2012-10-01 10:10',
               'visibility': 'private',
               'editable': True,
               'buttons': ('<button onclick="if (confirm(\'Are you sure you '
                           'want to make public artifact id: 2?\')) { '
                           'set_artifact_visibility(\'public\', 2) }" '
                           'class="btn btn-primary btn-sm">Make public'
                           '</button> <button onclick="if (confirm(\'Are you '
                           'sure you want to revert to sandbox artifact id: '
                           '2?\')) { set_artifact_visibility(\'sandbox\', 2) '
                           '}" class="btn btn-primary btn-sm">Revert to '
                           'sandbox</button> <a class="btn btn-primary '
                           'btn-sm" href="/ebi_submission/2"><span '
                           'class="glyphicon glyphicon-export"></span> '
                           'Submit to EBI</a> <a class="btn btn-primary '
                           'btn-sm" href="/vamps/2"><span class="glyphicon '
                           'glyphicon-export"></span> Submit to VAMPS</a>'),
               'processing_info': {
                    'command': 'Split libraries FASTQ',
                    'software': 'QIIME',
                    'software_version': '1.9.1',
                    'processing_parameters': {
                        'max_barcode_errors': '1.5', 'sequence_max_n': '0',
                        'max_bad_run_length': '3', 'phred_offset': u'auto',
                        'rev_comp': 'False', 'phred_quality_threshold': '3',
                        'input_data': '1', 'rev_comp_barcode': 'False',
                        'rev_comp_mapping_barcodes': 'False',
                        'min_per_read_length_fraction': '0.75',
                        'barcode_type': u'golay_12'}},
               'files': exp_files,
               'is_from_analysis': False,
               'summary': None,
               'job': None,
               'errored_summary_jobs': []}
        self.assertEqual(obs, exp)

        # analysis artifact
        obs = artifact_summary_get_request(user, 8)
        exp = {'name': 'noname',
               'artifact_id': 8,
               'artifact_type': 'BIOM',
               # this value changes on build so copy from obs
               'artifact_timestamp': obs['artifact_timestamp'],
               'visibility': 'sandbox',
               'editable': True,
               'buttons': '',
               'processing_info': {},
               'files': [(27, 'biom_table.biom (biom)')],
               'is_from_analysis': True,
               'summary': None,
               'job': None,
               'errored_summary_jobs': []}
        self.assertEqual(obs, exp)
Exemple #44
0
def sample_template_handler_patch_request(user, req_op, req_path,
                                          req_value=None, req_from=None):
    """Patches the sample template

    Parameters
    ----------
    user: qiita_db.user.User
        The user performing the request
    req_op : str
        The operation to perform on the sample template
    req_path : str
        The path to the attribute to patch
    req_value : str, optional
        The new value
    req_from : str, optional
        The original path of the element

    Returns
    -------

    Raises
    ------
    HTTPError
        400 If the path parameter doens't follow the expected format
        400 If the given operation is not supported
    """
    req_path = [v for v in req_path.split('/') if v]
    # At this point we know the path should be at least length 2
    if len(req_path) < 2:
        raise HTTPError(400, reason='Incorrect path parameter')

    study_id = int(req_path[0])
    # Check if the current user has access to the study and if the sample
    # template exists
    sample_template_checks(study_id, user, check_exists=True)

    if req_op == 'remove':
        # Path format
        # column: study_id/columns/column_name
        # sample: study_id/samples/sample_id
        if len(req_path) != 3:
            raise HTTPError(400, reason='Incorrect path parameter')

        attribute = req_path[1]
        attr_id = req_path[2]

        qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
        cmd = qiita_plugin.get_command('delete_sample_or_column')
        params = Parameters.load(
            cmd, values_dict={'obj_class': 'SampleTemplate',
                              'obj_id': study_id,
                              'sample_or_col': attribute,
                              'name': attr_id})
        job = ProcessingJob.create(user, params, True)
        # Store the job id attaching it to the sample template id
        r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id,
                     dumps({'job_id': job.id}))
        job.submit()
        return {'job': job.id}
    elif req_op == 'replace':
        # WARNING: Although the patch operation is a replace, is not a full
        # true replace. A replace is in theory equivalent to a remove + add.
        # In this case, the replace operation doesn't necessarily removes
        # anything (e.g. when only new columns/samples are being added to the)
        # sample information.
        # Path format: study_id/data
        # Forcing to specify data for extensibility. In the future we may want
        # to use this function to replace other elements of the sample
        # information
        if len(req_path) != 2:
            raise HTTPError(400, reason='Incorrect path parameter')

        attribute = req_path[1]

        if attribute == 'data':
            # Update the sample information
            if req_value is None:
                raise HTTPError(400, reason="Value is required when updating "
                                "sample information")

            # Check if the file exists
            fp_rsp = check_fp(study_id, req_value)
            if fp_rsp['status'] != 'success':
                raise HTTPError(404, reason='Filepath not found')
            filepath = fp_rsp['file']

            qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
            cmd = qiita_plugin.get_command('update_sample_template')
            params = Parameters.load(
                cmd, values_dict={'study': study_id,
                                  'template_fp': filepath})
            job = ProcessingJob.create(user, params, True)

            # Store the job id attaching it to the sample template id
            r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id,
                         dumps({'job_id': job.id}))

            job.submit()
            return {'job': job.id}
        else:
            raise HTTPError(404, reason='Attribute %s not found' % attribute)

    else:
        raise HTTPError(400, reason='Operation %s not supported. Current '
                        'supported operations: remove, replace' % req_op)
Exemple #45
0
    def test_artifact_summary_get_request(self):
        # Artifact w/o summary
        obs = artifact_summary_get_request('*****@*****.**', 1)
        exp_p_jobs = [
            ['063e553b-327c-4818-ab4a-adfe58e49860', 'Split libraries FASTQ',
             'queued', None, None],
            ['bcc7ebcd-39c1-43e4-af2d-822e3589f14d', 'Split libraries',
             'running', 'demultiplexing', None]]
        exp_files = [
            (1L, '1_s_G1_L001_sequences.fastq.gz (raw forward seqs)'),
            (2L, '1_s_G1_L001_sequences_barcodes.fastq.gz (raw barcodes)')]
        exp = {'status': 'success',
               'message': '',
               'name': 'Raw data 1',
               'summary': None,
               'job': None,
               'processing_jobs': exp_p_jobs,
               'errored_jobs': [],
               'visibility': 'private',
               'buttons': '<button onclick="if (confirm(\'Are you sure you '
                          'want to make public artifact id: 1?\')) { '
                          'set_artifact_visibility(\'public\', 1) }" '
                          'class="btn btn-primary btn-sm">Make public</button>'
                          ' <button onclick="if (confirm(\'Are you sure you '
                          'want to revert to sandbox artifact id: 1?\')) '
                          '{ set_artifact_visibility(\'sandbox\', 1) }" '
                          'class="btn btn-primary btn-sm">Revert to '
                          'sandbox</button>',
               'files': exp_files,
               'editable': True}
        self.assertEqual(obs, exp)

        # Artifact with summary being generated
        job = ProcessingJob.create(
            User('*****@*****.**'),
            Parameters.load(Command(7), values_dict={'input_data': 1})
        )
        job._set_status('queued')
        obs = artifact_summary_get_request('*****@*****.**', 1)
        exp = {'status': 'success',
               'message': '',
               'name': 'Raw data 1',
               'summary': None,
               'job': [job.id, 'queued', None],
               'processing_jobs': exp_p_jobs,
               'errored_jobs': [],
               'visibility': 'private',
               'buttons': '<button onclick="if (confirm(\'Are you sure you '
                          'want to make public artifact id: 1?\')) { '
                          'set_artifact_visibility(\'public\', 1) }" '
                          'class="btn btn-primary btn-sm">Make public</button>'
                          ' <button onclick="if (confirm(\'Are you sure you '
                          'want to revert to sandbox artifact id: 1?\')) { '
                          'set_artifact_visibility(\'sandbox\', 1) }" '
                          'class="btn btn-primary btn-sm">Revert to '
                          'sandbox</button>',
               'files': exp_files,
               'editable': True}
        self.assertEqual(obs, exp)

        # Artifact with summary
        fd, fp = mkstemp(suffix=".html")
        close(fd)
        with open(fp, 'w') as f:
            f.write('<b>HTML TEST - not important</b>\n')
        a = Artifact(1)
        a.html_summary_fp = fp
        self._files_to_remove.extend([fp, a.html_summary_fp[1]])
        exp_files.append(
            (a.html_summary_fp[0],
             '%s (html summary)' % basename(a.html_summary_fp[1])))
        obs = artifact_summary_get_request('*****@*****.**', 1)
        exp = {'status': 'success',
               'message': '',
               'name': 'Raw data 1',
               'summary': '<b>HTML TEST - not important</b>\n',
               'job': None,
               'processing_jobs': exp_p_jobs,
               'errored_jobs': [],
               'visibility': 'private',
               'buttons': '<button onclick="if (confirm(\'Are you sure you '
                          'want to make public artifact id: 1?\')) { '
                          'set_artifact_visibility(\'public\', 1) }" '
                          'class="btn btn-primary btn-sm">Make public</button>'
                          ' <button onclick="if (confirm(\'Are you sure you '
                          'want to revert to sandbox artifact id: 1?\')) { '
                          'set_artifact_visibility(\'sandbox\', 1) }" '
                          'class="btn btn-primary btn-sm">Revert to '
                          'sandbox</button>',
               'files': exp_files,
               'editable': True}
        self.assertEqual(obs, exp)

        # No access
        obs = artifact_summary_get_request('*****@*****.**', 1)
        exp = {'status': 'error',
               'message': 'User does not have access to study'}
        self.assertEqual(obs, exp)

        # A non-owner/share user can't see the files
        a.visibility = 'public'
        obs = artifact_summary_get_request('*****@*****.**', 1)
        exp = {'status': 'success',
               'message': '',
               'name': 'Raw data 1',
               'summary': '<b>HTML TEST - not important</b>\n',
               'job': None,
               'processing_jobs': exp_p_jobs,
               'errored_jobs': [],
               'visibility': 'public',
               'buttons': '',
               'files': [],
               'editable': False}
        self.assertEqual(obs, exp)
    def test_complete_job(self):
        # Complete success
        pt = npt.assert_warns(
            QiitaDBWarning, PrepTemplate.create,
            pd.DataFrame({'new_col': {'1.SKD6.640190': 1}}),
            Study(1), '16S')
        c_job = ProcessingJob.create(
            User('*****@*****.**'),
            Parameters.load(
                Command.get_validator('BIOM'),
                values_dict={'template': pt.id,
                             'files': dumps({'BIOM': ['file']}),
                             'artifact_type': 'BIOM'}), True)
        c_job._set_status('running')
        fd, fp = mkstemp(suffix='_table.biom')
        close(fd)
        with open(fp, 'w') as f:
            f.write('\n')
        self._clean_up_files.append(fp)
        exp_artifact_count = get_count('qiita.artifact') + 1
        payload = dumps(
            {'success': True, 'error': '',
             'artifacts': {'OTU table': {'filepaths': [(fp, 'biom')],
                                         'artifact_type': 'BIOM'}}})
        job = self._create_job('complete_job', {'job_id': c_job.id,
                                                'payload': payload})
        private_task(job.id)
        self.assertEqual(job.status, 'success')
        self.assertEqual(c_job.status, 'success')
        self.assertEqual(get_count('qiita.artifact'), exp_artifact_count)

        # Complete job error
        payload = dumps({'success': False, 'error': 'Job failure'})
        job = self._create_job(
            'complete_job', {'job_id': 'bcc7ebcd-39c1-43e4-af2d-822e3589f14d',
                             'payload': payload})
        private_task(job.id)
        self.assertEqual(job.status, 'success')
        c_job = ProcessingJob('bcc7ebcd-39c1-43e4-af2d-822e3589f14d')
        self.assertEqual(c_job.status, 'error')
        self.assertEqual(c_job.log, LogEntry.newest_records(numrecords=1)[0])
        self.assertEqual(c_job.log.msg, 'Job failure')

        # Complete internal error
        pt = npt.assert_warns(
            QiitaDBWarning, PrepTemplate.create,
            pd.DataFrame({'new_col': {'1.SKD6.640190': 1}}),
            Study(1), '16S')
        c_job = ProcessingJob.create(
            User('*****@*****.**'),
            Parameters.load(
                Command.get_validator('BIOM'),
                values_dict={'template': pt.id,
                             'files': dumps({'BIOM': ['file']}),
                             'artifact_type': 'BIOM'}), True)
        c_job._set_status('running')
        fp = '/surprised/if/this/path/exists.biom'
        payload = dumps(
            {'success': True, 'error': '',
             'artifacts': {'OTU table': {'filepaths': [(fp, 'biom')],
                                         'artifact_type': 'BIOM'}}})
        job = self._create_job('complete_job', {'job_id': c_job.id,
                                                'payload': payload})
        private_task(job.id)
        self.assertEqual(job.status, 'success')
        self.assertEqual(c_job.status, 'error')
        self.assertIn('No such file or directory', c_job.log.msg)
Exemple #47
0
def prep_template_patch_req(user_id, req_op, req_path, req_value=None,
                            req_from=None):
    """Modifies an attribute of the prep template

    Parameters
    ----------
    user_id : str
        The id of the user performing the patch operation
    req_op : str
        The operation to perform on the prep information
    req_path : str
        The prep information and attribute to patch
    req_value : str, optional
        The value that needs to be modified
    req_from : str, optional
        The original path of the element

    Returns
    -------
    dict of {str, str, str}
        A dictionary with the following keys:
        - status: str, whether if the request is successful or not
        - message: str, if the request is unsuccessful, a human readable error
        - row_id: str, the row_id that we tried to delete
    """
    req_path = [v for v in req_path.split('/') if v]
    if req_op == 'replace':
        # The structure of the path should be /prep_id/attribute_to_modify/
        # so if we don't have those 2 elements, we should return an error
        if len(req_path) != 2:
            return {'status': 'error',
                    'message': 'Incorrect path parameter'}
        prep_id = int(req_path[0])
        attribute = req_path[1]

        # Check if the user actually has access to the prep template
        prep = PrepTemplate(prep_id)
        access_error = check_access(prep.study_id, user_id)
        if access_error:
            return access_error

        status = 'success'
        msg = ''
        if attribute == 'investigation_type':
            prep.investigation_type = req_value
        elif attribute == 'data':
            fp = check_fp(prep.study_id, req_value)
            if fp['status'] != 'success':
                return fp
            fp = fp['file']
            qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
            cmd = qiita_plugin.get_command('update_prep_template')
            params = Parameters.load(
                cmd, values_dict={'prep_template': prep_id, 'template_fp': fp})
            job = ProcessingJob.create(User(user_id), params, True)

            r_client.set(PREP_TEMPLATE_KEY_FORMAT % prep_id,
                         dumps({'job_id': job.id}))
            job.submit()
        elif attribute == 'name':
            prep.name = req_value.strip()
        else:
            # We don't understand the attribute so return an error
            return {'status': 'error',
                    'message': 'Attribute "%s" not found. '
                               'Please, check the path parameter' % attribute}

        return {'status': status, 'message': msg}
    elif req_op == 'remove':
        # The structure of the path should be:
        # /prep_id/row_id/{columns|samples}/name
        if len(req_path) != 4:
            return {'status': 'error',
                    'message': 'Incorrect path parameter'}
        prep_id = int(req_path[0])
        row_id = req_path[1]
        attribute = req_path[2]
        attr_id = req_path[3]

        # Check if the user actually has access to the study
        pt = PrepTemplate(prep_id)
        access_error = check_access(pt.study_id, user_id)
        if access_error:
            return access_error

        qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
        cmd = qiita_plugin.get_command('delete_sample_or_column')
        params = Parameters.load(
            cmd, values_dict={'obj_class': 'PrepTemplate',
                              'obj_id': prep_id,
                              'sample_or_col': attribute,
                              'name': attr_id})
        job = ProcessingJob.create(User(user_id), params, True)
        # Store the job id attaching it to the sample template id
        r_client.set(PREP_TEMPLATE_KEY_FORMAT % prep_id,
                     dumps({'job_id': job.id}))
        job.submit()
        return {'status': 'success', 'message': '', 'row_id': row_id}
    else:
        return {'status': 'error',
                'message': 'Operation "%s" not supported. '
                           'Current supported operations: replace, remove'
                           % req_op,
                'row_id': '0'}
Exemple #48
0
def artifact_post_req(user_id, filepaths, artifact_type, name,
                      prep_template_id, artifact_id=None):
    """Creates the initial artifact for the prep template

    Parameters
    ----------
    user_id : str
        User adding the atrifact
    filepaths : dict of str
        Comma-separated list of files to attach to the artifact,
        keyed by file type
    artifact_type : str
        The type of the artifact
    name : str
        Name to give the artifact
    prep_template_id : int or str castable to int
        Prep template to attach the artifact to
    artifact_id : int or str castable to int, optional
        The id of the imported artifact

    Returns
    -------
    dict of objects
        A dictionary containing the new artifact ID
        {'status': status,
         'message': message,
         'artifact': id}
    """
    prep_template_id = int(prep_template_id)
    prep = PrepTemplate(prep_template_id)
    study_id = prep.study_id

    # First check if the user has access to the study
    access_error = check_access(study_id, user_id)
    if access_error:
        return access_error

    if artifact_id:
        # if the artifact id has been provided, import the artifact
        job_id = safe_submit(user_id, copy_raw_data, prep, artifact_id)
        is_qiita_job = False
    else:
        uploads_path = get_mountpoint('uploads')[0][1]
        path_builder = partial(join, uploads_path, str(study_id))
        cleaned_filepaths = {}
        for ftype, file_list in viewitems(filepaths):
            # JavaScript sends us this list as a comma-separated list
            for fp in file_list.split(','):
                # JavaScript will send this value as an empty string if the
                # list of files was empty. In such case, the split will
                # generate a single element containing the empty string. Check
                # for that case here and, if fp is not the empty string,
                # proceed to check if the file exists
                if fp:
                    # Check if filepath being passed exists for study
                    full_fp = path_builder(fp)
                    exists = check_fp(study_id, full_fp)
                    if exists['status'] != 'success':
                        return {'status': 'error',
                                'message': 'File does not exist: %s' % fp}
                    if ftype not in cleaned_filepaths:
                        cleaned_filepaths[ftype] = []
                    cleaned_filepaths[ftype].append(full_fp)

        # This should never happen, but it doesn't hurt to actually have
        # a explicit check, in case there is something odd with the JS
        if not cleaned_filepaths:
            return {'status': 'error',
                    'message': "Can't create artifact, no files provided."}

        command = Command.get_validator(artifact_type)
        job = ProcessingJob.create(
            User(user_id),
            Parameters.load(command, values_dict={
                'template': prep_template_id,
                'files': dumps(cleaned_filepaths),
                'artifact_type': artifact_type
                }))
        job.submit()
        job_id = job.id
        is_qiita_job = True

    r_client.set(PREP_TEMPLATE_KEY_FORMAT % prep.id,
                 dumps({'job_id': job_id, 'is_qiita_job': is_qiita_job}))

    return {'status': 'success', 'message': ''}
Exemple #49
0
    def test_job_ajax_patch_req(self):
        # Create a new job - through a workflow since that is the only way
        # of creating jobs in the interface
        exp_command = Command(1)
        json_str = (
            '{"input_data": 1, "max_barcode_errors": 1.5, '
            '"barcode_type": "golay_12", "max_bad_run_length": 3, '
            '"rev_comp": false, "phred_quality_threshold": 3, '
            '"rev_comp_barcode": false, "rev_comp_mapping_barcodes": false, '
            '"min_per_read_length_fraction": 0.75, "sequence_max_n": 0}')
        exp_params = Parameters.load(exp_command, json_str=json_str)
        exp_user = User('*****@*****.**')
        name = "Test processing workflow"

        # tests success
        wf = ProcessingWorkflow.from_scratch(
            exp_user, exp_params, name=name, force=True)

        graph = wf.graph
        nodes = list(graph.nodes())
        job_id = nodes[0].id

        # Incorrect path parameter
        obs = job_ajax_patch_req('remove', '/%s/somethingelse' % job_id)
        exp = {'status': 'error',
               'message': 'Incorrect path parameter: missing job id'}
        self.assertEqual(obs, exp)

        obs = job_ajax_patch_req('remove', '/')
        exp = {'status': 'error',
               'message': 'Incorrect path parameter: missing job id'}
        self.assertEqual(obs, exp)

        # Job id is not like a job id
        obs = job_ajax_patch_req('remove', '/notAJobId')
        exp = {'status': 'error',
               'message': 'Incorrect path parameter: '
                          'notAJobId is not a recognized job id'}
        self.assertEqual(obs, exp)

        # Job doesn't exist
        obs = job_ajax_patch_req('remove',
                                 '/6d368e16-2242-4cf8-87b4-a5dc40bc890b')
        exp = {'status': 'error',
               'message': 'Incorrect path parameter: '
                          '6d368e16-2242-4cf8-87b4-a5dc40bc890b is not a '
                          'recognized job id'}
        self.assertEqual(obs, exp)

        # in_construction job
        obs = job_ajax_patch_req('remove', '/%s' % job_id)
        exp = {'status': 'error',
               'message': "Can't delete job %s. It is 'in_construction' "
                          "status. Please use /study/process/workflow/"
                          % job_id}
        self.assertEqual(obs, exp)

        # job status != 'error'
        job = ProcessingJob(job_id)
        job._set_status('queued')
        obs = job_ajax_patch_req('remove', '/%s' % job_id)
        exp = {'status': 'error',
               'message': 'Only jobs in "error" status can be deleted.'}
        self.assertEqual(obs, exp)

        # Operation not supported
        job._set_status('queued')
        obs = job_ajax_patch_req('add', '/%s' % job_id)
        exp = {'status': 'error',
               'message': 'Operation "add" not supported. Current supported '
                          'operations: remove'}
        self.assertEqual(obs, exp)

        # Test success
        job._set_error('Killed for testing')
        obs = job_ajax_patch_req('remove', '/%s' % job_id)
        exp = {'status': 'success',
               'message': ''}
        self.assertEqual(obs, exp)
Exemple #50
0
    def test_artifact_summary_get_request(self):
        # Artifact w/o summary
        obs = artifact_summary_get_request('*****@*****.**', 1)
        exp_p_jobs = [
            ['063e553b-327c-4818-ab4a-adfe58e49860', 'Split libraries FASTQ',
             'queued', None, None],
            ['bcc7ebcd-39c1-43e4-af2d-822e3589f14d', 'Split libraries',
             'running', 'demultiplexing', None]]
        exp_files = [
            (1L, '1_s_G1_L001_sequences.fastq.gz (raw forward seqs)'),
            (2L, '1_s_G1_L001_sequences_barcodes.fastq.gz (raw barcodes)')]
        exp = {'status': 'success',
               'message': '',
               'name': 'Raw data 1',
               'processing_parameters': {},
               'summary': None,
               'job': None,
               'processing_jobs': exp_p_jobs,
               'errored_jobs': [],
               'visibility': 'private',
               'buttons': ('<button onclick="if (confirm(\'Are you sure you '
                           'want to make public artifact id: 1?\')) { '
                           'set_artifact_visibility(\'public\', 1) }" '
                           'class="btn btn-primary btn-sm">Make public'
                           '</button> <button onclick="if (confirm(\'Are you '
                           'sure you want to revert to sandbox artifact id: '
                           '1?\')) { set_artifact_visibility(\'sandbox\', 1) '
                           '}" class="btn btn-primary btn-sm">Revert to '
                           'sandbox</button>'),
               'files': exp_files,
               'editable': True,
               'prep_id': 1,
               'study_id': 1}
        self.assertEqual(obs, exp)

        # Artifact with summary being generated
        job = ProcessingJob.create(
            User('*****@*****.**'),
            Parameters.load(Command(7), values_dict={'input_data': 1})
        )
        job._set_status('queued')
        obs = artifact_summary_get_request('*****@*****.**', 1)
        exp = {'status': 'success',
               'message': '',
               'name': 'Raw data 1',
               'processing_parameters': {},
               'summary': None,
               'job': [job.id, 'queued', None],
               'processing_jobs': exp_p_jobs,
               'errored_jobs': [],
               'visibility': 'private',
               'buttons': ('<button onclick="if (confirm(\'Are you sure you '
                           'want to make public artifact id: 1?\')) { '
                           'set_artifact_visibility(\'public\', 1) }" '
                           'class="btn btn-primary btn-sm">Make public'
                           '</button> <button onclick="if (confirm(\'Are you '
                           'sure you want to revert to sandbox artifact id: '
                           '1?\')) { set_artifact_visibility(\'sandbox\', 1) '
                           '}" class="btn btn-primary btn-sm">Revert to '
                           'sandbox</button>'),
               'files': exp_files,
               'editable': True,
               'prep_id': 1,
               'study_id': 1}
        self.assertEqual(obs, exp)

        # Artifact with summary
        fd, fp = mkstemp(suffix=".html")
        close(fd)
        with open(fp, 'w') as f:
            f.write('<b>HTML TEST - not important</b>\n')
        a = Artifact(1)
        a.html_summary_fp = fp
        self._files_to_remove.extend([fp, a.html_summary_fp[1]])
        exp_files.append(
            (a.html_summary_fp[0],
             '%s (html summary)' % basename(a.html_summary_fp[1])))
        obs = artifact_summary_get_request('*****@*****.**', 1)
        exp = {'status': 'success',
               'message': '',
               'name': 'Raw data 1',
               'processing_parameters': {},
               'summary': '<b>HTML TEST - not important</b>\n',
               'job': None,
               'processing_jobs': exp_p_jobs,
               'errored_jobs': [],
               'visibility': 'private',
               'buttons': ('<button onclick="if (confirm(\'Are you sure you '
                           'want to make public artifact id: 1?\')) { '
                           'set_artifact_visibility(\'public\', 1) }" '
                           'class="btn btn-primary btn-sm">Make public'
                           '</button> <button onclick="if (confirm(\'Are you '
                           'sure you want to revert to sandbox artifact id: '
                           '1?\')) { set_artifact_visibility(\'sandbox\', 1) '
                           '}" class="btn btn-primary btn-sm">Revert to '
                           'sandbox</button>'),
               'files': exp_files,
               'editable': True,
               'prep_id': 1,
               'study_id': 1}
        self.assertEqual(obs, exp)

        # No access
        obs = artifact_summary_get_request('*****@*****.**', 1)
        exp = {'status': 'error',
               'message': 'User does not have access to study'}
        self.assertEqual(obs, exp)

        # A non-owner/share user can't see the files
        a.visibility = 'public'
        obs = artifact_summary_get_request('*****@*****.**', 1)
        exp = {'status': 'success',
               'message': '',
               'name': 'Raw data 1',
               'processing_parameters': {},
               'summary': '<b>HTML TEST - not important</b>\n',
               'job': None,
               'processing_jobs': exp_p_jobs,
               'errored_jobs': [],
               'visibility': 'public',
               'buttons': '',
               'files': [],
               'editable': False,
               'prep_id': 1,
               'study_id': 1}
        self.assertEqual(obs, exp)

        # returnig to private
        a.visibility = 'sandbox'

        # admin gets buttons
        obs = artifact_summary_get_request('*****@*****.**', 2)
        exp_p_jobs = [
            ['d19f76ee-274e-4c1b-b3a2-a12d73507c55',
             'Pick closed-reference OTUs', 'error', 'generating demux file',
             'Error message']]
        exp_files = [
            (3L, '1_seqs.fna (preprocessed fasta)'),
            (4L, '1_seqs.qual (preprocessed fastq)'),
            (5L, '1_seqs.demux (preprocessed demux)')]
        exp = {'status': 'success',
               'files': exp_files,
               'errored_jobs': [],
               'editable': True,
               'visibility': 'private',
               'job': None,
               'message': '',
               'name': 'Demultiplexed 1',
               'processing_jobs': exp_p_jobs,
               'processing_parameters': {
                   'max_barcode_errors': 1.5, 'sequence_max_n': 0,
                   'max_bad_run_length': 3, 'phred_offset': u'auto',
                   'rev_comp': False, 'phred_quality_threshold': 3,
                   'input_data': 1, 'rev_comp_barcode': False,
                   'rev_comp_mapping_barcodes': False,
                   'min_per_read_length_fraction': 0.75,
                   'barcode_type': u'golay_12'},
               'summary': None,
               'buttons': ('<button onclick="if (confirm(\'Are you sure you '
                           'want to make public artifact id: 2?\')) { '
                           'set_artifact_visibility(\'public\', 2) }" '
                           'class="btn btn-primary btn-sm">Make public'
                           '</button> <button onclick="if (confirm(\'Are you '
                           'sure you want to revert to sandbox artifact id: '
                           '2?\')) { set_artifact_visibility(\'sandbox\', 2) '
                           '}" class="btn btn-primary btn-sm">Revert to '
                           'sandbox</button> <a class="btn btn-primary '
                           'btn-sm" href="/vamps/2"><span class="glyphicon '
                           'glyphicon-export"></span> Submit to VAMPS</a>'),
               'study_id': 1,
               'prep_id': 1}
        self.assertEqual(obs, exp)
Exemple #51
0
def job_ajax_patch_req(req_op, req_path, req_value=None, req_from=None):
    """Patches a job

    Parameters
    ----------
    req_op : str
        The operation to perform on the job
    req_path : str
        Path parameter with the job to patch
    req_value : str, optional
        The value that needs to be modified
    req_from : str, optional
        The original path of the element

    Returns
    -------
    dict of {str: str}
        A dictionary of the form: {'status': str, 'message': str} in which
        status is the status of the request ('error' or 'success') and message
        is a human readable string with the error message in case that status
        is 'error'.
    """
    if req_op == 'remove':
        req_path = [v for v in req_path.split('/') if v]
        if len(req_path) != 1:
            return {
                'status': 'error',
                'message': 'Incorrect path parameter: missing job id'
            }

        # We have ensured that we only have one element on req_path
        job_id = req_path[0]
        try:
            job = ProcessingJob(job_id)
        except QiitaDBUnknownIDError:
            return {
                'status':
                'error',
                'message':
                'Incorrect path parameter: '
                '%s is not a recognized job id' % job_id
            }
        except Exception as e:
            e = str(e)
            if "invalid input syntax for uuid" in e:
                return {
                    'status':
                    'error',
                    'message':
                    'Incorrect path parameter: '
                    '%s is not a recognized job id' % job_id
                }
            else:
                return {
                    'status':
                    'error',
                    'message':
                    'An error occured while accessing the '
                    'job: %s' % e
                }

        job_status = job.status

        if job_status == 'in_construction':
            # A job that is in construction is in a workflow. Use the methods
            # defined for workflows to keep everything consistent. This message
            # should never be presented to the user, but rather to the
            # developer if it makes a mistake during changes in the interface
            return {
                'status':
                'error',
                'message':
                "Can't delete job %s. It is 'in_construction' "
                "status. Please use /study/process/workflow/" % job_id
            }
        elif job_status == 'error':
            # When the job is in error status, we just need to hide it
            job.hide()
            return {'status': 'success', 'message': ''}
        else:
            # In any other state, we currently fail. Adding the else here
            # because it can be useful to have it for fixing issue #2307
            return {
                'status': 'error',
                'message': 'Only jobs in "error" status can be deleted.'
            }
    else:
        return {
            'status':
            'error',
            'message':
            'Operation "%s" not supported. Current supported '
            'operations: remove' % req_op
        }
Exemple #52
0
def correct_redis_data(key, cmd, values_dict, user):
    """Corrects the data stored in the redis DB

    Parameters
    ----------
    key: str
        The redis key to fix
    cmd : qiita_db.software.Command
        Command to use to create the processing job
    values_dict : dict
        Dictionary used to instantiate the parameters of the command
    user : qiita_db.user. User
        The user that will own the job
    """
    info = r_client.get(key)
    if info:
        info = loads(info)
        if info['job_id'] is not None:
            if 'is_qiita_job' in info:
                if info['is_qiita_job']:
                    try:
                        job = ProcessingJob(info['job_id'])
                        payload = {'job_id': info['job_id'],
                                   'alert_type': info['status'],
                                   'alert_msg': info['alert_msg']}
                        r_client.set(key, dumps(payload))
                    except (QiitaDBUnknownIDError, KeyError):
                        # We shomehow lost the information of this job
                        # Simply delete the key
                        r_client.delete(key)
                else:
                    # These jobs don't contain any information on the live
                    # dump. We can safely delete the key
                    r_client.delete(key)
            else:
                # These jobs don't contain any information on the live
                # dump. We can safely delete the key
                r_client.delete(key)
        else:
            # Job is null, we have the information here
            if info['status'] == 'success':
                # In the success case no information is stored. We can
                # safely delete the key
                r_client.delete(key)
            elif info['status'] == 'warning':
                # In case of warning the key message stores the warning
                # message. We need to create a new job, mark it as
                # successful and store the error message as expected by
                # the new structure
                params = Parameters.load(cmd, values_dict=values_dict)
                job = ProcessingJob.create(user, params)
                job._set_status('success')
                payload = {'job_id': job.id,
                           'alert_type': 'warning',
                           'alert_msg': info['message']}
                r_client.set(key, dumps(payload))
            else:
                # The status is error. The key message stores the error
                # message. We need to create a new job and mark it as
                # failed with the given error message
                params = Parameters.load(cmd, values_dict=values_dict)
                job = ProcessingJob.create(user, params)
                job._set_error(info['message'])
                payload = {'job_id': job.id}
                r_client.set(key, dumps(payload))
    else:
        # The key doesn't contain any information. Delete the key
        r_client.delete(key)