def setUp(self):
    self.dbconfig = 'data/dbconfig.json'
    dbparam=read_dbconf_json(self.dbconfig)
    base = BaseAdaptor(**dbparam)
    self.engine = base.engine
    self.dbname=dbparam['dbname']
    Base.metadata.drop_all(self.engine)
    if os.path.exists(self.dbname):
      os.remove(self.dbname)
    Base.metadata.create_all(self.engine)
    self.session_class=base.get_session_class()

    base = BaseAdaptor(**{'session_class':self.session_class})
    base.start_session()
    platform_data=[{ "platform_igf_id" : "M001",
                     "model_name" : "MISEQ" ,
                     "vendor_name" : "ILLUMINA" ,
                     "software_name" : "RTA",
                     "software_version" : "RTA1.18.54"}]                        # platform data
    flowcell_rule_data=[{"platform_igf_id":"M001",
                         "flowcell_type":"MISEQ",
                         "index_1":"NO_CHANGE",
                         "index_2":"NO_CHANGE"}]                                # flowcell rule data
    pl=PlatformAdaptor(**{'session':base.session})
    pl.store_platform_data(data=platform_data)                                  # loading platform data
    pl.store_flowcell_barcode_rule(data=flowcell_rule_data)                     # loading flowcell rules data
    project_data=[{'project_igf_id':'ProjectA'}]                                # project data
    pa=ProjectAdaptor(**{'session':base.session})
    pa.store_project_and_attribute_data(data=project_data)                      # load project data
    sample_data=[{'sample_igf_id':'SampleA',
                  'project_igf_id':'ProjectA'}]                                 # sample data
    sa=SampleAdaptor(**{'session':base.session})
    sa.store_sample_and_attribute_data(data=sample_data)                        # store sample data
    seqrun_data=[{'seqrun_igf_id':'SeqrunA', 
                  'flowcell_id':'000000000-D0YLK', 
                  'platform_igf_id':'M001',
                  'flowcell':'MISEQ'}]                                          # seqrun data
    sra=SeqrunAdaptor(**{'session':base.session})
    sra.store_seqrun_and_attribute_data(data=seqrun_data)                       # load seqrun data
    experiment_data=[{'experiment_igf_id':'ExperimentA',
                      'sample_igf_id':'SampleA',
                      'library_name':'SampleA',
                      'platform_name':'MISEQ',
                      'project_igf_id':'ProjectA'}]                             # experiment data
    ea=ExperimentAdaptor(**{'session':base.session})
    ea.store_project_and_attribute_data(data=experiment_data)                   # load experiment data
    base.commit_session()
    base.close_session()
Esempio n. 2
0
    def _check_and_register_data(self, data, project_info_file):
        '''
    An internal method for checking and registering data

    :param data: A dictionary containing following keys
    
          project_data
          user_data
          project_user_data
          sample_data
    :param project_info_file: A filepath for project info
    '''
        try:
            db_connected = False
            project_data = pd.DataFrame(data['project_data'])
            user_data = pd.DataFrame(data['user_data'])
            project_user_data = pd.DataFrame(data['project_user_data'])
            sample_data = pd.DataFrame(data['sample_data'])
            base = BaseAdaptor(**{'session_class': self.session_class})
            base.start_session()  # connect_to db
            db_connected = True
            project_data = project_data[project_data[
                self.project_lookup_column].isnull() == False]
            project_data = project_data.drop_duplicates()
            if project_data.index.size > 0:
                project_data=project_data.\
                             apply(lambda x: \
                                   self._check_existing_data(\
                                      data=x,\
                                      dbsession=base.session, \
                                      table_name='project',
                                      check_column='EXISTS'),\
                                   axis=1)                                              # get project map
                project_data = project_data[project_data['EXISTS'] ==
                                            False]  # filter existing projects
                project_data.drop('EXISTS', axis=1,
                                  inplace=True)  # remove extra column

            user_data = user_data[user_data[self.user_lookup_column].isnull()
                                  == False]
            user_data = user_data.drop_duplicates()
            if user_data.index.size > 0:
                user_data=user_data.apply(lambda x: \
                                        self._assign_username_and_password(x), \
                                        axis=1)                                         # check for use account and password
                user_data=user_data.\
                          apply(lambda x: \
                                self._check_existing_data(\
                                      data=x,\
                                      dbsession=base.session, \
                                      table_name='user',
                                      check_column='EXISTS'),\
                                axis=1)                                                 # get user map
                user_data = user_data[user_data['EXISTS'] ==
                                      False]  # filter existing users
                user_data.drop('EXISTS', axis=1,
                               inplace=True)  # remove extra column

            sample_data = sample_data[sample_data[
                self.sample_lookup_column].isnull() == False]
            sample_data = sample_data.drop_duplicates()
            if sample_data.index.size > 0:
                sample_data=sample_data.\
                             apply(lambda x: \
                                   self._check_existing_data(\
                                      data=x,\
                                      dbsession=base.session, \
                                      table_name='sample',
                                      check_column='EXISTS'),\
                                   axis=1)                                              # get sample map
                sample_data = sample_data[sample_data['EXISTS'] ==
                                          False]  # filter existing samples
                sample_data.drop('EXISTS', axis=1,
                                 inplace=True)  # remove extra column

            project_user_data = project_user_data.drop_duplicates()
            project_user_data_mask=(project_user_data[self.project_lookup_column].isnull()==False) & \
                                   (project_user_data[self.user_lookup_column].isnull()==False)
            project_user_data = project_user_data[
                project_user_data_mask]  # not allowing any empty values for project or user lookup
            if project_user_data.index.size > 0:
                project_user_data = self._add_default_user_to_project(
                    project_user_data
                )  # update project_user_data with default users
                project_user_data=project_user_data.\
                                  apply(lambda x: \
                                   self._check_existing_data(\
                                      data=x,\
                                      dbsession=base.session, \
                                      table_name='project_user',
                                      check_column='EXISTS'),\
                                   axis=1)                                              # get project user map
                project_user_data = project_user_data[project_user_data[
                    'EXISTS'] == False]  # filter existing project user
                project_user_data.drop('EXISTS', axis=1,
                                       inplace=True)  # remove extra column

            if len(project_data.index) > 0:  # store new projects
                pa1 = ProjectAdaptor(**{'session': base.session
                                        })  # connect to project adaptor
                pa1.store_project_and_attribute_data(
                    data=project_data, autosave=False)  # load project data

            if len(user_data.index) > 0:  # store new users
                ua = UserAdaptor(**{'session': base.session})
                ua.store_user_data(data=user_data,
                                   autosave=False)  # load user data

            if len(project_user_data.index) > 0:  # store new project users
                pa2 = ProjectAdaptor(**{'session': base.session
                                        })  # connect to project adaptor
                project_user_data = project_user_data.to_dict(
                    orient='records')  # convert dataframe to dictionary
                pa2.assign_user_to_project(
                    data=project_user_data,
                    autosave=False)  # load project user data

            if len(sample_data.index) > 0:  # store new samples
                sa = SampleAdaptor(**{'session': base.session
                                      })  # connect to sample adaptor
                sa.store_sample_and_attribute_data(
                    data=sample_data, autosave=False)  # load samples data

            if self.setup_irods:
                user_data.apply(lambda x: self._setup_irods_account(data=x),
                                axis=1)  # create irods account

            file_checksum = calculate_file_checksum(filepath=project_info_file)
            file_size = os.path.getsize(project_info_file)
            file_data=[{'file_path':project_info_file,\
                        'location':'ORWELL',\
                        'md5':file_checksum,\
                        'size':file_size,\
                      }]
            fa = FileAdaptor(**{'session':
                                base.session})  # connect to file adaptor
            fa.store_file_data(data=file_data, autosave=False)

        except:
            if db_connected:
                base.rollback_session()  # rollback session
            raise
        else:
            if db_connected:
                base.commit_session()  # commit changes to db
                if len(user_data.index) > 0 and self.notify_user:
                    user_data.apply(lambda x: self._notify_about_new_user_account(x),\
                                    axis=1)                                               # send mail to new user with their password and forget it
        finally:
            if db_connected:
                base.close_session()  # close db connection
Esempio n. 3
0
    def setUp(self):
        self.path = 'data/seqrun_dir'
        self.dbconfig = 'data/dbconfig.json'
        self.md5_out_path = 'data/md5_dir'
        self.pipeline_name = 'demultiplexing_fastq'

        seqrun_json = 'data/seqrun_db_data.json'
        platform_json = 'data/platform_db_data.json'
        pipeline_json = 'data/pipeline_data.json'

        os.mkdir(self.md5_out_path)
        dbparam = None
        with open(self.dbconfig, 'r') as json_data:
            dbparam = json.load(json_data)
        base = BaseAdaptor(**dbparam)
        self.engine = base.engine
        self.dbname = dbparam['dbname']
        self.pipeline_name = ''
        Base.metadata.create_all(self.engine)
        base.start_session()
        user_data = [
            {
                'name': 'user1',
                'email_id': '*****@*****.**',
                'username': '******'
            },
        ]
        ua = UserAdaptor(**{'session': base.session})
        ua.store_user_data(data=user_data)
        project_data = [{
            'project_igf_id': 'project_1',
            'project_name': 'test_22-8-2017_rna',
            'description': 'Its project 1',
            'project_deadline': 'Before August 2017',
            'comments': 'Some samples are treated with drug X',
        }]
        pa = ProjectAdaptor(**{'session': base.session})
        pa.store_project_and_attribute_data(data=project_data)
        project_user_data = [{
            'project_igf_id': 'project_1',
            'email_id': '*****@*****.**',
            'data_authority': True
        }]
        pa.assign_user_to_project(data=project_user_data)
        sample_data = [
            {
                'sample_igf_id': 'IGF0001',
                'project_igf_id': 'project_1',
            },
            {
                'sample_igf_id': 'IGF0002',
                'project_igf_id': 'project_1',
            },
            {
                'sample_igf_id': 'IGF0003',
                'project_igf_id': 'project_1',
            },
        ]
        sa = SampleAdaptor(**{'session': base.session})
        sa.store_sample_and_attribute_data(data=sample_data)
        base.commit_session()
        with open(pipeline_json,
                  'r') as json_data:  # store pipeline data to db
            pipeline_data = json.load(json_data)
            pa = PipelineAdaptor(**{'session': base.session})
            pa.store_pipeline_data(data=pipeline_data)

        with open(platform_json,
                  'r') as json_data:  # store platform data to db
            platform_data = json.load(json_data)
            pl = PlatformAdaptor(**{'session': base.session})
            pl.store_platform_data(data=platform_data)

        with open(seqrun_json, 'r') as json_data:  # store seqrun data to db
            seqrun_data = json.load(json_data)
            sra = SeqrunAdaptor(**{'session': base.session})
            sra.store_seqrun_and_attribute_data(data=seqrun_data)
            base.close_session()
    def load_file_to_disk_and_db(self,
                                 input_file_list,
                                 withdraw_exisitng_collection=True,
                                 autosave_db=True,
                                 file_suffix=None,
                                 force=True,
                                 remove_file=False):
        '''
    A method for loading analysis results to disk and database. File will be moved to a new path if base_path is present.
    Directory structure of the final path is based on the collection_table information.
    
    Following will be the final directory structure if base_path is present
    
    project - base_path/project_igf_id/analysis_name
    sample - base_path/project_igf_id/sample_igf_id/analysis_name
    experiment - base_path/project_igf_id/sample_igf_id/experiment_igf_id/analysis_name
    run - base_path/project_igf_id/sample_igf_id/experiment_igf_id/run_igf_id/analysis_name
    
    :param input_file_list: A list of input file to load, all using the same collection info
    :param withdraw_exisitng_collection: Remove existing collection group, DO NOT use this while loading a list of files
    :param autosave_db: Save changes to database, default True
    :param file_suffix: Use a specific file suffix, use None if it should be same as original file
                        e.g. input.vcf.gz to  output.vcf.gz
    :param force: Toggle for removing existing file, default True
    :param remove_file: A toggle for removing existing file from disk, default False
    :returns: A list of final filepath
    '''
        try:
            project_igf_id = None
            sample_igf_id = None
            experiment_igf_id = None
            experiment_igf_id = None
            run_igf_id = None
            output_path_list = list()  # define empty output list
            dbconnected = False
            if self.collection_name is None or \
               self.collection_type is None or \
               self.collection_table is None:
                raise ValueError('File collection information is incomplete'
                                 )  # check for collection information

            base = BaseAdaptor(**{'session_class': self.dbsession_class})
            base.start_session()  # connect to db
            dbconnected = True
            if self.base_path is not None:
                if self.collection_table == 'sample':
                    sa = SampleAdaptor(**{'session': base.session})
                    sample_igf_id = self.collection_name
                    sample_exists = sa.check_sample_records_igf_id(
                        sample_igf_id=sample_igf_id)
                    if not sample_exists:
                        raise ValueError('Sample {0} not found in db'.\
                                         format(sample_igf_id))

                    project_igf_id = \
                      sa.fetch_sample_project(sample_igf_id=sample_igf_id)                # fetch project id for sample
                elif self.collection_table == 'experiment':
                    ea = ExperimentAdaptor(**{'session': base.session})
                    experiment_igf_id = self.collection_name
                    experiment_exists = \
                      ea.check_experiment_records_id(
                        experiment_igf_id=experiment_igf_id)
                    if not experiment_exists:
                        raise ValueError('Experiment {0} not present in database'.\
                                         format(experiment_igf_id))

                    (project_igf_id,sample_igf_id) = \
                        ea.fetch_project_and_sample_for_experiment(
                          experiment_igf_id=experiment_igf_id)                            # fetch project and sample id for experiment
                elif self.collection_table == 'run':
                    ra = RunAdaptor(**{'session': base.session})
                    run_igf_id = self.collection_name
                    run_exists = ra.check_run_records_igf_id(
                        run_igf_id=run_igf_id)
                    if not run_exists:
                        raise ValueError('Run {0} not found in database'.\
                                         format(run_igf_id))

                    (project_igf_id,sample_igf_id,experiment_igf_id) = \
                      ra.fetch_project_sample_and_experiment_for_run(
                        run_igf_id=run_igf_id)                                            # fetch project, sample and experiment id for run
                elif self.collection_table == 'project':
                    pa = ProjectAdaptor(**{'session': base.session})
                    project_igf_id = self.collection_name
                    project_exists = \
                      pa.check_project_records_igf_id(
                        project_igf_id=project_igf_id)
                    if not project_exists:
                        raise ValueError('Project {0} not found in database'.\
                                         format(project_igf_id))

            if self.rename_file and self.analysis_name is None:
                raise ValueError('Analysis name is required for renaming file'
                                 )  # check analysis name

            for input_file in input_file_list:
                final_path = ''
                if self.base_path is None:  # do not move file if base_path is absent
                    final_path = os.path.dirname(input_file)
                else:  # move file path
                    if self.collection_table == 'project':
                        if project_igf_id is None:
                            raise ValueError('Missing project id for collection {0}'.\
                                             format(self.collection_name))

                        final_path = \
                          os.path.join(
                            self.base_path,
                            project_igf_id,
                            self.analysis_name)                                             # final path for project
                    elif self.collection_table == 'sample':
                        if project_igf_id is None or \
                           sample_igf_id is None:
                            raise ValueError('Missing project and sample id for collection {0}'.\
                                             format(self.collection_name))

                        final_path = \
                          os.path.join(
                            self.base_path,
                            project_igf_id,
                            sample_igf_id,
                            self.analysis_name)                                             # final path for sample
                    elif self.collection_table == 'experiment':
                        if project_igf_id is None or \
                           sample_igf_id is None or \
                           experiment_igf_id is None:
                            raise ValueError('Missing project,sample and experiment id for collection {0}'.\
                                             format(self.collection_name))

                        final_path = \
                          os.path.join(
                            self.base_path,
                            project_igf_id,
                            sample_igf_id,
                            experiment_igf_id,
                            self.analysis_name)                                             # final path for experiment
                    elif self.collection_table == 'run':
                        if project_igf_id is None or \
                           sample_igf_id is None or \
                           experiment_igf_id is None or \
                           run_igf_id is None:
                            raise ValueError('Missing project,sample,experiment and run id for collection {0}'.\
                                             format(self.collection_name))

                        final_path = \
                          os.path.join(\
                            self.base_path,
                            project_igf_id,
                            sample_igf_id,
                            experiment_igf_id,
                            run_igf_id,
                            self.analysis_name)                                             # final path for run

                if self.rename_file:
                    new_filename = \
                      self.get_new_file_name(
                        input_file=input_file,
                        file_suffix=file_suffix)
                    final_path = \
                      os.path.join(
                        final_path,
                        new_filename)                                                     # get new filepath
                else:
                    final_path = \
                      os.path.join(
                        final_path,
                        os.path.basename(input_file))

                if final_path != input_file:  # move file if its required
                    final_path = preprocess_path_name(
                        input_path=final_path
                    )  # remove unexpected characters from file path
                    move_file(source_path=input_file,
                              destinationa_path=final_path,
                              force=force
                              )  # move or overwrite file to destination dir

                output_path_list.append(
                    final_path)  # add final path to the output list
                self.create_or_update_analysis_collection(
                    file_path=final_path,
                    dbsession=base.session,
                    withdraw_exisitng_collection=withdraw_exisitng_collection,
                    remove_file=remove_file,
                    autosave_db=autosave_db)  # load new file collection in db
                if autosave_db:
                    base.commit_session()  # save changes to db for each file

            base.commit_session()  # save changes to db
            base.close_session()  # close db connection
            return output_path_list
        except:
            if dbconnected:
                base.rollback_session()
                base.close_session()
            raise
    def setUp(self):
        self.dbconfig = 'data/dbconfig.json'
        dbparam = read_dbconf_json(self.dbconfig)
        base = BaseAdaptor(**dbparam)
        self.engine = base.engine
        self.dbname = dbparam['dbname']
        Base.metadata.drop_all(self.engine)
        if os.path.exists(self.dbname):
            os.remove(self.dbname)
        Base.metadata.create_all(self.engine)
        self.session_class = base.get_session_class()
        self.temp_work_dir = get_temp_dir()
        self.temp_base_dir = get_temp_dir()
        self.input_list = ['a.cram', 'a.vcf.gz', 'b.tar.gz']
        for file_name in self.input_list:
            file_path = os.path.join(self.temp_work_dir, file_name)
            with open(file_path, 'w') as fq:
                fq.write('AAAA')  # create input files

        base = BaseAdaptor(**{'session_class': self.session_class})
        base.start_session()
        platform_data = [{
            "platform_igf_id": "M001",
            "model_name": "MISEQ",
            "vendor_name": "ILLUMINA",
            "software_name": "RTA",
            "software_version": "RTA1.18.54"
        }]  # platform data
        flowcell_rule_data = [{
            "platform_igf_id": "M001",
            "flowcell_type": "MISEQ",
            "index_1": "NO_CHANGE",
            "index_2": "NO_CHANGE"
        }]  # flowcell rule data
        pl = PlatformAdaptor(**{'session': base.session})
        pl.store_platform_data(data=platform_data)  # loading platform data
        pl.store_flowcell_barcode_rule(
            data=flowcell_rule_data)  # loading flowcell rules data
        project_data = [{'project_igf_id': 'ProjectA'}]  # project data
        pa = ProjectAdaptor(**{'session': base.session})
        pa.store_project_and_attribute_data(
            data=project_data)  # load project data
        sample_data = [{
            'sample_igf_id': 'SampleA',
            'project_igf_id': 'ProjectA'
        }]  # sample data
        sa = SampleAdaptor(**{'session': base.session})
        sa.store_sample_and_attribute_data(
            data=sample_data)  # store sample data
        seqrun_data = [{
            'seqrun_igf_id': 'SeqrunA',
            'flowcell_id': '000000000-D0YLK',
            'platform_igf_id': 'M001',
            'flowcell': 'MISEQ'
        }]  # seqrun data
        sra = SeqrunAdaptor(**{'session': base.session})
        sra.store_seqrun_and_attribute_data(
            data=seqrun_data)  # load seqrun data
        experiment_data = [{
            'experiment_igf_id': 'ExperimentA',
            'sample_igf_id': 'SampleA',
            'library_name': 'SampleA',
            'platform_name': 'MISEQ',
            'project_igf_id': 'ProjectA'
        }]  # experiment data
        ea = ExperimentAdaptor(**{'session': base.session})
        ea.store_project_and_attribute_data(
            data=experiment_data)  # load experiment data
        run_data = [{
            'run_igf_id': 'RunA',
            'experiment_igf_id': 'ExperimentA',
            'seqrun_igf_id': 'SeqrunA',
            'lane_number': '1'
        }]  # run data
        ra = RunAdaptor(**{'session': base.session})
        ra.store_run_and_attribute_data(data=run_data)  # load run data
        base.commit_session()
        base.close_session()
 pipeline_seed_data = [{
     'pipeline_name': 'PrimaryAnalysis',
     'seed_id': 1,
     'seed_table': 'experiment'
 }, {
     'pipeline_name': 'PrimaryAnalysis',
     'seed_id': 2,
     'seed_table': 'experiment'
 }, {
     'pipeline_name': 'PrimaryAnalysis',
     'seed_id': 3,
     'seed_table': 'experiment'
 }]
 pla.store_pipeline_data(data=pipeline_data)
 pla.create_pipeline_seed(data=pipeline_seed_data)
 base.commit_session()
 base.close_session()
 ps = Project_status(igf_session_class=base.get_session_class(),
                     project_igf_id='ProjectA')
 #print(ps.get_seqrun_info(demultiplexing_pipeline='DemultiplexIlluminaFastq'))
 #print(ps.get_seqrun_info(active_seqrun_igf_id='SeqrunA'))
 #print(ps.get_seqrun_info(demultiplexing_pipeline='DemultiplexIlluminaFastq',
 #                         active_seqrun_igf_id='180410_K00345_0063_AHWL7CBBXX'))
 #print(ps.get_status_description())
 #print(ps.get_status_column_order())
 #print(ps.get_analysis_info(analysis_pipeline='PrimaryAnalysis'))
 #ps.generate_gviz_json_file(output_file='a',
 #                           demultiplexing_pipeline='DemultiplexIlluminaFastq',
 #                           analysis_pipeline='PrimaryAnalysis',
 #                           active_seqrun_igf_id='180410_K00345_0063_AHWL7CBBXX')
 Base.metadata.drop_all(engine)
    def _build_and_store_exp_run_and_collection_in_db(self,fastq_files_list, \
                                                      restricted_list=('10X')):
        '''
    An internal method for building db collections for the raw fastq files
    '''
        session_class = self.session_class
        db_connected = False
        try:
            restricted_list = list(restricted_list)
            dataframe = pd.DataFrame(fastq_files_list)
            # calculate additional detail
            dataframe=dataframe.apply(lambda data: \
                                      self._calculate_experiment_run_and_file_info(data,
                                                                     restricted_list),\
                                      axis=1)
            # get file data
            file_group_columns = [
                'name', 'type', 'location', 'R1', 'R1_md5', 'R1_size', 'R2',
                'R2_md5', 'R2_size'
            ]
            file_group_data = dataframe.loc[:, file_group_columns]
            file_group_data = file_group_data.drop_duplicates()
            (file_data, file_group_data) = self._reformat_file_group_data(
                data=file_group_data)
            # get base session
            base = BaseAdaptor(**{'session_class': session_class})
            base.start_session()
            db_connected = True
            # get experiment data
            experiment_columns=base.get_table_columns(table_name=Experiment, \
                                                      excluded_columns=['experiment_id',
                                                                        'project_id',
                                                                        'sample_id' ])
            experiment_columns.extend(['project_igf_id', 'sample_igf_id'])
            exp_data = dataframe.loc[:, experiment_columns]
            exp_data = exp_data.drop_duplicates()
            if exp_data.index.size > 0:
                exp_data=exp_data.apply(lambda x: \
                                        self._check_existing_data(\
                                              data=x,\
                                              dbsession=base.session,\
                                              table_name='experiment',\
                                              check_column='EXISTS'),\
                                        axis=1)
                exp_data = exp_data[exp_data['EXISTS'] ==
                                    False]  # filter existing experiments
                exp_data.drop('EXISTS', axis=1,
                              inplace=True)  # remove extra columns
                exp_data = exp_data[pd.isnull(exp_data['experiment_igf_id']) ==
                                    False]  # filter exp with null values
            # get run data
            run_columns=base.get_table_columns(table_name=Run, \
                                               excluded_columns=['run_id',
                                                                 'seqrun_id',
                                                                 'experiment_id',
                                                                 'date_created',
                                                                 'status'
                                                                ])
            run_columns.extend([
                'seqrun_igf_id', 'experiment_igf_id', 'R1_READ_COUNT',
                'R2_READ_COUNT'
            ])
            run_data = dataframe.loc[:, run_columns]
            run_data = run_data.drop_duplicates()
            if run_data.index.size > 0:
                run_data=run_data.apply(lambda x: \
                                        self._check_existing_data(\
                                              data=x,\
                                              dbsession=base.session,\
                                              table_name='run',\
                                              check_column='EXISTS'),\
                                        axis=1)
                run_data = run_data[run_data['EXISTS'] ==
                                    False]  # filter existing runs
                run_data.drop('EXISTS', axis=1,
                              inplace=True)  # remove extra columns
                run_data = run_data[pd.isnull(run_data['run_igf_id']) ==
                                    False]  # filter run with null values
            # get collection data
            collection_columns = ['name', 'type', 'table']
            collection_data = dataframe.loc[:, collection_columns]
            collection_data = collection_data.drop_duplicates()
            if collection_data.index.size > 0:
                collection_data=collection_data.apply(lambda x: \
                                        self._check_existing_data( \
                                              data=x, \
                                              dbsession=base.session, \
                                              table_name='collection', \
                                              check_column='EXISTS'), \
                                        axis=1)
                collection_data = collection_data[collection_data[
                    'EXISTS'] == False]  # filter existing collection
                collection_data.drop('EXISTS', axis=1,
                                     inplace=True)  # remove extra columns
                collection_data = collection_data[pd.isnull(
                    collection_data['name']
                ) == False]  # filter collection with null values
            # store experiment to db
            if exp_data.index.size > 0:
                ea = ExperimentAdaptor(**{'session': base.session})
                ea.store_project_and_attribute_data(data=exp_data,
                                                    autosave=False)
                base.session.flush()
            # store run to db
            if run_data.index.size > 0:
                ra = RunAdaptor(**{'session': base.session})
                ra.store_run_and_attribute_data(data=run_data, autosave=False)
                base.session.flush()
            # store file to db

            fa = FileAdaptor(**{'session': base.session})
            fa.store_file_and_attribute_data(data=file_data, autosave=False)
            base.session.flush()
            # store collection to db
            ca = CollectionAdaptor(**{'session': base.session})
            if collection_data.index.size > 0:
                ca.store_collection_and_attribute_data(data=collection_data,\
                                                       autosave=False)
                base.session.flush()
            ca.create_collection_group(data=file_group_data, autosave=False)
            base.commit_session()
            self._write_manifest_file(file_data)
        except:
            if db_connected:
                base.rollback_session()
            raise
        finally:
            if db_connected:
                base.close_session()