def setUp(self):
self.dbconfig='data/dbconfig.json'
self.new_project_data='data/check_project_data/new_project_data.csv'
dbparam = None
with open(self.dbconfig, 'r') as json_data:
dbparam = json.load(json_data)
base = BaseAdaptor(**dbparam)
self.engine = base.engine
self.dbname=dbparam['dbname']
Base.metadata.create_all(self.engine)
self.session_class=base.session_class
base.start_session()
ua=UserAdaptor(**{'session':base.session})
user_data=[{'name':'user1','email_id':'*****@*****.**','username':'******'},
{'name':'igf','email_id':'*****@*****.**','username':'******'}]
ua.store_user_data(data=user_data)
project_data=[{'project_igf_id':'IGFP0001_test_22-8-2017_rna',
'project_name':'test_22-8-2017_rna',
'description':'Its project 1',
'project_deadline':'Before August 2017',
'comments':'Some samples are treated with drug X',
}]
pa=ProjectAdaptor(**{'session':base.session})
pa.store_project_and_attribute_data(data=project_data)
project_user_data=[{'project_igf_id':'IGFP0001_test_22-8-2017_rna',
'email_id':'*****@*****.**',
'data_authority':True}]
pa.assign_user_to_project(data=project_user_data)
sample_data=[{'sample_igf_id':'IGF00001',
'project_igf_id':'IGFP0001_test_22-8-2017_rna',},
{'sample_igf_id':'IGF00002',
'project_igf_id':'IGFP0001_test_22-8-2017_rna',},
{'sample_igf_id':'IGF00003',
'project_igf_id':'IGFP0001_test_22-8-2017_rna',},
{'sample_igf_id':'IGF00004',
'project_igf_id':'IGFP0001_test_22-8-2017_rna',},
{'sample_igf_id':'IGF00005',
'project_igf_id':'IGFP0001_test_22-8-2017_rna',},
]
sa=SampleAdaptor(**{'session':base.session})
sa.store_sample_and_attribute_data(data=sample_data)
base.close_session()
new_project_data=[{'project_igf_id':'IGFP0002_test_23-5-2017_rna',
'name':'user2',
'email_id':'*****@*****.**',
'sample_igf_id':'IGF00006',
},
{'project_igf_id':'IGFP0003_test_24-8-2017_rna',
'name':'user2',
'email_id':'*****@*****.**',
'sample_igf_id':'IGF00007',
'barcode_check':'OFF'
}]
pd.DataFrame(new_project_data).to_csv(os.path.join('.',self.new_project_data))
def setUp(self):
self.dbconfig = 'data/dbconfig.json'
dbparam = read_dbconf_json(self.dbconfig)
base = BaseAdaptor(**dbparam)
self.engine = base.engine
self.dbname = dbparam['dbname']
Base.metadata.create_all(self.engine)
self.session_class = base.get_session_class()
project_data = [{
'project_igf_id': 'IGFP0001_test_22-8-2017_rna',
'project_name': 'test_22-8-2017_rna',
'description': 'Its project 1',
'project_deadline': 'Before August 2017',
'comments': 'Some samples are treated with drug X',
}, {
'project_igf_id': 'IGFP0002_test_22-8-2017_rna',
'project_name': 'test_23-8-2017_rna',
'description': 'Its project 2',
'project_deadline': 'Before August 2017',
'comments': 'Some samples are treated with drug X'
}]
user_data = [{
'name': 'UserA',
'email_id': '*****@*****.**',
'username': '******'
}]
project_user_data = [{
'project_igf_id': 'IGFP0001_test_22-8-2017_rna',
'email_id': '*****@*****.**',
'data_authority': True
}, {
'project_igf_id': 'IGFP0002_test_22-8-2017_rna',
'email_id': '*****@*****.**'
}]
base.start_session()
ua = UserAdaptor(**{'session': base.session})
ua.store_user_data(data=user_data)
pa = ProjectAdaptor(**{'session': base.session})
pa.store_project_and_attribute_data(data=project_data)
pa.assign_user_to_project(data=project_user_data)
base.close_session()
def _check_and_register_data(self, data, project_info_file):
'''
An internal method for checking and registering data
:param data: A dictionary containing following keys
project_data
user_data
project_user_data
sample_data
:param project_info_file: A filepath for project info
'''
try:
db_connected = False
project_data = pd.DataFrame(data['project_data'])
user_data = pd.DataFrame(data['user_data'])
project_user_data = pd.DataFrame(data['project_user_data'])
sample_data = pd.DataFrame(data['sample_data'])
base = BaseAdaptor(**{'session_class': self.session_class})
base.start_session() # connect_to db
db_connected = True
project_data = project_data[project_data[
self.project_lookup_column].isnull() == False]
project_data = project_data.drop_duplicates()
if project_data.index.size > 0:
project_data=project_data.\
apply(lambda x: \
self._check_existing_data(\
data=x,\
dbsession=base.session, \
table_name='project',
check_column='EXISTS'),\
axis=1) # get project map
project_data = project_data[project_data['EXISTS'] ==
False] # filter existing projects
project_data.drop('EXISTS', axis=1,
inplace=True) # remove extra column
user_data = user_data[user_data[self.user_lookup_column].isnull()
== False]
user_data = user_data.drop_duplicates()
if user_data.index.size > 0:
user_data=user_data.apply(lambda x: \
self._assign_username_and_password(x), \
axis=1) # check for use account and password
user_data=user_data.\
apply(lambda x: \
self._check_existing_data(\
data=x,\
dbsession=base.session, \
table_name='user',
check_column='EXISTS'),\
axis=1) # get user map
user_data = user_data[user_data['EXISTS'] ==
False] # filter existing users
user_data.drop('EXISTS', axis=1,
inplace=True) # remove extra column
sample_data = sample_data[sample_data[
self.sample_lookup_column].isnull() == False]
sample_data = sample_data.drop_duplicates()
if sample_data.index.size > 0:
sample_data=sample_data.\
apply(lambda x: \
self._check_existing_data(\
data=x,\
dbsession=base.session, \
table_name='sample',
check_column='EXISTS'),\
axis=1) # get sample map
sample_data = sample_data[sample_data['EXISTS'] ==
False] # filter existing samples
sample_data.drop('EXISTS', axis=1,
inplace=True) # remove extra column
project_user_data = project_user_data.drop_duplicates()
project_user_data_mask=(project_user_data[self.project_lookup_column].isnull()==False) & \
(project_user_data[self.user_lookup_column].isnull()==False)
project_user_data = project_user_data[
project_user_data_mask] # not allowing any empty values for project or user lookup
if project_user_data.index.size > 0:
project_user_data = self._add_default_user_to_project(
project_user_data
) # update project_user_data with default users
project_user_data=project_user_data.\
apply(lambda x: \
self._check_existing_data(\
data=x,\
dbsession=base.session, \
table_name='project_user',
check_column='EXISTS'),\
axis=1) # get project user map
project_user_data = project_user_data[project_user_data[
'EXISTS'] == False] # filter existing project user
project_user_data.drop('EXISTS', axis=1,
inplace=True) # remove extra column
if len(project_data.index) > 0: # store new projects
pa1 = ProjectAdaptor(**{'session': base.session
}) # connect to project adaptor
pa1.store_project_and_attribute_data(
data=project_data, autosave=False) # load project data
if len(user_data.index) > 0: # store new users
ua = UserAdaptor(**{'session': base.session})
ua.store_user_data(data=user_data,
autosave=False) # load user data
if len(project_user_data.index) > 0: # store new project users
pa2 = ProjectAdaptor(**{'session': base.session
}) # connect to project adaptor
project_user_data = project_user_data.to_dict(
orient='records') # convert dataframe to dictionary
pa2.assign_user_to_project(
data=project_user_data,
autosave=False) # load project user data
if len(sample_data.index) > 0: # store new samples
sa = SampleAdaptor(**{'session': base.session
}) # connect to sample adaptor
sa.store_sample_and_attribute_data(
data=sample_data, autosave=False) # load samples data
if self.setup_irods:
user_data.apply(lambda x: self._setup_irods_account(data=x),
axis=1) # create irods account
file_checksum = calculate_file_checksum(filepath=project_info_file)
file_size = os.path.getsize(project_info_file)
file_data=[{'file_path':project_info_file,\
'location':'ORWELL',\
'md5':file_checksum,\
'size':file_size,\
}]
fa = FileAdaptor(**{'session':
base.session}) # connect to file adaptor
fa.store_file_data(data=file_data, autosave=False)
except:
if db_connected:
base.rollback_session() # rollback session
raise
else:
if db_connected:
base.commit_session() # commit changes to db
if len(user_data.index) > 0 and self.notify_user:
user_data.apply(lambda x: self._notify_about_new_user_account(x),\
axis=1) # send mail to new user with their password and forget it
finally:
if db_connected:
base.close_session() # close db connection
def setUp(self):
self.path = 'data/seqrun_dir'
self.dbconfig = 'data/dbconfig.json'
self.md5_out_path = 'data/md5_dir'
self.pipeline_name = 'demultiplexing_fastq'
seqrun_json = 'data/seqrun_db_data.json'
platform_json = 'data/platform_db_data.json'
pipeline_json = 'data/pipeline_data.json'
os.mkdir(self.md5_out_path)
dbparam = None
with open(self.dbconfig, 'r') as json_data:
dbparam = json.load(json_data)
base = BaseAdaptor(**dbparam)
self.engine = base.engine
self.dbname = dbparam['dbname']
self.pipeline_name = ''
Base.metadata.create_all(self.engine)
base.start_session()
user_data = [
{
'name': 'user1',
'email_id': '*****@*****.**',
'username': '******'
},
]
ua = UserAdaptor(**{'session': base.session})
ua.store_user_data(data=user_data)
project_data = [{
'project_igf_id': 'project_1',
'project_name': 'test_22-8-2017_rna',
'description': 'Its project 1',
'project_deadline': 'Before August 2017',
'comments': 'Some samples are treated with drug X',
}]
pa = ProjectAdaptor(**{'session': base.session})
pa.store_project_and_attribute_data(data=project_data)
project_user_data = [{
'project_igf_id': 'project_1',
'email_id': '*****@*****.**',
'data_authority': True
}]
pa.assign_user_to_project(data=project_user_data)
sample_data = [
{
'sample_igf_id': 'IGF0001',
'project_igf_id': 'project_1',
},
{
'sample_igf_id': 'IGF0002',
'project_igf_id': 'project_1',
},
{
'sample_igf_id': 'IGF0003',
'project_igf_id': 'project_1',
},
]
sa = SampleAdaptor(**{'session': base.session})
sa.store_sample_and_attribute_data(data=sample_data)
base.commit_session()
with open(pipeline_json,
'r') as json_data: # store pipeline data to db
pipeline_data = json.load(json_data)
pa = PipelineAdaptor(**{'session': base.session})
pa.store_pipeline_data(data=pipeline_data)
with open(platform_json,
'r') as json_data: # store platform data to db
platform_data = json.load(json_data)
pl = PlatformAdaptor(**{'session': base.session})
pl.store_platform_data(data=platform_data)
with open(seqrun_json, 'r') as json_data: # store seqrun data to db
seqrun_data = json.load(json_data)
sra = SeqrunAdaptor(**{'session': base.session})
sra.store_seqrun_and_attribute_data(data=seqrun_data)
base.close_session()