def AtRoot(AtTair10):
if cf.test.force.COB:
co.del_dataset('Expr', 'AtRoot', safe=False)
if not co.available_datasets('Expr', 'AtRoot'):
Root = ['GSE14578', 'GSE46205', 'GSE7631', 'GSE10576', 'GSE42007',
'GSE34130', 'GSE21611', 'GSE22966', 'GSE7641', 'GSE5620',
'GSE8934', 'GSE5628', 'GSE30095', 'GSE30097', 'GSE5624',
'GSE5626', 'GSE5749', 'GSE5621', 'GSE5622',
'GSE5623', 'GSE5625', 'GSE5688']
RootFam = sum(
[co.Family.from_file(
os.path.join(
cf.options.testdir,
'raw', 'GSE', '{}_family.soft.gz'.format(x)
)
)
for x in Root ]
)
#RootFam.to_keepfile("RootKeep.tsv", keep_hint='root')
return co.COB.from_DataFrame(
RootFam.series_matrix(
keepfile=os.path.join(
cf.options.testdir,
'raw', 'GSE', 'RootKeep.tsv')
),
'AtRoot', 'Arab Root',
AtTair10,
rawtype='MICROARRAY',
quantile=True
)
else:
return co.COB('AtRoot')
def AtRoot(AtTair10):
if cf.test.force.COB:
co.del_dataset('Expr', 'AtRoot', force=True)
if not co.available_datasets('Expr', 'AtRoot'):
Root = [
'GSE14578', 'GSE46205', 'GSE7631', 'GSE10576', 'GSE42007',
'GSE34130', 'GSE21611', 'GSE22966', 'GSE7641', 'GSE5620',
'GSE8934', 'GSE5628', 'GSE30095', 'GSE30097', 'GSE5624', 'GSE5626',
'GSE5749', 'GSE5621', 'GSE5622', 'GSE5623', 'GSE5625', 'GSE5688'
]
RootFam = sum([
co.Family.from_file(
os.path.join(cf.options.testdir, 'raw', 'GSE',
'{}_family.soft.gz'.format(x))) for x in Root
])
#RootFam.to_keepfile("RootKeep.tsv", keep_hint='root')
return co.COB.from_DataFrame(
RootFam.series_matrix(keepfile=os.path.join(
cf.options.testdir, 'raw', 'GSE', 'RootKeep.tsv')),
'AtRoot',
'Arab Root',
AtTair10,
rawtype='MICROARRAY',
quantile=True)
else:
return co.COB('AtRoot')
def AtRoot(self):
co.del_dataset('Expr','AtRoot',safe=False)
Root = ['GSE14578','GSE46205','GSE7631','GSE10576','GSE42007',
'GSE34130','GSE21611','GSE22966','GSE7641','GSE5620',
'GSE8934','GSE5628','GSE30095','GSE30097','GSE5624',
'GSE5626','GSE5749','GSE5621','GSE5622',
'GSE5623','GSE5625','GSE5688']
RootFam = sum(
[co.Family.from_file(
os.path.join(
cf['options']['testdir'],
'raw','GSE','{}_family.soft'.format(x)
)
)
for x in Root ]
)
#RootFam.to_keepfile("RootKeep.tsv",keep_hint='root')
AtRoot = co.COB.from_DataFrame(
RootFam.series_matrix(
keepfile=pjoin(self.rawdir,'GSE','RootKeep.tsv')
),
'AtRoot','Arab Root',
co.RefGen('T10'),
rawtype='MICROARRAY'
)
self.assertIsInstance(AtRoot,co.COB)
def test_fromDataFrame(testRefGen):
'''
Test GWAS creation from DataFrame
'''
co.del_dataset('GWAS','testGWAS',force=True)
df = pd.DataFrame({
'Trait' : ['a','a','b','b'],
'CHR' : ['chr1','chr2','chr3','chr4'],
'POS' : [100,200,300,400],
'Start' : [100,200,300,400],
'End' : [1000,20000,3000,4000],
'id' : ['snp1','snp2','snp3','snp4'],
'pval' : [0.05,0.05,0.01,0.01]
})
gwas = co.GWAS.from_DataFrame(
df,
'testGWAS',
'Test GWAS Dataset',
testRefGen,
chr_col='CHR',
pos_col='POS',
id_col='id',
term_col='Trait'
)
assert len(gwas) == 2
def AtGen(AtTair10):
if cf.test.force.COB:
co.del_dataset('Expr', 'AtGen', safe=False)
if not co.available_datasets('Expr', 'AtGen'):
General = ['GSE18975', 'GSE39384', 'GSE19271', 'GSE5632', 'GSE39385',
'GSE5630', 'GSE15617', 'GSE5617', 'GSE5686', 'GSE2473',
'GSE5633', 'GSE5620', 'GSE5628', 'GSE5624',
'GSE5626', 'GSE5621', 'GSE5622', 'GSE5623', 'GSE5625', 'GSE5688']
GenFam = sum(
[co.Family.from_file(
os.path.join(
cf.options.testdir,
'raw', 'GSE', '{}_family.soft.gz'.format(x)
)
)
for x in General ]
)
#GenFam.to_keepfile("GenKeep.tsv")
return co.COB.from_DataFrame(
GenFam.series_matrix(
keepfile=os.path.join(
cf.options.testdir,
'raw', 'GSE', 'GenKeep.tsv'
)
),
'AtGen', 'Arab General',
AtTair10,
rawtype='MICROARRAY',
quantile=True
)
else:
return co.COB('AtGen')
def AtGen(AtTair10):
if cf.test.force.COB:
co.del_dataset('Expr', 'AtGen', force=True)
if not co.available_datasets('Expr', 'AtGen'):
General = [
'GSE18975', 'GSE39384', 'GSE19271', 'GSE5632', 'GSE39385',
'GSE5630', 'GSE15617', 'GSE5617', 'GSE5686', 'GSE2473', 'GSE5633',
'GSE5620', 'GSE5628', 'GSE5624', 'GSE5626', 'GSE5621', 'GSE5622',
'GSE5623', 'GSE5625', 'GSE5688'
]
GenFam = sum([
co.Family.from_file(
os.path.join(cf.options.testdir, 'raw', 'GSE',
'{}_family.soft.gz'.format(x))) for x in General
])
#GenFam.to_keepfile("GenKeep.tsv")
return co.COB.from_DataFrame(
GenFam.series_matrix(keepfile=os.path.join(
cf.options.testdir, 'raw', 'GSE', 'GenKeep.tsv')),
'AtGen',
'Arab General',
AtTair10,
rawtype='MICROARRAY',
quantile=True)
else:
return co.COB('AtGen')
def ZmIonome(Zm5bFGS):
# Delete the old dataset
if cf.test.force.Ontology:
co.del_dataset('GWAS', 'ZmIonome', force=True)
if not co.available_datasets('GWAS', 'ZmIonome'):
# Grab path the csv
csv = os.path.join(
cf.options.testdir, 'raw', 'GWAS', 'Ionome',
'sigGWASsnpsCombinedIterations.longhorn.allLoc.csv.gz')
# Define our reference geneome
df = pd.DataFrame.from_csv(csv, index_col=None)
# Import class from dataframe
IONS = co.GWAS.from_DataFrame(df,
'ZmIonome',
'Maize Ionome',
Zm5bFGS,
term_col='el',
chr_col='chr',
pos_col='pos')
# Get rid of pesky Cobalt
IONS.del_term('Co59')
# I guess we need a test in here too
return IONS
else:
return co.GWAS('ZmIonome')
def AtSeed(AtTair10):
if cf.test.force.COB:
co.del_dataset('Expr', 'AtSeed', safe=False)
if not co.available_datasets('Expr', 'AtSeed'):
Seed = ['GSE12404', #'GSE30223',
'GSE1051', 'GSE11852', 'GSE5634']
SeedFam = sum(
[co.Family.from_file(
os.path.join(
cf.options.testdir,
'raw', 'GSE', '{}_family.soft.gz'.format(x)
)
)
for x in Seed ]
)
#SeedFam.to_keepfile("SeedKeep.tsv", keep_hint='seed')
return co.COB.from_DataFrame(
SeedFam.series_matrix(
keepfile=os.path.join(
cf.options.testdir,
'raw', 'GSE', 'SeedKeep.tsv'
)
),
'AtSeed', 'Arabidopsis Seed',
AtTair10,
rawtype='MICROARRAY',
quantile=True
)
else:
return co.COB('AtSeed')
def AtSeed(AtTair10):
if cf.test.force.COB:
co.del_dataset('Expr', 'AtSeed', force=True)
if not co.available_datasets('Expr', 'AtSeed'):
Seed = [
'GSE12404', #'GSE30223',
'GSE1051',
'GSE11852',
'GSE5634'
]
SeedFam = sum([
co.Family.from_file(
os.path.join(cf.options.testdir, 'raw', 'GSE',
'{}_family.soft.gz'.format(x))) for x in Seed
])
#SeedFam.to_keepfile("SeedKeep.tsv", keep_hint='seed')
return co.COB.from_DataFrame(
SeedFam.series_matrix(keepfile=os.path.join(
cf.options.testdir, 'raw', 'GSE', 'SeedKeep.tsv')),
'AtSeed',
'Arabidopsis Seed',
AtTair10,
rawtype='MICROARRAY',
quantile=True)
else:
return co.COB('AtSeed')
def AtLeaf(AtTair10):
if cf.test.force.COB:
co.del_dataset('Expr', 'AtLeaf', safe=False)
if not co.available_datasets('Expr', 'AtLeaf'):
Leaf = ['GSE14578', 'GSE5630', 'GSE13739', #'GSE26199',
'GSE5686', 'GSE5615', 'GSE5620', 'GSE5628',
'GSE5624', 'GSE5626', 'GSE5621', 'GSE5622',
'GSE5623', 'GSE5625', 'GSE5688']
LeafFam = sum(
[co.Family.from_file(
os.path.join(
cf.options.testdir,
'raw', 'GSE', '{}_family.soft.gz'.format(x)
)
)
for x in Leaf ]
)
#LeafFam.to_keepfile("LeafKeep.tsv", keep_hint="lea")
return co.COB.from_DataFrame(
LeafFam.series_matrix(
keepfile=os.path.join(
cf.options.testdir,
'raw', 'GSE', 'LeafKeep.tsv'
)
),
'AtLeaf', 'Arabidopsis Leaf',
AtTair10,
rawtype='MICROARRAY',
max_gene_missing_data=0.3,
min_expr=0.01,
quantile=True,
)
else:
return co.COB('AtLeaf')
def AtLeaf(self):
co.del_dataset('Expr','AtLeaf',safe=False)
Leaf = ['GSE14578','GSE5630','GSE13739', #'GSE26199',
'GSE5686','GSE5615','GSE5620','GSE5628',
'GSE5624','GSE5626','GSE5621','GSE5622',
'GSE5623','GSE5625','GSE5688']
LeafFam = sum(
[co.Family.from_file(
pjoin(
self.rawdir,'GSE','{}_family.soft'.format(x)
)
)
for x in Leaf ]
)
# Generate the LeafKeep file
#LeafFam.to_keepfile("LeafKeep.tsv",keep_hint="lea")
AtLeaf = co.COB.from_DataFrame(
LeafFam.series_matrix(
keepfile=pjoin(self.rawdir,'GSE','LeafKeep.tsv')
),
'AtLeaf','Arabidopsis Leaf',
co.RefGen('T10'),rawtype='MICROARRAY',
max_gene_missing_data=0.3,
min_expr=0.01,
quantile=False,
)
self.assertIsInstance(AtLeaf,co.COB)
def ZmRNASeqTissueAtlas(Zm5bFGS):
if cf.test.force.COB:
co.del_dataset('COB', 'ZmRNASeqTissueAtlas', safe=False)
co.del_dataset('Expr', 'ZmRNASeqTissueAtlas', safe=False)
if not co.available_datasets('Expr', 'ZmRNASeqTissueAtlas'):
# Build it
return co.COB.from_table(
os.path.join(cf.options.testdir,
'raw', 'Expr', 'RNASEQ',
'MaizeRNASeqTissue.tsv.bz2',
),
'ZmRNASeqTissueAtlas',
'Maize RNASeq Tissue Atlas Network, Sekhon 2013, PLoS ONE',
Zm5bFGS,
rawtype='RNASEQ',
max_gene_missing_data=0.3,
max_accession_missing_data=0.08,
min_single_sample_expr=1,
min_expr=0.001,
quantile=False,
max_val=300,
dry_run=True
)
else:
return co.COB('ZmRNASeqTissueAtlas')
def ZmRNASeqTissueAtlas(Zm5bFGS):
if cf.test.force.COB:
print('Rebuilding ZmRNASeqTissueAtlas')
co.del_dataset('COB', 'ZmRNASeqTissueAtlas', force=True)
co.del_dataset('Expr', 'ZmRNASeqTissueAtlas', force=True)
if not co.available_datasets('Expr', 'ZmRNASeqTissueAtlas'):
# Build it
return co.COB.from_table(
os.path.join(
cf.options.testdir,
'raw',
'Expr',
'RNASEQ',
'MaizeRNASeqTissue.tsv.bz2',
),
'ZmRNASeqTissueAtlas',
'Maize RNASeq Tissue Atlas Network, Sekhon 2013, PLoS ONE',
Zm5bFGS,
rawtype='RNASEQ',
max_gene_missing_data=0.3,
max_accession_missing_data=0.08,
min_single_sample_expr=1,
min_expr=0.001,
quantile=False,
max_val=300,
dry_run=True)
else:
return co.COB('ZmRNASeqTissueAtlas')
def ZmIonome(Zm5bFGS):
# Delete the old dataset
if cf.test.force.Ontology:
co.del_dataset('GWAS','ZmIonome',safe=False)
if not co.available_datasets('GWAS','ZmIonome'):
# Grab path the csv
csv = os.path.join(
cf.options.testdir,
'raw','GWAS','Ionome',
'sigGWASsnpsCombinedIterations.longhorn.allLoc.csv.gz'
)
# Define our reference geneome
df = pd.DataFrame.from_csv(csv,index_col=None)
# Import class from dataframe
IONS = co.GWAS.from_DataFrame(
df,'ZmIonome','Maize Ionome',
Zm5bFGS,
term_col='el',chr_col='chr',pos_col='pos'
)
# Get rid of pesky Cobalt
IONS.del_term('Co59')
# I guess we need a test in here too
return IONS
else:
return co.GWAS('ZmIonome')
def AtRootHydroIonome(AtTair10):
if cf.test.force.Ontology:
co.del_dataset('GWAS','AtRootHydroIonome',safe=False)
if not co.available_datasets('GWAS', 'AtRootHydroIonome'):
# glob glob is god
csvs = glob.glob(os.path.join(
cf.options.testdir,
'raw','GWAS','AtIonome',
'AtRootHydroIonome','*.csv.gz'
))
# Read in each table individually then concat for GIANT table
df = pd.concat([pd.read_table(x,sep=' ') for x in csvs])
# Only keep significant pvals
df = df.loc[df.pval <= cf.options.alpha,:]
# Kill groups of SNPs that have identical (beta,pval)s
df = df.groupby(['beta','pval']).filter(lambda x: len(x) < 5)
# Add 'Chr' to chromosome column
df.CHR = df.CHR.apply(lambda x: 'Chr'+str(x))
# Chase dat refgen
# Import class from dataframe
return co.GWAS.from_DataFrame(
df,'AtRootHydroIonome','Arabidopsis second pass 1.6M',
AtTair10, term_col='Trait', chr_col='CHR', pos_col='POS'
)
else:
return co.GWAS('AtRootHydroIonome')
def AtLeafHydroIonome(AtTair10):
if cf.test.force.Ontology:
co.del_dataset('GWAS', 'AtLeafHydroIonome', force=True)
if not co.available_datasets('GWAS', 'AtLeafHydroIonome'):
# glob glob is god
csvs = glob.glob(
os.path.join(cf.options.testdir, 'raw', 'GWAS', 'AtIonome',
'AtLeafHydroIonome', '*.csv.gz'))
# Read in each table individually then concat for GIANT table
df = pd.concat([pd.read_table(x, sep=' ') for x in csvs])
df = df.loc[df.pval <= cf.options.alpha, :]
# Kill groups of SNPs that have identical (beta,pval)s
df = df.groupby(['beta', 'pval']).filter(lambda x: len(x) < 5)
# Add 'Chr' to chromosome column
df.CHR = df.CHR.apply(lambda x: 'Chr' + str(x))
# Import class from dataframe
return co.GWAS.from_DataFrame(df,
'AtLeafHydroIonome',
'Arabidopsis second pass 1.6M',
AtTair10,
term_col='Trait',
chr_col='CHR',
pos_col='POS')
else:
return co.GWAS('AtLeafHydroIonome')
def Tair10(self):
gff = os.path.join(
cf['options']['testdir'],
'raw','TAIR10_GFF3_genes.gff'
)
co.del_dataset('RefGen','T10',safe=False)
T10 = co.RefGen.from_gff(gff,'T10','Tair 10','10','Arabidopsis')
self.assertIsInstance(T10,co.RefGen)
def test_filtered_refgen(testRefGen):
co.del_dataset("RefGen", "test_filtered_refgen", safe=False)
random_genes = set(testRefGen.random_genes(n=500))
test_filtered_refgen = testRefGen.filtered_refgen(
"test_filtered_refgen", "test. please ignore", testRefGen, random_genes
)
assert len(test_filtered_refgen) == len(random_genes)
for x in random_genes:
assert x in test_filtered_refgen
co.del_dataset("RefGen", "test_filtered_refgen", safe=False)
def Zm5bFGS(self):
gff = os.path.join(
cf['options']['testdir'],
'raw','ZmB73_5b_FGS.gff'
)
co.del_dataset('RefGen','Zm5bFGS',safe=False)
ZM = co.RefGen.from_gff(
gff,'Zm5bFGS','Maize 5b Filtered Gene Set','5b','Zea Mays'
)
self.assertIsInstance(ZM,co.RefGen)
def ZmGO(Zm5bFGS):
if cf.test.force.Ontology:
co.del_dataset('GOnt', 'ZmGO', force=True)
if not co.available_datasets('GOnt', 'ZmGO'):
obo = os.path.join(cf.options.testdir, 'raw', 'GOnt', 'go.obo.bz2')
gene_map_file = os.path.join(cf.options.testdir, 'raw', 'GOnt',
'zm_go.tsv.bz2')
return co.GOnt.from_obo(obo, gene_map_file, 'ZmGO',
'Maize Gene Ontology', Zm5bFGS)
else:
return co.GOnt('ZmGO')
def TestGO(Zm5bFGS):
if cf.test.force.Ontology:
co.del_dataset('GOnt', 'TestGO', force=True)
if not co.available_datasets('GOnt', 'TestGO'):
obo = os.path.join(cf.options.testdir, 'raw', 'GOnt', 'go.test.obo')
gene_map_file = os.path.join(cf.options.testdir, 'raw', 'GOnt',
'go.test.tsv')
return co.GOnt.from_obo(obo, gene_map_file, 'TestGO', 'Test GO',
Zm5bFGS)
else:
return co.GOnt('TestGO')
def AtTair10():
if cf.test.force.RefGen:
co.del_dataset('RefGen', 'AtTair10', force=True)
if not co.available_datasets('RefGen', 'AtTair10'):
gff = os.path.expanduser(
os.path.join(cf.options.testdir, 'raw', 'RefGen',
'TAIR10_GFF3_genes.gff.gz'))
return co.RefGen.from_gff(gff, 'AtTair10', 'Tair 10', '10',
'Arabidopsis')
else:
return co.RefGen('AtTair10')
def test_filtered_refgen(testRefGen):
co.del_dataset('RefGen','test_filtered_refgen',force=True)
random_genes = set(testRefGen.random_genes(n=500))
test_filtered_refgen = testRefGen.filtered_refgen(
'test_filtered_refgen',
'test. please ignore',
testRefGen,
random_genes
)
assert len(test_filtered_refgen) == len(random_genes)
for x in random_genes:
assert x in test_filtered_refgen
co.del_dataset('RefGen','test_filtered_refgen',force=True)
def Zm5bFGS():
if cf.test.force.RefGen:
co.del_dataset('RefGen', 'Zm5bFGS', force=True)
if not co.available_datasets('RefGen', 'Zm5bFGS'):
# We have to build it
gff = os.path.expanduser(
os.path.join(cf.options.testdir, 'raw', 'RefGen',
'ZmB73_5b_FGS.gff.gz'))
# This is stupid and necessary because pytables wont let me open
# more than one table
co.RefGen.from_gff(gff, 'Zm5bFGS', 'Maize 5b Filtered Gene Set', '5b',
'Zea Mays')
return co.RefGen('Zm5bFGS')
def ZmRoot(self):
co.del_dataset('Expr','ZmRoot',safe=False)
ZM = co.RefGen('Zm5bFGS')
ZmRoot = co.COB.from_table(
os.path.join(cf.get('options','testdir'),'raw','Expression','ROOTFPKM.tsv'),
'ZmRoot',
'Maize Root Network',
ZM,
rawtype='RNASEQ',
max_gene_missing_data=0.4,
min_expr=0.1,
dry_run=False,
max_val=300
)
def AtTair10():
if cf.test.force.RefGen:
co.del_dataset('RefGen', 'AtTair10', safe=False)
if not co.available_datasets('RefGen', 'AtTair10'):
gff = os.path.expanduser(
os.path.join(
cf.options.testdir,
'raw', 'RefGen', 'TAIR10_GFF3_genes.gff.gz'
)
)
return co.RefGen.from_gff(
gff, 'AtTair10', 'Tair 10', '10', 'Arabidopsis'
)
else:
return co.RefGen('AtTair10')
def AtGO(AtTair10):
if cf.test.force.Ontology:
co.del_dataset('GOnt', 'AtGO', force=True)
if not co.available_datasets('GOnt', 'AtGO'):
obo = os.path.join(cf.options.testdir, 'raw', 'GOnt', 'go.obo.bz2')
gene_map_file = os.path.join(cf.options.testdir, 'raw', 'GOnt',
'ath_go.tsv.bz2')
return co.GOnt.from_obo(obo,
gene_map_file,
'AtGO',
'Arabidopsis Gene Ontology',
AtTair10,
id_col=0,
go_col=5)
else:
return co.GOnt('AtGO')
def build_gont(args):
refgen = co.RefGen(args.refgen)
# Check to see if this dataset is already built
if co.available_datasets('GOnt', args.name):
print('Warning! This dataset has already been built.')
co.del_dataset('GOnt', args.name, force=args.force)
go = co.GOnt.from_obo(args.obo_filename,
args.filename,
args.name,
args.description,
refgen,
go_col=args.go_col,
id_col=args.id_col)
print("Done: {}".format(go.summary()))
print('Build Successful')
def build_cob(args):
# Build the refgen
refgen = co.RefGen(args.refgen)
# Check that the sep is likely right.
if len(pd.read_table(args.filename, sep=args.sep).columns) == 1:
print(
("Detected only 1 column in {}, are you sure "
"colunms are separated by '{}'?").format(args.filename, args.sep))
return None
if args.allow_non_membership:
refgen = refgen.copy('{}_tmp'.format(refgen.name),
'temp refgen'.format(refgen.name))
# Add non membership genes
for gid in pd.read_table(args.filename, sep=args.sep).index:
refgen.add_gene(Gene(None, None, id=gid))
quality_control = False if args.skip_quality_control else True
normalize = False if args.skip_normalization else True
# Check to see if this dataset is already built
if co.available_datasets('Expr', args.name):
print('Warning! This dataset has already been built.')
co.del_dataset('Expr', args.name, safe=args.force)
# Basically just pass all the CLI arguments to the COB class method
cob = co.COB.from_table(
args.filename,
args.name,
args.description,
refgen,
# Optional arguments
sep=args.sep,
rawtype=args.rawtype,
# Data Processing
quality_control=quality_control,
normalization=normalize,
quantile=args.quantile,
# Data processing parameters
max_gene_missing_data=args.max_gene_missing_data,
max_accession_missing_data=args.max_accession_missing_data,
min_single_sample_expr=args.min_single_sample_expr,
min_expr=args.min_expr,
max_val=args.max_val,
dry_run=args.dry_run,
index_col=args.index_col)
print("Build successful!")
print(cob.summary())
def Zm5bFGS():
if cf.test.force.RefGen:
co.del_dataset('RefGen', 'Zm5bFGS', safe=False)
if not co.available_datasets('RefGen', 'Zm5bFGS'):
# We have to build it
gff = os.path.expanduser(
os.path.join(
cf.options.testdir,
'raw', 'RefGen', 'ZmB73_5b_FGS.gff.gz'
)
)
# This is stupid and necessary because pytables wont let me open
# more than one table
return co.RefGen.from_gff(
gff, 'Zm5bFGS', 'Maize 5b Filtered Gene Set', '5b', 'Zea Mays'
)
return co.RefGen('Zm5bFGS')
def ZmSAM(self):
co.del_dataset('Expr','ZmSAM',safe=False)
ZM = co.RefGen('Zm5bFGS')
ZmSAM = co.COB.from_table(
os.path.join(
cf.get('options','testdir'),'raw','Expression',
'TranscriptomeProfiling_B73_Atlas_SAM_FGS_LiLin_20140316.txt'
),
'ZmSAM',
'Maize Root Network',
ZM,
rawtype='RNASEQ',
max_gene_missing_data=0.4,
min_expr=0.1,
dry_run=False,
max_val=300
)
def ZmRoot(self):
co.del_dataset('Expr','ZmRoot',safe=False)
ZM = co.RefGen('Zm5bFGS')
ZmRoot = co.COB.from_table(
os.path.join(cf.get('options','testdir'),'raw','Expression',
'RNASEQ','ROOTFPKM.tsv'),
'ZmRoot',
'Maize Root Network',
ZM,
rawtype='RNASEQ',
max_gene_missing_data=0.3,
max_accession_missing_data=0.08,
min_single_sample_expr=1,
min_expr=0.001,
quantile=False,
max_val=300
)
self.assertIsInstance(ZmRoot,co.COB)
def ZmGO(Zm5bFGS):
if cf.test.force.Ontology:
co.del_dataset('GOnt','ZmGO',safe=False)
if not co.available_datasets('GOnt','ZmGO'):
obo = os.path.join(
cf.options.testdir,
'raw','GOnt','go.obo.bz2'
)
gene_map_file = os.path.join(
cf.options.testdir,
'raw','GOnt','zm_go.tsv.bz2'
)
return co.GOnt.from_obo(
obo, gene_map_file, 'ZmGO',
'Maize Gene Ontology', Zm5bFGS
)
else:
return co.GOnt('ZmGO')
def ZmSAM(Zm5bFGS):
if cf.test.force.COB:
co.del_dataset('Expr', 'ZmSAM', force=True)
if not co.available_datasets('Expr', 'ZmSAM'):
return co.COB.from_table(os.path.join(
cf.options.testdir, 'raw', 'Expr', 'RNASEQ',
'TranscriptomeProfiling_B73_Atlas_SAM_FGS_LiLin_20140316.txt.gz'),
'ZmSAM',
'Maize Root Network',
Zm5bFGS,
rawtype='RNASEQ',
max_gene_missing_data=0.4,
min_expr=0.1,
quantile=False,
dry_run=False,
max_val=250)
else:
return co.COB('ZmSAM')
def ZmRoot(Zm5bFGS):
if cf.test.force.COB:
co.del_dataset('Expr', 'ZmRoot', force=True)
if not co.available_datasets('Expr', 'ZmRoot'):
return co.COB.from_table(os.path.join(cf.options.testdir, 'raw',
'Expr', 'RNASEQ',
'ROOTFPKM.tsv.gz'),
'ZmRoot',
'Maize Root Network',
Zm5bFGS,
rawtype='RNASEQ',
max_gene_missing_data=0.3,
max_accession_missing_data=0.08,
min_single_sample_expr=1,
min_expr=0.001,
quantile=False,
max_val=300)
else:
return co.COB('ZmRoot')
def AtGO(AtTair10):
if cf.test.force.Ontology:
co.del_dataset('GOnt','AtGO',safe=False)
if not co.available_datasets('GOnt','AtGO'):
obo = os.path.join(
cf.options.testdir,
'raw','GOnt','go.obo.bz2'
)
gene_map_file = os.path.join(
cf.options.testdir,
'raw','GOnt','ath_go.tsv.bz2'
)
return co.GOnt.from_obo(
obo, gene_map_file, 'AtGO',
'Arabidopsis Gene Ontology', AtTair10,
id_col=0, go_col=5
)
else:
return co.GOnt('AtGO')
def ZmPAN(Zm5bFGS):
if cf.test.force.COB:
co.del_dataset('Expr', 'ZmPAN', force=True)
if not co.available_datasets('Expr', 'ZmPAN'):
return co.COB.from_table(os.path.join(cf.options.testdir, 'raw',
'Expr', 'RNASEQ',
'PANGenomeFPKM.txt.gz'),
'ZmPAN',
'Maize Root Network',
Zm5bFGS,
rawtype='RNASEQ',
max_gene_missing_data=0.4,
min_expr=1,
quantile=False,
dry_run=False,
sep=',',
max_val=300)
else:
return co.COB('ZmPAN')
def ZmIonome(self):
# Delete the old dataset
co.del_dataset('Ontology','ZmIonome',safe=False)
# Grab path the csv
csv = os.path.join(
cf.get('options','testdir'),
'raw','GWAS','Ionome',
'sigGWASsnpsCombinedIterations.longhorn.allLoc.csv'
)
# Define our reference geneome
ZM = co.RefGen('Zm5bFGS')
df = pd.DataFrame.from_csv(csv,index_col=None)
# Import class from dataframe
IONS = co.Ontology.from_DataFrame(
df,'ZmIonome','Maize Ionome',
ZM,term_col='el',chr_col='chr',pos_col='pos'
)
IONS.del_term('Co59')
# I guess we need a test in here too
self.assertIsInstance(IONS,co.Ontology)
def ZmPAN(self):
co.del_dataset('Expr','ZmPAN',safe=False)
ZM = co.RefGen('Zm5bFGS')
ZmPAN = co.COB.from_table(
os.path.join(
cf.get('options','testdir'),'raw','Expression','RNASEQ',
'PANGenomeFPKM.txt'
),
'ZmPAN',
'Maize Root Network',
ZM,
rawtype='RNASEQ',
max_gene_missing_data=0.4,
min_expr=1,
quantile=False,
dry_run=False,
sep=',',
max_val=300
)
self.assertIsInstance(ZmPAN,co.COB)
def AtLeafHydroIonome(self):
co.del_dataset('Ontology','AtLeafHydroIonome',safe=False)
# glob glob is god
csvs = glob.glob(os.path.join(
cf.get('options','testdir'),
'raw','GWAS','AtLeafHydro',
'*.sigsnps.csv'
))
# Read in each table individually then concat for GIANT table
df = pd.concat([pd.read_table(x,sep=',') for x in csvs])
# Add 'Chr' to chromosome column
df.CHR = df.CHR.apply(lambda x: 'Chr'+str(x))
# Chase dat refgen
T10 = co.RefGen('T10')
# Import class from dataframe
AtLeafHydroIonome = co.Ontology.from_DataFrame(
df,'AtLeafHydroIonome','Arabidopsis 1.6M EmmaX GWAS',
T10,term_col='Trait',chr_col='CHR',pos_col='BP'
)
self.assertIsInstance(AtLeafHydroIonome,co.Ontology)
def ZmWallace(Zm5bFGS):
if cf.test.force.Ontology:
co.del_dataset('GWAS', 'ZmWallace', force=True)
if not co.available_datasets('GWAS', 'ZmWallace'):
# Grab path the csv
csv = os.path.join(
cf.options.testdir, 'raw', 'GWAS', 'WallacePLoSGenet',
'Wallace_etal_2014_PLoSGenet_GWAS_hits-150112.txt.bz2')
# Define our reference geneome
df = pd.DataFrame.from_csv(csv, index_col=None, sep='\t')
# Import class from dataframe
gwas = co.GWAS.from_DataFrame(df,
'ZmWallace',
'Wallace PLoS ONE Dataset.',
Zm5bFGS,
term_col='trait',
chr_col='chr',
pos_col='pos')
return gwas
else:
return co.GWAS('ZmWallace')
def AtSeed(self):
co.del_dataset('Expr','AtSeed',safe=False)
Seed = ['GSE12404',#'GSE30223',
'GSE1051','GSE11852','GSE5634']
SeedFam = sum(
[co.Family.from_file(
pjoin(
self.rawdir,'GSE','{}_family.soft'.format(x)
)
)
for x in Seed ]
)
#SeedFam.to_keepfile("SeedKeep.tsv",keep_hint='seed')
AtSeed = co.COB.from_DataFrame(
SeedFam.series_matrix(
keepfile=pjoin(self.rawdir,'GSE','SeedKeep.tsv')
),
'AtSeed','Arabidopsis Seed',
co.RefGen('T10'),rawtype='MICROARRAY'
)
self.assertIsInstance(AtSeed,co.COB)
def testGWAS(testRefGen):
if cf.test.force.Ontology:
co.del_dataset('GWAS', 'testGWAS', force=True)
df = pd.DataFrame({
'Trait': ['a', 'a', 'b', 'b'],
'CHR': ['chr1', 'chr2', 'chr3', 'chr4'],
'POS': [100, 200, 300, 400],
'Start': [100, 200, 300, 400],
'End': [1000, 20000, 3000, 4000],
'id': ['snp1', 'snp2', 'snp3', 'snp4'],
'pval': [0.05, 0.05, 0.01, 0.01]
})
gwas = co.GWAS.from_DataFrame(df,
'testGWAS',
'Test GWAS Dataset',
testRefGen,
chr_col='CHR',
pos_col='POS',
id_col='id',
term_col='Trait')
return gwas
def ZmWallace(Zm5bFGS):
if cf.test.force.Ontology:
co.del_dataset('GWAS','ZmWallace',safe=False)
if not co.available_datasets('GWAS','ZmWallace'):
# Grab path the csv
csv = os.path.join(
cf.options.testdir,
'raw','GWAS','WallacePLoSGenet',
'Wallace_etal_2014_PLoSGenet_GWAS_hits-150112.txt.bz2'
)
# Define our reference geneome
df = pd.DataFrame.from_csv(csv,index_col=None,sep='\t')
# Import class from dataframe
gwas = co.GWAS.from_DataFrame(
df, 'ZmWallace', 'Wallace PLoS ONE Dataset.',
Zm5bFGS,
term_col='trait', chr_col='chr', pos_col='pos'
)
return gwas
else:
return co.GWAS('ZmWallace')
def AtLeaf(AtTair10):
if cf.test.force.COB:
co.del_dataset('Expr', 'AtLeaf', force=True)
if not co.available_datasets('Expr', 'AtLeaf'):
Leaf = [
'GSE14578',
'GSE5630',
'GSE13739', #'GSE26199',
'GSE5686',
'GSE5615',
'GSE5620',
'GSE5628',
'GSE5624',
'GSE5626',
'GSE5621',
'GSE5622',
'GSE5623',
'GSE5625',
'GSE5688'
]
LeafFam = sum([
co.Family.from_file(
os.path.join(cf.options.testdir, 'raw', 'GSE',
'{}_family.soft.gz'.format(x))) for x in Leaf
])
#LeafFam.to_keepfile("LeafKeep.tsv", keep_hint="lea")
return co.COB.from_DataFrame(
LeafFam.series_matrix(keepfile=os.path.join(
cf.options.testdir, 'raw', 'GSE', 'LeafKeep.tsv')),
'AtLeaf',
'Arabidopsis Leaf',
AtTair10,
rawtype='MICROARRAY',
max_gene_missing_data=0.3,
min_expr=0.01,
quantile=True,
)
else:
return co.COB('AtLeaf')
def ZmSAM(Zm5bFGS):
if cf.test.force.COB:
co.del_dataset('Expr','ZmSAM',safe=False)
if not co.available_datasets('Expr','ZmSAM'):
return co.COB.from_table(
os.path.join(
cf.options.testdir,
'raw','Expr','RNASEQ',
'TranscriptomeProfiling_B73_Atlas_SAM_FGS_LiLin_20140316.txt.gz'
),
'ZmSAM',
'Maize Root Network',
Zm5bFGS,
rawtype='RNASEQ',
max_gene_missing_data=0.4,
min_expr=0.1,
quantile=False,
dry_run=False,
max_val=250
)
else:
return co.COB('ZmSAM')
def ZmRoot(Zm5bFGS):
if cf.test.force.COB:
co.del_dataset('Expr','ZmRoot',safe=False)
if not co.available_datasets('Expr','ZmRoot'):
return co.COB.from_table(
os.path.join(
cf.options.testdir,
'raw','Expr',
'RNASEQ','ROOTFPKM.tsv.gz'
),
'ZmRoot',
'Maize Root Network',
Zm5bFGS,
rawtype='RNASEQ',
max_gene_missing_data=0.3,
max_accession_missing_data=0.08,
min_single_sample_expr=1,
min_expr=0.001,
quantile=False,
max_val=300
)
else:
return co.COB('ZmRoot')
def ZmPAN(Zm5bFGS):
if cf.test.force.COB:
co.del_dataset('Expr','ZmPAN',safe=False)
if not co.available_datasets('Expr','ZmPAN'):
return co.COB.from_table(
os.path.join(
cf.options.testdir,
'raw','Expr','RNASEQ',
'PANGenomeFPKM.txt.gz'
),
'ZmPAN',
'Maize Root Network',
Zm5bFGS,
rawtype='RNASEQ',
max_gene_missing_data=0.4,
min_expr=1,
quantile=False,
dry_run=False,
sep=',',
max_val=300
)
else:
return co.COB('ZmPAN')
def remove(args):
co.del_dataset(args.type,args.name,force=args.force)
print('Done')
def BuildT10(self):
gff = os.path.join(cf.get('options','testdir'),'raw','TAIR10_GFF3_genes.gff')
co.del_dataset('RefGen','T10',safe=False)
T10 = co.RefGen.from_gff(gff,'T10','Tair 10','10','Arabidopsis')
self.assertIsInstance(T10,co.RefGen)
def remove(args):
co.del_dataset(args.type,args.name,safe=args.force)
print('Done')
def BuildZm5bFGS(self):
gff = os.path.join(cf.get('options','testdir'),'raw','ZmB73_5b_FGS.gff')
co.del_dataset('RefGen','Zm5bFGS',safe=False)
ZM = co.RefGen.from_gff(gff,'Zm5bFGS','Maize 5b Filtered Gene Set','5b','Zea Mays')
self.assertIsInstance(ZM,co.RefGen)