def getPathwayAs(pathway_db, species_code, mod):
begin_time = time.time()
try:
export.deleteFolder(
'BuildDBs/WPs') ### clear any remaining pathway files
except Exception:
null = []
for wpid in pathway_db:
file_type = 'gpml'
wp_id_data = client.service.getPathwayAs(fileType=file_type,
pwId=wpid,
revision=0)
wp_id_data = base64.b64decode(wp_id_data)
gpml_path = filepath('BuildDBs/WPs/' + wpid + '.gpml')
outfile = export.ExportFile(gpml_path)
outfile.write(wp_id_data)
outfile.close()
gene_system_list = string.split(wp_id_data, '\n')
parent_path = export.findParentDir(gpml_path)
pathway_db = gene_associations.getGPMLGraphData(
parent_path, species_code, mod) ### get GPML data back
os.remove(gpml_path) ### Only store the file temporarily
end_time = time.time()
time_diff = float(end_time - begin_time)
"""
try: print "WikiPathways data imported in %d seconds" % time_diff
except Exception: null=None ### Occurs when transitioning back from the Official Database download window (not sure why) -- TclError: can't invoke "update" command
"""
return pathway_db
def getPathwayAs(pathway_db,species_code,mod):
begin_time = time.time()
for wpid in pathway_db:
#print [wpid],'pathway_db',len(pathway_db)
file_type = 'gpml'
#file_type = 'application/gpml+xml'
processor_time = str(time.clock())
#try: export.deleteFolder('BuildDBs/WPs') ### clear any remaining pathway files
#except Exception: pass
#wp_id_data = client.service.getPathwayAs(fileType = file_type,pwId = wpid, revision = 0)
kwargs = {
'identifier': 'WP2062',
'version': 0,
'file_format': 'application/gpml+xml'}
#wp_id_data = wikipathways_api_client_instance.get_pathway_as(**kwargs)
wp_id_data = wikipathways_api_client_instance.get_pathway_as(file_format = file_type,identifier = wpid, version = 0)
#wp_id_data = base64.b64decode(wp_id_data)
gpml_path = filepath('BuildDBs/WPs/'+processor_time+'/'+wpid+'.gpml')
#print gpml_path
outfile = export.ExportFile(gpml_path)
outfile.write(wp_id_data); outfile.close()
gene_system_list = string.split(wp_id_data,'\n')
parent_path = export.findParentDir(gpml_path)
pathway_db = gene_associations.getGPMLGraphData(parent_path,species_code,mod) ### get GPML data back
#os.remove(gpml_path) ### Only store the file temporarily
try: export.deleteFolder('BuildDBs/WPs/'+processor_time) ### clear any remaining pathway files
except Exception: pass
end_time = time.time(); time_diff = float(end_time-begin_time)
"""
try: print "WikiPathways data imported in %d seconds" % time_diff
except Exception: null=None ### Occurs when transitioning back from the Official Database download window (not sure why) -- TclError: can't invoke "update" command
"""
return pathway_db
def getPathwayAs(pathway_db, species_code, mod):
begin_time = time.time()
for wpid in pathway_db:
#print [wpid],'pathway_db',len(pathway_db)
file_type = 'gpml'
#file_type = 'application/gpml+xml'
processor_time = str(time.clock())
#try: export.deleteFolder('BuildDBs/WPs') ### clear any remaining pathway files
#except Exception: pass
#wp_id_data = client.service.getPathwayAs(fileType = file_type,pwId = wpid, revision = 0)
kwargs = {
'identifier': 'WP2062',
'version': 0,
'file_format': 'application/gpml+xml'
}
#wp_id_data = wikipathways_api_client_instance.get_pathway_as(**kwargs)
wp_id_data = wikipathways_api_client_instance.get_pathway_as(
file_format=file_type, identifier=wpid, version=0)
#wp_id_data = base64.b64decode(wp_id_data)
gpml_path = filepath('BuildDBs/WPs/' + processor_time + '/' + wpid +
'.gpml')
#print gpml_path
outfile = export.ExportFile(gpml_path)
outfile.write(wp_id_data)
outfile.close()
gene_system_list = string.split(wp_id_data, '\n')
parent_path = export.findParentDir(gpml_path)
pathway_db = gene_associations.getGPMLGraphData(
parent_path, species_code, mod) ### get GPML data back
#os.remove(gpml_path) ### Only store the file temporarily
try:
export.deleteFolder(
'BuildDBs/WPs/' +
processor_time) ### clear any remaining pathway files
except Exception:
pass
end_time = time.time()
time_diff = float(end_time - begin_time)
"""
try: print "WikiPathways data imported in %d seconds" % time_diff
except Exception: null=None ### Occurs when transitioning back from the Official Database download window (not sure why) -- TclError: can't invoke "update" command
"""
return pathway_db
def runMiDAS(apt_dir,array_type,dataset_name,array_group_list,array_group_db):
if '/bin' in apt_dir: apt_file = apt_dir +'/apt-midas' ### if the user selects an APT directory
elif os.name == 'nt':
import platform
if '32bit' in platform.architecture(): apt_file = apt_dir + '/PC/32bit/apt-midas'
elif '64bit' in platform.architecture(): apt_file = apt_dir + '/PC/64bit/apt-midas'
elif 'darwin' in sys.platform: apt_file = apt_dir + '/Mac/apt-midas'
elif 'linux' in sys.platform:
import platform
if '32bit' in platform.architecture(): apt_file = apt_dir + '/Linux/32bit/apt-midas'
elif '64bit' in platform.architecture(): apt_file = apt_dir + '/Linux/64bit/apt-midas'
apt_file = filepath(apt_file)
### Each input file for MiDAS requires a full file path, so get the parent path
midas_input_dir = 'AltResults/MIDAS/'
path=filepath(midas_input_dir)
### Remotely connect to the previously verified APT C+ midas program and run analysis
metafile = path + 'meta-'+dataset_name[0:-1]+'.txt'
exon_or_junction_file = path + array_type+'-exp-'+dataset_name[0:-1]+'.txt'
gene_exp_file = path + 'gene-exp-'+dataset_name[0:-1]+'.txt'
celfiles = path + 'celfiles-'+dataset_name[0:-1]+'.txt'
output_file = path + dataset_name[0:-1]+'-output'
### Delete the output folder if it already exists (may cause APT problems)
delete_status = export.deleteFolder(output_file)
try:
import subprocess
retcode = subprocess.call([
apt_file, "--cel-files", celfiles,"-g", gene_exp_file, "-e", exon_or_junction_file,
"-m", metafile, "-o", output_file])
if retcode: status = 'failed'
else: status = 'run'
except NameError: status = 'failed'
if status == 'failed':
try:
### Try running the analysis with old MiDAS file headers and command
exportMiDASArrayNames(array_group_list,array_group_db,dataset_name,'old')
import subprocess
retcode = subprocess.call([
apt_file, "-c", celfiles,"-g", gene_exp_file, "-e", exon_or_junction_file,
"-m", metafile, "-o", output_file])
if retcode: status = 'failed'
else: status = 'run'
except Exception: status = 'failed'
if status == 'failed': print "apt-midas failed"
else: print "apt-midas run successfully"
return status
def runMiDAS(apt_dir,array_type,dataset_name,array_group_list,array_group_db):
if '/bin' in apt_dir: apt_file = apt_dir +'/apt-midas' ### if the user selects an APT directory
elif os.name == 'nt':
import platform
if '32bit' in platform.architecture(): apt_file = apt_dir + '/PC/32bit/apt-midas'
elif '64bit' in platform.architecture(): apt_file = apt_dir + '/PC/64bit/apt-midas'
elif 'darwin' in sys.platform: apt_file = apt_dir + '/Mac/apt-midas'
elif 'linux' in sys.platform:
import platform
if '32bit' in platform.architecture(): apt_file = apt_dir + '/Linux/32bit/apt-midas'
elif '64bit' in platform.architecture(): apt_file = apt_dir + '/Linux/64bit/apt-midas'
apt_file = filepath(apt_file)
### Each input file for MiDAS requires a full file path, so get the parent path
midas_input_dir = 'AltResults/MIDAS/'
path=filepath(midas_input_dir)
### Remotely connect to the previously verified APT C+ midas program and run analysis
metafile = path + 'meta-'+dataset_name[0:-1]+'.txt'
exon_or_junction_file = path + array_type+'-exp-'+dataset_name[0:-1]+'.txt'
gene_exp_file = path + 'gene-exp-'+dataset_name[0:-1]+'.txt'
celfiles = path + 'celfiles-'+dataset_name[0:-1]+'.txt'
output_file = path + dataset_name[0:-1]+'-output'
### Delete the output folder if it already exists (may cause APT problems)
delete_status = export.deleteFolder(output_file)
try:
import subprocess
retcode = subprocess.call([
apt_file, "--cel-files", celfiles,"-g", gene_exp_file, "-e", exon_or_junction_file,
"-m", metafile, "-o", output_file])
if retcode: status = 'failed'
else: status = 'run'
except NameError: status = 'failed'
if status == 'failed':
try:
### Try running the analysis with old MiDAS file headers and command
exportMiDASArrayNames(array_group_list,array_group_db,dataset_name,'old')
import subprocess
retcode = subprocess.call([
apt_file, "-c", celfiles,"-g", gene_exp_file, "-e", exon_or_junction_file,
"-m", metafile, "-o", output_file])
if retcode: status = 'failed'
else: status = 'run'
except Exception: status = 'failed'
if status == 'failed': print "apt-midas failed"
else: print "apt-midas run successfully"
return status
def runICGStest(testType = "complete",inputData = "BAM"):
import UI
species_names = UI.getSpeciesInfo()
additional=None
genes = ''
genes_option = "--genes"
cluster_method = "ward"
rho = "0.4"
removeOutliers = "no"
if inputData == "text":
if 'Mm' not in species_names:
return 'WARNING!!! Species Mm database not installed.'
inputDataType = "Fluidigim_TPM"
species = "Mm"
SamplesDiffering = "4"
excludeCellCycle = "conservative"
restrictBy = "None"
input_path = unique.filepath(os.getcwd()+"/tests/demo_data/"+inputDataType+"/input/BoneMarrow-scRNASeq.txt")
expname = "BoneMarrow-scRNASeq"
expdir = "--expdir"
removeOutliers = "no"
elif inputData == "BAM":
if 'Hs' not in species_names:
return 'WARNING!!! Species Hs database not installed.'
species = "Hs"
inputDataType = inputData
SamplesDiffering = "3"
excludeCellCycle = "no"
restrictBy = "None"
expdir = "--bedDir"
expname = "test"
input_path = unique.filepath(os.getcwd()+"/tests/demo_data/"+inputDataType+"/input")
elif inputData == 'FASTQ':
if 'Hs' not in species_names:
return 'WARNING!!! Species Hs database not installed.'
species = "Hs"
inputDataType = inputData
SamplesDiffering = "3"
excludeCellCycle = "no"
restrictBy = "None"
expdir = "--fastq_dir"
expname = "test"
input_path = unique.filepath(os.getcwd()+"/tests/demo_data/"+inputDataType+"/input")
custom_FASTA_path = unique.filepath(os.getcwd()+"/tests/demo_data/FASTA/Homo_sapiens.GRCh37.72.cdna.all.filtered.fa")
additional = ["--runKallisto","True","--customFASTA",custom_FASTA_path]
elif inputData == '10X':
if 'Mm' not in species_names:
return 'WARNING!!! Species Hs database not installed.'
species = "Mm"
inputDataType = inputData
SamplesDiffering = "4"
rho = "0.3"
excludeCellCycle = "no"
removeOutliers = "yes"
restrictBy = "protein_coding"
expdir = "--ChromiumSparseMatrix"
expname = "test"
input_path = unique.filepath(os.getcwd()+"/tests/demo_data/10X/input/mm10/matrix.mtx")
if testType=="complete":
cluster_method = "hopach"
elif inputData == "text":
### Optionally, peed up the analysis by restricting to genes correlated to these guides
genes = "Gfi1 Irf8 Vwf Mmp9"
outputPath = unique.filepath(os.getcwd()+"/tests/demo_data/"+inputDataType+"/output/")
input_root_dir = unique.filepath(os.getcwd()+"/tests/demo_data/"+inputDataType+"/input/")
### Create the output directory path if not created
try: os.mkdir(outputPath)
except Exception:
### Remove this directory
print 'Removing previous output test directory...'
export.deleteFolder(outputPath)
try: os.mkdir(outputPath)
except Exception: pass
### Commands replicate Olsson et al. 2016 Nature
commands = ["python", "AltAnalyze.py", "--platform","RNASeq", "--species", species,
"--column_method", cluster_method, "--column_metric", "cosine", "--rho", rho,
"--ExpressionCutoff", "1", "--FoldDiff", "4", "--SamplesDiffering",SamplesDiffering,
"--restrictBy", restrictBy, "--excludeCellCycle", excludeCellCycle, "--removeOutliers",
removeOutliers, "--row_method", cluster_method, expdir, input_path, "--output", outputPath,
"--runICGS", "yes", genes_option, genes, "--expname",expname]
if additional != None: ### additional optional parameters
commands+=additional
print string.join(commands,' ')
retcode = subprocess.call(commands)
#sys.exit()
reference_log, ref_exceptions = readLogFile(input_root_dir)
test_log, test_exceptions = readLogFile(outputPath)
overlap = list(set(reference_log).intersection(test_log))
matching_content = float(len(overlap))/len(reference_log)
percent_matching = int(matching_content*100)
if percent_matching<90: status = "\nWARNING!!! "
else: status = "\nSUCCESSFUL OUTPUT!!! "
status+= str(percent_matching)+'% of reported ICGS outputs match to reference outputs. '
status+= str(len(test_exceptions))+' errors encountered.'
print status
return status
null = [] ### Occurs if dabg export failed
if analyze_metaprobesets == 'yes':
residual_destination_file = string.replace(
expression_file, 'exp.', 'residuals.')
residual_exp_file = output_dir + '/' + algorithm + '.residuals.txt'
#shutil.copyfile(residual_exp_file, residual_destination_file);os.remove(residual_exp_file)
reformatResidualFile(residual_exp_file,
residual_destination_file)
residual_dabg_file = output_dir + '/dabg.residuals.txt'
os.remove(residual_dabg_file)
except NameError:
status = 'failed'
#print traceback.format_exc()
cache_delete_status = export.deleteFolder(cache_dir)
if status == 'failed':
if architecture == '64bit' and platform.architecture(
)[0] == '64bit' and (os.name == 'nt' or 'linux' in sys.platform):
print 'Warning! 64bit version of APT encountered an error, trying 32bit.'
### If the above doesn't work, try 32bit architecture instead of 64bit (assuming the problem is related to known transient 64bit build issues)
for dataset in exp_file_location_db: ### Instance of the Class ExpressionFileLocationData
fl = exp_file_location_db[dataset]
fl.setArchitecture('32bit')
probesetSummarize(exp_file_location_db, analyze_metaprobesets,
probeset_type, species, root)
else:
print_out = 'apt-probeset-summarize failed. See log and report file in the output folder under "ExpressionInput/APT-output" for more details.'
try:
WarningWindow(print_out, 'Exit')
root.destroy()
os.remove(summary_exp_file)
except Exception:
print traceback.format_exc()
null=[] ### Occurs if dabg export failed
if analyze_metaprobesets == 'yes':
residual_destination_file = string.replace(expression_file,'exp.','residuals.')
residual_exp_file = output_dir+'/'+algorithm+'.residuals.txt'
#shutil.copyfile(residual_exp_file, residual_destination_file);os.remove(residual_exp_file)
reformatResidualFile(residual_exp_file,residual_destination_file)
residual_dabg_file = output_dir+'/dabg.residuals.txt'; os.remove(residual_dabg_file)
except NameError:
status = 'failed'
#print traceback.format_exc()
cache_delete_status = export.deleteFolder(cache_dir)
if status == 'failed':
if architecture == '64bit' and platform.architecture()[0] == '64bit' and (os.name == 'nt' or 'linux' in sys.platform):
print 'Warning! 64bit version of APT encountered an error, trying 32bit.'
### If the above doesn't work, try 32bit architecture instead of 64bit (assuming the problem is related to known transient 64bit build issues)
for dataset in exp_file_location_db: ### Instance of the Class ExpressionFileLocationData
fl = exp_file_location_db[dataset]; fl.setArchitecture('32bit')
probesetSummarize(exp_file_location_db,analyze_metaprobesets,probeset_type,species,root)
else:
print_out = 'apt-probeset-summarize failed. See log and report file in the output folder under "ExpressionInput/APT-output" for more details.'
try:
WarningWindow(print_out,'Exit')
root.destroy()
except Exception:
print print_out; force_exit
else:
