def CALIBRATE(self):
try:
self.materialModel=DataUtilities.materialModels[str(self.ui.comboBox_4.currentText())]
self.popt, self.pcov = DataUtilities.fitData(self.time, self.strain, self.stress, self.materialModel)
self.statusString += 'Optimizer routine completed \n'
#self.statusString += str(popt, pcov)
except:
self.statusString += 'Optimizer routine failed \n'
try:
self.stressEvaluated = DataUtilities.evaluateMaterialModel(self.strain,self.popt,self.materialModel)
finally:
self.UpdateStatus()
self.PLOT(self.strain,self.stressEvaluated)
def getAllSummary(): patientList = DataUtilities.getListofPlatPatients() getSummaryGender(patientList) getSummaryPrior_dx(patientList) getSummaryRace(patientList) getSummaryEthnicity(patientList) getSummaryPathologic_stage(patientList) getSummaryNumber_pack_years_smoked(patientList) getSummaryTobacco_smoking_history(patientList) getSummaryPrimary_therapy_outcome_success(patientList) getSummaryVital_status(patientList) getSummaryYears_to_Birth(patientList)
def importFile(self):
self.fileDialog = QtGui.QFileDialog(self)
self.dataPath = unicode(self.fileDialog.getOpenFileName())
try:
self.time, self.strain, self.stress = DataUtilities.readData(self.dataPath)
self.statusString += 'Read data routine completed \n'
except:
self.statusString += 'Read data routine failed \n'
finally:
self.PLOT(self.strain,self.stress)
#print (self.dataPath)
self.UpdateStatus()
def run_regression(survey, version): """Runs logistic regressions. survey: Survey version: which model to run Returns: Regressions object """ dep, control = DataUtilities.get_version(version) print dep, control reg = survey.make_logistic_regression(dep, control) #print reg return Regressions([reg])
def test_models(version=(30,-1), resample_flag=False, patients = -1, printReg = True): means = dict(educ_from_12=4, born_from_1960=10) if patients == -1: patients = DataUtilities.getDictReadofPatientsFilled() #patients = getDictReadofPatientsFilled() # read the survey survey, complete = read_complete(version, patients) print DataUtilities.get_version(version) #compare_survey_and_complete(survey, complete) #print 'all respondents', survey.len() #print 'complete', complete.len() # run the models if printReg: regs = run_regression_and_print(survey, version=version, means=means) else: regs = run_regression(survey, version) return regs
def read_complete(version,patients): survey = read_survey(patients) # give respondents random values #[r.clean_random() for r in survey.respondents()] # select complete records dep, control = DataUtilities.get_version(version) #for var in [dep] + control: # print r'\verb"%s",' % var attrs = [dep] + control complete = survey.subsample(lambda r: r.is_complete(attrs)) return survey, complete
def getSummaryNumber_pack_years_smoked(patientList=None):
if patientList is None:
patientList = DataUtilities.getListofPatients()
dataDict = {}
for patient in patientList:
data = patient.getNumber_pack_years_smoked()
if data == None:
data = "Unknown"
if data in dataDict.keys():
dataDict[data] += 1
else:
dataDict[data] = 1
print "Number_pack_years_smoked Breakdown"
for key, value in dataDict.iteritems():
print key + ": " + str(value)
print ""
return dataDict
def getSummaryPathologic_stage(patientList=None):
if patientList is None:
patientList = DataUtilities.getListofPatients()
dataDict = {}
for patient in patientList:
data = patient.getPathologic_stage()
if data == None:
data = "Unknown"
if data in dataDict.keys():
dataDict[data] += 1
else:
dataDict[data] = 1
print "Pathologic_stage Breakdown"
for key, value in dataDict.iteritems():
print key + ": " + str(value)
print ""
return dataDict
def getSummaryYears_to_Birth(patientList=None):
if patientList is None:
patientList = DataUtilities.getListofPatients()
dataDict = {}
for patient in patientList:
data = patient.getYears_to_birthClean()
if data == None:
data = "Unknown"
if data in dataDict.keys():
dataDict[data] += 1
else:
dataDict[data] = 1
print "Years_to_Birth Breakdown"
for key, value in dataDict.iteritems():
print str(key) + ": " + str(value)
print ""
return dataDict
def getSummaryPrimary_therapy_outcome_success(patientList=None):
if patientList is None:
patientList = DataUtilities.getListofPatients()
dataDict = {}
for patient in patientList:
data = patient.getPrimary_therapy_outcome_success()
if data == None:
data = "Unknown"
if data in dataDict.keys():
dataDict[data] += 1
else:
dataDict[data] = 1
print "Primary_therapy_outcome_success Breakdown"
for key, value in dataDict.iteritems():
print key + ": " + str(value)
print ""
return dataDict
def getSummaryTobacco_smoking_history(patientList=None):
if patientList is None:
patientList = DataUtilities.getListofPatients()
dataDict = {}
for patient in patientList:
data = patient.getTobacco_smoking_history()
if data == None:
data = "Unknown"
if data in dataDict.keys():
dataDict[data] += 1
else:
dataDict[data] = 1
print "getTobacco_smoking_history Breakdown"
for key, value in dataDict.iteritems():
print key + ": " + str(value)
print ""
return dataDict
def IBOrders(d,q_data,q_msg_o,q_err_o):
# Setup second logger-------------------------------------------------------
log2=logging.getLogger('IBOrdersScript')
log2.setLevel(logging.INFO)
formatter=logging.Formatter('%(asctime)s %(name)s %(levelname)s %(message)s')
file_handler=logging.FileHandler('IBOrdersScript.log')
file_handler.setFormatter(formatter)
stream_handler=logging.StreamHandler()
stream_handler.setFormatter(formatter)
log2.addHandler(file_handler)
log2.addHandler(stream_handler)
#---------------------------------------------------------------------------
ib=IB()
try:
# Start main process loop
while True:
# Check the message queue and either connect or quit
if not q_msg_o.empty():
message=q_msg_o.get() # Get the message
if message=='Start': # Start IB
log2.info('Starting IB ordering process')
_=util.logToFile('IBOrders.log',level=20,ibapiLevel=20) # Logger
_=ib.connect('127.0.0.1', 7497, 2) # Connect to IB
ib.sleep(1)
log2.info('Connected to IB ordering process')
elif message=='Stop': # Stop IB
if ib.isConnected():
ib.disconnect()
ib.sleep(1)
log2.info('Controlled disconnection from IB ordering process')
# While we are connected, do stuff
if ib.isConnected():
# If there is data in the queue, go through it and make trades
if not q_data.empty():
data=q_data.get() # Get data
ticker=data[0].symbol # tikcer
# Call the model, it should know your input JSON structure
trd=model.Position(data,d[ticker])
# Get model parameters, UNITS and stop percentage
params=model.Parameters(d[ticker])
# If we are going to open a position, check time of day here
if trd==1:
# Open position
log2.info('Placing market order for '+ticker+' with ticktime '+str(data[1])+' and ask '+str(data[3]))
order=MarketOrder('BUY',params[0])
#contract=ib.qualifyContracts(Stock(,'Smart','USD'))
trade=ib.placeOrder(data[0],order)
# Update model dynamic parameters
d[ticker][-1][0]=1 # Position flag
d[ticker][-1][1]=data[4] # Open price
d[ticker][-1][2]=data[4]*(1-params[1]) # Stop price
# Write the updated model dict to JSON for safety
_=DataUtilities.ModelOut(d)
# If we are going to sell an open position
elif trd==-1:
# Close position
log2.info('Closing position for '+ticker+' with ticktime '+str(data[1])+' and bid '+str(data[2]))
order=MarketOrder('SELL',params[0])
#contract=ib.qualifyContracts(Stock(str(data[0]),'Smart','USD'))
trade=ib.placeOrder(data[0],order)
# Update model dynamic parameters
d[ticker][-1]=[0,0,0,0]
# Write the updated model dict to JSON for safety
_=DataUtilities.ModelOut(d)
# Check if we need to adjust stop
elif trd==0:
# Set trailing stop flg is needed
if d[ticker][-1][0] and not d[ticker][-1][3]:
d[ticker][-1][3]=model.SetTrail(data[4],d[ticker])
# Write the updated model dict to JSON for safety
_=DataUtilities.ModelOut(d)
# Adjust stop price (trailing stop) if needed
if d[ticker][-1][0] and d[ticker][-1][3]:
d[ticker][-1][2]=model.NewStop(data[4],d[ticker])
# Write the updated model dict to JSON for safety
_=DataUtilities.ModelOut(d)
except KeyboardInterrupt:
ib.disconnect()
ib.sleep(1)
log2.info('Controlled disconnection from IB ordering process')
except Exception as e:
_=q_err_o.put(d)
ib.disconnect()
ib.sleep(1)
log2.error('Disconnected by exception IB ordering process')
log2.exception(e)
formatter=logging.Formatter('%(asctime)s %(name)s %(levelname)s %(message)s')
file_handler=logging.FileHandler('IBMain.log')
file_handler.setFormatter(formatter)
stream_handler=logging.StreamHandler()
stream_handler.setFormatter(formatter)
logger.addHandler(file_handler)
logger.addHandler(stream_handler)
#---------------------------------------------------------------------------
# Set up the ticker list and model------------------------------------------
# Read in the JSON and get existing positions (if you want)
tickerdata=DataUtilities.GetTickerDataCL()
if tickerdata==0: sys.exit()
tlist=list(tickerdata.keys())
positions=DataUtilities.GetPositionsCL()
if positions==0: positions={}
# Build the "model", it is just a dictionary where tickers are keys and
# values are lists of trading related parameters
d=DataUtilities.BuildModel(positions,tickerdata) # Model dictionary
#---------------------------------------------------------------------------
# Set up multiprocessing----------------------------------------------------
# Set up the queues for data and message traffic.
q_data=mp.Queue()
q_msg_f=mp.Queue()
import glob
import xml.etree.ElementTree as ET
import thinkplot
import DataUtilities
import numpy
matching = DataUtilities.findPatientFiles()
namespaces = DataUtilities.getPatientXMLNameSpaces()
drugList = {}
patientsWithDrugs = 0
drugsPerPatient = []
patientWithPlat = 0
"getDataOverview"
for patient in matching:
tree = ET.parse(patient)
rootElement = tree.getroot()
plat = False
for drugs in rootElement.findall("luad:patient/rx:drugs", namespaces=namespaces):
if len(drugs) != 0:
drugsPerPatient.append(len(drugs))
patientsWithDrugs += 1
for drug in drugs:
drugName = drug.find('rx:drug_name' , namespaces=namespaces).text
if drugName:
drugName = DataUtilities.getTrueDrugName(drugName.lower())
# I think this is where FixingErrors.py should go.
if drugName.lower() == "cisplatin" or drugName.lower() == "carboplatin":
plat = True
def errorTotal():
means = {}
version = (30,-1)
cumDict = {}
patientDict = DataUtilities.getDictReadofPatients()
for i in range(100):
patients = DataUtilities.getDictReadofPatientsFilled(patientDict = patientDict)
#resample
sPatients = sample_wr(patients, 91)
regs = TCGAlogReg.test_models(version = version, patients = sPatients, printReg = False)
for reg in regs.regs:
cumulative_odds = reg.report_odds(means, printCum = False)
for name, odds, p in cumulative_odds:
if name in cumDict.keys():
cumDict[name].append(odds)
else:
cumDict[name] = [odds]
print "ErrorTotal"
#create table
print r"\begin{table}[h]"
print r"\begin{tabular}{|l|l|l|l|}"
print r"\hline"
print r"\textbf{Type} & \textbf{Odds} & \textbf{Odds Lower} & \textbf{Odds Upper}\\ \hline"
for key, oddsList in sorted(cumDict.items(), key=lambda e: sum(e[1])):
if key not in mutationDict.getMutationsGreaterThan(1):
average = sum(oddsList) / float(len(oddsList))
lower = getLowerConfidence(oddsList)
upper= getUpperLowerConfidence(oddsList)
print key + " & " "{:.3f}".format(average) + " & " + "{:.3f}".format(lower) + " & " "{:.3f}".format(upper) + r"\\ \hline"
print "\end{tabular}"
print "\end{table}"
print ""
print r"\begin{table}[h]"
print r"\begin{tabular}{|l|l|l|l|}"
print r"\hline"
print r"\textbf{Type} & \textbf{Odds} & \textbf{Odds Lower} & \textbf{Odds Upper}\\ \hline"
names = []
averages = []
lowers = []
uppers = []
for key, oddsList in sorted(cumDict.items(), key=lambda e: sum(e[1])):
if key in mutationDict.getMutationsGreaterThan(1):
average = sum(oddsList) / float(len(oddsList))
lower = getLowerConfidence(oddsList)
upper= getUpperConfidence(oddsList)
print key + " & " "{:.3f}".format(average) + " & (" + "{:.3f}".format(lower) + ", " "{:.3f}".format(upper) + r")\\ \hline"
names.append(key)
averages.append(average)
lowers.append(lower)
uppers.append(upper)
print "\end{tabular}"
print "\caption{The above table shows the effect each mutation has when tested in conjunction with the other mutations}"
print "\end{table}"
plotRegMult(names, averages, lowers, uppers, "Mutation Log Odds With Total Error","Mutations", "Log Odds")
def errorMissing():
means = {}
version = (30,-1)
cumDict = {}
patientDict = DataUtilities.getDictReadofPatients()
for i in range(100):
patients = DataUtilities.getDictReadofPatientsFilled(patientDict = patientDict)
regs = TCGAlogReg.test_models(version = version, patients = patients, printReg = False)
for reg in regs.regs:
cumulative_odds = reg.report_odds(means, printCum = False)
for name, odds, p in cumulative_odds:
if name in cumDict.keys():
cumDict[name].append(odds)
else:
cumDict[name] = [odds]
print "ErrorMissing"
#create table
print r"\begin{table}[h]"
print r"\begin{tabular}{|l|l|l|}"
print r"\hline"
print r"\textbf{Type} & \textbf{Odds} & \textbf{Confidence Interval}\\ \hline"
namesL = []
averagesL = []
lowersL = []
uppersL = []
for key, oddsList in sorted(cumDict.items(), key=lambda e: sum(e[1])):
if key not in mutationDict.getMutationsGreaterThan(1):
average = sum(oddsList) / float(len(oddsList))
lower = getLowerConfidence(oddsList)
upper= getUpperConfidence(oddsList)
print key + " & " "{:.3f}".format(average) + " & (" + "{:.3f}".format(lower) + ", " "{:.3f}".format(upper) + r")\\ \hline"
namesL.append(key)
averagesL.append(average)
lowersL.append(lower)
uppersL.append(upper)
print "\end{tabular}"
print "\end{table}"
plotRegMult(namesL, averagesL, lowersL, uppersL, "Life Factor Log Odds With Sampling Error","Factor", "Log Odds")
print ""
print r"\begin{table}[h]"
print r"\begin{tabular}{|l|l|l|}"
print r"\hline"
print r"\textbf{Type} & \textbf{Odds} & \textbf{Confidence Interval}\\ \hline"
namesM = []
averagesM = []
lowersM = []
uppersM = []
for key, oddsList in sorted(cumDict.items(), key=lambda e: sum(e[1])):
if key in mutationDict.getMutationsGreaterThan(1):
average = sum(oddsList) / float(len(oddsList))
lower = getLowerConfidence(oddsList)
upper= getUpperConfidence(oddsList)
print key + " & " "{:.3f}".format(average) + " & (" + "{:.3f}".format(lower) + ", " "{:.3f}".format(upper) + r")\\ \hline"
namesM.append(key)
averagesM.append(average)
lowersM.append(lower)
uppersM.append(upper)
print "\end{tabular}"
print "\caption{The above table shows the effect each mutation has when tested independently the other mutations. The confidence intervals are shown for the error due to missing data.}"
print "\end{table}"
plotRegMult(namesM, averagesM, lowersM, uppersM, "Mutation Log Odds With Sampling Error","Mutation", "Log Odds")
plotRegMult(namesL + namesM, averagesL + averagesM, lowersL + lowersM, uppersL + uppersM, "Log Odds With Sampling Error","Factor", "Log Odds")
