from load import main import tensorflow as tf tf.logging.set_verbosity(tf.logging.INFO) dataset, model = main(train=False) session = model.restore_last_session() while True: line = input("Q: ") line = line.strip() if not line: continue inp = dataset.str2sequence(line) print("INPUT:") print(inp) output = model.predict(session, inp.T) print("OUTPUT:") print(output) q = dataset.sequence2str(inp[0][::-1]) a = dataset.sequence2str(output[0]) print("Q: {}\nA: {}".format(q, a))
from load import main import tensorflow as tf tf.logging.set_verbosity(tf.logging.INFO) batch_size = 256 dataset, model = main(train=True) batch_size = 256 training = dataset.getTrainingData() training_batch = dataset.getBatches(training, batch_size) model.train(training_batch)
#!/usr/bin/python3 # check python version is > 3.6 import load if __name__ == '__main__': load.main(args)
#!/usr/bin/env python3 # loadaverage exceution script # for more info, see github.com/qguv/loadaverage import sys import load sys.exit(load.main())
def main(binarize_qol, tensor_dir, min_date, cutoff): #Remove old tensors if os.path.exists(tensor_dir + "mm_joined.h5"): os.remove(tensor_dir + "mm_joined.h5") if os.path.exists(tensor_dir + "mm_separate.h5"): os.remove(tensor_dir + "mm_separate.h5") dataset = load.main("clinical", tensor_dir) clinical_tensor = dataset['tensor'] clinical_obs_tensor = dataset['obs_tensor'] clinical_feature_names = dataset['feature_names'] clinical_feature_types = dataset['feature_types'] people = dataset['people'] if binarize_qol: ##Binarize QOL variables for k in range(clinical_tensor.shape[2]): if 'qol' in clinical_feature_names[k] and clinical_feature_types[ k] == 'categorical': clinical_tensor[:, :, k] = (clinical_tensor[:, :, k] >= 2.) * 1. clinical_feature_types[k] = 'binary' #Clinical data before the start of treatment in 2-d initial_clinical_tensor, initial_clinical_mask = aggregateOverTime( clinical_tensor[:, :abs(min_date), :], clinical_obs_tensor[:, :abs(min_date), :], clinical_feature_types, FREQ=abs(min_date)) print "Initial clinical aggregated" #Clinical data after the start of treatment in bins of 90 days aggregated_clinical_tensor, binary_clinical_mask = aggregateOverTime( clinical_tensor[:, abs(min_date) + 1:, :], clinical_obs_tensor[:, abs(min_date) + 1:, :], clinical_feature_types) del clinical_tensor del clinical_obs_tensor print "Clinical aggregated" dataset = load.main("initial", tensor_dir) initial_tensor = dataset['tensor'] initial_obs_tensor = dataset['obs_tensor'] initial_feature_names = dataset['feature_names'] initial_feature_types = dataset['feature_types'] assert np.array_equal(dataset['people'], people) #Clinical data before the start of treatment in 2-d aggregated_initial_tensor, aggregated_initial_mask = aggregateOverTime( initial_tensor[:, :abs(min_date), :], initial_obs_tensor[:, :, :], initial_feature_types, FREQ=initial_tensor.shape[1]) del initial_tensor del initial_obs_tensor print "Initial aggregated" dataset = load.main("treatment", tensor_dir) treatment_tensor = dataset['tensor'] treatment_obs_tensor = dataset['obs_tensor'] treatment_feature_names = dataset['feature_names'] treatment_feature_types = dataset['feature_types'] assert np.array_equal(dataset['people'], people) #Treatment data after the start of treatment in bins of 90 days aggregated_treatment_tensor, binary_treatment_mask = aggregateOverTime( treatment_tensor[:, abs(min_date) + 1:, :], treatment_obs_tensor[:, abs(min_date) + 1:, :], treatment_feature_types) del treatment_tensor del treatment_obs_tensor print "Treatment aggregated" INITIAL_MATRIX = np.concatenate( [initial_clinical_tensor, aggregated_initial_tensor], axis=2) INITIAL_MASK = np.concatenate( [initial_clinical_mask, aggregated_initial_mask], axis=2) INITIAL_FEATURE_NAMES = np.concatenate( [clinical_feature_names, initial_feature_names]) INITIAL_FEATURE_TYPES = np.concatenate( [clinical_feature_types, initial_feature_types]) indices = [] for i in range(len(INITIAL_FEATURE_NAMES)): if len(np.nonzero(INITIAL_MATRIX[:, 0, i])[0]) >= cutoff: indices.append(i) INITIAL_MATRIX = INITIAL_MATRIX[:, :, indices] INITIAL_MASK = INITIAL_MASK[:, :, indices] INITIAL_FEATURE_NAMES = INITIAL_FEATURE_NAMES[indices] INITIAL_FEATURE_TYPES = INITIAL_FEATURE_TYPES[indices] #Treatment binary for now TREATMENT_TENSOR = binary_treatment_mask TREATMENT_MASK = binary_treatment_mask TREATMENT_FEATURE_NAMES = treatment_feature_names TREATMENT_FEATURE_TYPES = treatment_feature_types ##Shuffle features by type reshufidx = np.argsort(clinical_feature_types) CLINICAL_TENSOR = aggregated_clinical_tensor[:, :, reshufidx] CLINICAL_MASK = binary_clinical_mask[:, :, reshufidx] CLINICAL_FEATURE_NAMES = clinical_feature_names[reshufidx] CLINICAL_FEATURE_TYPES = clinical_feature_types[reshufidx] RESPONSE_IDX = list(CLINICAL_FEATURE_NAMES).index( "at_treatmentresp-treatment response-numerical") DEATH_IDX = list(CLINICAL_FEATURE_NAMES).index( "se_primaryreason-primary reason for discontinuation-death") patient_response = CLINICAL_TENSOR[:, :, RESPONSE_IDX] NpatientsProgressed = np.sum((patient_response.max(1) == 6) * 1.) prog_idx = np.where(patient_response.max(1) == 6)[0] print NpatientsProgressed, 'patients progressed' patient_death = CLINICAL_TENSOR[:, :, DEATH_IDX] NpatientsDied = np.sum((patient_death.max(1) == 1) * 1.) death_idx = np.where(patient_death.max(1) == 1)[0] print NpatientsDied, 'patients died' both_idx = np.array( list(set(prog_idx.tolist()).intersection(set(death_idx.tolist())))) print len(both_idx), ' progressed and died' only_prog = np.array([k for k in prog_idx if k not in both_idx.tolist()]) only_death = np.array([k for k in death_idx if k not in both_idx.tolist()]) print 'Only death: ', len(only_death), ' Only progressed', len(only_prog) np.random.seed(0) train_both, valid_both, test_both = splitRatio(both_idx) train_prog, valid_prog, test_prog = splitRatio(only_prog) train_death, valid_death, test_death = splitRatio(only_death) assert len(only_prog) + len(only_death) + len(both_idx) == len( set(prog_idx.tolist() + death_idx.tolist())), 'Check failed' remaining_idx = np.array( list( set(range(CLINICAL_TENSOR.shape[0])) - set(prog_idx.tolist() + death_idx.tolist()))) train_remain, valid_remain, test_remain = splitRatio(remaining_idx) train_total = np.array(train_both.tolist() + train_prog.tolist() + train_death.tolist() + train_remain.tolist()) valid_total = np.array(valid_both.tolist() + valid_prog.tolist() + valid_death.tolist() + valid_remain.tolist()) test_total = np.array(test_both.tolist() + test_prog.tolist() + test_death.tolist() + test_remain.tolist()) np.random.shuffle(train_total) np.random.shuffle(valid_total) np.random.shuffle(test_total) print train_total.shape[0] + test_total.shape[0] + valid_total.shape[ 0], CLINICAL_TENSOR.shape[0] #Find index under current feature names LABEL_NAMES = np.array([ 'progression', 'progression_days', 'surrogates for overall survival', 'os days', 'last obs days' ]) CLINICAL_Y = np.concatenate([ util.PDlabel(CLINICAL_TENSOR, RESPONSE_IDX).reshape(-1, 1), util.PDdays(CLINICAL_TENSOR, RESPONSE_IDX).reshape(-1, 1), util.deathLabel(CLINICAL_TENSOR, DEATH_IDX).reshape(-1, 1), util.deathDays(CLINICAL_TENSOR, DEATH_IDX).reshape(-1, 1), util.obsDays(CLINICAL_TENSOR).reshape(-1, 1) ], axis=1) #Separate Tensors ## Clinical CLINICAL_TRAIN = CLINICAL_TENSOR[train_total] CLINICAL_TRAIN_Y = CLINICAL_Y[train_total] CLINICAL_TRAIN_MASK = CLINICAL_MASK[train_total] CLINICAL_VALID = CLINICAL_TENSOR[valid_total] CLINICAL_VALID_Y = CLINICAL_Y[valid_total] CLINICAL_VALID_MASK = CLINICAL_MASK[valid_total] CLINICAL_TEST_Y = CLINICAL_Y[test_total] CLINICAL_TEST = CLINICAL_TENSOR[test_total] CLINICAL_TEST_MASK = CLINICAL_MASK[test_total] ## Treatment TREATMENT_TRAIN = TREATMENT_TENSOR[train_total] TREATMENT_TRAIN_MASK = TREATMENT_MASK[train_total] TREATMENT_VALID = TREATMENT_TENSOR[valid_total] TREATMENT_VALID_MASK = TREATMENT_MASK[valid_total] TREATMENT_TEST = TREATMENT_TENSOR[test_total] TREATMENT_TEST_MASK = TREATMENT_MASK[test_total] ## Initial INITIAL_TRAIN = INITIAL_MATRIX[train_total] INITIAL_VALID = INITIAL_MATRIX[valid_total] INITIAL_TEST = INITIAL_MATRIX[test_total] #Normalize all according to train data distributions INITIAL_TRAIN, INITIAL_VALID, INITIAL_TEST = normalize( INITIAL_TRAIN, INITIAL_VALID, INITIAL_TEST, INITIAL_FEATURE_TYPES) CLINICAL_TRAIN, CLINICAL_VALID, CLINICAL_TEST = normalize( CLINICAL_TRAIN, CLINICAL_VALID, CLINICAL_TEST, CLINICAL_FEATURE_TYPES) TREATMENT_TRAIN, TREATMENT_VALID, TREATMENT_TEST = normalize( TREATMENT_TRAIN, TREATMENT_VALID, TREATMENT_TEST, TREATMENT_FEATURE_TYPES) DATASET_SEPARATE = OrderedDict() DATASET_SEPARATE['train_y'] = CLINICAL_TRAIN_Y DATASET_SEPARATE['valid_y'] = CLINICAL_VALID_Y DATASET_SEPARATE['test_y'] = CLINICAL_TEST_Y DATASET_SEPARATE['y_names'] = LABEL_NAMES DATASET_SEPARATE['train_x'] = CLINICAL_TRAIN DATASET_SEPARATE['train_x_mask'] = CLINICAL_TRAIN_MASK DATASET_SEPARATE['valid_x'] = CLINICAL_VALID DATASET_SEPARATE['valid_x_mask'] = CLINICAL_VALID_MASK DATASET_SEPARATE['test_x'] = CLINICAL_TEST DATASET_SEPARATE['test_x_mask'] = CLINICAL_TEST_MASK DATASET_SEPARATE['x_names'] = CLINICAL_FEATURE_NAMES DATASET_SEPARATE['x_types'] = CLINICAL_FEATURE_TYPES DATASET_SEPARATE['train_u'] = TREATMENT_TRAIN DATASET_SEPARATE['train_u_mask'] = TREATMENT_TRAIN_MASK DATASET_SEPARATE['valid_u'] = TREATMENT_VALID DATASET_SEPARATE['valid_u_mask'] = TREATMENT_VALID_MASK DATASET_SEPARATE['test_u'] = TREATMENT_TEST DATASET_SEPARATE['test_u_mask'] = TREATMENT_TEST_MASK DATASET_SEPARATE['u_names'] = TREATMENT_FEATURE_NAMES DATASET_SEPARATE['u_types'] = TREATMENT_FEATURE_TYPES DATASET_SEPARATE['train_init'] = INITIAL_TRAIN[:, 0, :] DATASET_SEPARATE['valid_init'] = INITIAL_VALID[:, 0, :] DATASET_SEPARATE['test_init'] = INITIAL_TEST[:, 0, :] DATASET_SEPARATE['init_names'] = INITIAL_FEATURE_NAMES DATASET_SEPARATE['init_types'] = INITIAL_FEATURE_TYPES saveHDF5(tensor_dir + 'mm_separate.h5', DATASET_SEPARATE) #Combined Tensors print 'WARNING: ASSUMES BINARY TREATMENT!!!' BOTH_TRAIN = np.concatenate([TREATMENT_TRAIN, CLINICAL_TRAIN], axis=2) BOTH_TRAIN_MASK = np.concatenate( [TREATMENT_TRAIN_MASK, CLINICAL_TRAIN_MASK], axis=2) BOTH_VALID = np.concatenate([TREATMENT_VALID, CLINICAL_VALID], axis=2) BOTH_VALID_MASK = np.concatenate( [TREATMENT_VALID_MASK, CLINICAL_VALID_MASK], axis=2) BOTH_TEST = np.concatenate([TREATMENT_TEST, CLINICAL_TEST], axis=2) BOTH_TEST_MASK = np.concatenate([TREATMENT_TEST_MASK, CLINICAL_TEST_MASK], axis=2) BOTH_FEATURE_NAMES = np.array(TREATMENT_FEATURE_NAMES.tolist() + CLINICAL_FEATURE_NAMES.tolist()) BOTH_FEATURE_TYPES = np.array(TREATMENT_FEATURE_TYPES.tolist() + CLINICAL_FEATURE_TYPES.tolist()) DATASET_JOINED = OrderedDict() DATASET_JOINED['train_y'] = CLINICAL_TRAIN_Y DATASET_JOINED['valid_y'] = CLINICAL_VALID_Y DATASET_JOINED['test_y'] = CLINICAL_TEST_Y DATASET_JOINED['y_names'] = LABEL_NAMES DATASET_JOINED['train_x'] = BOTH_TRAIN DATASET_JOINED['train_mask'] = BOTH_TRAIN_MASK DATASET_JOINED['valid_x'] = BOTH_VALID DATASET_JOINED['valid_mask'] = BOTH_VALID_MASK DATASET_JOINED['test_x'] = BOTH_TEST DATASET_JOINED['test_mask'] = BOTH_TEST_MASK DATASET_JOINED['x_names'] = BOTH_FEATURE_NAMES DATASET_JOINED['x_types'] = BOTH_FEATURE_NAMES saveHDF5(tensor_dir + 'mm_joined.h5', DATASET_JOINED)
def load_env(environment): """load("dev") :param environment: """ print "Load {0} environment on box: {1}".format(environment, local_host) load.main(environment)
import load; import math; def dist(ax,ay,bx,by): x2 = (ax-bx)*(ax-bx); y2 = (ay-by)*(ay-by); res =math.sqrt(x2+y2); return(res); def calcQuad(x,y): x_int = int(str(x).split(".")[0]); stry = str(y); '''MAIN CODE ''' locations = load.main(); print(locations); pos = list(); buff = list(); pos = load.getpos(); midx = pos[0]; midy = pos[1]; x0 = midx-0.005; y0 = midy-0.005; x1 = midx+0.005; y1 = midy+0.005;
#!/usr/bin/env python3 # loadaverage execution script # for more info, see github.com/qguv/loadaverage import sys import load sys.exit(load.main())
yy=[xx_min[xindex], xx_max[xindex]] iindex=self.element_indices.index(index) calibr=self.calibrations[iindex] print yy """ xx=[ self.equation(calibr[0], xx_min), self.equation(calibr[0], xx_max) ] """ xx=[ self.equation(calibr[0], self.choose(self.intensities[1])), self.equation(calibr[0], self.choose(self.intensities[0])) ] line = biggles.Curve(xx, yy) line.label = "Calibrating line" #legend = biggles.PlotKey( .7, .9, [points, line] ) #p.add( line, points, legend) p.add( points, line) p.show() if __name__=='__main__': import load load.main()
def confirm(self,fichier): var.lat,var.lon,var.name=load.main(fichier+".asc") print (var.lat,var.lon,var.name)
tmpb=tmpb+tmpa flm.close() flm=open("tmp.txt","w") flm.write(tmpb) flm.close() xf=read_file("tmp.txt") os.remove("tmp.txt") xf.remove("") return xf """ DEBUT DES CLASSES DE L'INTERFACE """ var.lat,var.lon,var.name=load.main("Lidl_Fr"+".asc") class AffPrincipal(BoxLayout): def confirm(self,fichier): var.lat,var.lon,var.name=load.main(fichier+".asc") print (var.lat,var.lon,var.name) def confirmload(self,text): R=(var.name.index(text)) print (var.lat[R], var.lon[R], text) def update(self,*args): self.gridmain.box.boxf.spinaero.text=AffPrincipal.get_spin_f(self,1) self.gridmain.box.boxf.spinaero.values=AffPrincipal.get_spin_f(self,0) def update2(self,*args): self.gridmain.box.boxf.spinaero2.text=AffPrincipal.get_spin_f2(self,1) self.gridmain.box.boxf.spinaero2.values=AffPrincipal.get_spin_f2(self,0)