def expandSide(m, n, dM, dN): # print("\nExpanding Side") #? settings.staticPrefixes = settings.prefixes.copy() settings.currRootsDir = settings.DATA_FOLDER / "sideRoots" settings.currOldRootsDir = settings.DATA_FOLDER / "sideOldRoots" #initialize the root roots with path len 1 util.emptyDir(settings.DATA_FOLDER / "sideRoots") for x in range(m + 1, m + dM + 1): util.store(set([(x, )]), settings.DATA_FOLDER / f"sideRoots/{(x,)}.dat") settings.prefixes.add((x, )) util.store(settings.prefixes, settings.DATA_FOLDER / "prefixes.dat") for d in range(2, n + dN): expandSideLayer(d, m, dM) #take resulting side roots and add them to main roots files = os.listdir(settings.DATA_FOLDER / "sideRoots") for f in files: try: oldRoots = util.load(settings.DATA_FOLDER / f"roots/{f}") newRoots = util.load(settings.DATA_FOLDER / f"sideRoots/{f}") oldRoots.update(newRoots) combRoots = oldRoots util.store(combRoots, settings.DATA_FOLDER / f"roots/{f}") except OSError: os.rename(settings.DATA_FOLDER / f"sideRoots/{f}", settings.DATA_FOLDER / f"roots/{f}")
def load_dataset(fname, nb_lines): """Load the Amazon dataset if not already present on disc""" import os.path if os.path.isfile('safe/Amazon-'+str(nb_lines)+'.p'): return util.load('safe/Amazon-'+str(nb_lines)+'.p') count = 1 X = [] y = [] with open(fname) as f: for line in f: text, label = read_line(line) #print((label, text)) X.append(text) y.append(label) if count >= nb_lines: break count+=1 #load pretrained dictonary dico = util.load('safe/vocab_gensim.p') preprocessor = text_preprocessing.Preprocessor(dico=dico) X = preprocessor.preprocess(X) #save the loaded dataset in a pickle for speeding up next run util.save((X,y), 'safe/Amazon-'+str(nb_lines)+'.p') return (X, y)
def create_reference_table(): words = util.load(words_path) times = np.array(util.load(data_path)) pos = util.load(pos_path) table = collections.defaultdict(dict) count = collections.defaultdict(dict) for w, t, p in zip(words, times, pos): if w == "": continue l = len(w) if p not in table or l not in table[p]: table[p][l] = t count[p][l] = 1 else: table[p][l] += t count[p][l] += 1 for p in table: for l in table[p]: table[p][l] = table[p][l] / float(count[p][l]) print(table) util.save(table, 'pos_len_table')
def load(self, file_name): params_dict = {} util.load(file_name, params_dict) if not hasattr(self, '_lst_layers'): self._lst_layer_type, self._lst_num_hid, self._data_dim = \ params_dict['lst_layer_type'], \ params_dict['lst_num_hid'], \ params_dict['data_dim'] logging.info("Creating new layers from parameters in file: %s" % file_name) self._lst_layers = [] for (layer_name, layer_type) in zip(params_dict['lst_layer_names'], self._lst_layer_type): layer = create_empty_nnet_layer(layer_name, layer_type) layer.copy_params_from_dict(params_dict) self._lst_layers.append(layer) else: logging.info("Updating layer parameters using file: %s" % file_name) for layer_num, layer in enumerate(self._lst_layers): if params_dict.has_key(layer.name + "_wts") and \ layer_num < len(params_dict['lst_layer_type']): layer.copy_params_from_dict(params_dict) self.num_layers = len(self._lst_layers)
def codeFile(args,flag,data): PARAM_KEY = 1; PARAM_FILE = 2; # Output file location PARAM_FORMATTER = 3 ARGUMENTS = len(args)-1 # Ability to add a block of code through copy and paste and have it formatted correctly! if( keyExists("files",args[PARAM_KEY])): _file = json.loads(load("files/"+args[PARAM_KEY])); out = '' # loadJSON for x in _file: block = str(load("blocks/"+ x)) if(ARGUMENTS == PARAM_FORMATTER): # Alter all the blocks in said fashion block = format.block(block, args[PARAM_FORMATTER]) out += block out += "\n" # Adds some spacing between blocks # No file specified if(len(args) < 3 ): log(out) else: log("Saving to file "+ args[PARAM_FILE] ) save(args[PARAM_FILE],out) else: error("Error: File does not exist")
def openLoadFileDialog(self): fname = QtGui.QFileDialog.getOpenFileName(self, 'Open equation', '', "All (*.*)") if fname: load(fname, self, self.dialogUI) self.cloneOptionsMapInfo() self.solveButtonTrigger.emit()
def load_map(G, tiles='config/tiles.yml', borders='config/borders.yml'): tiles = load(tiles) borders = load(borders) for b in borders: n1, n2 = b.tile1, b.tile2 t = b.type if 'borders' not in tiles[n1]: tiles[n1].borders = tdict() tiles[n1].borders[n2] = t if 'borders' not in tiles[n2]: tiles[n2].borders = tdict() tiles[n2].borders[n1] = t G.tiles = tdict({name: idict(tile) for name, tile in tiles.items()}) for name, tile in G.tiles.items(): tile.__dict__['_id'] = name # tile.name = name tile.units = tset() if tile.type not in {'Sea', 'Ocean', 'Strait'}: for neighbor in tile.borders.keys(): if G.tiles[neighbor].type == 'Sea' or G.tiles[ neighbor].type == 'Ocean': tile.type = 'Coast' break # add tile to game objects tile.obj_type = 'tile' tile.visible = tset({'Axis', 'West', 'USSR'}) G.objects.table[name] = tile
def remove_player(self, world_name, name): print("Removing player", name) util.load(self.world_dir(world_name)) if name in util.all_dict: del util.all_dict[name] util.write() return self.players(world_name)
def codeProject(args,flag,data): PARAM_KEY = 1 PARAM_PATH = 2 PARAM_FORMATTER = 3 ARGUMENTS = len(args)-1 # JSON mapping files and storage of this if( keyExists("projects",args[1])): if( "stdout" in args[2]): project = json.loads(load("projects/"+args[PARAM_KEY])); # Uses key value storage directory = args[PARAM_PATH] + "/" + args[PARAM_KEY] mkdir(directory) for x in project.keys(): # Reflect that with here _file = json.loads(load("files/"+x)); out = ''; for y in _file: block = str(load("blocks/"+ y)) if(ARGUMENTS == PARAM_FORMATTER): # Alter all the blocks in said fashion block = format.block(block, args[PARAM_FORMATTER]) out += block # Output the file with the correct file name save(directory + "/" + project[x],out) else: error("Error: Project does not exist")
def load_map(G, tiles='config/tiles.yml', borders='config/borders.yml'): tiles = load(tiles) borders = load(borders) for b in borders: n1, n2 = b.tile1, b.tile2 t = b.type if 'borders' not in tiles[n1]: tiles[n1].borders = tdict() tiles[n1].borders[n2] = t if 'borders' not in tiles[n2]: tiles[n2].borders = tdict() tiles[n2].borders[n1] = t G.tiles = tiles for name, tile in G.tiles.items(): tile.name = name tile.units = tset() if tile.type != 'Sea' and tile.type != 'Ocean': for neighbor in tile.borders.keys(): if G.tiles[neighbor].type == 'Sea' or G.tiles[ neighbor].type == 'Ocean': tile.type = 'Coast' break # add tile to game objects tile.obj_type = 'tile' G.objects.table[name] = tile
def load_params(self, param_file): targetDict = {} util.load(param_file, targetDict, verbose=False) self.cmW = cm.CUDAMatrix(cm.reformat(targetDict['W'])) self.cmBiasesHid = cm.CUDAMatrix(cm.reformat(targetDict['biasesHid'])) self.cmBiasesVis = cm.CUDAMatrix(cm.reformat(targetDict['biasesVis'])) self.input_dim, self.num_units = self.cmW.shape
def main(): #previous mxn completed m, n = util.load(settings.DATA_FOLDER / "mXn.dat") settings.prefixes = util.load(settings.DATA_FOLDER / "prefixes.dat") startM = m startN = n firstST = time.time() #m and n are prev m and n expanded to while m < settings.MAX_M or n < settings.MAX_N: #dM and dN are how much to expand m and n by respectively dM = min(settings.DELTA_M, settings.MAX_M - m) dN = min(settings.DELTA_N, settings.MAX_N - n) print(f"\nExpanding from {m}X{n} to {m+dM}X{n+dN}") sT = time.time() #expand sideways by dM expand.expandSide(m, n, dM, dN) sideTime = time.time() print(f"Side time: {sideTime - sT}s") util.emptyDir(settings.DATA_FOLDER / "parents") util.emptyDir(settings.DATA_FOLDER / "oldRoots") #expand down by dN expand.expandDown(m, n, dM, dN) print(f"Down time: {time.time() - sideTime}s") endT = time.time() m += dM n += dN #load all evens just for us to check if it's working properly # print("genning all evens") allEvens = set() for x in range(1, n + 1): eX = util.load(settings.EVENS_FOLDER / f"evens{x}.dat") allEvens.update(eX) print(f"{m}X{n} total evens: {len(allEvens)}\t in {str(endT-sT)}s") if settings.printEvens: print(str(m) + "X" + str(n) + " evens: " + str(allEvens)) # print(f"size of all evens: {sys.getsizeof(allEvens)}") # print(f"Deep allEvens objSize: {get_deep_size(allEvens)}") # print() #store the m and n completed, evens are stored in side and down expand util.store((m, n), settings.DATA_FOLDER / "mXn.dat") util.store(settings.prefixes, settings.DATA_FOLDER / "prefixes.dat") print( f"\n\nTotal run time for {startM}X{startN} to {m}X{n}: {time.time() - firstST}s " )
def _buildmeta(ui, repo, args, partial=False, skipuuid=False): if repo is None: raise error.RepoError("There is no Mercurial repository" " here (.hg not found)") dest = None validateuuid = False if len(args) == 1: dest = args[0] validateuuid = True elif len(args) > 1: raise error.Abort('rebuildmeta takes 1 or no arguments') url = repo.ui.expandpath(dest or repo.ui.config('paths', 'default-push') or repo.ui.config('paths', 'default') or '') meta = svnmeta.SVNMeta(repo, skiperrorcheck=True) svn = None if meta.subdir is None: svn = svnrepo.svnremoterepo(ui, url).svn meta.subdir = svn.subdir youngest = 0 startrev = 0 branchinfo = {} if not partial: hgutil.unlinkpath(meta.revmap_file, ignoremissing=True) revmap = meta.revmap if partial: try: # we can't use meta.lastpulled here because we are bootstraping the # lastpulled and want to keep the cached value on disk during a # partial rebuild foundpartialinfo = False youngestpath = os.path.join(meta.metapath, 'lastpulled') if os.path.exists(youngestpath): youngest = util.load(youngestpath) lasthash = revmap.lasthash if len(revmap) > 0 and lasthash: startrev = repo[lasthash].rev() + 1 branchinfo = util.load(meta.branch_info_file) foundpartialinfo = True if not foundpartialinfo: ui.status('missing some metadata -- doing a full rebuild\n') partial = False except IOError, err: if err.errno != errno.ENOENT: raise ui.status('missing some metadata -- doing a full rebuild\n') except AttributeError: ui.status('no metadata available -- doing a full rebuild\n')
def copy_params_from_single_prediction_net(self, file_name): params_dict = {} util.load(file_name, params_dict) sys.stderr.write("Updating layer parameters from single time " + \ "nnet using file: %s\n"%file_name) logging.info("Updating layer parameters from single time nnet "+ \ " using file: %s"%file_name) for layer_num in range(self.num_layers - 1): sys.stderr.write("Updating layer # %d\n" % layer_num) sys.stderr.flush() logging.info("Updating layer # %d" % layer_num) self._lst_layers[layer_num].copy_params_from_dict(params_dict)
def greedy(stage=1): p = None m = DataParallel(ModelStack(1)).to(o.device) if stage > 1: p = DataParallel(ModelStack(stage - 1)).to(o.device) load(p, torch.load("save/01-10g.tar")) p.stage = stage - 1 train(m, p) # concat and save a = change_key(m.module.m[0].state_dict(), lambda x: f"m.{stage-1}." + x) if p: a.update(p.module.state_dict()) torch.save(a, f"save/01-10g.tar")
def _buildmeta(ui, repo, args, partial=False, skipuuid=False): if repo is None: raise error.RepoError("There is no Mercurial repository" " here (.hg not found)") dest = None validateuuid = False if len(args) == 1: dest = args[0] validateuuid = True elif len(args) > 1: raise hgutil.Abort('rebuildmeta takes 1 or no arguments') url = repo.ui.expandpath(dest or repo.ui.config('paths', 'default-push') or repo.ui.config('paths', 'default') or '') meta = svnmeta.SVNMeta(repo, skiperrorcheck=True) svn = None if meta.subdir is None: svn = svnrepo.svnremoterepo(ui, url).svn meta.subdir = svn.subdir youngest = 0 startrev = 0 sofar = [] branchinfo = {} if partial: try: # we can't use meta.lastpulled here because we are bootstraping the # lastpulled and want to keep the cached value on disk during a # partial rebuild foundpartialinfo = False youngestpath = os.path.join(meta.metapath, 'lastpulled') if os.path.exists(youngestpath): youngest = util.load(youngestpath) sofar = list(maps.RevMap.readmapfile(meta.revmap_file)) if sofar and len(sofar[-1].split(' ', 2)) > 1: lasthash = sofar[-1].split(' ', 2)[1] startrev = repo[lasthash].rev() + 1 branchinfo = util.load(meta.branch_info_file) foundpartialinfo = True if not foundpartialinfo: ui.status('missing some metadata -- doing a full rebuild\n') partial = False except IOError, err: if err.errno != errno.ENOENT: raise ui.status('missing some metadata -- doing a full rebuild\n') except AttributeError: ui.status('no metadata available -- doing a full rebuild\n')
def __init__(self, model_path): self.word_to_id = util.load(model_path + "_w2i.pkl") self.id_to_word = util.load(model_path + "_i2w.pkl") vocab_size = len(self.id_to_word) model = EncoderDecoder(encoder_layers=lstm_layers, decoder_layers=lstm_layers, input_vocab_size=vocab_size, output_vocab_size=vocab_size, embed_size=embed_size, hidden_size=hidden_size, dropout=dropout, ARR=np) chainer.serializers.load_npz(model_path, model) self.model = model
def __init__(self, game): ''' Constructor (game) -> Editor_menu ''' self.menu_background_surf = util.load('editor/menu_button.png') self.make_options_surf = util.load('editor/make_options.png') self.selection_surf = util.load('editor/selection.png') self.editor_menu_option = util.load('editor/editor_menu_options.png') self.world_name_dialog = util.load('editor/world_name_dialog.png') # ----------- Select rect -- credits - http://stackoverflow.com/questions/6339057/draw-a-transparent-rectangle-in-pygame self.select_rect_surf = pygame.Surface((220, 64)) self.select_rect_surf.set_alpha(150) self.select_rect_surf.fill((50, 255, 50)) self.make_option = 0 # 4 = collidable self.make_option_x = -991 self.option_specific = 1 # Starts with plain block making # Make the white collumn self.white_col = pygame.Surface((360, 150)) self.white_col.set_alpha(128) self.white_col.fill((255, 255, 255)) # idea self.game.block.append(Block(0, 0, pygame.Color(0,0,0))) # self.block_ctor(x, y, surface, **{k: v.current for k, v in self.variables.items()}) **{'jump': 2.5} # http://stackoverflow.com/questions/334655/passing-a-dictionary-to-a-function-in-python-as-keyword-parameters self.collidable = Environment_editor( [game.block], 'Collidable', game.collidable_images, Collidable_block, [('jump', Block_variable(1, .1, 4, .1, 'Jump')), ('ground_friction', Block_variable(1, .01, 2, .1, 'Ground Friction')), ('wall_friction', Block_variable(1, .01, 2, .1, 'Wall Friction')), ('wall_jump_up', Block_variable(1, .01, 2, .1, 'Wall Jump Up')), ('wall_jump_sideways', Block_variable(1, .01, 2, .1, 'Wall Jump Sideways'))]) self.scenery = Environment_editor( [game.scenery_background, game.scenery_foreground], 'scenery/still', game.scenery_images, Scenery_block, [('rotation', Block_variable(0, 0, 359, -1, 'Rotation', True))]) self.brush_tuple = (self.collidable, self.scenery ) # Tuple with different brushes try: self.myfont = pygame.font.SysFont("segoeuihistoric", 25) except: self.myfont = pygame.font.SysFont(None, 25) # For saving and quitting self.menu_open = False
def load(self, use_deltas_accs): self.use_deltas_accs = use_deltas_accs if os.path.isfile(self.SummaryFilePath) == True: print "Loading DB indices from file: ", self.SummaryFilePath params_dict = {} util.load(self.SummaryFilePath, params_dict, verbose=False) try: self.label_dim = int(params_dict['label_dim']) except AttributeError: print "No label_dim in file" try: self.UtteranceIds = params_dict['UtteranceIds'] except KeyError: print "No UtteranceIds in index file." self.RawFileList = params_dict['RawFileList'] self.data_dim = params_dict['data_dim'] try: self.NumFrames = params_dict['NumFrames'] except KeyError: self.NumFrames = 0 self.DataMeanVect = params_dict['DataMeanVect'].reshape(-1,1) self.DataStdVect = params_dict['DataStdVect'].reshape(-1,1) self.Utt2Speaker = params_dict['Utt2Speaker'] self.Speaker2Utt = params_dict['Speaker2Utt'] self.SpeakerMeans = params_dict['SpeakerMeans'] self.SpeakerStds = params_dict['SpeakerStds'] self._lst_ignored_files = params_dict['lst_ignored_files'] if not self.use_deltas_accs: self.data_dim /= 3 self.DataMeanVect = self.DataMeanVect[:self.data_dim,] self.DataStdVect = self.DataStdVect[:self.data_dim,] if self.SpeakerMeans is not None: for speaker in self.SpeakerMeans.keys(): self.SpeakerMeans[speaker] = \ self.SpeakerMeans[speaker][:self.data_dim,] self.SpeakerStds[speaker] = \ self.SpeakerStds[speaker][:self.data_dim,] self.LoadAligments(self.AliFile)
def MDSPlotTest(): import json import experiment resPath = "../experiments/ebook_color_pca_3" experiment.experimentCase("../params/ebook_color_pca_28x28_3.json",resPath) info = json.loads(util.fileString("../params/ebook_color_pca_28x28_3.json")) info = util.dotdict(info) x = util.load(resPath+"/x.pkl") print x.dtype compressed = util.load(resPath+"/compressed.pkl") MDSPlots(x,compressed,info.dataSet.shape) import matplotlib.pyplot as plt fig.savefig() print("show figure") plt.show()
def add_player(self, world_name, name, ignoresPlayerLimit, permission): print("Adding player", name) util.load(self.world_dir(world_name)) player = { "name": name, "ignoresPlayerLimit": ignoresPlayerLimit, "xuid": "" } if permission != "default": player["permission"] = permission util.all_dict[name] = player util.write() return self.players(world_name)
def get_patient_info(pid): # initialize postgres connection con = psycopg2.connect(database='mimic', user='******', host='localhost', password='******') # retrive comorbidity scores sql_query = """ SELECT * FROM elixhauser_ahrq WHERE SUBJECT_ID = %d ; """ % (pid) comorb = pd.read_sql_query(sql_query, con) # limit to first admission hadms = load("lists/adults_heart_discharged.csv", cols=['hadm_id'])['hadm_id'].tolist() comorb = comorb[comorb['hadm_id'].isin(hadms)] # only display comorbidity = 1 items positives = comorb.loc[:, (comorb != 0).any(axis=0)] col_names = positives.drop(['subject_id', 'hadm_id'], axis=1).columns.values return str(col_names.tolist())[1:-1]
def test_tour(par=1): ip.reset(par) scene = 'disrupt-11' scan = dset.Scan('../data/%s' % scene) texel_colors = ut.load(ut.pjoin(figures.make_path('noloo', 'interior-wide', scene), 'data.pk'))['ret'][0] mesh = box.load_from_mat(ut.pjoin(scan.path, 'cube.mat')) tour(scan, mesh, texel_colors, [0, 1, 2], plane_idx = 0, outline_start = 0, par = par)
def sgd_optimize(learning_rate=0.1, pretrain_learning_rate=0.001, pretrain_epochs=15, finetune_epochs=1000, batch_size=20): # Load datasets train, valid, test = util.load() print "loading 0 - ", train[0].shape[0], " train inputs in gpu memory" train_x, train_y = util.create_theano_shared(train) print "loading 0 - ", valid[0].shape[0], " validation inputs in gpu memory" valid_x, valid_y = util.create_theano_shared(valid) print "loading 0 - ", test[0].shape[0], " test inputs in gpu memory" test_x, test_y = util.create_theano_shared(test) n_train_batches = train[0].shape[0] / batch_size n_valid_batches = valid[0].shape[0] / batch_size n_test_batches = test[0].shape[0] / batch_size random_generator = numpy.random.RandomState(1) print "...Building model" sd = StackedDenoisingAutoEncoders(random_generator, hidden_layer_sizes=[1000, 1000, 1000]) print "...Getting pretrain functions" pretrain_fns = sd.pretrain(train_x, batch_size) ############# # Pretrain ############ print "... Pre-training model" start_time = time.clock() ## Pre-train layer-wise corruption_levels = [.1, .2, .3] for i in range(sd.n_layers): for epoch in range(pretrain_epochs): c = [] for batch_index in xrange(n_train_batches): c.append(pretrain_fns[i](index=batch_index, corruption_level=corruption_levels[i], learning_rate=pretrain_learning_rate)) print 'Pre-training layer %i, epoch %d, cost ' % (i, epoch), print numpy.mean(c) end_time = time.clock() print "Pretraining code ran for %.2fm" % (end_time - start_time) ############# # Finetune ############ print "...Fine-tuning model" train_model, valid_model, test_model = sd.finetune(train_x, train_y, valid_x, valid_y, test_x, test_y, batch_size, learning_rate) util.train_test_model(finetune_epochs, train_model, valid_model, test_model, n_train_batches, n_valid_batches, n_test_batches)
def _set_subdir(self, subdir): subdir = util.forceutf8(subdir) if subdir: subdir = '/'.join(p for p in subdir.split('/') if p) self.__subdir = None subdirfile = os.path.join(self.metapath, 'subdir') if os.path.isfile(subdirfile): stored_subdir = util.load(subdirfile) assert stored_subdir is not None if subdir is None: self.__subdir = stored_subdir elif subdir and subdir != stored_subdir: raise hgerror.Abort( 'unable to work on a different path in the ' 'repository') else: self.__subdir = subdir elif subdir is not None: util.dump(subdir, subdirfile) self.__subdir = subdir elif not self._skiperror: raise hgerror.Abort("hgsubversion metadata unavailable; " "please run 'hg svn rebuildmeta'")
def predict(data_json, model_path): preproc = util.load(os.path.dirname(model_path)) dataset = load.load_dataset(data_json) x, y = preproc.process(*dataset) y_test = [] for e, i in enumerate(dataset[1]): for j in range(len(i)): y_test.append(y[e, j, :]) y_result = np.array(y_test) model = keras.models.load_model(model_path) probs = model.predict(x, verbose=1) #update start y_test = [] y_predict = [] for e, i in enumerate(dataset[1]): for j in range(len(i)): y_test.append(y[e, j, :]) y_predict.append(probs[e, j, :]) y_test = np.array(y_test) y_predict = np.array(y_predict) #update stop return y_test, y_predict
def retrieve_notes_sql(): con = psycopg2.connect(database='mimic', user='******', host='localhost', password='******') # load list of first icus icu_list = load("notes/adults_heart_discharged.csv", cols=['subject_id', 'hadm_id']) adm_id = icu_list['hadm_id'].tolist() # then find the first icu stay not in above list query_list = "("+str(adm_id)[1:-1]+")" # get all relevant note during first stay sql_query = """ SELECT SUBJECT_ID,hadm_id, chartdate, charttime,text FROM mimiciii.noteevents WHERE (hadm_id IN %s AND category != 'Discharge summary') ORDER BY SUBJECT_ID, chartdate, charttime ; """ % query_list notes = pd.read_sql_query(sql_query, con) notes.to_csv("notes/notes.csv", index=False) # print notes.shape return True
def consistencyCheck(ref_csv, outputBshellFile=None, outPutResolutionFile=None): try: ref_imgs, _ = read_imgs_masks(ref_csv) except: ref_imgs = read_imgs(ref_csv) if isfile(outputBshellFile) and isfile(outPutResolutionFile): ref_bvals = read_bvals(outputBshellFile) ref_res = np.load(outPutResolutionFile) else: ref_bshell_img = ref_imgs[0] print(f'Using {ref_bshell_img} to determine b-shells') inPrefix = abspath(ref_bshell_img).split('.nii')[0] ref_bvals = findBShells(inPrefix + '.bval', outputBshellFile) ref_res = load(ref_bshell_img).header['pixdim'][1:4] np.save(outPutResolutionFile, ref_res) print('b-shells are', ref_bvals) print('\nSite', ref_csv, '\n') print('Checking consistency of b-shells among subjects') check_bshells(ref_imgs, ref_bvals) print('spatial resolution is', ref_res) print('Checking consistency of spatial resolution among subjects') check_resolution(ref_imgs, ref_res)
def setup_pre_phase(G, player_setup_path='config/faction_setup.yml'): player_setup = load(player_setup_path) # place fixed units for name, config in player_setup.items(): if 'units' not in config.setup: continue for unit in config.setup.units: add_unit(G, unit) # prep temp info G.temp = tdict() for name, faction in player_setup.items(): out = tdict() out.player = name if 'cadres' in faction.setup: out.info = faction.setup.cadres out.msg = 'Choose this many cadres to place into each of these territories' else: out.msg = 'Wait while other players place their cadres'
def main(args, cgi_args): if not (cgi_args.has_key("s") or cgi_args.has_key("w") or cgi_args.has_key("v") or cgi_args.has_key("d")): jsond() print json.dumps("No search terms provided.") return lexicon = cgi_args.getfirst("lexicon", None) if lexicon is None: lexicon = cgi_args.getfirst("l", None) if lexicon is None: jsond() print json.dumps("No lexicon provided.") return lex = util.db.lexicon(lexicon) if lex is None: jsond() print json.dumps("Invalid lexicon '%s'" % lexicon) if cgi_args.has_key("d"): return load(args, cgi_args, lex) elif cgi_args.has_key("v"): return save(args, cgi_args, lex) elif cgi_args.has_key("s"): return search(args, cgi_args, lex) elif cgi_args.has_key("w"): return challenge(args, cgi_args, lex) else: jsond() print json.dumps("Nothing.")
def main(): # establish postgresql connection con = psycopg2.connect(database='mimic', user='******', host='localhost', password='******') # 1. extract and export list of adults adults = extract_adults(con) adults.to_csv("lists/adults_admitted.csv", index=False, columns=['subject_id', 'combined_dod', 'outtime', 'age']) # ---- # 2. extract patients with cardiovascular conditions adults = load("lists/adults_admitted.csv") adults_list = set(adults['subject_id'].tolist()) heart_patients = generate_heart_patients(con, adults_list) heart_patients.to_csv("lists/heart_patients.csv", index=False, header=True) adults_heart = pd.merge(adults, heart_patients, on=['subject_id'], how='inner') # 3. then trim down the list to patients who were discharged alive discharged = lived(adults_heart) discharged.to_csv("lists/adults_heart_discharged.csv", index=False) # discharged = load("lists/adults_heart_discharged.csv") # 4. from those discharged patients, find the ones with a second # admission and calculate the data difference between first and second readmitted = readmission_diff(con, discharged['subject_id'].tolist()) readmitted.to_csv("lists/readmission_diff.csv", index=False) # 5. generate icu id's of first visits (could be used for awk) generate_icu_id(con, "lists/readmission_diff.csv", "lists/first_icu_list.txt")
def _load(self, resource): """ Load the requested resource from source. Valid resources are: "profile" player profiles "session" user sessions Parameters ---------- resource: str The resource requested to be loaded """ if self.backend == "aws": table = getattr(self, resource + "_table") return_dict = {} for item in table.scan()['Items']: # Even though numeric keys have the weird type Decimal('123') # they still compare fine when we try to access their values key = item.pop(RESOURCE_KEY_NAMES[resource]) return_dict[key] = item return return_dict elif self.backend == "file": try: return util.load(resource + '.pkl.gzip') except FileNotFoundError: return {} else: raise ValueError("Backend must be either aws or file")
def generate_features(provided_list=None): con = psycopg2.connect(database='mimic', user='******', host='localhost', password='******') # load list of first ham id first_icu = load("lists/adults_heart_discharged.csv", cols=['subject_id', 'hadm_id', 'icustay_id']).\ sort_values(by=['subject_id']) # if a patient list if provided, then only look at those patients if provided_list is not None: first_icu = first_icu[first_icu['subject_id'].isin(provided_list)] hids = set(first_icu['hadm_id'].tolist()) icu_list = first_icu['icustay_id'].tolist() p_list = first_icu['subject_id'].tolist() patients = first_icu['subject_id'] # get comorbidity scores comorb = comorb_scores(con, hids) # get oasis scores etc phys_scores = generate_phys_scores(con, icu_list, p_list) combined_scores = np.concatenate((np.asmatrix(comorb), phys_scores), axis=1) np.savetxt("X.csv", combined_scores, delimiter=",") return patients, combined_scores
def main(train_set, test_set, iter=2): split("yelp_cat.csv") iter = int(iter) X_train, X_test, Y_train, Y_test = load(train_set, test_set) p = perceptronAverage(iter, X_train, Y_train) p.train() print "ZERO-ONE LOSS=" + str(p.test(X_test,Y_test))
def load_ECG_model(): # load the model from disk filename='./model/classify_model.hdf5' preproc = util.load(os.path.dirname(filename)) loaded_model = keras.models.load_model(filename) return loaded_model, preproc
def load_game_info(G, seed=None, path='config/game_info.yml'): info = load(path) game = tdict() game.seed = seed G.random = random.Random(seed) # G.random = TestRandom(seed) game.year = info.first_year - 1 # zero based game.last_year = info.last_year num_rounds = game.last_year - game.year game.turn_order_options = info.turn_order_options game.sequence = ['Setup'] + num_rounds * info.year_order + ['Scoring'] game.index = 0 # start below 0, so after increment in next_phase() it starts at 0 #game.action_phases = tset(x for x in info.phases if info.phases[x]) # no need for action phases anymore (all action phases have a pre phase) game.peace_dividends = tlist( sum([[v] * n for v, n in info.peace_dividends.items()], [])) G.random.shuffle(game.peace_dividends) game.victory = info.victory G.game = game G.objects = tdict() G.objects.table = tdict()
def load_scenario2(path): # load from input file, or most recent checkpoint (more safe) data = load(path) #print(data) #tos = convert_from_saveable(data) #ftos = convert_to_saveable(tos) #print(ftos) return data
def generate_features(provided_list=None): con = psycopg2.connect(database='mimic', user='******', host='localhost', password='******') # load list of first ham id first_icu = load("lists/adults_heart_discharged.csv", cols=['subject_id', 'hadm_id', 'icustay_id']).sort_values(by=['subject_id']) # if a patient list if provided, then only look at those patients if provided_list is not None: first_icu = first_icu[first_icu['subject_id'].isin(provided_list)] hids = set(first_icu['hadm_id'].tolist()) icu_list = first_icu['icustay_id'].tolist() p_list = first_icu['subject_id'].tolist() patients = first_icu['subject_id'] # get comorbidity scores comorb = comorb_scores(con, hids) # get oasis scores etc phys_scores = get_phys_scores(con, icu_list, p_list) combined_scores = np.concatenate((np.asmatrix(comorb), phys_scores), axis=1) # below line is commented out because files with no # write-protection should not be used for multi-user # application !! # np.savetxt("X.csv", combined_scores, delimiter=",") return patients, combined_scores
def check_resolution(ref_imgs, ref_res): unmatched = [] for imgPath in ref_imgs: imgPath = local.path(imgPath) if not imgPath.exists(): FileNotFoundError(imgPath) res = load(imgPath._path).header['pixdim'][1:4] if (res - ref_res).sum() <= 10e-6: print('spatial resolution matched for', imgPath.name) else: print(f'\nUnmatched spatial resolution for {imgPath.name}') print(res) print(f'ref_res {ref_res}\n') unmatched.append(imgPath._path) print('') if len(unmatched): print('Unmatched cases:') print(unmatched) raise ValueError( 'Leave out the unmatched cases or change the reference case for determining spatial resolution to run multi-shell-dMRIharmonization' ) else: print( 'All cases have same spatial resolution. Data is good for running multi-shell-dMRIharmonization' ) print('')
def get_patient_scores(pid): # initialize postgres connection con = psycopg2.connect(database='mimic', user='******', host='localhost', password='******') stay_ids = load("lists/adults_heart_discharged.csv", cols=['icustay_id'])['icustay_id'].tolist() # retrive from database all relevant scores sql_query = """ SELECT * FROM mimiciii.oasis WHERE SUBJECT_ID = %d ; """ % (pid) oasis = pd.read_sql_query(sql_query, con) oasis = oasis[oasis['icustay_id'].isin(stay_ids)] sql_query = """ SELECT * FROM mimiciii.sofa WHERE SUBJECT_ID = %d ; """ % (pid) sofa = pd.read_sql_query(sql_query, con) oasis = oasis[oasis['icustay_id'].isin(stay_ids)] sql_query = """ SELECT * FROM mimiciii.sapsii WHERE SUBJECT_ID = %d ; """ % (pid) sapsii = pd.read_sql_query(sql_query, con) sapsii = sapsii[sapsii['icustay_id'].isin(stay_ids)] sql_query = """ SELECT * FROM mimiciii.sapsii_last WHERE SUBJECT_ID = %d ; """ % (pid) sapsii_last = pd.read_sql_query(sql_query, con) sapsii_last = sapsii_last[sapsii_last['icustay_id'].isin(stay_ids)] # combine all scores data = [oasis.iloc[0][2], sofa.iloc[0][2], sapsii.iloc[0][2], sapsii_last.iloc[0][2]] # TODO: make a plot and display in html ''' #barplot=ax.bar([0,1,2,3],data,0.6,color=['grey','white','grey','white']) #names = ax.set_xticklabels(['severity illness score', # 'organ failure assessment', # 'acute physiology score', # 'acute physiology score(last)']) #ax.set_xticks([0,1,2,3]) #ax.set_xlim(-0.3,3.8) #plt.gcf().subplots_adjust(bottom=0.25) #plt.setp(names,rotation=30,fontsize=13) #savefig("predict/fig.png") ''' return str(data)[1:-1]
def saveModelImages(modelPath,dstPath,color = False): info,sda = util.load(modelPath) import train x = train.createDataSet(info["dataSet"]).get_value(borrow=True) for name,img in createSdaImages(sda,x,color): dst = dstPath + "/" + name util.ensurePathExists(dst) img.save(dst)
def saveTest(): data = [0,1,2,3,4] name = "test.pkl" util.save(data,name) data2 = util.load(name) print data, data2 return data == data2
def predict(data_json, model_path): preproc = util.load(os.path.dirname(model_path)) dataset = load.load_dataset(data_json) x, y = preproc.process(*dataset) model = keras.models.load_model(model_path) probs = model.predict(x, verbose=1) return probs
def read_test_data(file): # assume if one is saved they all are if util.check_file_exists(CONST.DATASET_PATH + CONST.TEST_PATH): T_Data = util.load(CONST.DATASET_PATH + CONST.TEST_PATH) T_Labels = util.load(CONST.DATASET_PATH + CONST.TEST_PATH_LABELS) T_Queries = util.load(CONST.DATASET_PATH + CONST.TEST_PATH_Q) T_Docs = util.load(CONST.DATASET_PATH + CONST.TEST_PATH_DOCS) else: T_Data, T_Labels, T_Queries, T_Docs = read_train_data(file) util.save_pickle(CONST.DATASET_PATH + CONST.TEST_PATH, T_Data) util.save_pickle(CONST.DATASET_PATH + CONST.TEST_PATH_LABELS, T_Labels) util.save_pickle(CONST.DATASET_PATH + CONST.TEST_PATH_Q, T_Queries) util.save_pickle(CONST.DATASET_PATH + CONST.TEST_PATH_DOCS, T_Docs) return T_Data, T_Labels, T_Queries, T_Docs
def get_resource(resource_name): if RESOURCE_TO_ID.has_key(resource_name): return DATA_MAP[RESOURCE_TO_ID[resource_name]] resource_data = load(resource_name) resurouce_id = RESOURCE_COUNTER RESOURCE_COUNTER += 1 RESOURCE_TO_ID[resource_name] = resurouce_id DATA_MAP[resurouce_id] = resource_data return resource_data
def createFile(_file, formating, formatFlag): out = "" # Output string for x in _file: block = str(load("blocks/"+ x)) if(formatFlag): # Alter all the blocks in said fashion block = format.block(block, formating) out += block out += "\n" # Adds some spacing between blocks return out
def get_phys_scores(con, icu_list, p_list): oasis = get_sql(con, "subject_id,icustay_id, oasis", "mimiciii.oasis") oasis = oasis[oasis['icustay_id'].isin(icu_list)].sort_values( by=['subject_id']) oasis_last = get_sql(con, "subject_id,icustay_id, oasis", "mimiciii.oasis_last") oasis_last = oasis_last[oasis_last['icustay_id']. isin(icu_list)].sort_values(by=['subject_id']) saps = get_sql(con, 'subject_id,icustay_id, saps', 'mimiciii.saps') saps = saps[saps['icustay_id'].isin(icu_list)].sort_values( by=['subject_id']) sapsii = get_sql(con, 'subject_id,icustay_id, sapsii', 'mimiciii.sapsii') sapsii = sapsii[sapsii['icustay_id']. isin(icu_list)].sort_values(by=['subject_id']) sapsii_last = get_sql(con, 'subject_id,icustay_id, sapsii', 'mimiciii.sapsii_last') sapsii_last = sapsii_last[sapsii_last['icustay_id']. isin(icu_list)].sort_values(by=['subject_id']) sofa = get_sql(con, 'subject_id,icustay_id, sofa', 'mimiciii.sofa') sofa = sofa[sofa['icustay_id'].isin(icu_list)].\ sort_values(by=['subject_id']) # length of stay as numerical feature icu_los = get_sql(con, 'subject_id,icustay_id, los', 'mimiciii.ICUSTAYS') icu_los = icu_los[icu_los['icustay_id'].isin( icu_list)].sort_values(by=['subject_id']) # turn everything in to matrices oasis_m = np.asmatrix(oasis)[:, 2] oasis_last_m = np.asmatrix(oasis_last)[:, 2] sofa_m = np.asmatrix(sofa)[:, 2] saps_m = np.asmatrix(saps)[:, 2] sapsii_m = np.asmatrix(sapsii)[:, 2] sapsii_last_m = np.asmatrix(sapsii_last)[:, 2] icu_los_m = np.asmatrix(icu_los)[:, 2] age = load("lists/adults_admitted.csv", cols=['subject_id', 'age']) age = age[age['subject_id'].isin( p_list)].sort_values(by=['subject_id']) age_m = np.asmatrix(age)[:, 1] # combine all features into one feature vector and return v = np.concatenate((oasis_m, sofa_m, sapsii_m, sapsii_last_m, age_m, icu_los_m), axis=1) return v
def run(data, args): pdata = load(args.patchfile) newdata = data for i, d in enumerate(data): crc = str(crc32(d['orig'].encode('utf-8')) & 0xffffffff) if crc in pdata: newdata[i][args.lang] = pdata[crc] else: print(u"Unknown: {0}".format(crc)) return newdata
def generate_labels(limit=30): patients = load("lists/adults_heart_discharged.csv", cols=['subject_id']) cases = load("lists/readmission_diff.csv", cols=['subject_id', 'diff'])\ .sort_values(by=['subject_id']) labels = pd.merge(cases, patients, on='subject_id', how='right')\ .sort_values(by=['subject_id']) # generate labels def labeling(x): if x is not None: return 0 elif x < limit: return 1 else: return 0 labels['label'] = labels['diff'].apply(labeling) labels['label'].to_csv("Y.csv", header=False, index=False) return labels
def load_features(featurefile): (bugs, test_results) = load(featurefile) executed = set([]) bugs = set(bugs) for (_, features) in test_results: executed.update(features) num_bugs = len(bugs) num_execed_bugs = len(bugs.intersection(executed)) return (num_bugs, num_execed_bugs)
def run(data, args): pd = load(args.infile) d = {} for item in data: if not item['ctx'] in d: d[item['ctx']] = {} d[item['ctx']][item['orig']] = item for item in pd: if item[args.lang]: d[item['ctx']][item['orig']][args.lang] = item[args.lang] return data
def sdaImageTest(modelPath): info,model = util.load(modelPath) print info print model shape = info["dataSet"]["shape"] color = len(shape) == 3 import matplotlib.pyplot as plt for name,fig in sdaLayerImages2(model,3): fig.show() raw_input() plt.show()
def score(options): if not os.path.exists('output/combined.csv'): return if os.path.exists('output/results.csv'): os.remove('output/results.csv') data=load('output/combined.csv', False) detectors={} encoders={} for row in data: detector, tracename, d1, d2, d3, guess, answer, correct=row if guess==answer: if detector in detectors.keys(): stats=detectors[detector] stats[0]=stats[0]+1 detectors[detector]=stats else: detectors[detector]=[1,0] if answer in encoders.keys(): stats=encoders[answer] stats[0]=stats[0]+1 encoders[answer]=stats else: encoders[answer]=[1,0,0] else: if detector in detectors.keys(): stats=detectors[detector] stats[1]=stats[1]+1 detectors[detector]=stats else: detectors[detector]=[0,1] if guess in encoders.keys(): stats=encoders[guess] stats[1]=stats[1]+1 encoders[guess]=stats else: encoders[guess]=[0,1,0] if answer in encoders.keys(): stats=encoders[answer] stats[2]=stats[2]+1 encoders[answer]=stats else: encoders[answer]=[0,0,1] f=open('output/detectors.csv', 'wb') for detector in detectors: stats=detectors[detector] f.write(detector+','+str(stats[0])+','+str(stats[1])+"\n") f.close() f=open('output/encoders.csv', 'wb') for encoder in encoders: stats=encoders[encoder] f.write(encoder+','+str(stats[0])+','+str(stats[1])+','+str(stats[2])+"\n") f.close()
def errorRate(options): if not os.path.exists('output/detectors.csv'): return data=load('output/detectors.csv', False) for row in data: detector, correct, incorrect=row correct=float(correct) incorrect=float(incorrect) total=correct+incorrect pc=int(round((correct*100)/total)) print(detector+': '+str(pc)+'% detected') print('') if not os.path.exists('output/encoders.csv'): return data=load('output/encoders.csv', False) for row in data: encoder, correct, fpos, fneg=row correct=float(correct) fpos=float(fpos) fneg=float(fneg) total=correct+fpos+fneg pc=round((correct*100)/total) pp=round((fpos*100)/total) pn=round((fneg*100)/total) print(encoder+': '+str(int(pc))+'% ('+str(int(pp))+'%/'+str(int(pn))+'%) detected') print('') if not os.path.exists('output/diffs.csv'): return data=load('output/diffs.csv', False) for row in data: pair, correct, incorrect=row correct=float(correct) incorrect=float(incorrect) total=correct+incorrect pc=int(round((correct*100)/total)) print(pair+': '+str(pc)+'% distinguishable')
def _get_cachedconfig(self, name, filename, configname, default, pre): """Return a cached value for a config option. If the cache is uninitialized then try to read its value from disk. Option can be overridden by the commandline. name: property name, e.g. 'lastpulled' filename: name of file in .hg/svn configname: commandline option name default: default value pre: transformation to apply to a value before caching it. """ varname = '_' + name if getattr(self, varname) is None: # construct the file path from metapath (e.g. .hg/svn) plus the # filename f = os.path.join(self.metapath, filename) # load the config property (i.e. command-line or .hgrc) c = None if configname: # a little awkward but we need to convert the option from a # string to whatever type the default value is, so we use the # type of `default` to determine with ui.config method to call c = None if isinstance(default, bool): c = self.ui.configbool('hgsubversion', configname, default) elif isinstance(default, int): c = self.ui.configint('hgsubversion', configname, default) elif isinstance(default, list): c = self.ui.configlist('hgsubversion', configname, default) elif isinstance(default, dict): c = dict(self.ui.configitems(configname)) else: c = self.ui.config('hgsubversion', configname, default) # load the value from disk val = util.load(f, default=default) # prefer the non-default, and the one sent from command-line if c is not None and c != val and c != default: val = c # apply transformation if necessary if pre: val = pre(val) # set the value as the one from disk (or default if not found) setattr(self, varname, val) # save the value to disk by using the setter property setattr(self, name, val) return getattr(self, varname)
def load_evaluations(evalfs, metricnames): res = {} cumulative = {} benchnames = [] for fname in evalfs: try: scores = load(fname) except: continue collect(metricnames, scores, res, cumulative) return (res, cumulative)
def load_tasks(self, sender): """Retrieve the contents of the task file.""" task_file = self.load_dialog["txt_load"].text if task_file: task_file = util.validate_file(task_file) if task_file: self.load_dialog.close() self.tasklist.tasks = util.load(task_file) self.current_task_file = task_file tasklist.Task.last_id = len(self.tasklist.tasks) self.show_tasks(None) else: self.display_message(self.load_dialog["txt_load"].text + " is not a valid file") self.load_dialog["txt_load"].text = ""
def load_tasks(self, sender): """Retrieve the contents of the task file.""" task_file = self.load_dialog['textfield1'].text if not task_file == '': task_file = util.validate_file(task_file) if task_file: self.load_dialog.close() self.tasklist.tasks = util.load(task_file) self.current_task_file = task_file Task.last_id = len(self.tasklist.tasks) self.show_tasks(None) else: self.display_message(self.load_dialog['textfield1'].text + ' is not a valid file') self.load_dialog['textfield1'].text = ''
def _set_uuid(self, uuid): self.__uuid = None uuidfile = os.path.join(self.metapath, 'uuid') if os.path.isfile(uuidfile): stored_uuid = util.load(uuidfile) assert stored_uuid if uuid and uuid != stored_uuid: raise hgutil.Abort('unable to operate on unrelated repository') self.__uuid = uuid or stored_uuid elif uuid: util.dump(uuid, uuidfile) self.__uuid = uuid elif not self._skiperror: raise hgutil.Abort("hgsubversion metadata unavailable; " "please run 'hg svn rebuildmeta'")
def __init__(self, repo, uuid=None, subdir=None, skiperrorcheck=False): """path is the path to the target hg repo. subdir is the subdirectory of the edits *on the svn server*. It is needed for stripping paths off in certain cases. """ # simple and public variables self.ui = repo.ui self.repo = repo self.path = os.path.normpath(repo.join('..')) self.firstpulled = 0 self.lastdate = '1970-01-01 00:00:00 -0000' self.addedtags = {} self.deletedtags = {} # private variables self._skiperror = skiperrorcheck self._tags = None self._layoutobj = None self._revmap = None self._authors = None self._branchmap = None self._tagmap = None self._filemap = None self._layout = None # create .hg/svn folder if it doesn't exist if not os.path.isdir(self.metapath): os.makedirs(self.metapath) # properties that need .hg/svn to exist self.uuid = uuid self.subdir = subdir # generated properties that have a persistent file stored on disk self._gen_cachedconfig('lastpulled', 0, configname=False) self._gen_cachedconfig('defaultauthors', True) self._gen_cachedconfig('caseignoreauthors', False) self._gen_cachedconfig('mapauthorscmd', None) self._gen_cachedconfig('defaulthost', self.uuid) self._gen_cachedconfig('usebranchnames', True) self._gen_cachedconfig('defaultmessage', '') self._gen_cachedconfig('branch', '') self._gen_cachedconfig('layout', 'auto') # misc self.branches = util.load(self.branch_info_file) or {} self.prevbranches = dict(self.branches)