def decrypt(key, ct_path="ciphertext.enc", savePT="plaintext.dec"): with open(ct_path, 'rb') as input: u = dill.load(input) ciphertext = dill.load(input) v = modexp(u, key.x, key.p) uv = str(u)+str(v) k = SHA224.new(uv.encode('utf-8')).hexdigest().encode('utf-8') #symmetric key for compute the ciphertext with AES print("K: "+str(k)) bs = Blowfish.block_size iv = ciphertext[:bs] # Remove IV ciphertext = ciphertext[bs:] print("CT-LEN:"+str(len(ciphertext))) cipher = Blowfish.new(k, Blowfish.MODE_CBC, iv) plaintext = cipher.decrypt(ciphertext) # Remove padding last_byte = plaintext[-1] plaintext = plaintext[:- (last_byte if type(last_byte) is int else ord(last_byte))] # Write to file the plaintext decrypted #plaintext = plaintext.decode(plaintext, key.iNumBits) io.open(savePT,"wb").write(plaintext) return plaintext
def decrypt(sk, ct_path="ciphertext.enc", savePT="plaintext.dec"): # Load Y and ciphertext with open(ct_path, 'rb') as input: y = dill.load(input) ciphertext = dill.load(input) # Compute x = RSA^(-1)(y) as y^d mod N x = pow(y, sk.d, sk.n) # Compute symmetric key with Hash function (SHA224) k = SHA224.new(repr(x).encode('utf-8')).hexdigest().encode('utf-8') #symmetric key for compute the ciphertext print("K: "+str(k)) # Count block_size bs = Blowfish.block_size # Retreive IV vector iv = ciphertext[:bs] # Remove IV ciphertext = ciphertext[bs:] print("CT-LEN:"+str(len(ciphertext))) # Initialize Blowfish cipher cipher = Blowfish.new(k, Blowfish.MODE_CBC, iv) # Decrypt plaintext = cipher.decrypt(ciphertext) # Remove padding last_byte = plaintext[-1] plaintext = plaintext[:- (last_byte if type(last_byte) is int else ord(last_byte))] # Write to file the plaintext decrypted io.open(savePT,"wb").write(plaintext) return plaintext
def test_run_advanced(self, tmpdir): temp_path = os.path.join(str(tmpdir), 'temp') log_path = os.path.join(str(tmpdir), 'log') paths = { 'temp': temp_path, 'log': log_path } pipe = Pipeline(paths=paths, create_paths=True) pipe.add_step(test_func1, ['func1'], var1=3, var2=4) pipe.add_step(test_func2, ['func2'], var1=25, var2=10) pipe.add_step(test_func2, ['func2'], var1=1, var2=0) pipe.add_step(test_func3, var1=1, var2=0) with pytest.raises(PipelineError): pipe.run() pipe = dill.load(open(os.path.join(paths['log'], 'pipeline.p'), 'rb')) assert pipe.run_step_idx==2 assert pipe.steps[0].results=={'next_id': 4, 'status': 'success', 'step_id': 0, 'sum': 7} with pytest.raises(ZeroDivisionError): for step in pipe.steps: step.results = None pipe.run(resume=True, ignore_errors=True) new_pipe = dill.load(open(os.path.join(paths['log'], 'pipeline.p'), 'rb')) result = new_pipe.run(start_idx=1, ignore_errors=True, ignore_exceptions=True) assert result['status']=='success' assert new_pipe.steps[0].results==None assert new_pipe.steps[1].results=={'diff': 2.5, 'status': 'success'} assert new_pipe.steps[2].results=={'error': 'Division by 0', 'status': 'error'} assert new_pipe.steps[3].results['status']=='error'
def classify_patch_group(fn_rf, fn_kern, fn_patch_info): t0 = time() RF = dill.load(open(fn_rf, "rb")) # unpack the RF classifier kernels = dill.load(open(fn_kern, "rb")) # unpack the kernels patch_info = dill.load(open(fn_patch_info, "rb")) # unpack the patch info a, b, c = patch_info[0] # get the bounds of the set patches_a = patch_info[1] # grab the set to classify patches_r = patch_info[2] # grab the set to compare with (atlas mask) results = [] for i, patch in enumerate(patches_a): # go through each patch if np.all(patches_r[i]): # if the patch is entirely masked feat = compute_feats(patch, kernels).flatten().reshape(1, -1) intens = np.array(compute_intens(patch)).flatten().reshape(1, -1) feat = np.concatenate((feat, intens), axis=1) prediction = RF.predict(feat) # print("Classifying patch {}/{}: {}".format(i, len(patches), prediction)) results.append(np.full(patch.shape, prediction)) else: # the associated ROI patch is totally zero results.append(np.zeros(patch.shape)) dt = time() - t0 print("Classified group {}-{}/{} in {:.2f} time".format(a, b, c, dt)) return results
def get_SALICON_train(location=None): """ Loads or downloads and caches the SALICON training dataset. For memory reasons no fixation trains are provided. @type location: string, defaults to `None` @param location: If and where to cache the dataset. The dataset will be stored in the subdirectory `SALICON_train` of location and read from there, if already present. @return: Training stimuli, validation stimuli, testing stimuli, training fixation trains, validation fixation trains .. seealso:: Ming Jiang, Shengsheng Huang, Juanyong Duan*, Qi Zhao: SALICON: Saliency in Context, CVPR 2015 http://salicon.net/ """ if location: location = os.path.join(location, 'SALICON_train') if os.path.exists(location): stimuli = dill.load(open(os.path.join(location, 'stimuli.pydat'), 'rb')) fixations = dill.load(open(os.path.join(location, 'fixations.pydat'), 'rb')) return stimuli, fixations os.makedirs(location) stimuli, fixations = _get_SALICON('train', 'https://s3.amazonaws.com/salicon-dataset/2015r1/train.zip', 'd549761c16e59b80cd5981373ada5e98', 'https://s3.amazonaws.com/salicon-dataset/2015r1/fixations_train2014.json', 'ab60a090ee31642fbb4aa41f4953b8bd', location) return stimuli, fixations
def get_SALICON_val(location=None): """ Loads or downloads and caches the SALICON validation dataset. For memory reasons no fixation trains are provided. @type location: string, defaults to `None` @param location: If and where to cache the dataset. The dataset will be stored in the subdirectory `SALICON_train` of location and read from there, if already present. @return: Training stimuli, validation stimuli, testing stimuli, training fixation trains, validation fixation trains .. seealso:: Ming Jiang, Shengsheng Huang, Juanyong Duan*, Qi Zhao: SALICON: Saliency in Context, CVPR 2015 http://salicon.net/ """ if location: location = os.path.join(location, 'SALICON_val') if os.path.exists(location): stimuli = dill.load(open(os.path.join(location, 'stimuli.pydat'), 'rb')) fixations = dill.load(open(os.path.join(location, 'fixations.pydat'), 'rb')) return stimuli, fixations os.makedirs(location) stimuli, fixations = _get_SALICON('val', 'https://s3.amazonaws.com/salicon-dataset/2015r1/val.zip', '62cd6641a5354d3099a693ff90cb6dab', 'https://s3.amazonaws.com/salicon-dataset/2015r1/fixations_val2014.json', '3224f8cf86ea8d248d93583866b60c5f', location) return stimuli, fixations
def main(): with open('./word2index.dict', 'rb') as f: word2index = dill.load(f) with open('./index2word.dict', 'rb') as f: index2word = dill.load(f) model = Seq2Seq( vocab_size=len(word2index), embed_size=300, hidden_size=300, ) serializers.load_npz('seq2seq.npz', model) while True: s = input() test_input = Variable( np.array([word2index.get(word, word2index['UNK']) for word in mecab_wakati(s)], dtype='int32') ) print('ε ₯ε-> {}'.format(s)) print('εΊε-> ', end="") for index in model.predict(test_input): print(index2word[index], end='') print()
def _remove_nresults(self, traj, nresults, continue_folder): result_tuple_list = [] n = 0 for filename in os.listdir(continue_folder): _, ext = os.path.splitext(filename) if ext != '.rcnt': continue n += 1 cnt_file = open(os.path.join(continue_folder, filename), 'rb') try: result = dill.load(cnt_file) cnt_file.close() result_tuple_list.append((result)) except Exception: # delete broken files logging.getLogger().exception('Could not open continue snapshot ' 'file `%s`.' % filename) cnt_file.close() os.remove(filename) self.assertGreaterEqual(n, nresults) result_tuple_list = sorted(result_tuple_list, key=lambda x: x[0]) timestamp_list = [x[1]['finish_timestamp'] for x in result_tuple_list] timestamp_list = timestamp_list[-nresults:] for timestamp in timestamp_list: filename = os.path.join(continue_folder, 'result_%s.rcnt' % repr(timestamp).replace('.','_')) os.remove(filename) result_tuple_list = [] for filename in os.listdir(continue_folder): _, ext = os.path.splitext(filename) if ext != '.rcnt': continue cnt_file = open(os.path.join(continue_folder, filename), 'rb') result = dill.load(cnt_file) cnt_file.close() result_tuple_list.append((result)) name_set = set([x[1]['name'] for x in result_tuple_list]) removed = 0 for run_name in traj.f_iter_runs(): if run_name not in name_set: run_dict = traj.f_get_run_information(run_name, copy=False) run_dict['completed'] = 0 idx = run_dict['idx'] traj._updated_run_information.add(idx) removed += 1 self.assertGreaterEqual(removed, nresults) logging.getLogger().error('Removed %s runs for continuing' % removed) traj.f_store(only_init=True)
def _generate_image_segments_and_label_batch(images,img_dir,seg_dir,mean_img): #choosing 5 images at random test_batch = [] batch_labels = [] for i in range(0,images.__len__()): img = io.imread(img_dir+images[i]) segf = open(seg_dir+images[i][0:-3]+'slic','rb') segmap = dill.load(segf) segments_l = dill.load(segf) sal_l = dill.load(segf) segf.close() data = im2mdfin(img,mean_img,segmap,segments_l) for j in range(0,segments_l.__len__()): x = data.segments[j] dat = np.zeros([227,227,9],dtype = np.uint8) dat[:,:,0:3]= x.SP_Region dat[:,:,3:6]=x.SP_Neighbor dat[:,:,6:9]=x.Pic test_batch.append(sal_l[j]) test_batch.append(x.SP_Region) test_batch.append(x.SP_Neighbor) test_batch.append(x.Pic) return test_batch
def experimentLMKLIEP_per_feat(which_d): with open('feat_vec_out.'+which_d+'.en.pickle', 'rb') as handle: out_d = pickle.load(handle) with open('feat_vec_in.'+which_d+'.en.pickle', 'rb') as handle: in_d = pickle.load(handle) labels = - np.ones(out_d.shape[0]) predictions = - np.ones(out_d.shape[0]) labels[-50000:] = 1 W = np.zeros(out_d.shape) for i in xrange(W.shape[1]): kliep = KLIEP(init_b=100, seed=0) lm_out = out_d[:, i].reshape((out_d.shape[0], 1)) lm_in = in_d[:, i].reshape((in_d.shape[0], 1)) kliep.fit_CV(lm_out, lm_in) W[:, i] = kliep.predict(lm_out).ravel() w = np.mean(W, axis=1) predictions[np.where(w > 1.4)[0]] = 1 print 'total positive:', np.where(predictions == 1)[0].shape, ', out of:', out_d.shape[0] # sorted_ind = np.argsort(w, axis=None)[::-1] # predictions[sorted_ind[0:50000]] = 1 p, r, f, s = precision_recall_fscore_support(labels.astype(int), predictions.astype(int), pos_label=1, average='micro') print 'Precision:', p, print 'Recall:', r, print 'F1:', f, print 'Support:', s,
def experiment_LMSVM(which_d): with open('feat_vec_out.'+which_d+'.en.pickle', 'rb') as handle: out_d = pickle.load(handle) with open('feat_vec_in.'+which_d+'.en.pickle', 'rb') as handle: in_d = pickle.load(handle) labels = - np.ones(out_d.shape[0]) labels[-50000:] = 1 print 'Fitting one class SVM' clf = svm.OneClassSVM(kernel='linear') clf.fit(in_d) print 'Predicting for out domain' predictions = clf.predict(out_d) print 'total positive:', np.where(predictions == 1)[0].shape, ', out of:', out_d.shape[0] # sorted_ind = np.argsort(w, axis=None)[::-1] # predictions[sorted_ind[0:50000]] = 1 p, r, f, s = precision_recall_fscore_support(labels.astype(int), predictions.astype(int), pos_label=1, average='micro') print 'Precision:', p, print 'Recall:', r, print 'F1:', f, print 'Support:', s,
def test_run_advanced(self, tmpdir): temp_path = os.path.join(str(tmpdir), "temp") log_path = os.path.join(str(tmpdir), "log") paths = {"temp": temp_path, "log": log_path} pipe = pipeline.Pipeline(paths=paths, create_paths=True) pipe.add_step(test_func1, ["func1"], var1=3, var2=4) pipe.add_step(test_func2, ["func2"], var1=25, var2=10) pipe.add_step(test_func2, ["func2"], var1=1, var2=0) pipe.add_step(test_func3, var1=1, var2=0) with pytest.raises(pipeline.PipelineError): pipe.run() pipe = dill.load(open(os.path.join(paths["log"], "pipeline.p"), "rb")) assert pipe.run_step_idx == 2 assert pipe.steps[0].results == {"next_id": 4, "status": "success", "step_id": 0, "sum": 7} with pytest.raises(ZeroDivisionError): for step in pipe.steps: step.results = None pipe.run(resume=True, ignore_errors=True) new_pipe = dill.load(open(os.path.join(paths["log"], "pipeline.p"), "rb")) result = new_pipe.run(start_idx=1, ignore_errors=True, ignore_exceptions=True) assert result["status"] == "success" assert new_pipe.steps[0].results == None assert new_pipe.steps[1].results == {"diff": 2.5, "status": "success"} assert new_pipe.steps[2].results == {"error": "Division by 0", "status": "error"} assert new_pipe.steps[3].results["status"] == "error"
def put(self): hour = int(request.form['hour']) date = request.form['date'] prcp = float(request.form['prcp'])*100 snow = float(request.form['snow']) * 10 tmax = float(request.form['tmax']) * 10 tmin = float(request.form['tmin']) * 10 date = pd.to_datetime(date) with open(os.path.join(APP_STATIC, 'uniquegeohash.pkl'), 'rb') as f: uniquegeohash = dill.load(f) with open(os.path.join(APP_STATIC, 'predict_pickup_density.pkl'), 'rb') as f: model = dill.load(f) x_dict = [{"pickup_geohash": geostr, "hour": hour, "dayofweek": date.dayofweek, 'month': date.month,'PRCP':prcp,'SNOW':snow,'TMAX':tmax,'TMIN':tmin} for geostr in uniquegeohash] x_df = pd.DataFrame(x_dict) y = model.predict(x_df) geodecode = [Geohash.decode(geocode) for geocode in uniquegeohash] yzipgeo = zip(y, geodecode) sortedlist = sorted(yzipgeo, key=lambda x: -x[0]) top10address = [] top10dict = {} for y, geodecode in sortedlist[0:50]: key = ",".join(geodecode) top10dict[key] = top10dict.get(key,0) + y top10res = [] for key in top10dict: temptuple = (float(key.split(",")[0]),float(key.split(",")[1])) top10res.append([top10dict[key],temptuple]) top10res = sorted(top10res,key=lambda x:-x[0]) top10res = top10res[0:10] if len(top10res) > 10 else top10res for u,geodecode in top10res: g = geocoder.google([geodecode[0], geodecode[1]], method='reverse').address top10address.append(g) return {"top10": top10res,"top10address":top10address}
def create_cifar10(): # Load training data X, y = [], [] for num in range(1,5): f = open('data_batch_' + str(num), 'rb') batch = pickle.load(f) X.append(batch['data']) y.append(batch['labels']) X = np.concatenate(X).reshape(-1, 3, 32, 32).astype(np.float32) y = np.concatenate(y).astype(np.int32) # Load test data f = open('test_batch', 'rb') batch = pickle.load(f) X_test = batch['data'].reshape(-1, 3, 32, 32).astype(np.float32) y_test = np.array(batch['labels'], dtype=np.int32) # Split arrays ii = np.random.permutation(len(X)) X_train = X[ii[1000:]] y_train = y[ii[1000:]] X_val = X[ii[:1000]] y_val = y[ii[:1000]] # Offset data offset = np.mean(X_train, 0) scale = np.std(X_train, 0).clip(min=1) X_train = (X_train - offset) / scale X_val = (X_val - offset) / scale X_test = (X_test - offset) / scale # Save data pickle.dump(X_train, open('cifar10_X_train', "wb")) pickle.dump(y_train, open('cifar10_y_train', "wb")) pickle.dump(X_val, open('cifar10_X_val', "wb")) pickle.dump(y_val, open('cifar10_y_val', "wb")) pickle.dump(X_test, open('cifar10_X_test', "wb")) pickle.dump(y_test, open('cifar10_y_test', "wb")) return True
def get_data(): ''' get the predictor and target and return them ''' nids = dill.load(open('movie_ids.pkl', 'rb')) model = dill.load(open('all2.pkl', 'rb')) W = model.W df = pd.DataFrame(W, index=nids) c = MongoClient() db = c['movies'] boxoffice2 = db.boxoffice2 movie_info = db.movie_info r = list(boxoffice2.find({}, {'_id': 1, 'BoxOffice2': 1})) df_bf = pd.DataFrame(r).set_index('_id') r2 = list(movie_info.find({}, {'_id': 1, 'year': 1, 'title': 1})) df_year = pd.DataFrame(r2) df_year = df_year.set_index('_id') df = df.join(df_bf).join(df_year) cond1 = (df['year'] >= 2010) cond2 = ~ np.isnan(df['BoxOffice2']) cond = cond1 & cond2 df_subset = df[cond] y = df_subset['BoxOffice2'].values X = df_subset.iloc[:, :-3].values return X, y
def main(): argc = len(sys.argv) if argc < 2: print "Error: No input files" exit() data = pickle.load(open(sys.argv[1], "rb")) bwords = None vocabulary = None if argc > 2: vocabulary, bwords, _ = pickle.load(open(sys.argv[2], "rb")) corpus, topics, titles, time, grpid = data BV = BVsmVectorizer(vocabulary) # N = len(corpus) i = 0 time = np.array(time) # corpus = np.array(corpus) topics = np.array(topics) titles = np.array(titles) grpid = np.array(grpid) allDates = sorted(list(set(time))) for d in allDates: try: dcorpus = list() indeces = list(np.nonzero(time == d)[0]) for j in indeces: dcorpus.append(corpus[j]) c, t, b, sab, smb = BV.vectorize(dcorpus, vocab=vocabulary) vocabulary = BV.vocab except KeyboardInterrupt: print "Could Not Vectorise", d exit() try: pickle.dump( ((c, t, b, sab, smb), topics[time == d], titles[time == d], time[time == d], grpid[time == d]) , open("vectors/"+str(d), "wb") ) # pickle.dump( (BV.vocab, BV.bwords, BV.wordWise / BV.N, BV.N, d) , open("vectors/vectorizer."+str(i%2), "wb")) except KeyboardInterrupt: print "Could Not Pickle", d exit() i += 1 print d, (d%2), "\n" if not d % 30: pickle.dump( (BV.vocab, BV.bwords, BV.bvocab, BV.wordWise / BV.N, BV.N, d) , open("vectors/vectorizer."+str(d), "wb")) print len(set(BV.bwords)) print "Finishing..." pickle.dump( (BV.vocab, BV.bwords, BV.bvocab, BV.wordWise / BV.N, BV.N, d) , open("vectors/vectorizer."+str(d), "wb"))
def predict_checkins(id, n_docks, total_docks, features): if not (os.path.exists('estimators/checkin_estimator_high_'+str(id)+'.dill.bz2') and os.path.exists('estimators/checkout_estimator_mid_'+str(id)+'.dill.bz2')): return None, None, None checkin_est = dill.load(bz2.BZ2File('estimators/checkin_estimator_high_'+str(id)+'.dill.bz2', 'r')) checkout_est = dill.load(bz2.BZ2File('estimators/checkout_estimator_mid_'+str(id)+'.dill.bz2', 'r')) flux = checkout_est.predict(features)/15 - checkin_est.predict(features)/15 docks = np.clip(n_docks+flux.cumsum(), 0, total_docks) return int(docks[29]), int(docks[44]), int(docks[59])
def test_2(): max_len = 4 print 'Loading heldout phrase pairs...' with open('/home/christos/SSLP/Project2/heldout/phrase_pairs_.pickle') as handle: phrase_pairs_held = pickle.load(handle) print 'Loading regular phrase pairs...' with open('/home/said/git/SSLP/Project2/training/phrase_pairs_.pickle') as handle: reg_phrase_pairs = pickle.load(handle) print 'Loading combined phrase pairs...' with open('/home/said/git/SSLP/Project2/training/combined_phrase_pairs_4_10.pickle') as handle: comb_phrase_pairs = pickle.load(handle) print 'Measuring sparsity...' ex_sparsity = measure_exact(reg_phrase_pairs, phrase_pairs_held, max_len=max_len) print 'Regular phrase pairs:' for i in xrange(len(ex_sparsity)): print 'For phrases with n =', i+1 print 'In train and heldout:', ex_sparsity[i][0] print 'In train and not in heldout:', ex_sparsity[i][1] print 'In heldout and not in train:', ex_sparsity[i][2] print ex_sparsity = measure_exact(comb_phrase_pairs, phrase_pairs_held, max_len=max_len) print 'Combined phrase pairs:' for i in xrange(len(ex_sparsity)): print 'For phrases with n =', i+1 print 'In train and heldout:', ex_sparsity[i][0] print 'In train and not in heldout:', ex_sparsity[i][1] print 'In heldout and not in train:', ex_sparsity[i][2] print print 'Calcuating precision and recall against Moses...' print 'PR for regular:' precision, recall = pr_vs_moses('phrase-table', reg_phrase_pairs) print 'Precision:', precision, 'Recall:', recall print print 'PR for combined:' precision, recall = pr_vs_moses('phrase-table', comb_phrase_pairs) print 'Precision:', precision, 'Recall:', recall print print 'Calcuating precision and recall against Moses per length...' print 'PR for regular:' precision, recall = pr_vs_moses_per_len('phrase-table', reg_phrase_pairs) for i in xrange(len(precision)): print 'n=', i+1, ':', 'Precision:', precision[i], 'Recall:', recall[i] print print 'PR for combined:' precision, recall = pr_vs_moses_per_len('phrase-table', comb_phrase_pairs) for i in xrange(len(precision)): print 'n=', i+1, ':', 'Precision:', precision[i], 'Recall:', recall[i] print
def test_dynamic_classes(self): test_classes = { algorithms.GradientDescent: {}, algorithms.MinibatchGradientDescent: {'batch_size': 10}, algorithms.Momentum: {'momentum': 0.5}, } for algorithm_class, algorithm_params in test_classes.items(): optimization_classes = [algorithms.WeightDecay, algorithms.SearchThenConverge] bpnet = algorithm_class( (3, 5, 1), addons=optimization_classes, verbose=False, **algorithm_params ) data, target = datasets.make_regression(n_features=3, n_targets=1) data = preprocessing.MinMaxScaler().fit_transform(data) target_scaler = preprocessing.MinMaxScaler() target = target_scaler.fit_transform(target.reshape(-1, 1)) with tempfile.NamedTemporaryFile() as temp: valid_class_name = bpnet.__class__.__name__ dill.dump(bpnet, temp) temp.file.seek(0) restored_bpnet = dill.load(temp) restored_class_name = restored_bpnet.__class__.__name__ temp.file.seek(0) self.assertEqual(valid_class_name, restored_class_name) self.assertEqual(optimization_classes, restored_bpnet.addons) bpnet.train(data, target, epochs=10) real_bpnet_error = bpnet.prediction_error(data, target) updated_input_weight = ( bpnet.input_layer.weight.get_value().copy() ) dill.dump(bpnet, temp) temp.file.seek(0) restored_bpnet2 = dill.load(temp) temp.file.seek(0) restored_bpnet_error = restored_bpnet2.prediction_error( data, target ) np.testing.assert_array_equal( updated_input_weight, restored_bpnet2.input_layer.weight.get_value() ) # Error must be big, because we didn't normalize data self.assertEqual(real_bpnet_error, restored_bpnet_error)
def compute_predictions(self, model_name, inputs): ''' computes prediction for given sample and specified model ''' if self.approx_technique.lower() in ['rsm', 'moa', 'hda', 'gp']: model = gtapprox.Model(self.models_save_path+model_name+self.ext) prediction = model.calc(inputs) elif self.approx_technique.lower() == 'xgboost': model = Xgboost() model.load(self.models_save_path+model_name+self.ext) prediction = model.predict(inputs)[:, np.newaxis] elif self.approx_technique.lower() == 'quadxgboost': model = Xgboost() model.load(self.models_save_path+model_name+'_1_'+self.ext) prediction = model.predict(inputs)[:, np.newaxis] model0 = gtapprox.Model(self.models_save_path+model_name+'_0_'+self.ext) prediction = prediction + model0.calc(inputs) elif self.approx_technique.lower() == 'mars': with open(self.models_save_path+model_name+self.ext, 'rb') as file_object: model = pickle.load(file_object) prediction = model.predict(inputs)[:, np.newaxis] elif self.approx_technique.lower() == 'skgboost': with open(self.models_save_path+model_name+self.ext, 'rb') as file_object: model = pickle.load(file_object) prediction = model.predict(inputs)[:, np.newaxis] elif self.approx_technique.lower() == 'adaboost': with open(self.models_save_path+model_name+self.ext, 'rb') as file_object: model = pickle.load(file_object) prediction = model.predict(inputs)[:, np.newaxis] elif self.approx_technique.lower() == 'zeros': prediction = np.zeros((inputs.shape[0], 1)) elif self.approx_technique.lower() == 'means': with open(self.models_save_path+model_name+self.ext, 'rb') as file_object: model = pickle.load(file_object) prediction = model*np.ones((inputs.shape[0], 1)) else: raise Exception('Wrong approx type specified!') return prediction
def load_actions_from_file(): """ loads Qtrons from previous file, please check that file load_qtrons.pk1 exists in data directory """ with open('./data/load_qtrons.pk1', 'rb') as f: parameters = dill.load(f) actions = dill.load(f) config = dill.load(f) return parameters, actions, config
def _main(fd): with os.fdopen(fd, 'rb', closefd=True) as from_parent: process.current_process()._inheriting = True try: preparation_data = pickle.load(from_parent) prepare(preparation_data) self = pickle.load(from_parent) finally: del process.current_process()._inheriting return self._bootstrap()
def __init__(self, words_file, ep_cache_file, occ_cache_file, accuracy): with open(ep_cache_file, 'rb') as f: self.error_probability = dill.load(f) with open(occ_cache_file, 'rb') as f: self.occurrneces = dill.load(f) self.words = TextCorpusStatisticsCalculator([]).read_words(words_file) self.accuracy = accuracy self.n = sum([v for k, v in self.occurrneces.items()]) self.m = len(self.words) self.calc = ErrorProbabilityCalculator()
def update_session(fname=None): import dill as pickle if fname is None: fname = conf.session try: s = pickle.load(gzip.open(fname,"rb")) except IOError: s = pickle.load(open(fname,"rb")) kamene_session = builtins.__dict__["kamene_session"] kamene_session.update(s)
def load_phrases(folder): print 'Loading...' with open(folder+'phrase_pairs_.pickle', 'rb') as handle: phrase_pairs = pickle.load(handle) with open(folder+'en_given_nl_.pickle', 'rb') as handle: en_given_nl = pickle.load(handle) with open(folder+'nl_given_en_.pickle', 'rb') as handle: nl_given_en = pickle.load(handle) with open(folder+'joint_ennl_.pickle', 'rb') as handle: joint_ennl = pickle.load(handle) print 'Loaded.' return phrase_pairs, en_given_nl, nl_given_en, joint_ennl
def main(): classifier_file = open('saved_classifiers/spam_classifier.pickle', 'rb') classifier_object = dill.load(classifier_file) classifier_file.close() trainer_file = open('saved_classifiers/trainer.pickle', 'rb') trainer_object = dill.load(trainer_file) trainer_file.close() # Testing the accuracy test(trainer_object, classifier_object, 'spam') test(trainer_object, classifier_object, 'ham')
def category_model(record): test_tf={x:1 for x in record['categories']} f=open("./ml/dict_category_model", "r") a=dill.load(f) test_tf1=a.transform(test_tf) f.close() f=open("./ml/category_model","r") b=dill.load(f) f.close() try: return b.predict(test_tf1) except: return 0
def test_dynamic_classes(self): test_classes = { algorithms.Backpropagation: {}, algorithms.MinibatchGradientDescent: {"batch_size": 10}, algorithms.Momentum: {"momentum": 0.5}, algorithms.RPROP: {"maximum_step": 1}, algorithms.IRPROPPlus: {"maximum_step": 1}, algorithms.ConjugateGradient: {"update_function": "fletcher_reeves"}, algorithms.QuasiNewton: {"update_function": "bfgs"}, algorithms.HessianDiagonal: {"min_eigenvalue": 1e-5}, algorithms.LevenbergMarquardt: {"mu": 0.01}, } for algorithm_class, algorithm_params in test_classes.items(): optimization_classes = [algorithms.WeightDecay, algorithms.SearchThenConverge] bpnet = algorithm_class((3, 5, 1), optimizations=optimization_classes, verbose=False, **algorithm_params) data, target = datasets.make_regression(n_features=3, n_targets=1) data = preprocessing.MinMaxScaler().fit_transform(data) target_scaler = preprocessing.MinMaxScaler() target = target_scaler.fit_transform(target.reshape(-1, 1)) with tempfile.NamedTemporaryFile() as temp: valid_class_name = bpnet.__class__.__name__ dill.dump(bpnet, temp) temp.file.seek(0) restored_bpnet = dill.load(temp) restored_class_name = restored_bpnet.__class__.__name__ temp.file.seek(0) self.assertEqual(valid_class_name, restored_class_name) self.assertEqual(optimization_classes, restored_bpnet.optimizations) bpnet.train(data, target, epochs=10) real_bpnet_error = bpnet.error(bpnet.predict(data), target) updated_input_weight = bpnet.input_layer.weight.copy() dill.dump(bpnet, temp) temp.file.seek(0) restored_bpnet2 = dill.load(temp) temp.file.seek(0) actual = restored_bpnet2.predict(data) restored_bpnet_error = restored_bpnet2.error(actual, target) np.testing.assert_array_equal(updated_input_weight, restored_bpnet2.input_layer.weight) # Error must be big, because we didn't normalize data self.assertEqual(real_bpnet_error, restored_bpnet_error)
def sentiment(text): """ sentiment ΰΈ ΰΈ²ΰΈ©ΰΈ²ΰΉΰΈΰΈ’ ΰΉΰΈΰΉΰΈΰΉΰΈΰΈ‘ΰΈΉΰΈ₯ΰΈΰΈ²ΰΈ https://github.com/wannaphongcom/lexicon-thai/tree/master/ΰΈΰΉΰΈΰΈΰΈ§ΰΈ²ΰΈ‘/ ΰΈ£ΰΈ±ΰΈΰΈΰΉΰΈ²ΰΈͺΰΈΰΈ£ΰΈ΄ΰΈ str ΰΈΰΈ·ΰΈΰΈΰΉΰΈ² pos , neg""" with open(os.path.join(templates_dir, 'vocabulary.data'), 'rb') as in_strm: vocabulary = dill.load(in_strm) in_strm.close() with open(os.path.join(templates_dir, 'sentiment.data'), 'rb') as in_strm: classifier = dill.load(in_strm) in_strm.close() text=set(word_tokenize(text))-set(stopwords.words('thai')) featurized_test_sentence = {i:(i in text) for i in vocabulary} return classifier.classify(featurized_test_sentence)
#!/home/chenyang/Documents/twilio_webserver/env/bin/python2 # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 2008-2016 California Institute of Technology. # Copyright (c) 2016-2017 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - http://trac.mystic.cacr.caltech.edu/project/pathos/browser/dill/LICENSE if __name__ == '__main__': import sys import dill for file in sys.argv[1:]: print (dill.load(open(file,'rb')))
res = MyEval.F14Exp(pred, test_label) print(res) with open('../../stat/res_exp_for_paper.csv', 'a') as fout: fout.write('{0},{1},{2},{3},{4},{5},{6},{7}\n'.format( method, i_iter, i_fold, res[0], res[1], res[2], res[3], res[4])) i_fold += 1 if __name__ == "__main__": with open('../data/features_all_v2.5.pkl', 'rb') as my_input: all_pid = dill.load(my_input) all_feature = np.array(dill.load(my_input)) all_label = np.array(dill.load(my_input)) print('features_all shape: ', all_feature.shape) with open('../data/feat_deep_centerwave_v0.1.pkl', 'rb') as my_input: feat_deep_centerwave = np.array(dill.load(my_input)) print('feat_deep_centerwave shape: ', feat_deep_centerwave.shape) with open('../data/feat_resnet.pkl', 'rb') as my_input: feat_resnet = np.array(dill.load(my_input)) print('feat_resnet shape: ', feat_resnet.shape) all_feature = np.c_[all_feature, feat_deep_centerwave, feat_resnet] all_label = np.array(all_label) all_pid = np.array(all_pid)
filename = "51_rubidium87_relevant_2021-08-08T13_42_.pkl" filename = "51_rubidium87_relevant_21*021-08-08T14_56_.pkl" filename = "51_rubidium87_relevant_2021-08-08T18_54_.pkl" filename = "51_rubidium87_relevant_2021-08-09T02_34_.pkl" filename = "51_rubidium87_relevant_2021-08-09T13_23_.pkl" filename = "51_rubidium87_relevant_2021-08-09T13_47_.pkl" filename = "51_rubidium87_relevant_2021-08-09T14_36_.pkl" filename = "51_rubidium87_relevant_2021-08-09T15_28_.pkl" filename = "51_rubidium87_relevant_2021-08-09T16_27_.pkl" filename = "51_rubidium87_relevant_2021-08-09T17_12_.pkl" filename = "51_rubidium87_relevant_2021-08-09T17_37.pkl" filename = "51_rubidium87_relevant_2021-08-09T18_12.pkl" filename = "60_rubidium87_relevant_2021-08-19T16_58.pkl" with open(f"system/simulation/saved_simulations/{filename}", "rb") as f: simulation: Simulation = pickle.load(f) # _raw_dc_calculator = simulation.get_calculator((270, 230)) # simulation.dc_field_calculator = lambda _t: _raw_dc_calculator(_t).round(1) # _raw_dc_calculator = simulation.get_calculator((270, 210)) # simulation.dc_field_calculator = lambda _t: _raw_dc_calculator(_t).round(1) # # simulation.rf_freq_calculator = simulation.get_calculator(230e6 / 1e9) # simulation.rf_field_calculator = lambda t: 3 * np.sin(np.pi * t / 1000 / simulation.t) print(simulation.dc_field) print(simulation.rf_field) print(simulation.rf_freq) print(simulation.t) systems: List[qutip.Qobj] = simulation.results.states
lenDBuser = len(DBuser) lenUserVec = len(userVec) jaccard = matches / (lenDBuser + lenUserVec - matches) tempSim = jaccard * (1 - MSD) if tempSim > bestSim: bestSim = tempSim bestSimUser = u bestSimUser = icamf_recommender.rating_object.ids_user[bestSimUser] return bestSimUser, bestSim with open("dummy_model.pkl", "rb") as f: icamf_recommender = dill.load(f) if __name__ == '__main__': app.run(host="0.0.0.0", port=8080) #To run without clipping set to False or del argument #train_recommender_kfold(kfold=5, regularizer=0.001, learning_rate=0.001, num_factors=20, iterations=50, clipping=5) #train_recommender_kfold(kfold=5, regularizer=0.001, learning_rate=0.002, num_factors=20, iterations=50, clipping=5) #train_recommender_kfold(kfold=5, regularizer=0.001, learning_rate=0.005, num_factors=20, iterations=50, clipping=5) #train_and_save_model(regularizer=0.001, learning_rate=0.002, num_factors=20, iterations=1, clipping=5) #with open("dummy_model.pkl", "rb") as f: # icamf_recommender = dill.load(f)
def sample(save_dir): path_to_config = save_dir + "/config" if not os.path.isfile(path_to_config): raise IOError("Could not find " + path_to_config) with open(path_to_config, "rb") as f: gen_config = pickle.load(f) # # Load vocabulary encoder # glove_dir = '/Users/danfriedman/Box Sync/My Box Files/9 senior spring/gen/glove/glove.6B/glove.6B.50d.txt' # #glove_dir = '/data/corpora/word_embeddings/glove/glove.6B.50d.txt' if gen_config.use_glove: _, _, _, L = data_reader.glove_encoder(gen_config.glove_dir) else: L = None # Rebuild the model with tf.variable_scope("LSTM"): gen_model = lstm_ops.seq2seq_model( encoder_seq_length=gen_config.d_len, decoder_seq_length=1, num_layers=gen_config.num_layers, embed_size=gen_config.embed_size, batch_size=gen_config.batch_size, hidden_size=gen_config.hidden_size, vocab_size=gen_config.vocab_size, dropout=gen_config.dropout, max_grad_norm=gen_config.max_grad_norm, use_attention=gen_config.use_attention, embeddings=L, is_training=False, is_gen_model=True, token_type=gen_config.token_type, reuse=False) with tf.Session() as session: saver = tf.train.Saver() saver.restore(session, tf.train.latest_checkpoint('./' + args.save_dir)) def generate(description, temperature): return lstm_ops.generate_text_beam_search( session=session, model=gen_model, encode=gen_config.encode, decode=gen_config.decode, description=description, d_len=gen_config.d_len, beam=5, stop_length=gen_config.c_len, temperature=temperature) seed = "Three huge birds wait outside of the window of a man's room. The man is talking on the phone." temp = 1.0 print(generate(seed, temp)) while raw_input("Sample again? ([y]/n): ") != "n": new_seed = raw_input("seed: ") if len(gen_config.encode(seed)) > gen_config.d_len: print("Description must be < {} tokens".format( gen_config.d_len)) continue new_temp = raw_input("temp: ") if new_seed != "": seed = new_seed if new_temp != "": temp = float(new_temp) print(generate(seed, temp))
def resume(save_file): simulation = dill.load(save_file) simulation.run(resume=True)
def tokenizer(text): # create a tokenizer function # θΏε a list of <class 'spacy.tokens.token.Token'> return [tok.text for tok in spacy_en.tokenizer(text)] from torchtext import data import numpy as np from data import text_utils args = argument_parser() with open("seq2seq/bak/TEXT.Field", "rb") as f: TEXT = dill.load(f) LENGTH = data.Field(sequential=False, use_vocab=False) embeddings = np.random.random((len(TEXT.vocab.itos), args.embed_size)) args.TEXT = TEXT encoder = SN_MODELS["encoder"](embeddings, args) # atten = SN_MODELS["attention"](args.hidden_size * 4, 300) # decoder = SN_MODELS["decoder"](embeddings, args) atten = SN_MODELS["attention"](args.hidden_size, "general") decoder = SN_MODELS["decoder"](embeddings, args, atten) model_class = SN_MODELS[args.model_name] # model = model_class(encoder, decoder, args)
NAME = '*test13' pth = Path('~/Simulations/coop_extension') flist = list(pth.expanduser().glob(NAME)) def get_params(fname): par = str(fname).split('/')[5].split('-') return par data = [] for fl in flist: with open(fl, 'rb') as fh: d = dill.load(fh) p = get_params(fl) data.append((p, d)) D = len(data) N = len([ x for x in data[0][1][-1].keys() if isinstance(x, tuple) ]) - 1 dataset = np.zeros((D, 10)) NN = tuple(range(N)) for i, (par, dt) in enumerate(data): dataset[i][0] = dt[-1]['roi_coop_theo'] # Theo roi coop dataset[i][1] = dt[-1]['roi_coop_days'].mean() # Mean roi coop dataset[i][2] = 100 * ((dataset[i, 0] / dataset[i , 1]) - 1)
default=10000, help='number of points used for plotting') args = parser.parse_args() args.twofold = bool(args.twofold) if args.target_dims == 'all': args.mds_dims = args.latent_dims if not args.conditioning is None: args.name += "_%s" % args.conditioning #%% Data import import dill with open(args.data, 'rb') as f: audioSet = dill.load(f) #testSet = np.load('data/testSet.npy') if not args.filter is None: wrong_ids = np.where(audioSet.metadata['octave'] > args.filter) audioSet.files = np.delete(np.array(audioSet.files), wrong_ids).tolist() audioSet.data = np.delete(np.array(audioSet.data), wrong_ids).tolist() for k, v in audioSet.metadata.items(): audioSet.metadata[k] = np.delete(v, wrong_ids) if len(args.frames) == 0: print('taking the whole dataset...') audioSet.flattenData(lambda x: x[:]) elif len(args.frames) == 2: print('taking between %d and %d...' % (args.frames[0], args.frames[1])) audioSet.flattenData(lambda x: x[args.frames[0]:args.frames[1]])
def load_model(self, path_to_save): with open(path_to_save, 'rb') as f: hmm_tagger = dill.load(f) self.model = hmm_tagger return hmm_tagger
def load_model(): with open(MODEL_FILE_PATH,'rb') as file: return dill.load(file)
def _load_auxiliary_files(self): super(TextTrainer, self)._load_auxiliary_files() data_indexer_file = open("%s_data_indexer.pkl" % self.model_prefix, "rb") self.data_indexer = pickle.load(data_indexer_file) data_indexer_file.close()
def unpickle(path): with open(path, 'rb') as fp: return dill.load(fp)
def load(identifier): file_name = f'{se.local_storage}model_nav-mode_v{DILL_VERSION}_{identifier}.dill' with open(file_name, 'rb') as handle: bm = dill.load(handle) return bm
import tempfile import dill try: import pathos.multiprocessing as mp except ImportError: pass from spatialist.ancillary import HiddenPrints if __name__ == '__main__': # de-serialize the arguments written by function ancillary.multicore tmpfile = os.path.join(tempfile.gettempdir(), 'spatialist_dump') with open(tmpfile, 'rb') as tmp: func, cores, processlist = dill.load(tmp) # serialize the job arguments to be able to pass them to the processes processlist = [dill.dumps([func, x]) for x in processlist] # a simple wrapper to execute the jobs in the sub-processes # re-import of modules and passing pickled variables is necessary since on # Windows the environment is not shared between parent and child processes def wrapper(job): import dill function, proc = dill.loads(job) return function(**proc) # hide print messages in the sub-processes with HiddenPrints(): # start pool of processes and do the work
def __init__( self, Nlayers=1, # number of layers Ndirs=1, # unidirectional or bidirectional Nx=100, # input size Nh=100, # hidden layer size Ny=100, # output size Ah="relu", # hidden unit activation (e.g. relu, tanh, lstm) Ay="linear", # output unit activation (e.g. linear, sigmoid, softmax) predictPer="frame", # frame or sequence loss=None, # loss function (e.g. mse, ce, ce_group, hinge, squared_hinge) L1reg=0.0, # L1 regularization L2reg=0.0, # L2 regularization seed=15213, # random seed for initializing the weights frontEnd=None, # a lambda function for transforming the input filename=None, # initialize from file initParams=None, # initialize from given dict ): if filename is not None: # load parameters from file with smart_open(filename, "rb") as f: initParams = dill.load(f) if initParams is not None: # load parameters from given dict self.paramNames = [] self.params = [] for k, v in initParams.iteritems(): if type(v) is numpy.ndarray: self.addParam(k, v) else: setattr(self, k, v) self.paramNames.append(k) # F*ck, locals()[k] = v doesn't work; I have to do this statically Nlayers, Ndirs, Nx, Nh, Ny, Ah, Ay, predictPer, loss, L1reg, L2reg, frontEnd \ = self.Nlayers, self.Ndirs, self.Nx, self.Nh, self.Ny, self.Ah, self.Ay, self.predictPer, self.loss, self.L1reg, self.L2reg, self.frontEnd else: # Initialize parameters randomly # Names of parameters to save to file self.paramNames = [ "Nlayers", "Ndirs", "Nx", "Nh", "Ny", "Ah", "Ay", "predictPer", "loss", "L1reg", "L2reg", "frontEnd" ] for name in self.paramNames: value = locals()[name] setattr(self, name, value) # Values of parameters for building the computational graph self.params = [] # Initialize random number generators global rng rng = numpy.random.RandomState(seed) # Construct parameter matrices Nlstm = 4 if Ah == 'lstm' else 1 self.addParam("Win", rand_init((Nx, Nh * Ndirs * Nlstm), Ah)) self.addParam("Wrec", rand_init((Nlayers, Ndirs, Nh, Nh * Nlstm), Ah)) self.addParam( "Wup", rand_init((Nlayers - 1, Nh * Ndirs, Nh * Ndirs * Nlstm), Ah)) self.addParam("Wout", rand_init((Nh * Ndirs, Ny), Ay)) if Ah != "lstm": self.addParam("Bhid", zeros((Nlayers, Nh * Ndirs))) else: self.addParam( "Bhid", numpy.tile( numpy.hstack([ full((Nlayers, Nh), 1.0), zeros((Nlayers, Nh * 3)) ]), (1, Ndirs))) self.addParam("Bout", zeros(Ny)) self.addParam("h0", zeros((Nlayers, Ndirs, Nh))) if Ah == "lstm": self.addParam("c0", zeros((Nlayers, Ndirs, Nh))) # Compute total number of parameters self.nParams = sum(x.get_value().size for x in self.params) # Initialize accumulators for gradients self.aparams = [ theano.shared(zeros(x.get_value().shape)) for x in self.params ] # Build computation graph input = T.ftensor3() mask = T.imatrix() mask_int = [(mask % 2).nonzero(), (mask >= 2).nonzero()] mask_float = [ T.cast((mask % 2).dimshuffle((1, 0)).reshape( (mask.shape[1], mask.shape[0], 1)), theano.config.floatX), T.cast((mask >= 2).dimshuffle((1, 0)).reshape( (mask.shape[1], mask.shape[0], 1)), theano.config.floatX) ] # mask_int = [(mask & 1).nonzero(), (mask & 2).nonzero()] # mask_float = [T.cast((mask & 1).dimshuffle((1, 0)).reshape((mask.shape[1], mask.shape[0], 1)), theano.config.floatX), # T.cast(((mask & 2) / 2).dimshuffle((1, 0)).reshape((mask.shape[1], mask.shape[0], 1)), theano.config.floatX)] def step_rnn(x_t, mask, h_tm1, W, h0): h_tm1 = T.switch(mask, h0, h_tm1) return [ACTIVATION[Ah](x_t + h_tm1.dot(W))] def step_lstm(x_t, mask, c_tm1, h_tm1, W, c0, h0): c_tm1 = T.switch(mask, c0, c_tm1) h_tm1 = T.switch(mask, h0, h_tm1) a = x_t + h_tm1.dot(W) f_t = T.nnet.sigmoid(a[:, :Nh]) i_t = T.nnet.sigmoid(a[:, Nh:Nh * 2]) o_t = T.nnet.sigmoid(a[:, Nh * 2:Nh * 3]) c_t = T.tanh(a[:, Nh * 3:]) * i_t + c_tm1 * f_t h_t = T.tanh(c_t) * o_t return [c_t, h_t] x = input if frontEnd is None else frontEnd(input) for i in range(Nlayers): h = (x.dimshuffle((1, 0, 2)).dot(self.Win) if i == 0 else h.dot(self.Wup[i - 1])) + self.Bhid[i] rep = lambda x: T.extra_ops.repeat( x.reshape((1, -1)), h.shape[1], axis=0) if Ah != "lstm": h = T.concatenate([ theano.scan( fn=step_rnn, sequences=[ h[:, :, Nh * d:Nh * (d + 1)], mask_float[d] ], outputs_info=[rep(self.h0[i, d])], non_sequences=[self.Wrec[i, d], rep(self.h0[i, d])], go_backwards=(d == 1), )[0][::(1 if d == 0 else -1)] for d in range(Ndirs) ], axis=2) else: h = T.concatenate([ theano.scan( fn=step_lstm, sequences=[ h[:, :, Nh * 4 * d:Nh * 4 * (d + 1)], mask_float[d] ], outputs_info=[rep(self.c0[i, d]), rep(self.h0[i, d])], non_sequences=[ self.Wrec[i, d], rep(self.c0[i, d]), rep(self.h0[i, d]) ], go_backwards=(d == 1), )[0][1][::(1 if d == 0 else -1)] for d in range(Ndirs) ], axis=2) h = h.dimshuffle((1, 0, 2)) if predictPer == "sequence": h = T.concatenate([ h[mask_int[1 - d]][:, Nh * d:Nh * (d + 1)] for d in range(Ndirs) ], axis=1) output = ACTIVATION[Ay](h.dot(self.Wout) + self.Bout) # Compute loss function if loss is None: loss = { "linear": "mse", "sigmoid": "ce", "softmax": "ce_group" }[self.Ay] if loss == "ctc": label = T.imatrix() label_time = T.imatrix() tol = T.iscalar() cost = ctc_cost(output, mask, label, label_time, tol) else: if predictPer == "sequence": label = T.fmatrix() y = output t = label elif predictPer == "frame": label = T.ftensor3() indices = (mask >= 0).nonzero() y = output[indices] t = label[indices] cost = T.mean({ "ce": -T.mean(T.log(y) * t + T.log(1 - y) * (1 - t), axis=1), "ce_group": -T.log((y * t).sum(axis=1)), "mse": T.mean((y - t)**2, axis=1), "hinge": T.mean(relu(1 - y * (t * 2 - 1)), axis=1), "squared_hinge": T.mean(relu(1 - y * (t * 2 - 1))**2, axis=1), }[loss]) # Add regularization cost += sum(abs(x).sum() for x in self.params) / self.nParams * L1reg cost += sum(T.sqr(x).sum() for x in self.params) / self.nParams * L2reg # Compute updates for network parameters updates = [] lrate = T.fscalar() clip = T.fscalar() grad = T.grad(cost, self.params) grad_clipped = [T.maximum(T.minimum(g, clip), -clip) for g in grad] for w, a, g in zip(self.params, self.aparams, grad_clipped): updates.append((a, 0.9 * a + 0.1 * g**2)) updates.append((w, w - lrate * g / (a + 1e-8)**0.5)) # Create functions to be called from outside if loss == "ctc": inputs = [input, mask, label, label_time, tol, lrate, clip] else: inputs = [input, mask, label, lrate, clip] self.train = theano.function( inputs=inputs, outputs=cost, updates=updates, ) self.predict = theano.function(inputs=[input, mask], outputs=output)
def generate_mu_timelist( record_max=10**4, no_arms=5, mu_type='biggap', is_timevar=False, timevar_type='General', reward_type='Bernoulli', mu_list=None, direc=None, plot=True, ): ''' generate the value of each arm mean as a function of time ''' # if not provided the list of mean of arms to start with, then generate it if mu_list is None: mu_list = generate_mu_list(no_arms, mu_type, timevar_type, reward_type) # make sure the senatity no_arms = len(mu_list) if not is_timevar: # no changes mu_time_list = lambda t, i: mu_list[i] else: if timevar_type == 'Abrupt': # continuous changes if mu_type == "biggap": f_t = lambda t: 0.1 * pow(-1, np.floor(t / 50000)) * ( t >= 50000 * np.floor(t / 50000) and t < 50000 * (np.floor(t / 50000) + 1)) elif mu_type == "smallgap": f_t = lambda t: 0.01 * pow(-1, np.floor(t / 50000)) * ( t >= 50000 * np.floor(t / 50000) and t < 50000 * (np.floor(t / 50000) + 1)) else: f_t = lambda t: rand() * pow(-1, np.floor(t / 50000)) * ( t >= 50000 * np.floor(t / 50000) and t < 50000 * (np.floor(t / 50000) + 1)) mu_time_list = lambda t, i: mu_list[mod( i + np.int(np.divide(t, 50000)), no_arms)] + f_t(t) elif timevar_type == 'General': # continuous changes f_t = lambda t: sin(np.pi * t / 50000) + 1 mu_time_list = lambda t, i: mu_list[i] * f_t(t + 50000 * i) elif timevar_type == 'RealAbrupt': # Example from the Yahoo! dataset, from article "Nearly Optimal Adaptive Procedure with Change Detection for Piecewise-Stationary Bandit" (M-UCB) https://arxiv.org/abs/1802.03692 # 6 arms, 9 discrete change mu_list = [[0.071, 0.041, 0.032, 0.030, 0.020, 0.011], [0.055, 0.053, 0.032, 0.030, 0.008, 0.011], [0.040, 0.063, 0.032, 0.030, 0.008, 0.011], [0.040, 0.042, 0.043, 0.030, 0.008, 0.011], [0.030, 0.032, 0.055, 0.030, 0.008, 0.011], [0.030, 0.032, 0.020, 0.030, 0.008, 0.021], [0.020, 0.022, 0.020, 0.045, 0.008, 0.021], [0.020, 0.022, 0.020, 0.057, 0.008, 0.011], [0.020, 0.022, 0.034, 0.057, 0.022, 0.011]] mu_time_list = lambda t, i: mu_list[int(np.floor(t / 50000))][i] # save the data to direc if direc is not None: filename = 'noarms%d_mu_type%s_timevar_type%s_reward_type%s' % ( no_arms, mu_type, timevar_type, reward_type) if not os.path.exists("%s/%s.pkl" % (direc, filename)): savelambda(mu_time_list, direc, filename) else: with open('%s/%s.pkl' % (direc, filename), 'rb') as input: mu_dat = dill.load(input) mu_time_list = mu_dat plotfilename = 'mu_time_list%s' % (filename) plotdirec = direc + '/plots' if not os.path.exists("%s/%s.pdf" % (plotdirec, plotfilename)) and plot: plot_mu(plotdirec, plotfilename, no_arms, record_max, mu_time_list, mu_type) # return the list, which is a list of funtion, each represent the mean of arms as a function of time return mu_time_list
from flask import Flask from flask import request from flask import render_template app = Flask(__name__) import pickle import numpy as np import dill model = dill.load(open('./data/model.pkl')) c_feat_mat = model.feat_mat[:len(model.course_list), :] # Form page to submit text #============================================ # create page with a form on it @app.route('/index.html') @app.route('/index') @app.route('/') def submission_page(): return render_template('index.html') # Recommendation page #============================================ # create page with a form on it # Recommending
def LoadNetwork(network2Load): with open(network2Load, "rb") as networkPickled: return pickle.load(networkPickled)
#coding:utf-8 import dill as pickle tmp = pickle.load(open('bpe_deen/bpe_vocab.pkl', 'rb')) # open(opt.data_pkl, 'rb') print(tmp)
def main(_): tf.logging.set_verbosity(tf.logging.INFO) albert_config = modeling.AlbertConfig.from_json_file( FLAGS.albert_config_file) validate_flags_or_throw(albert_config) tf.gfile.MakeDirs(FLAGS.output_dir) # # tokenizer = fine_tuning_utils.create_vocab( # vocab_file=FLAGS.vocab_file, # do_lower_case=FLAGS.do_lower_case, # spm_model_file=FLAGS.spm_model_file, # hub_module=FLAGS.albert_hub_module_handle) tpu_cluster_resolver = None if FLAGS.use_tpu and FLAGS.tpu_name: tpu_cluster_resolver = contrib_cluster_resolver.TPUClusterResolver( FLAGS.tpu_name, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project) is_per_host = contrib_tpu.InputPipelineConfig.PER_HOST_V2 if FLAGS.do_train: iterations_per_loop = int( min(FLAGS.iterations_per_loop, FLAGS.save_checkpoints_steps)) else: iterations_per_loop = FLAGS.iterations_per_loop run_config = contrib_tpu.RunConfig( cluster=tpu_cluster_resolver, master=FLAGS.master, model_dir=FLAGS.output_dir, keep_checkpoint_max=0, save_checkpoints_steps=FLAGS.save_checkpoints_steps, tpu_config=contrib_tpu.TPUConfig( iterations_per_loop=iterations_per_loop, num_shards=FLAGS.num_tpu_cores, per_host_input_for_training=is_per_host)) train_examples = None num_train_steps = None num_warmup_steps = None if not tf.gfile.Exists(FLAGS.train_feature_file): raise Exception("Train tf-record missed...") cnt = 0 records = tf.python_io.tf_record_iterator(FLAGS.train_feature_file) for _ in records: cnt += 1 print(cnt) num_train_steps = int(cnt / FLAGS.train_batch_size * FLAGS.num_train_epochs) # train_examples = squad_utils.read_squad_examples( # input_file=FLAGS.train_file, is_training=True) # num_train_steps = int( # len(train_examples) / FLAGS.train_batch_size * FLAGS.num_train_epochs) if FLAGS.do_train: num_warmup_steps = int(num_train_steps * FLAGS.warmup_proportion) # # # Pre-shuffle the input to avoid having to make a very large shuffle # # buffer in in the `input_fn`. # rng = random.Random(12345) # rng.shuffle(train_examples) tag_info = squad_utils.TagInfo.load(FLAGS.tag_info_file) print(tag_info.__dict__) model_fn = squad_utils.v2_model_fn_builder( albert_config=albert_config, init_checkpoint=FLAGS.init_checkpoint, learning_rate=FLAGS.learning_rate, num_train_steps=num_train_steps, num_warmup_steps=num_warmup_steps, use_tpu=FLAGS.use_tpu, use_one_hot_embeddings=FLAGS.use_tpu, max_seq_length=FLAGS.max_seq_length, start_n_top=FLAGS.start_n_top, end_n_top=FLAGS.end_n_top, dropout_prob=FLAGS.dropout_prob, hub_module=FLAGS.albert_hub_module_handle, tag_info=tag_info) # If TPU is not available, this will fall back to normal Estimator on CPU # or GPU. estimator = contrib_tpu.TPUEstimator( use_tpu=FLAGS.use_tpu, model_fn=model_fn, config=run_config, train_batch_size=FLAGS.train_batch_size, predict_batch_size=FLAGS.predict_batch_size) if FLAGS.do_train: # We write to a temporary file to avoid storing very large constant tensors # in memory. # if not tf.gfile.Exists(FLAGS.train_feature_file): # train_writer = squad_utils.FeatureWriter( # filename=os.path.join(FLAGS.train_feature_file), is_training=True) # squad_utils.convert_examples_to_features( # examples=train_examples, # tokenizer=tokenizer, # max_seq_length=FLAGS.max_seq_length, # doc_stride=FLAGS.doc_stride, # max_query_length=FLAGS.max_query_length, # is_training=True, # output_fn=train_writer.process_feature, # do_lower_case=FLAGS.do_lower_case) # train_writer.close() tf.logging.info("***** Running training *****") # tf.logging.info(" Num orig examples = %d", len(train_examples)) # tf.logging.info(" Num split examples = %d", train_writer.num_features) tf.logging.info(" Batch size = %d", FLAGS.train_batch_size) tf.logging.info(" Num steps = %d", num_train_steps) # del train_examples train_input_fn = squad_utils.input_fn_builder( input_file=FLAGS.train_feature_file, seq_length=FLAGS.max_seq_length, is_training=True, drop_remainder=True, use_tpu=FLAGS.use_tpu, bsz=FLAGS.train_batch_size, is_v2=True) estimator.train(input_fn=train_input_fn, max_steps=num_train_steps) if FLAGS.do_predict: import dill # with tf.gfile.Open(FLAGS.predict_file) as predict_file: # prediction_json = json.load(predict_file)["data"] # eval_examples = squad_utils.read_squad_examples( # input_file=FLAGS.predict_file, is_training=False) if (tf.gfile.Exists(FLAGS.predict_feature_file) and tf.gfile.Exists(FLAGS.predict_feature_left_file) and tf.gfile.Exists(FLAGS.predict_example_file)): tf.logging.info("Loading eval features from {}".format( FLAGS.predict_feature_left_file)) with tf.gfile.Open(FLAGS.predict_feature_left_file, "rb") as fin: eval_features = dill.load(fin) with tf.gfile.Open(FLAGS.predict_example_file, "rb") as fin: eval_examples = dill.load(fin) # else: # eval_writer = squad_utils.FeatureWriter( # filename=FLAGS.predict_feature_file, is_training=False) # eval_features = [] # # def append_feature(feature): # eval_features.append(feature) # eval_writer.process_feature(feature) # # squad_utils.convert_examples_to_features( # examples=eval_examples, # tokenizer=tokenizer, # max_seq_length=FLAGS.max_seq_length, # doc_stride=FLAGS.doc_stride, # max_query_length=FLAGS.max_query_length, # is_training=False, # output_fn=append_feature, # do_lower_case=FLAGS.do_lower_case) # eval_writer.close() # # with tf.gfile.Open(FLAGS.predict_feature_left_file, "wb") as fout: # pickle.dump(eval_features, fout) tf.logging.info("***** Running predictions *****") tf.logging.info(" Num orig examples = %d", len(eval_examples)) tf.logging.info(" Num split examples = %d", len(eval_features)) tf.logging.info(" Batch size = %d", FLAGS.predict_batch_size) predict_input_fn = squad_utils.input_fn_builder( input_file=FLAGS.predict_feature_file, seq_length=FLAGS.max_seq_length, is_training=False, drop_remainder=False, use_tpu=FLAGS.use_tpu, bsz=FLAGS.predict_batch_size, is_v2=True) def get_result(checkpoint): """Evaluate the checkpoint on SQuAD v2.0.""" # If running eval on the TPU, you will need to specify the number of # steps. all_results = [] for result in estimator.predict(predict_input_fn, yield_single_examples=True, checkpoint_path=checkpoint): if len(all_results) % 1000 == 0: tf.logging.info("Processing example: %d" % (len(all_results))) unique_id = int(result["unique_ids"]) crf_logits = result["crf_logits"] transition_params = result["transition_params"] softmax = result["softmax"] all_results.append( squad_utils.RawResultV2( unique_id=unique_id, crf_logits=crf_logits, softmax=softmax, transition_params=transition_params, )) output_prediction_file = os.path.join(FLAGS.output_dir, "predictions.json") predictions = squad_utils.write_predictions_et( eval_examples, eval_features, all_results, FLAGS.max_answer_length, tag_info) import numpy class MyEncoder(json.JSONEncoder): def default(self, o): if isinstance(o, numpy.integer): return int(o) if isinstance(o, numpy.floating): return float(o) if isinstance(o, numpy.ndarray): return o.tolist() else: return super(MyEncoder, self).default(o) with tf.gfile.Open(output_prediction_file, 'w') as f: json.dump(predictions, f, ensure_ascii=False, cls=MyEncoder) latest_checkpoint = tf.train.latest_checkpoint(FLAGS.output_dir) get_result(latest_checkpoint)
def load_obj(path): """ Loads object """ return dill.load(open(path, 'rb'))
def load(location, do_unzip_and_model_type_check=True): """ Method used to load a container from the file system. Args: location: The location on the file system where to load the model. do_unzip_and_model_type_check: Whether to unzip the model and check the type. Returns: The loaded model. """ assert tvm_installed(), "TVM Container requires TVM installed." _load_param_dict = tvm._ffi.get_global_func("tvm.relay._load_param_dict") # We borrow this function directly from Relay. # Relay when imported tryies to download schedules data, # but at inference time access to disk or network could be blocked. def load_param_dict(param_bytes): if isinstance(param_bytes, (bytes, str)): param_bytes = bytearray(param_bytes) load_arr = _load_param_dict(param_bytes) return {v.name: v.array for v in load_arr} container = None if do_unzip_and_model_type_check: # Unzip the dir. zip_location = location if not location.endswith("zip"): zip_location = location + ".zip" else: location = zip_location[:-4] assert os.path.exists(zip_location), "Zip file {} does not exist.".format(zip_location) shutil.unpack_archive(zip_location, location, format="zip") assert os.path.exists(location), "Model location {} does not exist.".format(location) # Load the model type. with open(os.path.join(location, constants.SAVE_LOAD_MODEL_TYPE_PATH), "r") as file: model_type = file.readline() if model_type != "tvm": shutil.rmtree(location) raise RuntimeError("Expected TVM model type, got {}".format(model_type)) # Check the versions of the modules used when saving the model. if os.path.exists(os.path.join(location, constants.SAVE_LOAD_MODEL_CONFIGURATION_PATH)): with open(os.path.join(location, constants.SAVE_LOAD_MODEL_CONFIGURATION_PATH), "r") as file: configuration = file.readlines() check_dumped_versions(configuration, hummingbird, torch) else: warnings.warn( "Cannot find the configuration file with versions. You are likely trying to load a model saved with an old version of Hummingbird." ) # Load the actual model. path_lib = os.path.join(location, constants.SAVE_LOAD_TVM_LIB_PATH) graph = open(os.path.join(location, constants.SAVE_LOAD_TVM_GRAPH_PATH)).read() lib = tvm.runtime.module.load_module(path_lib) params = load_param_dict(open(os.path.join(location, constants.SAVE_LOAD_TVM_PARAMS_PATH), "rb").read()) # Load the container. with open(os.path.join(location, constants.SAVE_LOAD_CONTAINER_PATH), "rb") as file: container = dill.load(file) if container is None: shutil.rmtree(location) raise RuntimeError("Failed to load the model container.") # Setup the container. ctx = tvm.cpu() if container._ctx == "cpu" else tvm.gpu container._model = graph_runtime.create(graph, lib, ctx) container._model.set_input(**params) container._extra_config[constants.TVM_GRAPH] = graph container._extra_config[constants.TVM_LIB] = lib container._extra_config[constants.TVM_PARAMS] = params container._extra_config[constants.TVM_CONTEXT] = ctx container._ctx = ctx container._tvm_tensors = {name: container._to_tvm_array(np.array([])) for name in container._input_names} # Need to set the number of threads to use as set in the original container. os.environ["TVM_NUM_THREADS"] = str(container._n_threads) shutil.rmtree(location) return container
import os import dill as pickle import numpy as np import lda from rpy2 import robjects from rpy2.robjects.packages import importr from rpy2.robjects.vectors import IntVector, FloatVector # set working directory os.chdir( "/Users/annekespeijers/Desktop/BGSE/Term3/TextMining/Homework/Project/") # load in corpus with open('./data/corpus.pkl', 'rb') as input: corpus = pickle.load(input) # doc term matrix X = corpus.document_term_matrix(corpus.token_set) X = X.astype(int) # get vocab, article titles and article comments vocab = tuple(corpus.token_set) titles = tuple([t.title for t in corpus.docs]) comments = tuple([com.comments for com in corpus.docs]) ## fit the model: k = 2 k = 2 model = lda.LDA(n_topics=k, n_iter=500, random_state=1, eta=200 / float(len(vocab)),
def main(): if not os.path.exists(os.path.join("saved", model_name)): os.makedirs(os.path.join("saved", model_name)) data_path = '../../data/records_final.pkl' voc_path = '../../data/voc_final.pkl' device = torch.device('cuda:0') data = dill.load(open(data_path, 'rb')) voc = dill.load(open(voc_path, 'rb')) diag_voc, pro_voc, med_voc = voc['diag_voc'], voc['pro_voc'], voc[ 'med_voc'] split_point = int(len(data) * 2 / 3) data_train = data[:split_point] eval_len = int(len(data[split_point:]) / 2) data_test = data[split_point:split_point + eval_len] data_eval = data[split_point + eval_len:] voc_size = (len(diag_voc.idx2word), len(pro_voc.idx2word), len(med_voc.idx2word)) EPOCH = 30 LR = 0.0002 TEST = False END_TOKEN = voc_size[2] + 1 model = Leap(voc_size, device=device) if TEST: model.load_state_dict( torch.load( open(os.path.join("saved", model_name, resume_name), 'rb'))) # pass model.to(device=device) print('parameters', get_n_params(model)) optimizer = Adam(model.parameters(), lr=LR) if TEST: eval(model, data_test, voc_size, 0) else: history = defaultdict(list) for epoch in range(EPOCH): loss_record = [] start_time = time.time() model.train() for step, input in enumerate(data_train): for adm in input: loss_target = adm[2] + [END_TOKEN] output_logits = model(adm) loss = F.cross_entropy( output_logits, torch.LongTensor(loss_target).to(device)) loss_record.append(loss.item()) optimizer.zero_grad() loss.backward(retain_graph=True) optimizer.step() llprint('\rTrain--Epoch: %d, Step: %d/%d' % (epoch, step, len(data_train))) ddi_rate, ja, prauc, avg_p, avg_r, avg_f1 = eval( model, data_eval, voc_size, epoch) history['ja'].append(ja) history['ddi_rate'].append(ddi_rate) history['avg_p'].append(avg_p) history['avg_r'].append(avg_r) history['avg_f1'].append(avg_f1) history['prauc'].append(prauc) end_time = time.time() elapsed_time = (end_time - start_time) / 60 llprint( '\tEpoch: %d, Loss1: %.4f, One Epoch Time: %.2fm, Appro Left Time: %.2fh\n' % (epoch, np.mean(loss_record), elapsed_time, elapsed_time * (EPOCH - epoch - 1) / 60)) torch.save( model.state_dict(), open( os.path.join( 'saved', model_name, 'Epoch_%d_JA_%.4f_DDI_%.4f.model' % (epoch, ja, ddi_rate)), 'wb')) print('') dill.dump(history, open(os.path.join('saved', model_name, 'history.pkl'), 'wb')) # test torch.save( model.state_dict(), open(os.path.join('saved', model_name, 'final.model'), 'wb'))
def fine_tune(fine_tune_name=''): data_path = '../../data/records_final.pkl' voc_path = '../../data/voc_final.pkl' device = torch.device('cuda:0') data = dill.load(open(data_path, 'rb')) voc = dill.load(open(voc_path, 'rb')) diag_voc, pro_voc, med_voc = voc['diag_voc'], voc['pro_voc'], voc[ 'med_voc'] ddi_A = dill.load(open('../../data/ddi_A_final.pkl', 'rb')) split_point = int(len(data) * 2 / 3) data_train = data[:split_point] eval_len = int(len(data[split_point:]) / 2) data_test = data[split_point:split_point + eval_len] # data_eval = data[split_point+eval_len:] voc_size = (len(diag_voc.idx2word), len(pro_voc.idx2word), len(med_voc.idx2word)) model = Leap(voc_size, device=device) model.load_state_dict( torch.load( open(os.path.join("saved", model_name, fine_tune_name), 'rb'))) model.to(device) EPOCH = 30 LR = 0.0001 END_TOKEN = voc_size[2] + 1 optimizer = Adam(model.parameters(), lr=LR) ddi_rate_record = [] for epoch in range(1): loss_record = [] start_time = time.time() random_train_set = [ random.choice(data_train) for i in range(len(data_train)) ] for step, input in enumerate(random_train_set): model.train() K_flag = False for adm in input: target = adm[2] output_logits = model(adm) out_list, sorted_predict = sequence_output_process( output_logits.detach().cpu().numpy(), [voc_size[2], voc_size[2] + 1]) inter = set(out_list) & set(target) union = set(out_list) | set(target) jaccard = 0 if union == 0 else len(inter) / len(union) K = 0 for i in out_list: if K == 1: K_flag = True break for j in out_list: if ddi_A[i][j] == 1: K = 1 break loss = -jaccard * K * torch.mean( F.log_softmax(output_logits, dim=-1)) loss_record.append(loss.item()) optimizer.zero_grad() loss.backward(retain_graph=True) optimizer.step() llprint('\rTrain--Epoch: %d, Step: %d/%d' % (epoch, step, len(data_train))) if K_flag: ddi_rate, ja, prauc, avg_p, avg_r, avg_f1 = eval( model, data_test, voc_size, epoch) end_time = time.time() elapsed_time = (end_time - start_time) / 60 llprint( '\tEpoch: %d, Loss1: %.4f, One Epoch Time: %.2fm, Appro Left Time: %.2fh\n' % (epoch, np.mean(loss_record), elapsed_time, elapsed_time * (EPOCH - epoch - 1) / 60)) torch.save( model.state_dict(), open( os.path.join( 'saved', model_name, 'fine_Epoch_%d_JA_%.4f_DDI_%.4f.model' % (epoch, ja, ddi_rate)), 'wb')) print('') # test torch.save(model.state_dict(), open(os.path.join('saved', model_name, 'final.model'), 'wb'))
with open(filename, 'wb') as fh: dill.dump( dict(seed=seed, dhids=dhids, epochs=epochs, n_per_batch=n_per_batch, eta=eta, alpha=alpha, weights=weights, tau_rc=tau_rc, amp=amp, learners=learners), fh) else: with open(filename, 'rb') as fh: filedata = dill.load(fh) globals().update(filedata) # --- plot results (cols=[train, test], traces=learners) rows = 1 cols = 2 plt.figure(figsize=(7, 4)) # - train subplot ax = plt.subplot(rows, cols, 1) # filt = Alpha(3000, default_dt=n_per_batch) filt = Alpha(10000, default_dt=n_per_batch) for learner in learners:
srep.viz.plotting_style() #%% fig, ax = plt.subplots(2, 2, figsize=(9, 9)) # ########################################################################### # 95% HPD for identifiable expts promoters # ########################################################################### # # loop thru df, not all_samples keys, so we get deterministic order! expt_labels = ("O2_0p5ngmL", "O2_1ngmL", "Oid_1ngmL", "O3_10ngmL") var_labels = ["k_burst", "b", "kR_on", "kR_off"] color_keys = ["green", "blue", "red", "purple"] for i, expt in enumerate(expt_labels): # unpickle sampler, then convert to arviz InfDat obj pklfile = open(f"{repo_rootdir}/data/mcmc_samples/{expt}_sampler.pkl", 'rb') sampler = dill.load(pklfile) pklfile.close() inf_dat = az.convert_to_inference_data(sampler, var_names=var_labels) kR_on_samples = inf_dat.posterior.kR_on.values.flatten() kR_off_samples = inf_dat.posterior.kR_off.values.flatten() x_contour, y_contour = bebi103.viz.contour_lines_from_samples( kR_off_samples, kR_on_samples, levels=0.95, smooth=0.025) ax[0, 1].plot(x_contour[0], y_contour[0], label=expt, linewidth=0.6, color=colors[color_keys[i]]) # ax[0,1].set_xlim(-2,2) # ax[0,1].set_ylim(-2,2) ax[0, 1].set_ylabel(r'$log_{10}(k_R^+/\gamma)$')
import time from sklearn.preprocessing import MinMaxScaler from operator import add def testindexing(): graph_raw_data = open("graph.pkl", "rb") graph_data = pickle.load(graph_raw_data) print(graph_data) if __name__ == "__main__": testindexing() #need to update revised in gce raw_data = open("revised_total_data.pkl", "rb") X_total, X_tr, y_tr, X_te = pickle.load(raw_data) test = np.array([0]) print(X_total.shape) #remove null island graph_raw_data = open("graph.pkl", "rb") graph_data = pickle.load(graph_raw_data) first_ids = graph_data[:, 0] connected_ids = graph_data[:, 1] training_ids = X_total[:, 0] added_features = [] count = 0 length_graph = first_ids.shape[0] length_training = training_ids.shape[0] for row in X_total: id_first = row[0] index = np.searchsorted(first_ids, id_first)
def testindexing(): graph_raw_data = open("graph.pkl", "rb") graph_data = pickle.load(graph_raw_data) print(graph_data)
def experiment_LMKLIEP(which_d): with open('feat_vec_out.'+which_d+'.en.pickle', 'rb') as handle: out_d = pickle.load(handle) with open('feat_vec_in.'+which_d+'.en.pickle', 'rb') as handle: in_d = pickle.load(handle) labels = - np.ones(out_d.shape[0]) predictions = - np.ones(out_d.shape[0]) labels[-50000:] = 1 kliep = KLIEP(init_b=100, seed=0) kliep.fit_CV(out_d, in_d) w = kliep.predict(out_d).ravel() predictions[np.where(w > 1)[0]] = 1 # w = p_te/p_tr print 'total positive:', np.where(predictions == 1)[0].shape, ', out of:', out_d.shape[0] # sorted_ind = np.argsort(w, axis=None)[::-1] # predictions[sorted_ind[0:50000]] = 1 p, r, f, s = precision_recall_fscore_support(labels.astype(int), predictions.astype(int), pos_label=1, average='micro') print 'Precision:', p, print 'Recall:', r, print 'F1:', f, print 'Support:', s,