def plr_sv_feature(name): fst_name = f"fst_{name}" snd_name = f"snd_{name}" if fst_name in dct and snd_name in dct: fst_val = rl.SizedValue(*dct[fst_name]) snd_val = rl.SizedValue(*dct[snd_name]) self.features.append( feature.Feature(name=fst_name, value=fst_val, flip_value=snd_val) ) self.features.append( feature.Feature(name=snd_name, value=snd_val, flip_value=fst_val) )
def __init__(self, args, opt_column=None): """Constructs a collection features. Args: args: constructor argument. One of: 1) A string - assumed to be the name of a collection. 2) A number - assumed to be the ID of a Fusion Table. 3) A geometry. 4) A feature. 5) An array of features. 6) A computed object - reinterpreted as a collection. opt_column: The name of the geometry column to use. Only useful with the string or number constructor arguments. Raises: EEException: if passed something other than the above. """ self.initialize() # Wrap geometries with features. if isinstance(args, geometry.Geometry): args = feature.Feature(args) # Wrap single features in an array. if isinstance(args, feature.Feature): args = [args] if ee_types.isNumber(args) or ee_types.isString(args): # An ID. actual_args = {'tableId': args} if opt_column: actual_args['geometryColumn'] = opt_column super(FeatureCollection, self).__init__( apifunction.ApiFunction.lookup('Collection.loadTable'), actual_args) elif isinstance(args, (list, tuple)): # A list of features. super(FeatureCollection, self).__init__( apifunction.ApiFunction.lookup('Collection'), {'features': [feature.Feature(i) for i in args]}) elif isinstance(args, ee_list.List): # A computed list of features. super(FeatureCollection, self).__init__(apifunction.ApiFunction.lookup('Collection'), {'features': args}) elif isinstance(args, computedobject.ComputedObject): # A custom object to reinterpret as a FeatureCollection. super(FeatureCollection, self).__init__(args.func, args.args, args.varName) else: raise ee_exception.EEException( 'Unrecognized argument type to convert to a FeatureCollection: %s' % args)
def plr_feature(name): fst_name = f"fst_{name}" snd_name = f"snd_{name}" if fst_name in dct and snd_name in dct: fst_val = dct[fst_name] snd_val = dct[snd_name] self.features.append( feature.Feature(name=fst_name, value=fst_val, flip_value=snd_val) ) self.features.append( feature.Feature(name=snd_name, value=snd_val, flip_value=fst_val) )
def get_feature(self, sep, algorithm): #特征选择 feature_inst = feature.Feature() feature_inst.data2vector(data = self.data_inst, sep = sep) feature_inst.select_feat(feat_num = self.feat_num, algorithm = algorithm) self.feat_names = feature_inst.select_feat_name self.feat_inst = feature_inst
def main(argv): del argv o_data_source = tushare_data.DataSource(20000101, '', '', 1, 20120101, 20200106, False, False, True) o_feature = feature.Feature(30, feature.FUT_D5_NORM, 1, False, False) # o_feature = feature.Feature(30, feature.FUT_D5_NORM, 1, False, False) # o_feature = feature.Feature(30, feature.FUT_5REGION5_NORM, 5, False, False) # o_feature = feature.Feature(30, feature.FUT_2AVG5_NORM, 5, False, False) o_wave = ExtremeWave(o_data_source, o_feature, 2, 2, False, 0, 0.1, 5) split_date = 20180101 o_dl_model = dl_model.DLModel('%s_%u' % (o_wave.setting_name, split_date), o_feature.feature_unit_num, o_feature.feature_unit_size, 32, 10240, 0.004, 'mean_absolute_tp0_max_ratio_error') if FLAGS.mode == 'data': o_data_source.DownloadData() o_data_source.UpdatePPData() elif FLAGS.mode == 'testall': o_wave.TradeTestAll() elif FLAGS.mode == 'test': o_wave.TradeTestStock(FLAGS.c, FLAGS.show) elif FLAGS.mode == 'show': o_wave.ShowTradePP(FLAGS.c) elif FLAGS.mode == 'train': tf, tl, vf, vl, td = o_wave.GetDataset(split_date) tl = tl * 100.0 vl = vl * 100.0 o_dl_model.Train(tf, tl, vf, vl, FLAGS.epoch) elif FLAGS.mode == 'rtest': tf, tl, tf, tl, ta = o_wave.GetDataset(split_date) o_dl_model.LoadModel(FLAGS.epoch) o_wave.RTest(o_dl_model, tf, ta, False) exit()
def lr_main(train, test, outfilepath): ''' Linear Regression main function train: filepath of train.csv test: filepath of test_X.csv outfilepath: filepath of predicted_result.csv ''' # Load in data & preprocessing training_data = pd.read_csv(train, sep=",", encoding="big5") # Training W_best = [] for i in range(30): train = feature.Feature(training_data) train, d = train.scaling() train.add_bias() train, _ = train.bagging(train) W = train_lr(train, iternum=100000, lamb=1e-2) W_best.append(W) with open("./model/W_best.pkl", "wb") as o: pickle.dump(W_best, o) with open("./model/W_best.csv", "w") as o: for W in W_best: for line in W: o.write(str(line)) o.write("\n") # Testing and output result test = pd.read_csv(test, sep=",", header=None) test_lr(W_best, test, d, outfilepath, train)
def build_aprx(self): self.features = [] for feature_n in xrange(len(self.X[0])): data = [] for i in xrange(len(self.X)): data.append([self.X[i][feature_n], self.Y[i]]) ftr = feature.Feature() ftr.create_vars(data) ftr.solve_theta() self.features.append(ftr)
def binary_feature(self, sentence, type_feature): self.sentence = sentence #define unclean temporary array data train and label train = [] #jika training iis maka lakukan pencarian binary feature dengan label #contoh : (dict(f1=0, f2=0, f3=0, f4=0, f5=0, f6=0, f7=1, f8=1, f9=0, f12=0, f10=0, f11=0}, "NUM")) if type_feature == "train_iis": for index, data in enumerate(sentence): label = [] token = word_tokenize(data) for index, data in enumerate(token): if "/" in data : #add label to array label.append(self.lbl.search(token[index]).group(1)) #add word to array token[index] = self.w.search(token[index]).group(1) else: label.append("O") for index, data in enumerate(token): #feature processing panggil class feature featuretrain = f.Feature() result = featuretrain.template_feature(token, label, index) #result = template_feature(token, label, index) train.append(result) else: #jika training ner atau selain iis maka hanya melakukan pencarian binary feature, tidak dengan label #contoh : (dict(f1=0, f2=0, f3=0, f4=0, f5=0, f6=0, f7=1, f8=1, f9=0, f12=0, f10=0, f11=0})) token = word_tokenize(sentence) label = [] for index, data in enumerate(token): #feature processing panggil class feature featuretrain = f.Feature() result = featuretrain.template_feature(token, label, index) #result = template_feature(token, label, index) train.append(result) # filter array empty/none karena Other atau entitas O tidak diproses train_set = filter(None, train) #print train_set return train_set
def window_func(win): col_time = 'time' col_vals = list(win.columns) col_vals.remove('time') col_vals.remove('event_seizure') col_event = 'event_seizure' time = win[col_time] vals = win[col_vals] event = win[col_event] ## TODO: problem with data stack if add info data func = feature.Feature(time=time, vals=vals, event=event).func() return func
def add_feature_dif_bonus(features, feature_name, sex, pid1, pid2, min_date=None, max_date=None): value = get_dif_bonus(sex, pid1, pid2, min_date=min_date, max_date=max_date) features.append( feature.Feature(name=feature_name, value=value, flip_value=-value))
def get_feature(self, fname): feature = self.features.get(fname) if feature is None: feature_info = self.content['features'].get(fname) if feature_info is None: return None import feature as F feature = F.Feature(fname, feature_info.get('name', None), self.version, updatesite=self.site_root, **feature_info) self.features[fname] = feature return feature
def find_features(where): import path, feature features = [] where = path.path(where) for plat_dir in where.dirs(): if plat_dir.name not in feature.compatible_platform_versions: continue for feat_dir in plat_dir.dirs(): feat = feature.Feature(feat_dir.name, platform_version=plat_dir.name, dir=where) features.append(feat) return features
def feature_generate(pair): ftr = feature.Feature() res = [] # ekstraksi fitur-fitur pasangan kalimat dalam bentuk list for x, y in pair: fc = ftr.first_capital(x, y) cd = ftr.capital_diff(x, y) sld = ftr.sent_lenght_diff(x, y) cod = ftr.comma_diff(x, y) dd = ftr.dot_diff(x, y) it = ftr.italic_diff(x, y) cb = ftr.count_bigram(x, y) cp = ftr.count_postag(x, y) sim = ftr.similarity(x, y) res.append([fc, cd, sld, cod, dd, it, cp, cd, sim]) return res
def main(argv): del argv o_data_source = tushare_data.DataSource(20000101, '', '', 1, 20000101, 20200403, False, False, True) o_feature = feature.Feature(30, feature.FUT_D5_NORM_PCT, 1, False, False) # o_feature = feature.Feature(30, feature.FUT_D5_NORM, 1, False, False) # o_feature = feature.Feature(30, feature.FUT_5REGION5_NORM, 5, False, False) # o_feature = feature.Feature(30, feature.FUT_2AVG5_NORM, 5, False, False) o_trade = Breakup(o_data_source, o_feature, PPI_close_100_avg, 10, 10, 3.0, 0.1) split_date = 20180101 o_dl_model = dl_model.DLModel('%s_%u' % (o_trade.setting_name, split_date), o_feature.feature_unit_num, o_feature.feature_unit_size, 32, 10240, 0.004, 'mean_absolute_tp0_max_ratio_error') if FLAGS.mode == 'data': o_data_source.DownloadData() o_data_source.UpdatePPData() elif FLAGS.mode == 'testall': o_trade.TradeTestAll(True, FLAGS.show) elif FLAGS.mode == 'test': o_data_source.DownloadStockData(FLAGS.c) o_data_source.UpdateStockPPData(FLAGS.c) start_time = time.time() # o_trade.TradeTestStock(FLAGS.c, FLAGS.show) o_trade.Test(FLAGS.c) elif FLAGS.mode == 'train': tf, tl, vf, vl, td = o_trade.GetDataset(split_date) tl = tl * 100.0 vl = vl * 100.0 o_dl_model.Train(tf, tl, vf, vl, FLAGS.epoch) elif FLAGS.mode == 'rtest': tf, tl, tf, tl, ta = o_trade.GetDataset(split_date) o_dl_model.LoadModel(FLAGS.epoch) o_trade.RTest(o_dl_model, tf, ta, False) elif FLAGS.mode == 'dsw': dataset = o_trade.ShowDSW3DDataset() elif FLAGS.mode == 'show': dataset = o_trade.ShowTradePP(FLAGS.c) exit()
def lr_main(train, test, outfilepath): ''' Linear Regression main function train: filepath of train.csv test: filepath of test_X.csv outfilepath: filepath of predicted_result.csv ''' training_data = pd.read_csv(train, sep=",", encoding="big5") train = feature.Feature(training_data) W_best = train_lr(train, model=1) ''' with open("./model/W.pkl", "wb") as o: pickle.dump(W_best, o) ''' # Testing and output result test = pd.read_csv(test, sep=",", header=None) test_lr(W_best, test, outfilepath, train)
def main(): """ Controls the flow """ start = time.time() # load the datasets loader = load.DataLoader() # return the training and the testing datasets train, test, strat, streets = loader.load_data() featurer = feature.Feature(train, test, streets) train, test, targets_tr, features = featurer.format() runner = classifier.Modeler(train, test, targets_tr, strat, features) runner.run_predictions() runner.run_Kfold() end = time.time() print "Total Time: " + str(end - start)
def add_features(features, date, h2hobj, time_discounts=(0.95, 0.90, 0.85, 0.80, 0.75)): assert h2hobj is not None assert date is not None items = [ Item( is_left_win=h2hobj.fst_player == mch.first_player, date=mch.date if mch.date else tour.date, ) for tour, mch, _ in h2hobj.tour_match_aset ] calc = Calc(date, items) for time_discount in time_discounts: feat_name = "h2h_{:.2f}".format(time_discount).replace(".", "") calc.time_discount = time_discount value = calc.direct() features.append( feature.Feature( name=feat_name, value=value, flip_value=None if value is None else 1.0 - value, ) )
def lr_main(train, test, outfilepath): ''' Linear Regression main function train: filepath of train.csv test: filepath of test_X.csv outfilepath: filepath of predicted_result.csv ''' training_data = pd.read_csv(train, sep=",", encoding="big5") train = feature.Feature(training_data) train, d = train.scaling() train.add_bias() with open("./model/W_sbl.pkl", "rb") as w: W_best = pickle.load(w) # Testing and output result test = pd.read_csv(test, sep=",", header=None) test_lr(W_best, test, d, outfilepath, train) '''
def main(argv): del argv o_data_source = tushare_data.DataSource(20000101, '', '', 1, 20100101, 20200306, False, False, True) o_feature = feature.Feature(7, feature.FUT_D5_NORM, 1, False, False) o_vol_wave = VolWave(o_data_source, o_feature, 0.1) # split_date = 20180101 # o_dl_model = dl_model.DLModel('%s_%u' % (o_vol_wave.setting_name, split_date), # o_feature.feature_unit_num, # o_feature.feature_unit_size, # 32, 10240, 0.004, 'mean_absolute_tp0_max_ratio_error') if FLAGS.mode == 'data': o_data_source.DownloadData() o_data_source.UpdatePPData() elif FLAGS.mode == 'testall': o_vol_wave.TradeTestAll(True, FLAGS.show) elif FLAGS.mode == 'test': o_data_source.DownloadStockData(FLAGS.c) o_data_source.UpdateStockPPData(FLAGS.c) start_time = time.time() o_vol_wave.TradeTestStock(FLAGS.c, FLAGS.show) print(time.time() - start_time) elif FLAGS.mode == 'train': tf, tl, vf, vl, td = o_vol_wave.GetDataset(split_date) tl = tl * 100.0 vl = vl * 100.0 o_dl_model.Train(tf, tl, vf, vl, FLAGS.epoch) elif FLAGS.mode == 'rtest': tf, tl, tf, tl, ta = o_vol_wave.GetDataset(split_date) o_dl_model.LoadModel(FLAGS.epoch) o_vol_wave.RTest(o_dl_model, tf, ta, False) elif FLAGS.mode == 'dsw': dataset = o_vol_wave.ShowDSW3DDataset() elif FLAGS.mode == 'show': dataset = o_vol_wave.ShowTradePP(FLAGS.c) exit()
savestring = bstring[8] + '/' + bstring[9] # save_csv1=os.path.join(saving_path1,str(save_name)+'M.txt') save_image = os.path.join(save_map, str(save_name)) ### get the image and label data_source = (np.loadtxt(label_source_path, dtype=np.str, delimiter=","))[1:, 1:].astype(np.float) img_source = cv2.imread(file_source_path, 1) img_target = cv2.imread(file_target_path, 1) ### pre-register pre = register.Pre(img_source, img_target, fixedsize) M_warp = pre.pre_register() width2 = pre.width2 height2 = pre.height2 img_warp = cv2.warpAffine(img_source, M_warp, (width2, height2)) ### feature register feature1 = feature.Feature(img_warp, img_target, shrink_num2) M2, I, lenmatch, threshold, in_num, Ir = feature1.register() ### get the shrink image after transform,and the mix of two image shrink1 = feature1.dimg1 shrink2 = feature1.dimg2 w2, h2 = feature1.width2, feature1.height2 warp = cv2.warpAffine(shrink1, M2, (w2, h2)) merge = np.uint8(shrink2 * 0.5 + warp * 0.5) cv2.imwrite(str(save_image) + '-mix.jpg', merge) plt.imshow(merge) plt.show() ### get the transform of the total size M = M2 M[0, 2] = 5 * M2[0, 2] M[1, 2] = 5 * M2[1, 2] M_warp = np.vstack((M_warp, [0, 0, 1]))
def main(argv): del argv end_date = 20200306 split_date = 20100101 o_data_source = tushare_data.DataSource(20000101, '', '', 10, 20000101, end_date, False, False, True) o_feature = feature.Feature(10, feature.FUT_D5_NORM, 1, False, False) obj = OpenClose(o_data_source, o_feature, not FLAGS.overlap_feature) o_dl_model = dl_model.DLModel( '%s_%u' % (obj.setting_name, split_date), o_feature.feature_unit_num, o_feature.feature_unit_size, # 32, 10240, 0.04, 'mean_absolute_tp0_max_ratio_error') # rtest<0 # 4, 10240, 0.04, 'mean_absolute_tp0_max_ratio_error') # rtest<0 # 4, 10240, 0.01, 'mean_absolute_tp0_max_ratio_error') # rtest:0.14 32, 10240, 0.03, 'mean_absolute_tp_max_ratio_error_tanhmap', 50) # rtest:0.62 # 16, 10240, 0.01, 'mean_absolute_tp0_max_ratio_error') # rtest<0 # 16, 10240, 0.01, 'mean_absolute_tp_max_ratio_error_tanhmap', 100) if FLAGS.mode == 'datasource': o_data_source.DownloadData() o_data_source.UpdatePPData() elif FLAGS.mode == 'dataset': obj.CreateDataSet() elif FLAGS.mode == 'public_dataset': obj.CreateDataSet() public_dataset = obj.PublicDataset() file_name = './public/data/dataset.npy' np.save(file_name, public_dataset) elif FLAGS.mode == 'train': tf, tl, vf, vl, td = obj.GetDataset(split_date) # tf, tl, vf, vl, va = obj.GetDatasetRandom(0.5) train_epoch = FLAGS.epoch if FLAGS.epoch > 0 else 250 o_dl_model.Train(tf, tl, vf, vl, train_epoch) elif FLAGS.mode == 'rtest': tf, tl, vf, vl, va = obj.GetDataset(split_date) # tf, tl, vf, vl, va = obj.GetDatasetRandom(0.5) o_dl_model.LoadModel(FLAGS.epoch) obj.RTest(o_dl_model, vf, va, False) # elif FLAGS.mode == 'dqntest': # o_dl_model.LoadModel(FLAGS.epoch) # o_dsfa = dsfa3d_dataset.DSFa3DDataset(o_data_source, o_feature) # o_dqn_test = dqn_test.DQNTest(o_dsfa, split_date, o_dl_model) # o_dqn_test.Test(1, FLAGS.pt, True, FLAGS.show) # elif FLAGS.mode == 'dqntestall': # o_dl_model.LoadModel(FLAGS.epoch) # o_dsfa = dsfa3d_dataset.DSFa3DDataset(o_data_source, o_feature) # o_dqn_test = dqn_test.DQNTest(o_dsfa, split_date, o_dl_model) # o_dqn_test.TestAllModels(1, FLAGS.pt) # elif FLAGS.mode == 'predict': # o_dl_model.LoadModel(FLAGS.epoch) # o_data_source.SetPPDataDailyUpdate(20180101, 20200323) # o_dsfa = dsfa3d_dataset.DSFa3DDataset(o_data_source, o_feature) # o_dqn_test = dqn_test.DQNTest(o_dsfa, split_date, o_dl_model) # o_dqn_test.Test(1, FLAGS.pt, True, FLAGS.show) elif FLAGS.mode == 'dsw': dataset = obj.ShowDSW3DDataset() elif FLAGS.mode == 'show': dataset = obj.ShowTradePP(FLAGS.c) elif FLAGS.mode == 'showlabel': dataset = obj.ShowLabel() elif FLAGS.mode == 'debug': dataset = np.load(obj.FileNameDataset()) print("dataset: {}".format(dataset.shape)) dataset = np_common.Sort2D(dataset, [obj.index_increase], [False]) dataset = dataset[:5] obj.ShowDataSet(dataset, 'dataset') elif FLAGS.mode == 'clean': obj.Clean() o_dl_model.Clean() elif FLAGS.mode == 'pp': o_data_source.ShowStockPPData(FLAGS.c, FLAGS.date) elif FLAGS.mode == 'vol': o_data_source.ShowAvgVol(100000) exit()
def build_data(fname, user_cluster=None, pv_model=None, cv=10): """ Loads and process data. """ feature_extractor = feature.Feature() revs = [] vocab = defaultdict(float) ins_idx, tw_w_clu = 0, 0 users = set() if pv_model is not None: dim = pv_model.layer1_size keys = ["PV_%d" % i for i in range(dim)] if user_cluster is not None: key_clus = ["PV_clu_%d" % i for i in range(dim)] with open(fname, "rb") as f: for line in f: line = line.strip() spidx = line.rfind("|") rev = line[:spidx].strip() metas = line[spidx + 1:].strip().split(",") label = int(metas[0]) cluster = 0 if user_cluster is not None and metas[1] in user_cluster: cluster = user_cluster[metas[1]] users.add(metas[1]) tw_w_clu += 1 if len(metas) > 2: split = int(metas[2]) else: #split = np.random.randint(0,cv) split = ins_idx % cv nrc_feat = feature_extractor.NRC_feature_extractor(line) orig_rev = clean_str(rev) words = set(orig_rev.split()) for word in words: vocab[word] += 1 if pv_model is not None: pv_dict = dict(zip(keys, pv_model['SENT_%d' % ins_idx])) if user_cluster is not None: pv_dict.update( dict(zip(key_clus, pv_model['CLUSTER_%d' % cluster]))) else: pv_dict = {} datum = { "y": label, "text": orig_rev, "features": nrc_feat, "pv": pv_dict, "num_words": len(orig_rev.split()), "cluster": cluster, "split": split } revs.append(datum) ins_idx += 1 max_l = np.max(pd.DataFrame(revs)["num_words"]) logger.info( "finish building data: %d tweets, in which %d tweets of %d users have link information" % (ins_idx, tw_w_clu, len(users))) logger.info( "tweets of users have no links are associated with cluster ID 0 (the biggest cluster)" ) logger.info("vocab size: %d, max tweet length: %d" % (len(vocab), max_l)) return revs, vocab, max_l
img2_i = np.zeros(img2.shape, dtype=np.uint8) mask2 = mask2s[m][:] > 0 for i in range(3): img1_i[:, :, i] = img1[:, :, i] * mask1 #+(1-mask1)*255 img2_i[:, :, i] = img2[:, :, i] * mask2 #+(1-mask2)*255 ### pre register the connected domains pre = register.Pre(img1_i, img2_i, 200) M_warp = pre.pre_register() width2 = pre.width2 height2 = pre.height2 img_warp_i = cv2.warpAffine(img1_i, M_warp, (width2, height2)) plt.subplot(121), plt.imshow(img_warp_i) plt.subplot(122), plt.imshow(img2_i) plt.show() ### get feature and register feature1 = feature.Feature(img_warp_i, img2_i, 1) ### get the M ,register match image ,string lenmatch,number of interpoints,match image M, I_, lenmatch, threshold, interiornum, Ir = feature1.register() match_path = os.path.join(result_path, 'match') match_filename = os.path.join(match_path, str(base) + str(m) + '-r.jpg') cv2.imwrite(match_filename, Ir) # M=cv2.getRotationMatrix2D((0,0),0,1) img3_i = cv2.warpAffine(img_warp_i, M, (width2, height2)) ### get the NGF of image and image after register pre2 = register.Pre(img3_i, img2_i, 200) dst1, dst2 = pre2.crop_cross() distance, ngf = pre2.NGF(dst1, dst2) ### get the total martix of pre-register and feature register M2_ = np.vstack((M_warp, [0, 0, 1])) M1_ = np.vstack((M, [0, 0, 1]))
title = file_pt.readline() output = open(data_io.get_paths()["submission_path"], "a") tot_fet = None for l in file_pt.readlines(): res = l.split(",") fet = f.create_features_from_res(res) if tot_fet == None: tot_fet = fet else: tot_fet = numpy.vstack((tot_fet, fet)) pred = classifier.predict_proba(fet[:, 3:]) sorted_pred = sorted(zip(res[1].split(), pred[:, 1]), key=lambda a: a[1], reverse=True) #print sorted_pred output.write(res[0] + "," + " ".join(map(lambda a: a[0], sorted_pred)) + "\n") output.close() numpy.savetxt(data_io.get_paths()["test_feature_path"], tot_fet.astype(float), fmt='%f', delimiter=",") if __name__ == "__main__": p = parser.Parser() p.parse_csv() f = feature.Feature(p) classifier = train(f, data_io.get_paths()["train_path"]) predict(f, classifier, data_io.get_paths()["valid_path"])
def main(): ### write the name of information with open(result_filename, 'a', encoding='utf-8') as f1: f1.writelines('{0:<20}{1:<65}{2:<25}{3:<25}{4:<15}{5:<15}{6:<20}\n'. format('dir ','name', 'rTRE(median_rTRE)','match_num','threshold','inliers','times')) ### get the path of image/label file for dir1 in image_dir1: print(dir1) image_path1=os.path.join(image_path, dir1) image_dir2 = sorted(os.listdir(image_path1)) readlabel = os.path.join(label_path, dir1) m=os.listdir(readlabel)[0] for dir2 in image_dir2: saving_path = os.path.join(result_path, str(dir1),str(dir2)) if not os.path.exists(saving_path): os.makedirs(saving_path) image_path2=os.path.join(image_path1, dir2) file_image=os.listdir(image_path2) for x in file_image: str_img=str(x)[-4:] file_target_path = os.path.join(image_path2,str(dir2)+str_img) file_target=os.path.basename(file_target_path) file = [x for x in file_image if x !=file_target] label_target_path=os.path.join(readlabel,m,str(file_target)[:-4]+'.csv') for idx in file: file_source_path=os.path.join(image_path2,str(idx)) label_source_path=os.path.join(readlabel,m,str(os.path.basename(idx)[:-4])+'.csv') ### get the time of the begining of register start_time1 = time.time() ### get the image and the label img_source = cv2.imread(file_source_path,1) img_target = cv2.imread(file_target_path,1) data_source = (np.loadtxt(label_source_path, dtype=np.str, delimiter=","))[1:, 1:].astype(np.float) data_target = (np.loadtxt(label_target_path, dtype=np.str, delimiter=","))[1:, 1:].astype(np.float) ### define path to save save_name = os.path.join(os.path.basename(idx)[:-4]+'to'+file_target[:-4]) save_path = os.path.join(saving_path, save_name) error_image=os.path.join(error_path,str(save_name)+str(dir1)+'.jpg') save_data_path=os.path.join(saving_path,'datalabel') if not os.path.exists(save_data_path): os.makedirs(save_data_path) print('load:',time.time()-start_time1) ### pre-register pre=register.Pre(img_source,img_target,fixedsize) M_warp=pre.pre_register() width2=pre.width2 height2=pre.height2 img_warp=cv2.warpAffine(img_source, M_warp, (width2, height2)) print('pre:',time.time()-start_time1) ### feature register feature1=feature.Feature(img_warp,img_target,shrink_num) M,I,lenmatch,threshold,in_num,Ir=feature1.register() print('feature:',time.time()-start_time1) ### get the shrink image after transform,and the mix of two image shrink1=feature1.dimg1 shrink2=feature1.dimg2 w2,h2=feature1.width2,feature1.height2 warp = cv2.warpAffine(shrink1, M, (w2, h2)) merge = np.uint8(shrink2* 0.5 + warp * 0.5) plt.imshow(merge) plt.show() ### save images cv2.imwrite(str(save_path)+'-merge.jpg',merge) cv2.imwrite(str(save_path)+'-warp.jpg', warp) cv2.imwrite(str(save_path)+'-match.jpg', Ir) cv2.imwrite(str(save_path)+'-register.jpg', I) ### get the transform of the total size M1=M M1[0,2]=5*M1[0,2] M1[1,2]=5*M1[1,2] M1_=np.vstack((M1,[0,0,1])) M2_=np.vstack((M_warp,[0,0,1])) M3=np.dot(M1_,M2_) # np.savetxt(os.path.join(save_M ,str(idx)[:-4]+'.txt'),M1) # data_warp=(f.get_data(data_source,M_warp)[1:,1:]).astype('float') ### get the rTRE and the new data of landmarks,save data rTRE,_,_,data_save= c.get_TRE(data_source, data_target, M3,height2,width2) data_save1=c.get_data(data_source,M3) np.savetxt(os.path.join(save_data_path,str(idx)[:-4]+'.csv'),data_save, fmt=('%s,%s,%s'),delimiter=',') np.savetxt(os.path.join(save_data_path,str(idx)[:-4]+'-1.csv'),data_save1, fmt=('%s,%s,%s'),delimiter=',') print('save:',time.time()-start_time1) print(rTRE) ### save image that rTRE is bigger than 0.02 if rTRE>0.02: cv2.imwrite(str(error_image), merge) del data_save,data_source,data_target,#inliers,nkp1_s,nkp1_t gc.collect() del img_source,img_target,I gc.collect() # ============================================================================= # ### draw the landmarks in the image # for i in range(len(data_source)): # x=data_source[i][0] # y=data_source[i][1] # k=np.array((x,y),np.int32).reshape((-1,1,2)) # cv2.polylines(img_target,k,True,(0,255,0),100) # for i in range(len(data_target)): # x=data_target[i][0] # y=data_target[i][1] # k=np.array((x,y),np.int32).reshape((-1,1,2)) # cv2.polylines(img_target,k,True,(255,0,0),100) # for i in range(1,len(data_save)): # x3=np.float32(data_save[i][1]) # y3=np.float32(data_save[i][2]) # k=np.array((x3,y3),np.int32).reshape((-1,1,2)) # cv2.polylines(img_target,k,True,(0,0,255),100) # ============================================================================= end_time1 = time.time() print(idx) print(save_name) time_register =(end_time1 - start_time1) ### write some information needed with open(result_filename, 'a', encoding='utf-8') as f1: f1.writelines('{0:<20}{1:<65s}{2:<25s}{3:<25s}{4:<15s}{5:<15s}{6:<20s}\n' .format(str(dir1),str(save_name),str(rTRE),str(lenmatch), str(threshold),str(in_num),str(time_register))) # with open(result_filename, 'a', encoding='utf-8') as f1: # f1.writelines('{0:<20}{1:<65s}{2:<25s}{3:<15s}{4:<15s}{5:<15s}\n' # .format(str(dir1),str(save_name),str(rTRE),str(err)[:8],str(iteration),str(time_register))) end_time = time.time() sumtime =(end_time-start_time) with open(result_filename, 'a', encoding='utf-8') as f1: f1.writelines('{0:<20}{1:<20n}\n'.format('all of time:',sumtime)) print(result_path)
feature_list = kwargs.get('feature_list', None) if not feature_list: self.name = self.name+'(-irt)' self.train_x = self.select_features(self.feature.features_train, feature_list) self.train_y = self.feature.label_train.values self.feature_names = self.train_x.columns # 评估训练集上的效果 self.train_y_pred = self.predict(self.train_x) self.train_ev = self.evaluation.evaluate(y_true=self.train_y, y_pred=self.train_y_pred, threshold=0.5) return self def predict(self, x: pd.DataFrame = None) -> np.ndarray: if x is None: x = self.feature.features_test y_pred = self.predict_by_theta(x) return y_pred if __name__ == "__main__": import feature ft = feature.Feature() ft.fit() # ft.select() model = Irt(ft) model.predict() model.evaluate()
CAR_SIMPLE = "simple" CAR_USER = "******" CAR_NESTED = "nested" CAR_BELIEF = "belief" CAR_CANNED = "canned" CAR_NEURAL = "neural" CAR_COPY = "copy" # Rewards are various custom reward functions REWARD_MIDDLE_LANE = "r_middle_lane" REWARD_LEFT_LANE = "r_left_lane" REWARD_RIGHT_LANE = "r_right_lane" REWARD_FUNCTIONS = { REWARD_MIDDLE_LANE: feature.Feature(lambda t, x, u: -(x[0])**2), REWARD_LEFT_LANE: feature.Feature(lambda t, x, u: np.exp((-0.5 * (x[0] + 0.13)**2) / .04)), REWARD_RIGHT_LANE: feature.Feature(lambda t, x, u: -(x[0] - 0.13)**2), } # car_from {{{ # car_from constructs a car from a car declarative # definition. e.g., # # def = { # "kind": CAR_SIMPLE, # "x0": [-.13, 0.0, math.pi/2., 0.5],
(self.code_index_map[ts_code], ts_code)) def CreateDSFa3DDataset(self): dataset_file_name = self.FileNameDSFa3DDataset() self.dataset = np.zeros( (len(self.date_list), len(self.code_list), self.feature.unit_size)) # base_common.ListMultiThread(CreateDSFa3DSplitMTFunc, self, 1, self.code_list) for ts_code in self.code_list: self.CreateDSFa3DSplitStock(ts_code) base_common.MKFileDirs(dataset_file_name) np.save(dataset_file_name, self.dataset) self.dataset = None def GetDSFa3DDataset(self): dataset_file_name = self.FileNameDSFa3DDataset() if not os.path.exists(dataset_file_name): self.CreateDSFa3DDataset() return np.load(dataset_file_name) if __name__ == "__main__": data_source = tushare_data.DataSource(20000101, '', '', 1, 20000101, 20200106, False, False, True) data_source.ShowStockCodes() o_feature = feature.Feature(30, feature.FUT_D5_NORM, 1, False, False) o_dataset = DSFa3DDataset(data_source, o_feature) temp_dataset = o_dataset.GetDSFa3DDataset() print("dataset: {}".format(temp_dataset.shape))
def main(argv): del argv code_filter = '\ 000001.SZ,\ 000002.SZ,\ 000063.SZ,\ 000538.SZ,\ 000541.SZ,\ 000550.SZ,\ 000560.SZ,\ 000561.SZ,\ 000584.SZ,\ 000625.SZ,\ 000650.SZ,\ 000651.SZ,\ 000721.SZ,\ 000800.SZ,\ 000802.SZ,\ 000858.SZ,\ 000898.SZ,\ 000913.SZ,\ 000927.SZ,\ 000932.SZ,\ 000937.SZ,\ 000938.SZ,\ 000951.SZ,\ 000959.SZ,\ 001696.SZ,\ 600000.SH,\ 600006.SH,\ 600085.SH,\ 600104.SH,\ 600109.SH,\ 600115.SH,\ 600137.SH,\ 600177.SH,\ 600198.SH,\ 600199.SH,\ 600600.SH,\ 600601.SH,\ 600609.SH,\ 600612.SH,\ 600623.SH,\ 600624.SH,\ 600664.SH,\ 600679.SH,\ 600702.SH,\ 600718.SH,\ 600809.SH' o_data_source = tushare_data.DataSource(20000101, '', code_filter, 1, 20130101, 20200106, False, False, True) o_feature = feature.Feature(30, feature.FUT_D5_NORM, 1, False, False) # o_feature = feature.Feature(30, feature.FUT_D5_NORM, 1, False, False) # o_feature = feature.Feature(30, feature.FUT_5REGION5_NORM, 5, False, False) # o_feature = feature.Feature(30, feature.FUT_2AVG5_NORM, 5, False, False) o_avg_wave = AvgWave(o_data_source, o_feature, PPI_close_30_avg, MODE_GRAD, 0, 0, 0.1) split_date = 20180101 o_dl_model = dl_model.DLModel( '%s_%u' % (o_avg_wave.setting_name, split_date), o_feature.feature_unit_num, o_feature.feature_unit_size, 32, 10240, 0.004, 'mean_absolute_tp0_max_ratio_error') if FLAGS.mode == 'data': o_data_source.DownloadData() o_data_source.UpdatePPData() elif FLAGS.mode == 'testall': o_avg_wave.TradeTestAll(True, FLAGS.show) elif FLAGS.mode == 'test': o_data_source.DownloadStockData(FLAGS.c) o_data_source.UpdateStockPPData(FLAGS.c) start_time = time.time() o_avg_wave.TradeTestStock(FLAGS.c, FLAGS.show) print(time.time() - start_time) elif FLAGS.mode == 'train': tf, tl, vf, vl, td = o_avg_wave.GetDataset(split_date) tl = tl * 100.0 vl = vl * 100.0 o_dl_model.Train(tf, tl, vf, vl, FLAGS.epoch) elif FLAGS.mode == 'rtest': tf, tl, tf, tl, ta = o_avg_wave.GetDataset(split_date) o_dl_model.LoadModel(FLAGS.epoch) o_avg_wave.RTest(o_dl_model, tf, ta, False) elif FLAGS.mode == 'dsw': dataset = o_avg_wave.ShowDSW3DDataset() elif FLAGS.mode == 'show': dataset = o_avg_wave.ShowTradePP(FLAGS.c) exit()
import selector import feature # TEST 1 - Paper candidates = ["Gun", "Bell"] s1 = selector.Selector(candidates) f1 = feature.Feature("Applicator Cost", 5) s1.add_ndn_measurable_feature(f1, feature.Mode.MINIMAZING, [120e3, 140e3]) f2 = feature.Feature("Safety Issues", 5) s1.add_immeasurable_feature(f2, [4, 3]) s1.calc_premium_values() # TEST 2 - Paper candidates = ["Gun", "Bell"] s2 = selector.Selector(candidates) f1 = feature.Feature("Transfer Efficiency", 3) s2.add_immeasurable_feature(f1, [2, 4]) f2 = feature.Feature("Part Ground Sensitivity", 2) s2.add_immeasurable_feature(f2, [4, 2]) f3 = feature.Feature("Penetration Into Dificult Areas", 2) s2.add_immeasurable_feature(f3, [4, 1]) f4 = feature.Feature("Ease of Path Teaching", 3) s2.add_immeasurable_feature(f4, [2, 4])