def int_communication(data_dir, freq, single, adr_change, backend, skip_meas, clk_enable): GPIO_DR = 6 i = 0 while os.path.exists(os.path.join(data_dir, 'data_' + str(i) + '.dat')): i += 1 global write_file if not skip_meas: import te_setup #import RPi.GPIO as GPIO import RPIO as GPIO import spidev spi = spidev.SpiDev() write_file = open(os.path.join(data_dir, 'data_' + str(i) + '.dat'), 'w') global spi spi.open(0, 0) spi.max_speed_hz = (freq) GPIO.setmode(GPIO.BCM) GPIO.setup(GPIO_DR, GPIO.IN, pull_up_down=GPIO.PUD_DOWN) GPIO.add_event_detect(GPIO_DR, GPIO.RISING) if not adr_change: GPIO.add_event_callback(GPIO_DR, callback) else: GPIO.add_event_callback(GPIO_DR, callback_adr) if single: if not adr_change: callback(6) else: callback_adr(6) else: try: if clk_enable: GPIO.setup(5, RPIO.ALT0) while True: time.sleep(1) except KeyboardInterrupt: GPIO.setup(5, RPIO.INPUT) GPIO.cleanup() #global write_file time.sleep(0.1) write_file.close() print('Written to data_' + str(i) + '.dat') if backend == 'display_graph': data_manager.data_manager(directory=data_dir, data_size=1000, train_size=100, split_mode=False, attenate_flag=False, save_as_png=False).plot()
def int_communication(data_dir, freq, single, adr_change, backend, skip_meas, clk_enable): GPIO_DR = 6 i = 0 while os.path.exists(os.path.join(data_dir, 'data_' + str(i) + '.dat')): i += 1 global write_file if not skip_meas: import te_setup #import RPi.GPIO as GPIO import RPIO as GPIO import spidev spi = spidev.SpiDev() write_file = open(os.path.join(data_dir, 'data_' + str(i) + '.dat'), 'w') global spi spi.open(0,0) spi.max_speed_hz=(freq) GPIO.setmode(GPIO.BCM) GPIO.setup(GPIO_DR, GPIO.IN, pull_up_down=GPIO.PUD_DOWN) GPIO.add_event_detect(GPIO_DR, GPIO.RISING) if not adr_change: GPIO.add_event_callback(GPIO_DR, callback) else: GPIO.add_event_callback(GPIO_DR, callback_adr) if single: if not adr_change: callback(6) else: callback_adr(6) else: try: if clk_enable: GPIO.setup(5, RPIO.ALT0) while True: time.sleep(1) except KeyboardInterrupt: GPIO.setup(5, RPIO.INPUT) GPIO.cleanup() #global write_file time.sleep(0.1) write_file.close() print('Written to data_' + str(i) + '.dat') if backend == 'display_graph': data_manager.data_manager(directory=data_dir, data_size=1000, train_size=100, split_mode=False, attenate_flag=False, save_as_png=False).plot()
def __init__(self, client): super().__init__(client) self.func_tbl = { 'status': [self.cmd_status, 'bbコマンドグループのステータス出力'], 'resetLeague': [self.cmd_resetLeague, 'リーグの状態を完全初期化'], 'createTeam': [self.cmd_createTeam, 'チームをつくる'], 'createAth': [self.cmd_createAth, 'create athlete 選手データを登録する'], 'showTeam': [self.cmd_showTeam, 'チームデータを閲覧する'], 'addMatch': [self.cmd_addMatch, '試合を追加する'], 'addStadium': [self.cmd_addStadium, '球場を登録する'], 'startSeason': [self.cmd_startSeason, 'レギュラーシーズンを開幕する'], 'setBattingOrder': [self.cmd_setBattingOrder, 'バッターを打順に組み込む'], 'setStartingPitcher': [self.cmd_setStartingPitcher, '先発ピッチャーを指名する'], 'setBattingOrderGroup': [self.cmd_setBattingOrderGroup, 'バッターを打順に組み込む'], 'imReady': [self.cmd_imReady, '試合前の準備が完了したことを通知する'], 'drawIn': [self.cmd_drawIn, '前進守備'], 'walk': [self.cmd_walk, '敬遠'], 'showStartMem': [self.cmd_showStartMem, '現在登録されているスタメンを見る'], 'bunt': [self.cmd_bunt, 'バント'], 'changePitcher': [self.cmd_changePitcher, 'ピッチャー交代'], 'debug': [self.cmd_dbg, 'おちんちんびろーん'], 'debugSetIning': [self.cmd_dbg_setIning, 'おちんちんびろーん'], } self.cmd_group_name = "bb" self.client = client self.league_status = LeagueStatus.OFF_SEASON self.db = data_manager(client) self.rule = bb_rule(self.db, self.client)
def object_free(self, num): dm = data_manager(self.data_file_name) values = dm.unpickle_data() values["obj"].append(int(num)) values["obj"].sort() dm.pickle_data(values) return True
def standard_dm(ref_cell_num, stage_fname="data/base_data/stage_hpf.json", no10ss=False, **kwargs): stage_dict = json.load(open(stage_fname)) dm = data_manager(ref_cell_num, **kwargs) dm.register_tomoseq("data/base_data/tomo_seq/zfshield", stage_dict["shield"]) dm.register_sc_seq("data/base_data/sc/staged_exp/shield.csv", stage_dict["shield"], stage="shield") dm.register_sc_seq("data/base_data/sc/staged_exp/epiboly_75.csv", stage_dict["epiboly75"], stage="epiboly75") dm.register_sc_seq("data/base_data/sc/staged_exp/epiboly_90.csv", stage_dict["epiboly90"], stage="epiboly90") # dm.register_sc_seq("data/base_data/sc/staged_exp/epiboly_bud.csv", stage_dict["bud"], stage="bud") dm.register_sc_seq("data/base_data/sc/staged_exp/somite_3.csv", stage_dict["3ss"], stage="3ss") dm.register_sc_seq("data/base_data/sc/staged_exp/somite_6.csv", stage_dict["6ss"], stage="6ss") if not no10ss: dm.register_tomoseq_ss("data/base_data/tomo_seq/zf10ss", stage_dict["10ss"]) gene_df = pd.read_csv("data/gene_list/common_cluster_gene_set.csv") dm.stage_time_dict = stage_dict dm.process() dm.normalize_sc_dict() return (dm)
def standard_no10ss_dm(ref_cell_num, default=False): stage_dict = json.load(open("data/base_data/stage_hpf.json")) dm = data_manager(ref_cell_num, default=default) dm.register_tomoseq("data/base_data/tomo_seq/zfshield", stage_dict["shield"]) dm.register_sc_seq("data/base_data/sc/staged_exp/shield.csv", stage_dict["shield"], stage="shield") dm.register_sc_seq("data/base_data/sc/staged_exp/epiboly_75.csv", stage_dict["epiboly75"], stage="epiboly75") dm.register_sc_seq("data/base_data/sc/staged_exp/epiboly_90.csv", stage_dict["epiboly90"], stage="epiboly90") # dm.register_sc_seq("data/base_data/sc/staged_exp/epiboly_bud.csv", stage_dict["bud"], stage="bud") dm.register_sc_seq("data/base_data/sc/staged_exp/somite_3.csv", stage_dict["3ss"], stage="3ss") dm.register_sc_seq("data/base_data/sc/staged_exp/somite_6.csv", stage_dict["6ss"], stage="6ss") gene_df = pd.read_csv("data/gene_list/common_cluster_gene_set.csv") dm.process() dm.normalize_sc_dict() return (dm)
def object_create(self): dm = data_manager(self.data_file_name) values = dm.unpickle_data() values["counter"] = values["counter"] + 1 values["obj"].append(int(values["counter"])) dm.pickle_data(values) return True
def ts_dm(ref_cell_num): stage_dict = json.load(open("data/base_data/stage_hpf.json")) dm = data_manager(ref_cell_num) dm.register_tomoseq("data/base_data/tomo_seq/zfshield", stage_dict["shield"]) dm.register_tomoseq_ss("data/base_data/tomo_seq/zf10ss", stage_dict["10ss"]) return (dm)
def object_get(self): dm = data_manager(self.data_file_name) values = dm.unpickle_data() obj = None if len(values["obj"]) > 0: obj = random.choice(values["obj"]) values["obj"].remove(obj) dm.pickle_data(values) return obj
def shield_dm(ref_cell_num): stage_dict = json.load(open("data/base_data/stage_hpf.json")) dm = data_manager(ref_cell_num) dm.register_tomoseq("data/base_data/tomo_seq/zfshield", stage_dict["shield"]) dm.register_sc_seq("data/base_data/sc/staged_exp/shield.csv", stage_dict["shield"], stage="shield") dm.process() return (dm)
def set_up_data_manager(sc_data_dict, ct, ts_prefix_dict, stage_time_dict, gene_df, fix_angle): print("update") dm = data_manager() dm.register_use_gene(gene_df) dm.point_num = ct.point_num dm.ct = ct dm.stage_time_dict = stage_time_dict dm.register_sc_dict(sc_data_dict) dm.register_tomoseq(ts_prefix_dict["shield"], stage_time_dict["shield"]) return (dm)
def set_sample(pre_train_size, pre_test_size, train_size, test_size, auto_encoder=False, split_mode='slide', **keywords): print('fetch data') sections = [pre_train_size, pre_test_size, train_size, test_size] sample = data_manager.data_manager(DATA_DIR, data_size=300, split_mode=split_mode, attenate_flag=True, auto_encoder=auto_encoder, **keywords ).make_sample(sections) p_x_train, p_x_test, x_train, x_test, _ = sample.data _, _, y_train, y_test, _ = sample.target return p_x_train, p_x_test, x_train, x_test, y_train, \ y_test, sample.input_matrix_size, sample.output_matrix_size
def evaluate(self): log_errors = "" vectors_file_input = "../../../../Data/processed/evaluation_vectors/" \ + self.w2v_model_name + "/KORE_input_vectors.txt" vectors_file_output = "../../../../Data/processed/evaluation_vectors/" \ + self.w2v_model_name + "/KORE_output_vectors.txt" data_mng = data_manager(vectors_file_input, vectors_file_output, self.w2v_model_name) input_vectors, output_vectors = data_mng.retrieve_vectors() groups = data_mng.read_file() scores = list() left_entities_df = pd.DataFrame({"id": list(groups.keys())}) left_merged, left_ignored = data_mng.intersect_vectors_goldStandard( output_vectors, left_entities_df) self._store_ignored(left_ignored) if left_merged.size == 0: log_errors += "EntityRelatedness: no left entities of KORE in " \ + "vectors.\n" print("EntityRelatedness: no left entities of KORE in vectors.\n") else: right_merged_list = list() right_ignored_list = list() for key in groups.keys(): right_entities_df = pd.DataFrame({"id": groups[key]}) right_merged, right_ignored = \ data_mng.intersect_vectors_goldStandard( input_vectors, right_entities_df) right_ignored["related_to"] = key right_merged_list.append(right_merged) right_ignored_list.append(right_ignored) self._store_ignored(right_ignored) model = EntityRelatednessModel() scores = model.train(left_merged, left_ignored, right_merged_list, right_ignored_list, groups) self._store_results(scores) results_df = self._resultAsDataFrame(scores)
def standard_light_dm(ref_cell_num, default=False, no_10ss=False): stage_dict = json.load(open("data/base_data/stage_hpf.json")) dm = data_manager(ref_cell_num, default=default) dm.register_tomoseq("data/base_data/tomo_seq/zfshield", stage_dict["shield"]) dm.register_tomoseq_ss("data/base_data/tomo_seq/zf10ss", stage_dict["10ss"]) sc_id_dict = { "shield": "ZFS", "epiboly75": "ZF75", "epiboly90": "ZF90", "bud": "ZFB", "3ss": "ZF3S", "6ss": "ZF6S" } for stage in sc_id_dict.keys(): exp_file = "data/base_data/sc/stage_exp/" + sc_id_dict[ stage] + ".csv" dm.register_sc_seq(exp_file, stage_dict[stage], stage=stage) gene_df = pd.read_csv("data/gene_list/common_cluster_gene_set.csv") dm.process() dm.normalize_sc_dict() return (dm)
def evaluate(self): log_errors = "" vectors_file = "../../../../Data/processed/evaluation_vectors/" \ + self.w2v_model_name + "/LP50.txt" stats_file = "../../../../Data/raw/evaluation_sets/LP50/" \ + "LP50_averageScores.csv" data_mng = data_manager(vectors_file, self.w2v_model_name) stats = data_mng.read_file(stats_file, ["doc1", "doc2", "average"]) vectors = data_mng.retrieve_vectors() data, ignored = data_mng.intersect_vectors_goldStandard(vectors) self._store_ignored(ignored) scores = defaultdict(dict) if data.size == 0: log_errors += "Document similarity : Problems in merging vector " \ + "with gold standard LP50.\n" print( "Document similarity : Problems in merging vector with gold" + " standard LP50.\n") else: try: with_weights = False model = DocumentSimilarityModel(with_weights) result = model.train(data, stats) scores["without_weights"] = result with_weights = True model = DocumentSimilarityModel(with_weights) result = model.train(data, stats) scores["with_weights"] = result self._store_results(scores) results_df = self._resultAsDataFrame(scores) except Exception as e: log_errors += "File used as gold standard: LP50.\n" log_errors += "Document similarity, with weights: " \ + str(with_weights) + "\n" log_errors += str(e) + "\n"
class regresor(): data_man = data_manager() cluster_eng = clustering_engine() filtros_cubo_cdmx_departamentos_media = { "CLAVE_ENTIDAD_FEDERATIVA_UBICACION_INMUEBLE": ['09'], "CAT_TIPO_INMUEBLE": ['4'], "CVE_USO_CONSTRUCCION": ['1'], "CVE_CLASE_INMUEBLE": ['4'], "CVE_NIVEL_SOCIO_ECONOMICO_ZONA": ['4'], "CVE_CLASIFICACION_ZONA": ['3', '4'], "GRADO_TERMINACION_OBRA": ['100', '100.0', '100.00'] } filtros_cubo_cdmx_departamentos_plus = { "CLAVE_ENTIDAD_FEDERATIVA_UBICACION_INMUEBLE": ['09'], "CAT_TIPO_INMUEBLE": ['4'], "CVE_USO_CONSTRUCCION": ['1'], "CVE_CLASE_INMUEBLE": ['5'], "CVE_NIVEL_SOCIO_ECONOMICO_ZONA": ['5'], "CVE_CLASIFICACION_ZONA": ['3', '2'], "GRADO_TERMINACION_OBRA": ['100', '100.0', '100.00'] } filtros_cubo_cdmx_departamentos_alta = { "CLAVE_ENTIDAD_FEDERATIVA_UBICACION_INMUEBLE": ['09'], "CAT_TIPO_INMUEBLE": ['4'], "CVE_USO_CONSTRUCCION": ['1'], "CVE_CLASE_INMUEBLE": ['6', '7'], "CVE_NIVEL_SOCIO_ECONOMICO_ZONA": ['6'], "CVE_CLASIFICACION_ZONA": ['3', '4'], "GRADO_TERMINACION_OBRA": ['100', '100.0', '100.00'] } filtros_cubo_jalisco_departamentos_media = { "CLAVE_ENTIDAD_FEDERATIVA_UBICACION_INMUEBLE": ['14'], "CAT_TIPO_INMUEBLE": ['4'], "CVE_USO_CONSTRUCCION": ['1'], "CVE_CLASE_INMUEBLE": ['4'], "CVE_NIVEL_SOCIO_ECONOMICO_ZONA": ['4'], "CVE_CLASIFICACION_ZONA": ['3', '4'], "GRADO_TERMINACION_OBRA": ['100', '100.0', '100.00'] } filtros_cubo_jalisco_departamentos_plus = { "CLAVE_ENTIDAD_FEDERATIVA_UBICACION_INMUEBLE": ['14'], "CAT_TIPO_INMUEBLE": ['4'], "CVE_USO_CONSTRUCCION": ['1'], "CVE_CLASE_INMUEBLE": ['5'], "CVE_NIVEL_SOCIO_ECONOMICO_ZONA": ['5'], "CVE_CLASIFICACION_ZONA": ['3', '2'], "GRADO_TERMINACION_OBRA": ['100', '100.0', '100.00'] } filtros_cubo_jalisco_departamentos_alta = { "CLAVE_ENTIDAD_FEDERATIVA_UBICACION_INMUEBLE": ['14'], "CAT_TIPO_INMUEBLE": ['4'], "CVE_USO_CONSTRUCCION": ['1'], "CVE_CLASE_INMUEBLE": ['6', '7'], "CVE_NIVEL_SOCIO_ECONOMICO_ZONA": ['6'], "CVE_CLASIFICACION_ZONA": ['3', '2'], "GRADO_TERMINACION_OBRA": ['100', '100.0', '100.00'] } filtros_cubo_nuevo_leon_departamentos_media = { "CLAVE_ENTIDAD_FEDERATIVA_UBICACION_INMUEBLE": ['19'], "CAT_TIPO_INMUEBLE": ['4'], "CVE_USO_CONSTRUCCION": ['1'], "CVE_CLASE_INMUEBLE": ['4'], "CVE_NIVEL_SOCIO_ECONOMICO_ZONA": ['4'], "CVE_CLASIFICACION_ZONA": ['3', '4'], "GRADO_TERMINACION_OBRA": ['100', '100.0', '100.00'] } filtros_cubo_nuevo_leon_departamentos_plus = { "CLAVE_ENTIDAD_FEDERATIVA_UBICACION_INMUEBLE": ['19'], "CAT_TIPO_INMUEBLE": ['4'], "CVE_USO_CONSTRUCCION": ['1'], "CVE_CLASE_INMUEBLE": ['5'], "CVE_NIVEL_SOCIO_ECONOMICO_ZONA": ['5'], "CVE_CLASIFICACION_ZONA": ['3', '2'], "GRADO_TERMINACION_OBRA": ['100', '100.0', '100.00'] } filtros_cubo_nuevo_leon_departamentos_alta = { "CLAVE_ENTIDAD_FEDERATIVA_UBICACION_INMUEBLE": ['19'], "CAT_TIPO_INMUEBLE": ['4'], "CVE_USO_CONSTRUCCION": ['1'], "CVE_CLASE_INMUEBLE": ['6', '7'], "CVE_NIVEL_SOCIO_ECONOMICO_ZONA": ['6'], "CVE_CLASIFICACION_ZONA": ['2', '1'], "GRADO_TERMINACION_OBRA": ['100', '100.0', '100.00'] } def clusterizar(self, filtros_cubo, prefijo): avaluos = self.data_man.get_avaluos_server(filtros_cubo) num_clusters = len(avaluos) // 40 print "calculando ", num_clusters, " clusters" clusters = self.cluster_eng.get_clusters(prefijo, avaluos, num_clusters, 200, False) for each_cluster in clusters: each_cluster.update_statistics def analizar_clusters(self, prefijo, num_clusters): # filtros_cubo_cdmx_departamentos_plus43 for c in range(num_clusters): file_name = prefijo + str(c) + ".csv" avaluos = self.data_man.get_avaluos_csv(file_name) print "cluster: ", c, " contiene: ", len(avaluos), " avaluos" def __init__(self): print "Iniciando experimento..." #self.clusterizar(self.filtros_cubo_cdmx_departamentos_plus, "filtros_cubo_cdmx_departamentos_plus") self.clusterizar(self.filtros_cubo_cdmx_departamentos_media, "filtros_cubo_cdmx_departamentos_media") self.clusterizar(self.filtros_cubo_cdmx_departamentos_plus, "filtros_cubo_cdmx_departamentos_plus") self.clusterizar(self.filtros_cubo_cdmx_departamentos_alta, "filtros_cubo_cdmx_departamentos_alta") self.clusterizar(self.filtros_cubo_jalisco_departamentos_media, "filtros_cubo_jalisco_departamentos_media") self.clusterizar(self.filtros_cubo_jalisco_departamentos_plus, "filtros_cubo_jalisco_departamentos_plus") self.clusterizar(self.filtros_cubo_jalisco_departamentos_alta, "filtros_cubo_jalisco_departamentos_alta") self.clusterizar(self.filtros_cubo_nuevo_leon_departamentos_media, "filtros_cubo_nuevo_leon_departamentos_media") self.clusterizar(self.filtros_cubo_nuevo_leon_departamentos_plus, "filtros_cubo_nuevo_leon_departamentos_plus") self.clusterizar(self.filtros_cubo_nuevo_leon_departamentos_alta, "filtros_cubo_nuevo_leon_departamentos_alta")
import threading import datetime import time from flask import Flask, render_template, jsonify, request, redirect, url_for import data_crawler import data_manager import settings import math app = Flask(__name__, template_folder=settings.template_dir, static_folder=settings.static_dir) dm = data_manager.data_manager() @app.route('/') def main_page(): recent_etfs = [{ 'ticker': x[0][1:], 'name': x[1] } for x in dm.get_recent_etfs(5)] return render_template('index.html', recent=recent_etfs) @app.route('/', methods=['GET', 'POST']) def route_to_etf(): ticker = dm.name_to_ticker(request.form['etf']) if ticker != None: return redirect(url_for('etf_info', ticker=ticker[1:])) else:
def evaluate(self): log_errors = "" gold_standard_filenames = ["CitiesQualityOfLiving", "AAUP", "Forbes2013", "MetacriticMovies", "MetacriticAlbums"] total_scores = defaultdict(dict) if self.task_name == "Classification": column_score_name = "label" model_names = ["NB", "KNN", "C45", "SVM"] SVM_configurations = [pow(10, -3), pow(10, -2), 0.1, 1.0, 10.0, pow(10, 2), pow(10, 3)] Model = ClassificationModel elif self.task_name == "Regression": column_score_name = "rating" model_names = ["LR", "KNN", "M5"] Model = RegressionModel else: print("WRONG TASK NAME!") for gold_standard_filename in gold_standard_filenames: print("Evaluating dataset: {}.\n\n".format(gold_standard_filename)) gold_standard_file = "../../../../Data/raw/evaluation_sets/" + \ gold_standard_filename + "/CompleteDataset15.tsv" vectors_file = "../../../../Data/processed/evaluation_vectors/" \ + self.w2v_model_name + "/" + gold_standard_filename + ".txt" data_mng = data_manager(gold_standard_file, vectors_file, self.w2v_model_name) vectors = data_mng.retrieve_vectors() scores = defaultdict(list) total_scores_element = defaultdict(list) data, ignored = data_mng.intersect_vectors_goldStandard( vectors, column_score_name) self._store_ignored(gold_standard_filename, ignored) if data.size == 0: log_errors += "Classification : Problems in merging vector " "with gold standard " + gold_standard_file + "\n" print("Classification : Problems in merging vector with gold " "standard " + gold_standard_file + "\n") else: for i in range(10): data = data.sample(frac=1, random_state=i).reset_index( drop=True) for model_name in model_names: if model_name != "SVM": # Initialize the model model = Model(model_name) # Train and print score try: result = model.train(data) result["gold_standard_file"] = \ gold_standard_filename scores[model_name].append(result) total_scores_element[model_name].append(result) except Exception as e: log_errors += "File used as gold standard" + \ gold_standard_filename + "\n" log_errors += str(self.task_name) + \ " method: " + model_name + "\n" log_errors += str(e) + "\n" else: for conf in SVM_configurations: # Initialize the model model = Model("SVM", conf) try: result = model.train(data) result['gold_standard_file'] = \ gold_standard_filename scores["SVM"].append(result) total_scores_element["SVM_" + str(conf) ].append(result) except Exception as e: log_errors += "File used as gold " \ + "standard: " \ + gold_standard_filename + "\n" log_errors += str(self.task_name) \ + " method: SVM " + str(conf) + "\n" log_errors += str(e) + "\n" self._store_results(gold_standard_filename, scores) total_scores[gold_standard_filename] = total_scores_element results_df = self._resultsAsDataFrame(total_scores) return results_df
from datetime import datetime from portfolio import Portfolio from data_manager import data_manager from strategy import TradingStrategy start = datetime(2013, 1, 1) end = datetime(2017, 2, 28) feed = data_manager('SPY', start, end) stock_data = feed.get_close('SPY') #Create strategy and signals myStrategy = TradingStrategy(stock_data) signals = myStrategy.signals # Create a portfolio of SPY with $10,000 initial capital # This will also test the porfolio performance based on the signals we # created with the trading strategy portfolio = Portfolio('SPY', stock_data, signals, 10000.0) # Print performance print("Return: " + str(portfolio.roi)) print("My sharpe ratio: " + str(portfolio.sharpe_ratio)) print("Base Return: " + str(portfolio.baseline_roi)) print("Base sharpe ratio: " + str(portfolio.baseline_sharpe)) #Plotter functions to follow. Have raw code in notebook
def object_list(self): dm = data_manager(self.data_file_name) values = dm.unpickle_data() return values["obj"]
type=float, help='hyper parameter: t_corr') parser.add_argument('--l_corr', '-l', default=100.0, type=float, help='hyper parameter: l_corr') parser.add_argument('--opt_flag', '-f', default="slt", type=str, help='specifing optimized parameters') parser.add_argument('--no_ss', action='store_true', help='no somite stage') args = parser.parse_args() # data preparation dm = data_manager(args.sample_opt) hpf = args.hpf refhpf = {"shield": 7.5, "ss10": 16.8} dm.register_tomoseq('data/base_data/tomo_seq/zfshield', refhpf["shield"]) if not args.no_ss: dm.register_tomoseq_ss('data/base_data/tomo_seq/zf10ss', refhpf["ss10"]) dm.zero_num = 0 dm.process(args.sample_opt) # optimize parameter stge = STGE() stge.register_data_manager(dm) ## 1 at index of parameter which estimate opt_flag_vec = [int(var in args.opt_flag) for var in ['l', 't', 's']] if np.sum(opt_flag_vec) > 0: opt_params = stge.optimize_parameters(args.gene_id, args.l_corr,
def object_reset(self): dm = data_manager(self.data_file_name) objs = [] dm.pickle_data({"obj": objs, "counter": len(objs)}) return True
def new_data_manager(cfg, si): data_mana = data_manager(cfg, si) return data_mana
if DATA_FRACTION_TO_USE != 1: n=len(Y_train) nf=int(n*DATA_FRACTION_TO_USE) perm=np.random.permutation(n) X_train=X_train[perm[:nf]] Y_train=Y_train[perm[:nf]] #At the moment we are not taking a subset of the Y data if TEST_DATA_MANAGER: print '*'*50 print 'TEST_DATA_MANAGER' drs=[.1,.3,.5,.7,.9] dm=data_manager.data_manager( csr_train_feats=X_train,\ train_labels___0_means_unlabelled=Y_train,\ csr_test_feats=X_test,\ test_labels=Y_test,\ #dropout_rates=set(),\ dropout_rates=set(drs),\ max_num_dropout_corruptions_per_point=3\ ) print 1 def print_data(dr): print "DROPOUT RATE",dr print "len(tr_X.nonzero()[0])", len(tr_X.nonzero()[0]) # print "len(te_X.nonzero()[0])", len(te_X.nonzero()[0]) # print "Train - Labelled" # misc.print_labels_1_feats(tr_X,tr_Y,max_examples=25,max_feats=15) # print "tr_XU\n",tr_XU.todense() # print "Test - Labelled" # misc.print_labels_1_feats(te_X,te_Y,max_examples=25,max_feats=15) # print "tr_X",tr_X.shape # print "tr_Y",tr_Y.shape
def load_model_from_json(model_path, model_name): with open(model_path + model_name + '.json', 'r') as file: model_json = file.read() model = model_from_json(model_json) model.load_weights(model_path + model_name + '_weights.h5') return model if __name__ == '__main__': # 加载测试数据 import data_manager #test_set_path = './KolektorSDD/test_set' test_set_path = './KolektorSDD/all' sort_list = data_manager.data_manager(test_set_path) # 模型的输入形状 target_size = (500, 500) output_size = (62, 62) n_label = 2 # 加载模型 / 权重 model_path = './model/' model_name = 'best_model_10' try: ''' 两者选其一 1. 直接加载模型 2. 加载json文件和weights权重 ''' #model = load_model(model_path+model_name+'.h5') # 1.
parser = argparse.ArgumentParser() parser.add_argument('-r', '--repeat', type=int, help='repeat time') parser.add_argument('-n', '--num', type=int, help='particle number') parser.add_argument('-t', '--test', type=str, help='test_set') parser.add_argument('-o', '--output', type=str, help='out directory') args = parser.parse_args() mpl.rc('font', family='Times New Roman') # read parameters from environment si = 0.01 process_snr = 45 obs_snr = 20 test = 'test' if args.test is None else args.test N = 150 if args.num is None else args.num data_cfg = '{}/RO.cfg'.format(test) data_mana = data_manager(data_cfg, si) log_path = 'log/RO/{}'.format(args.output) if not os.path.isdir(log_path): os.makedirs(log_path) logging.basicConfig(filename=os.path.join(log_path, 'log.txt'), level=logging.INFO) for k in range(args.repeat): for i in range(len(data_mana.data)): msg = data_mana.get_info(i, prt=False) state = data_mana.select_states(i) state_with_noise = data_mana.select_states(i, process_snr) output = data_mana.select_outputs(i) output_with_noise = data_mana.select_outputs(i, obs_snr) modes = data_mana.select_modes(i)
model_name = args.model_name start = 0 if args.start is None else args.start log_path = 'log/RO/{}'.format(args.output) # set log path if not os.path.exists(log_path): os.makedirs(log_path) use_cuda = False si = 0.01 # test data set obs_snr = 20 obs_scale = np.array([1, 1, 1, 10, 10e8]) state_scale = np.array([1, 1, 1, 10, 10e8, 10e8]) data_cfg = '{}/RO.cfg'.format(args.data_set) data_mana = data_manager(data_cfg, si) # diagnoser diagnoser = 'model/{}'.format(model_name) diagnoser = torch.load(diagnoser, map_location='cpu') diagnoser.eval() # set log logging.basicConfig(filename=os.path.join(log_path, 'log.txt'), level=logging.INFO) # data 0 data_mana0 = data_manager('test_n\\RO.cfg', si) output_n = data_mana0.select_outputs(0, norm=obs_scale) # ro ro = RO(si)
from nn_classifier import NN from trainer import Solver import matplotlib.pyplot as plt CLASS_LABELS = [ 'apple', 'banana', 'nectarine', 'plum', 'peach', 'watermelon', 'pear', 'mango', 'grape', 'orange', 'strawberry', 'pineapple', 'radish', 'carrot', 'potato', 'tomato', 'bellpepper', 'broccoli', 'cabbage', 'cauliflower', 'celery', 'eggplant', 'garlic', 'spinach', 'ginger' ] image_size = 90 classes = CLASS_LABELS dm = data_manager(classes, image_size) val_data = dm.val_data train_data = dm.train_data K = [1, 20, 100] test_losses = [] train_losses = [] for k in K: nn = NN(train_data, val_data, n_neighbors=k) nn.train_model() test_losses.append(nn.get_validation_error()) train_losses.append(nn.get_train_error())
fname = '../models/homeview_ld_%d_conv_%d_id_%d_e_%d_history.pkl' % (latent_dim, num_conv, intermediate_dim, epochs) try: with open(fname, 'rb') as fo: history = cPickle.load(fo) #print history except: print "training history not saved" ''' # load dataset to plot latent space # load dataset img_source = sys.argv[1] age_group = img_source[img_source.rfind('/') + 1:] print("Loading data...") X = data_manager(dim_x=img_cols, dim_y=img_rows, batch_size=batch_size) x_train, x_validation, x_test = X.load_imgs(img_source, testing=True) # testing set x_test = x_test.astype('float32') / 255. x_test = x_test.reshape((x_test.shape[0], ) + original_img_size) # load saved models vae_path = os.path.join( '../models', age_group, 'homeview_ld_%d_conv_%d_id_%d_e_%d_vae.h5' % (latent_dim, num_conv, intermediate_dim, epochs)) encoder_path = os.path.join( '../models', age_group, 'homeview_ld_%d_conv_%d_id_%d_e_%d_encoder.h5' % (latent_dim, num_conv, intermediate_dim, epochs)) generator_path = os.path.join( '../models', age_group, 'homeview_ld_%d_conv_%d_id_%d_e_%d_generator.h5' % (latent_dim, num_conv, intermediate_dim, epochs))
import embedding_manager import xml_parser if len(sys.argv) != 2: print('Usage: python sentence_extract.py <config>') exit(0) hyper_params = xml_parser.parse(file=sys.argv[1], flat=False) # Process dataset data_process_params = hyper_params['data_process'] force_flag = data_process_params['force'] data_manager_params = data_process_params['data_manager_params'] file_sets = data_process_params['file_sets'] my_data_manager = data_manager.data_manager(data_manager_params) #my_data_manager.analyze_documents() #my_data_manager.build_files(force=force_flag) my_data_manager.load_dict() for key in file_sets: file_set = file_sets[key] my_data_manager.init_batch_gen(set_label=key, file_list=file_set, permutation=True) # Process word embedding embedding_params = hyper_params['embedding'] embedding_manager_params = embedding_params['embedding_manager'] source = embedding_params['source'] format = embedding_params['format'] force = embedding_params['force']
import data_manager as dt if __name__ == "__main__": manager = dt.data_manager('kerimdeveci', 'data4') values = manager.fetch_data() manager.write_col(values)
import numpy as np import data_manager import segment_manager from matplotlib import pyplot as plt warnings.filterwarnings("ignore") start_time = time.time() ### Initialize data_managerager class. ### Initialize with sigma, w and mode (rest or mean). sigma = 4 w = 50 mode = "mean" segment_manager = segment_manager.segment_manager(sigma, w, mode) data_manager = data_manager.data_manager() ### Initialize a list to store the events from all the datasets. all_data = [] all_segments = [] ### Define the names of the datasets that we will use filenames = ['7501394_PHAAET_rec16112018_PRincon_S1', '7501709_PHAAET_rec18112018_PRincon_S1', '7501755_PHAAET_rec27112018_PRincon_S1', '8200163_PHAAET_rec14052019_PRoque_S1', '8200445_PHAAET_rec290422019_PRincon_S1', '8200473_PHAAET_rec24052019_PRincon_S2', '8200487_PHAAET_rec04052019_PRincon_S1', '8200718_PHAAET_rec08032019_PRincon', '8201653_PHAAET_I.Cima_rec21012021_ninho 39_36_S1', '8201667_PHAAET_I.Cima_rec21012021_ninho 68_21_S1',
def main(moves_to_play = 3): # Get arguments parser = argparse.ArgumentParser(description='Starts a game of go in the terminal.') parser.add_argument('-s', '--size', type=int, default=19, help='size of board') args = parser.parse_args() if args.size < 7 or args.size > 19: sys.stdout.write('Board size must be between 7 and 19!\n') sys.exit(0) # Initialize board and view board = Board(args.size) view = View(board) err = None global move_count, prev_move_count, dm, mm, npboard, pred pred = (1,1) dm = data_manager.data_manager() dm.load_popularity_boards() mm = model_manager.model_manager() mm.load_many_models(1,moves_to_play) move_count = 0 prev_move_count = move_count #actions def goboard_to_npboard(goboard): global move_count goboard_np = np.array(goboard) goboard_array = [] for i in range(19): goboard_array.append([.5]*19) i,j=0,0 for row in goboard: if j >18: j=0 for col in row: if i >18: i=0 if col._type == 'white': goboard_array[i][j] = 1.0 elif col._type == 'black': goboard_array[i][j] = 0.0 else: goboard_array[i][j] = 0.5 i+=1 j+=1 for i in range(len(goboard_array)): goboard_array[i] = goboard_array[i][::-1] goboard_array = np.array(goboard_array).T return np.concatenate(goboard_array) #return npboard def cpu_play(): global mm, move_count, npboard global pred if move_count > 0: if (move_count % 2) == 0: color = 'b' else: color = 'w' predictions = mm.guess_list(npboard, move_count, dm) x, y = predictions[0] pred = predictions[0] move = (y+1, 18-x+1) board.move(move[0], move[1]) view.redraw() def move(): """ Makes a move at the current position of the cursor for the current turn. """ board.move(*view.cursor) view.redraw() def undo(): """ Undoes the last move. """ board.undo() view.redraw() def redo(): """ Redoes an undone move. """ board.redo() view.redraw() def exit(): """ Exits the game. """ sys.exit(0) # Action keymap KEYS = { 'w': view.cursor_up, 's': view.cursor_down, 'a': view.cursor_left, 'd': view.cursor_right, ' ': move, 'u': undo, 'r': redo, 'c': cpu_play, '\x1b': exit, } # Main loop while True: clear() global pred sys.stdout.write('{0}\n'.format(view)) print "move #:", move_count sys.stdout.write('Black: {black} <===> White: {white}\n'.format(**board.score)) sys.stdout.write('{0}\'s prediction '.format(pred)) sys.stdout.write('{0}\'s '.format(mm.most_popular_moves)) sys.stdout.write('{0}\'s move... '.format(board.turn)) if err: sys.stdout.write('\n' + err + '\n') err = None # Get action key c = getch() global move_count, prev_move_count change_flag = 0 try: # Execute selected action KEYS[c]() prev_move_count = move_count if c == ' ' or c == 'r' or c == 'c': move_count += 1 change_flag = 1 elif c == 'u': move_count = max( [0, move_count-1] ) change_flag = 1 except BoardError as be: # Board error (move on top of other piece, suicidal move, etc.) if change_flag == 1: move_count = prev_move_count change_flag = 1 err = be.message except KeyError: # Action not found, do nothing pass if change_flag == 1: # update global npboard global npboard npboard = goboard_to_npboard(board._state.board)