def int_communication(data_dir, freq, single, adr_change, backend, skip_meas,
clk_enable):
GPIO_DR = 6
i = 0
while os.path.exists(os.path.join(data_dir, 'data_' + str(i) + '.dat')):
i += 1
global write_file
if not skip_meas:
import te_setup
#import RPi.GPIO as GPIO
import RPIO as GPIO
import spidev
spi = spidev.SpiDev()
write_file = open(os.path.join(data_dir, 'data_' + str(i) + '.dat'),
'w')
global spi
spi.open(0, 0)
spi.max_speed_hz = (freq)
GPIO.setmode(GPIO.BCM)
GPIO.setup(GPIO_DR, GPIO.IN, pull_up_down=GPIO.PUD_DOWN)
GPIO.add_event_detect(GPIO_DR, GPIO.RISING)
if not adr_change:
GPIO.add_event_callback(GPIO_DR, callback)
else:
GPIO.add_event_callback(GPIO_DR, callback_adr)
if single:
if not adr_change:
callback(6)
else:
callback_adr(6)
else:
try:
if clk_enable:
GPIO.setup(5, RPIO.ALT0)
while True:
time.sleep(1)
except KeyboardInterrupt:
GPIO.setup(5, RPIO.INPUT)
GPIO.cleanup()
#global write_file
time.sleep(0.1)
write_file.close()
print('Written to data_' + str(i) + '.dat')
if backend == 'display_graph':
data_manager.data_manager(directory=data_dir,
data_size=1000,
train_size=100,
split_mode=False,
attenate_flag=False,
save_as_png=False).plot()
def int_communication(data_dir, freq, single, adr_change, backend, skip_meas, clk_enable):
GPIO_DR = 6
i = 0
while os.path.exists(os.path.join(data_dir, 'data_' + str(i) + '.dat')):
i += 1
global write_file
if not skip_meas:
import te_setup
#import RPi.GPIO as GPIO
import RPIO as GPIO
import spidev
spi = spidev.SpiDev()
write_file = open(os.path.join(data_dir, 'data_' + str(i) + '.dat'), 'w')
global spi
spi.open(0,0)
spi.max_speed_hz=(freq)
GPIO.setmode(GPIO.BCM)
GPIO.setup(GPIO_DR, GPIO.IN, pull_up_down=GPIO.PUD_DOWN)
GPIO.add_event_detect(GPIO_DR, GPIO.RISING)
if not adr_change:
GPIO.add_event_callback(GPIO_DR, callback)
else:
GPIO.add_event_callback(GPIO_DR, callback_adr)
if single:
if not adr_change:
callback(6)
else:
callback_adr(6)
else:
try:
if clk_enable:
GPIO.setup(5, RPIO.ALT0)
while True:
time.sleep(1)
except KeyboardInterrupt:
GPIO.setup(5, RPIO.INPUT)
GPIO.cleanup()
#global write_file
time.sleep(0.1)
write_file.close()
print('Written to data_' + str(i) + '.dat')
if backend == 'display_graph':
data_manager.data_manager(directory=data_dir,
data_size=1000,
train_size=100,
split_mode=False,
attenate_flag=False,
save_as_png=False).plot()
def __init__(self, client):
super().__init__(client)
self.func_tbl = {
'status': [self.cmd_status, 'bbコマンドグループのステータス出力'],
'resetLeague': [self.cmd_resetLeague, 'リーグの状態を完全初期化'],
'createTeam': [self.cmd_createTeam, 'チームをつくる'],
'createAth': [self.cmd_createAth, 'create athlete 選手データを登録する'],
'showTeam': [self.cmd_showTeam, 'チームデータを閲覧する'],
'addMatch': [self.cmd_addMatch, '試合を追加する'],
'addStadium': [self.cmd_addStadium, '球場を登録する'],
'startSeason': [self.cmd_startSeason, 'レギュラーシーズンを開幕する'],
'setBattingOrder': [self.cmd_setBattingOrder, 'バッターを打順に組み込む'],
'setStartingPitcher':
[self.cmd_setStartingPitcher, '先発ピッチャーを指名する'],
'setBattingOrderGroup':
[self.cmd_setBattingOrderGroup, 'バッターを打順に組み込む'],
'imReady': [self.cmd_imReady, '試合前の準備が完了したことを通知する'],
'drawIn': [self.cmd_drawIn, '前進守備'],
'walk': [self.cmd_walk, '敬遠'],
'showStartMem': [self.cmd_showStartMem, '現在登録されているスタメンを見る'],
'bunt': [self.cmd_bunt, 'バント'],
'changePitcher': [self.cmd_changePitcher, 'ピッチャー交代'],
'debug': [self.cmd_dbg, 'おちんちんびろーん'],
'debugSetIning': [self.cmd_dbg_setIning, 'おちんちんびろーん'],
}
self.cmd_group_name = "bb"
self.client = client
self.league_status = LeagueStatus.OFF_SEASON
self.db = data_manager(client)
self.rule = bb_rule(self.db, self.client)
def object_free(self, num):
dm = data_manager(self.data_file_name)
values = dm.unpickle_data()
values["obj"].append(int(num))
values["obj"].sort()
dm.pickle_data(values)
return True
def standard_dm(ref_cell_num,
stage_fname="data/base_data/stage_hpf.json",
no10ss=False,
**kwargs):
stage_dict = json.load(open(stage_fname))
dm = data_manager(ref_cell_num, **kwargs)
dm.register_tomoseq("data/base_data/tomo_seq/zfshield",
stage_dict["shield"])
dm.register_sc_seq("data/base_data/sc/staged_exp/shield.csv",
stage_dict["shield"],
stage="shield")
dm.register_sc_seq("data/base_data/sc/staged_exp/epiboly_75.csv",
stage_dict["epiboly75"],
stage="epiboly75")
dm.register_sc_seq("data/base_data/sc/staged_exp/epiboly_90.csv",
stage_dict["epiboly90"],
stage="epiboly90")
# dm.register_sc_seq("data/base_data/sc/staged_exp/epiboly_bud.csv", stage_dict["bud"], stage="bud")
dm.register_sc_seq("data/base_data/sc/staged_exp/somite_3.csv",
stage_dict["3ss"],
stage="3ss")
dm.register_sc_seq("data/base_data/sc/staged_exp/somite_6.csv",
stage_dict["6ss"],
stage="6ss")
if not no10ss:
dm.register_tomoseq_ss("data/base_data/tomo_seq/zf10ss",
stage_dict["10ss"])
gene_df = pd.read_csv("data/gene_list/common_cluster_gene_set.csv")
dm.stage_time_dict = stage_dict
dm.process()
dm.normalize_sc_dict()
return (dm)
def standard_no10ss_dm(ref_cell_num, default=False):
stage_dict = json.load(open("data/base_data/stage_hpf.json"))
dm = data_manager(ref_cell_num, default=default)
dm.register_tomoseq("data/base_data/tomo_seq/zfshield",
stage_dict["shield"])
dm.register_sc_seq("data/base_data/sc/staged_exp/shield.csv",
stage_dict["shield"],
stage="shield")
dm.register_sc_seq("data/base_data/sc/staged_exp/epiboly_75.csv",
stage_dict["epiboly75"],
stage="epiboly75")
dm.register_sc_seq("data/base_data/sc/staged_exp/epiboly_90.csv",
stage_dict["epiboly90"],
stage="epiboly90")
# dm.register_sc_seq("data/base_data/sc/staged_exp/epiboly_bud.csv", stage_dict["bud"], stage="bud")
dm.register_sc_seq("data/base_data/sc/staged_exp/somite_3.csv",
stage_dict["3ss"],
stage="3ss")
dm.register_sc_seq("data/base_data/sc/staged_exp/somite_6.csv",
stage_dict["6ss"],
stage="6ss")
gene_df = pd.read_csv("data/gene_list/common_cluster_gene_set.csv")
dm.process()
dm.normalize_sc_dict()
return (dm)
def object_create(self):
dm = data_manager(self.data_file_name)
values = dm.unpickle_data()
values["counter"] = values["counter"] + 1
values["obj"].append(int(values["counter"]))
dm.pickle_data(values)
return True
def ts_dm(ref_cell_num):
stage_dict = json.load(open("data/base_data/stage_hpf.json"))
dm = data_manager(ref_cell_num)
dm.register_tomoseq("data/base_data/tomo_seq/zfshield",
stage_dict["shield"])
dm.register_tomoseq_ss("data/base_data/tomo_seq/zf10ss",
stage_dict["10ss"])
return (dm)
def object_get(self):
dm = data_manager(self.data_file_name)
values = dm.unpickle_data()
obj = None
if len(values["obj"]) > 0:
obj = random.choice(values["obj"])
values["obj"].remove(obj)
dm.pickle_data(values)
return obj
def shield_dm(ref_cell_num):
stage_dict = json.load(open("data/base_data/stage_hpf.json"))
dm = data_manager(ref_cell_num)
dm.register_tomoseq("data/base_data/tomo_seq/zfshield",
stage_dict["shield"])
dm.register_sc_seq("data/base_data/sc/staged_exp/shield.csv",
stage_dict["shield"],
stage="shield")
dm.process()
return (dm)
def set_up_data_manager(sc_data_dict, ct, ts_prefix_dict, stage_time_dict,
gene_df, fix_angle):
print("update")
dm = data_manager()
dm.register_use_gene(gene_df)
dm.point_num = ct.point_num
dm.ct = ct
dm.stage_time_dict = stage_time_dict
dm.register_sc_dict(sc_data_dict)
dm.register_tomoseq(ts_prefix_dict["shield"], stage_time_dict["shield"])
return (dm)
def set_sample(pre_train_size, pre_test_size, train_size, test_size, auto_encoder=False, split_mode='slide', **keywords):
print('fetch data')
sections = [pre_train_size, pre_test_size, train_size, test_size]
sample = data_manager.data_manager(DATA_DIR,
data_size=300,
split_mode=split_mode,
attenate_flag=True,
auto_encoder=auto_encoder,
**keywords
).make_sample(sections)
p_x_train, p_x_test, x_train, x_test, _ = sample.data
_, _, y_train, y_test, _ = sample.target
return p_x_train, p_x_test, x_train, x_test, y_train, \
y_test, sample.input_matrix_size, sample.output_matrix_size
def evaluate(self):
log_errors = ""
vectors_file_input = "../../../../Data/processed/evaluation_vectors/" \
+ self.w2v_model_name + "/KORE_input_vectors.txt"
vectors_file_output = "../../../../Data/processed/evaluation_vectors/" \
+ self.w2v_model_name + "/KORE_output_vectors.txt"
data_mng = data_manager(vectors_file_input, vectors_file_output,
self.w2v_model_name)
input_vectors, output_vectors = data_mng.retrieve_vectors()
groups = data_mng.read_file()
scores = list()
left_entities_df = pd.DataFrame({"id": list(groups.keys())})
left_merged, left_ignored = data_mng.intersect_vectors_goldStandard(
output_vectors, left_entities_df)
self._store_ignored(left_ignored)
if left_merged.size == 0:
log_errors += "EntityRelatedness: no left entities of KORE in " \
+ "vectors.\n"
print("EntityRelatedness: no left entities of KORE in vectors.\n")
else:
right_merged_list = list()
right_ignored_list = list()
for key in groups.keys():
right_entities_df = pd.DataFrame({"id": groups[key]})
right_merged, right_ignored = \
data_mng.intersect_vectors_goldStandard(
input_vectors, right_entities_df)
right_ignored["related_to"] = key
right_merged_list.append(right_merged)
right_ignored_list.append(right_ignored)
self._store_ignored(right_ignored)
model = EntityRelatednessModel()
scores = model.train(left_merged, left_ignored, right_merged_list,
right_ignored_list, groups)
self._store_results(scores)
results_df = self._resultAsDataFrame(scores)
def standard_light_dm(ref_cell_num, default=False, no_10ss=False):
stage_dict = json.load(open("data/base_data/stage_hpf.json"))
dm = data_manager(ref_cell_num, default=default)
dm.register_tomoseq("data/base_data/tomo_seq/zfshield",
stage_dict["shield"])
dm.register_tomoseq_ss("data/base_data/tomo_seq/zf10ss",
stage_dict["10ss"])
sc_id_dict = {
"shield": "ZFS",
"epiboly75": "ZF75",
"epiboly90": "ZF90",
"bud": "ZFB",
"3ss": "ZF3S",
"6ss": "ZF6S"
}
for stage in sc_id_dict.keys():
exp_file = "data/base_data/sc/stage_exp/" + sc_id_dict[
stage] + ".csv"
dm.register_sc_seq(exp_file, stage_dict[stage], stage=stage)
gene_df = pd.read_csv("data/gene_list/common_cluster_gene_set.csv")
dm.process()
dm.normalize_sc_dict()
return (dm)
def evaluate(self):
log_errors = ""
vectors_file = "../../../../Data/processed/evaluation_vectors/" \
+ self.w2v_model_name + "/LP50.txt"
stats_file = "../../../../Data/raw/evaluation_sets/LP50/" \
+ "LP50_averageScores.csv"
data_mng = data_manager(vectors_file, self.w2v_model_name)
stats = data_mng.read_file(stats_file, ["doc1", "doc2", "average"])
vectors = data_mng.retrieve_vectors()
data, ignored = data_mng.intersect_vectors_goldStandard(vectors)
self._store_ignored(ignored)
scores = defaultdict(dict)
if data.size == 0:
log_errors += "Document similarity : Problems in merging vector " \
+ "with gold standard LP50.\n"
print(
"Document similarity : Problems in merging vector with gold" +
" standard LP50.\n")
else:
try:
with_weights = False
model = DocumentSimilarityModel(with_weights)
result = model.train(data, stats)
scores["without_weights"] = result
with_weights = True
model = DocumentSimilarityModel(with_weights)
result = model.train(data, stats)
scores["with_weights"] = result
self._store_results(scores)
results_df = self._resultAsDataFrame(scores)
except Exception as e:
log_errors += "File used as gold standard: LP50.\n"
log_errors += "Document similarity, with weights: " \
+ str(with_weights) + "\n"
log_errors += str(e) + "\n"
class regresor():
data_man = data_manager()
cluster_eng = clustering_engine()
filtros_cubo_cdmx_departamentos_media = {
"CLAVE_ENTIDAD_FEDERATIVA_UBICACION_INMUEBLE": ['09'],
"CAT_TIPO_INMUEBLE": ['4'],
"CVE_USO_CONSTRUCCION": ['1'],
"CVE_CLASE_INMUEBLE": ['4'],
"CVE_NIVEL_SOCIO_ECONOMICO_ZONA": ['4'],
"CVE_CLASIFICACION_ZONA": ['3', '4'],
"GRADO_TERMINACION_OBRA": ['100', '100.0', '100.00']
}
filtros_cubo_cdmx_departamentos_plus = {
"CLAVE_ENTIDAD_FEDERATIVA_UBICACION_INMUEBLE": ['09'],
"CAT_TIPO_INMUEBLE": ['4'],
"CVE_USO_CONSTRUCCION": ['1'],
"CVE_CLASE_INMUEBLE": ['5'],
"CVE_NIVEL_SOCIO_ECONOMICO_ZONA": ['5'],
"CVE_CLASIFICACION_ZONA": ['3', '2'],
"GRADO_TERMINACION_OBRA": ['100', '100.0', '100.00']
}
filtros_cubo_cdmx_departamentos_alta = {
"CLAVE_ENTIDAD_FEDERATIVA_UBICACION_INMUEBLE": ['09'],
"CAT_TIPO_INMUEBLE": ['4'],
"CVE_USO_CONSTRUCCION": ['1'],
"CVE_CLASE_INMUEBLE": ['6', '7'],
"CVE_NIVEL_SOCIO_ECONOMICO_ZONA": ['6'],
"CVE_CLASIFICACION_ZONA": ['3', '4'],
"GRADO_TERMINACION_OBRA": ['100', '100.0', '100.00']
}
filtros_cubo_jalisco_departamentos_media = {
"CLAVE_ENTIDAD_FEDERATIVA_UBICACION_INMUEBLE": ['14'],
"CAT_TIPO_INMUEBLE": ['4'],
"CVE_USO_CONSTRUCCION": ['1'],
"CVE_CLASE_INMUEBLE": ['4'],
"CVE_NIVEL_SOCIO_ECONOMICO_ZONA": ['4'],
"CVE_CLASIFICACION_ZONA": ['3', '4'],
"GRADO_TERMINACION_OBRA": ['100', '100.0', '100.00']
}
filtros_cubo_jalisco_departamentos_plus = {
"CLAVE_ENTIDAD_FEDERATIVA_UBICACION_INMUEBLE": ['14'],
"CAT_TIPO_INMUEBLE": ['4'],
"CVE_USO_CONSTRUCCION": ['1'],
"CVE_CLASE_INMUEBLE": ['5'],
"CVE_NIVEL_SOCIO_ECONOMICO_ZONA": ['5'],
"CVE_CLASIFICACION_ZONA": ['3', '2'],
"GRADO_TERMINACION_OBRA": ['100', '100.0', '100.00']
}
filtros_cubo_jalisco_departamentos_alta = {
"CLAVE_ENTIDAD_FEDERATIVA_UBICACION_INMUEBLE": ['14'],
"CAT_TIPO_INMUEBLE": ['4'],
"CVE_USO_CONSTRUCCION": ['1'],
"CVE_CLASE_INMUEBLE": ['6', '7'],
"CVE_NIVEL_SOCIO_ECONOMICO_ZONA": ['6'],
"CVE_CLASIFICACION_ZONA": ['3', '2'],
"GRADO_TERMINACION_OBRA": ['100', '100.0', '100.00']
}
filtros_cubo_nuevo_leon_departamentos_media = {
"CLAVE_ENTIDAD_FEDERATIVA_UBICACION_INMUEBLE": ['19'],
"CAT_TIPO_INMUEBLE": ['4'],
"CVE_USO_CONSTRUCCION": ['1'],
"CVE_CLASE_INMUEBLE": ['4'],
"CVE_NIVEL_SOCIO_ECONOMICO_ZONA": ['4'],
"CVE_CLASIFICACION_ZONA": ['3', '4'],
"GRADO_TERMINACION_OBRA": ['100', '100.0', '100.00']
}
filtros_cubo_nuevo_leon_departamentos_plus = {
"CLAVE_ENTIDAD_FEDERATIVA_UBICACION_INMUEBLE": ['19'],
"CAT_TIPO_INMUEBLE": ['4'],
"CVE_USO_CONSTRUCCION": ['1'],
"CVE_CLASE_INMUEBLE": ['5'],
"CVE_NIVEL_SOCIO_ECONOMICO_ZONA": ['5'],
"CVE_CLASIFICACION_ZONA": ['3', '2'],
"GRADO_TERMINACION_OBRA": ['100', '100.0', '100.00']
}
filtros_cubo_nuevo_leon_departamentos_alta = {
"CLAVE_ENTIDAD_FEDERATIVA_UBICACION_INMUEBLE": ['19'],
"CAT_TIPO_INMUEBLE": ['4'],
"CVE_USO_CONSTRUCCION": ['1'],
"CVE_CLASE_INMUEBLE": ['6', '7'],
"CVE_NIVEL_SOCIO_ECONOMICO_ZONA": ['6'],
"CVE_CLASIFICACION_ZONA": ['2', '1'],
"GRADO_TERMINACION_OBRA": ['100', '100.0', '100.00']
}
def clusterizar(self, filtros_cubo, prefijo):
avaluos = self.data_man.get_avaluos_server(filtros_cubo)
num_clusters = len(avaluos) // 40
print "calculando ", num_clusters, " clusters"
clusters = self.cluster_eng.get_clusters(prefijo, avaluos,
num_clusters, 200, False)
for each_cluster in clusters:
each_cluster.update_statistics
def analizar_clusters(self, prefijo, num_clusters):
# filtros_cubo_cdmx_departamentos_plus43
for c in range(num_clusters):
file_name = prefijo + str(c) + ".csv"
avaluos = self.data_man.get_avaluos_csv(file_name)
print "cluster: ", c, " contiene: ", len(avaluos), " avaluos"
def __init__(self):
print "Iniciando experimento..."
#self.clusterizar(self.filtros_cubo_cdmx_departamentos_plus, "filtros_cubo_cdmx_departamentos_plus")
self.clusterizar(self.filtros_cubo_cdmx_departamentos_media,
"filtros_cubo_cdmx_departamentos_media")
self.clusterizar(self.filtros_cubo_cdmx_departamentos_plus,
"filtros_cubo_cdmx_departamentos_plus")
self.clusterizar(self.filtros_cubo_cdmx_departamentos_alta,
"filtros_cubo_cdmx_departamentos_alta")
self.clusterizar(self.filtros_cubo_jalisco_departamentos_media,
"filtros_cubo_jalisco_departamentos_media")
self.clusterizar(self.filtros_cubo_jalisco_departamentos_plus,
"filtros_cubo_jalisco_departamentos_plus")
self.clusterizar(self.filtros_cubo_jalisco_departamentos_alta,
"filtros_cubo_jalisco_departamentos_alta")
self.clusterizar(self.filtros_cubo_nuevo_leon_departamentos_media,
"filtros_cubo_nuevo_leon_departamentos_media")
self.clusterizar(self.filtros_cubo_nuevo_leon_departamentos_plus,
"filtros_cubo_nuevo_leon_departamentos_plus")
self.clusterizar(self.filtros_cubo_nuevo_leon_departamentos_alta,
"filtros_cubo_nuevo_leon_departamentos_alta")
import threading
import datetime
import time
from flask import Flask, render_template, jsonify, request, redirect, url_for
import data_crawler
import data_manager
import settings
import math
app = Flask(__name__,
template_folder=settings.template_dir,
static_folder=settings.static_dir)
dm = data_manager.data_manager()
@app.route('/')
def main_page():
recent_etfs = [{
'ticker': x[0][1:],
'name': x[1]
} for x in dm.get_recent_etfs(5)]
return render_template('index.html', recent=recent_etfs)
@app.route('/', methods=['GET', 'POST'])
def route_to_etf():
ticker = dm.name_to_ticker(request.form['etf'])
if ticker != None:
return redirect(url_for('etf_info', ticker=ticker[1:]))
else:
def evaluate(self):
log_errors = ""
gold_standard_filenames = ["CitiesQualityOfLiving", "AAUP",
"Forbes2013", "MetacriticMovies",
"MetacriticAlbums"]
total_scores = defaultdict(dict)
if self.task_name == "Classification":
column_score_name = "label"
model_names = ["NB", "KNN", "C45", "SVM"]
SVM_configurations = [pow(10, -3), pow(10, -2), 0.1, 1.0, 10.0,
pow(10, 2), pow(10, 3)]
Model = ClassificationModel
elif self.task_name == "Regression":
column_score_name = "rating"
model_names = ["LR", "KNN", "M5"]
Model = RegressionModel
else:
print("WRONG TASK NAME!")
for gold_standard_filename in gold_standard_filenames:
print("Evaluating dataset: {}.\n\n".format(gold_standard_filename))
gold_standard_file = "../../../../Data/raw/evaluation_sets/" + \
gold_standard_filename + "/CompleteDataset15.tsv"
vectors_file = "../../../../Data/processed/evaluation_vectors/" \
+ self.w2v_model_name + "/" + gold_standard_filename + ".txt"
data_mng = data_manager(gold_standard_file, vectors_file,
self.w2v_model_name)
vectors = data_mng.retrieve_vectors()
scores = defaultdict(list)
total_scores_element = defaultdict(list)
data, ignored = data_mng.intersect_vectors_goldStandard(
vectors, column_score_name)
self._store_ignored(gold_standard_filename, ignored)
if data.size == 0:
log_errors += "Classification : Problems in merging vector "
"with gold standard " + gold_standard_file + "\n"
print("Classification : Problems in merging vector with gold "
"standard " + gold_standard_file + "\n")
else:
for i in range(10):
data = data.sample(frac=1, random_state=i).reset_index(
drop=True)
for model_name in model_names:
if model_name != "SVM":
# Initialize the model
model = Model(model_name)
# Train and print score
try:
result = model.train(data)
result["gold_standard_file"] = \
gold_standard_filename
scores[model_name].append(result)
total_scores_element[model_name].append(result)
except Exception as e:
log_errors += "File used as gold standard" + \
gold_standard_filename + "\n"
log_errors += str(self.task_name) + \
" method: " + model_name + "\n"
log_errors += str(e) + "\n"
else:
for conf in SVM_configurations:
# Initialize the model
model = Model("SVM", conf)
try:
result = model.train(data)
result['gold_standard_file'] = \
gold_standard_filename
scores["SVM"].append(result)
total_scores_element["SVM_" + str(conf)
].append(result)
except Exception as e:
log_errors += "File used as gold " \
+ "standard: " \
+ gold_standard_filename + "\n"
log_errors += str(self.task_name) \
+ " method: SVM " + str(conf) + "\n"
log_errors += str(e) + "\n"
self._store_results(gold_standard_filename, scores)
total_scores[gold_standard_filename] = total_scores_element
results_df = self._resultsAsDataFrame(total_scores)
return results_df
from datetime import datetime
from portfolio import Portfolio
from data_manager import data_manager
from strategy import TradingStrategy
start = datetime(2013, 1, 1)
end = datetime(2017, 2, 28)
feed = data_manager('SPY', start, end)
stock_data = feed.get_close('SPY')
#Create strategy and signals
myStrategy = TradingStrategy(stock_data)
signals = myStrategy.signals
# Create a portfolio of SPY with $10,000 initial capital
# This will also test the porfolio performance based on the signals we
# created with the trading strategy
portfolio = Portfolio('SPY', stock_data, signals, 10000.0)
# Print performance
print("Return: " + str(portfolio.roi))
print("My sharpe ratio: " + str(portfolio.sharpe_ratio))
print("Base Return: " + str(portfolio.baseline_roi))
print("Base sharpe ratio: " + str(portfolio.baseline_sharpe))
#Plotter functions to follow. Have raw code in notebook
def object_list(self):
dm = data_manager(self.data_file_name)
values = dm.unpickle_data()
return values["obj"]
type=float,
help='hyper parameter: t_corr')
parser.add_argument('--l_corr',
'-l',
default=100.0,
type=float,
help='hyper parameter: l_corr')
parser.add_argument('--opt_flag',
'-f',
default="slt",
type=str,
help='specifing optimized parameters')
parser.add_argument('--no_ss', action='store_true', help='no somite stage')
args = parser.parse_args()
# data preparation
dm = data_manager(args.sample_opt)
hpf = args.hpf
refhpf = {"shield": 7.5, "ss10": 16.8}
dm.register_tomoseq('data/base_data/tomo_seq/zfshield', refhpf["shield"])
if not args.no_ss:
dm.register_tomoseq_ss('data/base_data/tomo_seq/zf10ss',
refhpf["ss10"])
dm.zero_num = 0
dm.process(args.sample_opt)
# optimize parameter
stge = STGE()
stge.register_data_manager(dm)
## 1 at index of parameter which estimate
opt_flag_vec = [int(var in args.opt_flag) for var in ['l', 't', 's']]
if np.sum(opt_flag_vec) > 0:
opt_params = stge.optimize_parameters(args.gene_id, args.l_corr,
def object_reset(self):
dm = data_manager(self.data_file_name)
objs = []
dm.pickle_data({"obj": objs, "counter": len(objs)})
return True
def new_data_manager(cfg, si):
data_mana = data_manager(cfg, si)
return data_mana
if DATA_FRACTION_TO_USE != 1: n=len(Y_train) nf=int(n*DATA_FRACTION_TO_USE) perm=np.random.permutation(n) X_train=X_train[perm[:nf]] Y_train=Y_train[perm[:nf]] #At the moment we are not taking a subset of the Y data if TEST_DATA_MANAGER: print '*'*50 print 'TEST_DATA_MANAGER' drs=[.1,.3,.5,.7,.9] dm=data_manager.data_manager( csr_train_feats=X_train,\ train_labels___0_means_unlabelled=Y_train,\ csr_test_feats=X_test,\ test_labels=Y_test,\ #dropout_rates=set(),\ dropout_rates=set(drs),\ max_num_dropout_corruptions_per_point=3\ ) print 1 def print_data(dr): print "DROPOUT RATE",dr print "len(tr_X.nonzero()[0])", len(tr_X.nonzero()[0]) # print "len(te_X.nonzero()[0])", len(te_X.nonzero()[0]) # print "Train - Labelled" # misc.print_labels_1_feats(tr_X,tr_Y,max_examples=25,max_feats=15) # print "tr_XU\n",tr_XU.todense() # print "Test - Labelled" # misc.print_labels_1_feats(te_X,te_Y,max_examples=25,max_feats=15) # print "tr_X",tr_X.shape # print "tr_Y",tr_Y.shape
def load_model_from_json(model_path, model_name):
with open(model_path + model_name + '.json', 'r') as file:
model_json = file.read()
model = model_from_json(model_json)
model.load_weights(model_path + model_name + '_weights.h5')
return model
if __name__ == '__main__':
# 加载测试数据
import data_manager
#test_set_path = './KolektorSDD/test_set'
test_set_path = './KolektorSDD/all'
sort_list = data_manager.data_manager(test_set_path)
# 模型的输入形状
target_size = (500, 500)
output_size = (62, 62)
n_label = 2
# 加载模型 / 权重
model_path = './model/'
model_name = 'best_model_10'
try:
'''
两者选其一
1. 直接加载模型
2. 加载json文件和weights权重
'''
#model = load_model(model_path+model_name+'.h5') # 1.
parser = argparse.ArgumentParser()
parser.add_argument('-r', '--repeat', type=int, help='repeat time')
parser.add_argument('-n', '--num', type=int, help='particle number')
parser.add_argument('-t', '--test', type=str, help='test_set')
parser.add_argument('-o', '--output', type=str, help='out directory')
args = parser.parse_args()
mpl.rc('font', family='Times New Roman')
# read parameters from environment
si = 0.01
process_snr = 45
obs_snr = 20
test = 'test' if args.test is None else args.test
N = 150 if args.num is None else args.num
data_cfg = '{}/RO.cfg'.format(test)
data_mana = data_manager(data_cfg, si)
log_path = 'log/RO/{}'.format(args.output)
if not os.path.isdir(log_path):
os.makedirs(log_path)
logging.basicConfig(filename=os.path.join(log_path, 'log.txt'),
level=logging.INFO)
for k in range(args.repeat):
for i in range(len(data_mana.data)):
msg = data_mana.get_info(i, prt=False)
state = data_mana.select_states(i)
state_with_noise = data_mana.select_states(i, process_snr)
output = data_mana.select_outputs(i)
output_with_noise = data_mana.select_outputs(i, obs_snr)
modes = data_mana.select_modes(i)
model_name = args.model_name
start = 0 if args.start is None else args.start
log_path = 'log/RO/{}'.format(args.output)
# set log path
if not os.path.exists(log_path):
os.makedirs(log_path)
use_cuda = False
si = 0.01
# test data set
obs_snr = 20
obs_scale = np.array([1, 1, 1, 10, 10e8])
state_scale = np.array([1, 1, 1, 10, 10e8, 10e8])
data_cfg = '{}/RO.cfg'.format(args.data_set)
data_mana = data_manager(data_cfg, si)
# diagnoser
diagnoser = 'model/{}'.format(model_name)
diagnoser = torch.load(diagnoser, map_location='cpu')
diagnoser.eval()
# set log
logging.basicConfig(filename=os.path.join(log_path, 'log.txt'),
level=logging.INFO)
# data 0
data_mana0 = data_manager('test_n\\RO.cfg', si)
output_n = data_mana0.select_outputs(0, norm=obs_scale)
# ro
ro = RO(si)
from nn_classifier import NN
from trainer import Solver
import matplotlib.pyplot as plt
CLASS_LABELS = [
'apple', 'banana', 'nectarine', 'plum', 'peach', 'watermelon', 'pear',
'mango', 'grape', 'orange', 'strawberry', 'pineapple', 'radish', 'carrot',
'potato', 'tomato', 'bellpepper', 'broccoli', 'cabbage', 'cauliflower',
'celery', 'eggplant', 'garlic', 'spinach', 'ginger'
]
image_size = 90
classes = CLASS_LABELS
dm = data_manager(classes, image_size)
val_data = dm.val_data
train_data = dm.train_data
K = [1, 20, 100]
test_losses = []
train_losses = []
for k in K:
nn = NN(train_data, val_data, n_neighbors=k)
nn.train_model()
test_losses.append(nn.get_validation_error())
train_losses.append(nn.get_train_error())
fname = '../models/homeview_ld_%d_conv_%d_id_%d_e_%d_history.pkl' % (latent_dim, num_conv, intermediate_dim, epochs)
try:
with open(fname, 'rb') as fo:
history = cPickle.load(fo)
#print history
except:
print "training history not saved"
'''
# load dataset to plot latent space
# load dataset
img_source = sys.argv[1]
age_group = img_source[img_source.rfind('/') + 1:]
print("Loading data...")
X = data_manager(dim_x=img_cols, dim_y=img_rows, batch_size=batch_size)
x_train, x_validation, x_test = X.load_imgs(img_source, testing=True)
# testing set
x_test = x_test.astype('float32') / 255.
x_test = x_test.reshape((x_test.shape[0], ) + original_img_size)
# load saved models
vae_path = os.path.join(
'../models', age_group, 'homeview_ld_%d_conv_%d_id_%d_e_%d_vae.h5' %
(latent_dim, num_conv, intermediate_dim, epochs))
encoder_path = os.path.join(
'../models', age_group, 'homeview_ld_%d_conv_%d_id_%d_e_%d_encoder.h5' %
(latent_dim, num_conv, intermediate_dim, epochs))
generator_path = os.path.join(
'../models', age_group, 'homeview_ld_%d_conv_%d_id_%d_e_%d_generator.h5' %
(latent_dim, num_conv, intermediate_dim, epochs))
import embedding_manager
import xml_parser
if len(sys.argv) != 2:
print('Usage: python sentence_extract.py <config>')
exit(0)
hyper_params = xml_parser.parse(file=sys.argv[1], flat=False)
# Process dataset
data_process_params = hyper_params['data_process']
force_flag = data_process_params['force']
data_manager_params = data_process_params['data_manager_params']
file_sets = data_process_params['file_sets']
my_data_manager = data_manager.data_manager(data_manager_params)
#my_data_manager.analyze_documents()
#my_data_manager.build_files(force=force_flag)
my_data_manager.load_dict()
for key in file_sets:
file_set = file_sets[key]
my_data_manager.init_batch_gen(set_label=key,
file_list=file_set,
permutation=True)
# Process word embedding
embedding_params = hyper_params['embedding']
embedding_manager_params = embedding_params['embedding_manager']
source = embedding_params['source']
format = embedding_params['format']
force = embedding_params['force']
import data_manager as dt
if __name__ == "__main__":
manager = dt.data_manager('kerimdeveci', 'data4')
values = manager.fetch_data()
manager.write_col(values)
import numpy as np
import data_manager
import segment_manager
from matplotlib import pyplot as plt
warnings.filterwarnings("ignore")
start_time = time.time()
### Initialize data_managerager class.
### Initialize with sigma, w and mode (rest or mean).
sigma = 4
w = 50
mode = "mean"
segment_manager = segment_manager.segment_manager(sigma, w, mode)
data_manager = data_manager.data_manager()
### Initialize a list to store the events from all the datasets.
all_data = []
all_segments = []
### Define the names of the datasets that we will use
filenames = ['7501394_PHAAET_rec16112018_PRincon_S1',
'7501709_PHAAET_rec18112018_PRincon_S1',
'7501755_PHAAET_rec27112018_PRincon_S1',
'8200163_PHAAET_rec14052019_PRoque_S1',
'8200445_PHAAET_rec290422019_PRincon_S1',
'8200473_PHAAET_rec24052019_PRincon_S2',
'8200487_PHAAET_rec04052019_PRincon_S1',
'8200718_PHAAET_rec08032019_PRincon',
'8201653_PHAAET_I.Cima_rec21012021_ninho 39_36_S1',
'8201667_PHAAET_I.Cima_rec21012021_ninho 68_21_S1',
def main(moves_to_play = 3):
# Get arguments
parser = argparse.ArgumentParser(description='Starts a game of go in the terminal.')
parser.add_argument('-s', '--size', type=int, default=19, help='size of board')
args = parser.parse_args()
if args.size < 7 or args.size > 19:
sys.stdout.write('Board size must be between 7 and 19!\n')
sys.exit(0)
# Initialize board and view
board = Board(args.size)
view = View(board)
err = None
global move_count, prev_move_count, dm, mm, npboard, pred
pred = (1,1)
dm = data_manager.data_manager()
dm.load_popularity_boards()
mm = model_manager.model_manager()
mm.load_many_models(1,moves_to_play)
move_count = 0
prev_move_count = move_count
#actions
def goboard_to_npboard(goboard):
global move_count
goboard_np = np.array(goboard)
goboard_array = []
for i in range(19):
goboard_array.append([.5]*19)
i,j=0,0
for row in goboard:
if j >18:
j=0
for col in row:
if i >18:
i=0
if col._type == 'white':
goboard_array[i][j] = 1.0
elif col._type == 'black':
goboard_array[i][j] = 0.0
else:
goboard_array[i][j] = 0.5
i+=1
j+=1
for i in range(len(goboard_array)):
goboard_array[i] = goboard_array[i][::-1]
goboard_array = np.array(goboard_array).T
return np.concatenate(goboard_array)
#return npboard
def cpu_play():
global mm, move_count, npboard
global pred
if move_count > 0:
if (move_count % 2) == 0:
color = 'b'
else:
color = 'w'
predictions = mm.guess_list(npboard, move_count, dm)
x, y = predictions[0]
pred = predictions[0]
move = (y+1, 18-x+1)
board.move(move[0], move[1])
view.redraw()
def move():
"""
Makes a move at the current position of the cursor for the current
turn.
"""
board.move(*view.cursor)
view.redraw()
def undo():
"""
Undoes the last move.
"""
board.undo()
view.redraw()
def redo():
"""
Redoes an undone move.
"""
board.redo()
view.redraw()
def exit():
"""
Exits the game.
"""
sys.exit(0)
# Action keymap
KEYS = {
'w': view.cursor_up,
's': view.cursor_down,
'a': view.cursor_left,
'd': view.cursor_right,
' ': move,
'u': undo,
'r': redo,
'c': cpu_play,
'\x1b': exit,
}
# Main loop
while True:
clear()
global pred
sys.stdout.write('{0}\n'.format(view))
print "move #:", move_count
sys.stdout.write('Black: {black} <===> White: {white}\n'.format(**board.score))
sys.stdout.write('{0}\'s prediction '.format(pred))
sys.stdout.write('{0}\'s '.format(mm.most_popular_moves))
sys.stdout.write('{0}\'s move... '.format(board.turn))
if err:
sys.stdout.write('\n' + err + '\n')
err = None
# Get action key
c = getch()
global move_count, prev_move_count
change_flag = 0
try:
# Execute selected action
KEYS[c]()
prev_move_count = move_count
if c == ' ' or c == 'r' or c == 'c':
move_count += 1
change_flag = 1
elif c == 'u':
move_count = max( [0, move_count-1] )
change_flag = 1
except BoardError as be:
# Board error (move on top of other piece, suicidal move, etc.)
if change_flag == 1:
move_count = prev_move_count
change_flag = 1
err = be.message
except KeyError:
# Action not found, do nothing
pass
if change_flag == 1: # update global npboard
global npboard
npboard = goboard_to_npboard(board._state.board)
