def main():
try:
args = get_args()
data = preprocess(args)
if args.visualize_data:
visualize(data)
sys.exit(1)
train_set, test_set = split(data)
num_examples = train_set.shape[0]
num_features = train_set.shape[1] - 1
if args.mini_batch:
batch_size = 32 # or 64
epochs = 1500
else:
batch_size = num_examples
epochs = 30000
nn = NeuralNetwork(num_features, batch_size, epochs)
if args.train:
nn.train(data, train_set, test_set, num_examples, args.quiet)
if args.evaluation:
y_pred = probability_to_class( nn.output.T)
get_validation_metrics(y_pred[:, 0], nn.y.T[:, 0])
# mini-batch learning is noisy, so we don't plot it
if not args.mini_batch:
plot_learning(nn.train_losses, nn.test_losses)
# save network params
if args.save_model:
W1, W2, W3, W4 = nn.weights1.tolist(), nn.weights2.tolist(), nn.weights3.tolist(), nn.weights4.tolist()
B1, B2, B3, B4 = nn.bias1.tolist(), nn.bias2.tolist(), nn.bias3.tolist(), nn.bias4.tolist()
model = dict(weights1=W1, weights2=W2, weights3=W3, weights4=W4, bias1=B1, bias2=B2, bias3=B3, bias4=B4)
with open("model.json", "w") as f:
json.dump(model, f, separators=(',', ':'), indent=4)
if args.predict and (args.predict == "model.json"):
try:
with open(args.predict) as file:
model = json.load(file)
except:
error_exit("please provide a valid model")
nn.load_model(model)
nn.predict(test_set, epochs)
except:
pass
def main():
# 获取命令行参数
command_args = get_args()
# 设置logging日志级别
if command_args['debug'] == True:
logging.getLogger().setLevel(logging.DEBUG)
else:
logging.getLogger().setLevel(logging.INFO)
# 获取配置文件参数
file_opts = get_file_opts(command_args)
# 实例化配置
collector_opts = CollectorConfig(file_opts=file_opts, command_args=command_args)
# 实例化AliyunRDSCollector
aliyun_rds_collector = AliyunRDSCollector(config=collector_opts)
# 注册到Prometheus的registry里面
REGISTRY.register(aliyun_rds_collector)
app = make_wsgi_app()
httpd = make_server(
host=str(collector_opts.server['host']),
port=int(collector_opts.server['port']),
app=app,
)
logging.info("Start exporter, listen on {}:{}".format(str(collector_opts.server['host']), int(collector_opts.server['port'])))
httpd.serve_forever()
confidences = t.split(';')[1].split(
' ') # ['', 'x_confs', '0.5362149']
confidences = confidences[2:]
for c in confidences:
confidences_sum += float(c)
tot += 1
mean = confidences_sum / tot if tot != 0 else 0
confidences_per_line[cpt] = mean
cpt += 1
return confidences_per_line
if __name__ == '__main__':
# Options parser
options = tools.get_args()
dir_path = options.data_dir
level = options.level
# For each file in the html directory
for file in glob.glob(dir_path + '*.html'):
# Parsing
inFile = io.open(file, mode='r', encoding='utf-8')
html = inFile.read()
soup = BeautifulSoup(html, 'lxml') # parsing
if level == 'page':
doc_name = file.split('/')[-1]
sum_, tot = get_confidences__page_level(soup)
mean = sum_ / tot if tot != 0 else 0
print(doc_name + '\t' + str(mean))
elif level == 'line':
doc_name = file.split('/')[-1]
if cpt_proc % 100 == 0:
sys.stdout.write("\r%i processed, %i relevant" %
(cpt_proc, cpt_rel))
# sys.stdout.flush() # if the buffer gets big, which is not the case
output_path = write_output(output_dic, options)
list_docs_not_found(missing_docs)
return cpt_proc, cpt_rel, output_path
if __name__ == "__main__":
start = time.clock()
options = get_args()
print(options)
if options.corpus == None:
print(
"Please specify a Json file (-c), see README.txt for more informations about the format. To use the default example :\n -c docs/Indonesian_GL.json"
)
exit()
else:
options.document_path = "None"
try:
os.makedirs("tmp")
except:
pass
cpt_doc, cpt_rel, output_path = start_detection(options)
end = time.clock()
print("\n%s docs proc. in %s seconds" %
return self.bhx_net.encode(state)
def obs_encode(self, obs, hx=None):
if hx is None:
hx = Variable(torch.zeros(1, self.gru_units))
if next(self.parameters()).is_cuda:
hx = hx.cuda()
_, _, _, (_, _, _, input_x) = self.gru_net((obs, hx),
input_fn=self.obx_net,
hx_fn=self.bhx_net,
inspect=True)
return input_x
if __name__ == '__main__':
args = tl.get_args()
env = gym.make(args.env)
env.seed(args.env_seed)
obs = env.reset()
# create directories to store results
result_dir = tl.ensure_directory_exits(
os.path.join(args.result_dir, 'Classic_Control'))
env_dir = tl.ensure_directory_exits(os.path.join(result_dir, args.env))
gru_dir = tl.ensure_directory_exits(
os.path.join(env_dir, 'gru_{}'.format(args.gru_size)))
gru_net_path = os.path.join(gru_dir, 'model.p')
gru_plot_dir = tl.ensure_directory_exits(os.path.join(gru_dir, 'Plots'))
bhx_dir = tl.ensure_directory_exits(
logging.basicConfig(
level=logging.INFO,
format=fmt,
datefmt=datefmt,
)
if filename_log:
build_log = logging.FileHandler(filename=filename_log, mode='w')
build_log.setLevel(logging.INFO)
formatter = logging.Formatter(fmt, datefmt=datefmt)
build_log.setFormatter(formatter)
logging.getLogger('').addHandler(build_log)
if __name__ == '__main__':
args = get_args()
logger.info(f"Reading test file")
test_file = os.path.join(args.input_path, args.test_filename)
test = process_data_classification(test_file)
X_test = test.drop('AlexLabel', axis=1)
texts_test = X_test["Text-EN"].str.lower()
cards_test = X_test.filter(
regex="card_.*"
) # Using a regular expression cause some cards may not be in test.
y_test = test['AlexLabel']
logger.info(f" Using test dataset with {X_test.shape[0]} instances")
logger.info(f"Reading train file")
train_file = os.path.join(args.input_path, args.train_filename)
train = process_data_classification(train_file)
X_train = train.drop('AlexLabel', axis=1)
name = 'RandomForest'
if args.no_ppi:
name += '_NO-PPI'
if args.expression:
name += '_EXP'
if args.sublocs:
name += '_SUB'
if args.orthologs:
name += '_ORT'
return name
if __name__ == '__main__':
args = tools.get_args()
mean, std = main(args)
name = get_name(args)
df_path = 'results/results.csv'
df = pd.read_csv(df_path)
df.loc[len(df)] = [
name, args.organism, args.ppi, args.expression, args.orthologs,
args.sublocs, args.n_runs, mean, std
]
df.to_csv(df_path, index=False)
print(df.head())
