def download_submissoes(submissoes_url, output_path):
print 'Getting submissions from: %s' % submissoes_url
# request html
html = utils.request_get(submissoes_url)
# save last table as dataframe
df = utils.get_last_table(html)
print 'Saving as csv at %s' % output_path
# save df as csv
utils.save_as_csv(df, output_path)
def download_scoreboard(scoreboard_url, output_path):
print 'Getting scoreboard from: %s' % scoreboard_url
# request html
html = utils.request_get(scoreboard_url)
# save last table as dataframe
df = utils.get_last_table(html)
print 'Saving as csv at %s' % output_path
# save df as csv
utils.save_as_csv(df, output_path)
def download_regions_champions_data(url, output_path_regional_champions):
print 'Getting region champions from: %s' % url
# request html
html = utils.request_get(url)
soup = BeautifulSoup(html, 'lxml')
df_region_champs = generate_dataframes(url, soup,
output_path_regional_champions)
print 'Saving as csv at %s' % output_path_regional_champions
utils.save_as_csv(df_region_champs, output_path_regional_champions)
def download_estatisticas(statistics_url, output_path):
print 'Getting statistics from: %s' % statistics_url
# request html
html = utils.request_get(statistics_url)
# save as dataframe
# ignoreing first table because it is just the page header
for df in pd.read_html(html)[1:len(pd.read_html(html))]:
title = df.loc[0, 0].lower().replace(' ', '_')
print 'Saving as csv at %s' % output_path + title + '.csv'
# save df as csv
utils.save_as_csv(df, output_path + title + '.csv')
def download_competidor_data(url, output_path_competitor, output_path_coach):
global position
position = 1
print 'Getting competitors from: %s' % url
# request html
html = utils.request_get(url)
soup = BeautifulSoup(html, 'lxml')
df_competitors, df_coaches = generate_dataframes(url, soup,
output_path_competitor,
output_path_coach)
print 'Saving as csv at %s' % output_path_competitor
utils.save_as_csv(df_competitors, output_path_competitor)
print 'Saving as csv at %s' % output_path_coach
utils.save_as_csv(df_coaches, output_path_coach)
def call_run(modelname="toy", plot_trajectories=False, verbose=True):
names = map(lambda x: modelname + "/" + x,
[default_model, default_observations])
params_in = read_files(names[0], names[1])
try:
res = pattern_solver(params_in, verbose=verbose)
except ValueError:
print("ERROR: Something occurred during execution")
return (None)
if (not res[0]):
return (None)
## Dump pattern + trajectory matrices as CSV files
files = ["pattern_matrix", "trajectories", "grns"]
## Avoid special characters such as whitespaces
modelname = concat(modelname.split(" "), "-")
diffusion_rate = params_in[3]["diffusion-rate"]
for model_id in range(len(res[0])):
pattern_matrix, trajectories, grns = res[0][model_id]
patterns = res[1]
multi_binary_dict = res[2]
bvectors = res[3]
for j in range(len(files)):
print("\nMSG: Saving \'" + files[j] + "\' object from solution #" +
str(model_id + 1) + " of model " + modelname)
save_as_csv(res[0][model_id][j], model_id, modelname, files[j],
patterns, multi_binary_dict, bvectors)
## Visualization of the solution
model_to_igraph(pattern_matrix,
model_id,
modelname,
patterns,
verbose=verbose)
if (plot_trajectories):
to_plottable_matrix(trajectories, model_id, modelname,
multi_binary_dict)
print("\nMSG: Results saved as csv files!")
return (None)
from data import dataprocessor
from utils import save_as_csv
from tests import feature_selection
# --------------------------------------------------------------------------------
# This file can be ran to initialize datasets and perform feature analysis
# --------------------------------------------------------------------------------
if __name__ == '__main__':
# Preprocess raw data to desired format
interim = dataprocessor.process_raw_data()
save_as_csv(interim, 'data/datasets/interim/game_stats.csv')
# Create datasets with simple game statistics
simple_train_data, simple_test_data = dataprocessor.get_simplified_data(
interim)
save_as_csv(simple_train_data,
'data/datasets/processed/simple_train_data.csv')
save_as_csv(simple_test_data,
'data/datasets/processed/simple_test_data.csv')
# Create datasets with advanced game statistics
adv_train_data, adv_test_data, full_data = dataprocessor.get_advanced_data(
interim)
save_as_csv(adv_train_data, 'data/datasets/processed/adv_train_data.csv')
save_as_csv(adv_test_data, 'data/datasets/processed/adv_test_data.csv')
save_as_csv(full_data, 'data/datasets/processed/full_data.csv')
# Analyze feature importance
# If you want to select different features for your optimized models based on the analysis,
url = 'http://s.wanfangdata.com.cn/periodical?q=%28%28{}%29%20%29%20Date%3A{}-{}&s=50'.format(
quote('刊名:{}'.format(periodicalName)), year, year)
response = get_page(url)
total_number = get_page_number(response)
print(periodicalName, '共有', total_number, '页')
for page in range(1, total_number + 1):
print("当前爬取《" + periodicalName + "》第" + str(year) + "年" +
"第" + str(page) + "页")
url = 'http://s.wanfangdata.com.cn/periodical?q=%28%28{}%29%20%29%20Date%3A{}-{}&p={}&s=50'.format(
quote('刊名:{}'.format(periodicalName)), year, year,
str(page))
attempts = 0
success = False
while attempts < 10 and not success:
try:
response = get_page(url)
utils.save_as_csv(get_content(response))
time.sleep(2 + random.randint(0, 9) * 0.1)
success = True
except:
print("----------爬取错误，当前爬取至" + periodicalName +
"第" + str(page) + "页,正在重试:" + str(attempts) +
"---------")
attempts += 1
if attempts == 10:
txtfile = open('./log.txt', 'a', encoding='utf-8')
txtfile.write(periodicalName + '第' + str(page) +
'页读取错误 \n')
txtfile.close()
def save_as_csv(self):
fieldnames = [
'memory', 'vCPUs', 'transfer', 'disk', 'hour_price', 'month_price'
]
save_as_csv('digital_items.csv', self.items, fieldnames)
def save_as_csv(self):
fieldnames = ['storage', 'cpu', 'memory', 'bandwith', 'price']
save_as_csv('vultr_items.csv', self.items, fieldnames)
def call_predict(modelname,
model_id,
q0,
grn,
qf=None,
solmax=None,
plot_trajectories=False,
verbose=True):
print("\n-----------\nMODEL = " + modelname)
print("Solution " +
ifthenelse(model_id < 0, "\'no pattern selection\'", "#" +
str(model_id + 1)) + "\n-----------")
from numpy import shape, zeros, reshape
names = map(lambda x: modelname + "/" + x,
[default_model, default_observations])
params_in = read_files(names[0], names[1], return_phenotypes=True)
## Delete previous observations
del params_in[-3]
## Delete previous fix points
del params_in[-3]
nfields = len(params_in[4])
if (not (nfields == len(q0))):
print("ERROR: Wrong initial condition vector length: " + str(len(q0)) +
" instead of " + str(nfields))
return (None)
if (not (nfields == len(grn))):
print("ERROR: Wrong initial GRN vector length: " + str(len(q0)) +
" instead of " + str(nfields))
return (None)
Patterns = params_in[-1]
patterning_step = params_in[2]["patterning_step"]
k = params_in[2]["nsteps"]
nsteps = ifthenelse(patterning_step == 0, k, min(patterning_step, k))
## Add observations for initial states
Observations = filter_nones([
ifthenelse(
len(q0[id_field]) > 0,
{
"name": "Trajectory",
"step": 0,
"field": id_field,
## name of GRN instead of GRN itself: GRNs.get(di.get("GRN"))
"GRN": grn[id_field],
"phenotype": Patterns.get(q0[id_field])
}) for id_field in range(nfields)
])
## Add observations for final states
if (qf):
Observations += filter_nones([
ifthenelse(
len(qf[id_field]) > 0,
{
"name": "Trajectory",
"step": k,
"field": id_field,
## name of GRN instead of GRN itself: GRNs.get(di.get("GRN"))
"GRN": None,
"phenotype": Patterns.get(qf[id_field])
}) for id_field in range(nfields)
])
params_in = params_in[:-2] + [Observations, []] + [params_in[-2]]
diffusion_rate = params_in[3]["diffusion-rate"]
if (model_id < 0):
pattern_matrix = zeros((len(params_in[1]), nsteps))
else:
filename = path_to_results + modelname + "_rate=" + str(
diffusion_rate) + "/"
filename += "result_" + concat(
modelname.split("-")) + "_" + str(model_id +
1) + "_pattern_matrix.csv"
pattern_matrix = np.loadtxt(open(filename, "rb"),
delimiter=",",
skiprows=1)
print("\n* Pattern matrix:")
print("t= " +
concat([str(i) + " " * (3 - len(str(i)) + 1)
for i in range(nsteps)]))
print(pattern_matrix)
print("\n--- Starting predictions!")
try:
res = pattern_solver(params_in,
pattern_matrix0=pattern_matrix,
solmax=solmax,
verbose=True)
except ValueError:
print("ERROR: Something occurred during execution")
return (None)
if (not res[0]):
return (None)
## Avoid special characters such as whitespaces
modelname = "prediction_" + concat(modelname.split(" "),
"-") + "_solution=" + str(model_id + 1)
for model_id in range(len(res[0])):
_, trajectories, grns = res[0][model_id]
patterns = res[1]
multi_binary_dict = res[2]
bvectors = res[3]
## Dump trajectory matrices as CSV files
print("\nMSG: Saving \'trajectories\' object from solution #" +
str(model_id + 1) + " of model " + modelname)
save_as_csv(res[0][model_id][1], model_id, modelname, "trajectories",
patterns, multi_binary_dict, bvectors)
if (plot_trajectories):
to_plottable_matrix(trajectories, model_id, modelname,
multi_binary_dict)
print("\nMSG: Results saved as csv files!")
return (None)
parser.add_argument('--dataset', type=str, default='CIFAR')
args = vars(parser.parse_args())
if args['dataset'] == 'CIFAR':
model = CIFAR_Model()
train_loader, valid_loader, _, _ = create_loaders(is_train=True,
is_valid=True,
is_test=False)
else:
model = MNIST_Model()
train_loader, valid_loader, _, _ = create_loaders(
which_dataset='MNIST', is_train=True, is_valid=True, is_test=False)
mask = create_mask(model)
if args['first_time']:
torch.save(model.state_dict(), 'environment\\initial_model.pth')
torch.save(mask.state_dict(), 'environment\\initial_mask.pth')
save_as_csv(model)
else:
model.load_state_dict(torch.load('environment\\model.pth'))
mask.load_state_dict(torch.load('environment\\mask.pth'))
criterion, optimizer = create_criterion_optimizer(model)
value = train(args['epochs'], train_loader, valid_loader, optimizer,
criterion, model, mask)
f = open('wrapping\\result.txt', 'w')
print(value, file=f, end='')
f.close()