def fix_experiment(file_loc):
full_path = os.path.join(wd, file_loc)
if 'params.experiment' in file_loc:
exp_params = load_obj(full_path)
if 'stimuli_sets' in exp_params:
exp_params.pop('stimuli_sets')
save_obj(exp_params, full_path)
def prepare_one_day_late_trade_date_list(daily_date, opt_date_list):
all_trade_date_list = daily_date['trade_date'].drop_duplicates().to_list()
all_trade_date_list.sort()
trade_date_list = []
opt2trade = {}
for opt_date in opt_date_list:
idx = all_trade_date_list.index(opt_date)
trade_date = all_trade_date_list[idx + 1]
trade_date_list.append(trade_date)
opt2trade[opt_date] = trade_date
save_obj(config.opt2trade, opt2trade)
return trade_date_list, opt2trade
def get_name2code(wgt_data):
file_path = config.name2code
if os.path.exists(file_path):
name2code = load_obj(file_path)
return name2code
name2code = {}
all_code = wgt_data['ts_code'].drop_duplicates().to_list()
for code in all_code:
stock_name = wgt_data[wgt_data.ts_code == code]['stock_name']
stock_name = stock_name.iloc[0]
name2code[stock_name] = code
save_obj(file_path, name2code)
return name2code
def get_gics2name():
file_path = config.gics2name
if os.path.exists(file_path):
gics2name = load_obj(file_path)
return gics2name
hs300_industry_df = prepare_hs300_industry_data_df()
industry_name = hs300_industry_df['industry_name'].to_list()
gics_code = hs300_industry_df['gics_code'].to_list()
tuple_list = [(i[0], i[1]) for i in zip(gics_code, industry_name)]
tuple_set = set(tuple_list)
gics2name = dict(list(tuple_set))
save_obj(file_path, gics2name)
return gics2name
def get_code2gics():
file_path = config.code2gics
if os.path.exists(file_path):
code2gics_info = load_obj(file_path)
return code2gics_info
hs300_industry_df = prepare_hs300_industry_data_df()
code = hs300_industry_df['ts_code'].to_list()
gics_code = hs300_industry_df['gics_code'].to_list()
tuple_list = [(i[0], i[1]) for i in zip(code, gics_code)]
tuple_set = set(tuple_list)
code2gics = dict(list(tuple_set))
save_obj(file_path, code2gics)
return code2gics
def collect_data(self, process_new=None):
if process_new == None:
process_new = len(self.data)
for key in self.all_json.keys():
self.process(self.all_json[key])
start = len(self.all_json)
end_range = process_new + len(self.all_json)
if end_range > len(self.data):
end_range = len(self.data)
for i in range(start, end_range):
r = self.process_post_request(self.data[i])
self.all_json[self.data[i]] = r
self.process(r)
self.set_max_freq()
tools.save_obj(cfg.SAVING_FILE, self.all_json)
wait_for_request_availability(w)
player_match_history = w.get_match_list(p_id,end_time=match_list_end_time , season='SEASON2016',ranked_queues=('RANKED_SOLO_5x5', 'TEAM_BUILDER_DRAFT_RANKED_5x5'))
wait_for_request_availability(w)
player_champion_match_history = w.get_match_list(p_id,end_time=match_created-1 , season='SEASON2016',ranked_queues=('RANKED_SOLO_5x5', 'TEAM_BUILDER_DRAFT_RANKED_5x5'), champion_ids=participants_champion_played[p_id])
match_data['participantMatchHistory'][p_id] = get_match_details_from_history(player_match_history, match_created)
match_data['participantChampionMatchHistory'][p_id] = get_match_details_from_history(player_champion_match_history, match_created, alt_list=match_data['participantMatchHistory'][p_id])
match_data['participantMatchListHistory'][p_id] = player_match_history
break
except Exception, e:
print("Error occured when trying to get summoner history data: " + str(e))
time.sleep(1)
continue
pulled_matches_count += 1
#match_data_to_save.append(match_data)
print "pulled {0} matches so far".format(pulled_matches_count)
#save match data to file every match
#if len(match_data_to_save) >= 1:
save_obj(match_data,'game_data_{0}.pkl'.format(saved_files))
match_data = None
with zipfile.ZipFile('game_data_{0}.zip'.format(saved_files), 'w', zipfile.ZIP_DEFLATED) as myzip:
myzip.write('game_data_{0}.pkl'.format(saved_files))
os.remove('game_data_{0}.pkl'.format(saved_files))
saved_files += 1
#match_data_to_save = list()
def save(self, folder_location, experiment_name=''):
"""
:param folder_location: Folder to save at
:param name: name for both files
"""
# Append underscored to experiment name if one was provided
if experiment_name:
if not experiment_name[-1] == '_':
experiment_name = experiment_name + '_'
check_folder(folder_location=folder_location)
# --- Saving Tempotron ---
# Determine file name for tempotron parameters
tempotron_params_location = os.path.join(folder_location, f'{experiment_name}params.tempotron')
# Get reference dictionary of tempotron parameters
model_params_dict = self.model.__dict__
# Select only the params to be saved
model_save_params = ['number_of_neurons', 'tau', 'threshold', 'stimulus_duration', 'weights', 'eqs']
# Create new dictionary with only desired parameters
model_params_savedict = {key: model_params_dict[key] for key in model_save_params}
# Handle network saving
model_networks = model_params_dict['networks']
# Excluding the plotting network
if 'plot' in model_networks:
model_networks.pop('plot')
# Adding network sizes to parameter dictionary
network_sizes = {name: network['number_of_stimuli'] for name, network in model_networks.items()}
model_params_savedict['network_sizes'] = network_sizes
# Saving tempotron parameters
save_obj(model_params_savedict, tempotron_params_location)
# --- Saving Experiment params and data ---
# Determine file name for experiment parameters
experiment_params_location = os.path.join(folder_location, f'{experiment_name}params.experiment')
# Get reference dictionary of experiment parameters
experiment_params_dict = self.__dict__
# Select only the params to be saved
experiment_save_params = ['stimuli_creation_params', 'training_params',
'origin_transform_function', 'origin_transform_params',
'set_transform_function', 'set_transform_params', 'repetitions',
'last_run_time', 'results', 'random_seed']
# Create new dictionary with only desired parameters
experiment_params_savedict = {key: experiment_params_dict[key] for key in experiment_save_params}
# Save experiment parameters
save_obj(experiment_params_savedict, experiment_params_location)
# Determine file name for results file
csv_full_path = os.path.join(folder_location, f'{experiment_name}results.csv')
# Save experiment results file
self.results.to_csv(csv_full_path)
@staticmethod
def load(location):
"""
Loads experiment file from specified location
:param location: the full location of the experiment file to load
"""
raise NotImplemented('Automatic loaded for experiments not yet implemented')
# then append the data you want to save to the list that we will save.
if(save_this_match(match_data)):
# Save only specific information from each match (see the function)
match_data = get_match_information_to_save(match_data)
match_data_to_save.append(match_data)
num_matches += 1
matched_you_actually_wanted += 1
# When we get 100+ games worth of new data, save them all to a new file.
# This is nice because it prevents your files from becoming enormous,
# and if you lose one you don't lose them all. It creates a bit more code
# but I think it's worth it.
# Note that there will not necessarily be exactly 100 games per file.
if(num_matches >= 100):
print("SAVING {0} MATCHES!".format(num_matches))
# This saves the data as a cPickle. See Python's documentation for more info.
save_obj(match_data_to_save,'game_data_{0}.pkl'.format(step))
match_data_to_save = []
num_matches = 0
for mid in these_pulled_matches:
pulled_matchIds_file.write("{0}\n".format(mid))
these_pulled_matches = []
# Save (append) the summonerIds we pulled to file
for summoner in these_pulled_summoners:
pulled_summoners_file.write("{0}\n".format(summoner))
these_pulled_summoners = []
# Save (overwrite) unpulled_summoners to file
unpulled_summoners_file = open('unpulled_summoners.txt','w')
for summoner in unpulled_summoners:
unpulled_summoners_file.write("{0}\n".format(summoner))
unpulled_summoners_file.close()
# Increment step
def main():
load_data_from_pickles = True
if(not load_data_from_pickles): # If false, will skip to just load the matrix from file
modelfiles = sys.argv[1:]
modelfiles = [file for file in modelfiles if '.xyz' in file]
modelfiles = natsort.natsorted(modelfiles)
modelfiles = modelfiles[0:5000]
new_modelfiles = [file for file in modelfiles if(not os.path.isfile(file[:-4]+'_VP.txt'))]
cutoff = 3.6
#print(modelfiles)
print("Calculating VP for all models...")
calculate_and_save_vp(new_modelfiles,cutoff)
print("Finding all transformed VPs...")
model_indexes = []
for model in modelfiles:
indexes = open(model[:-4]+'_VP.txt').readlines()
indexes = [eval(ind.strip()) for ind in indexes]
model_indexes.append(copy.copy(indexes))
d_vpIndex_to_matIndex, d_matIndex_to_vpIndex, matrix = analyze(model_indexes)
save_obj(d_vpIndex_to_matIndex, 'd_vpIndex_to_matIndex.pkl')
save_obj(d_matIndex_to_vpIndex, 'd_matIndex_to_vpIndex.pkl')
save_obj(matrix, 'matrix.pkl')
f = open('output.txt','w')
for row in matrix:
row = ','.join([str(x) for x in row]) + '\n'
f.write(row)
f.close()
# Post analysis
if(load_data_from_pickles):
d_vpIndex_to_matIndex = load_obj('d_vpIndex_to_matIndex.pkl')
d_matIndex_to_vpIndex = load_obj('d_matIndex_to_vpIndex.pkl')
matrix = load_obj('matrix.pkl')
normalized_matrix = copy.copy(matrix)
for i,row in enumerate(normalized_matrix):
normalized_matrix[i][i] = 0
row_total = [ 1.0*sum(row) for row in normalized_matrix ]
#print(row_total)
normalized_matrix = [ [x/row_total[i] for x in row] for i,row in enumerate(normalized_matrix) ]
if(True):
count = 0
to_print = [[] for row in normalized_matrix]
for i,row in enumerate(normalized_matrix):
for j,x in enumerate(row):
#if(i==j): continue
if(x > .01):
line = "{0}:\t{1} -> {2}".format(round(100.0*x,3), d_matIndex_to_vpIndex[i], d_matIndex_to_vpIndex[j])
to_print[i].append(tuple([row_total[i],line]))
count += x*row_total[i]/100.0
to_print = natsort.natsorted(to_print)
to_print = [natsort.natsorted(row) for row in to_print]
for row in to_print:
for x,line in row:
print(line + '\t' + str(100.0*x))
print('')
print("Total transformations: {0}".format(count))
# Find shortest paths to (0, 0, 12, 0) and (0, 6, 0, 8)
ico_index = (0, 0, 12, 0, 0, 0, 0, 0)
bcc_index = (0, 6, 0, 8, 0, 0, 0, 0)
graph = nx.Graph()
for i,row in enumerate(normalized_matrix):
for j,x in enumerate(row):
if(i==j): continue
if(x > 0.00):
#graph.add_edge( d_matIndex_to_vpIndex[i], d_matIndex_to_vpIndex[j] )
#graph[d_matIndex_to_vpIndex[i]][d_matIndex_to_vpIndex[j]]['weight'] = x
graph.add_edge( d_matIndex_to_vpIndex[j], d_matIndex_to_vpIndex[i] )
graph[d_matIndex_to_vpIndex[j]][d_matIndex_to_vpIndex[i]]['weight'] = 1-x
#test = []
bcc_dist = {}
ico_dist = {}
for ind in d_vpIndex_to_matIndex.keys():
try:
path = nx.shortest_path(graph, source=ind, target=ico_index, weight='weight')
dist = 1.0
for i in range(len(path)-1):
dist = dist * (1-graph[path[i]][path[i+1]]['weight'])
ico_dist[ind] = dist
except nx.exception.NetworkXNoPath:
ico_dist[ind] = 0.0
#test.append(tuple([ dist*100,ind, len(path) ]))
try:
path = nx.shortest_path(graph, source=ind, target=bcc_index, weight='weight')
dist = 1.0
for i in range(len(path)-1):
dist = dist * (1-graph[path[i]][path[i+1]]['weight'])
bcc_dist[ind] = dist
except nx.exception.NetworkXNoPath:
bcc_dist[ind] = 0.0
#test.sort()
#for t in test:
# print(t)
test = []
for key in ico_dist:
#print(key, ico_dist[key], bcc_dist[key], ico_dist[key]/bcc_dist[key], sum(matrix[d_vpIndex_to_matIndex[key]))
test.append([key, ico_dist[key], bcc_dist[key], ico_dist[key]/bcc_dist[key], sum(matrix[d_vpIndex_to_matIndex[key]])])
test.sort(key=operator.itemgetter(3))
test.reverse()
for t in test:
t = [str(x) for x in t]
t = '$'.join(t)
print(t)
p_id] = get_match_details_from_history(
player_champion_match_history,
match_created,
alt_list=match_data['participantMatchHistory']
[p_id])
match_data['participantMatchListHistory'][
p_id] = player_match_history
break
except Exception, e:
print(
"Error occured when trying to get summoner history data: "
+ str(e))
time.sleep(1)
continue
pulled_matches_count += 1
#match_data_to_save.append(match_data)
print "pulled {0} matches so far".format(pulled_matches_count)
#save match data to file every match
#if len(match_data_to_save) >= 1:
save_obj(match_data, 'game_data_{0}.pkl'.format(saved_files))
match_data = None
with zipfile.ZipFile('game_data_{0}.zip'.format(saved_files), 'w',
zipfile.ZIP_DEFLATED) as myzip:
myzip.write('game_data_{0}.pkl'.format(saved_files))
os.remove('game_data_{0}.pkl'.format(saved_files))
saved_files += 1
#match_data_to_save = list()