def multiple_figures():
mpl.style.use(['seaborn-white', 'seaborn-paper', 'grayscale'])
#width = 3.39
#height = 3.39 * (np.sqrt(5) - 1.0) / 2.0
latexify()
for title, model in zip(['without', 'gaussian', 'constant', ], [without, guassian, constant, ]):
y, y_pred = model(pipe)
print(title, classifier[0])
print(class_counter(y))
print(metrics.classification_report(y, y_pred, labels=labels))
acc = metrics.accuracy_score(y, y_pred)
cm = metrics.confusion_matrix(y, y_pred, labels=labels)
cm = norm_cm(cm)
cm = pd.DataFrame(cm, index=labels, columns=labels)
fig, ax = plt.subplots(dpi=92)
sns.heatmap(cm, vmin=0, vmax=1, annot=True, fmt='.2f', cmap='Greys', ax=ax, cbar=False, square=True)
format_axes_for_cm(ax)
ax.set_title(f'accuracy = {acc:.3f}')
fig.tight_layout()
ensure_dir('./output/interpolations/')
fig.savefig(f'./output/interpolations/{title}.pdf', dpi=92, bbox_inches='tight')
plt.clf()
def generate_unique_log_folder():
while True:
log_folder = os.path.join(LOG_FOLDER, str(uuid.uuid4()))
if not os.path.exists(log_folder):
break
tools.ensure_dir(log_folder)
return log_folder
def run(self):
tools.ensure_dir(self._weights_dir)
self.init_cfg()
self.init_dataset()
self._steps_per_epoch = len(self.train_dataloader)
self._loss_print_interval = self._steps_per_epoch // 10
self.init_model()
for layer in self.model.module.module_list:
if isinstance(layer, YOLOLayer):
self.yolos.append(layer)
state_dict = deepcopy(self.model.state_dict())
for i in range(200):
self.model.load_state_dict(state_dict)
hypers = self.random_hyper()
print(i, hypers)
fitness = self.fit()
print(fitness)
self.records.append({
'hyper': hypers,
'fitness': fitness,
})
json.dump({'data': self.records}, open('evol2.json', 'w'))
def toMaya(params):
# print "To Maya - Start"
# search for camera point group...
campg = checkCameraGroups()
if campg is None:
tde4.postQuestionRequester("Export Maya...",
"Error, there is no camera point group.",
"Ok")
return
# getParams
filepath = params['file_browser'] + '.mel'
cameras = params['cameras']
camerasOutSize = params['camerasOutSize']
camerasFrameStart = params['camerasFirstFrame']
# openFile
tools.ensure_dir(filepath)
f = open(filepath, 'w')
# write header
f.write("//\n")
f.write("// Maya/MEL export data written by %s\n" % tde4.get3DEVersion())
f.write("//\n")
f.write("// All lengths are in centimeter, all angles are in degree.\n")
f.write("//\n\n")
# write scene group...
f.write("// create scene group...\n")
f.write("string $sceneGroupName = `group -em -name \"Scene\"`;\n")
# wrtie cameras
exportCameras(f, cameras, camerasFrameStart, params['file_browser'], campg,
camerasOutSize, params['date'])
# write camera points : campg
exportCameraPoints(f, campg)
# write moca / Objects
exportMocaObjects(f, camerasFrameStart[0])
# write global (scene node) transformation...
p3d = tde4.getScenePosition3D()
p3d = convertZup(p3d, yup)
r3d = tde4.getSceneRotation3D()
rot = convertToAngles(r3d)
s = tde4.getSceneScale3D()
f.write(
"xform -zeroTransformPivots -rotateOrder zxy -translation %.15f %.15f %.15f -scale %.15f %.15f %.15f -rotation %.15f %.15f %.15f $sceneGroupName;\n\n"
% (p3d[0], p3d[1], p3d[2], s, s, s, rot[0], rot[1], rot[2]))
f.write("\n")
# close file
f.close()
def save_mp3_to_file(self, mp3_data):
mp3_folder = os.path.join(self.log_folder, MP3_FOLDERNAME)
tools.ensure_dir(mp3_folder)
files = tools.list_files(mp3_folder, ['*.mp3'])
mp3_filename = os.path.join(mp3_folder, '{:04}.mp3'.format(len(files)))
with open(mp3_filename, 'wb') as f:
f.write(mp3_data)
return mp3_filename
def save_classifier_map_to_file(self, classifier_map):
map_folder = os.path.join(self.log_folder, MAP_FOLDERNAME)
tools.ensure_dir(map_folder)
files = tools.list_files(map_folder, ['*.png'])
map_filename = os.path.join(map_folder, '{:04}.png'.format(len(files)))
web_tools.save_map_to_file(classifier_map, map_filename)
return map_filename
def save_drawing_to_file(self, drawing_data):
drawing_folder = os.path.join(self.log_folder, DRAWING_FOLDERNAME)
tools.ensure_dir(drawing_folder)
drawing_files = tools.list_files(drawing_folder, ['*.json'])
drawing_filename = os.path.join(
drawing_folder, '{:04}.json'.format(len(drawing_files)))
tools.save_json(drawing_filename, drawing_data)
return drawing_filename
def preform_multilinear_regression(df_anova):
params = ['u', 'm', 'n', 'h', 'q']
combinations = []
### create all possible combinations between u, m, n, h, q
for r in range(1, len(params) + 1):
combinations += list(itertools.combinations(params, r))
### create all possible formulas from the combinations
formulas = []
for comb in combinations:
formula = 'err' + '~'
for c in comb:
formula += '+' + c
formulas += [formula]
### making dataframes for the different probabilities
df_a = df_anova[df_anova['prob'] == 'a']
df_b = df_anova[df_anova['prob'] == 'b']
df_ab = df_anova[df_anova['prob'] == 'ab']
dfs = {'a': df_a, 'b': df_b, 'ab': df_ab}
### calculating the regression model for each combinations for each probability
for formula in formulas:
for prob, current_df in dfs.items():
reg_model = ols(formula, data=current_df).fit()
# reg_model.summary()
temp = pd.DataFrame.from_dict({
'prob': [prob],
'formula': [formula],
'R_sq': [reg_model.rsquared],
'p_value': [reg_model.f_pvalue]
})
if 'df_reg_models' not in locals():
df_reg_models = temp.copy()
else:
df_reg_models = df_reg_models.append(temp)
df_reg_models = df_reg_models.reset_index(drop=True)
df_reg_models = df_reg_models.sort_values(by='R_sq')
df_reg_models.to_csv('analysis/reg_results.csv')
# plot R^2 as function of parameters in the model.
for prob, current_df in dfs.items():
fig, ax = plt.subplots(1, 1)
cdf = df_reg_models[df_reg_models['prob'] == prob]
g = sns.pointplot(x='formula', y='R_sq', data=cdf, ax=ax)
g.set_xticklabels(g.get_xticklabels(), rotation=45)
g.set_title(r"$R^2$" " of {} as function of combination".format(prob))
fig_name = 'figs/r_sq_p{}.png'.format(prob)
ensure_dir(fig_name)
fig.savefig(fig_name, dpi=300)
def main():
mpl.style.use(['seaborn-white', 'seaborn-paper', 'grayscale'])
latexify()
w_sizes = (5, 10, 50, 100)
for (w_prr, w_history) in it.product(w_sizes, repeat=2):
y, y_pred = different_window_sizes(w_prr, w_history)
acc = metrics.accuracy_score(y, y_pred)
prec = metrics.precision_score(y,
y_pred,
average='weighted',
labels=labels)
recall = metrics.recall_score(y,
y_pred,
average='weighted',
labels=labels)
f1 = metrics.f1_score(y, y_pred, average='weighted', labels=labels)
print(
f'& {w_history}\t& {w_prr}\t& {acc:.3f}\t& {prec:.3f}\t& {recall:.3f}\t& {f1:.3f}'
)
#print(f'Wh=={w_history}; Wprr=={w_prr}')
#print(metrics.classification_report(y, y_pred, labels=labels))
cm = metrics.confusion_matrix(y, y_pred, labels=labels)
cm = norm_cm(cm)
cm = pd.DataFrame(cm, index=labels, columns=labels)
fig, ax = plt.subplots(dpi=92)
sns.heatmap(cm,
vmin=0,
vmax=1,
annot=True,
fmt='.2f',
cmap='Greys',
ax=ax,
cbar=False,
square=True)
#ax.set_title(f'$\\mathrm{{Acc}}(W_{{\\mathrm{{PRR}}}}={w_prr}, W_{{\\mathrm{{history}}}}={w_history})={acc:.3f}$')
ax.set_title(
f'Accuracy = {acc:.3f}\n(prec = {prec:.3f}, rec = {recall:.3f})')
format_axes_for_cm(ax)
fig.tight_layout()
ensure_dir('./output/w_sizes/')
fig.savefig(f'./output/w_sizes/Wprr{w_prr}_Wh{w_history}.pdf',
dpi=92,
bbox_inches='tight')
plt.close(fig)
def save_learner_logs_to_file(self, learner_logs):
learner_logs_folder = os.path.join(self.log_folder,
LEARNER_LOGS_FOLDERNAME)
tools.ensure_dir(learner_logs_folder)
files = tools.list_files(learner_logs_folder, ['*.json'])
learner_logs_filename = os.path.join(learner_logs_folder,
'{:04}.json'.format(len(files)))
tools.save_json(learner_logs_filename, learner_logs)
return learner_logs_filename
def run(self):
tools.ensure_dir(self._weights_dir)
self.init_cfg()
self.init_dataset()
# 一轮训练的步数
self._steps_per_epoch = len(self.train_dataloader)
# 每一轮训练打印多少次损失
self._loss_print_interval = self._steps_per_epoch // 5
self.init_model()
self.init_evaluator()
self.init_optimizer()
self.init_losses()
if self._sparse_train:
self.bns = tools.get_bn_layers(self.model)
self.train()
def play(self, n_episodes = 10, save_images = True, render_on_screen = False):
log.info("Playing catch with the trained agent ({} episodes)".format(n_episodes));
if save_images:
output_folder = "{}/images".format(self.output_folder);
ensure_dir(output_folder);
log.info("Images will be saved in the folder: {}".format(output_folder));
try:
wins = 0
for e in range(n_episodes):
# The episode starts with resetting the environment.
observation = self.env.reset();
if render_on_screen:
self.env.render(mode='human');
loss = 0.;
done = False;
c = 0;
if save_images:
# save the initial observation as an image
if not render_on_screen:
self.env.render(mode='matplotlib');
plt.savefig("{}/{:02d}_{:02d}.png".format(output_folder, e, c));
while not done:
c += 1;
# get next action
action = self.choose_action(observation, 'greedy');
# apply action, get rewards and new state
observation, reward, done, info = self.env.step(action);
if render_on_screen:
self.env.render(mode='human');
# Count the number of wins
if reward == 1:
wins += 1;
if save_images:
# save the observation as an image
if not render_on_screen:
self.env.render(mode='matplotlib');
plt.savefig("{}/{:02d}_{:02d}.png".format(output_folder, e, c))
except KeyboardInterrupt:
print("");
log.warn('Playing interrupted by user');
log.info("Won {} out of {} games ({} %)".format(wins, n_episodes, (100*wins/n_episodes)));
def save_audio(self, save_audio_dir, save_audio_name):
dir = tools.ensure_dir(save_audio_dir)
if self.audioclip is None:
print('Video has no audio')
else:
self.audioclip.write_audiofile(os.path.join(save_audio_dir, save_audio_name), nbytes=2, codec='pcm_s16le',
bitrate='1000k', verbose=True)
def save_frames(self, save_video_dir):
dir = tools.ensure_dir(save_video_dir)
num_frame = int(self.videoclip.fps * self.videoclip.duration)
for i, frame in enumerate(self.videoclip.iter_frames()):
Image.fromarray(frame).save('{0}/{1}.jpeg'.format(dir, i))
print('saving frame {0}/{1}'.format(i, num_frame - 1))
with open(config.frames_output_dir+'/fps.txt', 'w') as f:
f.write(str(self.videoclip.fps))
def save_latest(self, directory, model_and_loss, stats_dict, store_as_best=False):
# -----------------------------------------------------------------------------------------
# Make sure directory exists
# -----------------------------------------------------------------------------------------
tools.ensure_dir(directory)
# check previous latest file version
latest_statistics_filename = os.path.join(
directory, self._prefix + self._latest_postfix + ".json")
if os.path.isfile(latest_statistics_filename):
statistics = tools.read_json(latest_statistics_filename)
shadow_is_latest = statistics['shadow']
else:
shadow_is_latest = True
stats_dict['shadow'] = not shadow_is_latest
# -----------------------------------------------------------------------------------------
# Save
# -----------------------------------------------------------------------------------------
save_dict = dict(stats_dict)
save_dict[self._model_key] = model_and_loss.state_dict()
if shadow_is_latest:
latest_checkpoint_filename = os.path.join(
directory, self._prefix + self._latest_postfix + self._extension)
else:
latest_checkpoint_filename = os.path.join(
directory, self._prefix + self._latest_postfix + '_shadow' + self._extension)
torch.save(save_dict, latest_checkpoint_filename)
tools.write_json(data_dict=stats_dict, filename=latest_statistics_filename)
# -----------------------------------------------------------------------------------------
# Possibly store as best
# -----------------------------------------------------------------------------------------
if store_as_best:
best_checkpoint_filename = os.path.join(
directory, self._prefix + self._best_postfix + self._extension)
best_statistics_filename = os.path.join(
directory, self._prefix + self._best_postfix + ".json")
logging.info("Saved checkpoint as best model..")
shutil.copyfile(latest_checkpoint_filename, best_checkpoint_filename)
shutil.copyfile(latest_statistics_filename, best_statistics_filename)
def __init__( self, env
, memory_maxlen = 1000
, net = None
, discount = 0.9
, output_folder = "output"
):
log.info("Creating an agent to play Catch");
self.env = env;
self.memory = deque(maxlen=memory_maxlen);
if net is None:
self.net = qnet( (None, self.env.grid_size, self.env.grid_size)
, self.env.action_space.n);
else:
self.net = net;
self.discount = discount;
ensure_dir(output_folder);
self.output_folder = output_folder;
self.results_filename = "{}/results.csv".format(self.output_folder);
def __init__(self, broker, account, local_base_dir, export_base_dir):
self.trading_day = time.strftime('%Y%m%d', time.localtime(time.time()))
self.broker = broker
self.account = account
self.local_log_path = '%s\\%s\\%s.log' % (local_base_dir, account,
self.trading_day)
self.export_basedir = '%s\\%s' % (export_base_dir, account)
self.export_dir = '%s\\%s\\%s' % (export_base_dir, account,
self.trading_day)
self.export_log_path = '%s\\export.log' % (self.export_dir)
self.src_property_path = '%s\\property.txt' % (self.export_dir)
self.src_entrust_path = '%s\\entrust.txt' % (self.export_dir)
self.src_trade_path = '%s\\trade.txt' % (self.export_dir)
self.dst_fund_path = '%s\\_fund.txt' % (self.export_dir)
self.dst_position_path = '%s\\_position.txt' % (self.export_dir)
self.dst_entrust_path = '%s\\_entrust.txt' % (self.export_dir)
self.dst_trade_path = '%s\\_trade.txt' % (self.export_dir)
tools.ensure_dir(self.local_log_path)
tools.ensure_dir(self.export_log_path)
def save_latest(self,
directory,
model_and_loss,
stats_dict,
store_as_best=False):
# -----------------------------------------------------------------------------------------
# Make sure directory exists
# -----------------------------------------------------------------------------------------
tools.ensure_dir(directory)
# -----------------------------------------------------------------------------------------
# Save
# -----------------------------------------------------------------------------------------
save_dict = dict(stats_dict)
save_dict[self._model_key] = model_and_loss.state_dict()
latest_checkpoint_filename = os.path.join(
directory, self._prefix + self._latest_postfix + self._extension)
latest_statistics_filename = os.path.join(
directory, self._prefix + self._latest_postfix + ".json")
torch.save(save_dict, latest_checkpoint_filename)
tools.write_json(data_dict=stats_dict,
filename=latest_statistics_filename)
# -----------------------------------------------------------------------------------------
# Possibly store as best
# -----------------------------------------------------------------------------------------
if store_as_best:
best_checkpoint_filename = os.path.join(
directory, self._prefix + self._best_postfix + self._extension)
best_statistics_filename = os.path.join(
directory, self._prefix + self._best_postfix + ".json")
logging.info("Saved checkpoint as best model..")
shutil.copyfile(latest_checkpoint_filename,
best_checkpoint_filename)
shutil.copyfile(latest_statistics_filename,
best_statistics_filename)
def multiplots():
mpl.style.use(['seaborn-white', 'seaborn-paper', 'grayscale'])
latexify()
for model, title in zip([no_resample, undersample, oversample], ['none', 'undersample', 'oversample']):
y, y_pred, c = model(classifier)
acc = metrics.accuracy_score(y, y_pred)
#prec = metrics.precision_score(y, y_pred, labels=labels, average='macro')
#rec = metrics.recall_score(y, y_pred, labels=labels, average='macro')
#f1 = metrics.f1_score(y, y_pred, labels=labels, average='macro')
cm = metrics.confusion_matrix(y, y_pred, labels=labels)
cm = norm_cm(cm)
cm = pd.DataFrame(cm, index=labels, columns=labels)
fig, ax = plt.subplots(dpi=92, constrained_layout=True)
#print(f'{title}\t-- Acc.: {acc:.3f};\t Prec.: {prec:.3f}\t Rec.: {rec:.3f}\t F1: {f1:.3f}')
print('Resample:', title, classifier[0], f'accuracy={acc:.3f}')
print(metrics.classification_report(y, y_pred, labels=labels))
plt.suptitle(f'accuracy={acc:.3f}')
#plt.suptitle(f'good={c["good"]:,}\ninterm.={c["interm."]:,}\nbad={c["bad"]:,}', ha='left')
sns.heatmap(cm, vmin=0, vmax=1, annot=True, fmt='.2f', cmap='Greys', ax=ax, cbar=False, square=True)
#ax.set_title(f'Accuracy = {acc:.3f}', loc='center')
ax.set_title(
f'good: {c["good"]:,}\ninterm.: {c["interm."]:,}\nbad: {c["bad"]:,}',
fontdict={'fontsize': 9},
loc='left'
)
format_axes_for_cm(ax)
#fig.tight_layout()
ensure_dir('./output/resampling/')
fig.savefig(f'./output/resampling/{title}.pdf', dpi=92, bbox_inches='tight')
plt.close(fig)
def advanced_charts():
mpl.style.use(['seaborn-white', 'seaborn-paper', 'grayscale'])
latexify(columns=2)
w_sizes = (2, 5, 10, 15, 20, 30, 50, 80,
100)[::-1] # [::-1] will reverse order
fig, ax = plt.subplots(dpi=92)
colors = sns.color_palette("cubehelix", len(w_sizes))
for w_prr, color in zip(w_sizes, colors):
acc = []
for w_history in w_sizes:
y, y_pred = different_window_sizes(w_prr, w_history)
acc.append(metrics.accuracy_score(y, y_pred))
ax.plot(w_sizes,
acc,
label=f'W$_\\mathrm{{PRR}}={w_prr}$',
color=color)
ax.set_ylabel('accuracy')
ax.set_xlabel(f'W$_\\mathrm{{history}}$')
ax.set_xticks(w_sizes[::-1])
ax.set_xlim(min(w_sizes), max(w_sizes))
format_axes_for_chart(ax)
fig.tight_layout()
fig.legend(loc='right')
ensure_dir('./output/')
fig.savefig('./output/different_window_sizes_linechart.pdf',
dpi=92,
bbox_inches='tight')
plt.close(fig)
def multiple_figures():
mpl.style.use(['seaborn-white', 'seaborn-paper', 'grayscale'])
latexify()
pipe = pipe_dtree
for features in feature_sets:
y, y_pred = different_features(pipe, features)
acc = metrics.accuracy_score(y, y_pred)
prec = metrics.precision_score(y,
y_pred,
average='weighted',
labels=labels)
recall = metrics.recall_score(y,
y_pred,
average='weighted',
labels=labels)
#prec = metrics.precision_score(y, y_pred, labels=labels, average='micro')
#rec = metrics.recall_score(y, y_pred, labels=labels, average='micro')
cm = metrics.confusion_matrix(y, y_pred, labels=labels)
cm = norm_cm(cm)
cm = pd.DataFrame(cm, index=labels, columns=labels)
fig, ax = plt.subplots(dpi=92)
sns.heatmap(cm,
vmin=0,
vmax=1,
annot=True,
fmt='.2f',
cmap='Greys',
ax=ax,
cbar=False,
square=True)
format_axes_for_cm(ax)
feature_str = stringify_features(features)
ax.set_title(
f'Accuracy = {acc:.3f}\n(prec = {prec:.3f}; rec = {recall:.3f})')
fig.tight_layout()
ensure_dir('./output/features/dtree/')
fig.savefig(f'./output/features/dtree/{feature_str}.pdf',
dpi=92,
bbox_inches='tight')
plt.close(fig)
print(f'Done {features}')
pipe = pipe_logreg
for features in feature_sets:
print('Features', features)
y, y_pred = different_features(pipe, features)
acc = metrics.accuracy_score(y, y_pred)
prec = metrics.precision_score(y,
y_pred,
average='micro',
labels=labels)
recall = metrics.recall_score(y,
y_pred,
average='micro',
labels=labels)
#prec = metrics.precision_score(y, y_pred, labels=labels, average='micro')
#rec = metrics.recall_score(y, y_pred, labels=labels, average='micro')
cm = metrics.confusion_matrix(y, y_pred, labels=labels)
cm = norm_cm(cm)
cm = pd.DataFrame(cm, index=labels, columns=labels)
fig, ax = plt.subplots(dpi=92)
sns.heatmap(cm,
vmin=0,
vmax=1,
annot=True,
fmt='.2f',
cmap='Greys',
ax=ax,
cbar=False,
square=True)
format_axes_for_cm(ax)
feature_str = stringify_features(features)
ax.set_title(
f'Accuracy = {acc:.3f}\n(prec = {prec:.3f}, rec = {recall:.3f})')
fig.tight_layout()
ensure_dir('./output/features/logistic/')
fig.savefig(f'./output/features/logistic/{feature_str}.pdf',
dpi=92,
bbox_inches='tight')
plt.close(fig)
print(f'Done {features}')
import numpy as np
# In[50]:
allmean =[]
freq=[]
standard_deviation=[]
# In[51]:
EXP_FOLDERNAME = os.path.join('experiments', 'sampling_rate')
tools.ensure_dir(EXP_FOLDERNAME)
experimental_data= (os.path.join(EXP_FOLDERNAME,'calculated_data.json'))
# In[52]:
def getdata(name):
EXP_FILENAME =(os.path.join(EXP_FOLDERNAME, name + '.json'))
EXP_TIME_SECOND = 20
if __name__ == '__main__':
handler = SerialHandler('/dev/ttyACM3', 115200)
handler.current_state = {'time': 0}
# Step1: Split video and audio
GENERATE_FRAMES = True
if GENERATE_FRAMES:
sp = AVSplit(config.source_video)
sp.save_frames(config.frames_output_dir)
sp.save_audio(config.audio_output_dir, config.audio_output_name)
frame_names = tools.load_frames(config.FRAMES_PATH_BASE)
# Step2: Detect logo frame-wise
num_frames = len(frame_names)
for i, frame in enumerate(frame_names):
try:
print('detecting frame {0}/{1}'.format(i, num_frames))
img = LogoDetect(config.DETECT_OBJ, frame)
directory = tools.ensure_dir(config.save_detect_dir)
Image.fromarray(img).save('{0}/{1}.jpeg'.format(directory, i))
finally:
pass
# Step3: export video file
detected_frames = tools.load_frames(config.DETECTED_PATH_BASE)
clip = moviepy.editor.ImageSequenceClip(detected_frames, fps=10)
if GENERATE_FRAMES:
fps = sp.videoclip.fps
clip = clip.set_fps(fps)
if sp.audioclip is not None:
clip = clip.set_audio(sp.audioclip)
else:
with open(config.frames_output_dir + '/fps.txt', 'r') as f:
fps = float(f.read())
import inspect
HERE_PATH = os.path.dirname(
os.path.abspath(inspect.getfile(inspect.currentframe())))
from datetime import datetime
import pandas as pd
import tinydb
import tools
DB_FOLDERNAME = os.path.join(HERE_PATH, 'db')
TRACKING_DB_FILENAME = os.path.join(DB_FOLDERNAME, 'tracking_info.json')
CSV_FOLDERNAME = os.path.join(HERE_PATH, 'csv')
tools.ensure_dir(CSV_FOLDERNAME)
CSV_FILENAME = os.path.join(CSV_FOLDERNAME, 'tracking_data.csv')
UNKNOWN_REFERENCE = ''
LOCALHOST_MARKER = 'LOCALHOST'
def get_location_info(client_ip_info):
location_info = {}
# fill list with unknown values
field_list = ['ip', 'country', 'region', 'city', 'geo']
for field_name in field_list:
location_info[field_name] = UNKNOWN_REFERENCE
def save(self, net_file):
ensure_dir(os.path.dirname(net_file));
log.info("Saving net file: {}".format(net_file));
with open(net_file, 'wb') as pkl_file:
param_values = lasagne.layers.get_all_param_values(self.net['output']);
pickle.dump(param_values, pkl_file);
import os
# this get our current location in the file system
import inspect
HERE_PATH = os.path.dirname(
os.path.abspath(inspect.getfile(inspect.currentframe())))
import tinydb
import tools
DB_FOLDERNAME = os.path.join(HERE_PATH, 'db')
tools.ensure_dir(DB_FOLDERNAME)
TRACKING_DB_FILENAME = os.path.join(DB_FOLDERNAME, 'tracking_info.json')
PROCESSED_DB_FILENAME = os.path.join(DB_FOLDERNAME, 'log_processed.json')
CONNECTION_INFO_FILENAME = 'connection_info.json'
URL_INFO_FILENAME = 'url_info.json'
# LOG_ROOT_FOLDER = os.path.join(HERE_PATH, 'logs')
LOG_ROOT_FOLDER = os.path.join(HERE_PATH,
'../server/logs') ## for local server log build
if __name__ == '__main__':
tracking_info_db = tinydb.TinyDB(TRACKING_DB_FILENAME)
log_processed_db = tinydb.TinyDB(PROCESSED_DB_FILENAME)
log_folders = tools.list_folders(LOG_ROOT_FOLDER)
for log_folder in log_folders:
def toNuke(params, oneFile=False):
log.debug('To Nuke - Start')
cameras = params['cameras']
camerasOutSize = params['camerasOutSize']
camerasFrameStart = params['camerasFirstFrame']
filesGenerated = []
argsToBatchs = []
if oneFile:
filepath = params['file_browser'] + '_undisto.nk'
tools.ensure_dir(filepath)
fileObj = open(filepath, 'w')
for index, cam in enumerate(cameras):
fileLines = []
argsExportNuke = [
cam, index, camerasOutSize[index], filepath, params['date'],
camerasFrameStart[index]
]
fileLines, argsToBatch = exportNuke(*argsExportNuke)
if fileLines == -1:
# print ("Problem with camera "+ str(cam) )
continue
writeLines(fileObj, fileLines)
argsToBatchs.append(argsToBatch)
filesGenerated.append(filepath)
else:
# create Directory
folderCameras = params['file_browser'] + '_undisto/'
try:
os.makedirs(folderCameras)
except:
pass # Todo check if is only that the folder exists
for index, cam in enumerate(cameras):
cameraName = "%s_%s_1" % (tools.validName(
tde4.getCameraName(cam)), index)
filepath = folderCameras + cameraName + '_undisto.nk'
tools.ensure_dir(filepath)
fileObj = open(filepath, 'w')
fileLines = []
argsExportNuke = [
cam, index, camerasOutSize[index], filepath, params['date'],
camerasFrameStart[index]
]
fileLines, argsToBatch = exportNuke(*argsExportNuke)
if fileLines == -1:
# print ("Problem with camera "+ str(cam) )
continue
writeLines(fileObj, fileLines)
argsToBatchs.append(argsToBatch)
filesGenerated.append(filepath)
# Launch batch Render
log.debug('Launching batch Render')
import toBatch
for argsToBatch in argsToBatchs:
toBatch.batch3DE(*(argsToBatch))
return filesGenerated
log.debug('To Nuke - End')
def multiple_figures():
mpl.style.use(['seaborn-white', 'seaborn-paper', 'grayscale'])
latexify()
cv = model_selection.StratifiedKFold(n_splits=10, shuffle=True)
scaler = preprocessing.StandardScaler()
resample = over_sampling.RandomOverSampler()
baseline = pipeline.make_pipeline(
scaler, resample,
dummy.DummyClassifier(strategy='constant', constant='good'))
logreg = pipeline.make_pipeline(
scaler,
resample,
linear_model.LogisticRegression(solver='lbfgs', multi_class='ovr'),
)
dtree = pipeline.make_pipeline(
scaler,
resample,
tree.DecisionTreeClassifier(),
)
knn = pipeline.make_pipeline(
scaler,
resample,
neighbors.KNeighborsClassifier(),
)
mlp = pipeline.make_pipeline(
scaler,
resample,
neural_network.MLPClassifier(hidden_layer_sizes=(
100,
100,
100,
),
activation='relu',
solver='adam'),
)
svc = pipeline.make_pipeline(
scaler,
resample,
svm.LinearSVC(),
)
RForest = pipeline.make_pipeline(
scaler,
resample,
ensemble.RandomForestClassifier(n_estimators=100),
)
models = (
('Constant', baseline),
('Logistic Regression', logreg),
('Decision Tree', dtree),
#('kNN', knn),
('Multi-Layer Perceptron', mlp),
('linearSVM', svc),
('Random Forest', RForest),
)
# Special case of baseline
filename = 'baseline-link-overall'
df = prepare_data()
y, y_pred = df['class'].ravel(), df['class_overall'].ravel()
acc = metrics.accuracy_score(y, y_pred)
prec = metrics.precision_score(y,
y_pred,
average='weighted',
labels=labels)
recall = metrics.recall_score(y, y_pred, average='weighted', labels=labels)
cm = metrics.confusion_matrix(y, y_pred, labels=labels)
cm = norm_cm(cm)
cm = pd.DataFrame(cm, index=labels, columns=labels)
fig, ax = plt.subplots(dpi=92)
sns.heatmap(cm,
vmin=0,
vmax=1,
annot=True,
fmt='.2f',
cmap='Greys',
ax=ax,
cbar=False,
square=True)
ax.set_title(
f'accuracy = {acc:.3f}\n(prec = {prec:.3f}, rec = {recall:.3f})')
format_axes_for_cm(ax)
fig.tight_layout()
ensure_dir('./output/models/')
fig.savefig(f'./output/models/{filename}.pdf', dpi=92, bbox_inches='tight')
plt.close(fig)
print(f'Done {filename}')
for name, pipe in models:
filename = name.lower().replace(' ', '_')
y, y_pred = different_models(pipe)
acc = metrics.accuracy_score(y, y_pred)
#prec = metrics.precision_score(y, y_pred, average='weighted', labels=labels)
#recall = metrics.recall_score(y, y_pred, average='weighted', labels=labels)
print(name)
print(metrics.classification_report(y, y_pred, labels=labels))
cm = metrics.confusion_matrix(y, y_pred, labels=labels)
cm = norm_cm(cm)
cm = pd.DataFrame(cm, index=labels, columns=labels)
fig, ax = plt.subplots(dpi=92)
sns.heatmap(cm,
vmin=0,
vmax=1,
annot=True,
fmt='.2f',
cmap='Greys',
ax=ax,
cbar=False,
square=True)
ax.set_title(f'accuracy={acc:.3f}')
format_axes_for_cm(ax)
fig.tight_layout()
ensure_dir('./output/models/')
fig.savefig(f'./output/models/{filename}.pdf',
dpi=92,
bbox_inches='tight')
plt.close(fig)
def main():
mpl.style.use(['seaborn-white', 'seaborn-paper', 'grayscale'])
latexify(columns=2)
#cv = model_selection.StratifiedKFold(n_splits=10, shuffle=True)
#poly = preprocessing.PolynomialFeatures(degree=2)
scaler = preprocessing.StandardScaler()
resample = over_sampling.RandomOverSampler()
baseline = pipeline.make_pipeline(
scaler, resample, dummy.DummyClassifier(strategy='constant',
constant=0))
logreg = pipeline.make_pipeline(
scaler,
resample,
linear_model.LogisticRegression(),
)
sgd = pipeline.make_pipeline(
scaler,
resample,
linear_model.SGDClassifier(),
)
dtree = pipeline.make_pipeline(
scaler,
resample,
tree.DecisionTreeClassifier(),
)
mlp = pipeline.make_pipeline(scaler, resample,
neural_network.MLPClassifier())
svc = pipeline.make_pipeline(scaler, resample, svm.LinearSVC())
RForest = pipeline.make_pipeline(scaler, resample,
ensemble.RandomForestClassifier())
models = (
('Constant', baseline),
('Logistic Reg.', logreg),
('Decision Tree', dtree),
#('kNN', knn),
('Multi-Layer Perceptron', mlp),
('SVM (linear kernel)', svc),
('Random Forest', RForest),
)
colors = sns.color_palette("cubehelix", len(models))
fig, ax = plt.subplots(dpi=92) # Setup a figure
#ax.set_title('Precision-Recall curve')
#ax.set_xlim(0, 1)
#ax.set_ylim(0, 1)
ax.set_xlabel('Recall = $\\frac{{TP}}{{TP+FN}}$')
ax.set_ylabel('Precision = $\\frac{{TP}}{{TP+FP}}$')
# Prepare data for processing
data = prepare_data()
X, y = data[['rssi', 'rssi_avg', 'rssi_std']].values, data['class'].ravel()
Y = preprocessing.label_binarize(y, classes=classes)
X_train, X_test, y_train, y_test = model_selection.train_test_split(
X, Y, test_size=0.2, random_state=random_state)
for (name, model), color in zip(models, colors):
classifier = multiclass.OneVsRestClassifier(
model) # Make model support *.decision_function
classifier.fit(X_train, y_train)
# generate y_score
if hasattr(classifier, 'decision_function'):
y_score = classifier.decision_function(X_test)
else:
y_score = classifier.predict_proba(X_test)
#continue
# generate probabilities
#y_proba = classifier.predict_proba(X_test)
# generate predictions
y_pred = classifier.predict(X_test)
precision = dict()
recall = dict()
average_precision = dict()
acc = metrics.accuracy_score(y_test, y_pred)
for i in [1]: # We observe only intermediate class
precision[i], recall[i], _ = metrics.precision_recall_curve(
y_test[:, i], y_score[:, i])
average_precision[i] = metrics.average_precision_score(
y_test[:, i], y_score[:, i])
ax.step(recall[i],
precision[i],
where='post',
color=color,
alpha=0.65,
label=f'{name}')
print(f'Plotted {name}')
ax.legend(loc="best")
format_axes_for_chart(ax)
fig.tight_layout()
ensure_dir('./output/')
fig.savefig('./output/precision-recall-curve.pdf',
dpi=92,
bbox_inches='tight')
#plt.show()
plt.close(fig)
