def download_track(track):
download_dir = settings.getSetting('folder')
downloaded, path, folder = getTrackPath(download_dir, track, codec)
if not downloaded:
checkFolder(folder)
info = get_track_download_info(track, codec, high_res)
log("download: %s, %s" % (info.codec, info.bitrate_in_kbps))
info.download(fixPath(path))
if codec == "mp3":
audio = mp3.MP3(path, ID3=easyid3.EasyID3)
audio["title"] = track.title
audio["length"] = str(track.duration_ms)
if track.artists:
audio["artist"] = track.artists[0].name
if track.albums:
audio["album"] = track.albums[0].title
audio["tracknumber"] = str(
track.albums[0].track_position.index)
audio["date"] = str(track.albums[0].year)
audio["genre"] = track.albums[0].genre
audio.save()
elif codec == "aac":
log("TODO: add tags to aac files")
# notify("Download", "Done: %s" % path, 1)
return path
def get_folder(cfg, experiment, check=False):
"""Returns the experiment folder. Creates it if necessary."""
folder = get_raw_folder(cfg)
utils.checkFolder(folder)
# add experiment subfolder
folder = os.path.join(folder, get_name('exp', experiment))
if check:
utils.checkFolder(folder)
return folder
def drw_small_overview(resOT, plt_deviation=0, alt=0, jEE_first=0):
def prepareData(resOT):
res = np.mean(resOT, axis=0)
res = np.mean(res, axis=0)[2:]
y_errors = np.std(resOT, axis=(0, 1), dtype=np.float64)[2:]
return res, y_errors,
def Other_data(resOT):
res = np.mean(resOT, axis=0)
y_errors = np.std(resOT, axis=(0), dtype=np.float64)
print(y_errors)
return res, y_errors
### Acutal Function Body ###
if len(resOT.shape) > 2:
res, y_errors = prepareData(resOT)
else:
res, y_errors = Other_data(resOT)
fig = plt.figure()
ax = fig.add_subplot(111)
x_ax = np.arange(len(res))
ax.errorbar(x_ax, res, yerr=y_errors, capsize=8, linestyle='--')
plt.fill_between(x_ax, res + y_errors, res - y_errors, alpha=0.3)
ax.set_xlabel('Temporal Distance of Prediction')
ax.set_ylabel('Prediction Accuracy')
folder = "../figs"
name = "OneMemory"
folder = utils.checkFolder(folder)
fullname = utils.testTheName(folder + name, "pdf")
plt.savefig(fullname)
utils.plotMessage(fullname)
plt.show()
plt.close(fig)
def one_vs_zero():
data_array = np.load("ml3_data.npy", allow_pickle=True)
for data in data_array[0]:
print(data)
naughts = np.array([row for row in data_array if row[0][0] < 0.05])
fulls = np.array([row for row in data_array if row[0][0] > 0.2])
naughts_mean = np.mean(naughts, axis=0)
naughts_std = np.std(naughts, axis=0)
fulls_mean = np.mean(fulls, axis=0)
fulls_std = np.std(fulls, axis=0)
add_base = np.array([1, 1])[:, None]
print(add_base.shape)
print(naughts_mean.shape)
# naughts_mean = np.concatenate((add_base,naughts_mean),axis=1)
naughts_mean = np.hstack((add_base, naughts_mean))
naughts_std = np.hstack((add_base * 0, naughts_std))
fulls_mean = np.concatenate((add_base * 0, fulls_mean), axis=1)
fulls_std = np.concatenate((add_base * 0, fulls_std), axis=1)
all_mean = np.concatenate((naughts_mean, fulls_mean), axis=0)
all_std = np.concatenate((naughts_std, fulls_std), axis=0)
# all_mean = all_mean[:,1:]
# all_std = all_std[:,1:]
# all_mean = np.concatenate((naughts_mean, fulls_mean), axis=0)
print(all_mean.shape)
x_ax = np.arange(all_mean.shape[1])
description = [
"fully activated network, excitatory",
"fully activated network, inhibitory",
"dead network, excitatory",
"dead network, inhibitory",
]
cols = ['b', 'r', 'b', 'r']
stil = ['-', '-', '--', '--']
fig = plt.figure(tight_layout=True)
ax = fig.add_subplot(111)
for i in range(4):
plt.errorbar(x_ax,
all_mean[i],
all_std[i],
capsize=4,
label=description[i],
c=cols[i],
linestyle=stil[i])
plt.legend()
ax.set_xlabel('Timesteps since start')
ax.set_ylabel('Share of all Neurons')
# ax.set_xticks(np.linspace(0,dur-1,5))
folder = "../figs"
name = "one_or_zero"
folder = utils.checkFolder(folder)
fullname = utils.testTheName(folder + name, "pdf")
plt.savefig(fullname)
utils.plotMessage(fullname)
plt.show()
plt.close(fig)
def hamming_analyse():
diff = np.load("hamming3_control.npy")[0][2]
control = np.load("hamming3_control.npy")[1][2]
diff = np.delete(diff, 3, 0)
# diff = np.load("hamming3.npy")[0][:,:21]
# control = np.load("hamming3.npy")[1][:,:21]
# diff = np.delete(diff,1, 0)
diff_mean = np.mean(diff, axis=0)
# idx = np.argmin(np.mean(diff,axis=1))
# diff_mean = diff[idx]
diff_std = np.std(diff, axis=0)
control_mean = np.mean(control, axis=0)
control_std = np.std(control, axis=0)
fig = plt.figure()
ax = fig.add_subplot(111)
dur = diff_mean.shape[0]
time = np.arange(dur)
# ax.plot(diff_mean)
diff_txt = "One flipped"
control_txt = "Random Pairings"
ax.errorbar(time, diff_mean, yerr=diff_std, label=diff_txt)
ax.errorbar(time, control_mean, yerr=control_std, label=control_txt)
ax.set_xlabel('Timesteps since start')
ax.set_ylabel('Share of all Neurons')
ax.set_xticks(np.linspace(0, dur - 1, 5))
ax.legend()
folder = "../figs"
name = "hamming"
title = "Hamming Distance"
plt.title(title)
folder = utils.checkFolder(folder)
fullname = utils.testTheName(folder + name, "pdf")
plt.savefig(fullname)
utils.plotMessage(fullname)
plt.show()
plt.close(fig)
def drw_overview_delay_1(resOT):
def prepareData(resOT):
res = np.mean(resOT, 0)
idx = np.arange(len(res[0, :]))
idx[0], idx[1] = 1, 0
res = res[:, idx]
y_errors = np.std(resOT, 0, dtype=np.float64)
y_errors = y_errors[:, idx]
dist = len(res[0]) - 2
return res, y_errors, dist
### style of plotted vars ###
# print(col)
def prepareColor(res):
for i in range(len(res)):
if res[i][0] != res[i + 1][0]:
length_line = i + 1
amount_lines = int(len(res) / length_line)
break
cmap = mpl.cm.plasma
col = cmap(np.linspace(0, .9, amount_lines))
return col, length_line
### plot all lines ###
def actualPlot_mean(ax, res, y_errors, length_line):
cleg = []
amount_of_lines = range(0, len(res), length_line)
print(list(amount_of_lines))
delay = 1
j = delay
for i in amount_of_lines:
plo = np.transpose(res[i:i + length_line])
pl_err = np.transpose(y_errors[i:i + length_line])
if ax:
ax.errorbar(
plo[1],
plo[j + 2],
yerr=pl_err[j + 2],
capsize=8,
c=col[int(i / length_line)],
)
else:
plt.errorbar(
plo[1],
plo[j + 2],
yerr=pl_err[j + 2],
capsize=8,
c=col[int(i / length_line)],
)
cleg.append(r"$E_E$" + ": %3.2f" % (plo[0, 0]))
return cleg
def make_leg(fig, cleg):
leg_elem = []
for i in range(len(cleg)):
line = mpl.lines.Line2D([0], [0], c=col[i], label=cleg[i])
leg_elem.append(line)
fig.legend(
bbox_to_anchor=(.97, .97),
# fontsize='small',
handles=leg_elem,
loc="upper right")
def make_labels():
plt.ylabel('true prediction rate')
plt.xlabel(r"$J_{EE}$")
### Acutal Function Body ###
res, y_errors, dist = prepareData(resOT)
col, length_line = prepareColor(res)
fig = plt.figure(tight_layout=True, figsize=(8, 6))
cleg = actualPlot_mean(
None,
res,
y_errors,
length_line,
)
make_leg(fig, cleg)
make_labels()
folder = "../figs"
name = "delayOne"
folder = utils.checkFolder(folder)
fullname = utils.testTheName(folder + name, "pdf")
plt.savefig(fullname)
utils.plotMessage(fullname)
plt.show()
plt.close(fig)
def drw_overviewOT(resOT, y_name, save_name):
plt_deviation = 0,
alt = 0,
jEE_first = 0
def prepareData(resOT):
res = np.mean(resOT, 0)
idx = np.arange(len(res[0, :]))
idx[0], idx[1] = 1, 0
res = res[:, idx]
y_errors = np.std(resOT, 0, dtype=np.float64)
y_errors = y_errors[:, idx]
dist = len(res[0]) - 2
return res, y_errors, dist
### style of plotted vars ###
# print(col)
def prepareColor(res):
for i in range(len(res)):
if res[i][0] != res[i + 1][0]:
length_line = i + 1
amount_lines = int(len(res) / length_line)
break
cmap = mpl.cm.plasma
col = cmap(np.linspace(0, .9, amount_lines))
return col, length_line
### plot all lines ###
def actualPlot_mean(ax, res, y_errors, length_line, lstyle, jEE_first):
cleg = []
lleg = []
amount_of_lines = range(0, len(res), length_line)
print(list(amount_of_lines))
for j in range(dist):
# for j in range(3):
lleg.append("delay: %d" % (j))
for i in amount_of_lines:
# for i in [0, 24, 60]:
plo = np.transpose(res[i:i + length_line])
pl_err = np.transpose(y_errors[i:i + length_line])
if ax:
ax.errorbar(plo[1],
plo[j + 2],
yerr=pl_err[j + 2],
capsize=8,
c=col[int(i / length_line)],
linestyle=lstyle[j])
else:
plt.errorbar(plo[1],
plo[j + 2],
yerr=pl_err[j + 2],
capsize=8,
c=col[int(i / length_line)],
linestyle=lstyle[j])
if j == 0:
# print(plo[0])
if (jEE_first):
cleg.append("J_EE: %3.2f" % (plo[0, 0]))
else:
cleg.append(r"$E_E$" + ": %3.2f" % (plo[0, 0]))
return cleg, lleg
def actualPlot_std(ax, res, y_errors, length_line, lstyle, jEE_first):
cleg = []
lleg = []
amount_of_lines = range(0, len(res), length_line)
for j in range(dist):
lleg.append("delay: %d" % (j))
for i in amount_of_lines:
plo = np.transpose(res[i:i + length_line])
pl_err = np.transpose(y_errors[i:i + length_line])
if ax:
ax.plot(plo[1],
pl_err[j + 2],
c=col[int(i / length_line)],
linestyle=lstyle[j])
else:
plt.plot(plo[1],
plo[j + 2],
c=col[int(i / length_line)],
linestyle=lstyle[j])
if j == 0:
# print(plo[0])
if (jEE_first):
cleg.append("J_EE: %3.2f" % (plo[0, 0]))
else:
cleg.append(r"$E_E$" + ": %3.2f" % (plo[0, 0]))
return cleg, lleg
def actualPlot_simp(ax, res, length_line, lstyle, jEE_first):
cleg = []
lleg = []
amount_of_lines = range(0, len(res), length_line)
for j in range(dist):
lleg.append("delay: %d" % (j))
for i in amount_of_lines:
plo = np.transpose(res[i:i + length_line])
plt.plot(plo[1],
plo[j + 2],
c=col[int(i / length_line)],
linestyle=lstyle[j])
if j == 0:
# print(plo[0])
if (jEE_first):
cleg.append("J_EE: %3.2f" % (plo[0, 0]))
else:
cleg.append(r"$E_E$" + ": %3.2f" % (plo[0, 0]))
return cleg, lleg
### color predictive capabilities ###
# plt.axhspan(0.9, 1, facecolor='red', alpha=0.3)
### Custom Legend for less lines on display ###
def make_leg(fig, cleg, lleg, lstyle):
leg_elem = []
for i in range(len(cleg)):
line = mpl.lines.Line2D([0], [0], c=col[i], label=cleg[i])
leg_elem.append(line)
fig.legend(
bbox_to_anchor=(.98, .36),
# fontsize='small',
handles=leg_elem,
# loc="lower right"
)
def make_labels(jEE_first):
plt.ylabel(y_name)
if (jEE_first):
plt.xlabel(r'$E_E$"+"')
else:
plt.xlabel(r"$J_{EE}$")
### Acutal Function Body ###
res, y_errors, dist = prepareData(resOT)
col, length_line = prepareColor(res)
lstyle = np.hstack((np.array(['-', '-.', '--',
':']), np.array(linestyle_tuple)[:, 1]))
if alt:
fig = plt.figure(tight_layout=True, figsize=(8, 6))
cleg, lleg = actualPlot_std(None, res, y_errors, length_line, lstyle,
jEE_first)
make_leg(fig, cleg, lleg, lstyle)
make_labels(jEE_first)
else:
if plt_deviation:
fig3 = plt.figure(constrained_layout=True)
ax1 = fig3.add_subplot(211)
ax2 = fig3.add_subplot(212)
cleg, lleg = actualPlot_mean(ax1, res, y_errors, length_line,
lstyle, jEE_first)
cleg, lleg = actualPlot_std(ax2, res, y_errors, length_line,
lstyle, jEE_first)
make_leg(fig3, cleg, lleg, lstyle)
else:
fig = plt.figure(tight_layout=True, figsize=(8, 6))
cleg, lleg = actualPlot_mean(None, res, y_errors, length_line,
lstyle, jEE_first)
make_leg(fig, cleg, lleg, lstyle)
make_labels(jEE_first)
folder = "../figs"
name = save_name
folder = utils.checkFolder(folder)
fullname = utils.testTheName(folder + name, "pdf")
plt.savefig(fullname)
utils.plotMessage(fullname)
plt.show()
plt.close(fig)
def flybyVectors3D(V2,
V4,
Vinfinityt,
VinfinitytStar,
Vplanet,
planetName,
mjd,
folder='output/flybyExamples',
save=True,
show=True):
DeltaV = V4 - V2
fig = plt.figure()
ax = plt.axes(projection='3d')
quivV2 = plt.quiver(
0,
0,
0,
V2[0],
V2[1],
V2[2],
# angles='xy', scale_units='xy',
# scale=1,
arrow_length_ratio=0.05,
color='C0',
zorder=3)
quivV4 = plt.quiver(
0,
0,
0,
V4[0],
V4[1],
V4[2],
# angles='xy', scale_units='xy', scale=1,
arrow_length_ratio=0.05,
color='C3',
zorder=3)
quivVplanet = plt.quiver(
0,
0,
0,
Vplanet[0],
Vplanet[1],
Vplanet[2],
# angles='xy', scale_units='xy', scale=1,
arrow_length_ratio=0.05,
color='C5',
zorder=3)
quivDeltaV = plt.quiver(
V2[0],
V2[1],
V2[2],
DeltaV[0],
DeltaV[1],
DeltaV[2],
# angles='xy', scale_units='xy', scale=1,
arrow_length_ratio=0.5,
color='C2',
zorder=3)
quivVinf = plt.quiver(
0,
0,
0,
Vinfinityt[0],
Vinfinityt[1],
Vinfinityt[2],
# angles='xy', scale_units='xy', scale=1,
arrow_length_ratio=0.5,
color='C4',
zorder=3)
quivVinfStar = plt.quiver(
0,
0,
0,
VinfinitytStar[0],
VinfinitytStar[1],
VinfinitytStar[2],
# angles='xy', scale_units='xy', scale=1,
arrow_length_ratio=0.5,
color='C1',
zorder=3)
with warnings.catch_warnings():
warnings.simplefilter(action='ignore', category=FutureWarning)
ax.text(V2[0] / 2, V2[1] / 2, V2[2] / 2, r'$V_2$', V2, color='C0')
ax.text(V4[0] / 2, V4[1] / 2, V4[2] / 2, r'$V_4$', V4, color='C3')
ax.text(V2[0] + DeltaV[0] / 2,
V2[1] + DeltaV[1] / 2,
V2[2] + DeltaV[2] / 2,
r'$\Delta V$',
V2,
color='C2')
ax.text(Vplanet[0] / 2,
Vplanet[1] / 2,
Vplanet[2] / 2,
r'$V_{planet}$',
Vplanet,
color='C5')
ax.text(Vinfinityt[0] / 2,
Vinfinityt[1] / 2,
Vinfinityt[2] / 2,
r'$V_{\infty t}$',
Vinfinityt,
color='C4')
ax.text(VinfinitytStar[0] / 2,
VinfinitytStar[1] / 2,
VinfinitytStar[2] / 2,
r'$V_{\infty t}^\star$',
VinfinitytStar,
color='C1')
axisEqual3D(ax)
ax.set_ylabel('v_y')
ax.set_xlabel('v_x')
ax.set_zlabel('v_z')
plt.title('Flyby @ ' + planetName + '\n' + str(mjd) + ' mjd')
plt.grid(zorder=0)
if save == True:
checkFolder(os.path.join(folder))
plt.savefig(os.path.join(folder,
'flyby-' + planetName + str(mjd) + '.pdf'),
dpi=300)
plt.savefig(os.path.join(folder,
'flyby-' + planetName + str(mjd) + '.png'),
dpi=300)
if show:
plt.show()
else:
plt.show(block=False)
def flybyVectors(V2,
V4,
Vinfinityt,
VinfinitytStar,
Vplanet,
planetName,
mjd,
folder='output/flybyExamples',
save=True,
show=True):
DeltaV = V4 - V2
fig = newFigure(height=6.2 / 2, half=True, target='thesis')
ax = plt.gca()
quivV2 = plt.quiver(0,
0,
V2[0],
V2[1],
angles='xy',
scale_units='xy',
scale=1,
color='C0',
zorder=3)
quivV4 = plt.quiver(0,
0,
V4[0],
V4[1],
angles='xy',
scale_units='xy',
scale=1,
color='C3',
zorder=3)
quivVplanet = plt.quiver(0,
0,
Vplanet[0],
Vplanet[1],
angles='xy',
scale_units='xy',
scale=1,
color='C5',
zorder=3)
quivDeltaV = plt.quiver(V2[0],
V2[1],
DeltaV[0],
DeltaV[1],
angles='xy',
scale_units='xy',
scale=1,
color='C2',
zorder=3)
quivVinf = plt.quiver(0,
0,
Vinfinityt[0],
Vinfinityt[1],
angles='xy',
scale_units='xy',
scale=1,
color='C4',
zorder=3)
quivVinfStar = plt.quiver(0,
0,
VinfinitytStar[0],
VinfinitytStar[1],
angles='xy',
scale_units='xy',
scale=1,
color='C1',
zorder=3)
plt.quiverkey(quivV2,
V2[0] / 2,
V2[1] / 2,
2,
r'$V_{\mathrm{in}}$',
coordinates='data',
labelpos='W',
labelcolor='C0')
plt.quiverkey(quivV4,
V4[0] / 3,
V4[1] / 3,
2,
r'$V_{\mathrm{out}}$',
coordinates='data',
labelpos='E',
labelcolor='C3')
plt.quiverkey(quivDeltaV,
V2[0] + DeltaV[0] / 2,
V2[1] + DeltaV[1] / 2,
2,
r'$\Delta V$',
coordinates='data',
labelpos='W',
labelcolor='C2')
plt.quiverkey(quivVplanet,
Vplanet[0] / 2,
Vplanet[1] / 2,
2,
r'$V_\mathrm{p}$',
coordinates='data',
labelpos='E',
labelcolor='C5')
plt.quiverkey(quivVinf,
Vinfinityt[0] / 2,
Vinfinityt[1] / 2,
2,
r'$\tilde{V}_{\mathrm{in}}$',
coordinates='data',
labelpos='W',
labelcolor='C4')
plt.quiverkey(quivVinfStar,
VinfinitytStar[0] / 3,
VinfinitytStar[1] / 3,
2,
r'$\tilde{V}_{\mathrm{out}}$',
coordinates='data',
labelpos='E',
labelcolor='C1')
ax.arrow(Vinfinityt[0],
Vinfinityt[1],
DeltaV[0],
DeltaV[1],
length_includes_head=True,
head_length=0,
zorder=3)
ax.arrow(0,
0,
Vinfinityt[0] + DeltaV[0] / 2,
Vinfinityt[1] + DeltaV[1] / 2,
length_includes_head=True,
head_length=0,
zorder=3)
factor = 1.1
plt.xlim(
factor * np.amin([
0, V2[0], V4[0], Vinfinityt[0], VinfinitytStar[0], DeltaV[0],
Vplanet[0]
]),
factor * np.amax([
0, V2[0], V4[0], Vinfinityt[0], VinfinitytStar[0], DeltaV[0],
Vplanet[0]
]))
plt.ylim(
1000 + factor * np.amin([
0, V2[1], V4[1], Vinfinityt[1], VinfinitytStar[1], DeltaV[1],
Vplanet[1]
]), 1000 + factor * np.amax([
0, V2[1], V4[1], Vinfinityt[1], VinfinitytStar[1], DeltaV[1],
Vplanet[1]
]))
# constant value for the Verification section
# plt.xlim([-25e3, 8e3])
# plt.ylim([-35e3, 5e3])
ax.set_aspect('equal', 'box')
plt.ylabel(r'$v_y$ $[\mathrm{m} \mathrm{s}^{-1}]$')
plt.xlabel(r'$v_x$ $[\mathrm{m} \mathrm{s}^{-1}]$')
plt.title('Flyby @ ' + planetName + '\n' + str(mjd) + ' mjd')
plt.grid(zorder=0)
plt.tight_layout()
if save == True:
checkFolder(os.path.join(folder))
plt.savefig(os.path.join(folder,
'flyby2D-' + planetName + str(mjd) + '.pdf'),
dpi=300)
plt.savefig(os.path.join(folder,
'flyby2D-' + planetName + str(mjd) + '.png'),
dpi=300)
if show:
plt.show()
else:
plt.show(block=False)
def main(cargs):
# folder from where dude is called
cfolder = os.getcwd()
# parse command line
(options, cargs) = parser.parse_args(cargs)
# check if a command has been given
if cargs == []:
parser.print_help()
sys.exit()
# create requires no Dudefile, so we deal with it right here
if cargs[0] == "create":
if len(cargs) < 2:
expgen.create()
else:
expgen.create(cargs[1])
sys.exit(0)
# all other commands require a Dudefile, so we first load it (in "cfg")
cfg = None
# use a given dudefile in options
if options.expfile != None:
try:
cfg = imp.load_source('', options.expfile)
except IOError:
print >> sys.stderr, 'ERROR: Loading', options.expfile, 'failed'
parser.print_help()
sys.exit(1)
else: # try default file names
current = os.getcwd()
max_folder = 10 # arbitrary number of parent directories
i = 0
while i < max_folder:
for f in ['desc.py', 'dudefile', 'Dudefile', 'dudefile.py']:
try:
if os.path.exists(f) and i > 0:
print "Opening Dudefile: ", os.path.abspath(f)
cfg = imp.load_source('', f)
break
except IOError:
pass
if cfg != None:
break
else:
i += 1
parent, last = os.path.split(current)
os.chdir(parent)
current = parent
if cfg == None:
print >> sys.stderr, 'ERROR: no dudefile found'
parser.print_help()
sys.exit(1)
# add to actual folder as root in cfg
cfg.root = os.getcwd()
# check if cfg can be used for core functions
core.check_cfg(cfg)
# check if cfg can be used for summaries
summary.check_cfg(cfg)
# parse arguments to module
if options.margs:
margs = args.parse(";".join(options.margs))
print "Passing arguments:", margs
args.set_args(cfg, margs)
if hasattr(cfg, 'dude_version') and cfg.dude_version >= 3:
dimensions.update(cfg)
# collect filters
filters = []
if options.filter and options.filter != []:
for fi in options.filter:
for f in fi.split(','):
filters.append(cfg.filters[f])
if options.filter_inline and options.filter_inline != []:
filters += filt.create_inline_filter(cfg, options.filter_inline)
if options.filter_path:
current = os.getcwd()
if current != cfolder:
# this assumes Dudefile is in the root of the experiment folder
os.chdir(cfolder)
path = os.path.abspath(options.filter_path)
os.chdir(current)
path = os.path.relpath(path) # get raw_output_dir/exp_... format
else:
path = options.filter_path
filters += filt.filter_path(cfg, path)
# get experiments
experiments = core.get_experiments(cfg)
# select the set of experiments to be considered (successful,
# failed or pending)
if (options.success and options.failed and options.pending) or\
not (options.success or options.failed or options.pending):
pass
else:
failed, pending = core.get_failed_pending_exp(cfg, experiments)
expin = []
expout = []
if options.failed:
expin += failed
else:
expout += failed
if options.pending:
expin += pending
else:
expout += pending
if options.success:
experiments = [exp for exp in experiments if exp not in expout]
else:
experiments = expin
# apply filters
if filters != []:
experiments = filt.filter_experiments(cfg, filters, options.invert,
False, experiments)
cmd = cargs[0]
if cmd == 'run':
if options.force:
clean.clean_experiments(cfg, experiments)
execute.run(cfg, experiments, options)
elif cmd == 'run-once':
assert len(experiments) == 1
optpt = experiments[0]
folder = "once"
utils.checkFolder(folder) # create if necessary
if options.force:
clean.clean_experiment(folder)
execute.execute_isolated(cfg, optpt, folder, options.show_output)
elif cmd == 'sum':
summary.summarize(cfg, experiments, cargs[1:], options.backend,
options.ignore_status)
elif cmd == 'list':
for experiment in experiments:
if options.dict:
print "experiment:", experiment
else:
print core.get_folder(cfg, experiment)
elif cmd == 'failed':
failed = core.get_failed(cfg, experiments, False)
for ffile in failed:
print ffile
elif cmd == 'missing':
failed = core.get_failed(cfg, experiments, True)
for exp in failed:
print exp
elif cmd == 'clean':
if options.invalid:
clean.clean_invalid_experiments(cfg, experiments)
else:
# TODO if no filter applied, ask if that's really what the
# user wants.
r = 'y'
if options.filter == None and \
options.filter_inline == None:
print "sure to wanna delete everything? [y/N]"
r = utils.getch() #raw_input("Skip, quit, or continue?
#[s/q/c]")
if r == 'y':
clean.clean_experiments(cfg, experiments)
elif cmd == 'visit':
if len(cargs) < 2:
print "Specify a bash command after visit"
sys.exit(1)
elif len(cargs) > 2:
print "Surround multi-term bash commands with \"\"."
print "e.g., \"%s\"" % ' '.join(cargs[1:])
sys.exit(1)
visit.visit_cmd_experiments(cfg, experiments, cargs[1])
elif cmd == 'info':
info.show_info(cfg, experiments)
elif cmd == 'status':
info.print_status(cfg, experiments)
else:
print >> sys.stderr, "ERROR: wrong command. %s" % cargs[0]
parser.print_help()
def ml_save(name,
predic,
activity,
ratio,
internal,
string,
info=None,
toDo=None):
""" Saves results of ML function
:param name: [description]
:type name: [type]
:param predic: [description]
:type predic: [type]
:param activity: [description]
:type activity: [type]
:param ratio: [description]
:type ratio: [type]
:param string: [description]
:type string: [type]
:param info: [description], defaults to None
:type info: [type], optional
:param toDo: [description], defaults to None
:type toDo: [type], optional
"""
### Setup File Names ###
folder = utils.checkFolder("data/" + name + time.strftime("%m%d"))
info_name = "info"
info_path = folder + info_name
info_path = utils.testTheName(info_path, "txt")
predic_name = "prediction"
pred_path = folder + predic_name
pred_path = utils.testTheName(pred_path, "npy")
act_name = "activity"
act_path = folder + act_name
act_path = utils.testTheName(act_path, "npy")
rat_name = "ratio"
rat_path = folder + rat_name
rat_path = utils.testTheName(rat_path, "npy")
int_name = "internal"
int_path = folder + int_name
int_path = utils.testTheName(int_path, "npy")
### Save Array ###
np.save(pred_path, predic)
np.save(act_path, activity)
np.save(rat_path, ratio)
np.save(int_path, internal)
### Get System Info ###
if not info:
info = numParam()
if not toDo:
toDo = doParam()[1]
paramX = ["doThresh"]
for param in paramX:
if param in toDo:
info[param] = toDo[param]
### Save Comment and System Info
f = open(info_path, "w")
f.write(string)
f.write("\n")
for key, val in info.items():
f.write(key + ", ")
f.write(str(val) + "\n")
f.close()
def get_raw_folder(cfg):
"""Gets the raw folder of the experiments. Create it if necessary"""
# get raw output folder
folder = cfg.raw_output_dir
utils.checkFolder(folder)
return folder
def do_test():
device = "cuda:0"
#set device
sr_destination_base = 'Z:/SuperResolution/Labeled_Tiled_Datasets_Fix/BSDS100\Scale_3/'
test_base_200 = 'Z:/SuperResolution/Labeled_Tiled_Datasets_Fix/BSDS200\Scale_3/'
model_type = 'Skip_SRCNN'
out_base = 'Z:\SuperResolution\Outputs\\' + model_type + '\\'
model_base = 'Z:\SuperResolution\\Models\\' + model_type + '\\'
checkFolder(out_base)
checkFolder(model_base)
#Hyper Params
#===================
epochs = 400
batch_size = 16
workers = 6
#==================
sr_dataset = ImageLabelDataset(sr_destination_base,
transform=transforms.ToTensor(),
resize=False)
sr_dataloader = DataLoader(sr_dataset,
batch_size=batch_size,
shuffle=True,
num_workers=workers,
drop_last=True)
test_dataset_200 = ImageLabelDataset(test_base_200,
transform=transforms.ToTensor(),
resize=False)
test_dataloader_200 = DataLoader(test_dataset_200,
batch_size=batch_size,
shuffle=False,
num_workers=workers,
drop_last=True)
#==================
#model = Basic_DRC(n_channels = 3)
model = SRCNN(num_channels=3, do_skip=True)
model = model.to(device)
mse = nn.MSELoss()
#SGD optimizer where each layer has their own weights
# opt = torch.optim.SGD(params=[{'params': model.conv1.parameters(), 'lr': 10e-4},
# {'params': model.conv2.parameters(), 'lr': 10e-4},
# {'params': model.conv3.parameters(), 'lr': 10e-5}],
# momentum=0.9)
opt = torch.optim.Adam(params=[{
'params': model.conv1.parameters(),
'lr': 1e-4
}, {
'params': model.conv2.parameters(),
'lr': 1e-4
}, {
'params': model.conv3.parameters(),
'lr': 1e-5
}])
#==================
train_loss_list = []
test_loss_list = []
test_psnr_list = []
test_ssim_list = []
avg_loss = 0
epoch_imgs = []
for e in range(epochs):
print("Train Epoch: " + str(e))
for i, sample in tqdm(enumerate(sr_dataloader, 0),
total=len(sr_dataloader)):
model.train()
x = sample['input'].to(device)
y = sample['label'].to(device)
opt.zero_grad()
out = model(x)
loss = mse(out, y).to(device)
avg_loss += loss.item()
loss.backward()
opt.step()
epoch_imgs.append(out[0].permute(1, 2, 0).detach().cpu().numpy())
epoch_train_loss = avg_loss / len(sr_dataloader)
train_loss_list.append(epoch_train_loss)
print("Train Loss: " + str(epoch_train_loss))
avg_test_loss = 0
avg_psnr = 0
avg_ssim = 0
avg_loss = 0
force_test = False
if e % 10 == 0 or force_test:
print("Testing Epoch: " + str(e))
for i, sample in tqdm(enumerate(test_dataloader_200, 0),
total=len(test_dataloader_200)):
model.eval()
x = sample['input'].to(device)
y = sample['label'].to(device)
opt.zero_grad()
out = model(x)
if out.dtype != y.dtype:
print("Dtype mixmatch")
if out.shape != y.shape:
print("shape mismatch")
loss = mse(out, y)
avg_test_loss += loss.item()
avg_ssim += ssim(y.permute(0, 2, 3, 1).detach().cpu().numpy(),
out.permute(0, 2, 3,
1).detach().cpu().numpy(),
multichannel=True)
avg_psnr += psnr(y.detach().cpu().numpy(),
out.detach().cpu().numpy())
if i == 50:
t_o = out[0].permute(1, 2, 0).detach().cpu().numpy()
t_y = y[0].permute(1, 2, 0).detach().cpu().numpy()
t_x = x[0].permute(1, 2, 0).detach().cpu().numpy()
epoch_test_loss = avg_test_loss / len(test_dataloader_200)
avg_ssim /= len(test_dataloader_200)
avg_psnr /= len(test_dataloader_200)
test_loss_list.append(epoch_test_loss)
test_psnr_list.append(avg_psnr)
test_ssim_list.append(avg_ssim)
print("Test Loss: " + str(epoch_test_loss))
print("Avg SSIM: " + str(avg_ssim))
print("Avg PSNR: " + str(avg_psnr))
avg_test_loss = 0
fig, ax = plt.subplots(3)
ax[0].imshow(t_y)
ax[1].imshow(t_x)
ax[2].imshow(t_o)
nb_out = len(os.listdir(out_base))
fig.savefig(out_base + str(nb_out) + '.png', dpi=800)
plotLosses(train_loss_list, test_loss_list, test_psnr_list,
test_ssim_list)
torch.save(model.state_dict(), model_base + model_type + '.pt')
def do_test():
device = "cuda:0"
#set device
sr_destination_base = 'Z:/SuperResolution/Labeled_Tiled_Datasets_Fix/BSDS200\Scale_3/'
test_base_200 = 'Z:/SuperResolution/Labeled_Tiled_Datasets_Fix/BSDS100\Scale_3/'
out_base = 'Z:\SuperResolution\Outputs\DRCNN_Baisc\\'
checkFolder(out_base)
#training online for a whole day
batch_size = 32
epochs = 500
momentum = 0.9
decay = 0.0001
workers = 4
sr_dataset = ImageLabelDataset(sr_destination_base,
transform=transforms.ToTensor(),
resize=False)
sr_dataloader = DataLoader(sr_dataset,
batch_size=batch_size,
shuffle=True,
num_workers=workers,
drop_last=True)
test_dataset_200 = ImageLabelDataset(test_base_200,
transform=transforms.ToTensor(),
resize=False)
test_dataloader_200 = DataLoader(test_dataset_200,
batch_size=batch_size,
shuffle=False,
num_workers=workers,
drop_last=True)
model = Basic_DRC(n_recursions=8, n_channels=3)
model = model.to(device)
mse = nn.MSELoss()
#SGD optimizer where each layer has their own weights
opt = torch.optim.SGD(params=[
{
'params': model.parameters(),
'lr': 0.01
},
],
momentum=momentum,
weight_decay=decay)
sched = lr_scheduler.ReduceLROnPlateau(opt,
'min',
factor=0.001,
patience=5,
min_lr=10e-6)
avg_loss = 0
avg_test_loss = 0
train_loss_list = []
test_loss_list = []
test_psnr_list = []
test_ssim_list = []
for e in range(epochs):
#print("Train Epoch: " + str(e))
for i, sample in tqdm(enumerate(sr_dataloader, 0),
total=len(sr_dataloader)):
model.train()
x = sample['input'].to(device)
y = sample['label'].to(device)
opt.zero_grad()
out = model(x)
loss = mse(out, y).to(device)
avg_loss += loss.item()
loss.backward()
opt.step()
epoch_train_loss = avg_loss / len(sr_dataloader)
train_loss_list.append(epoch_train_loss)
print("Train Loss: " + str(epoch_train_loss))
avg_loss = 0
avg_psnr = 0
avg_ssim = 0
force_test = False
if e % 10 == 0 or force_test:
with torch.no_grad():
print("Testing Epoch: " + str(e))
for i, sample in tqdm(enumerate(test_dataloader_200, 0),
total=len(test_dataloader_200)):
model.eval()
x = sample['input'].to(device)
y = sample['label'].to(device)
opt.zero_grad()
out = model(x)
test_loss = mse(out, y).to(device)
sched.step(test_loss)
if out.dtype != y.dtype:
print("Dtype mixmatch")
if out.shape != y.shape:
print("shape mismatch")
avg_test_loss += test_loss.item()
avg_ssim += ssim(y.permute(0, 2, 3,
1).detach().cpu().numpy(),
out.permute(0, 2, 3,
1).detach().cpu().numpy(),
multichannel=True)
avg_psnr += psnr(y.detach().cpu().numpy(),
out.detach().cpu().numpy())
if i == 50:
t_o = out[0].permute(1, 2, 0).detach().cpu().numpy()
t_y = y[0].permute(1, 2, 0).detach().cpu().numpy()
t_x = x[0].permute(1, 2, 0).detach().cpu().numpy()
epoch_test_loss = avg_test_loss / len(test_dataloader_200)
avg_ssim /= len(test_dataloader_200)
avg_psnr /= len(test_dataloader_200)
test_loss_list.append(epoch_test_loss)
test_psnr_list.append(avg_psnr)
test_ssim_list.append(avg_ssim)
print("Test Loss: " + str(epoch_test_loss))
print("Avg SSIM: " + str(avg_ssim))
print("Avg PSNR: " + str(avg_psnr))
avg_test_loss = 0
fig, ax = plt.subplots(3)
ax[0].imshow(t_y)
ax[1].imshow(t_x)
ax[2].imshow(t_o)
nb_out = len(os.listdir(out_base))
fig.savefig(out_base + str(nb_out) + '.png', dpi=800)
fig_l, ax_l = plt.subplots(4)
ax_l[0].plot(train_loss_list, color='blue')
ax_l[0].set_title("Train Loss")
ax_l[1].plot(test_loss_list, color='red')
ax_l[1].set_title("Test Loss")
ax_l[2].plot(test_psnr_list)
ax_l[2].set_title("Test Avg PSNR")
ax_l[3].plot(test_ssim_list)
ax_l[3].set_title("Test Avg SSIM")
fig_l.tight_layout()
fig_l.savefig(out_base + "test_metrics" + '.png', dpi=800)
if __name__ == "__main__":
if '--debug' in sys.argv or par.maintaining:
api.add_resource(gitPull,'/autodeploy/git')
logging.basicConfig(level=logging.DEBUG, format='[%(levelname)s] %(message)s')
else:
logging.basicConfig(level=logging.INFO , format='[%(levelname)s] %(message)s')
formatter=logging.Formatter('%(asctime)s [%(levelname)s] %(message)s',"%Y-%m-%d %H:%M:%S")
#fh = logging.FileHandler('./log/{:%Y-%m-%d}_error.log'.format(datetime.now()))
import logging.handlers as lh
fh = lh.TimedRotatingFileHandler("./log/error",'midnight',1)
fh.suffix="%Y-%m-%d.log"
fh.setLevel(logging.ERROR)
fh.setFormatter(formatter)
logging.getLogger('').addHandler(fh)
dbCleaningOnLaunch()
checkFolder()
algoChecker()
scheduler = BackgroundScheduler()
scheduler.add_job(purge, 'cron',day_of_week='0-6', hour=1, minute=27)
scheduler.start()
if '--port' not in sys.argv:
port=par.port
else:
port=int(sys.argv[sys.argv.index('--port')+1])
logging.info(f'InCore running at port {par.port}')
app.run(debug='--debug' in sys.argv or par.maintaining,port=port,host='0.0.0.0')
# █████▒█ ██ ▄████▄ ██ ▄█▀ ██████╗ ██╗ ██╗ ██████╗
# ▓██ ▒ ██ ▓██▒▒██▀ ▀█ ██▄█▒ ██╔══██╗██║ ██║██╔════╝
# ▒████ ░▓██ ▒██░▒▓█ ▄ ▓███▄░ ██████╔╝██║ ██║██║ ███╗
