def download_track(track): download_dir = settings.getSetting('folder') downloaded, path, folder = getTrackPath(download_dir, track, codec) if not downloaded: checkFolder(folder) info = get_track_download_info(track, codec, high_res) log("download: %s, %s" % (info.codec, info.bitrate_in_kbps)) info.download(fixPath(path)) if codec == "mp3": audio = mp3.MP3(path, ID3=easyid3.EasyID3) audio["title"] = track.title audio["length"] = str(track.duration_ms) if track.artists: audio["artist"] = track.artists[0].name if track.albums: audio["album"] = track.albums[0].title audio["tracknumber"] = str( track.albums[0].track_position.index) audio["date"] = str(track.albums[0].year) audio["genre"] = track.albums[0].genre audio.save() elif codec == "aac": log("TODO: add tags to aac files") # notify("Download", "Done: %s" % path, 1) return path
def get_folder(cfg, experiment, check=False): """Returns the experiment folder. Creates it if necessary.""" folder = get_raw_folder(cfg) utils.checkFolder(folder) # add experiment subfolder folder = os.path.join(folder, get_name('exp', experiment)) if check: utils.checkFolder(folder) return folder
def drw_small_overview(resOT, plt_deviation=0, alt=0, jEE_first=0): def prepareData(resOT): res = np.mean(resOT, axis=0) res = np.mean(res, axis=0)[2:] y_errors = np.std(resOT, axis=(0, 1), dtype=np.float64)[2:] return res, y_errors, def Other_data(resOT): res = np.mean(resOT, axis=0) y_errors = np.std(resOT, axis=(0), dtype=np.float64) print(y_errors) return res, y_errors ### Acutal Function Body ### if len(resOT.shape) > 2: res, y_errors = prepareData(resOT) else: res, y_errors = Other_data(resOT) fig = plt.figure() ax = fig.add_subplot(111) x_ax = np.arange(len(res)) ax.errorbar(x_ax, res, yerr=y_errors, capsize=8, linestyle='--') plt.fill_between(x_ax, res + y_errors, res - y_errors, alpha=0.3) ax.set_xlabel('Temporal Distance of Prediction') ax.set_ylabel('Prediction Accuracy') folder = "../figs" name = "OneMemory" folder = utils.checkFolder(folder) fullname = utils.testTheName(folder + name, "pdf") plt.savefig(fullname) utils.plotMessage(fullname) plt.show() plt.close(fig)
def one_vs_zero(): data_array = np.load("ml3_data.npy", allow_pickle=True) for data in data_array[0]: print(data) naughts = np.array([row for row in data_array if row[0][0] < 0.05]) fulls = np.array([row for row in data_array if row[0][0] > 0.2]) naughts_mean = np.mean(naughts, axis=0) naughts_std = np.std(naughts, axis=0) fulls_mean = np.mean(fulls, axis=0) fulls_std = np.std(fulls, axis=0) add_base = np.array([1, 1])[:, None] print(add_base.shape) print(naughts_mean.shape) # naughts_mean = np.concatenate((add_base,naughts_mean),axis=1) naughts_mean = np.hstack((add_base, naughts_mean)) naughts_std = np.hstack((add_base * 0, naughts_std)) fulls_mean = np.concatenate((add_base * 0, fulls_mean), axis=1) fulls_std = np.concatenate((add_base * 0, fulls_std), axis=1) all_mean = np.concatenate((naughts_mean, fulls_mean), axis=0) all_std = np.concatenate((naughts_std, fulls_std), axis=0) # all_mean = all_mean[:,1:] # all_std = all_std[:,1:] # all_mean = np.concatenate((naughts_mean, fulls_mean), axis=0) print(all_mean.shape) x_ax = np.arange(all_mean.shape[1]) description = [ "fully activated network, excitatory", "fully activated network, inhibitory", "dead network, excitatory", "dead network, inhibitory", ] cols = ['b', 'r', 'b', 'r'] stil = ['-', '-', '--', '--'] fig = plt.figure(tight_layout=True) ax = fig.add_subplot(111) for i in range(4): plt.errorbar(x_ax, all_mean[i], all_std[i], capsize=4, label=description[i], c=cols[i], linestyle=stil[i]) plt.legend() ax.set_xlabel('Timesteps since start') ax.set_ylabel('Share of all Neurons') # ax.set_xticks(np.linspace(0,dur-1,5)) folder = "../figs" name = "one_or_zero" folder = utils.checkFolder(folder) fullname = utils.testTheName(folder + name, "pdf") plt.savefig(fullname) utils.plotMessage(fullname) plt.show() plt.close(fig)
def hamming_analyse(): diff = np.load("hamming3_control.npy")[0][2] control = np.load("hamming3_control.npy")[1][2] diff = np.delete(diff, 3, 0) # diff = np.load("hamming3.npy")[0][:,:21] # control = np.load("hamming3.npy")[1][:,:21] # diff = np.delete(diff,1, 0) diff_mean = np.mean(diff, axis=0) # idx = np.argmin(np.mean(diff,axis=1)) # diff_mean = diff[idx] diff_std = np.std(diff, axis=0) control_mean = np.mean(control, axis=0) control_std = np.std(control, axis=0) fig = plt.figure() ax = fig.add_subplot(111) dur = diff_mean.shape[0] time = np.arange(dur) # ax.plot(diff_mean) diff_txt = "One flipped" control_txt = "Random Pairings" ax.errorbar(time, diff_mean, yerr=diff_std, label=diff_txt) ax.errorbar(time, control_mean, yerr=control_std, label=control_txt) ax.set_xlabel('Timesteps since start') ax.set_ylabel('Share of all Neurons') ax.set_xticks(np.linspace(0, dur - 1, 5)) ax.legend() folder = "../figs" name = "hamming" title = "Hamming Distance" plt.title(title) folder = utils.checkFolder(folder) fullname = utils.testTheName(folder + name, "pdf") plt.savefig(fullname) utils.plotMessage(fullname) plt.show() plt.close(fig)
def drw_overview_delay_1(resOT): def prepareData(resOT): res = np.mean(resOT, 0) idx = np.arange(len(res[0, :])) idx[0], idx[1] = 1, 0 res = res[:, idx] y_errors = np.std(resOT, 0, dtype=np.float64) y_errors = y_errors[:, idx] dist = len(res[0]) - 2 return res, y_errors, dist ### style of plotted vars ### # print(col) def prepareColor(res): for i in range(len(res)): if res[i][0] != res[i + 1][0]: length_line = i + 1 amount_lines = int(len(res) / length_line) break cmap = mpl.cm.plasma col = cmap(np.linspace(0, .9, amount_lines)) return col, length_line ### plot all lines ### def actualPlot_mean(ax, res, y_errors, length_line): cleg = [] amount_of_lines = range(0, len(res), length_line) print(list(amount_of_lines)) delay = 1 j = delay for i in amount_of_lines: plo = np.transpose(res[i:i + length_line]) pl_err = np.transpose(y_errors[i:i + length_line]) if ax: ax.errorbar( plo[1], plo[j + 2], yerr=pl_err[j + 2], capsize=8, c=col[int(i / length_line)], ) else: plt.errorbar( plo[1], plo[j + 2], yerr=pl_err[j + 2], capsize=8, c=col[int(i / length_line)], ) cleg.append(r"$E_E$" + ": %3.2f" % (plo[0, 0])) return cleg def make_leg(fig, cleg): leg_elem = [] for i in range(len(cleg)): line = mpl.lines.Line2D([0], [0], c=col[i], label=cleg[i]) leg_elem.append(line) fig.legend( bbox_to_anchor=(.97, .97), # fontsize='small', handles=leg_elem, loc="upper right") def make_labels(): plt.ylabel('true prediction rate') plt.xlabel(r"$J_{EE}$") ### Acutal Function Body ### res, y_errors, dist = prepareData(resOT) col, length_line = prepareColor(res) fig = plt.figure(tight_layout=True, figsize=(8, 6)) cleg = actualPlot_mean( None, res, y_errors, length_line, ) make_leg(fig, cleg) make_labels() folder = "../figs" name = "delayOne" folder = utils.checkFolder(folder) fullname = utils.testTheName(folder + name, "pdf") plt.savefig(fullname) utils.plotMessage(fullname) plt.show() plt.close(fig)
def drw_overviewOT(resOT, y_name, save_name): plt_deviation = 0, alt = 0, jEE_first = 0 def prepareData(resOT): res = np.mean(resOT, 0) idx = np.arange(len(res[0, :])) idx[0], idx[1] = 1, 0 res = res[:, idx] y_errors = np.std(resOT, 0, dtype=np.float64) y_errors = y_errors[:, idx] dist = len(res[0]) - 2 return res, y_errors, dist ### style of plotted vars ### # print(col) def prepareColor(res): for i in range(len(res)): if res[i][0] != res[i + 1][0]: length_line = i + 1 amount_lines = int(len(res) / length_line) break cmap = mpl.cm.plasma col = cmap(np.linspace(0, .9, amount_lines)) return col, length_line ### plot all lines ### def actualPlot_mean(ax, res, y_errors, length_line, lstyle, jEE_first): cleg = [] lleg = [] amount_of_lines = range(0, len(res), length_line) print(list(amount_of_lines)) for j in range(dist): # for j in range(3): lleg.append("delay: %d" % (j)) for i in amount_of_lines: # for i in [0, 24, 60]: plo = np.transpose(res[i:i + length_line]) pl_err = np.transpose(y_errors[i:i + length_line]) if ax: ax.errorbar(plo[1], plo[j + 2], yerr=pl_err[j + 2], capsize=8, c=col[int(i / length_line)], linestyle=lstyle[j]) else: plt.errorbar(plo[1], plo[j + 2], yerr=pl_err[j + 2], capsize=8, c=col[int(i / length_line)], linestyle=lstyle[j]) if j == 0: # print(plo[0]) if (jEE_first): cleg.append("J_EE: %3.2f" % (plo[0, 0])) else: cleg.append(r"$E_E$" + ": %3.2f" % (plo[0, 0])) return cleg, lleg def actualPlot_std(ax, res, y_errors, length_line, lstyle, jEE_first): cleg = [] lleg = [] amount_of_lines = range(0, len(res), length_line) for j in range(dist): lleg.append("delay: %d" % (j)) for i in amount_of_lines: plo = np.transpose(res[i:i + length_line]) pl_err = np.transpose(y_errors[i:i + length_line]) if ax: ax.plot(plo[1], pl_err[j + 2], c=col[int(i / length_line)], linestyle=lstyle[j]) else: plt.plot(plo[1], plo[j + 2], c=col[int(i / length_line)], linestyle=lstyle[j]) if j == 0: # print(plo[0]) if (jEE_first): cleg.append("J_EE: %3.2f" % (plo[0, 0])) else: cleg.append(r"$E_E$" + ": %3.2f" % (plo[0, 0])) return cleg, lleg def actualPlot_simp(ax, res, length_line, lstyle, jEE_first): cleg = [] lleg = [] amount_of_lines = range(0, len(res), length_line) for j in range(dist): lleg.append("delay: %d" % (j)) for i in amount_of_lines: plo = np.transpose(res[i:i + length_line]) plt.plot(plo[1], plo[j + 2], c=col[int(i / length_line)], linestyle=lstyle[j]) if j == 0: # print(plo[0]) if (jEE_first): cleg.append("J_EE: %3.2f" % (plo[0, 0])) else: cleg.append(r"$E_E$" + ": %3.2f" % (plo[0, 0])) return cleg, lleg ### color predictive capabilities ### # plt.axhspan(0.9, 1, facecolor='red', alpha=0.3) ### Custom Legend for less lines on display ### def make_leg(fig, cleg, lleg, lstyle): leg_elem = [] for i in range(len(cleg)): line = mpl.lines.Line2D([0], [0], c=col[i], label=cleg[i]) leg_elem.append(line) fig.legend( bbox_to_anchor=(.98, .36), # fontsize='small', handles=leg_elem, # loc="lower right" ) def make_labels(jEE_first): plt.ylabel(y_name) if (jEE_first): plt.xlabel(r'$E_E$"+"') else: plt.xlabel(r"$J_{EE}$") ### Acutal Function Body ### res, y_errors, dist = prepareData(resOT) col, length_line = prepareColor(res) lstyle = np.hstack((np.array(['-', '-.', '--', ':']), np.array(linestyle_tuple)[:, 1])) if alt: fig = plt.figure(tight_layout=True, figsize=(8, 6)) cleg, lleg = actualPlot_std(None, res, y_errors, length_line, lstyle, jEE_first) make_leg(fig, cleg, lleg, lstyle) make_labels(jEE_first) else: if plt_deviation: fig3 = plt.figure(constrained_layout=True) ax1 = fig3.add_subplot(211) ax2 = fig3.add_subplot(212) cleg, lleg = actualPlot_mean(ax1, res, y_errors, length_line, lstyle, jEE_first) cleg, lleg = actualPlot_std(ax2, res, y_errors, length_line, lstyle, jEE_first) make_leg(fig3, cleg, lleg, lstyle) else: fig = plt.figure(tight_layout=True, figsize=(8, 6)) cleg, lleg = actualPlot_mean(None, res, y_errors, length_line, lstyle, jEE_first) make_leg(fig, cleg, lleg, lstyle) make_labels(jEE_first) folder = "../figs" name = save_name folder = utils.checkFolder(folder) fullname = utils.testTheName(folder + name, "pdf") plt.savefig(fullname) utils.plotMessage(fullname) plt.show() plt.close(fig)
def flybyVectors3D(V2, V4, Vinfinityt, VinfinitytStar, Vplanet, planetName, mjd, folder='output/flybyExamples', save=True, show=True): DeltaV = V4 - V2 fig = plt.figure() ax = plt.axes(projection='3d') quivV2 = plt.quiver( 0, 0, 0, V2[0], V2[1], V2[2], # angles='xy', scale_units='xy', # scale=1, arrow_length_ratio=0.05, color='C0', zorder=3) quivV4 = plt.quiver( 0, 0, 0, V4[0], V4[1], V4[2], # angles='xy', scale_units='xy', scale=1, arrow_length_ratio=0.05, color='C3', zorder=3) quivVplanet = plt.quiver( 0, 0, 0, Vplanet[0], Vplanet[1], Vplanet[2], # angles='xy', scale_units='xy', scale=1, arrow_length_ratio=0.05, color='C5', zorder=3) quivDeltaV = plt.quiver( V2[0], V2[1], V2[2], DeltaV[0], DeltaV[1], DeltaV[2], # angles='xy', scale_units='xy', scale=1, arrow_length_ratio=0.5, color='C2', zorder=3) quivVinf = plt.quiver( 0, 0, 0, Vinfinityt[0], Vinfinityt[1], Vinfinityt[2], # angles='xy', scale_units='xy', scale=1, arrow_length_ratio=0.5, color='C4', zorder=3) quivVinfStar = plt.quiver( 0, 0, 0, VinfinitytStar[0], VinfinitytStar[1], VinfinitytStar[2], # angles='xy', scale_units='xy', scale=1, arrow_length_ratio=0.5, color='C1', zorder=3) with warnings.catch_warnings(): warnings.simplefilter(action='ignore', category=FutureWarning) ax.text(V2[0] / 2, V2[1] / 2, V2[2] / 2, r'$V_2$', V2, color='C0') ax.text(V4[0] / 2, V4[1] / 2, V4[2] / 2, r'$V_4$', V4, color='C3') ax.text(V2[0] + DeltaV[0] / 2, V2[1] + DeltaV[1] / 2, V2[2] + DeltaV[2] / 2, r'$\Delta V$', V2, color='C2') ax.text(Vplanet[0] / 2, Vplanet[1] / 2, Vplanet[2] / 2, r'$V_{planet}$', Vplanet, color='C5') ax.text(Vinfinityt[0] / 2, Vinfinityt[1] / 2, Vinfinityt[2] / 2, r'$V_{\infty t}$', Vinfinityt, color='C4') ax.text(VinfinitytStar[0] / 2, VinfinitytStar[1] / 2, VinfinitytStar[2] / 2, r'$V_{\infty t}^\star$', VinfinitytStar, color='C1') axisEqual3D(ax) ax.set_ylabel('v_y') ax.set_xlabel('v_x') ax.set_zlabel('v_z') plt.title('Flyby @ ' + planetName + '\n' + str(mjd) + ' mjd') plt.grid(zorder=0) if save == True: checkFolder(os.path.join(folder)) plt.savefig(os.path.join(folder, 'flyby-' + planetName + str(mjd) + '.pdf'), dpi=300) plt.savefig(os.path.join(folder, 'flyby-' + planetName + str(mjd) + '.png'), dpi=300) if show: plt.show() else: plt.show(block=False)
def flybyVectors(V2, V4, Vinfinityt, VinfinitytStar, Vplanet, planetName, mjd, folder='output/flybyExamples', save=True, show=True): DeltaV = V4 - V2 fig = newFigure(height=6.2 / 2, half=True, target='thesis') ax = plt.gca() quivV2 = plt.quiver(0, 0, V2[0], V2[1], angles='xy', scale_units='xy', scale=1, color='C0', zorder=3) quivV4 = plt.quiver(0, 0, V4[0], V4[1], angles='xy', scale_units='xy', scale=1, color='C3', zorder=3) quivVplanet = plt.quiver(0, 0, Vplanet[0], Vplanet[1], angles='xy', scale_units='xy', scale=1, color='C5', zorder=3) quivDeltaV = plt.quiver(V2[0], V2[1], DeltaV[0], DeltaV[1], angles='xy', scale_units='xy', scale=1, color='C2', zorder=3) quivVinf = plt.quiver(0, 0, Vinfinityt[0], Vinfinityt[1], angles='xy', scale_units='xy', scale=1, color='C4', zorder=3) quivVinfStar = plt.quiver(0, 0, VinfinitytStar[0], VinfinitytStar[1], angles='xy', scale_units='xy', scale=1, color='C1', zorder=3) plt.quiverkey(quivV2, V2[0] / 2, V2[1] / 2, 2, r'$V_{\mathrm{in}}$', coordinates='data', labelpos='W', labelcolor='C0') plt.quiverkey(quivV4, V4[0] / 3, V4[1] / 3, 2, r'$V_{\mathrm{out}}$', coordinates='data', labelpos='E', labelcolor='C3') plt.quiverkey(quivDeltaV, V2[0] + DeltaV[0] / 2, V2[1] + DeltaV[1] / 2, 2, r'$\Delta V$', coordinates='data', labelpos='W', labelcolor='C2') plt.quiverkey(quivVplanet, Vplanet[0] / 2, Vplanet[1] / 2, 2, r'$V_\mathrm{p}$', coordinates='data', labelpos='E', labelcolor='C5') plt.quiverkey(quivVinf, Vinfinityt[0] / 2, Vinfinityt[1] / 2, 2, r'$\tilde{V}_{\mathrm{in}}$', coordinates='data', labelpos='W', labelcolor='C4') plt.quiverkey(quivVinfStar, VinfinitytStar[0] / 3, VinfinitytStar[1] / 3, 2, r'$\tilde{V}_{\mathrm{out}}$', coordinates='data', labelpos='E', labelcolor='C1') ax.arrow(Vinfinityt[0], Vinfinityt[1], DeltaV[0], DeltaV[1], length_includes_head=True, head_length=0, zorder=3) ax.arrow(0, 0, Vinfinityt[0] + DeltaV[0] / 2, Vinfinityt[1] + DeltaV[1] / 2, length_includes_head=True, head_length=0, zorder=3) factor = 1.1 plt.xlim( factor * np.amin([ 0, V2[0], V4[0], Vinfinityt[0], VinfinitytStar[0], DeltaV[0], Vplanet[0] ]), factor * np.amax([ 0, V2[0], V4[0], Vinfinityt[0], VinfinitytStar[0], DeltaV[0], Vplanet[0] ])) plt.ylim( 1000 + factor * np.amin([ 0, V2[1], V4[1], Vinfinityt[1], VinfinitytStar[1], DeltaV[1], Vplanet[1] ]), 1000 + factor * np.amax([ 0, V2[1], V4[1], Vinfinityt[1], VinfinitytStar[1], DeltaV[1], Vplanet[1] ])) # constant value for the Verification section # plt.xlim([-25e3, 8e3]) # plt.ylim([-35e3, 5e3]) ax.set_aspect('equal', 'box') plt.ylabel(r'$v_y$ $[\mathrm{m} \mathrm{s}^{-1}]$') plt.xlabel(r'$v_x$ $[\mathrm{m} \mathrm{s}^{-1}]$') plt.title('Flyby @ ' + planetName + '\n' + str(mjd) + ' mjd') plt.grid(zorder=0) plt.tight_layout() if save == True: checkFolder(os.path.join(folder)) plt.savefig(os.path.join(folder, 'flyby2D-' + planetName + str(mjd) + '.pdf'), dpi=300) plt.savefig(os.path.join(folder, 'flyby2D-' + planetName + str(mjd) + '.png'), dpi=300) if show: plt.show() else: plt.show(block=False)
def main(cargs): # folder from where dude is called cfolder = os.getcwd() # parse command line (options, cargs) = parser.parse_args(cargs) # check if a command has been given if cargs == []: parser.print_help() sys.exit() # create requires no Dudefile, so we deal with it right here if cargs[0] == "create": if len(cargs) < 2: expgen.create() else: expgen.create(cargs[1]) sys.exit(0) # all other commands require a Dudefile, so we first load it (in "cfg") cfg = None # use a given dudefile in options if options.expfile != None: try: cfg = imp.load_source('', options.expfile) except IOError: print >> sys.stderr, 'ERROR: Loading', options.expfile, 'failed' parser.print_help() sys.exit(1) else: # try default file names current = os.getcwd() max_folder = 10 # arbitrary number of parent directories i = 0 while i < max_folder: for f in ['desc.py', 'dudefile', 'Dudefile', 'dudefile.py']: try: if os.path.exists(f) and i > 0: print "Opening Dudefile: ", os.path.abspath(f) cfg = imp.load_source('', f) break except IOError: pass if cfg != None: break else: i += 1 parent, last = os.path.split(current) os.chdir(parent) current = parent if cfg == None: print >> sys.stderr, 'ERROR: no dudefile found' parser.print_help() sys.exit(1) # add to actual folder as root in cfg cfg.root = os.getcwd() # check if cfg can be used for core functions core.check_cfg(cfg) # check if cfg can be used for summaries summary.check_cfg(cfg) # parse arguments to module if options.margs: margs = args.parse(";".join(options.margs)) print "Passing arguments:", margs args.set_args(cfg, margs) if hasattr(cfg, 'dude_version') and cfg.dude_version >= 3: dimensions.update(cfg) # collect filters filters = [] if options.filter and options.filter != []: for fi in options.filter: for f in fi.split(','): filters.append(cfg.filters[f]) if options.filter_inline and options.filter_inline != []: filters += filt.create_inline_filter(cfg, options.filter_inline) if options.filter_path: current = os.getcwd() if current != cfolder: # this assumes Dudefile is in the root of the experiment folder os.chdir(cfolder) path = os.path.abspath(options.filter_path) os.chdir(current) path = os.path.relpath(path) # get raw_output_dir/exp_... format else: path = options.filter_path filters += filt.filter_path(cfg, path) # get experiments experiments = core.get_experiments(cfg) # select the set of experiments to be considered (successful, # failed or pending) if (options.success and options.failed and options.pending) or\ not (options.success or options.failed or options.pending): pass else: failed, pending = core.get_failed_pending_exp(cfg, experiments) expin = [] expout = [] if options.failed: expin += failed else: expout += failed if options.pending: expin += pending else: expout += pending if options.success: experiments = [exp for exp in experiments if exp not in expout] else: experiments = expin # apply filters if filters != []: experiments = filt.filter_experiments(cfg, filters, options.invert, False, experiments) cmd = cargs[0] if cmd == 'run': if options.force: clean.clean_experiments(cfg, experiments) execute.run(cfg, experiments, options) elif cmd == 'run-once': assert len(experiments) == 1 optpt = experiments[0] folder = "once" utils.checkFolder(folder) # create if necessary if options.force: clean.clean_experiment(folder) execute.execute_isolated(cfg, optpt, folder, options.show_output) elif cmd == 'sum': summary.summarize(cfg, experiments, cargs[1:], options.backend, options.ignore_status) elif cmd == 'list': for experiment in experiments: if options.dict: print "experiment:", experiment else: print core.get_folder(cfg, experiment) elif cmd == 'failed': failed = core.get_failed(cfg, experiments, False) for ffile in failed: print ffile elif cmd == 'missing': failed = core.get_failed(cfg, experiments, True) for exp in failed: print exp elif cmd == 'clean': if options.invalid: clean.clean_invalid_experiments(cfg, experiments) else: # TODO if no filter applied, ask if that's really what the # user wants. r = 'y' if options.filter == None and \ options.filter_inline == None: print "sure to wanna delete everything? [y/N]" r = utils.getch() #raw_input("Skip, quit, or continue? #[s/q/c]") if r == 'y': clean.clean_experiments(cfg, experiments) elif cmd == 'visit': if len(cargs) < 2: print "Specify a bash command after visit" sys.exit(1) elif len(cargs) > 2: print "Surround multi-term bash commands with \"\"." print "e.g., \"%s\"" % ' '.join(cargs[1:]) sys.exit(1) visit.visit_cmd_experiments(cfg, experiments, cargs[1]) elif cmd == 'info': info.show_info(cfg, experiments) elif cmd == 'status': info.print_status(cfg, experiments) else: print >> sys.stderr, "ERROR: wrong command. %s" % cargs[0] parser.print_help()
def ml_save(name, predic, activity, ratio, internal, string, info=None, toDo=None): """ Saves results of ML function :param name: [description] :type name: [type] :param predic: [description] :type predic: [type] :param activity: [description] :type activity: [type] :param ratio: [description] :type ratio: [type] :param string: [description] :type string: [type] :param info: [description], defaults to None :type info: [type], optional :param toDo: [description], defaults to None :type toDo: [type], optional """ ### Setup File Names ### folder = utils.checkFolder("data/" + name + time.strftime("%m%d")) info_name = "info" info_path = folder + info_name info_path = utils.testTheName(info_path, "txt") predic_name = "prediction" pred_path = folder + predic_name pred_path = utils.testTheName(pred_path, "npy") act_name = "activity" act_path = folder + act_name act_path = utils.testTheName(act_path, "npy") rat_name = "ratio" rat_path = folder + rat_name rat_path = utils.testTheName(rat_path, "npy") int_name = "internal" int_path = folder + int_name int_path = utils.testTheName(int_path, "npy") ### Save Array ### np.save(pred_path, predic) np.save(act_path, activity) np.save(rat_path, ratio) np.save(int_path, internal) ### Get System Info ### if not info: info = numParam() if not toDo: toDo = doParam()[1] paramX = ["doThresh"] for param in paramX: if param in toDo: info[param] = toDo[param] ### Save Comment and System Info f = open(info_path, "w") f.write(string) f.write("\n") for key, val in info.items(): f.write(key + ", ") f.write(str(val) + "\n") f.close()
def get_raw_folder(cfg): """Gets the raw folder of the experiments. Create it if necessary""" # get raw output folder folder = cfg.raw_output_dir utils.checkFolder(folder) return folder
def do_test(): device = "cuda:0" #set device sr_destination_base = 'Z:/SuperResolution/Labeled_Tiled_Datasets_Fix/BSDS100\Scale_3/' test_base_200 = 'Z:/SuperResolution/Labeled_Tiled_Datasets_Fix/BSDS200\Scale_3/' model_type = 'Skip_SRCNN' out_base = 'Z:\SuperResolution\Outputs\\' + model_type + '\\' model_base = 'Z:\SuperResolution\\Models\\' + model_type + '\\' checkFolder(out_base) checkFolder(model_base) #Hyper Params #=================== epochs = 400 batch_size = 16 workers = 6 #================== sr_dataset = ImageLabelDataset(sr_destination_base, transform=transforms.ToTensor(), resize=False) sr_dataloader = DataLoader(sr_dataset, batch_size=batch_size, shuffle=True, num_workers=workers, drop_last=True) test_dataset_200 = ImageLabelDataset(test_base_200, transform=transforms.ToTensor(), resize=False) test_dataloader_200 = DataLoader(test_dataset_200, batch_size=batch_size, shuffle=False, num_workers=workers, drop_last=True) #================== #model = Basic_DRC(n_channels = 3) model = SRCNN(num_channels=3, do_skip=True) model = model.to(device) mse = nn.MSELoss() #SGD optimizer where each layer has their own weights # opt = torch.optim.SGD(params=[{'params': model.conv1.parameters(), 'lr': 10e-4}, # {'params': model.conv2.parameters(), 'lr': 10e-4}, # {'params': model.conv3.parameters(), 'lr': 10e-5}], # momentum=0.9) opt = torch.optim.Adam(params=[{ 'params': model.conv1.parameters(), 'lr': 1e-4 }, { 'params': model.conv2.parameters(), 'lr': 1e-4 }, { 'params': model.conv3.parameters(), 'lr': 1e-5 }]) #================== train_loss_list = [] test_loss_list = [] test_psnr_list = [] test_ssim_list = [] avg_loss = 0 epoch_imgs = [] for e in range(epochs): print("Train Epoch: " + str(e)) for i, sample in tqdm(enumerate(sr_dataloader, 0), total=len(sr_dataloader)): model.train() x = sample['input'].to(device) y = sample['label'].to(device) opt.zero_grad() out = model(x) loss = mse(out, y).to(device) avg_loss += loss.item() loss.backward() opt.step() epoch_imgs.append(out[0].permute(1, 2, 0).detach().cpu().numpy()) epoch_train_loss = avg_loss / len(sr_dataloader) train_loss_list.append(epoch_train_loss) print("Train Loss: " + str(epoch_train_loss)) avg_test_loss = 0 avg_psnr = 0 avg_ssim = 0 avg_loss = 0 force_test = False if e % 10 == 0 or force_test: print("Testing Epoch: " + str(e)) for i, sample in tqdm(enumerate(test_dataloader_200, 0), total=len(test_dataloader_200)): model.eval() x = sample['input'].to(device) y = sample['label'].to(device) opt.zero_grad() out = model(x) if out.dtype != y.dtype: print("Dtype mixmatch") if out.shape != y.shape: print("shape mismatch") loss = mse(out, y) avg_test_loss += loss.item() avg_ssim += ssim(y.permute(0, 2, 3, 1).detach().cpu().numpy(), out.permute(0, 2, 3, 1).detach().cpu().numpy(), multichannel=True) avg_psnr += psnr(y.detach().cpu().numpy(), out.detach().cpu().numpy()) if i == 50: t_o = out[0].permute(1, 2, 0).detach().cpu().numpy() t_y = y[0].permute(1, 2, 0).detach().cpu().numpy() t_x = x[0].permute(1, 2, 0).detach().cpu().numpy() epoch_test_loss = avg_test_loss / len(test_dataloader_200) avg_ssim /= len(test_dataloader_200) avg_psnr /= len(test_dataloader_200) test_loss_list.append(epoch_test_loss) test_psnr_list.append(avg_psnr) test_ssim_list.append(avg_ssim) print("Test Loss: " + str(epoch_test_loss)) print("Avg SSIM: " + str(avg_ssim)) print("Avg PSNR: " + str(avg_psnr)) avg_test_loss = 0 fig, ax = plt.subplots(3) ax[0].imshow(t_y) ax[1].imshow(t_x) ax[2].imshow(t_o) nb_out = len(os.listdir(out_base)) fig.savefig(out_base + str(nb_out) + '.png', dpi=800) plotLosses(train_loss_list, test_loss_list, test_psnr_list, test_ssim_list) torch.save(model.state_dict(), model_base + model_type + '.pt')
def do_test(): device = "cuda:0" #set device sr_destination_base = 'Z:/SuperResolution/Labeled_Tiled_Datasets_Fix/BSDS200\Scale_3/' test_base_200 = 'Z:/SuperResolution/Labeled_Tiled_Datasets_Fix/BSDS100\Scale_3/' out_base = 'Z:\SuperResolution\Outputs\DRCNN_Baisc\\' checkFolder(out_base) #training online for a whole day batch_size = 32 epochs = 500 momentum = 0.9 decay = 0.0001 workers = 4 sr_dataset = ImageLabelDataset(sr_destination_base, transform=transforms.ToTensor(), resize=False) sr_dataloader = DataLoader(sr_dataset, batch_size=batch_size, shuffle=True, num_workers=workers, drop_last=True) test_dataset_200 = ImageLabelDataset(test_base_200, transform=transforms.ToTensor(), resize=False) test_dataloader_200 = DataLoader(test_dataset_200, batch_size=batch_size, shuffle=False, num_workers=workers, drop_last=True) model = Basic_DRC(n_recursions=8, n_channels=3) model = model.to(device) mse = nn.MSELoss() #SGD optimizer where each layer has their own weights opt = torch.optim.SGD(params=[ { 'params': model.parameters(), 'lr': 0.01 }, ], momentum=momentum, weight_decay=decay) sched = lr_scheduler.ReduceLROnPlateau(opt, 'min', factor=0.001, patience=5, min_lr=10e-6) avg_loss = 0 avg_test_loss = 0 train_loss_list = [] test_loss_list = [] test_psnr_list = [] test_ssim_list = [] for e in range(epochs): #print("Train Epoch: " + str(e)) for i, sample in tqdm(enumerate(sr_dataloader, 0), total=len(sr_dataloader)): model.train() x = sample['input'].to(device) y = sample['label'].to(device) opt.zero_grad() out = model(x) loss = mse(out, y).to(device) avg_loss += loss.item() loss.backward() opt.step() epoch_train_loss = avg_loss / len(sr_dataloader) train_loss_list.append(epoch_train_loss) print("Train Loss: " + str(epoch_train_loss)) avg_loss = 0 avg_psnr = 0 avg_ssim = 0 force_test = False if e % 10 == 0 or force_test: with torch.no_grad(): print("Testing Epoch: " + str(e)) for i, sample in tqdm(enumerate(test_dataloader_200, 0), total=len(test_dataloader_200)): model.eval() x = sample['input'].to(device) y = sample['label'].to(device) opt.zero_grad() out = model(x) test_loss = mse(out, y).to(device) sched.step(test_loss) if out.dtype != y.dtype: print("Dtype mixmatch") if out.shape != y.shape: print("shape mismatch") avg_test_loss += test_loss.item() avg_ssim += ssim(y.permute(0, 2, 3, 1).detach().cpu().numpy(), out.permute(0, 2, 3, 1).detach().cpu().numpy(), multichannel=True) avg_psnr += psnr(y.detach().cpu().numpy(), out.detach().cpu().numpy()) if i == 50: t_o = out[0].permute(1, 2, 0).detach().cpu().numpy() t_y = y[0].permute(1, 2, 0).detach().cpu().numpy() t_x = x[0].permute(1, 2, 0).detach().cpu().numpy() epoch_test_loss = avg_test_loss / len(test_dataloader_200) avg_ssim /= len(test_dataloader_200) avg_psnr /= len(test_dataloader_200) test_loss_list.append(epoch_test_loss) test_psnr_list.append(avg_psnr) test_ssim_list.append(avg_ssim) print("Test Loss: " + str(epoch_test_loss)) print("Avg SSIM: " + str(avg_ssim)) print("Avg PSNR: " + str(avg_psnr)) avg_test_loss = 0 fig, ax = plt.subplots(3) ax[0].imshow(t_y) ax[1].imshow(t_x) ax[2].imshow(t_o) nb_out = len(os.listdir(out_base)) fig.savefig(out_base + str(nb_out) + '.png', dpi=800) fig_l, ax_l = plt.subplots(4) ax_l[0].plot(train_loss_list, color='blue') ax_l[0].set_title("Train Loss") ax_l[1].plot(test_loss_list, color='red') ax_l[1].set_title("Test Loss") ax_l[2].plot(test_psnr_list) ax_l[2].set_title("Test Avg PSNR") ax_l[3].plot(test_ssim_list) ax_l[3].set_title("Test Avg SSIM") fig_l.tight_layout() fig_l.savefig(out_base + "test_metrics" + '.png', dpi=800)
if __name__ == "__main__": if '--debug' in sys.argv or par.maintaining: api.add_resource(gitPull,'/autodeploy/git') logging.basicConfig(level=logging.DEBUG, format='[%(levelname)s] %(message)s') else: logging.basicConfig(level=logging.INFO , format='[%(levelname)s] %(message)s') formatter=logging.Formatter('%(asctime)s [%(levelname)s] %(message)s',"%Y-%m-%d %H:%M:%S") #fh = logging.FileHandler('./log/{:%Y-%m-%d}_error.log'.format(datetime.now())) import logging.handlers as lh fh = lh.TimedRotatingFileHandler("./log/error",'midnight',1) fh.suffix="%Y-%m-%d.log" fh.setLevel(logging.ERROR) fh.setFormatter(formatter) logging.getLogger('').addHandler(fh) dbCleaningOnLaunch() checkFolder() algoChecker() scheduler = BackgroundScheduler() scheduler.add_job(purge, 'cron',day_of_week='0-6', hour=1, minute=27) scheduler.start() if '--port' not in sys.argv: port=par.port else: port=int(sys.argv[sys.argv.index('--port')+1]) logging.info(f'InCore running at port {par.port}') app.run(debug='--debug' in sys.argv or par.maintaining,port=port,host='0.0.0.0') # █████▒█ ██ ▄████▄ ██ ▄█▀ ██████╗ ██╗ ██╗ ██████╗ # ▓██ ▒ ██ ▓██▒▒██▀ ▀█ ██▄█▒ ██╔══██╗██║ ██║██╔════╝ # ▒████ ░▓██ ▒██░▒▓█ ▄ ▓███▄░ ██████╔╝██║ ██║██║ ███╗