def __init__(self, z_piezo, atto, lockin, daq, cap_input, low_temp = False): self.touchdown = False self.V_to_C = 2530e3 # 2530 pF/V * (1e3 fF/pF), calibrated 20160423 by BTS, see ipython notebook self.attosteps = 100 #number of attocube steps between sweeps self.numpts = 10 # number of points to sweep at a time self.z_piezo = z_piezo self.atto = atto self.lockin = lockin self.daq = daq self.lockin.ch1_daq_input = cap_input # isLowTemp = input('Testing at low temperature? [y/(n)]') # if isLowTemp in ('y', 'Y'): # self.low_temp = True # else: # self.low_temp = False self.low_temp = low_temp self.configure_attocube() self.configure_lockin() self.configure_piezo() self.Vpiezo_full = [] self.cap_full = [] self.rsquared = [] self.fig = plt.figure() self.ax = plt.gca() display.clear_output()
def observer(change): kwargs = {k:v.value for k,v in controls.items()} show_inline_matplotlib_plots() with out: clear_output(wait=True) f(**kwargs) show_inline_matplotlib_plots()
def deepdream( net, base_img, iter_n=10, octave_n=4, octave_scale=1.4, end="inception_4c/output", clip=True, **step_params ): # prepare base images for all octaves octaves = [preprocess(net, base_img)] for i in xrange(octave_n - 1): octaves.append(nd.zoom(octaves[-1], (1, 1.0 / octave_scale, 1.0 / octave_scale), order=1)) src = net.blobs["data"] detail = np.zeros_like(octaves[-1]) # allocate image for network-produced details for octave, octave_base in enumerate(octaves[::-1]): h, w = octave_base.shape[-2:] if octave > 0: # upscale details from the previous octave h1, w1 = detail.shape[-2:] detail = nd.zoom(detail, (1, 1.0 * h / h1, 1.0 * w / w1), order=1) src.reshape(1, 3, h, w) # resize the network's input image size src.data[0] = octave_base + detail for i in xrange(iter_n): make_step(net, end=end, clip=clip, **step_params) # visualization vis = deprocess(net, src.data[0]) if not clip: # adjust image contrast if clipping is disabled vis = vis * (255.0 / np.percentile(vis, 99.98)) showarray(vis) print octave, i, end, vis.shape clear_output(wait=True) # extract details produced on the current octave detail = src.data[0] - octave_base # returning the resulting image return deprocess(net, src.data[0])
def data_set_load_handler(widget): clear_output() pyplot.close("all") dir = frame.get_attribute("address_bar", "value") data_set_count = frame.get_state_data("data_set_count") name = frame.get_attribute("data_set_name", "value") if name == "": name = "Data set - " + "%03d" % (data_set_count + 1, ) data = ppm.yprofile(dir) if data.files != []: frame.set_state("data_loaded") old_cycs = frame.get_state_data("cycles") cycs = data.cycles cycs = list(set(cycs) | set(old_cycs)) cycs.sort() frame.set_state_data("cycles", cycs) frame.set_state_data("yaxis_options", data.dcols, "plot_prof_time") frame.set_state_data("yaxis_options", data.dcols + data.cattrs, "get") frame.set_state_attribute("cycle", min=cycs[0], max=cycs[-1], value=cycs[-1]) frame.set_state_attribute("cycle_range", min=cycs[0], max=cycs[-1], value=(cycs[0], cycs[-1])) add_data_set(data, name) frame.update()
def update(self, *args): """ Call the interact function and update the output widget with the result of the function call. Parameters ---------- *args : ignored Required for this method to be used as traitlets callback. """ self.kwargs = {} if self.manual: self.manual_button.disabled = True try: show_inline_matplotlib_plots() with self.out: if self.clear_output: clear_output(wait=True) for widget in self.kwargs_widgets: value = widget.get_interact_value() self.kwargs[widget._kwarg] = value self.result = self.f(**self.kwargs) show_inline_matplotlib_plots() if self.auto_display and self.result is not None: display(self.result) except Exception as e: ip = get_ipython() if ip is None: self.log.warn("Exception in interact callback: %s", e, exc_info=True) else: ip.showtraceback() finally: if self.manual: self.manual_button.disabled = False
def get_from_nersc(user, relative_path): """Load a remote data file from NERSC to an H5 file Parameters ---------- user : str NERSC user account relative_path : str Path to file from "/project/projectdirs/metatlas/original_data/<user>/" """ import pexpect from IPython.display import clear_output cmd = 'scp -o StrictHostKeyChecking=no ' path = "/project/projectdirs/metatlas/original_data/%s/%s" path = path % (user, relative_path) cmd += '%[email protected]:%s . && echo "Download Complete"' cmd = cmd % (user, path) print(cmd) proc = pexpect.spawn(cmd) proc.expect("assword:*") if sys.version.startswith('3'): passwd = input() else: passwd = raw_input() clear_output() proc.send(passwd) proc.send('\r') proc.expect('Download Complete') proc.close() return os.path.abspath(os.path.basename(relative_path))
def print_processing_status(current_line, tot_line, s_time, frequency=None, pre_message=None): """ A function that prints to screen the processing status of another function. Current_line is the line being processed right now, tot_line the total number of lines to be processed, s_time the time the computation started, frequency how often the print message should be printed (if None, 1/100 of tot_line), pre_message the string that should be printed pefore the status message. """ if frequency is None: frequency = int(tot_line / 100) message = 'Processing line {0:.1f}{4} (of {1:.1f}{4}). \t Elapsed time: {2:.0f} secs, \t ETA: {3:.0f} secs.' if current_line == 1: print('Processing the first line...') elif current_line > 1: if current_line % frequency == 0: unit, etodivide = get_magnitude(tot_line) loc_time = time.time() - s_time clear_output(wait=True) if pre_message is not None: print(pre_message) print(message.format(current_line / (10 ** etodivide), tot_line / (10 ** etodivide), loc_time, (tot_line / current_line - 1) * loc_time, unit)) sys.stdout.flush()
def get_and_show(): env.host = '180.150.190.52:6' env.user = "******" while True: clear_output() remote_filename = REMOTE_PATH+'/'+'ocr_20150902.log' #local_filename = LOCAL_PATH +'_logs/'+'audit_logSessionServer.txt' + '.'+env.host remote_ret = run(""" tail -n10 %s | grep -E 'Downloading http|Return result' """ % remote_filename) lines = remote_ret.split('\n'); for line in lines: items = line.split() #print items if len(items)>=13 and items[11]=="Downloading": process_id = items[10] img_url = items[12] #print img_url if (process_id in records) and records[process_id] != img_url: print process_id,":", "Exception! does't complete it work." display(Image(url=records[process_id])) records.pop(process_id); records[process_id] = img_url continue if len(items)>=15 and items[12]=="result:": process_id = items[10][:-1] ocr_result = " ".join(items[14:-1]) if (process_id in records): print process_id,":", ocr_result display(Image(url=records[process_id])) records.pop(process_id); continue print "processing:", records print "b" time.sleep(2)
def cv_fit_model(model, X_train, y_train, nb_epoch=2, weights=False): """ Fit RNN model with ... :param model: :param X_train: :param y_train: :param nb_epoch: :param weights: :return: """ if weights: sample_weights = init_weights(y_train) else: sample_weights = np.ones_like(y_train) for i in range(nb_epoch): print "Epoch " + str(i) + '/' + str(nb_epoch) model.fit(X_train, y_train, batch_size=1024, nb_epoch=1, shuffle=False, sample_weight=sample_weights ) clear_output(wait=True) return model
def bowtie(self, line=''): """Build and serve a Bowtie app.""" opts, appvar = self.parse_options(line, 'w:h:b:p:') width = opts.get('w', 1500) height = opts.get('h', 1000) border = opts.get('b', 2) port = opts.get('p', 9991) host = '0.0.0.0' sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) result = sock.connect_ex((host, port)) if result == 0: raise Exception(f'Port {port} is unavailable on host {host}, aborting.') global_ns = self.shell.user_global_ns local_ns = self.shell.user_ns try: # pylint: disable=eval-used app = eval(appvar, global_ns, local_ns) except NameError: raise UsageError(f'Could not find App {appvar}') if not isinstance(app, App): raise UsageError(f'App is of type {type(app)} needs to be type <bowtie.App>') app._build(notebook=get_notebook_name()) # pylint: disable=protected-access self.process = Process(target=app._serve) # pylint: disable=protected-access self.process.start() time.sleep(5) clear_output() display(HTML( f'<iframe src=http://localhost:9991 width={width} height={height} ' f'frameBorder={border}></iframe>' ))
def totalGain(widget): print 'totalGain' global dbname global lists print 'dbname',dbname con = sqlite3.connect(dbname) query = "SELECT * FROM sqlite_master WHERE type='table'" global df df = pd.io.sql.read_frame(query,con) # print df.tail() tablelen = len(df['name']) clear_output() gainsum = 0.0 count = 0 for cnt in range(tablelen): if df['name'][cnt].find('signal') != -1: # print df['name'][cnt] title = df['name'][cnt].split('_') # print 'title',title,title[2] con1 = sqlite3.connect(dbname) global tablename tablename = df['name'][cnt] tabledf = pd.io.sql.read_frame("SELECT * from "+tablename, con1) if tabledf[-1:]['totalGain'] > 0.0: # print title[2],float(tabledf[-1:]['totalGain'].values) # gainsum += float(tabledf[-1:]['totalGain'].values) if float(tabledf[-1:]['totalGain'].values) >= 0.3: count += 1 print title[2],float(tabledf[-1:]['totalGain'].values) gainsum += float(tabledf[-1:]['totalGain'].values) print gainsum,count
def EvaluateGenomeList_Parallel(genome_list, evaluator, cores=8, display=True, ipython_client=None): ''' If ipython_client is None, will use concurrent.futures. Pass an instance of Client() in order to use an IPython cluster ''' fitnesses = [] curtime = time.time() if ipython_client is None or not ipython_installed: with ProcessPoolExecutor(max_workers=cores) as executor: for i, fitness in enumerate(executor.map(evaluator, genome_list)): fitnesses += [fitness] if display: if ipython_installed: clear_output(wait=True) print('Individuals: (%s/%s) Fitness: %3.4f' % (i, len(genome_list), fitness)) else: if type(ipython_client) == Client: lbview = ipython_client.load_balanced_view() amr = lbview.map(evaluator, genome_list, ordered=True, block=False) for i, fitness in enumerate(amr): if display: if ipython_installed: clear_output(wait=True) print('Individual:', i, 'Fitness:', fitness) fitnesses.append(fitness) else: raise ValueError('Please provide valid IPython.parallel Client() as ipython_client') elapsed = time.time() - curtime if display: print('seconds elapsed: %3.4f' % elapsed) return fitnesses
def EvaluateGenomeList_Serial(genome_list, evaluator, display=True, show_elapsed=False): fitnesses = [] count = 0 if display and show_elapsed: curtime = time.time() if display and pbar_installed: pbar = ProgressBar() pbar.max_value = len(genome_list) pbar.min_value = 0 for i,g in enumerate(genome_list): f = evaluator(g) fitnesses.append(f) if display: if not pbar_installed: if ipython_installed: clear_output(wait=True) print('Individuals: (%s/%s) Fitness: %3.4f' % (count, len(genome_list), f)) else: pbar.update(i) count += 1 if display and pbar_installed: pbar.finish() if display and show_elapsed: elapsed = time.time() - curtime print('seconds elapsed: %s' % elapsed) return fitnesses
def update_perceptron_plot(h, f, ax, x_plus, x_minus, i, w, b, x_select): """Update plots after decision boundary has changed.""" # Helper function for updating plots h["select"].set_xdata(x_select[0]) h["select"].set_ydata(x_select[1]) # Re-plot the hyper plane plot_limits = {} plot_limits["x"] = np.asarray(ax[0].get_xlim()) plot_limits["y"] = np.asarray(ax[0].get_ylim()) x0, x1 = hyperplane_coordinates(w, b, plot_limits) strt = -b / w[1] h["arrow"].remove() del (h["arrow"]) h["arrow"] = ax[0].arrow(0, strt, w[0], w[1] + strt, head_width=0.2) h["plane"].set_xdata(x0) h["plane"].set_ydata(x1) h["iter"].set_text("Update " + str(i)) ax[1].cla() bins = 15 f_minus = np.dot(x_minus, w) f_plus = np.dot(x_plus, w) ax[1].hist(f_plus, bins, alpha=0.5, label="+1", color="r") ax[1].hist(f_minus, bins, alpha=0.5, label="-1", color="g") ax[1].legend(loc="upper right") display(f) clear_output(wait=True) if i < 3: time.sleep(0.5) else: time.sleep(0.25) return h
def param_prompt(self): """ Check and confirm values of parameters """ correct = False while not correct: for param in ['rate', 'Rbias', 'Rbias_mod', 'Irampspan', 'Irampstep']: print('IV', param, ':', getattr(self.IV, param)) for parammod in ['Imodspan','Imodstep']: print(parammod, ':', getattr(self, parammod)) for paramamp in ['gain','filter']: print('IV preamp', paramamp, ':', getattr(self.IV.preamp, paramamp)) if self.IV.rate > self.IV.preamp.filter[1]: print("You're filtering out your signal... fix the preamp cutoff\n") if self.IV.Irampspan > 200e-6: print("You want the SQUID biased above 100 uA?... don't kill the SQUID!\n") if self.Imodspan > 300e-6: print("You want the SQUID mod biased above 150 uA?... don't kill the SQUID!\n") try: inp = input('Are these parameters correct? Enter a command to change parameters, or press enter to continue (e.g. IV.preamp.gain = 100): ') if inp == '': correct = True else: exec('self.'+inp) self.IV.calc_ramp() self.calc_ramp() # recalculate daq output display.clear_output() except: display.clear_output() print('Invalid command\n')
def save_plot(self, filename): plt.ion() targarr = np.array(self.targvalue) self.posi[0].set_xdata(self.wt_positions[:,0]) self.posi[0].set_ydata(self.wt_positions[:,1]) while len(self.plotel)>0: self.plotel.pop(0).remove() self.plotel = self.shape_plot.plot(np.array([self.wt_positions[[i,j],0] for i, j in self.elnet_layout.keys()]).T, np.array([self.wt_positions[[i,j],1] for i, j in self.elnet_layout.keys()]).T, 'y-', linewidth=1) for i in range(len(self.posb)): self.posb[i][0].set_xdata(self.iterations) self.posb[i][0].set_ydata(targarr[:,i]) self.legend.texts[i].set_text('%s = %8.2f'%(self.targname[i], targarr[-1,i])) self.objf_plot.set_xlim([0, self.iterations[-1]]) self.objf_plot.set_ylim([0.5, 1.2]) if not self.title == '': plt.title('%s = %8.2f'%(self.title, getattr(self, self.title))) plt.draw() #print self.iterations[-1] , ': ' + ', '.join(['%s=%6.2f'%(self.targname[i], targarr[-1,i]) for i in range(len(self.targname))]) with open(self.result_file+'.results','a') as f: f.write( '%d:'%(self.inc) + ', '.join(['%s=%6.2f'%(self.targname[i], targarr[-1,i]) for i in range(len(self.targname))]) + '\n') #plt.show() #plt.savefig(filename) display(plt.gcf()) #plt.show() clear_output(wait=True)
def deepdream(net, base_imarray, iter_n=50, octave_n=4, octave_scale=1.4, end='inception_4c/output', clip=True, **step_params): octaves = [preprocess(net, base_imarray)] for i in xrange(octave_n-1): octaves.append(nd.zoom(octaves[-1], (1, 1.0/octave_scale,1.0/octave_scale), order=1)) src = net.blobs['data'] detail = np.zeros_like(octaves[-1]) for octave, octave_base in enumerate(octaves[::-1]): h, w = octave_base.shape[-2:] if octave > 0: h1, w1 = detail.shape[-2:] detail = nd.zoom(detail, (1, 1.0*h/h1,1.0*w/w1), order=1) src.reshape(1,3,h,w) src.data[0] = octave_base+detail for i in xrange(iter_n): make_step(net, end=end, clip=clip, **step_params) vis = deprocess(net, src.data[0]) if not clip: vis = vis*(255.0/np.percentile(vis, 99.98)) showarray(vis) print octave, i, end, vis.shape clear_output(wait=True) detail = src.data[0]-octave_base return deprocess(net, src.data[0])
def solveCSP(problem): statesExplored = 0 frontier = [problem] while frontier: state = frontier.pop() statesExplored += 1 if state.complete(): print 'Number of explored: ' + str(statesExplored) return state else: successors = state.getAllSuccessors() if args.debug: if not successors: print "DEADEND BACKTRACKING \n" frontier.extend(successors) if args.debug: os.system("clear") print state raw_input("Press Enter to continue...") if args.debug_ipython: from time import sleep from IPython import display display.display(display.HTML(state.prettyprinthtml())) display.clear_output(True) sleep(0.5) return None
def ntraintest(self, x, y, clear=True, scoresign=-1, **params): """ fit until stopping criteria met and retain best model """ xtrain, xtest, ytrain, ytest = train_test_split(x, y, test_size=0.1, random_state=0) params.setdefault("verbose", 1) # keras verbose is 0 or 1 only if params["verbose"] > 0: params["verbose"] = 1 earlystopping = EarlyStopping(monitor='val_loss', patience=3, verbose=0, mode='auto') best_file = './best_weights.txt' savebestmodel = ModelCheckpoint(best_file, monitor='val_loss', verbose=0, save_best_only=True, mode='auto') keraslog = Keraslog() self.fit(xtrain, ytrain, validation_data=(xtest.toarray(), to_categorical(ytest)), callbacks=[earlystopping, savebestmodel, keraslog], **params) self.model.load_weights(best_file) losses = np.array(keraslog.losses) losses = losses * scoresign self.best_score = max(losses) best_iteration = list(losses).index(self.best_score) + 1 if clear: clear_output() log.info("best score %s with %s iterations" %(self.best_score, best_iteration+1)) return losses
def _createUrl(self, *args): pullName = self.stormStuff.loc[self.p2.groups[self.menuNames.value]] full = pullName.Filename.values[0] # Creates different path for 2014 storms if self.menuYears.value == 2014: url = "http://ftp.nhc.noaa.gov/atcf/aid_public/a%8s.dat.gz" % (full) urlb = "http://ftp.nhc.noaa.gov/atcf/btk/b%8s.dat" % (full) else: url = "http://ftp.nhc.noaa.gov/atcf/archive/%4s/a%8s.dat.gz" % (self.menuYears.value,full) urlb = "http://ftp.nhc.noaa.gov/atcf/archive/%4s/b%8s.dat.gz" % (self.menuYears.value,full) self.urls = {"Forecast":url, "Best":urlb} self.aExists = self.test_url(url) self.bExists = self.test_url(urlb) clear_output() # Creating message for intances in which the url test passes or fails if self.aExists and self.bExists: message = 'Selected data for: {}'.format(self.menuNames.value) self.plottingButton.disabled = False self.plottingButton.on_click(toPlottingFunction) else: self.stormSelector.visible = True self.plottingButton.disabled = True message = 'No data found for: ' if (not self.aExists) and (not self.bExists): message = message + "forecast tracks and best track" elif not self.aExists: message = message + "forecast tracks" else: message = message + "best track" print (message)
def init_display(self): clear_output() for key, w in self.widgets.iteritems(): w.visible = False self.widgets['searchTerm'].visible = True self.widgets['searchButton'].visible = True
def to_map(self): from IPython.display import clear_output clear_output() with self.app_context: session['emit'] = self.emit super(LocalGameMaster, self).to_map() places = ['%s | %s' % (p.key, p.preview) for p in self.node.places] print 'Current map:', self.node.name print 'Go to' print '\n'.join(places) flag = True while flag: user_input = raw_input() if user_input == '': break for p in self.node.places: if p.key.startswith(user_input): self.transition(p.label) flag = False break
def on_marker_change(self, _): a = self.node_marker_map_sel.marker.active.value b = self.edge_marker_map_sel.marker.active.value if a or b: clear_output() self.init_plot() return show(self.plot)
def runkMeans(X, initial_centroids, max_iters, plot_progress=False): #RUNKMEANS runs the K-Means algorithm on data matrix X, where each row of X #is a single example # Initialize values m, n = X.shape K = initial_centroids.shape[0] centroids = initial_centroids previous_centroids = centroids idx = np.zeros([m, 1]) # Run K-Means for i in range(max_iters): # For each example in X, assign it to the closest centroid idx = findClosestCentroids(X, centroids) # Optionally, plot progress here if plot_progress: plotProgresskMeans(X, centroids, previous_centroids, idx, K, i) previous_centroids = centroids display.clear_output(wait=True) display.display(plt.gcf()) sleep(1) #input('Press enter to continue') # Given the memberships, compute new centroids centroids = computeCentroids(X, idx, K) plt.close() return centroids,idx
def plot_test_acc(plot_handles): plt.legend(handles=plot_handles, loc="center right") plt.xlabel("Iterations") plt.ylabel("Test Accuracy") plt.ylim(0,1) display.display(plt.gcf()) display.clear_output(wait=True)
def onclick(category, filters): extra_values = [] if extras is not None: extra_values = [w.value for w in extras.children] clear_output() callback(category, filter_category(filters), *extra_values)
def redraw(figure): if 'inline' in matplotlib.get_backend(): from IPython import display display.clear_output(wait=True) display.display(figure) else: plt.draw()
def solveLocal(problem): for r in range(1): problem.randomRestart() state = problem for i in range(100000): originalConflicts = state.numConflicts() v1, v2 = state.randomSwap() state.gradientDescent(v1, v2) print '('+str(i)+')',"CONFLICTS: ",state.numConflicts() if args.debug_ipython: from time import sleep from IPython import display state.lastMoves = [s1, s2] display.display(display.HTML(state.prettyprinthtml())) display.clear_output(True) sleep(0.5) if state.numConflicts() == 0: return state break if args.debug: os.system("clear") print state raw_input("Press Enter to continue...")
def change_module(widget, value): if value == "History": frame.set_state_attribute("select_plot", states_mesa[1:], options={"":"mesa_w_data", "HR-Diagram":"hrd", "Plot":"plot", "Kippenhahn":"kippenhahn", "Kippenhahan contour":"kip_cont", "TCRhoC plot":"tcrhoc", "Get data":"get_data"}) frame.set_state_attribute("contain_model_select", states_mesa, visible=False) frame.set_attributes("contain_model_select", visible=False) elif value == "Profile": frame.set_attributes("load_data", disabled=True) frame.set_state_attribute("select_plot", states_mesa[1:], options={"":"mesa_w_data", "Plot":"plot", "Get data":"get_data"}) mass = float(frame.get_attribute("mass", "value")) Z = float(frame.get_attribute("Z", "value")) dir = frame.get_attribute("address_bar", "value") if manual_data_select: mdir, mmodel = frame.get_state_data("model_data") if (mdir != dir) or (mmodel == None): clear_output() pre_data = ms.mesa_profile(dir) mmodel = pre_data.model frame.set_state_data("model_data", (dir, mmodel)) else: mmass, mZ, mmodel = frame.get_state_data("model_data") if (mmass != mass) or (mZ != Z) or (mmodel == None): clear_output() pre_data = ms.mesa_profile(mass=mass, Z=Z) mmodel = pre_data.model frame.set_state_data("model_data", (mass, Z, mmodel)) frame.set_state_attribute("contain_model_select", states_mesa, visible=True) frame.set_attributes("contain_model_select", visible=True) frame.set_attributes("model_select", value=str(mmodel[-1])) frame.set_attributes("load_data", disabled=False) else: frame.set_state_attribute("contain_model_select", states_mesa, visible=False) frame.set_attributes("contain_model_select", visible=False)
def to_message(self): from IPython.display import clear_output, HTML, display from markdown import markdown clear_output() with self.app_context: session['emit'] = self.emit super(LocalGameMaster, self).to_message() print 'Current message:', self.node.name display(HTML(markdown(self.node.text, ['markdown.extensions.extra']))) s = '\n'.join('%s | %s' % (c.key, c.label) for c in self.node.choices) print 'Choose:\n' + s flag = True while flag: user_input = raw_input() if user_input == '': break for c in self.node.choices: if c.key == user_input or c.key == '*': self.transition(c.name) flag = False break
def plot_layout(self, button): plt.close() clear_output() self.notebook.net.draw()
def animate_solution(maze, path): # Map a color to each cell in the maze col_map = { 0: WHITE, 1: BLACK, 2: LIGHT_GREEN, -6: LIGHT_RED, -1: LIGHT_RED } # Size of the maze rows, cols = maze.shape # Create figure of the size of the maze fig = plt.figure(1, figsize=(cols, rows)) # Remove the axis ticks and add title title ax = plt.gca() ax.set_title('Policy simulation') ax.set_xticks([]) ax.set_yticks([]) # Give a color to each cell colored_maze = [[col_map[maze[j, i]] for i in range(cols)] for j in range(rows)] # Create figure of the size of the maze fig = plt.figure(1, figsize=(cols, rows)) # Create a table to color grid = plt.table(cellText=None, cellColours=colored_maze, cellLoc='center', loc=(0, 0), edges='closed') # Modify the hight and width of the cells in the table tc = grid.properties()['children'] for cell in tc: cell.set_height(1.0 / rows) cell.set_width(1.0 / cols) # Update the color at each frame for i in range(len(path)): grid.get_celld()[(path[i][0], path[i][1])].set_facecolor(LIGHT_ORANGE) grid.get_celld()[(path[i][0], path[i][1])].get_text().set_text('Player') grid.get_celld()[(path[i][2], path[i][3])].set_facecolor(LIGHT_PURPLE) grid.get_celld()[(path[i][2], path[i][3])].get_text().set_text('Minotaur') if i > 0: if maze[path[i][0]][ path[i][1]] == 2: # (, ) == (path[i-1][0], path[i-1][1]) grid.get_celld()[(path[i][0], path[i][1])].set_facecolor(LIGHT_GREEN) grid.get_celld()[( path[i][0], path[i][1])].get_text().set_text('Player is out') grid.get_celld()[(path[i - 1][2], path[i - 1][3])].set_facecolor( col_map[maze[path[i - 1][2], path[i - 1][3]]]) grid.get_celld()[(path[i - 1][2], path[i - 1][3])].get_text().set_text('') else: grid.get_celld()[(path[i - 1][2], path[i - 1][3])].set_facecolor( col_map[maze[path[i - 1][2], path[i - 1][3]]]) grid.get_celld()[(path[i - 1][2], path[i - 1][3])].get_text().set_text('') grid.get_celld()[(path[i - 1][0], path[i - 1][1])].set_facecolor( col_map[maze[path[i - 1][0], path[i - 1][1]]]) grid.get_celld()[(path[i - 1][0], path[i - 1][1])].get_text().set_text('') display.display(fig) display.clear_output(wait=True) time.sleep(1)
test_dir='Test' train_dir='Train' traincsv = pd.read_csv('myntra_train_dataset.csv') #testcsv = pd.read_csv('myntra_test.csv') traincsv.fillna(method='bfill',inplace=True) if not os.path.exists(train_dir): os.mkdir(train_dir) start = time.time() for i in tqdm(range(10001,20001)): link = traincsv.iloc[i]['Link_to_the_image'] name = (traincsv.iloc[i]['Sub_category']) full_name = str(i)+'_'+name+'.jpg' img_name = full_name full_name = os.path.join(train_dir, img_name) if not os.path.exists(full_name): try: clear_output(wait=True) urllib.request.urlretrieve(link, full_name) print(colored(img_name+' downloaded', 'green')) except: clear_output(wait=True) print(colored('Link Missing', color='red')) else: clear_output(wait=True) print(img_name,' has already been downloaded') end = time.time() print('Time taken: ', end-start)
def exitJDWeb(): runSh("pkill -9 -e -f java") clear_output(wait=True) createButton("Start", func=startJDService, style="info")
model = CONV_VAE(latent_vec) def generate_and_save_images(model, epochs, input_data): preds = model.sampling(input_data) fig = plt.figure(figsize=(4,4)) for i in range(preds.shape[0]): plt.subplot(4, 4, i+1) plt.imshow(preds[i, :, :, 0], cmap='gray') plt.axis('off') plt.savefig('generated_digits/img_at_epoch{:04d}.png'.format(epochs)) plt.show() generate_and_save_images(model, 0, rand_vec) for epoch in range(1, epochs + 1): start_time = time.time() for x in train_batch: gradient_func(model, x, optimizer_func) end_time = time.time() loss = tf.keras.metrics.Mean() for y in test_batch: loss(loss_func(model, y)) elbo = -loss.result() display.clear_output(wait=False) print('Epoch no.: {}, Test batch ELBO: {}, ' 'elapsed time for current epoch {}'.format(epoch, elbo, end_time - start_time)) generate_and_save_images(model, epochs, rand_vec)
_, loss_t = sess.run([train_step, td_loss], sample_batch(exp_replay, batch_size=64)) td_loss_history.append(loss_t) # adjust agent parameters if i % 500 == 0: # You could think that loading weights onto a target network is simply # load_weigths_into_target_network(agent, target_network) # but actually calling this function repeatedly creates a TF copy operator # again and again, which bloats memory consumption with each training step. # Instead, you should create 'copy_step' once. sess.run(copy_step) agent.epsilon = max(agent.epsilon * 0.99, 0.01) mean_rw_history.append(evaluate(make_env(), agent, n_games=3)) if i % 100 == 0: clear_output(True) print("buffer size = %i, epsilon = %.5f" % (len(exp_replay), agent.epsilon)) plt.subplot(1, 2, 1) plt.title("mean reward per game") plt.plot(mean_rw_history) plt.grid() assert not np.isnan(loss_t) plt.figure(figsize=[12, 4]) plt.subplot(1, 2, 2) plt.title("TD loss history (moving average)") plt.plot(moving_average(np.array(td_loss_history), span=100, min_periods=100)) plt.grid() plt.show()
def _selection_changed(df, selection_name): with output: clear_output(wait=True) f_interact(df, selection_name)
def train_NN(model, game_state, observe=False): cur_ep = load_obj("episode") + 1 # We will train as episode for episode in range(cur_ep, EPISODES): print( f"----------------------------EPISODE {episode} --------------------------" ) # Reset Environment # game_state._game.restart_game() # Load experince # Load the past experience that we store the previous action buffer = load_obj("exp_replay") # Declare array of action array_of_actions = np.zeros(ACTIONS) # Clear all screenshot from the prevous episode t = load_obj("time") # Every episode, clear everything out shutil.rmtree(screenshot_path) os.mkdir(screenshot_path) # Make capture screen for every state # Initial state by doing nothing image, reward, is_over, game_score = game_state.get_state( array_of_actions) print("Initial state") # Convert image into 4 stack frames s_t = np.stack((image, image, image, image), axis=2) s_t = s_t.reshape(1, s_t.shape[0], s_t.shape[1], s_t.shape[2]) initial_state = s_t # Load model weight that we have already updated from the previous round model.load_weights(object_path + "model.h5") adam = Adam(lr=LEARNING_RATE) model.compile(loss='mse', optimizer=adam) if observe: state = "Observe" OBSERVE = 999999999 # Set epsilon to very low score --> so that tend to use Q value epsilon = FINAL_EPSILON else: state = "training" # Now we start to do training mode OBSERVE = OBSERVATION # Download the previous epsilon to train more epsilon = load_obj("epsilon") done = False chick_step = 0 loss = 0 total_rewards = 0 while (done != True): loss_return = 0 array_of_actions = np.zeros(ACTIONS) if random.random() < epsilon: action_index = random.randrange(ACTIONS) q_value = np.zeros(ACTIONS) array_of_actions[action_index] = 1 result_model = "Random" else: q_value = model.predict(s_t) norm_target_next_state = np.linalg.norm(q_value) q_value = q_value / norm_target_next_state action_index = np.argmax(q_value) array_of_actions[action_index] = 1 result_model = "Max Q" if epsilon > FINAL_EPSILON and t > OBSERVE: epsilon -= (INITIAL_EPSILON - FINAL_EPSILON) / EXPLORE # Feed new image image, reward, is_over, game_score = game_state.get_state( array_of_actions) image = image.reshape(1, image.shape[0], image.shape[1], 1) # 1x80x80x1 # append the new image to input stack and remove the first one s_t_new = np.append(image, s_t[:, :, :, :3], axis=3) #1*80*80*4 # # Prepare for the next state # s_t_new = np.stack((image,image,image,image), axis=2) # s_t_new = s_t_new.reshape(1,s_t_new.shape[0],s_t_new.shape[1], s_t_new.shape[2]) # Need to save the past experince into buffer if len(buffer) > REPLAY_SIZE: buffer.popleft() # Save the past experience in the buffer then we will use it rather for training on batch buffer.append((s_t, action_index, reward, is_over, s_t_new)) if (t > OBSERVE): print("We are doing the updated weight by sampling on batch") inputs = np.zeros( (BATCH_SIZE, s_t.shape[1], s_t.shape[2], s_t.shape[3])) targets = np.zeros((inputs.shape[0], ACTIONS)) loss_return, max_q = sampling_batch(model, buffer, BATCH_SIZE, GAMMA, inputs, targets, loss) loss += loss_return loss_df.loc[len(loss_df)] = loss q_values_df.loc[len(q_values_df)] = max_q if (t % 500 == 0): print("Now we save model") model.save_weights(object_path + "model.h5", overwrite=True) save_obj(buffer, "exp_replay") #saving episodes save_obj( epsilon, "epsilon" ) #cache epsilon to avoid repeated randomness in actions loss_df.to_csv(loss_file_path, index=False) scores_df.to_csv(scores_file_path, index=False) actions_df.to_csv(actions_file_path, index=False) q_values_df.to_csv(q_value_file_path, index=False) log_df.to_csv(log_file_path, index=False) with open(object_path + "model.json", "w") as outfile: json.dump(model.to_json(), outfile) if (is_over): done = True score = game_score if (chick_step > LIMIT_STEP): done = True score = game_score chick_step += 1 total_rewards += reward if t % 20 == 0: clear_output() save_obj(t, "time") #caching time steps log_out = "TIMESTEP:" + str( t ) + " STATE:" + state + " ACTION_TYPE:" + result_model + " EPSILON:" + str( epsilon) + " ACTION:" + str( action_index) + " REWARD:" + str(reward) + " Q_MAX:" + str( np.max(q_value)) + " Loss:" + str(loss_return) print(log_out) # image_render() log_df.loc[len(log_df)] = { 'episode': episode, 'timestep': t, 'state': state, 'action_type': result_model, 'eps': epsilon, 'action': action_index, 'reward': reward, 'qmax': np.max(q_value), 'loss': loss_return } s_t = initial_state if is_over else s_t_new t += 1 print("Now we save episode") episode_df.loc[len(episode_df)] = { 'episodes': episode, 'score': score, 'steps': chick_step, 'total_loss': loss, 'average_loss': (loss / chick_step), 'total_reward': total_rewards } episode_df.to_csv(episode_file_path, index=False) save_obj(episode, "episode") print("EPISODE END -----------------------------------------") game.end()
def trainNetwork(model, game_state, observe=False): last_time = time.time() # store the previous observations in replay memory D = load_obj("D") #load from file system # get the first state by doing nothing do_nothing = np.zeros(ACTIONS) do_nothing[0] = 1 #0 => do nothing, #1=> jump x_t, r_0, terminal = game_state.get_state( do_nothing) # get next step after performing the action s_t = np.stack((x_t, x_t, x_t, x_t), axis=2) # stack 4 images to create placeholder input s_t = s_t.reshape(1, s_t.shape[0], s_t.shape[1], s_t.shape[2]) #1*20*40*4 initial_state = s_t if observe: OBSERVE = 999999999 #We keep observe, never train epsilon = FINAL_EPSILON print("Now we load weight") model.load_weights("model.h5") adam = Adam(lr=LEARNING_RATE) model.compile(loss='mse', optimizer=adam) print("Weight load successfully") else: #We go to training mode OBSERVE = OBSERVATION epsilon = load_obj("epsilon") model.load_weights("model.h5") adam = Adam(lr=LEARNING_RATE) model.compile(loss='mse', optimizer=adam) t = load_obj( "time") # resume from the previous time step stored in file system while (True): #endless running loss = 0 Q_sa = 0 action_index = 0 r_t = 0 #reward at 4 a_t = np.zeros([ACTIONS]) # action at t #choose an action epsilon greedy if t % FRAME_PER_ACTION == 0: #parameter to skip frames for actions if random.random() <= epsilon: #randomly explore an action print("----------Random Action----------") action_index = random.randrange(ACTIONS) a_t[0] = 1 else: # predict the output q = model.predict( s_t) #input a stack of 4 images, get the prediction max_Q = np.argmax(q) # chosing index with maximum q value action_index = max_Q a_t[action_index] = 1 # o=> do nothing, 1=> jump #We reduced the epsilon (exploration parameter) gradually if epsilon > FINAL_EPSILON and t > OBSERVE: epsilon -= (INITIAL_EPSILON - FINAL_EPSILON) / EXPLORE #run the selected action and observed next state and reward x_t1, r_t, terminal = game_state.get_state(a_t) print('fps: {0}'.format( 1 / (time.time() - last_time))) # helpful for measuring frame rate last_time = time.time() x_t1 = x_t1.reshape(1, x_t1.shape[0], x_t1.shape[1], 1) #1x20x40x1 s_t1 = np.append( x_t1, s_t[:, :, :, :3], axis=3 ) # append the new image to input stack and remove the first one # store the transition in D D.append((s_t, action_index, r_t, s_t1, terminal)) if len(D) > REPLAY_MEMORY: D.popleft() #only train if done observing if t > OBSERVE: #sample a minibatch to train on minibatch = random.sample(D, BATCH) inputs = np.zeros((BATCH, s_t.shape[1], s_t.shape[2], s_t.shape[3])) #32, 20, 40, 4 targets = np.zeros((inputs.shape[0], ACTIONS)) #32, 2 #Now we do the experience replay for i in range(0, len(minibatch)): state_t = minibatch[i][0] # 4D stack of images action_t = minibatch[i][1] #This is action index reward_t = minibatch[i][2] #reward at state_t due to action_t state_t1 = minibatch[i][3] #next state terminal = minibatch[i][ 4] #wheather the agent died or survided due the action inputs[i:i + 1] = state_t targets[i] = model.predict(state_t) # predicted q values Q_sa = model.predict(state_t1) #predict q values for next step if terminal: targets[ i, action_t] = reward_t # if terminated, only equals reward else: targets[i, action_t] = reward_t + GAMMA * np.max(Q_sa) loss += model.train_on_batch(inputs, targets) loss_df.loc[len(loss_df)] = loss q_values_df.loc[len(q_values_df)] = np.max(Q_sa) s_t = initial_state if terminal else s_t1 #reset game to initial frame if terminate t = t + 1 # save progress every 1000 iterations if t % 1000 == 0: print("Now we save model") game_state._game.pause() #pause game while saving to filesystem model.save_weights("model.h5", overwrite=True) save_obj(D, "D") #saving episodes save_obj(t, "time") #caching time steps save_obj(epsilon, "epsilon" ) #cache epsilon to avoid repeated randomness in actions loss_df.to_csv("./objects/loss_df.csv", index=False) scores_df.to_csv("./objects/scores_df.csv", index=False) actions_df.to_csv("./objects/actions_df.csv", index=False) q_values_df.to_csv(q_value_file_path, index=False) with open("model.json", "w") as outfile: json.dump(model.to_json(), outfile) clear_output() game_state._game.resume() # print info state = "" if t <= OBSERVE: state = "observe" elif t > OBSERVE and t <= OBSERVE + EXPLORE: state = "explore" else: state = "train" print("TIMESTEP", t, "/ STATE", state, "/ EPSILON", epsilon, "/ ACTION", action_index, "/ REWARD", r_t, "/ Q_MAX ", np.max(Q_sa), "/ Loss ", loss) print("Episode finished!") print("************************")
def plot_dynamics(self, button): plt.close() clear_output() display(self.widget) self.notebook.sim.Plot_TimeCourse(self.widget_selectInput.value, cell=self.widget_selectCell.value)
def display_board(board): clear_output() print( board[1] +'|'+board[2]+'|'+board[3]) print( board[4] +'|'+board[5]+'|'+board[6]) print( board[7] +'|'+board[8]+'|'+board[9])
def train(self, dataset, steps_per_epoch, labels, saving_gap=15, save_img=True, img_path='./generated_imgs', save_loss_batch=False, save_loss_epoch=True, loss_path='./losses'): ''' Train the model and print results over each epochs. Parameters ------------------- dataset (np.array, np.array): Image generator tuple with the images and its labels (images, labels). steps_per_epoch (int): Number of steps per epoch (total images / batch size). labels (np.array): Input labels for sample images to generate. saving_gap (int): Gaps to save checkpoints, images and losses over epochs. Default 15. save_img (bool): Whether to save images produced by 'generate_and_save_images' function for each 'saving_gap' epochs. Default True. img_path (str): Folder path to save images. Default './generated_imgs'. save_loss_batch (bool): Whether to save plots of losses over batches produced by 'summarize_epoch' function for each 'saving_gap' epochs. Default False. save_loss_epoch (bool): Whether to save the plot of total loss over epochs produced by 'plot_loss_over_epoch' function. Default True. loss_path (str): Folder path to save losses. Default './losses'. ''' # save losses over epochs losses_on_epoch = { 'g_losses': [], 'd_losses': [], 'epochs': [], 'perc_fake_true': [] } for epoch in range(self.epochs): print(f"Starting epoch {epoch+1}.") start = time.time() #save losses over batches g_losses = [] d_losses = [] num_fake_true = [] # number of classify fake image as real batch = 0 for image_batch, image_label in dataset: if batch % 50 == 0: print(f" Starting batch {batch+1}/{steps_per_epoch}") # training part g_loss, d_loss, fake_arr = self.train_step( image_batch, image_label) # get number of images that fools discriminator fake_true = np.sum(fake_arr.numpy() > 0) # append losses over batches g_losses.append(g_loss.numpy()) d_losses.append(d_loss.numpy()) num_fake_true.append(fake_true) ## batch += 1 if batch >= steps_per_epoch: break print(" Done train step.") # Produce images for the GIF as we go display.clear_output(wait=True) # save images for each epoch self.generate_and_save_images(self.generator, epoch + 1, self.seed, labels, img_path, save_img, 1) # Save the model every 15 epochs if saving_gap != 0: if ((epoch + 1) % saving_gap == 0) | ((epoch + 1) == self.epochs): self.checkpoint.save(file_prefix=self.checkpoint_prefix) # plot loss on each epoch g_loss_mean, d_loss_mean, perc_fake_true = self.summarize_epoch( epoch, d_losses, g_losses, steps_per_epoch, num_fake_true, loss_path, save_loss_batch, saving_gap) # save the losses losses_on_epoch['g_losses'].append(g_loss_mean) losses_on_epoch['d_losses'].append(d_loss_mean) losses_on_epoch['epochs'].append(epoch + 1) losses_on_epoch['perc_fake_true'].append(perc_fake_true) print('Time taken for epoch {} is {} sec.'.format( epoch + 1, time.time() - start)) # Generate after the final epoch display.clear_output(wait=True) self.generate_and_save_images( self.generator, self.epochs, self.seed, labels, img_path, True, # default save last image saving_gap) # plot loss after all epochs self.plot_loss_over_epoch(losses_on_epoch['g_losses'], losses_on_epoch['d_losses'], losses_on_epoch['epochs'], losses_on_epoch['perc_fake_true'], loss_path, save_loss_epoch) # print loss after all epochs print( 'g_loss: {} \nd_loss: {} \npercentage of classify fake image as real: {}' .format(losses_on_epoch['g_losses'][-1], losses_on_epoch['d_losses'][-1], losses_on_epoch['perc_fake_true'][-1]))
def record_motion(self, btn, gyro): clear_output(wait=True) print('데이터 수집을 시작합니다.') print('ready') time.sleep(1) print('3') time.sleep(1) print('2') time.sleep(1) p = Process(target=play_beep, args=('sound02.wav', )) p.start() print('1') time.sleep(1) # p = Process(target=play_beep, args=('sound02.wav',)) # p.run() # p.start() # p.join() # self._temp_pool.submit(self.play_beep('sound02.wav')) # play_beep('sound02.wav') print('start!') self.l = [] self.X = None self.X_df = None self.X_sc = None while True: time.sleep(0.01) self.aX = gyro.acceleration_x self.aY = gyro.acceleration_y self.aZ = gyro.acceleration_z # self.gX = gyro.angular_vel_x # self.gY = gyro.angular_vel_y # self.gZ = gyro.angular_vel_z self.roll = gyro.roll self.pitch = gyro.pitch self.yaw = gyro.yaw # self.vi = gyro.vibration self.l.append((self.aX, self.aY, self.aZ, self.roll, self.pitch, self.yaw)) # self.gX,self.gY,self.gZ, ,self.vi # print('aX, aY, aZ, gX, gY, gZ, roll, pitch, yaw, vi', self.aX, self.aY, self.aZ, # self.gX, self.gY, self.gZ, # self.roll, self.pitch, self.yaw, # self.vi) print("데이터 수집 중. 수집을 종료하려면 버튼을 누르세요.") print( '현재 자이로 센서 데이터', self.aX, self.aY, self.aZ, #self.gX, self.gY, self.gZ, self.roll, self.pitch, self.yaw) # , self.vi clear_output(wait=True) if btn.clicked: p.join() p = Process(target=play_beep, args=('sound04.wav', )) p.start() # play_beep('sound04.wav') self.X = np.array(self.l)[5:-5] if len(self.X) > 50: itr = len(self.X) // 25 self.X_tr = self.X[::itr][:25] self.X_df = pd.DataFrame( self.X_tr, columns=['aX', 'aY', 'aZ', 'roll', 'pitch', 'yaw']) self.X_df.plot() plt.show() scaler = MaxAbsScaler() scaler.fit(self.X_df) self.X_sc = DataFrame( scaler.transform(self.X_df), columns=['aX', 'aY', 'aZ', 'roll', 'pitch', 'yaw']) time.sleep(1) else: print('데이터 길이가 너무 짧습니다. 데이터가 저장되지 않았습니다.') time.sleep(1) print('조금만 천천히 그려보세요.') time.sleep(1) #raise ValueError('data is too short') # 전처리 필요 # 최대 길이에 맞출 경우, 1 처럼 모션 시간이 짧은 데이터는 결측값이 존재하게 됨. 이부분을 평균값으로 처리할지, 이전값으로 처리할지에 대한 부분 # self.X_df = pd.DataFrame(self.X_tr, columns=['aX', 'aY', 'aZ', 'gX', 'gY', 'gZ', 'roll', 'pitch', 'yaw', 'vi']) p.join() break return self.X_sc
#Problema 3 import numpy as np import time from IPython.display import clear_output pista = 100 c = [] p = [] cavalos = int(input()) for i in range(cavalos): r = input().split() r = [float(j) for j in r] c.append(r) p = [0]*len(c) while max(p) < pista: clear_output() for i in range(cavalos): p[i] += int(abs(np.random.normal(c[i][0], c[i][1]))) print("{:.0f} {}{}{}".format(i+1, '-'*p[i], ' '*(pista-p[i]), '|')) time.sleep(0.5) print("Cavalo {:.0f} Venceu!".format(p.index(max(p))+1))
def bowtie(self, line=''): """Build and serve a Bowtie app.""" opts, appvar = self.parse_options(line, 'w:h:b:p:') width = opts.get('w', 1500) height = opts.get('h', 1000) border = opts.get('b', 2) port = opts.get('p', 9991) host = '0.0.0.0' sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) result = sock.connect_ex((host, port)) if result == 0: raise Exception( 'Port {} is unavailable on host {}, aborting.'.format( port, host)) global_ns = self.shell.user_global_ns local_ns = self.shell.user_ns try: # pylint: disable=eval-used app = eval(appvar, global_ns, local_ns) except NameError: raise UsageError('Could not find App {}'.format(appvar)) if not isinstance(app, App): raise UsageError( 'App is of type {} needs to be type <bowtie.App>'.format( type(app))) # pylint: disable=protected-access app._build(notebook=get_notebook_name()) filepath = './{}/src/server.py'.format(_DIRECTORY) if os.path.isfile(filepath): self.server = Popen(['python', '-u', filepath], stdout=PIPE, stderr=STDOUT) else: raise FileNotFoundError( 'Cannot find "{}". Did you build the app?'.format(filepath)) def flush_stdout(cmd): """Flush stdout from command continuously.""" while True: line = cmd.stdout.readline() if line == b'' and cmd.poll() is not None: return cmd.poll() print(line.decode('utf-8'), end='') raise Exception() thread = Thread(target=flush_stdout, args=(self.server, )) thread.daemon = True thread.start() while self.server.poll() is None: try: if requests.get('http://localhost:9991').ok: break except requests.exceptions.RequestException: continue time.sleep(1) else: print(self.server.stdout.read().decode('utf-8'), end='') clear_output() display( HTML('<iframe src=http://localhost:9991 width={} height={} ' 'frameBorder={}></iframe>'.format(width, height, border)))
def clear_screen(): print(chr(27) + "[2J") clear_output()
def on_epoch_end(self, epoch, logs=None): clear_output(wait=True) if not away_from_computer: show_predictions() print('\nSample Prediction after epoch {}\n'.format(epoch + 1))
def gathering_motion(self, btn, gyro, filename): while True: # exit trigger if btn.double_clicked: clear_output(wait=True) print("데이터 수집을 종료합니다.") time.sleep(0.1) break #check csv file and return index of data try: df = pd.read_csv("../data/" + filename + ".csv") record_index = int((df.shape[0] + 1) / 25) except: record_index = 0 #collect data print(filename, "데이터의 ", str(record_index + 1), "번째 데이터 수집을 시작하려면 버튼을 클릭하세요. ", "데이터 수집을 종료하려면 버튼을 더블클릭하세요.", end='\r') time.sleep(0.1) if btn.clicked: clear_output(wait=True) print(str(record_index + 1), ' 번째 데이터 수집을 시작합니다.') print('ready') time.sleep(0.5) print('3') time.sleep(0.5) print('2') time.sleep(0.5) print('1') time.sleep(0.5) print('start!') clear_output(wait=True) self.l = [] self.X = None self.X_df = None while True: time.sleep(0.01) self.aX = gyro.acceleration_x self.aY = gyro.acceleration_y self.aZ = gyro.acceleration_z # self.gX = gyro.angular_vel_x # self.gY = gyro.angular_vel_y # self.gZ = gyro.angular_vel_z self.roll = gyro.roll self.pitch = gyro.pitch self.yaw = gyro.yaw # self.vi = gyro.vibration self.l.append( (self.aX, self.aY, self.aZ, self.roll, self.pitch, self.yaw)) # self.gX,self.gY,self.gZ,,self.vi print(str(record_index + 1), " 번째 데이터 수집 중. 수집을 종료하려면 버튼을 누르세요.") print( '현재 자이로 센서 데이터', self.aX, self.aY, self.aZ, #self.gX, self.gY, self.gZ, self.roll, self.pitch, self.yaw) # , self.vi clear_output(wait=True) if btn.clicked: dg = DataFrame(self.l) #dg.plot() #plt.show() #time.sleep(3) #Normalizer().fit(self.l) #dg_n = DataFrame(Normalizer().transform(self.l)) #dg_n.plot() #plt.show() #time.sleep(3) self.X = np.array(self.l)[5:-5] #print(self.X) if len(self.X) > 50: itr = len(self.X) // 25 self.X_tr = self.X[::itr][:25] self.X_df = pd.DataFrame( self.X_tr, #columns=['aX', 'aY', 'aZ', 'gX', 'gY', 'gZ', 'roll', 'pitch', 'yaw', 'vi']) columns=[ 'aX', 'aY', 'aZ', 'roll', 'pitch', 'yaw' ]) self.X_df.plot() plt.show() scaler = MaxAbsScaler() scaler.fit(self.X_df) self.X_sc = DataFrame(scaler.transform(self.X_df), columns=[ 'aX', 'aY', 'aZ', 'roll', 'pitch', 'yaw' ]) #self.X_sc.plot() #plt.show() time.sleep(1) else: print('데이터 길이가 너무 짧습니다. 데이터가 저장되지 않았습니다.') time.sleep(1) print('조금만 천천히 그려보세요.') time.sleep(1) break #raise ValueError('data is too short') # 전처리 필요 # 최대 길이에 맞출 경우, 1 처럼 모션 시간이 짧은 데이터는 결측값이 존재하게 됨. 이부분을 평균값으로 처리할지, 이전값으로 처리할지에 대한 부분 # self.X_df = pd.DataFrame(self.X_tr, columns=['aX', 'aY', 'aZ', 'gX', 'gY', 'gZ', 'roll', 'pitch', 'yaw', 'vi']) filepath = "../data/" + filename + ".csv" with open(filepath, 'a') as f: f.write('\n') if record_index == 0: self.X_sc.to_csv(filepath, mode='a', header=True) else: self.X_sc.to_csv(filepath, mode='a', header=False) #self.X_df.to_csv(filepath, mode='a', header=False) #print(record_index) print(str(record_index + 1), '번째 데이터가 저장되었습니다.') time.sleep(1) # clear_output(wait=True) break
def run(self): """ Display the network """ if self.args['notebook']: while True: try: self.coll_plot['node'][1] = [] self.coll_plot['label'][1] = [] self.coll_plot['edge'][1] = [] Cl = [] except: self.coll_plot['node'] = [[]] self.coll_plot['node'].append([]) self.coll_plot['label'] = [[]] self.coll_plot['label'].append([]) self.coll_plot['edge'] = [[]] self.coll_plot['edge'].append([]) Cl = [] rloop = self.net.RAT.keys() for ii, rl in enumerate(rloop): pos = self.net.get_pos(rl) self.coll_plot['node'][1].append(nx.draw_networkx_nodes(self.net, pos=pos, nodelist=self.net.SubNet[rl].nodes(), node_size=100., node_color='r')) # if self.option['estimate']: # try: # pose = self.net.get_pos_est(rl) # pdb.set_trace() # self.coll_plot['node'][1].append(nx.draw_networkx_nodes(self.net, pos=pose, nodelist=self.net.SubNet[rl].nodes(), node_size=100., node_color='b',alpha=0.5)) # except: # pass Cl = nx.draw_networkx_labels(self.net.SubNet[rl], pos=pos, font_size=10) self.coll_plot['label'][1].extend(Cl.values()) self.coll_plot['edge'][1].append((nx.draw_networkx_edges(self.net, pos=pos, edgelist=self.net.SubNet[rl].edges(), width=2., alpha=0.9, edge_color=self.RATcolor[rl], style=self.RATes[rl]))) if self.legend: self.ax.legend((self.coll_plot['edge'][1]), (rloop), loc=3) if self.info: L = nx.get_edge_attributes(self.net, 'TOA') if self.ion: try: [jj.remove() for jj in self.coll_plot['node'][0]] [jj.remove() for jj in self.coll_plot[ 'edge'][0] if jj is not None] [jj.remove() for jj in self.coll_plot['label'][0]] except: pass self.coll_plot['node'][0] = self.coll_plot['node'][1] self.coll_plot['edge'][0] = self.coll_plot['edge'][1] self.coll_plot['label'][0] = self.coll_plot['label'][1] clear_output() display(self.fig) yield hold, self, float(self.update['meca']) else : while True: plt.figure(self.fname) self.ax.axis('scaled') try: self.coll_plot['node'][1] = [] self.coll_plot['label'][1] = [] self.coll_plot['edge'][1] = [] Cl = [] except: self.coll_plot['node'] = [[]] self.coll_plot['node'].append([]) self.coll_plot['label'] = [[]] self.coll_plot['label'].append([]) self.coll_plot['edge'] = [[]] self.coll_plot['edge'].append([]) Cl = [] rloop = self.net.RAT.keys() for ii, rl in enumerate(rloop): pos = self.net.get_pos(rl) self.coll_plot['node'][1].append(nx.draw_networkx_nodes(self.net, pos=pos, nodelist=self.net.SubNet[rl].nodes(), node_size=100., node_color='r')) # if self.option['estimate']: # try: # pose = self.net.get_pos_est(rl) # pdb.set_trace() # self.coll_plot['node'][1].append(nx.draw_networkx_nodes(self.net, pos=pose, nodelist=self.net.SubNet[rl].nodes(), node_size=100., node_color='b',alpha=0.5)) # except: # pass Cl = nx.draw_networkx_labels(self.net.SubNet[rl], pos=pos, font_size=10) self.coll_plot['label'][1].extend(Cl.values()) self.coll_plot['edge'][1].append((nx.draw_networkx_edges(self.net, pos=pos, edgelist=self.net.SubNet[rl].edges(), width=2., alpha=0.9, arrows=False, edge_color=self.RATcolor[rl], style=self.RATes[rl]))) if self.legend: self.ax.legend((self.coll_plot['edge'][1]), (rloop), loc=3) if self.info: L = nx.get_edge_attributes(self.net, 'TOA') if self.ion: try: [jj.remove() for jj in self.coll_plot['node'][0]] [jj.remove() for jj in self.coll_plot[ 'edge'][0] if jj is not None] [jj.remove() for jj in self.coll_plot['label'][0]] except: pass plt.draw() self.coll_plot['node'][0] = self.coll_plot['node'][1] self.coll_plot['edge'][0] = self.coll_plot['edge'][1] self.coll_plot['label'][0] = self.coll_plot['label'][1] yield hold, self, float(self.update['meca'])
def get_imdb_template(imdb_id): imdb_url = f'http://www.omdbapi.com/?i={imdb_id}&apikey={api_key}&r=json&plot=full' #@markdown --- #@markdown ###Movie Information Trailer = "" #@param {type:"string"} Screenshot = "" #@param {type:"string"} youtube_match = re.match(youtube_pattern, Trailer) if Trailer != "": if not youtube_match: clear_output() print( 'Your trailer link is invalid. Please only put in youtube link!' ) sys.exit() else: trailer_enabled = True else: trailer_enabled = False if Screenshot != "": screenshot_list = Screenshot.split(" ") screenshot_enabled = True else: screenshot_enabled = False Path = "" #@param {type:"string"} if Path == "": mediainfo_enabled = False else: mediainfo_enabled = True tmp = subprocess.Popen( 'mediainfo --Logfile=/root/.nfo "{}"'.format(Path), stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) time.sleep(5) # Wait 5s for the nfo file to populate properly with open('/root/.nfo', 'r', encoding='utf-8') as nfo_file: nfo = nfo_file.readlines() del nfo[1] nfo[1] = "Complete name : {}\n".format( os.path.basename(Path)) nfo = ''.join(nfo) Link = "" #@param {type:"string"} if Link == "": print("You forgot putting in the link to download!") sys.exit() #@markdown --- #@markdown ###Forum Requirements Likes = 0 #@param {type: "number"} Posts = 0 #@param {type: "number"} Thanks = True #@param {type:"boolean"} def get_story_line(imdb_id): url = 'https://www.imdb.com/title/{}'.format(imdb_id) r = requests.get(url, headers=headers) data = r.text soup = BeautifulSoup(data, 'html.parser') story_line = soup.find_all( 'div', class_='inline canwrap')[0].find('span').text.lstrip() return story_line # Fetch data with urllib.request.urlopen(imdb_url) as imdb_url: imdb_data = json.loads(imdb_url.read().decode()) # Process data text_dump = "[center][img]{}[/img]\n".format(imdb_data['Poster']) text_dump += "[color=rgb(250, 197, 28)][b][size=6]{} ({})[/size][/b][/color]\n".format( imdb_data['Title'], imdb_data['Year']) text_dump += "[url=https://www.imdb.com/title/{}][img]https://i.imgur.com/rcSipDw.png[/img][/url][size=6][b] {}[/b]/10[/size]".format( imdb_id, imdb_data['imdbRating']) text_dump += "[size=6] --- [img]https://i.imgur.com/sEpKj3O.png[/img]{}[/size][/center]\n".format( imdb_data['imdbVotes']) text_dump += "[hr][/hr][indent][size=6][color=rgb(250, 197, 28)][b]Plot Summary[/b][/color][/size][/indent]\n" text_dump += "{}\n".format(get_story_line(imdb_id)) if trailer_enabled: text_dump += "[hr][/hr][indent][size=6][color=rgb(250, 197, 28)][b]Trailer[/b][/color][/size][/indent]\n" text_dump += "[media=youtube]{}[/media]\n".format( youtube_match.group(6)) if screenshot_enabled: text_dump += "[hr][/hr][indent][size=6][color=rgb(250, 197, 28)][b]Screenshot[/b][/color][/size][/indent]\n" text_dump += "[spoiler='Click here to view screenshots']\n" for i in screenshot_list: text_dump += "[img]{}[/img]\n".format(i) text_dump += "[/spoiler]" text_dump += "[hr][/hr][indent][size=6][color=rgb(250, 197, 28)][b]Movie Info[/b][/color][/size][/indent]\n" text_dump += "[LIST][*][color=rgb(255, 204, 0)]Rating:[/color] {}\n".format( imdb_data['Rated']) text_dump += "[*][color=rgb(255, 204, 0)]Genre:[/color] {}\n".format( imdb_data['Genre']) text_dump += "[*][color=rgb(255, 204, 0)]Directed By:[/color] {}\n".format( imdb_data['Director']) text_dump += "[*][color=rgb(255, 204, 0)]Written By:[/color] {}\n".format( imdb_data['Writer']) text_dump += "[*][color=rgb(255, 204, 0)]Starring:[/color] {}\n".format( imdb_data['Actors']) text_dump += "[*][color=rgb(255, 204, 0)]Release Date:[/color] {}\n".format( imdb_data['Released']) try: text_dump += "[*][color=rgb(255, 204, 0)]Runtime:[/color] {}\n".format( imdb_data['Runtime']) except: text_dump += "[*][color=rgb(255, 204, 0)]Runtime:[/color] N/A\n" try: text_dump += "[*][color=rgb(255, 204, 0)]On Disk/Streaming:[/color] {}\n".format( imdb_data['DVD']) except: text_dump += "[*][color=rgb(255, 204, 0)]On Disk/Streaming:[/color] N/A\n" try: text_dump += "[*][color=rgb(255, 204, 0)]Box Office:[/color] {}\n".format( imdb_data['BoxOffice']) except: text_dump += "[*][color=rgb(255, 204, 0)]Box Office:[/color] N/A\n" text_dump += "[*][color=rgb(255, 204, 0)]Awards:[/color] {}\n".format( imdb_data['Awards']) try: text_dump += "[*][color=rgb(255, 204, 0)]Studio:[/color] {}[/LIST]\n".format( imdb_data['Production']) except: text_dump += "[*][color=rgb(255, 204, 0)]Studio:[/color] N/A[/LIST]\n" if mediainfo_enabled: text_dump += "[hr][/hr][indent][size=6][color=rgb(250, 197, 28)][b]Media Info[/b][/color][/size][/indent]\n" text_dump += "[spoiler='Click here to view Media Info'][code]{}[/code][/spoiler]\n".format( nfo) text_dump += "[hr][/hr][center][size=6][color=rgb(250, 197, 28)][b]Download Link[/b][/color][/size][/center]\n" if Thanks: if Likes != 0 and Posts == 0: text_dump += "[center][thanks][likes={}]{}[/likes][/thanks][/center]".format( Likes, Link) elif Likes == 0 and Posts != 0: text_dump += "[center][thanks][posts={}]{}[/posts][/thanks][/center]".format( Posts, Link) elif Likes != 0 and Posts != 0: text_dump += "[center][thanks][likes={}][posts={}]{}[/posts][/likes][/thanks][/center]".format( Likes, Posts, Link) else: text_dump += "[center][thanks]{}[/thanks][/center]".format(Link) else: if Likes != 0 and Posts == 0: text_dump += "[center][likes={}]{}[/likes][/center]".format( Likes, Link) elif Likes == 0 and Posts != 0: text_dump += "[center][posts={}]{}[/posts][/center]".format( Posts, Link) elif Likes != 0 and Posts != 0: text_dump += "[center][likes={}][posts={}]{}[/posts][/likes][/center]".format( Likes, Posts, Link) else: text_dump += "[center]{}[/center]".format(Link) return text_dump
x, x_nesterov, dataLoss, regLoss = SQSOneStep(reconNet, x, x_nesterov, x_dict, prjs, weights, normImg, projectorNorm, args, showPlots) # dictionary x_dict, coefs = csc(x, D1, args) coefLoss = np.sum(np.abs(coefs)) rmse_roi = np.sqrt(np.mean((x - refs)[0, 128:-128, 128:-128, 0]**2)) vals.append([dataLoss, regLoss, coefLoss, rmse_roi]) if (iIter + 1) % args.outputInterval == 0: if showPlots: display.clear_output() plt.figure(figsize=[18, 6]) plt.subplot(131) plt.imshow(refs[0, 128:-128, 128:-128, 0].T, 'gray', vmin=args.vmin, vmax=args.vmax) plt.subplot(132) plt.imshow(x[0, 128:-128, 128:-128, 0].T, 'gray', vmin=args.vmin, vmax=args.vmax) plt.subplot(133) plt.imshow(x0[0, 128:-128, 128:-128, 0].T, 'gray', vmin=args.vmin,
# The initial state of the environment. isGameOver = False fruitRow, fruitColumn, basket = env.reset() currentState = env.observe() drawState(fruitRow, fruitColumn, basket) while (isGameOver != True): # Forward the current state through the network. q = sess.run(output_layer, feed_dict={X: currentState}) # Find the max index (the chosen action). index = q.argmax() action = index + 1 nextState, reward, gameOver, stateInfo = env.act(action) fruitRow = stateInfo[0] fruitColumn = stateInfo[1] basket = stateInfo[2] # Count game results if (reward == 1): winCount = winCount + 1 elif (reward == -1): loseCount = loseCount + 1 currentState = nextState isGameOver = gameOver drawState(fruitRow, fruitColumn, basket) time.sleep(0.4) display.clear_output(wait=True)
def animate(k): # clear the previous panel for next slide ax1.cla() ax2.cla() # plot function ax1.plot(w_plot,g_plot,color = 'k',zorder = 0) # plot function # print rendering update if np.mod(k+1,25) == 0: print ('rendering animation frame ' + str(k+1) + ' of ' + str(num_frames)) if k == num_frames - 1: print ('animation rendering complete!') time.sleep(1.5) clear_output() # plot initial point and evaluation if k == 0: w_val = self.w_init g_val = self.g(w_val) ax1.scatter(w_val,g_val,s = 100,c = 'm',edgecolor = 'k',linewidth = 0.7,zorder = 2) # plot point of tangency ax1.scatter(w_val,0,s = 100,c = 'm',edgecolor = 'k',linewidth = 0.7, zorder = 2, marker = 'X') # plot function alone first along with initial point if k > 0: epsilon = self.epsilon_range[k-1] # run gradient descent method self.w_hist = [] self.run_newtons_method(epsilon) # colors for points s = np.linspace(0,1,len(self.w_hist[:round(len(self.w_hist)/2)])) s.shape = (len(s),1) t = np.ones(len(self.w_hist[round(len(self.w_hist)/2):])) t.shape = (len(t),1) s = np.vstack((s,t)) self.colorspec = [] self.colorspec = np.concatenate((s,np.flipud(s)),1) self.colorspec = np.concatenate((self.colorspec,np.zeros((len(s),1))),1) # plot everything for each iteration for j in range(len(self.w_hist)): w_val = self.w_hist[j] g_val = self.g(w_val) ax1.scatter(w_val,g_val,s = 90,c = self.colorspec[j],edgecolor = 'k',linewidth = 0.7,zorder = 3) # plot point of tangency ax1.scatter(w_val,0,s = 90,facecolor = self.colorspec[j],marker = 'X',edgecolor = 'k',linewidth = 0.7, zorder = 2) # plug in value into func and derivative g_val = self.g(w_val) g_grad_val = self.grad(w_val) g_hess_val = self.hess(w_val) # determine width of plotting area for second order approximator width = 0.5 if g_hess_val < 0: width = - width # compute second order approximation wrange = np.linspace(w_val - 3,w_val + 3, 100) h = g_val + g_grad_val*(wrange - w_val) + 0.5*(g_hess_val + epsilon)*(wrange - w_val)**2 # plot all ax1.plot(wrange,h,color = self.colorspec[j],linewidth = 2,alpha = 0.4,zorder = 1) # plot approx ### plot all on cost function decrease plot ax2.scatter(j,g_val,s = 90,c = self.colorspec[j],edgecolor = 'k',linewidth = 0.7,zorder = 3) # plot point of tangency # place title title = r'$\epsilon = $' + r'{:.2f}'.format(epsilon) ax1.set_title(title,fontsize = 15) # plot connector between points for visualization purposes if j > 0: w_old = self.w_hist[j-1] w_new = self.w_hist[j] g_old = self.g(w_old) g_new = self.g(w_new) ax2.plot([j-1,j],[g_old,g_new],color = self.colorspec[j],linewidth = 2,alpha = 0.4,zorder = 1) # plot approx else: title = r'$\,\,\,$' ax1.set_title(title,fontsize = 15) # clean up axis in each panel ax2.set_xlabel('iteration',fontsize = 13) ax2.set_ylabel(r'$g(w)$',fontsize = 13,labelpad = 15,rotation = 0) ax1.set_xlabel(r'$w$',fontsize = 13) ax1.set_ylabel(r'$g(w)$',fontsize = 13,labelpad = 15,rotation = 0) # fix viewing limits ax1.set_xlim([-wmax,wmax]) ax1.set_ylim([min(g_plot) - ggap,max(g_plot) + ggap]) ax2.set_xlim([-0.5,self.max_its + 0.5]) ax2.set_ylim([min(g_plot) - ggap,max(g_plot) + ggap]) # set tickmarks ax1.set_xticks(np.arange(-round(wmax), round(wmax) + 1, 1.0)) ax1.set_yticks(np.arange(round(min(g_plot) - ggap), round(max(g_plot) + ggap) + 1, 1.0)) ax2.set_xticks(np.arange(0,self.max_its + 1, 1.0)) return artist,
def on_epoch_end(self, epoch_callback, logs=None): clear_output(wait=True) show_predictions(epoch_callback) print('\nSample Prediction after epoch {}\n'.format(epoch_callback + 1))
def train(self): generator = self.generator discriminator = self.discriminator g_optim = self.g_optim d_optim = self.d_optim g_decay = self.g_decay d_decay = self.d_decay iterator_type = self.schedule['iterator_type'] device = self.device epoch = 0 epochs = self.schedule['epochs'] pseudo_epoch_iters = self.schedule['pseudo_epoch_iters'] train_loader = self.train_loader train_loader_len = len(self.train_loader) test_loader = self.train_loader perceptual_loss = self.perceptual_loss z_transform = self.schedule.get('z_transform') self.z_transform = z_transform data_iters = 0 for epoch in range(epochs): i = 0 while True: if data_iters >= train_loader_len or data_iters is 0: train_iter = iter(train_loader) test_iter = iter(test_loader) self.validate(self.generator, test_iter, iterator_type, device) self.save_parts(epoch, i) data_iters = 0 epoch_break = self.pseudo_epoch(i, pseudo_epoch_iters) if epoch_break: g_decay.step() d_decay.step() break inputs, targets = parse_data(train_iter, device, iterator_type, z_transform) d_loss = spectral_d_iter(inputs, targets, generator, discriminator, d_optim) inputs, targets = parse_data(train_iter, device, iterator_type, z_transform) g_loss_adv, g_loss_percep = spectral_g_iter( inputs, targets, generator, discriminator, g_optim, perceptual_loss) i += 1 data_iters += 2 self.g_loss_adv.append(g_loss_adv) self.g_loss_percep.append(g_loss_percep) self.d_loss.append(d_loss) if i % 5 is 0: print(f'e{epoch}i{i}') if i % 95 is 0 or (i in [1, 2, 3, 4, 5, 10, 20, 50, 75, 125] and epoch is 0): os.system('clear') clear_output() g_lr = g_optim.param_groups[0]['lr'] d_lr = d_optim.param_groups[0]['lr'] print(f'g_lr {g_lr}') print(f'd_lr {d_lr}') self.validate(self.generator, test_iter, iterator_type, device) if i % 95 is 0: self.save_parts(epoch, i)
def download_project(project_name, class_info, email, use_path=None, skip_download=False): ''' use_path: 지정 폴더에 다운로드 skip_download: 폴더에 데이터가 존재할 시 SKIP. 단 CHECKSUM 비교는 안함. ''' global project, files try: project = Project(project_name, class_info=class_info, email=email) # data 폴더 경로 지정 DATA_DIR = 'data' if use_path is not None: DATA_DIR = use_path with open(PROJECT_DATA_PATH) as f: datasets = json.load(f) # data 폴더 생성 if not os.path.exists(os.path.join(DATA_DIR, project_name)): os.makedirs(os.path.join(DATA_DIR, project_name)) project_files = datasets[project_name] # skip download 체크 file_to_remove = [] if skip_download: for filename, _ in project_files.items(): filepath = os.path.join(DATA_DIR, project_name, filename) if os.path.exists(filepath): file_to_remove.append(filename) print( f'{filename} 파일이 {filepath} 이미 존재하여 다운로드를 SKIP 합니다..') print(f'카운트 다운!') time_cnt = 1 while time_cnt <= 3: print(f'{time_cnt}', end=' ') time.sleep(1) time_cnt += 1 clear_output(wait=True) for k in file_to_remove: project_files.pop(k) # auth username = '******' password = '******' print( f'프로젝트: {project_name}\n==============================\n파일 다운로드\n') for filename, fileurl in project_files.items(): r = requests.get(fileurl, auth=HTTPBasicAuth(username, password), stream=True) filepath = os.path.join(DATA_DIR, project_name, filename) print(f'{filename}', end=' ') ## 다운로드 progress bar 추가 ## total_size_in_bytes = int(r.headers.get('content-length', 0)) block_size = 1024 progress_bar = tqdm(total=total_size_in_bytes, unit='B', unit_scale=True) with open(filepath, 'wb') as file: for data in r.iter_content(block_size): progress_bar.update(len(data)) file.write(data) progress_bar.close() if total_size_in_bytes != 0 and progress_bar.n != total_size_in_bytes: print("ERROR: 다운로드 도중 에러가 발생하였습니다.") else: if filepath.endswith('.zip'): print(f'압축 해제 및 프로젝트 파일 구성중...') zipfile.ZipFile(filepath).extractall( os.path.join(DATA_DIR, project_name)) ## 다운로드 progress bar 추가 ## clear_output(wait=True) print(f'\n==============================') print(f'프로젝트: {project_name}\n==============================\n파일 목록\n') for f in [ fs for fs in os.listdir(os.path.join('data', project_name)) if 'csv' in fs ]: print(f'{f}\n- {os.path.join(DATA_DIR, project_name, f)}\n') print(f'==============================') except Exception as err: print(err) raise Exception('잘못된 정보입니다.')
def StartTraining(): training_Canvas = tk.Label(window) trainingAndVal_Canvas = tk.Label(window) resultTextbox.insert("end", "Training Start\n") window.update_idletasks() resultTextbox.see("end") ##### Init Args args.dataset = datasetSelected.get() args.vocab_size = int(vocab_size.get()) args.validation_portion = float(validation_portion.get()) args.test_portion = float(test_portion.get()) args.batch_size = int(batch_size.get()) args.L2 = float(L2.get()) args.lr = float(lr.get()) args.n_epoch = int(n_epoch.get()) args.earlyStopStep = int(earlyStopStep.get()) args.earlyStopEpoch = int(earlyStopEpoch.get()) args.val_freq = int(val_freq.get()) args.val_steps = int(val_steps.get()) args.log_freq = int(log_freq.get()) args.model_name = model_name.get() args.scheduler_step = int(scheduler_step.get()) args.scheduler_gamma = float(scheduler_gamma.get()) args.scheduler_minLr = float(scheduler_minLr.get()) args.usingWeightRandomSampling = bool(usingWeightRandomSampling.get()) args.GatedCNN_embedingDim = int(GatedCNN_embedingDim.get()) args.GatedCNN_convDim = int(GatedCNN_convDim.get()) args.GatedCNN_kernel = int(GatedCNN_kernel.get()) args.GatedCNN_stride = int(GatedCNN_stride.get()) args.GatedCNN_pad = int(GatedCNN_pad.get()) args.GatedCNN_layers = int(GatedCNN_layers.get()) args.GatedCNN_dropout = float(GatedCNN_dropout.get()) args.SSCL_embedingDim = int(SSCL_embedingDim.get()) args.SSCL_RNNHidden = int(SSCL_RNNHidden.get()) args.SSCL_CNNDim = int(SSCL_CNNDim.get()) args.SSCL_CNNKernel = int(SSCL_CNNKernel.get()) args.SSCL_CNNDropout = float(SSCL_CNNDropout.get()) args.SSCL_LSTMDropout = float(SSCL_LSTMDropout.get()) args.SSCL_LSTMLayers = int(SSCL_LSTMLayers.get()) args.SelfAttn_LenMaxSeq = int(SelfAttn_LenMaxSeq.get()) args.SelfAttn_ModelDim = int(SelfAttn_ModelDim.get()) args.SelfAttn_FFInnerDim = int(SelfAttn_FFInnerDim.get()) args.SelfAttn_NumLayers = int(SelfAttn_NumLayers.get()) args.SelfAttn_NumHead = int(SelfAttn_NumHead.get()) args.SelfAttn_KDim = int(SelfAttn_KDim.get()) args.SelfAttn_VDim = int(SelfAttn_VDim.get()) args.SelfAttn_Dropout = float(SelfAttn_Dropout.get()) args.model_path = './' + args.dataset + '_Log/' + args.model_name + '/Model/' args.log_path = './' + args.dataset + '_Log/' + args.model_name + '/Log/' device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') args.device = device if not os.path.exists(args.model_path): os.makedirs(args.model_path) if not os.path.exists(args.log_path): os.makedirs(args.log_path) training_dataset, validation_dataset, test_dataset, text = TkloadingData(args, resultTextbox, window) resultTextbox.see("end") args.numberOfSpammer = sum([t[-1] for t in training_dataset]) args.numberOfNonSpammer = len(training_dataset)-args.numberOfSpammer args.len_max_seq = training_dataset[0][2] resultTextbox.insert("end", ("Number of Spammer: " +str(args.numberOfSpammer.item()) +"\n" )) resultTextbox.insert("end", ("Number of NonSpammer: " + str(args.numberOfNonSpammer.item()) +"\n" )) window.update_idletasks() if args.usingWeightRandomSampling: resultTextbox.insert("end", "Using WeightRandomSampling... \n") sampler = getSampler(training_dataset) else: resultTextbox.insert("end", "Not Using WeightRandomSampling... \n") sampler = None window.update_idletasks() train_loader = DataLoader( training_dataset, batch_size=args.batch_size, shuffle=False, drop_last=False, sampler = sampler) valid_loader = DataLoader( validation_dataset, batch_size=args.batch_size, shuffle=True, drop_last=False) if modelSelected.get() == 'SSCL': resultTextbox.insert("end", "Using SSCL\n") trainModel = SSCL elif modelSelected.get() == 'GatedCNN': resultTextbox.insert("end", "Using GatedCNN\n") trainModel = GatedCNN elif modelSelected.get() == 'SelfAttn': resultTextbox.insert("end", "Using SelfAttn\n") trainModel = SelfAttnModel else: resultTextbox.insert("end", 'No Support For this Model') raise ValueError window.update_idletasks() trainer = TkTrainer(trainModel, args, resultTextbox, window).to(device) resultTextbox.insert("end", ("Number of Parameters in this Model: " + str(trainer.num_all_params()) +"\n")) resultTextbox.insert("end", ("Using device: " + str(device) +"\n")) window.update_idletasks() args.using_scheduler = args.scheduler_step and args.scheduler_gamma < 1 and args.scheduler_gamma > 0 if args.using_scheduler: scheduler = optim.lr_scheduler.StepLR(trainer.optim, args.scheduler_step, gamma=args.scheduler_gamma) # trainer.optim.param_groups[0]['lr']= allStep = 0 epoch = 0 resultTextbox.insert("end",("Model Structure: \n" + str(trainer.model) + "\n")) resultTextbox.see("end") window.update_idletasks() while epoch < args.n_epoch: for i, (texts, X , X_len, y) in enumerate(train_loader): trainer.train() X, X_len, y = X.to(device), X_len.to(device), y.to(device) if args.using_scheduler: if trainer.optim.param_groups[0]['lr'] >= args.scheduler_minLr: scheduler.step() start_t = time.time() # trainer.train_step((X, X_len), y) trainer.train_step(X, y) end_t = time.time() allStep += 1 resultTextbox.insert("end", ('| Epoch [%d] | Step [%d] | lr [%.6f] | Loss: [%.4f] | Acc: [%.4f] | Time: %.1fs \n' % (epoch, allStep, trainer.optim.param_groups[0]['lr'], trainer.loss.item(), trainer.accuracy.item(), end_t - start_t))) window.update_idletasks() resultTextbox.see("end") # if trainer.accuracy.item() > 0.95: # Stop early # resultTextbox.insert("end", "Train Accuracy Reach the Stop Accuracy ") # raise StopIteration if allStep % args.log_freq == 0: ################################################# trainer.plot_train_hist(args.model_name) TrainImg = ImageTk.PhotoImage(Image.open(args.log_path+"Train_Loss&Acc_Hist_"+ str(args.model_name) +".png").resize((1000, 470), Image.ANTIALIAS)) training_Canvas.config(image=TrainImg) training_Canvas.image = TrainImg training_Canvas.grid(column=5, row=0, rowspan= 16) window.update_idletasks() if args.earlyStopStep: if allStep >= args.earlyStopStep: resultTextbox.insert("end", "EarlyStopStep Reach") break if allStep % args.val_freq == 0: for _ in range(args.val_steps): trainer.eval() stIdx = np.random.randint( 0, len(validation_dataset) - args.batch_size) v_text, v_X, v_X_len, v_y = validation_dataset[stIdx: stIdx + args.batch_size] v_X, v_X_len, v_y = v_X.to( device), v_X_len.to(device), v_y.to(device) start_t = time.time() # trainer.test_step((v_X, v_X_len), v_y) trainer.test_step(v_X, v_y) end_t = time.time() resultTextbox.insert("end", ('| Epoch [%d] | Validation | Step [%d] | Loss: [%.4f] | Acc: [%.4f] | Time: %.1fs \n' % (epoch, allStep, trainer.loss.item(), trainer.accuracy.item(), end_t - start_t))) window.update_idletasks() resultTextbox.see("end") trainer.calculateAverage() clear_output() resultTextbox.insert("end",("TrainConfusion Matrix: \n")) resultTextbox.insert("end",pd.DataFrame(trainer.cms['Train'][-1])) resultTextbox.insert("end","\n\n") resultTextbox.insert("end",("ValidationConfusion Matrix: \n")) resultTextbox.insert("end",pd.DataFrame(trainer.cms['Val'][-1])) resultTextbox.insert("end","\n\n") window.update_idletasks() resultTextbox.see("end") ################################################# trainer.plot_all(args.model_name) TrainAndValImg = ImageTk.PhotoImage(Image.open(args.log_path + "All_Hist_" + str(args.model_name) + ".png").resize((1000, 470), Image.ANTIALIAS)) trainingAndVal_Canvas.config(image=TrainAndValImg) trainingAndVal_Canvas.image = TrainAndValImg trainingAndVal_Canvas.grid(column = 5 , row = 17) window.update_idletasks() epoch += 1 trainer.model_save(epoch) if args.earlyStopEpoch: if epoch >= args.earlyStopEpoch: resultTextbox.insert("end", "EarlyStopEpoch Reach") break test_text, test_X, test_X_len, test_y = zip(test_dataset[0:]) test_text, test_X, test_X_len, test_y = test_text[0], test_X[0].to(device), test_X_len[0].to(device), test_y[0].to(device) test_loss, test_accuracy, test_cm = trainer.test_step(test_X, test_y) resultTextbox.insert("end",("\n\n=========================================================================================" )) resultTextbox.insert("end",("\nThe Test Loss: " + str(test_loss.item()) + "\n" )) resultTextbox.insert("end",("The Test Accuracy: " + str(test_accuracy.item()) + "\n" )) resultTextbox.insert("end",("Test Confusion Matrix: \n")) resultTextbox.insert("end",(pd.DataFrame(test_cm))) resultTextbox.insert("end", "\n\n") resultTextbox.see("end")
def animate_it(self,epsilon_range,savepath,**kwargs): self.epsilon_range = epsilon_range if 'w_init' in kwargs: self.w_init = float(kwargs['w_init']) if 'max_its' in kwargs: self.max_its = float(kwargs['max_its']) wmax = 3 if 'wmax' in kwargs: wmax = kwargs['wmax'] # initialize figure fig = plt.figure(figsize = (10,4)) artist = fig gs = gridspec.GridSpec(1, 2, width_ratios=[2,1]) ax1 = plt.subplot(gs[0],aspect = 'auto'); ax2 = plt.subplot(gs[1],sharey=ax1); # generate function for plotting on each slide w_plot = np.linspace(-wmax,wmax,1000) g_plot = self.g(w_plot) g_range = max(g_plot) - min(g_plot) ggap = g_range*0.5 w_vals = np.linspace(-2.5,2.5,50) # animation sub-function print ('starting animation rendering...') num_frames = len(self.epsilon_range) + 1 def animate(k): # clear the previous panel for next slide ax1.cla() ax2.cla() # plot function ax1.plot(w_plot,g_plot,color = 'k',zorder = 0) # plot function # print rendering update if np.mod(k+1,25) == 0: print ('rendering animation frame ' + str(k+1) + ' of ' + str(num_frames)) if k == num_frames - 1: print ('animation rendering complete!') time.sleep(1.5) clear_output() # plot initial point and evaluation if k == 0: w_val = self.w_init g_val = self.g(w_val) ax1.scatter(w_val,g_val,s = 100,c = 'm',edgecolor = 'k',linewidth = 0.7,zorder = 2) # plot point of tangency ax1.scatter(w_val,0,s = 100,c = 'm',edgecolor = 'k',linewidth = 0.7, zorder = 2, marker = 'X') # plot function alone first along with initial point if k > 0: epsilon = self.epsilon_range[k-1] # run gradient descent method self.w_hist = [] self.run_newtons_method(epsilon) # colors for points s = np.linspace(0,1,len(self.w_hist[:round(len(self.w_hist)/2)])) s.shape = (len(s),1) t = np.ones(len(self.w_hist[round(len(self.w_hist)/2):])) t.shape = (len(t),1) s = np.vstack((s,t)) self.colorspec = [] self.colorspec = np.concatenate((s,np.flipud(s)),1) self.colorspec = np.concatenate((self.colorspec,np.zeros((len(s),1))),1) # plot everything for each iteration for j in range(len(self.w_hist)): w_val = self.w_hist[j] g_val = self.g(w_val) ax1.scatter(w_val,g_val,s = 90,c = self.colorspec[j],edgecolor = 'k',linewidth = 0.7,zorder = 3) # plot point of tangency ax1.scatter(w_val,0,s = 90,facecolor = self.colorspec[j],marker = 'X',edgecolor = 'k',linewidth = 0.7, zorder = 2) # plug in value into func and derivative g_val = self.g(w_val) g_grad_val = self.grad(w_val) g_hess_val = self.hess(w_val) # determine width of plotting area for second order approximator width = 0.5 if g_hess_val < 0: width = - width # compute second order approximation wrange = np.linspace(w_val - 3,w_val + 3, 100) h = g_val + g_grad_val*(wrange - w_val) + 0.5*(g_hess_val + epsilon)*(wrange - w_val)**2 # plot all ax1.plot(wrange,h,color = self.colorspec[j],linewidth = 2,alpha = 0.4,zorder = 1) # plot approx ### plot all on cost function decrease plot ax2.scatter(j,g_val,s = 90,c = self.colorspec[j],edgecolor = 'k',linewidth = 0.7,zorder = 3) # plot point of tangency # place title title = r'$\epsilon = $' + r'{:.2f}'.format(epsilon) ax1.set_title(title,fontsize = 15) # plot connector between points for visualization purposes if j > 0: w_old = self.w_hist[j-1] w_new = self.w_hist[j] g_old = self.g(w_old) g_new = self.g(w_new) ax2.plot([j-1,j],[g_old,g_new],color = self.colorspec[j],linewidth = 2,alpha = 0.4,zorder = 1) # plot approx else: title = r'$\,\,\,$' ax1.set_title(title,fontsize = 15) # clean up axis in each panel ax2.set_xlabel('iteration',fontsize = 13) ax2.set_ylabel(r'$g(w)$',fontsize = 13,labelpad = 15,rotation = 0) ax1.set_xlabel(r'$w$',fontsize = 13) ax1.set_ylabel(r'$g(w)$',fontsize = 13,labelpad = 15,rotation = 0) # fix viewing limits ax1.set_xlim([-wmax,wmax]) ax1.set_ylim([min(g_plot) - ggap,max(g_plot) + ggap]) ax2.set_xlim([-0.5,self.max_its + 0.5]) ax2.set_ylim([min(g_plot) - ggap,max(g_plot) + ggap]) # set tickmarks ax1.set_xticks(np.arange(-round(wmax), round(wmax) + 1, 1.0)) ax1.set_yticks(np.arange(round(min(g_plot) - ggap), round(max(g_plot) + ggap) + 1, 1.0)) ax2.set_xticks(np.arange(0,self.max_its + 1, 1.0)) return artist, anim = animation.FuncAnimation(fig, animate,frames=num_frames, interval=num_frames, blit=True) # produce animation and save fps = 50 if 'fps' in kwargs: fps = kwargs['fps'] anim.save(savepath, fps=fps, extra_args=['-vcodec', 'libx264']) clear_output()
def on_epoch_end(self, epoch, metrics): if epoch % self.interval == 0: display.clear_output(wait=True) self.plot_losses(self.trainer.metrics['train']['losses'], self.trainer.metrics['valid']['losses'])
# Iterate through epochs for epoch in range(NUM_EPOCHS): for n_batch, (batch,_) in enumerate(dataloader): # 1. Train Discriminator X_batch = batch.permute(0, 2, 3, 1).numpy() feed_dict = {X: X_batch, Z: noise(BATCH_SIZE, NOISE_SIZE)} _, d_error, d_pred_real, d_pred_fake = session.run( [D_opt, D_loss, D_real, D_fake], feed_dict=feed_dict ) # 2. Train Generator feed_dict = {Z: noise(BATCH_SIZE, NOISE_SIZE)} _, g_error = session.run( [G_opt, G_loss], feed_dict=feed_dict ) if n_batch % 100 == 0: display.clear_output(True) # Generate images from test noise test_images = session.run( G_sample, feed_dict={Z: test_noise} ) # Log Images logger.log_images(test_images, num_test_samples, epoch, n_batch, num_batches, format='NHWC'); # Log Status logger.display_status( epoch, num_epochs, n_batch, num_batches, d_error, g_error, d_pred_real, d_pred_fake )