def parser(): def chunks(l, n): list = [] for i in range(0, len(l), n): list.append(l[i:i+n]) return list url = 'http://www.banki.ru/banks/memory/?PAGEN_1=' site = urllib.request.urlopen(url) html = site.read().decode('windows-1251') n_pages = round(int(re.findall(r'totalItems:(.*?);', str(html), re.S)[0]) / 50.0 + 0.48) + 1 table = re.findall(r'<table class="standard-table standard-table--row-highlight">(.*?)</table>', str(html), re.S) thead = re.findall(r'<th>(.*?)(?: <nobr>.*?</nobr>)?</th>', str(table)) i = 0 for th in thead: thead[i] = th.replace(' ', '_') i += 1 trs = re.findall(r'<td[^>]*>(?:<strong><a[^>]*>)?(.*?)(?:</a></strong>)?</td>', str(table)) print_progress(0, n_pages, prefix='Формирую файл с отозванными банками:', suffix='Выполнено', barLength=40) for i in range(2, n_pages): cur_url = url + str(i) site = urllib.request.urlopen(cur_url) html = site.read().decode('windows-1251') table = re.findall(r'<table class="standard-table standard-table--row-highlight">(.*?)</table>', str(html), re.S) trs.extend(re.findall(r'<td[^>]*>(?:<strong><a[^>]*>)?(.*?)(?:</a></strong>)?</td>', str(table))) print_progress(i, n_pages, prefix='Формирую файл с отозванными банками:', suffix='Выполнено', barLength=40) defunct = pd.DataFrame(data=chunks(trs, len(thead)), columns=thead) return defunct
def download_images(self, directory): print("[+] Downloading images") html = requests.get('https://www.google.com/search?q=' + self.get_args().term + '&source=lnms&tbm=isch') soup = BeautifulSoup(html.text, "html.parser") links = soup.find_all("img") print_progress(links) i = 0 for a in links: if i < int(self.get_args().volume): _download(directory, a['src']) i += 1
# if fewer tweets found than max count, use that number totlen = len(alltweets) else: # otherwise, process maxcCount at most totlen = maxCount searchresults = [] if alltweets: for i, tweet in enumerate(alltweets): # this searches on tweet at a time searchresults.extend(tweetsearcher.searchTweet(tweet)) # count in progress_bar is i+1 because we start at zero and # this should not be zero-based counter if showProgressBar: progress_bar.print_progress(i + 1, totlen) if i == totlen: break # send email if search results come back if searchresults: # format message to send msg = '' for sr in searchresults: url = 'https://twitter.com/' +\ sr.reference.user.screen_name +\ '/status/' +\ sr.reference.id_str
] _, acc, loss = sess.run( [train_step, accuracy, cross_entropy], feed_dict={ x: batch[0], y: batch[1], keep_prob: FLAGS.keep_prob, is_training: True }) acc *= 100 num_iter = max_size * 10 progress = ( (epoch_num * max_size + num) % num_iter + 1) / num_iter num_ep = epoch_num + 1 print_progress(progress, num_ep, loss, acc) if FLAGS.online_test and (epoch_num + 1) % 10 == 0: print(' ' * 12, 'Online-Testing ========== (。・`ω´・) ========') imgs_t, labels_t = test_data.test_image.reshape( (-1, 90, 75)), test_data.test_labels test_acc, test_loss = sess.run([accuracy, cross_entropy], feed_dict={ x: imgs_t, y: labels_t, keep_prob: 1.0, is_training: False }) test_acc *= 100 print(
test_summary_dir = os.path.join(FLAGS.events_path, 'summaries', 'test') test_summary_writer = tf.summary.FileWriter(test_summary_dir) test_summary_writer.add_graph(sess.graph) # 随机梯度下降进行训练 for num_epoch in range(FLAGS.num_epochs): image_batch, label_batch = mnist.train.next_batch(FLAGS.batch_size) batch_loss, batch_acc, _, batch_summ, num_train, lr = sess.run([loss_tensor, accuracy, train_op, summary_train, global_step, learning_rate], feed_dict={image_place: image_batch, label_place: label_batch, keep_prob: FLAGS.keep_prob}) batch_acc *= 100 progress = float(num_epoch % FLAGS.num_batch + 1) / FLAGS.num_batch num_epoch_batch = num_epoch // FLAGS.num_batch + 1 print_progress(progress, num_epoch_batch, batch_loss, batch_acc) # print('Epoch '+str(num_epoch+1) + ', learning rate is %.4f' % lr + ', train accuracy is '+'{:.2f}%.'.format(batch_acc)) # print('Epoch '+str(num_epoch+1) + ', learning rate is %.4f' % lr + ', train loss is '+ '{:.4f}, '.format(batch_loss) + \ # 'accuracy is '+'{:.2f}%.'.format(batch_acc)) train_summary_writer.add_summary(batch_summ, num_epoch) checkpoints_prefix = 'model.ckpt' if (num_epoch+1) % FLAGS.num_batch == 0: print('Learning rate is %.4f' % lr + ', train accuracy is '+'{:.2f}%.'.format(batch_acc)) if (batch_loss <= min_cross) & (batch_acc > max_acc): # 按照要求保存网络模型 min_cross = batch_loss max_acc = batch_acc saver.save(sess, os.path.join(FLAGS.checkpoints_path, checkpoints_prefix), global_step=num_epoch+1) # print("\033[0;31;40m\tModel restored ... ...\033[0m\n") printRed("Model restored ... ...\n") print('\n')
nn = DeepQ(num_actions, num_states, lr=IL_LEARNING_RATE) # Load the samples we want to train on states = np.load(os.path.join( 'generated_data', 'states_{}_{}_{}_{}.npy'.format(path_grid, n, num_states, run_id)), allow_pickle=True) rewards = np.load(os.path.join( 'generated_data', 'rewards_{}_{}_{}_{}.npy'.format(path_grid, n, num_actions, run_id)), allow_pickle=True) loss = 0 for episode in range(NUM_EPISODES): print_progress(episode, NUM_EPISODES, prefix='Episode {}/{}'.format(episode, NUM_EPISODES)) # Train the network on batches of the generated samples for iteration in range(iterations): start_index = iteration * IL_BATCH_SIZE end_index = start_index + IL_BATCH_SIZE state_batch = states[start_index:end_index] target_batch = rewards[start_index:end_index] loss = nn.model.train_on_batch(state_batch, target_batch) # print("We had an imitation loss equal to ", loss) network_path = os.path.join('saved_networks', 'imitation_learning', '{}_{}_{}_il'.format(path_grid, n, run_id)) nn.save_network(network_path)
def train(self, env, num_iterations=10000, network_path=None): """ Train the agent. The agent runs in the environment for a specified number of iterations. When the done state is reached the environment is reset. When their are enough samples in the replay buffer. Training of the network starts. """ # Initialize Tensorboard writer log_path = os.path.join('logs', 'train', self.id + '_' + str(time.time())) tf_writer = tf.summary.create_file_writer(log_path) transformed_observation = self.convert_obs(env.reset()) if self.deep_q is None: self.init_deep_q(transformed_observation) epsilon = INITIAL_EPSILON epsilon_decay = (INITIAL_EPSILON - FINAL_EPSILON) / num_iterations total_reward = 0 reset_count = 0 current_loss = np.inf for iteration in range(num_iterations): print_progress(iteration + 1, num_iterations, prefix='Step {}/{}'.format(iteration + 1, num_iterations), suffix='Episode count: {}'.format(reset_count)) # Decay epsilon over time epsilon -= epsilon_decay # Predict the next step ... curr_state = self.convert_obs(env.get_obs()) predict_movement_int, predict_q_value = self.deep_q.predict_movement( curr_state, epsilon) act = self.convert_act(predict_movement_int) # ... and observe the action observation, reward, done, _ = env.step(act) new_state = self.convert_obs(env.get_obs()) self.replay_buffer.add(curr_state, predict_movement_int, reward, done, new_state) self.action_history.append(predict_movement_int) self.reward_history.append(reward) total_reward += reward # reset the environment if done: env.reset() reset_count += 1 # Start training the network when the replay buffer is large enough to sample batches from if iteration > BATCH_SIZE: s_batch, a_batch, r_batch, d_batch, s2_batch = self.replay_buffer.sample( BATCH_SIZE) current_loss = self.deep_q.train(s_batch, a_batch, r_batch, d_batch, s2_batch) self.deep_q.target_train() # Save the network every 100 iterations if iteration % 100 == 99: self.smallest_loss = current_loss print("Saving Network, current loss:", current_loss) self.deep_q.save_network(network_path) # Write to tensorboard with tf_writer.as_default(): tf.summary.scalar("loss", current_loss, iteration) tf.summary.scalar("action", predict_movement_int, iteration) tf.summary.scalar("reward", reward, iteration) tf.summary.scalar("max q-value", predict_q_value, iteration) env.close()
# # initialize report report = {} for metric in config.METRICS_TO_EVAL: report[metric] = {"dashboards": {}, "monitors": {}} # generate dashboard report from file if os.path.isfile(config.DB_CACHE_PATH): print("Cache file found, loading dashboards from", config.DB_CACHE_PATH) with open(config.DB_CACHE_PATH, "r") as file: db_count = file.readline() print("db_count:", db_count) for i, line in enumerate(file): search_dashboard(json.loads(line), report) progress_bar.print_progress(i + 1, db_count, bar_length=50) # generate dashboard report from API calls else: print("No cache file found, writing dashboard API results to", config.DB_CACHE_PATH) all_dashboards = api.Dashboard.get_all() db_ids = [db['id'] for db in all_dashboards.get("dashboards")] with open(config.DB_CACHE_PATH, "w") as file: db_count = len(db_ids) file.write(str(db_count) + "\n") for i, db_id in enumerate(db_ids): db_response = api.Dashboard.get(db_id) search_dashboard(db_response, report)
num_states = convert_obs(obs).shape[0] num_actions = 191 # Specific for TopologyChangeAction on case 14 print('State space size:', num_states) print('Action space size:', num_actions) converter = IdToAct(env.action_space) converter.init_converter() states = np.zeros((n, num_states)) rewards = np.zeros((n, num_actions)) cum_reward = 0. reset_count = 0 start_time = time.time() # Generate n samples ... for i in range(n): print_progress(i+1, n, prefix='Sample {}/{}'.format(i+1, n), suffix='Episode count: {}'.format(reset_count)) states[i] = convert_obs(obs) st = time.time() # ... by simulating all actions and storing the rewards for act_id in range(num_actions): act = converter.convert_act(act_id) _, reward, _, _ = obs.simulate(act) rewards[i, act_id] = reward my_act = np.argmax(rewards[i]) obs, reward, done, _ = env.step(converter.convert_act(int(my_act))) # Reset environment when game over state is reached if done: reset_count += 1
self.astar_solved = astar_solved self.astar_nodes = astar_nodes # Get all puzzle files from the puzzle directory puzzle_directories = ["./puzzles_iter_5x5", "./puzzles_iter_7x7"] puzzles = [] for puzzle_directory in puzzle_directories: for filename in os.listdir(puzzle_directory): if filename.endswith(".txt"): puzzles.append(puzzle_directory + "/" + filename) # Progress bar total_puzzles = len(puzzles) i = 0 print_progress(i, total_puzzles, prefix="Puzzles solved") # Test algorithms results = [] for input_file in puzzles: start_board = load_map(input_file) # Uninformed time_start = process_time() node_i, depth, tree_i, visit_i = iterative_dfs(start_board) time_i = process_time() - time_start # Informed time_start = process_time() node_a, tree_a, visit_a = a_star(start_board) time_a = process_time() - time_start # Saving Results res = Result()
def simulate(self): """ Section 7 - UWG main section self.N # Total hours in simulation self.ph # per hour self.dayType # 3=Sun, 2=Sat, 1=Weekday self.ceil_time_step # simulation timestep (dt) fitted to weather file timestep # Output of object instance vector self.WeatherData # Nx1 vector of forc instance self.UCMData # Nx1 vector of UCM instance self.UBLData # Nx1 vector of UBL instance self.RSMData # Nx1 vector of RSM instance self.USMData # Nx1 vector of USM instance """ self.N = int(self.simTime.days * 24) # total number of hours in simulation n = 0 # weather time step counter self.ph = self.simTime.dt / 3600. # dt (simulation time step) in hours # Data dump variables time = range(self.N) self.WeatherData = [None for x in xrange(self.N)] self.UCMData = [None for x in xrange(self.N)] self.UBLData = [None for x in xrange(self.N)] self.RSMData = [None for x in xrange(self.N)] self.USMData = [None for x in xrange(self.N)] print '\nSimulating new temperature and humidity values for {} days from {}/{}.\n'.format( int(self.nDay), int(self.Month), int(self.Day)) self.logger.info("Start simulation") # Start progress bar at zero progress_bar.print_progress(0, 100.0, prefix="Progress:", bar_length=25) for it in range( 1, self.simTime.nt, 1 ): # for every simulation time-step (i.e 5 min) defined by uwg # Update water temperature (estimated) if self.is_near_zero(self.nSoil): self.forc.deepTemp = sum(self.forcIP.temp) / float( len(self.forcIP.temp) ) # for BUBBLE/CAPITOUL/Singapore only self.forc.waterTemp = sum(self.forcIP.temp) / float( len(self.forcIP.temp) ) - 10. # for BUBBLE/CAPITOUL/Singapore only else: self.forc.deepTemp = self.Tsoil[ self.soilindex1][self.simTime.month - 1] #soil temperature by depth, by month self.forc.waterTemp = self.Tsoil[2][self.simTime.month - 1] # There's probably a better way to update the weather... self.simTime.UpdateDate() self.logger.info("\n{0} m={1}, d={2}, h={3}, s={4}".format( __name__, self.simTime.month, self.simTime.day, self.simTime.secDay / 3600., self.simTime.secDay)) self.ceil_time_step = int( math.ceil(it * self.ph) ) - 1 # simulation time increment raised to weather time step # minus one to be consistent with forcIP list index # Updating forcing instance self.forc.infra = self.forcIP.infra[ self. ceil_time_step] # horizontal Infrared Radiation Intensity (W m-2) self.forc.wind = max(self.forcIP.wind[self.ceil_time_step], self.geoParam.windMin) # wind speed (m s-1) self.forc.uDir = self.forcIP.uDir[ self.ceil_time_step] # wind direction self.forc.hum = self.forcIP.hum[ self.ceil_time_step] # specific humidty (kg kg-1) self.forc.pres = self.forcIP.pres[ self.ceil_time_step] # Pressure (Pa) self.forc.temp = self.forcIP.temp[ self.ceil_time_step] # air temperature (C) self.forc.rHum = self.forcIP.rHum[ self.ceil_time_step] # Relative humidity (%) self.forc.prec = self.forcIP.prec[ self.ceil_time_step] # Precipitation (mm h-1) self.forc.dif = self.forcIP.dif[ self. ceil_time_step] # horizontal solar diffuse radiation (W m-2) self.forc.dir = self.forcIP.dir[ self.ceil_time_step] # normal solar direct radiation (W m-2) self.UCM.canHum = copy.copy( self.forc.hum) # Canyon humidity (absolute) same as rural # Update solar flux self.solar = SolarCalcs(self.UCM, self.BEM, self.simTime, self.RSM, self.forc, self.geoParam, self.rural) self.rural, self.UCM, self.BEM = self.solar.solarcalcs() # Update building & traffic schedule # Assign day type (1 = weekday, 2 = sat, 3 = sun/other) if self.is_near_zero(self.simTime.julian % 7): self.dayType = 3 # Sunday elif self.is_near_zero(self.simTime.julian % 7 - 6.): self.dayType = 2 # Saturday else: self.dayType = 1 # Weekday # Update anthropogenic heat load for each hour (building & UCM) self.UCM.sensAnthrop = self.sensAnth * ( self.SchTraffic[self.dayType - 1][self.simTime.hourDay]) # Update the energy components for building types defined in initialize.uwg for i in xrange(len(self.BEM)): # Set temperature self.BEM[i].building.coolSetpointDay = self.Sch[i].Cool[ self.dayType - 1][self.simTime. hourDay] + 273.15 # add from temperature schedule for cooling self.BEM[i].building.coolSetpointNight = self.BEM[ i].building.coolSetpointDay self.BEM[i].building.heatSetpointDay = self.Sch[i].Heat[ self.dayType - 1][self.simTime. hourDay] + 273.15 # add from temperature schedule for heating self.BEM[i].building.heatSetpointNight = self.BEM[ i].building.heatSetpointDay # Internal Heat Load Schedule (W/m^2 of floor area for Q) self.BEM[i].Elec = self.Sch[i].Qelec * self.Sch[i].Elec[ self.dayType - 1][self.simTime.hourDay] # Qelec x elec fraction for day self.BEM[i].Light = self.Sch[i].Qlight * self.Sch[i].Light[ self.dayType - 1][self.simTime.hourDay] # Qlight x light fraction for day self.BEM[i].Nocc = self.Sch[i].Nocc * self.Sch[i].Occ[ self.dayType - 1][self.simTime. hourDay] # Number of occupants x occ fraction for day self.BEM[i].Qocc = self.sensOcc * (1 - self.LatFOcc) * self.BEM[ i].Nocc # Sensible Q occupant * fraction occupant sensible Q * number of occupants # SWH and ventilation schedule self.BEM[i].SWH = self.Sch[i].Vswh * self.Sch[i].SWH[ self.dayType - 1][self.simTime. hourDay] # litres per hour x SWH fraction for day self.BEM[i].building.vent = self.Sch[ i].Vent # m^3/s/m^2 of floor self.BEM[i].Gas = self.Sch[i].Qgas * self.Sch[i].Gas[ self.dayType - 1][self.simTime. hourDay] # Gas Equip Schedule, per m^2 of floor # This is quite messy, should update # Update internal heat and corresponding fractional loads intHeat = self.BEM[i].Light + self.BEM[i].Elec + self.BEM[ i].Qocc self.BEM[ i].building.intHeatDay = intHeat # W/m2 from light, electricity, occupants self.BEM[i].building.intHeatNight = intHeat self.BEM[i].building.intHeatFRad = ( self.RadFLight * self.BEM[i].Light + self.RadFEquip * self.BEM[i].Elec ) / intHeat # fraction of radiant heat from light and equipment of whole internal heat self.BEM[ i].building.intHeatFLat = self.LatFOcc * self.sensOcc * self.BEM[ i].Nocc / intHeat # fraction of latent heat (from occupants) of whole internal heat # Update envelope temperature layers self.BEM[i].T_wallex = self.BEM[i].wall.layerTemp[0] self.BEM[i].T_wallin = self.BEM[i].wall.layerTemp[-1] self.BEM[i].T_roofex = self.BEM[i].roof.layerTemp[0] self.BEM[i].T_roofin = self.BEM[i].roof.layerTemp[-1] # Update rural heat fluxes & update vertical diffusion model (VDM) self.rural.infra = self.forc.infra - self.rural.emissivity * self.SIGMA * self.rural.layerTemp[ 0]**4. # Infrared radiation from rural road self.rural.SurfFlux(self.forc, self.geoParam, self.simTime, self.forc.hum, self.forc.temp, self.forc.wind, 2., 0.) self.RSM.VDM(self.forc, self.rural, self.geoParam, self.simTime) # Calculate urban heat fluxes, update UCM & UBL self.UCM, self.UBL, self.BEM = urbflux(self.UCM, self.UBL, self.BEM, self.forc, self.geoParam, self.simTime, self.RSM) self.UCM.UCModel(self.BEM, self.UBL.ublTemp, self.forc, self.geoParam) self.UBL.UBLModel(self.UCM, self.RSM, self.rural, self.forc, self.geoParam, self.simTime) """ # Experimental code to run diffusion model in the urban area # N.B Commented out in python UWG because computed wind speed in # urban VDM: y = =0.84*ln((2-x/20)/0.51) results in negative log # for building heights >= 40m. Uroad = copy.copy(self.UCM.road) Uroad.sens = copy.copy(self.UCM.sensHeat) Uforc = copy.copy(self.forc) Uforc.wind = copy.copy(self.UCM.canWind) Uforc.temp = copy.copy(self.UCM.canTemp) self.USM.VDM(Uforc,Uroad,self.geoParam,self.simTime) """ self.logger.info("dbT = {}".format(self.UCM.canTemp - 273.15)) if n > 0: logging.info("dpT = {}".format(self.UCM.Tdp)) logging.info("RH = {}".format(self.UCM.canRHum)) if self.is_near_zero(self.simTime.secDay % self.simTime.timePrint) and n < self.N: self.logger.info("{0} ----sim time step = {1}----\n\n".format( __name__, n)) self.WeatherData[n] = copy.copy(self.forc) _Tdb, _w, self.UCM.canRHum, _h, self.UCM.Tdp, _v = psychrometrics( self.UCM.canTemp, self.UCM.canHum, self.forc.pres) self.UBLData[n] = copy.copy(self.UBL) self.UCMData[n] = copy.copy(self.UCM) self.RSMData[n] = copy.copy(self.RSM) self.logger.info("dbT = {}".format(self.UCMData[n].canTemp - 273.15)) self.logger.info("dpT = {}".format(self.UCMData[n].Tdp)) self.logger.info("RH = {}".format(self.UCMData[n].canRHum)) # Print progress bar sim_it = round((it / float(self.simTime.nt)) * 100.0, 1) progress_bar.print_progress(sim_it, 100.0, prefix="Progress:", bar_length=25) n += 1