def parser():
def chunks(l, n):
list = []
for i in range(0, len(l), n):
list.append(l[i:i+n])
return list
url = 'http://www.banki.ru/banks/memory/?PAGEN_1='
site = urllib.request.urlopen(url)
html = site.read().decode('windows-1251')
n_pages = round(int(re.findall(r'totalItems:(.*?);', str(html), re.S)[0]) / 50.0 + 0.48) + 1
table = re.findall(r'<table class="standard-table standard-table--row-highlight">(.*?)</table>', str(html), re.S)
thead = re.findall(r'<th>(.*?)(?: <nobr>.*?</nobr>)?</th>', str(table))
i = 0
for th in thead:
thead[i] = th.replace(' ', '_')
i += 1
trs = re.findall(r'<td[^>]*>(?:<strong><a[^>]*>)?(.*?)(?:</a></strong>)?</td>', str(table))
print_progress(0, n_pages, prefix='Формирую файл с отозванными банками:', suffix='Выполнено', barLength=40)
for i in range(2, n_pages):
cur_url = url + str(i)
site = urllib.request.urlopen(cur_url)
html = site.read().decode('windows-1251')
table = re.findall(r'<table class="standard-table standard-table--row-highlight">(.*?)</table>', str(html), re.S)
trs.extend(re.findall(r'<td[^>]*>(?:<strong><a[^>]*>)?(.*?)(?:</a></strong>)?</td>', str(table)))
print_progress(i, n_pages, prefix='Формирую файл с отозванными банками:', suffix='Выполнено', barLength=40)
defunct = pd.DataFrame(data=chunks(trs, len(thead)), columns=thead)
return defunct
def download_images(self, directory):
print("[+] Downloading images")
html = requests.get('https://www.google.com/search?q=' +
self.get_args().term + '&source=lnms&tbm=isch')
soup = BeautifulSoup(html.text, "html.parser")
links = soup.find_all("img")
print_progress(links)
i = 0
for a in links:
if i < int(self.get_args().volume):
_download(directory, a['src'])
i += 1
# if fewer tweets found than max count, use that number
totlen = len(alltweets)
else:
# otherwise, process maxcCount at most
totlen = maxCount
searchresults = []
if alltweets:
for i, tweet in enumerate(alltweets):
# this searches on tweet at a time
searchresults.extend(tweetsearcher.searchTweet(tweet))
# count in progress_bar is i+1 because we start at zero and
# this should not be zero-based counter
if showProgressBar:
progress_bar.print_progress(i + 1, totlen)
if i == totlen:
break
# send email if search results come back
if searchresults:
# format message to send
msg = ''
for sr in searchresults:
url = 'https://twitter.com/' +\
sr.reference.user.screen_name +\
'/status/' +\
sr.reference.id_str
]
_, acc, loss = sess.run(
[train_step, accuracy, cross_entropy],
feed_dict={
x: batch[0],
y: batch[1],
keep_prob: FLAGS.keep_prob,
is_training: True
})
acc *= 100
num_iter = max_size * 10
progress = (
(epoch_num * max_size + num) % num_iter + 1) / num_iter
num_ep = epoch_num + 1
print_progress(progress, num_ep, loss, acc)
if FLAGS.online_test and (epoch_num + 1) % 10 == 0:
print(' ' * 12,
'Online-Testing ========== (。・`ω´・) ========')
imgs_t, labels_t = test_data.test_image.reshape(
(-1, 90, 75)), test_data.test_labels
test_acc, test_loss = sess.run([accuracy, cross_entropy],
feed_dict={
x: imgs_t,
y: labels_t,
keep_prob: 1.0,
is_training: False
})
test_acc *= 100
print(
test_summary_dir = os.path.join(FLAGS.events_path, 'summaries', 'test')
test_summary_writer = tf.summary.FileWriter(test_summary_dir)
test_summary_writer.add_graph(sess.graph)
# 随机梯度下降进行训练
for num_epoch in range(FLAGS.num_epochs):
image_batch, label_batch = mnist.train.next_batch(FLAGS.batch_size)
batch_loss, batch_acc, _, batch_summ, num_train, lr = sess.run([loss_tensor, accuracy, train_op, summary_train, global_step, learning_rate],
feed_dict={image_place: image_batch, label_place: label_batch,
keep_prob: FLAGS.keep_prob})
batch_acc *= 100
progress = float(num_epoch % FLAGS.num_batch + 1) / FLAGS.num_batch
num_epoch_batch = num_epoch // FLAGS.num_batch + 1
print_progress(progress, num_epoch_batch, batch_loss, batch_acc)
# print('Epoch '+str(num_epoch+1) + ', learning rate is %.4f' % lr + ', train accuracy is '+'{:.2f}%.'.format(batch_acc))
# print('Epoch '+str(num_epoch+1) + ', learning rate is %.4f' % lr + ', train loss is '+ '{:.4f}, '.format(batch_loss) + \
# 'accuracy is '+'{:.2f}%.'.format(batch_acc))
train_summary_writer.add_summary(batch_summ, num_epoch)
checkpoints_prefix = 'model.ckpt'
if (num_epoch+1) % FLAGS.num_batch == 0:
print('Learning rate is %.4f' % lr + ', train accuracy is '+'{:.2f}%.'.format(batch_acc))
if (batch_loss <= min_cross) & (batch_acc > max_acc): # 按照要求保存网络模型
min_cross = batch_loss
max_acc = batch_acc
saver.save(sess, os.path.join(FLAGS.checkpoints_path, checkpoints_prefix), global_step=num_epoch+1)
# print("\033[0;31;40m\tModel restored ... ...\033[0m\n")
printRed("Model restored ... ...\n")
print('\n')
nn = DeepQ(num_actions, num_states, lr=IL_LEARNING_RATE)
# Load the samples we want to train on
states = np.load(os.path.join(
'generated_data', 'states_{}_{}_{}_{}.npy'.format(path_grid, n, num_states,
run_id)),
allow_pickle=True)
rewards = np.load(os.path.join(
'generated_data', 'rewards_{}_{}_{}_{}.npy'.format(path_grid, n,
num_actions, run_id)),
allow_pickle=True)
loss = 0
for episode in range(NUM_EPISODES):
print_progress(episode,
NUM_EPISODES,
prefix='Episode {}/{}'.format(episode, NUM_EPISODES))
# Train the network on batches of the generated samples
for iteration in range(iterations):
start_index = iteration * IL_BATCH_SIZE
end_index = start_index + IL_BATCH_SIZE
state_batch = states[start_index:end_index]
target_batch = rewards[start_index:end_index]
loss = nn.model.train_on_batch(state_batch, target_batch)
# print("We had an imitation loss equal to ", loss)
network_path = os.path.join('saved_networks', 'imitation_learning',
'{}_{}_{}_il'.format(path_grid, n, run_id))
nn.save_network(network_path)
def train(self, env, num_iterations=10000, network_path=None):
""" Train the agent.
The agent runs in the environment for a specified number of iterations. When the done state is reached the
environment is reset. When their are enough samples in the replay buffer. Training of the network starts.
"""
# Initialize Tensorboard writer
log_path = os.path.join('logs', 'train',
self.id + '_' + str(time.time()))
tf_writer = tf.summary.create_file_writer(log_path)
transformed_observation = self.convert_obs(env.reset())
if self.deep_q is None:
self.init_deep_q(transformed_observation)
epsilon = INITIAL_EPSILON
epsilon_decay = (INITIAL_EPSILON - FINAL_EPSILON) / num_iterations
total_reward = 0
reset_count = 0
current_loss = np.inf
for iteration in range(num_iterations):
print_progress(iteration + 1,
num_iterations,
prefix='Step {}/{}'.format(iteration + 1,
num_iterations),
suffix='Episode count: {}'.format(reset_count))
# Decay epsilon over time
epsilon -= epsilon_decay
# Predict the next step ...
curr_state = self.convert_obs(env.get_obs())
predict_movement_int, predict_q_value = self.deep_q.predict_movement(
curr_state, epsilon)
act = self.convert_act(predict_movement_int)
# ... and observe the action
observation, reward, done, _ = env.step(act)
new_state = self.convert_obs(env.get_obs())
self.replay_buffer.add(curr_state, predict_movement_int, reward,
done, new_state)
self.action_history.append(predict_movement_int)
self.reward_history.append(reward)
total_reward += reward
# reset the environment
if done:
env.reset()
reset_count += 1
# Start training the network when the replay buffer is large enough to sample batches from
if iteration > BATCH_SIZE:
s_batch, a_batch, r_batch, d_batch, s2_batch = self.replay_buffer.sample(
BATCH_SIZE)
current_loss = self.deep_q.train(s_batch, a_batch, r_batch,
d_batch, s2_batch)
self.deep_q.target_train()
# Save the network every 100 iterations
if iteration % 100 == 99:
self.smallest_loss = current_loss
print("Saving Network, current loss:", current_loss)
self.deep_q.save_network(network_path)
# Write to tensorboard
with tf_writer.as_default():
tf.summary.scalar("loss", current_loss, iteration)
tf.summary.scalar("action", predict_movement_int, iteration)
tf.summary.scalar("reward", reward, iteration)
tf.summary.scalar("max q-value", predict_q_value, iteration)
env.close()
# # initialize report
report = {}
for metric in config.METRICS_TO_EVAL:
report[metric] = {"dashboards": {}, "monitors": {}}
# generate dashboard report from file
if os.path.isfile(config.DB_CACHE_PATH):
print("Cache file found, loading dashboards from",
config.DB_CACHE_PATH)
with open(config.DB_CACHE_PATH, "r") as file:
db_count = file.readline()
print("db_count:", db_count)
for i, line in enumerate(file):
search_dashboard(json.loads(line), report)
progress_bar.print_progress(i + 1, db_count, bar_length=50)
# generate dashboard report from API calls
else:
print("No cache file found, writing dashboard API results to",
config.DB_CACHE_PATH)
all_dashboards = api.Dashboard.get_all()
db_ids = [db['id'] for db in all_dashboards.get("dashboards")]
with open(config.DB_CACHE_PATH, "w") as file:
db_count = len(db_ids)
file.write(str(db_count) + "\n")
for i, db_id in enumerate(db_ids):
db_response = api.Dashboard.get(db_id)
search_dashboard(db_response, report)
num_states = convert_obs(obs).shape[0]
num_actions = 191 # Specific for TopologyChangeAction on case 14
print('State space size:', num_states)
print('Action space size:', num_actions)
converter = IdToAct(env.action_space)
converter.init_converter()
states = np.zeros((n, num_states))
rewards = np.zeros((n, num_actions))
cum_reward = 0.
reset_count = 0
start_time = time.time()
# Generate n samples ...
for i in range(n):
print_progress(i+1, n, prefix='Sample {}/{}'.format(i+1, n), suffix='Episode count: {}'.format(reset_count))
states[i] = convert_obs(obs)
st = time.time()
# ... by simulating all actions and storing the rewards
for act_id in range(num_actions):
act = converter.convert_act(act_id)
_, reward, _, _ = obs.simulate(act)
rewards[i, act_id] = reward
my_act = np.argmax(rewards[i])
obs, reward, done, _ = env.step(converter.convert_act(int(my_act)))
# Reset environment when game over state is reached
if done:
reset_count += 1
self.astar_solved = astar_solved
self.astar_nodes = astar_nodes
# Get all puzzle files from the puzzle directory
puzzle_directories = ["./puzzles_iter_5x5", "./puzzles_iter_7x7"]
puzzles = []
for puzzle_directory in puzzle_directories:
for filename in os.listdir(puzzle_directory):
if filename.endswith(".txt"):
puzzles.append(puzzle_directory + "/" + filename)
# Progress bar
total_puzzles = len(puzzles)
i = 0
print_progress(i, total_puzzles, prefix="Puzzles solved")
# Test algorithms
results = []
for input_file in puzzles:
start_board = load_map(input_file)
# Uninformed
time_start = process_time()
node_i, depth, tree_i, visit_i = iterative_dfs(start_board)
time_i = process_time() - time_start
# Informed
time_start = process_time()
node_a, tree_a, visit_a = a_star(start_board)
time_a = process_time() - time_start
# Saving Results
res = Result()
def simulate(self):
""" Section 7 - UWG main section
self.N # Total hours in simulation
self.ph # per hour
self.dayType # 3=Sun, 2=Sat, 1=Weekday
self.ceil_time_step # simulation timestep (dt) fitted to weather file timestep
# Output of object instance vector
self.WeatherData # Nx1 vector of forc instance
self.UCMData # Nx1 vector of UCM instance
self.UBLData # Nx1 vector of UBL instance
self.RSMData # Nx1 vector of RSM instance
self.USMData # Nx1 vector of USM instance
"""
self.N = int(self.simTime.days *
24) # total number of hours in simulation
n = 0 # weather time step counter
self.ph = self.simTime.dt / 3600. # dt (simulation time step) in hours
# Data dump variables
time = range(self.N)
self.WeatherData = [None for x in xrange(self.N)]
self.UCMData = [None for x in xrange(self.N)]
self.UBLData = [None for x in xrange(self.N)]
self.RSMData = [None for x in xrange(self.N)]
self.USMData = [None for x in xrange(self.N)]
print '\nSimulating new temperature and humidity values for {} days from {}/{}.\n'.format(
int(self.nDay), int(self.Month), int(self.Day))
self.logger.info("Start simulation")
# Start progress bar at zero
progress_bar.print_progress(0,
100.0,
prefix="Progress:",
bar_length=25)
for it in range(
1, self.simTime.nt, 1
): # for every simulation time-step (i.e 5 min) defined by uwg
# Update water temperature (estimated)
if self.is_near_zero(self.nSoil):
self.forc.deepTemp = sum(self.forcIP.temp) / float(
len(self.forcIP.temp)
) # for BUBBLE/CAPITOUL/Singapore only
self.forc.waterTemp = sum(self.forcIP.temp) / float(
len(self.forcIP.temp)
) - 10. # for BUBBLE/CAPITOUL/Singapore only
else:
self.forc.deepTemp = self.Tsoil[
self.soilindex1][self.simTime.month -
1] #soil temperature by depth, by month
self.forc.waterTemp = self.Tsoil[2][self.simTime.month - 1]
# There's probably a better way to update the weather...
self.simTime.UpdateDate()
self.logger.info("\n{0} m={1}, d={2}, h={3}, s={4}".format(
__name__, self.simTime.month, self.simTime.day,
self.simTime.secDay / 3600., self.simTime.secDay))
self.ceil_time_step = int(
math.ceil(it * self.ph)
) - 1 # simulation time increment raised to weather time step
# minus one to be consistent with forcIP list index
# Updating forcing instance
self.forc.infra = self.forcIP.infra[
self.
ceil_time_step] # horizontal Infrared Radiation Intensity (W m-2)
self.forc.wind = max(self.forcIP.wind[self.ceil_time_step],
self.geoParam.windMin) # wind speed (m s-1)
self.forc.uDir = self.forcIP.uDir[
self.ceil_time_step] # wind direction
self.forc.hum = self.forcIP.hum[
self.ceil_time_step] # specific humidty (kg kg-1)
self.forc.pres = self.forcIP.pres[
self.ceil_time_step] # Pressure (Pa)
self.forc.temp = self.forcIP.temp[
self.ceil_time_step] # air temperature (C)
self.forc.rHum = self.forcIP.rHum[
self.ceil_time_step] # Relative humidity (%)
self.forc.prec = self.forcIP.prec[
self.ceil_time_step] # Precipitation (mm h-1)
self.forc.dif = self.forcIP.dif[
self.
ceil_time_step] # horizontal solar diffuse radiation (W m-2)
self.forc.dir = self.forcIP.dir[
self.ceil_time_step] # normal solar direct radiation (W m-2)
self.UCM.canHum = copy.copy(
self.forc.hum) # Canyon humidity (absolute) same as rural
# Update solar flux
self.solar = SolarCalcs(self.UCM, self.BEM, self.simTime, self.RSM,
self.forc, self.geoParam, self.rural)
self.rural, self.UCM, self.BEM = self.solar.solarcalcs()
# Update building & traffic schedule
# Assign day type (1 = weekday, 2 = sat, 3 = sun/other)
if self.is_near_zero(self.simTime.julian % 7):
self.dayType = 3 # Sunday
elif self.is_near_zero(self.simTime.julian % 7 - 6.):
self.dayType = 2 # Saturday
else:
self.dayType = 1 # Weekday
# Update anthropogenic heat load for each hour (building & UCM)
self.UCM.sensAnthrop = self.sensAnth * (
self.SchTraffic[self.dayType - 1][self.simTime.hourDay])
# Update the energy components for building types defined in initialize.uwg
for i in xrange(len(self.BEM)):
# Set temperature
self.BEM[i].building.coolSetpointDay = self.Sch[i].Cool[
self.dayType -
1][self.simTime.
hourDay] + 273.15 # add from temperature schedule for cooling
self.BEM[i].building.coolSetpointNight = self.BEM[
i].building.coolSetpointDay
self.BEM[i].building.heatSetpointDay = self.Sch[i].Heat[
self.dayType -
1][self.simTime.
hourDay] + 273.15 # add from temperature schedule for heating
self.BEM[i].building.heatSetpointNight = self.BEM[
i].building.heatSetpointDay
# Internal Heat Load Schedule (W/m^2 of floor area for Q)
self.BEM[i].Elec = self.Sch[i].Qelec * self.Sch[i].Elec[
self.dayType -
1][self.simTime.hourDay] # Qelec x elec fraction for day
self.BEM[i].Light = self.Sch[i].Qlight * self.Sch[i].Light[
self.dayType -
1][self.simTime.hourDay] # Qlight x light fraction for day
self.BEM[i].Nocc = self.Sch[i].Nocc * self.Sch[i].Occ[
self.dayType -
1][self.simTime.
hourDay] # Number of occupants x occ fraction for day
self.BEM[i].Qocc = self.sensOcc * (1 - self.LatFOcc) * self.BEM[
i].Nocc # Sensible Q occupant * fraction occupant sensible Q * number of occupants
# SWH and ventilation schedule
self.BEM[i].SWH = self.Sch[i].Vswh * self.Sch[i].SWH[
self.dayType -
1][self.simTime.
hourDay] # litres per hour x SWH fraction for day
self.BEM[i].building.vent = self.Sch[
i].Vent # m^3/s/m^2 of floor
self.BEM[i].Gas = self.Sch[i].Qgas * self.Sch[i].Gas[
self.dayType -
1][self.simTime.
hourDay] # Gas Equip Schedule, per m^2 of floor
# This is quite messy, should update
# Update internal heat and corresponding fractional loads
intHeat = self.BEM[i].Light + self.BEM[i].Elec + self.BEM[
i].Qocc
self.BEM[
i].building.intHeatDay = intHeat # W/m2 from light, electricity, occupants
self.BEM[i].building.intHeatNight = intHeat
self.BEM[i].building.intHeatFRad = (
self.RadFLight * self.BEM[i].Light +
self.RadFEquip * self.BEM[i].Elec
) / intHeat # fraction of radiant heat from light and equipment of whole internal heat
self.BEM[
i].building.intHeatFLat = self.LatFOcc * self.sensOcc * self.BEM[
i].Nocc / intHeat # fraction of latent heat (from occupants) of whole internal heat
# Update envelope temperature layers
self.BEM[i].T_wallex = self.BEM[i].wall.layerTemp[0]
self.BEM[i].T_wallin = self.BEM[i].wall.layerTemp[-1]
self.BEM[i].T_roofex = self.BEM[i].roof.layerTemp[0]
self.BEM[i].T_roofin = self.BEM[i].roof.layerTemp[-1]
# Update rural heat fluxes & update vertical diffusion model (VDM)
self.rural.infra = self.forc.infra - self.rural.emissivity * self.SIGMA * self.rural.layerTemp[
0]**4. # Infrared radiation from rural road
self.rural.SurfFlux(self.forc, self.geoParam, self.simTime,
self.forc.hum, self.forc.temp, self.forc.wind,
2., 0.)
self.RSM.VDM(self.forc, self.rural, self.geoParam, self.simTime)
# Calculate urban heat fluxes, update UCM & UBL
self.UCM, self.UBL, self.BEM = urbflux(self.UCM, self.UBL,
self.BEM, self.forc,
self.geoParam, self.simTime,
self.RSM)
self.UCM.UCModel(self.BEM, self.UBL.ublTemp, self.forc,
self.geoParam)
self.UBL.UBLModel(self.UCM, self.RSM, self.rural, self.forc,
self.geoParam, self.simTime)
"""
# Experimental code to run diffusion model in the urban area
# N.B Commented out in python UWG because computed wind speed in
# urban VDM: y = =0.84*ln((2-x/20)/0.51) results in negative log
# for building heights >= 40m.
Uroad = copy.copy(self.UCM.road)
Uroad.sens = copy.copy(self.UCM.sensHeat)
Uforc = copy.copy(self.forc)
Uforc.wind = copy.copy(self.UCM.canWind)
Uforc.temp = copy.copy(self.UCM.canTemp)
self.USM.VDM(Uforc,Uroad,self.geoParam,self.simTime)
"""
self.logger.info("dbT = {}".format(self.UCM.canTemp - 273.15))
if n > 0:
logging.info("dpT = {}".format(self.UCM.Tdp))
logging.info("RH = {}".format(self.UCM.canRHum))
if self.is_near_zero(self.simTime.secDay %
self.simTime.timePrint) and n < self.N:
self.logger.info("{0} ----sim time step = {1}----\n\n".format(
__name__, n))
self.WeatherData[n] = copy.copy(self.forc)
_Tdb, _w, self.UCM.canRHum, _h, self.UCM.Tdp, _v = psychrometrics(
self.UCM.canTemp, self.UCM.canHum, self.forc.pres)
self.UBLData[n] = copy.copy(self.UBL)
self.UCMData[n] = copy.copy(self.UCM)
self.RSMData[n] = copy.copy(self.RSM)
self.logger.info("dbT = {}".format(self.UCMData[n].canTemp -
273.15))
self.logger.info("dpT = {}".format(self.UCMData[n].Tdp))
self.logger.info("RH = {}".format(self.UCMData[n].canRHum))
# Print progress bar
sim_it = round((it / float(self.simTime.nt)) * 100.0, 1)
progress_bar.print_progress(sim_it,
100.0,
prefix="Progress:",
bar_length=25)
n += 1
