def status(self, interval=1., timeout=-1, fo=sys.stdout):
"""Waits for the jobs, printing progress at regular intervals
Args:
interval (float): a time in seconds, after which to print the progress.
timeout (float): a time in seconds, after which to give up waiting.
fo (file): a file object to which the progress is printed.
"""
if self.ar is None:
return
if timeout is None:
timeout = -1
# Make sure to write the job results into the job objects.
self.wait(1e-3)
tic = time.time()
while not self.ar.ready() and (timeout < 0 or time.time() - tic <= timeout):
self.wait(interval)
clear_output(wait=True)
dt = datetime.timedelta(seconds=self.ar.elapsed)
fo.write('{}/{} tasks finished after {}'.format(self.ar.progress, len(self.ar), str(dt)))
fo.flush()
else:
fo.write('\n')
dt = datetime.timedelta(seconds=self.ar.elapsed)
clear_output(wait=True)
fo.write('{} tasks completed in {}\n'.format(len(self.ar), str(dt)))
fo.flush()
def __call__(self, percentage):
" Update the progress bar within the specified percent_range"
if self.start_time is None: self.start_time = time.time()
span = (self.percent_range[1]-self.percent_range[0])
percentage = self.percent_range[0] + ((percentage/100.0) * span)
if self.display == 'disabled': return
elif self.display == 'stdout':
if percentage==100 and self.elapsed_time:
elapsed = time.time() - self.start_time
if clear_output and not ipython2: clear_output()
if clear_output and ipython2: clear_output(wait=True)
sys.stdout.write('\r' + '100%% %s %02d:%02d:%02d'
% (self.label.lower(), elapsed//3600,
elapsed//60, elapsed%60))
return
else:
self._stdout_display(percentage)
return
if 'socket' not in self.cache:
self.cache['socket'] = self._get_socket()
if self.cache['socket'] is not None:
self.cache['socket'].send('%s|%s' % (percentage, self.label))
def df(self, value):
"""This allows for on the fly data updating so you only need
to create one BarPlot Dashboard and instantiate the widgets once"""
self._update_columns(list(value.columns.values))
self._df = value
self.update()
clear_output()
def print_update(message):
try:
clear_output()
except Exception:
pass
print message
sys.stdout.flush()
def CreateWeightHist(net, selectedLayers):
firstLayer = int(selectedLayers.split("->")[0])
weights = net["layers"][firstLayer]["Weights"]
n1 = int(weights["row"])
n2 = int(weights["cols"])
m = ROOT.TMatrixD(n1, n2+1)
vec = weights["data"]
for i in xrange(n1):
for j in xrange(n2):
m[i][j] = vec[j+n2*i]
bvec = net["layers"][firstLayer]["Biases"]["data"]
if n1!=len(bvec):
print("Something wrong.. Number of bias weights not equal with the neuron number ("+str(n1)+"!="+str(len(bvec))+")")
return
for i in xrange(n1):
m[i][n2] = bvec[i]
th2 = ROOT.TH2D(m)
th2.SetTitle("Weight map for DNN")
for i in xrange(n2):
th2.GetXaxis().SetBinLabel(i + 1, str(i))
th2.GetXaxis().SetBinLabel(n2+1, "B")
for i in xrange(n1):
th2.GetYaxis().SetBinLabel(i + 1, str(i))
th2.GetXaxis().SetTitle("Layer: "+str(firstLayer))
th2.GetYaxis().SetTitle("Layer: "+str(firstLayer+1))
th2.SetStats(0)
th2.SetMarkerSize(1.5)
th2.SetMarkerColor(0)
labelSize = 0.040
th2.GetXaxis().SetLabelSize(labelSize)
th2.GetYaxis().SetLabelSize(labelSize)
th2.LabelsOption("d")
th2.SetLabelOffset(0.011)
clear_output()
JPyInterface.JsDraw.Draw(th2, 'drawDNNMap')
def print_update(message):
"Print a message immediately; do not wait for current execution to finish."
try:
clear_output()
except Exception:
pass
print(message)
sys.stdout.flush()
def flush_ipython(self):
try:
clear_output()
except Exception:
# terminal IPython has no clear_output
pass
print('\r', end='')
sys.stdout.flush()
def BookDNN(self, loader, title="DNN"):
global __BookDNNHelper
def __bookDNN(optString):
self.BookMethod(loader, ROOT.TMVA.Types.kDNN, title, optString)
return
__BookDNNHelper = __bookDNN
clear_output()
JPyInterface.JsDraw.InsertCSS("NetworkDesigner.min.css")
JPyInterface.JsDraw.Draw("", "NetworkDesigner", True)
def animate_ipython(self, iter):
try:
clear_output()
except Exception:
# terminal IPython has no clear_output
pass
print '\r', self,
sys.stdout.flush()
self.update_iteration(iter + 1)
def refresh_ptcl(change=None, force=False):
"""
Refresh the current particle figure
Parameters :
------------
change: dictionary
Dictionary passed by the widget to a callback functions
whenever a change of a widget happens
(see docstring of ipywidgets.Widget.observe)
This is mainline a place holder ; not used in this function
force: bool
Whether to force the update
"""
# Determine whether to do the refresh
do_refresh = False
if self.avail_species is not None:
if force or ptcl_refresh_toggle.value:
do_refresh = True
# Do the refresh
if do_refresh:
plt.figure(ptcl_figure_button.value, figsize=figsize)
plt.clf()
# When working in inline mode, in an ipython notebook,
# clear the output (prevents the images from stacking
# in the notebook)
if 'inline' in matplotlib.get_backend():
clear_output()
if ptcl_use_button.value:
i_power = ptcl_magnitude_button.value
vmin = ptcl_range_button.value[0] * 10 ** i_power
vmax = ptcl_range_button.value[1] * 10 ** i_power
else:
vmin = None
vmax = None
if ptcl_yaxis_button.value == 'None':
# 1D histogram
self.get_particle(t=self.current_t, output=False,
var_list=[ptcl_xaxis_button.value],
select=ptcl_select_widget.to_dict(),
species=ptcl_species_button.value, plot=True,
vmin=vmin, vmax=vmax, cmap=ptcl_color_button.value,
nbins=ptcl_bins_button.value)
else:
# 2D histogram
self.get_particle(t=self.current_t, output=False,
var_list=[ptcl_xaxis_button.value,
ptcl_yaxis_button.value],
select=ptcl_select_widget.to_dict(),
species=ptcl_species_button.value, plot=True,
vmin=vmin, vmax=vmax, cmap=ptcl_color_button.value,
nbins=ptcl_bins_button.value)
def display_ipython(self):
try:
clear_output()
except Exception:
# terminal IPython has no clear_output
pass
print('\r',self,end="")
sys.stdout.flush()
time.sleep(0.001)
def progressBar(cell, starting , current_step, stopping):
progressing = float(current_step - starting) / (stopping - starting)
from IPython.core.display import clear_output
clear_output(wait = True)
import time
current_time = time.time()
print('[{:6.6f} second {:} {:}% {:}]'.format(current_time - cell.start_time
, int(10 * progressing) * '--'
, int(100 * progressing)
, int(10 - 10 * progressing) * '++'))
def clicked(b):
if treeSelector.value>tr.getNTrees():
treeSelector.value = tr.getNTrees()
clear_output()
toJs = {
"variables": variables,
"tree": tr.getTree(treeSelector.value)
}
json_str = json.dumps(toJs)
JPyInterface.JsDraw.Draw(json_str, "drawDecisionTree", True)
def animate_ipython(self, *args, **kwargs):
self.update(*args, **kwargs)
try:
clear_output()
except Exception:
# terminal IPython has no clear_output
pass
print('\r {}'.format(self.progressbar), end='')
sys.stdout.flush()
def print_termlogs(self):
termlogs = self.termlogs()
if self.has_ipython and not self.has_curses:
clear_output(wait=True)
else:
for i in range(len(self.data) + 1):
termlogs += "\033[A"
sys.stdout.write(termlogs)
sys.stdout.flush()
def wait_interactive(self, interval=1., timeout=None):
"""interactive wait, printing progress at regular intervals"""
N = len(self)
tic = time.time()
while not self.ready() and (timeout is None or time.time() - tic <= timeout):
self.wait(interval)
clear_output()
print "%4i/%i tasks finished after %4i s" % (self.progress, N, self.elapsed),
sys.stdout.flush()
print
print "done"
def update(self, percentage):
" Update the progress bar to the given percentage value "
if clear_output: clear_output()
percent_per_char = 100.0 / self.width
char_count = int(math.floor(percentage/percent_per_char) if percentage<100.0 else self.width)
blank_count = self.width - char_count
print '\r', "[%s%s] %0.1f%%" % (self.fill_char * char_count,
' '*len(self.fill_char)*blank_count,
percentage)
sys.stdout.flush()
time.sleep(0.0001)
def wait_for_kill(asyncres, view, updateInterval = 1.):
from IPython.core.display import clear_output
N = len(asyncres)
print "%4i/%i tasks finished after %4i s" % (asyncres.progress, N, asyncres.elapsed),
while not asyncres.ready():
asyncres.wait(updateInterval)
clear_output()
print "%4i/%i tasks finished after %4i s" % (asyncres.progress, N, asyncres.elapsed),
sys.stdout.flush()
print ""
view.shutdown()
print "done"
def wait_interactive(self, interval=1.0, timeout=-1):
"""interactive wait, printing progress at regular intervals"""
if timeout is None:
timeout = -1
N = len(self)
tic = time.time()
while not self.ready() and (timeout < 0 or time.time() - tic <= timeout):
self.wait(interval)
clear_output(wait=True)
print("%4i/%i tasks finished after %4i s" % (self.progress, N, self.elapsed), end="")
sys.stdout.flush()
print("\ndone")
def _stdout_display(self, percentage):
if clear_output and not ipython2: clear_output()
if clear_output and ipython2: clear_output(wait=True)
percent_per_char = 100.0 / self.width
char_count = int(math.floor(percentage/percent_per_char)
if percentage<100.0 else self.width)
blank_count = self.width - char_count
sys.stdout.write('\r' + "%s[%s%s] %0.1f%%"
% (self.label+':\n' if self.label else '',
self.fill_char * char_count,
' '*len(self.fill_char) * blank_count,
percentage))
sys.stdout.flush()
time.sleep(0.0001)
def wazzup_slaves(direct_view, n_lines=3):
while True:
from IPython.core.display import clear_output
clear_output()
wazzups = direct_view.apply(wazzup, n_lines).get()
for hostname, wazzup_val in zip(get_hostnames(direct_view).iteritems(), wazzups):
print "\x1b[31m"+str(hostname)+"\x1b[0m"
print "".join(wazzup_val)
print ""
sys.stdout.flush()
time.sleep(1)
def render(self, chart_type, data):
if chart_type == 'linechart':
chart = pygal.XY(height=self.height,
width=self.width,
x_label_rotation=45)
chart.title = self.title
for series_name in data.keys():
series = data[series_name]
chart.add(series_name, series)
elif chart_type == 'timechart':
chart = pygal.DateTimeLine(height=self.height,
width=self.width,
x_label_rotation=45,
x_value_formatter=moment.datetime_to_iso8601)
chart.title = self.title
for series_name in data.keys():
series = data[series_name]
chart.add(series_name, series)
elif chart_type == 'barchart':
chart = pygal.Bar(height=self.height, width=self.width)
chart.title = self.title
chart.x_labels = [category for (category, _) in data[list(data.keys())[0]]]
for series_name in data.keys():
series = [value for (_, value) in data[series_name]]
chart.add(series_name, series)
else:
raise FlumeException('unsupported chart type "%s" for pygal view'
% chart_type)
if is_ipython():
from IPython.core.display import display, HTML, clear_output
clear_output(wait=True)
display(HTML(chart.render()))
elif self.filename.endswith('.png'):
chart.render_to_png(self.filename)
elif self.filename.endswith('.html'):
with open(self.filename, 'w') as output:
output.write(HTML_TEMPLATE % chart.render())
def print_termlogs(self, training_state):
termlogs = self.termlogs(
step=training_state.step,
global_loss=training_state.global_loss,
global_acc=training_state.global_acc)
if self.has_ipython and not self.has_curses:
clear_output(wait=True)
else:
for i in range(len(self.data) + 1):
termlogs += "\033[A"
sys.stdout.write(termlogs)
sys.stdout.flush()
def update_ipython(self, value):
"Updates the progress bar to a new value."
assert 0 <= value <= self.maxval
self.currval = value
if not self._need_update() or self.finished:
return
if not self.start_time:
self.start_time = time.time()
self.seconds_elapsed = time.time() - self.start_time
self.prev_percentage = self.percentage()
if self.clear_output is True:
clear_output()
print "\r", self._format_line(),
sys.stdout.flush()
if value == self.maxval:
self.finished = True
def print_termlogs(self, training_state):
termlogs = self.termlogs(
step=training_state.step,
global_loss=training_state.global_loss,
global_acc=training_state.global_acc,
step_time=training_state.step_time_total)
if self.has_ipython and not CURSES_SUPPORTED:
clear_output(wait=True)
else:
for i in range(len(self.data) + 1):
termlogs += "\033[A"
sys.stdout.write(termlogs)
sys.stdout.flush()
def _stdout_display(self, percentage, display=True):
if clear_output and not ipython2: clear_output()
if clear_output and ipython2: clear_output(wait=True)
percent_per_char = 100.0 / self.width
char_count = int(math.floor(percentage/percent_per_char)
if percentage<100.0 else self.width)
blank_count = self.width - char_count
prefix = '\n' if len(self.current_progress) > 1 else ''
self.out = prefix + ("%s[%s%s] %0.1f%%" %
(self.label+':\n' if self.label else '',
self.fill_char * char_count,
' '*len(self.fill_char) * blank_count,
percentage))
if display:
sys.stdout.write(''.join([pg.out for pg in self.current_progress]))
sys.stdout.flush()
time.sleep(0.0001)
def update_display(self, force=False):
"""Updates display on the frontend.
Retrieves the latest execution status by querying CacheManager and updates
display on the fronend. The assumption is that there is only one pipeline in
a cell, because it clears up everything in the cell output every update
cycle.
Args:
force: (bool) whether to force updating when no stats change happens.
"""
with self._lock:
stats_updated = False
for pcoll_id, stats in self._pcollection_stats.items():
cache_label = stats['cache_label']
version = stats['version']
if force or not self._cache_manager.is_latest_version(
version, 'sample', cache_label):
pcoll_list, version = self._cache_manager.read('sample', cache_label)
stats['sample'] = pcoll_list
stats['version'] = version
stats_updated = True
if pcoll_id in self._analyzer.tl_referenced_pcoll_ids():
self._text_to_print[pcoll_id] = (str(
'%s produced %s' % (
self._producers[pcoll_id],
interactive_pipeline_graph.format_sample(pcoll_list, 5))))
if force or stats_updated:
self._pipeline_graph.update_pcollection_stats(self._pcollection_stats)
if IPython:
from IPython.core import display
display.clear_output(True)
rendered_graph = self._renderer.render_pipeline_graph(
self._pipeline_graph)
display.display(display.HTML(rendered_graph))
_display_progress('Running...')
for text in self._text_to_print.values():
if text != "":
_display_progress(text)
def refresh_field(change=None, force=False):
"""
Refresh the current field figure
Parameters :
------------
change: dictionary
Dictionary passed by the widget to a callback functions
whenever a change of a widget happens
(see docstring of ipywidgets.Widget.observe)
This is mainline a place holder ; not used in this function
force: bool
Whether to force the update
"""
# Determine whether to do the refresh
do_refresh = False
if (self.avail_fields is not None):
if force or fld_refresh_toggle.value:
do_refresh = True
# Do the refresh
if do_refresh:
plt.figure(fld_figure_button.value, figsize=figsize)
plt.clf()
# When working in inline mode, in an ipython notebook,
# clear the output (prevents the images from stacking
# in the notebook)
if 'inline' in matplotlib.get_backend():
clear_output()
if fld_use_button.value:
i_power = fld_magnitude_button.value
vmin = fld_range_button.value[0] * 10 ** i_power
vmax = fld_range_button.value[1] * 10 ** i_power
else:
vmin = None
vmax = None
self.get_field(t=self.current_t, output=False, plot=True,
field=fieldtype_button.value, coord=coord_button.value,
m=convert_to_int(mode_button.value),
slicing=slicing_button.value, theta=theta_button.value,
slicing_dir=slicing_dir_button.value,
vmin=vmin, vmax=vmax, cmap=fld_color_button.value)
def notify_complete(self, del_time, return_name, return_shape):
"""Execute JavaScipt code that shows when a query is complete."""
pretty_time = time.strftime('%I:%M:%S %p %Z')
cell_id = int(time.time())
# fixes issue where browser pops up on reload
# without this, javascript is executed every
# time notebook is loaded
cur_time = (1+time.time())*1000.
string_args = {
'pretty_time': pretty_time,
'del_time': del_time,
'cell_id': cell_id,
'return_name': return_name,
'return_shape': return_shape,
'cur_time': cur_time
}
add_cell_id = '<a id="{cell_id}"></a>\n'.format(cell_id=cell_id)
alert_str = '''
<script>
function notifyMe() {{
if (Notification.permission !== "granted")
Notification.requestPermission();
else {{
var notification = new Notification('Query Finished in {del_time:2.2f} m', {{
icon: 'https://raw.githubusercontent.com/crawles/Logos/master/jupyter.png?raw=true',
body: "Name: {return_name}\\nDimensions: {return_shape}",
}});
notification.onclick = function () {{
document.getElementById('{cell_id}').scrollIntoView();
}};
}}
}}
var isIE = /*@cc_on!@*/false || !!document.documentMode;
// prevents notifications from popping up when notebook is re-opened
if (Date.now() < {cur_time} && !isIE) {{
notifyMe(); }};
</script>
'''.format(**string_args)
html_str = add_cell_id + alert_str
self.shell.displayhook(HTML(html_str))
# clear output bc displayhook creates [Out] cell, cluttering notebook
clear_output()
def CreateWeightHist(net, selectedLayers):
weightStartIndex = 0
numberOfWeights = 0
firstLayer=int(selectedLayers.split("->")[0])
for i in xrange(firstLayer):
n1=int(net["layers"][i-1]["Nodes"])
n2=int(net["layers"][i]["Nodes"])
weightStartIndex += int(n1*n2)
n1 = 1
n2 = 1
if firstLayer>0:
n1 = int(net["layers"][firstLayer-1]["Nodes"])
n2 = int(net["layers"][firstLayer]["Nodes"])
else:
n2 = int(net["layers"][firstLayer]["Nodes"])
numberOfWeights = n1*n2
m = ROOT.TMatrixD(n1, n2)
for i in xrange(n1):
for j in xrange(n2):
idx = j+n2*i
if idx>numberOfWeights:
print("Something is wrong...")
continue
m[i][j] = float(net["synapses"]["synapses"][weightStartIndex+idx])
th2 = ROOT.TH2D(m)
th2.SetTitle("Weight map for DNN")
for i in xrange(n2):
th2.GetXaxis().SetBinLabel(i + 1, str(i))
for i in xrange(n1):
th2.GetYaxis().SetBinLabel(i + 1, str(i))
th2.GetXaxis().SetTitle("Layer: "+str(firstLayer+1))
th2.GetYaxis().SetTitle("Layer: "+str(firstLayer))
th2.SetStats(0)
th2.SetMarkerSize(1.5)
th2.SetMarkerColor(0)
labelSize = 0.040
th2.GetXaxis().SetLabelSize(labelSize)
th2.GetYaxis().SetLabelSize(labelSize)
th2.LabelsOption("d")
th2.SetLabelOffset(0.011)
clear_output()
JPyInterface.JsDraw.Draw(th2, 'drawDNNMap')
def __clear(self):
try:
clear_output()
except Exception:
"# terminal IPython has no clear_output"
pass
def create_features(self,
low_memory=True,
broke_age=True,
save_files=None,
skip_number=0,
stop_number=0):
'''Финальная функция, которая запускает все остальные. Подгружает датасеты и перерабатывает их в
train and test DataFrames. Low_memory предназначена для ПК с малым количеством оперативной памяти.
Информация из датасета по покупкам подгружается постепенно.
broke_age - включает простую модель, которая предсказывает возраст клиента и исправляет явно ошибочный возраст.
skip_number - число chunk , которое нужно пропустить. Применяется если часть файла уже обработана.
stop_number - чило chunks на котором остановиться подрузка файла.
Это нужно, чтобы обработать файл в несколько подходов, для малой оперативной памяти.
'''
self.low_memory = low_memory
start_time = time.time()
df_clients = pd.read_csv(self.clients_path)
train = pd.read_csv(self.train_path)
test = pd.read_csv(self.test_path)
df_features = self.features_processing_from_clients_df(df_clients)
if low_memory:
purchases_chunks = pd.read_csv(self.purchases_path,
chunksize=10**6)
features_set = pd.DataFrame()
data = []
count = 0
for chunk in purchases_chunks:
count += 1
if count <= skip_number:
clear_output()
print(f'Пропущен {count}-й chunk из {skip_number} chunks')
else:
print(f'Обрабатывается {count}-й chunk из {46} chunks')
chunk.fillna(0, inplace=True)
chunk['hour'] = pd.to_datetime(
chunk['transaction_datetime']).dt.hour
chunk['dayofweek'] = pd.to_datetime(
chunk['transaction_datetime']).dt.dayofweek
data += self.features_processing_from_purchases(chunk)
if count == stop_number:
break
print('Создается общий датафрейм...')
features_set = pd.DataFrame(data)
else:
#print('Создается общий датафрейм...')
purchases = pd.read_csv(self.purchases_path)
purchases['hour'] = pd.to_datetime(
purchases['transaction_datetime']).dt.hour
purchases['dayofweek'] = pd.to_datetime(
purchases['transaction_datetime']).dt.dayofweek
features_set = pd.DataFrame(
self.features_processing_from_purchases(purchases))
features_set = pd.merge(features_set, df_features, how='inner')
if broke_age:
features_set = self.predict_broke_age(features_set)
print('Создаются раздельные датафреймы...')
features_set_train = pd.merge(features_set, train, how='inner')
features_set_test = pd.merge(features_set, test, how='inner')
if save_files:
if stop_number != 0:
new_path = self.path + 'feat_sets/'
print(f'Данные сохраняются в директорию {new_path}...')
features_set_train.to_csv(new_path + 'features_set_train_to_' +
str(stop_number) + '.csv',
index=False)
features_set_test.to_csv(new_path + 'features_set_test_to_' +
str(stop_number) + '.csv',
index=False)
print(f'Файлы успешно сохранены в директорию {new_path}')
else:
print(f'Данные сохраняются в директорию {self.path}...')
features_set_train.to_csv(self.path + 'features_set_train.csv',
index=False)
features_set_test.to_csv(self.path + 'features_set_test.csv',
index=False)
print(f'Файлы успешно сохранены в директорию {self.path}')
print(
f'Весь процесс обработки данных занял {int(time.time()-start_time)} секунд'
)
print(
f'Тренировочный датасет состоит из обработанных данных {features_set_train.shape[0]} клиентов и {features_set_train.shape[1]-2} сгенерированных признаков'
)
print(
f'Тестовый датасет состоит из обработанных данных {features_set_test.shape[0]} клиентов и {features_set_test.shape[1]} сгенерированных признаков'
)
if save_files == False:
return features_set_train, features_set_test
def run(self, num_iter=None, plot_stats=None, plot_period=1, history=None):
# initialize plots
plot_stats = plot_stats or []
if plot_stats:
num_plots = len(plot_stats)
fig, axs = plt.subplots(num_plots,
1,
squeeze=False,
figsize=(10, 5 * num_plots))
axs = axs.ravel()
# initialize history dict
history = history or {}
history = defaultdict(list, history)
total_steps = history.get("total_steps", [0])[-1]
total_episodes = history.get("total_episodes", [0])[-1]
num_iter = num_iter or self.num_iter
for i in range(num_iter):
# Store statistics of the current update step
stats = defaultdict(list)
# sample a training batch
train_batch, train_batch_stats = self.sample_batch()
# create a datatset
obs = train_batch[SampleBatch.OBS]
actions_old = train_batch[SampleBatch.ACTIONS]
action_old_log_p = train_batch[SampleBatch.ACTION_LOGP]
advantages = train_batch[SampleBatch.ADVANTAGES].astype("float32")
value_targets = train_batch[SampleBatch.VALUE_TARGETS].astype(
"float32")
old_value_pred = train_batch[SampleBatch.VF_PREDS].astype(
"float32")
dataset = (tf.data.Dataset.from_tensor_slices((
obs,
advantages,
action_old_log_p,
actions_old,
value_targets,
old_value_pred,
)).batch(self.sgd_minibatch_size, drop_remainder=True).shuffle(1))
for (
obs_batch,
advantage_batch,
action_old_log_p_batch,
action_old_batch,
value_target_batch,
old_value_pred_batch,
) in dataset:
# if critic is not used critic loss is zero
critic_loss = tf.constant([0])
# update critic
if self.use_critic:
critic_loss = self.train_op_critic(obs_batch,
value_target_batch,
old_value_pred_batch)
# update actor
policy_loss, mean_entropy, actor_loss = self.train_op_actor(
obs_batch, action_old_batch, action_old_log_p_batch,
advantage_batch)
stats["policy_loss"].append(policy_loss.numpy().item())
stats["mean_entropy"].append(mean_entropy.numpy().item())
stats["actor_loss"].append(actor_loss.numpy().item())
stats["critic_loss"].append(critic_loss.numpy().item())
mean_stats = {k: np.mean(v) for k, v in stats.items()}
mean_stats = dict(**mean_stats, **train_batch_stats)
# record total steps per game and episodes per iteration
total_steps += mean_stats["num_steps"]
total_episodes += mean_stats["num_episodes"]
mean_stats["total_steps"] = total_steps
mean_stats["total_episodes"] = total_episodes
for k, v in mean_stats.items():
history[k].append(v)
if self.use_tensorboard:
tf.summary.scalar(k, v, self.total_iters)
if plot_stats:
if (i + 1) % plot_period == 0:
for ax, stat_name in zip(axs, plot_stats):
ax.clear()
if isinstance(stat_name, str):
stat_name = [stat_name]
for s in stat_name:
sns.lineplot(
x=np.arange(len(history[s])),
y=history[s],
ax=ax,
)
ax.set_title(stat_name)
display.display(fig)
display.clear_output(wait=True)
else:
print(f"Iteration: {i+1}/{self.num_iter} | {mean_stats}", )
self.total_iters += 1
return history
def display(self):
"""
This function is used to display all widgets of the UI.
"""
clear_output()
display(ipywidgets.VBox(self._widget_list))
def main():
#Lists to iterate through to switch the url year and page number
years_url = [str(i) for i in range(2000, 2019)]
pages = ['0', '51', '101', '151']
#Lists that will store the movie data
names = []
years = []
view_ratings = []
genres = []
imdb_ratings = []
metascores = []
poster_links = []
numRequests = 0
for year_url in years_url:
for page in pages:
url = 'https://www.imdb.com/search/title?release_date={}-01-01,{}-12-31&sort=num_votes,desc&start={}&ref_=adv_prv'.format(
year_url, year_url, page)
response = get(url)
#Pause the loop anywhere from 8 to 15 seconds
sleep(randint(8, 15))
numRequests += 1
print('Request: {}'.format(numRequests))
clear_output(wait=True)
html_soup = BeautifulSoup(response.text, 'html.parser')
movie_containers = html_soup.find_all(
'div', class_='lister-item mode-advanced')
for movie in movie_containers:
if movie.find('div', class_='ratings-metascore') != None:
#Add name of movie to names list
name = movie.h3.a.text
names.append(name)
#Add year of movie to years list
year = movie.h3.find('span',
class_='lister-item-year').text
years.append(year)
#Add the legal view rating to the view ratings list
view_rating = movie.p.span.text
view_ratings.append(view_rating)
#Add genre to the genres list
genre = movie.p.find('span', class_='genre').text
genres.append(genre)
#Add imdb rating to imdb ratings list
imdb_rating = float(movie.strong.text)
imdb_ratings.append(imdb_rating)
#Add metascore to metascores list
metascore = movie.find('span', class_='metascore').text
metascores.append(int(metascore))
#Add poster link to the poster links list
poster_link = movie.a.img.get('loadlate')
poster_links.append(poster_link)
movie_data = makeDataFrame(names, years, view_ratings, genres,
imdb_ratings, metascores, poster_links)
movie_data.to_csv('movie_data.csv', index=False)
def _run(self, _=None):
# Get widgets state
search_text = self.text.value
show_url = self.show_url.value
show_abstract = self.show_abstract.value
n_results = int(self.n_results.value)
# Check if this is a new search or simply new stuff to show
if self._last_search != search_text:
self.progress_label.value = "Searching!"
# Search through wikipedia
search_results = self.wikipedia.search(query=search_text)
self._last_search = search_text
# Get most probably indices
self._search_indices = search_results.index.tolist()
# Get documents
documents = [
self.wikipedia.documents[val]
for val in self._search_indices[:n_results]
]
# Output table
titles = [doc.title for doc in documents]
if all(["Wikipedia: " in val for val in titles]):
titles = [val[len("Wikipedia: "):] for val in titles]
table = [titles]
header = ["title"]
# Add content
if show_url:
table.append([doc.url for doc in documents])
header.append("URL")
if show_abstract:
table.append([doc.abstract for doc in documents])
header.append("Abstract")
# Transpose
table = list(zip(*table))
# Make dataframe
self.table = pd.DataFrame(table,
columns=header,
index=list(range(1,
len(table) + 1)))
def make_hyperlink(val):
return '<a href="{}" rel="noopener noreferrer" target="_blank">{}</a>'.format(
val, val)
# Set table style and use 1-indexing
styles = [
dict(selector="th",
props=[("text-align", "left"), ("font-size", "120%")]),
dict(selector="td", props=[("text-align", "left")]),
]
table_display = self.table.style.set_table_styles(styles) \
.format({'URL': make_hyperlink})
# Clear output and show widgets + results
clear_output()
# noinspection PyTypeChecker
display(self.dashboard)
# noinspection PyTypeChecker
display(table_display)
self.table = self.table
# Update is done
self.progress_label.value = ""
def process_and_plot(self):
"""Process data and plot it"""
self.logger.debug('At')
x = self.view.viz_ddn_plot_xaxis.value
y = self.view.viz_ddn_plot_yaxis.value
pivot = self.view.viz_ddn_plot_pivot.value
fill = self.view.viz_ddn_plot_fill.value
harm_row = self.view.viz_ddn_plot_harm_row.value
harm_col = self.view.viz_ddn_plot_harm_col.value
# Specify numeric axis(es)
numeric_xy = (x == F_VAL or x == F_YER, y == F_VAL or y == F_YER)
# Plot will be based on model's "processed" data
self.model.init_processed()
# Clear pivot table data
with self.view.viz_out_plot_data:
clear_output(wait=True)
self.model.pivot(x, pivot, y, self.view.viz_ddn_plot_aggfunc.value)
# Fill missing values (interpolate)?
if not fill == NONE_ITEM:
self.model.fill(fill)
# Index to year?
indexed_by = None
if self.view.viz_ckb_plot_index.value:
if x == F_YER and not harm_row == NONE_ITEM:
indexed_by = harm_row
elif y == F_YER and not harm_col == NONE_ITEM:
self.model.index(harm_col, on_row=False)
indexed_by = harm_col
if indexed_by is not None:
self.model.index(indexed_by)
# Harmonize?
harmonized = False
if (not harm_row == NONE_ITEM) and (not harm_col == NONE_ITEM):
self.model.harmonize(harm_row, harm_col)
harmonized = True
# Title
title = y + ' for ' + pivot + ' by ' + x
if indexed_by is not None:
if harmonized:
title += ', Harmonized: '
if indexed_by == harm_row:
title += str(harm_col)
else:
title += str(harm_row)
title += ', Index: ' + str(indexed_by) + '=100'
elif harmonized:
title += ', Harmonized: ' + str(harm_row) + ', ' + str(harm_col)
self.model.dropna()
# Show plot data
with self.view.viz_out_plot_data:
self.model.set_disp(self.model.processed, wide=True)
clear_output(wait=True)
display(self.model.processed)
# Draw plot based on processed data
self.draw_plot(title, x, y, numeric_xy)
def main():
url = 'https://www.lagou.com/jobs/positionAjax.json?needAddtionalResult=false&isSchoolJob=0'
headers = {
'Host': 'www.lagou.com',
'User-Agent':
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36',
'Accept': 'application/json, text/javascript, */*; q=0.01',
'Accept-Language': 'zh-CN,en-US;q=0.7,en;q=0.3',
'Accept-Encoding': 'gzip, deflate, br',
'Referer':
'https://www.lagou.com/jobs/list_Python?labelWords=&fromSearch=true&suginput=',
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
'X-Requested-With': 'XMLHttpRequest',
'X-Anit-Forge-Token': 'None',
'X-Anit-Forge-Code': '0',
'Content-Length': '26',
# 'Cookie': 'user_trace_token=20171103191801-9206e24f-9ca2-40ab-95a3-23947c0b972a; _ga=GA1.2.545192972.1509707889; LGUID=20171103191805-a9838dac-c088-11e7-9704-5254005c3644; JSESSIONID=ABAAABAACDBABJB2EE720304E451B2CEFA1723CE83F19CC; _gat=1; LGSID=20171228225143-9edb51dd-ebde-11e7-b670-525400f775ce; PRE_UTM=; PRE_HOST=www.baidu.com; PRE_SITE=https%3A%2F%2Fwww.baidu.com%2Flink%3Furl%3DKkJPgBHAnny1nUKaLpx2oDfUXv9ItIF3kBAWM2-fDNu%26ck%3D3065.1.126.376.140.374.139.129%26shh%3Dwww.baidu.com%26sht%3Dmonline_3_dg%26wd%3D%26eqid%3Db0ec59d100013c7f000000055a4504f6; PRE_LAND=https%3A%2F%2Fwww.lagou.com%2F; LGRID=20171228225224-b6cc7abd-ebde-11e7-9f67-5254005c3644; index_location_city=%E5%85%A8%E5%9B%BD; TG-TRACK-CODE=index_search; SEARCH_ID=3ec21cea985a4a5fa2ab279d868560c8',
'Connection': 'keep-alive',
'Pragma': 'no-cache',
'Cache-Control': 'no-cache',
}
cookies = {
'Cookie':
'user_trace_token=20180309150141-1071c3b86d9340849221995337e696ab;'
'_ga=GA1.2.26456326.1520578906; _gid=GA1.2.1110652441.1520578906; '
'LGUID=20180309150146-bb3feaf5-2367-11e8-a4d0-525400f775ce; _gat=1; '
'LGSID=20180309220646-1a31dfca-23a3-11e8-a7fa-525400f775ce; '
'PRE_UTM=m_cf_cpt_baidu_pc; '
'PRE_HOST=www.baidu.com; PRE_SITE=https%3A%2F%2Fwww.baidu.com%2Fs%3Fwd%3D%25E6%258B%2589%25E5%258B%25BE%25E7%25BD%2591%26rsv_spt%3D1%26rsv_iqid%3D0xc5bd295800005d78%26issp%3D1%26f%3D8%26rsv_bp%3D0%26rsv_idx%3D2%26ie%3Dutf-8%26tn%3Dbaiduhome_pg%26rsv_enter%3D1%26rsv_sug3%3D4%26rsv_sug1%3D4%26rsv_sug7%3D100; '
'PRE_LAND=https%3A%2F%2Fwww.lagou.com%2Flp%2Fhtml%2Fcommon.html%3Futm_source%3Dm_cf_cpt_baidu_pc; '
'JSESSIONID=ABAAABAAADEAAFIFD520A39CCC6F8DB23972BE849E096FD; '
'Hm_lvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1520578906,1520604405,1520604412; '
'index_location_city=%E5%85%A8%E5%9B%BD; hideSliderBanner20180305WithTopBannerC=1; '
'TG-TRACK-CODE=index_search; SEARCH_ID=00df79401e894f288caf5da82944a99d; '
'Hm_lpvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1520604432; '
'LGRID=20180309220712-2a00fa16-23a3-11e8-b1a7-5254005c3644',
}
data = {
'first': 'false',
'pn': 1,
'kd': 'python',
}
page = get_page(url, cookies, headers, data)
page_num = get_page_num(page)
pages = [i for i in range(1, page_num)]
start_time = time.time()
# moniter parameter
requests = 0
for page in range(1, pages):
# a request would go here
print('爬取第{}页'.format(page))
data['pn'] = page
html = get_page(url, cookies, headers, data)
time.sleep(random.randint(5, 8))
requests += 1
elapsed_time = time.time() - start_time
print('Request: {}; Frequency: {} requests/s'.format(
requests, requests / elapsed_time))
clear_output(wait=True)
contents = parse_job(html)
save_to_file(contents)
def show_record(sender):
'''displays wins and losses. can only be called at the end of a round.'''
global wins, losses, submit_flag
if not submit_flag:
clear_output()
print('wins: {}\nlosses:{}'.format(wins, losses))
def print_stats(self) -> None:
clear_output()
self.display_cp()
self.print_graphs()
def run_agent(self, render: bool = False, print_swarm: bool = False):
"""
:param render:
:param print_swarm:
:return:
"""
self.tree.reset()
i_step, self._agent_reward, end = 0, 0, False
self._save_steps = []
# Clone emulator state w/ system state including pseudorandomness.
# Restoring this state will give an identical environment.
state, obs = self._env.reset(return_state=True)
self.tree.append_leaf(i_step,
parent_id=i_step - 1,
state=state,
action=0,
dt=1)
reward_sum = 0
net = dqn_model.DQN(self._env.observation_space.shape,
self._env.action_space.n).to(device)
tgt_net = dqn_model.DQN(self._env.observation_space.shape,
self._env.action_space.n).to(device)
buffer = dqn_agent.ExperienceBuffer(REPLAY_SIZE)
agent = dqn_agent.Agent(self._env, buffer)
optimizer = optim.Adam(net.parameters(), lr=LEARNING_RATE)
current_obs = obs
start_time = time.time()
while not end and self._agent_reward < self.reward_limit:
i_step += 1
self.run_swarm(state=state.copy(), obs=obs)
action = self.weight_actions()
state, obs, _reward, _end, info = self._env.step(
state=state, action=action, n_repeat_action=self.min_dt)
# if not _end:
# next_obs = obs
# else:
# next_obs = None
#
next_obs = obs
#print("come here")
exp = Experience(current_obs, action, _reward, _end, next_obs)
current_obs = next_obs
agent.exp_buffer.append(exp)
reward_sum += _reward
self.tree.append_leaf(i_step,
parent_id=i_step - 1,
state=state,
action=action,
dt=self._env.n_repeat_action)
self._agent_reward += _reward
self._last_action = action
end = info.get("terminal", _end)
#if _reward != 0:
# print("i_step,_reward,_end", i_step, _reward, _end)
if _end:
print('ep %d: game over. episode reward total was %f' %
(i_step, reward_sum))
if render:
self._env.render()
if print_swarm:
print(self)
clear_output(True)
self.update_parameters()
# train dqn model
print("experiences exploration time/seconds:",
time.time() - start_time)
print("**************dqn agent training...*******************")
#num_episodes = 1000
num_episodes = 10000
reward_sum = 0
start_time = time.time()
for i_episode in range(num_episodes):
optimizer.zero_grad()
batch = agent.exp_buffer.sample(BATCH_SIZE)
loss_t = dqn_agent.calc_loss(batch, net, tgt_net, device=device)
loss_t.backward()
optimizer.step()
print("Train time/seconds:", time.time() - start_time)
print("#############dqn agent testing...############")
env = gym.make('Pong-v0')
current_obs = env.reset()
start_time = time.time()
while True:
state_a = np.array([current_obs], copy=False)
state_v = torch.tensor(state_a, dtype=torch.float).to(device)
q_vals_v = net(state_v)
_, act_v = torch.max(q_vals_v, dim=1)
action = int(act_v.item())
new_state, _reward, _end, _ = env.step(action)
current_obs = new_state
#if _reward != 0:
# print("_reward,_end", _reward, _end)
reward_sum += _reward
if _end:
print('game over. reward total was %f' % reward_sum)
break
print("Test time:/seconds:", time.time() - start_time)
print("################test over##########################")
def on_button_update(_):
clear_output()
#display(Javascript('''var c = IPython.notebook.get_selected_index();
#IPython.notebook.execute_cells([c])'''))
add_attachment()
def train(self, env, q_table
): #This fuction updates the Q table using Bellman's Equation
#These are the hyperparameters
alpha = .1
gamma = .6
epsilon = .1
#For plotting metrics
all_epochs = []
all_penalties = []
total_epsiodes = []
print("Training Started")
for i in range(1, 100001):
state = env.reset()
epochs, penalties, reward, = 0, 0, 0
done = False
while not done:
if np.random.uniform(0, 1) < epsilon:
action = env.action_space.sample() # Explore action space
else:
action = np.argmax(
q_table[state]) # Exploit learned values
next_state, reward, done, info = env.step(action)
old_value = q_table[state, action]
next_max = np.max(q_table[next_state])
new_value = (1 - alpha) * old_value + alpha * (
reward + gamma * next_max
) #The sate value function which updates the Q value
q_table[state, action] = new_value
if reward == -10:
penalties += 1
state = next_state
epochs += 1
all_epochs.append(epochs)
all_penalties.append(penalties)
if i % 100 == 0:
clear_output(wait=True)
print(f"Episode: {i}")
total_epsiodes.append(i)
all_epochs = all_epochs[:1000]
all_penalties = all_penalties[:1000]
#Plotting the STEPS vs EPISODES graph
sns.lineplot(total_epsiodes, all_epochs)
plt.xlabel("Episode")
plt.ylabel("Steps")
plt.show()
# Plotting the PENALTIES vs EPISODES graph
sns.lineplot(total_epsiodes, all_penalties)
plt.xlabel("Episode")
plt.ylabel("penalties")
plt.show()
print("Training finished.\n")
def emit(self, s):
clear_output(wait=True)
print(s.getMessage())
if movie.find('span', class_='metascore') is not None:
# print('movie')
movie_name = movie.find('div', class_='lister-item-content')
# print(movie_name)
names.append(movie_name.h3.a.text)
# print(movie_name.strong.text)
ratings.append(str(movie_name.strong.text))
year = movie_name.h3.find('span', class_='lister-item-year text-muted unbold').text
# print(year)
years.append(year)
metascore = movie_name.find('div', class_='ratings-bar')
metascore = metascore.find('div', class_='inline-block ratings-metascore')
# print(metascore.span.text)
metascores.append(str(metascore.span.text))
vote = movie_name.find('span', attrs={'name':'nv'})['data-value']
# print(int(vote))
votes.append(vote)
clear_output(wait=True) # send request once the respond has received. wait before sending another request
table_df = pd.DataFrame({'movie': names,
'year': years,
'imdb': ratings,
'metascore': metascores,
'votes': votes
})
table_df.loc[:, 'year'] = table_df['year'].str[-5:-1].astype(int)
table_df.to_csv('imdb_ratings.csv')
print("Done!")
def get_article_links(start_year, end_year, num_pages):
"""Collect and save a list of links that correspond to
articles from The Dartmouth daily newspaper.
Parameters
----------
start_year : int
Year in the archives to start looking for articles
end_year : int
Year in the archives to stop looking for articles
num_pages : int
Number of archive pages that fit each year range to
inspect
Returns
-------
text file
A text file ('article_links.txt' that contains the
link to every article that appears in specificed
year and page range.
"""
# change range to 0,50 when using cluster
pages = list(map(str, range(num_pages)))
# change range to 2012,2019 when using cluster
year_urls = list(range(start_year, end_year))
article_links = []
start_time = time()
requests = 0
for year_url in year_urls:
for page in pages:
html_page = urllib.request.urlopen(
"http://www.thedartmouth.com/search?order"
"=date&direction=desc&begin={}0101&end="
"{}1231&page={}&per_page=20".format(year_url, year_url, page))
# html_page = urllib.request.urlopen(
# "http://www.thedartmouth.com/search?q=0&page={}&"
# "ti=0&tg=0&ty=0&ts_month=0&ts_day=0&ts_year={}&"
# "te_month=0&te_day=0&te_year={}&s=0&au=0&o=0&"
# "a=1".format(page, year_url, year_url+1))
# Pause the loop
sleep(randint(8, 15))
# Monitor the requests
requests += 1
elapsed_time = time() - start_time
print('Request:{}; Frequency: {} requests/s'.format(
requests, requests / elapsed_time))
clear_output(wait=True)
soup = BeautifulSoup(html_page, 'html.parser')
links = []
for link in soup.find_all('a'):
links.append(link.get('href'))
for link in links:
if link not in article_links:
if 'article' in link.split('/'):
article_links.append(link)
file = open('article_links.txt', 'w')
for article_link in article_links:
file.write("{}\n".format(article_link))
file.close()
def collect_data(self, render: bool = False, print_swarm: bool = False):
"""
:param render:
:param print_swarm:
:return:
"""
self.tree.reset()
self.env.reset()
state, obs = self.data_env.reset(return_state=True)
i_step, self._agent_reward, end = 0, 0, False
_end = False
for i in range(self.skip_initial_frames):
i_step += 1
action = 0
state, obs, _reward, _end, info = self.data_env.step(
state=state, action=action, n_repeat_action=1
)
self.tree.append_leaf(
i_step,
parent_id=i_step - 1,
state=state,
action=np.ones(self.env.n_actions),
dt=1,
reward=np.ones(self.env.n_actions),
terminal=bool(info["terminal"]),
obs=obs,
)
self._agent_reward += _reward
self._last_action = action
end = info.get("terminal", _end)
if end:
break
self._save_steps = []
while not end and self._agent_reward < self.reward_limit:
i_step += 1
self.run_swarm(state=copy.deepcopy(state), obs=obs)
action_dist, reward_dist = self.estimate_distributions(state=state, obs=obs)
action = (action_dist + reward_dist).argmax()
state, obs, _reward, _end, info = self.data_env.step(
state=state, action=action, n_repeat_action=1
)
self.tree.append_leaf(
i_step,
parent_id=i_step - 1,
state=state,
action=action_dist,
dt=1,
reward=reward_dist,
terminal=bool(_end),
obs=obs,
)
self._agent_reward += _reward
self._last_action = action
end = info.get("terminal", _end)
self._best_id = i_step
if render:
self.data_env.render()
if print_swarm:
from IPython.core.display import clear_output
print(self)
clear_output(True)
if self._update_parameters:
self.update_parameters()
def refresh_and_print(iterator_output):
clear_output(wait=True)
n, viz = iterator_output
print(n, viz, sep='\n')
def update(self, final=False):
if IPY:
clear_output(wait=True)
display(HTML(self.__html__(final)))
else:
print(self.__str__())
for page in pages:
url = "https://www.elespectador.com/search/" + keyword + "?page=" + page
print(url)
html = get(url)
htmlsoup = soup(html.content, 'html.parser')
time.sleep(randint(2, 4))
requests += 1
count += 1
elapsed_time = time.time() - start_time
print("")
print('Palabra: {}; Página: {}; Tiempo: {} min'.format(
keyword, page, round(elapsed_time / 60, 3)))
print("")
clear_output(wait=True)
articles = htmlsoup.find_all(
'div',
class_=
"node-title field field--name-title field--type-ds field--label-hidden"
)
if not articles:
print("There were no more articles found with your keyword")
break
else:
for oneArticle in articles:
title = oneArticle.a.text.strip()
link = oneArticle.a['href']
content = ''
url2 = "http://www.elespectador.com" + link
def evolve(central_mass, num_threads, length, length_units, resol, duration,
duration_units, step_factor, save_number, save_options, save_path,
npz, npy, hdf5, s_mass_unit, s_position_unit, s_velocity_unit,
solitons, start_time):
print('Initialising...')
##########################################################################################
#SET INITIAL CONDITIONS
if (length_units == ''):
gridlength = length
else:
gridlength = convert(length, length_units, 'l')
if (duration_units == ''):
t = duration
else:
t = convert(duration, duration_units, 't')
if (duration_units == ''):
t0 = start_time
else:
t0 = convert(start_time, duration_units, 't')
if (s_mass_unit == ''):
cmass = central_mass
else:
cmass = convert(central_mass, s_mass_unit, 'm')
Vcell = (gridlength / float(resol))**3
ne.set_num_threads(num_threads)
initsoliton_jit = numba.jit(initsoliton)
##########################################################################################
# CREATE THE TIMESTAMPED SAVE DIRECTORY AND CONFIG.TXT FILE
save_path = os.path.expanduser(save_path)
tm = time.localtime()
talt = ['0', '0', '0']
for i in range(3, 6):
if tm[i] in range(0, 10):
talt[i - 3] = '{}{}'.format('0', tm[i])
else:
talt[i - 3] = tm[i]
timestamp = '{}{}{}{}{}{}{}{}{}{}{}{}{}'.format(tm[0], '.', tm[1], '.',
tm[2], '_', talt[0], ':',
talt[1], ':', talt[2], '_',
resol)
file = open('{}{}{}'.format('./', save_path, '/timestamp.txt'), "w+")
file.write(timestamp)
os.makedirs('{}{}{}{}'.format('./', save_path, '/', timestamp))
file = open(
'{}{}{}{}{}'.format('./', save_path, '/', timestamp, '/config.txt'),
"w+")
file.write(('{}{}'.format('resol = ', resol)))
file.write('\n')
file.write(('{}{}'.format('axion_mass (kg) = ', axion_mass)))
file.write('\n')
file.write(('{}{}'.format('length (code units) = ', gridlength)))
file.write('\n')
file.write(('{}{}'.format('duration (code units) = ', t)))
file.write('\n')
file.write(('{}{}'.format('start_time (code units) = ', t0)))
file.write('\n')
file.write(('{}{}'.format('step_factor = ', step_factor)))
file.write('\n')
file.write(('{}{}'.format('central_mass (code units) = ', cmass)))
file.write('\n\n')
file.write(
('{}'.format('solitons ([mass, [x, y, z], [vx, vy, vz], phase]): \n')))
for s in range(len(solitons)):
file.write(('{}{}{}{}{}'.format('soliton', s, ' = ', solitons[s],
'\n')))
file.write(
('{}{}{}{}{}{}'.format('\ns_mass_unit = ', s_mass_unit,
', s_position_unit = ', s_position_unit,
', s_velocity_unit = ', s_velocity_unit)))
file.write(
'\n\nNote: If the above units are blank, this means that the soliton parameters were specified in code units'
)
file.close()
loc = save_path + '/' + timestamp
##########################################################################################
# SET UP THE REAL SPACE COORDINATES OF THE GRID
gridvec = np.linspace(-gridlength / 2.0 + gridlength / float(2 * resol),
gridlength / 2.0 - gridlength / float(2 * resol),
resol)
xarray, yarray, zarray = np.meshgrid(
gridvec,
gridvec,
gridvec,
sparse=True,
indexing='ij',
)
distarray = ne.evaluate(
"(xarray**2+yarray**2+zarray**2)**0.5") # Radial coordinates
##########################################################################################
# SET UP K-SPACE COORDINATES FOR COMPLEX DFT (NOT RHO DFT)
kvec = 2 * np.pi * np.fft.fftfreq(resol, gridlength / float(resol))
kxarray, kyarray, kzarray = np.meshgrid(
kvec,
kvec,
kvec,
sparse=True,
indexing='ij',
)
karray2 = ne.evaluate("kxarray**2+kyarray**2+kzarray**2")
##########################################################################################
# INITIALISE SOLITONS WITH SPECIFIED MASS, POSITION, VELOCITY, PHASE
f = np.load('./Soliton Profile Files/initial_f.npy')
delta_x = 0.00001 # Needs to match resolution of soliton profile array file. Default = 0.00001
warn = 0
psi = pyfftw.zeros_aligned((resol, resol, resol), dtype='complex128')
funct = pyfftw.zeros_aligned((resol, resol, resol), dtype='complex128')
for k in range(len(solitons)):
if (k != 0):
if (not overlap_check(solitons[k], solitons[:k])):
warn = 1
else:
warn = 0
for s in solitons:
mass = convert(s[0], s_mass_unit, 'm')
position = convert(np.array(s[1]), s_position_unit, 'l')
velocity = convert(np.array(s[2]), s_velocity_unit, 'v')
# Note that alpha and beta parameters are computed when the initial_f.npy soliton profile file is generated.
alpha = (mass / 3.883)**2
beta = 2.454
phase = s[3]
funct = initsoliton_jit(funct, xarray, yarray, zarray, position, alpha,
f, delta_x)
####### Impart velocity to solitons in Galilean invariant way
velx = velocity[0]
vely = velocity[1]
velz = velocity[2]
funct = ne.evaluate(
"exp(1j*(alpha*beta*t0 + velx*xarray + vely*yarray + velz*zarray -0.5*(velx*velx+vely*vely+velz*velz)*t0 + phase))*funct"
)
psi = ne.evaluate("psi + funct")
rho = ne.evaluate("real(abs(psi)**2)")
fft_psi = pyfftw.builders.fftn(psi, axes=(0, 1, 2), threads=num_threads)
ifft_funct = pyfftw.builders.ifftn(funct,
axes=(0, 1, 2),
threads=num_threads)
##########################################################################################
# COMPUTE SIZE OF TIMESTEP (CAN BE INCREASED WITH step_factor)
delta_t = (gridlength / float(resol))**2 / np.pi
min_num_steps = t / delta_t
min_num_steps_int = int(min_num_steps + 1)
min_num_steps_int = int(min_num_steps_int / step_factor)
if save_number >= min_num_steps_int:
actual_num_steps = save_number
its_per_save = 1
else:
rem = min_num_steps_int % save_number
actual_num_steps = min_num_steps_int + save_number - rem
its_per_save = actual_num_steps / save_number
h = t / float(actual_num_steps)
##########################################################################################
# SETUP K-SPACE FOR RHO (REAL)
rkvec = 2 * np.pi * np.fft.fftfreq(resol, gridlength / float(resol))
krealvec = 2 * np.pi * np.fft.rfftfreq(resol, gridlength / float(resol))
rkxarray, rkyarray, rkzarray = np.meshgrid(rkvec,
rkvec,
krealvec,
sparse=True,
indexing='ij')
rkarray2 = ne.evaluate("rkxarray**2+rkyarray**2+rkzarray**2")
rfft_rho = pyfftw.builders.rfftn(rho, axes=(0, 1, 2), threads=num_threads)
phik = rfft_rho(rho) # not actually phik but phik is defined in next line
phik = ne.evaluate("-4*3.141593*phik/rkarray2")
phik[0, 0, 0] = 0
irfft_phi = pyfftw.builders.irfftn(phik,
axes=(0, 1, 2),
threads=num_threads)
##########################################################################################
# COMPUTE INTIAL VALUE OF POTENTIAL
phisp = pyfftw.zeros_aligned((resol, resol, resol), dtype='float64')
phisp = irfft_phi(phik)
phisp = ne.evaluate("phisp-(cmass)/distarray")
##########################################################################################
# PRE-LOOP ENERGY CALCULATION
if (save_options[3]):
egylist = []
egpcmlist = []
egpsilist = []
ekandqlist = []
mtotlist = []
calculate_energies(
save_options,
resol,
psi,
cmass,
distarray,
Vcell,
phisp,
karray2,
funct,
fft_psi,
ifft_funct,
egpcmlist,
egpsilist,
ekandqlist,
egylist,
mtotlist,
)
##########################################################################################
# PRE-LOOP SAVE I.E. INITIAL CONFIG
save_grid(
rho,
psi,
resol,
save_options,
npy,
npz,
hdf5,
loc,
-1,
1,
)
##########################################################################################
# LOOP NOW BEGINS
halfstepornot = 1 # 1 for a half step 0 for a full step
tenth = float(
save_number / 10
) #This parameter is used if energy outputs are saved while code is running.
# See commented section below (line 585)
clear_output()
print("The total number of steps is %.0f" % actual_num_steps)
if warn == 1:
print(
"WARNING: Significant overlap between solitons in initial conditions"
)
print('\n')
tinit = time.time()
for ix in range(actual_num_steps):
if halfstepornot == 1:
psi = ne.evaluate("exp(-1j*0.5*h*phisp)*psi")
halfstepornot = 0
else:
psi = ne.evaluate("exp(-1j*h*phisp)*psi")
funct = fft_psi(psi)
funct = ne.evaluate("funct*exp(-1j*0.5*h*karray2)")
psi = ifft_funct(funct)
rho = ne.evaluate("real(abs(psi)**2)")
phik = rfft_rho(
rho) # not actually phik but phik is defined on next line
phik = ne.evaluate("-4*3.141593*(phik)/rkarray2")
phik[0, 0, 0] = 0
phisp = irfft_phi(phik)
phisp = ne.evaluate("phisp-(cmass)/distarray")
#Next if statement ensures that an extra half step is performed at each save point
if (((ix + 1) % its_per_save) == 0) and halfstepornot == 0:
psi = ne.evaluate("exp(-1j*0.5*h*phisp)*psi")
rho = ne.evaluate("real(abs(psi)**2)")
halfstepornot = 1
#Next block calculates the energies at each save, not at each timestep.
if (save_options[3]):
calculate_energies(
save_options,
resol,
psi,
cmass,
distarray,
Vcell,
phisp,
karray2,
funct,
fft_psi,
ifft_funct,
egpcmlist,
egpsilist,
ekandqlist,
egylist,
mtotlist,
)
#Uncomment next section if partially complete energy lists desired as simulation runs.
#In this way, some energy data will be saved even if the simulation is terminated early.
# if (save_options[3]):
# if (ix+1) % tenth == 0:
# label = (ix+1)/tenth
# file_name = "{}{}".format(label,'egy_cumulative.npy')
# np.save(os.path.join(os.path.expanduser(loc), file_name), egylist)
# file_name = "{}{}".format(label,'egpcm_cumulative.npy')
# np.save(os.path.join(os.path.expanduser(loc), file_name), egpcmlist)
# file_name = "{}{}".format(label,'egpsi_cumulative.npy')
# np.save(os.path.join(os.path.expanduser(loc), file_name), egpsilist)
# file_name = "{}{}".format(label,'ekandq_cumulative.npy')
# np.save(os.path.join(os.path.expanduser(loc), file_name), ekandqlist)
################################################################################
# SAVE DESIRED OUTPUTS
if ((ix + 1) % its_per_save) == 0:
save_grid(
rho,
psi,
resol,
save_options,
npy,
npz,
hdf5,
loc,
ix,
its_per_save,
)
################################################################################
# UPDATE INFORMATION FOR PROGRESS BAR
tint = time.time() - tinit
tinit = time.time()
prog_bar(actual_num_steps, ix + 1, tint)
################################################################################
# LOOP ENDS
clear_output()
print('\n')
print("Complete.")
if warn == 1:
print(
"WARNING: Significant overlap between solitons in initial conditions"
)
if (save_options[3]):
file_name = "egylist.npy"
np.save(os.path.join(os.path.expanduser(loc), file_name), egylist)
file_name = "egpcmlist.npy"
np.save(os.path.join(os.path.expanduser(loc), file_name), egpcmlist)
file_name = "egpsilist.npy"
np.save(os.path.join(os.path.expanduser(loc), file_name), egpsilist)
file_name = "ekandqlist.npy"
np.save(os.path.join(os.path.expanduser(loc), file_name), ekandqlist)
file_name = "masslist.npy"
np.save(os.path.join(os.path.expanduser(loc), file_name), mtotlist)
def run(self,
num_episodes=None,
plot_stats=None,
plot_period=1,
history=None):
# initialize plots
plot_stats = plot_stats or []
if plot_stats:
num_plots = len(plot_stats)
fig, axs = plt.subplots(num_plots,
1,
squeeze=False,
figsize=(10, 5 * num_plots))
axs = axs.ravel()
# initialize history dict
history = history or {}
history = defaultdict(list, history)
num_episodes = num_episodes or self.num_episodes
for i in range(num_episodes):
# Keep track of current episode statistics
stats = {}
# sample a trajectory
train_batch, score = self.sample_trajectory()
# Get data from the training batch
obs = train_batch[SampleBatch.OBS]
actions_old = train_batch[SampleBatch.ACTIONS]
returns = train_batch[SampleBatch.RETURNS].astype(
"float32").squeeze()
# normalize returns on the batch level
if self.standardize_returns:
returns = np_standardized(returns)
# perform gradient descent
policy_loss, total_loss, entropy_bonus = self.train_op(
obs, actions_old, returns)
# record statistics
stats["policy_loss"] = policy_loss.numpy().item()
stats["entropy"] = entropy_bonus.numpy().item()
stats["total_loss"] = total_loss.numpy().item()
stats["score"] = score
stats["steps_per_episode"] = len(train_batch)
for k, v in stats.items():
history[k].append(v)
if self.use_tensorboard:
tf.summary.scalar(k, v, self.total_episodes)
self.total_episodes += 1
if plot_stats:
if (i + 1) % plot_period == 0:
for ax, stat_name in zip(axs, plot_stats):
ax.clear()
# print(stat_name, len(history[stat_name]))
sns.lineplot(
x=np.arange(len(history[stat_name])),
y=history[stat_name],
ax=ax,
)
ax.set_title(stat_name)
display.display(fig)
display.clear_output(wait=True)
else:
print(f"episode {i}/{self.num_episodes} | {stats}", )
return history
def features_processing_from_purchases(self, purchases):
df_products = pd.read_csv(self.products_path).fillna('None')
df_products.loc[df_products['netto'] == 'None', 'netto'] = 0
'''Это функция извлечения признаков из данных о покупках. Данные групируются вокруг клиента'''
all_client_n = purchases['client_id'].nunique()
start_time_for_print = strftime("%Y-%m-%d %H:%M:%S", gmtime())
start_time = time.time()
if self.low_memory == True:
dict_product = dict(
zip(df_products['product_id'].values,
df_products.iloc[:, 5].values))
else:
dict_product = dict(
zip(df_products['product_id'].values,
df_products.iloc[:, 1:6].values))
dict_product_number = dict(
zip(df_products['product_id'].values,
df_products.iloc[:, -3:].values))
print(f'Начало обработки клиентов {start_time_for_print}')
features_product = self.features_dict(df_products, )
features_product['netto'] = 0
features_product['trademark'] = 0
features_product['alhocol'] = 0
features_product['regular_points_received'] = 0
features_product['express_points_received'] = 0
features_product['regular_points_spent'] = 0
features_product['express_points_spent'] = 0
for i in range(24):
if i < 7:
features_product['dayofweek_' + str(i)] = 0
features_product['hour_' + str(i)] = 0
points_list = ['regular_points_received', 'express_points_received',\
'regular_points_spent', 'express_points_spent']
data = []
for i, (id_c,
client) in enumerate(purchases.groupby('client_id',
sort=False)):
if (i + 1) % 5000 == 0:
clear_output()
print(f'Начало обработки клиентов {start_time_for_print}')
print(f'Обработано {i+1} клиентов из {all_client_n}...')
print(f'C начало прошло {int(time.time()-start_time)} секунд')
features = features_product.copy()
n_trans = client['transaction_id'].nunique()
features['transactions'] = n_trans
features['sum'] = client['trn_sum_from_iss'].sum()
features['trn_sum_from_red'] = client['trn_sum_from_red'].sum()
features['n_store'] = client['store_id'].nunique()
features['n_product'] = client['product_id'].nunique()
features['max_price'] = client['trn_sum_from_iss'].max()
features['min_price'] = client['trn_sum_from_iss'].min()
features['quantity'] = client['product_quantity'].sum()
features['first_buy_sum'] = client['purchase_sum'].iloc[0]
features['last_buy_sum'] = client['purchase_sum'].iloc[-1]
try:
features['almost_last_buy'] = client['purchase_sum'].unique(
)[-2]
except:
features['almost_last_buy'] = client['purchase_sum'].unique(
)[0]
features['client_id'] = client['client_id'].iloc[0]
features['transaction_max_delay'] = self.transaction_max_delay(
client)
#Features from products
count_products = Counter(client['product_id'])
if self.low_memory == True:
for product in count_products.keys():
features[
'segment_id_' +
str(dict_product[product])] += count_products[product]
else:
for product in count_products.keys():
values = dict_product[product]
for value in values:
if type(value) != str:
features['segment_id_' +
str(value)] += count_products[product]
else:
features[value] = count_products[product]
temp_dict_quantity = dict(
zip(client['product_id'], client['product_quantity']))
for product, quantity in temp_dict_quantity.items():
features['netto'] += quantity * dict_product_number[product][0]
features[
'trademark'] += quantity * dict_product_number[product][1]
features[
'alhocol'] += quantity * dict_product_number[product][2]
#Features from date
temp_dict_date = dict(
zip(client['transaction_id'].values,
client['dayofweek'].values))
for dayofweek in temp_dict_date.values():
features['dayofweek_' + str(dayofweek)] += 1
temp_dict_date = dict(
zip(client['transaction_id'].values, client['hour'].values))
for hour in temp_dict_date.values():
features['hour_' + str(hour)] += 1
#Features from points
points_dict = dict(
zip(client['transaction_id'].values,
client[points_list].values))
for point in points_dict.values():
features['regular_points_received'] += point[0]
features['express_points_received'] += point[1]
features['regular_points_spent'] += point[2]
features['express_points_spent'] += point[3]
#Average features
features['avg_regular_points_received'] = features[
'regular_points_received'] / n_trans
features['avg_express_points_received'] = features[
'express_points_received'] / n_trans
features['avg_regular_points_spent'] = features[
'regular_points_spent'] / n_trans
features['avg_express_points_spent'] = features[
'express_points_spent'] / n_trans
features[
'avg_sum_from_red'] = features['trn_sum_from_red'] / n_trans
features['avg_price_product'] = features['sum'] / n_trans
features['avg_delay_beetwen_transc'] = features[
'transaction_max_delay'] / n_trans
features['avg_sum'] = features['sum'] / n_trans
features['avg_quantity'] = features['quantity'] / n_trans
features['avg_netto'] = features['netto'] / n_trans
features['avg_trademark'] = features['trademark'] / n_trans
features['avg_alhocol'] = features['alhocol'] / n_trans
data.append(features)
clear_output()
print(f'Начало обработки клиентов {start_time_for_print}')
print(
f'Обработка данных клиентов завершена за {int(time.time()-start_time)} секунд'
)
return data
def animate(self):
display.clear_output()
display.display(self)
sys.stdout.flush()
def start(self):
def startImport(b):
b.disabled = True
print('start import...')
parent_dir, files = self.previous_step.data
self.progressbar.max = int(self.sample_num.value)
if not parent_dir.lower().endswith('.zip'):
sampled_ids = random.sample(range(0, len(files)),
self.progressbar.max)
for i in range(0, self.progressbar.max):
file = files[sampled_ids[i]]
base_name = path.basename(file)
self.progressbar.value = i
with open(parent_dir + '/' + file) as f:
text = f.read()
if file.lower().endswith('.xml'):
self.loadTextFromXml(file, text, self.data)
else:
self.dataset_name = 'unknown'
self.data = self.data.append(
pd.DataFrame(
[[None, base_name, text, None, None]],
columns=[
'BUNCH_ID', 'DOC_NAME', 'TEXT', 'DATE',
'REF_DATE'
]))
self.progressbar.value = self.progressbar.max
else:
sampled_ids = random.sample(range(0, len(self.file_list)),
self.progressbar.max)
for i in range(0, self.progressbar.max):
finfo = self.file_list[sampled_ids[i]]
ifile = self.zfile.open(finfo)
doc_text = ifile.read().decode("utf-8")
base_name = path.basename(finfo.filename)
self.progressbar.value = i
if finfo.filename.lower().endswith('.xml'):
self.loadTextFromXml(finfo, doc_text, self.data)
else:
self.dataset_name = 'unknown'
self.data = self.data.append(
pd.DataFrame(
[[None, base_name, doc_text, None, None]],
columns=[
'BUNCH_ID', 'DOC_NAME', 'TEXT', 'DATE',
'REF_DATE'
]))
self.zfile.close()
self.progressbar.value = self.progressbar.max
self.next_button.disabled = False
# self.data.set_index('DOC_NAME', inplace=True)
if self.dataset_name == 'n2c2':
self.inferRefDate(self.data)
print("Totally " + str(len(sampled_ids)) +
" files have been imported into dataframe.\n"
"They are parsed into " + str(len(self.data)) +
" records in dataframe.")
pass
if self.previous_step.data is None:
self.previous_step.start()
parent_dir, files = self.previous_step.data
if not parent_dir.lower().endswith('.zip'):
label = widgets.HTML(
"<h4>Read %s files from: </h4><p>%s</p>".format(
len(files), parent_dir))
self.start_import_btn.on_click(startImport)
self.sample_num.value = str(len(files))
self.progressbar.max = len(files)
rows = [
label, self.sample_num, self.start_import_btn, self.progressbar
] + self.addSeparator(top='10px') + [
self.addPreviousNext(self.show_previous, self.show_next)
]
else:
import zipfile, os
parent_dir = os.path.join(self.previous_step.path, parent_dir)
self.zfile = zipfile.ZipFile(parent_dir)
print('reading file list from {} ...'.format(parent_dir))
self.file_list = [
f for f in self.zfile.infolist() if not f.is_dir()
]
label = widgets.HTML(
"<h4>Read {} files from: </h4><p>{}</p>".format(
len(self.file_list), parent_dir))
self.sample_num.value = str(len(self.file_list))
self.progressbar.max = len(self.file_list)
self.start_import_btn.on_click(startImport)
rows = [
label, self.sample_num, self.start_import_btn, self.progressbar
] + self.addSeparator(top='10px') + [
self.addPreviousNext(self.show_previous, self.show_next)
]
vbox = widgets.VBox(rows)
vbox.layout.flex_grown = 'column'
clear_output()
display(vbox)
return self.data
def update(self, _=None):
"""Map the widgets values to cufflinks parameters and creates a plot"""
clear_output(wait=True) #Is there any way to avoid the clipping?
self.update_layout()
if self.bargap.value == 1.0:
bgap = None
else:
bgap = self.bargap.value
if self.bargroupgap.value == 1.0:
bgroupgap = None
else:
bgroupgap = self.bargroupgap.value
if self.bins.value == 0:
bins = None
else:
bins = self.bins.value
#try:
if self.apply_layout.value:
self.layout_d.visible = True
layout = self.layout_d.output.copy()
layout[
'barmode'] = self.barmode.value #barmode must be in layout dict to take effect if you are using layout param
self._df[list(self.columns.value)].iplot(
barmode=self.barmode.value,
colors=self.cmap.map_data(range(len(self.columns.value)),
hex=True),
theme=self.theme.value,
x=self.x.value,
y=self.y.value,
z=self.z.value,
mode=self.marker_mode.value,
title=self.title.value,
xTitle=self.xtitle.value,
yTitle=self.ytitle.value,
zTitle=self.ztitle.value,
colorscale=self.color_scale.value,
text=self.text.value,
size=self.size.value,
layout_update=layout
) #weird bug where kind and colors sometimes doesnt apply propperly
else:
self.layout_d.visible = False
self._df[list(self.columns.value)].iplot(
kind=self.kind.value,
barmode=self.barmode.value,
colors=self.cmap.map_data(range(len(self.columns.value)),
hex=True),
theme=self.theme.value,
x=self.x.value,
y=self.y.value,
z=self.z.value,
mode=self.marker_mode.value,
title=self.title.value,
xTitle=self.xtitle.value,
yTitle=self.ytitle.value,
zTitle=self.ztitle.value,
colorscale=self.color_scale.value,
text=self.text.value,
size=self.size.value,
fill=self.fill.value,
subplots=self.subplots.value,
shared_xaxes=self.shared_xaxes.value,
shared_yaxes=self.shared_yaxes.value,
symbol=self.symbol.value,
bins=bins,
bargap=bgap,
bargroupgap=bgroupgap,
orientation=self.ori.value,
dash=self.dash.value,
histnorm=self.histnorm.value,
histfunc=self.histfunc.value,
categories=self.categories.value,
secondary_y=list(self.secondary_y.value))
def cisco_data(url):
requests = 0
start_time = time()
for page in pages:
#make a get request
#cisco = get('https://www.trustradius.com/products/cisco-webex-meetings/reviews?f='+page, headers = headers)
cisco = get(url + '?f=' + page, headers=headers)
#pause the loop for 8-20 seconds
sleep(randint(8, 20))
#monitor the requests
requests += 1
elapsed_time = time() - start_time
print('Request:{}; Frequency: {} requests/s'.format(
requests, requests / elapsed_time))
clear_output(wait=True)
#show a warning if a non 200 status code is returned
if cisco.status_code != 200:
warn('Request: {}; Status code: {}'.format(requests,
response.status_code))
cisco_soup = BeautifulSoup(cisco.text, 'html.parser')
#find the major tag peculiar to each review
container = cisco_soup.find_all("div", class_='serp-header')
container2 = cisco_soup.find_all("div", class_='serp-body')
#iterate through the major tag
for con in container:
#scrape the review body
if not con.find('h3'):
review_title = '-'
else:
review_title = con.find('h3').text
review_titles.append(review_title)
#scrape the review dates
if not con.select('div.review-date'):
review_date = '-'
else:
review_date = con.select('div.review-date')[0].text
review_dates.append(review_date)
#scrape the user scores
if not con.select('div.trust-score__score span'):
user_score = '-'
else:
user_score = con.select('div.trust-score__score span')[1].text
user_scores.append(user_score)
#scrape the user names.
if not con.select('div.name'):
name = '-'
else:
name = con.select('div.name')[0].text
names.append(name)
#scrape the user positions.
if not con.select('div.position'):
position = '-'
else:
position = con.select('div.position')[0].text
positions.append(position)
#scrape the company
if not con.select('span.company'):
company = '-'
else:
company = con.select('span.company')[0].text
companys.append(company)
#scrape the industry tupe.
if not con.select('span.industry-type'):
IndustryType = '-'
else:
IndustryType = con.select('span.industry-type')[0].text
IndustryTypes.append(IndustryType)
#scrape the employees count
if not con.select('span.size'):
Employee = '-'
else:
Employee = con.select('span.size')[0].text
Employees.append(Employee)
#scrape the review body
for con in container2:
if not con.select('div.question-response-container')[0]:
review_body = '-'
else:
review_body = con.select(
'div.question-response-container')[0].text
review_bodys.append(review_body)
#scrape the review pros
if not con.select('ul.pros'):
review_pro = '-'
else:
review_pro = con.select('ul.pros')[0].text
review_pros.append(review_pro)
#scrape the review cons
if not con.select('ul.cons'):
review_con = '-'
else:
review_con = con.select('ul.cons')[0].text
review_cons.append(review_con)
def anime_scraper(self, tot_page_num, pause_sec, is_clear_output=True):
'''
Method to scrape a list of animes
Arguments:
tot_page_num: total number of pages to scrape
pause_sec: number of seconds to pause before going to the next page
is_clear_output: whether to clear output
'''
# address of the page to start scraping
start_site_string = 'https://myanimelist.net/topanime.php'
# initialize some variable
start_time = time()
num_request = 0
page_num = 1
anime_num = 0
# initialize the dataframe with column names
anime_dataset_columns = ['anime', 'score', 'popularity',
'season_start', 'year_start', 'season_end', 'year_end',
'num_episodes', 'anime_type']
self.df_ = pd.DataFrame(columns=anime_dataset_columns)
# request the initial page
source = requests.get(start_site_string).text
soup = BeautifulSoup(source, 'lxml')
# update number of requests
num_request += 1
# pause for the number of second specified so as not to overburden the website
sleep(pause_sec)
# loop until some specified condition
while True:
# loop through each animes on the page
for anime in soup.find_all('tr', class_='ranking-list'):
# get information about the anime
info_list = anime.find('div', class_='information di-ib mt4').text.strip().split()
# get the information only if the information is stored in a nice format
if len(info_list) == 10:
# get the title of the anime
anime_code = anime.find('a', class_='hoverinfo_trigger fl-l fs14 fw-b')
anime_name = anime_code.text if anime_code is not None else np.nan
# get the anime score
score_code = anime.find('span', class_='text on')
score = score_code.text if score_code is not None else np.nan
# get the type of the anime
anime_type = info_list[0]
# get the number of episodes and replace the missing values with -500
num_episodes = int(info_list[1].split('(')[-1].replace('?', '-500'))
# get the months when the animes started and ended
season_start, season_end = info_list[3], info_list[6]
# get the years when the animes started and ended
year_start, year_end = int(info_list[4]), int(info_list[7])
# get the popularity of the animes
popularity = int(info_list[-2].replace(',', ''))
# store the above information into a list and append it to the dataframe
all_info_list = [anime_name, score, popularity,
season_start, year_start, season_end, year_end,
num_episodes, anime_type]
info_dict = {column_name: info for column_name, info in zip(anime_dataset_columns, all_info_list)}
self.df_ = self.df_.append(info_dict, ignore_index=True)
else:
continue
# get the time elapsed and print the number of requests per second
time_elapsed = time() - start_time
print('processed {0:.3f} requests/s'.format(num_request/time_elapsed))
# print the number of pages processed
print('{}/{} pages processed'.format(page_num, tot_page_num))
# break the while loop if the specified number of pages has reached
if page_num == tot_page_num:
print('terminated because total page specified reached')
break
# request the next page and break if it failed, meaning the scraper has reached the maximum possible
# number of pages
try:
next_code = soup.find('a', class_='link-blue-box next')
source = requests.get(start_site_string + next_code['href']).text
soup = BeautifulSoup(source, 'lxml')
except TypeError:
print('terminated because no more pages can be accessed from the website')
break
# update the number of requests and pause for number of seconds specified
num_request += 1
sleep(pause_sec)
# clear output
if is_clear_output:
clear_output(wait=True)
# update the page number
page_num += 1
def on_button_confirm(_):
global attach
clear_output()
print('Allegato: ' + str(attach))
open_attachment(attach)
variable_parameters={name: []
for name in variables['names']},
model_reporters=model_reporters)
count = 0
for i in range(replicates):
for vals in param_values:
# Change parameters that should be integers
vals = list(vals)
vals[0] = int(vals[0])
# Transform to dict with parameter names and their values
variable_parameters = {}
for name, val in zip(variables['names'], vals):
variable_parameters[name] = val
variable_parameters['floorplan'] = floor_plans[0]
batch.run_iteration(variable_parameters, tuple(vals), count)
count += 1
clear_output()
print(f'{count / (len(param_values) * replicates) * 100:.2f}% done')
data = batch.get_model_vars_dataframe()
data.to_csv('data_results.csv')
print(data)
plt.scatter(data['human_count'], data['Total_steps'])
plt.savefig('human_count_total_steps.png')
plt.show()
