def data_prepare(x, verbous=True):
X = x.copy()
#Пока так
if verbous:
print('Склеиваем строки...')
X['Text'] = X['Txt'].apply(lambda x: ' '.join(x))
if verbous:
print('Готово\nНормализиция текстов...')
X['Words'] = pd.Series([[]]*len(X))
X['WrdCnt'] = pd.Series([0]*len(X))
X['Tokens'] = pd.Series([[]]*len(X))
X['TokCnt'] = pd.Series([0]*len(X))
if verbous:
pb = ProgressBar(max_value = len(X))
pb.start()
for i in range(0,len(X)):
if verbous:
pb.update(i)
w,t = text_preprocess(X.loc[i,'Text'])
X.at[i,'Words'] = w
X.at[i,'WrdCnt'] = len(w)
X.at[i,'Tokens'] = t
X.at[i,'TokCnt'] = len(t)
if verbous:
pb.finish()
if verbous:
X['TokCnt'].hist(bins=100)
X['WrdCnt'].hist(bins=100)
return X
def iterate_urls(driver, df):
length = len(df['reuters_url'])
bar = PB(max_value=length).start()
result = [
get_the_time(driver, link, bar, index)
for index, link in enumerate(df['reuters_url'])
]
bar.finish()
return driver, result
class SimulationProgressBar(object):
""" Simulation progress bar
Shows the progress of a simulation towards the time it is scheduled to end, in simulation time
"""
def __init__(self, use=False):
""" Create a simulation progress bar
A `SimulationProgressBar` does nothing by default, so that it can be used without an
`if` statement and configured at run-time.
Args:
use (:obj:`bool`): whether to use a progress bar
"""
self.use = use
def start(self, time_max):
""" Start the simulation's progress bar
Args:
time_max (:obj:`float`): the simulation's end time
"""
if self.use:
self.bar = ProgressBar(widgets=[
widgets.Percentage(),
' ',
widgets.SimpleProgress(
format='%(value)d/%(max_value)d (time_max)'),
' ',
widgets.Bar(),
' ',
widgets.Timer(),
' ',
widgets.AdaptiveETA(),
],
max_value=time_max).start()
def progress(self, sim_time):
""" Advance the simulation's progress bar
Args:
sim_time (:obj:`float`): the simulation time
"""
if self.use:
self.bar.update(sim_time)
def end(self):
""" End the simulation's progress bar
"""
if self.use:
self.bar.finish()
def run(self, target, *args_iter, verbose=False):
workers_idle = [False] * self.num_workers
tasks = list(zip(*args_iter))
n_tasks = len(tasks)
if verbose:
pbar = ProgressBar(max_value=n_tasks)
while not all(workers_idle):
for i in range(self.num_workers):
if not self._pool[i].is_alive():
self._pool[i].terminate()
if len(tasks) > 0:
if verbose:
pbar.update(n_tasks - len(tasks))
#print(n_tasks-len(tasks))
next_task = tasks.pop(0)
self._pool[i] = _start_process(target, next_task)
else:
workers_idle[i] = True
if verbose:
pbar.finish()
codes = []
quotes = []
pb = ProgressBar(max_value=len(data))
pb.start()
for i, rec in enumerate(data):
msg_ids.append(rec['MsgId'])
creators.append(rec['Creator'])
timestamps.append(rec['Time'])
src_links.append(rec['SrcLink'])
del_reasons.append(rec['DelReason'])
texts.append(rec['Txt'])
codes.append(rec['Code'])
quotes.append(rec['Quotes'])
pb.update(i)
pb.finish()
data = None
print('Создаю DataFrame...')
df = pd.DataFrame({
'TopId': tid,
'MsgId': msg_ids,
'Creator': creators,
'Time': timestamps,
'SrcLink': src_links,
'DelReason': del_reasons,
'Txt': texts,
'Code': codes,
'Quotes': quotes
})
def flash_program(self, addr, data, verify=True):
flash_region = int(self.regions['spiflash'][0], 0)
flash_len = int(self.regions['spiflash'][1], 0)
if (addr + len(data) > flash_len):
print("Write data out of bounds! Aborting.")
exit(1)
# ID code check
code = self.flash_rdid(1)
print("ID code bytes 1-2: 0x{:08x}".format(code))
if code != 0x8080c2c2:
print("ID code mismatch")
exit(1)
code = self.flash_rdid(2)
print("ID code bytes 2-3: 0x{:08x}".format(code))
if code != 0x3b3b8080:
print("ID code mismatch")
exit(1)
# block erase
progress = ProgressBar(min_value=0, max_value=len(data), prefix='Erasing ').start()
erased = 0
while erased < len(data):
self.ping_wdt()
if (len(data) - erased >= 65536) and ((addr & 0xFFFF) == 0):
blocksize = 65536
else:
blocksize = 4096
while True:
self.flash_wren()
status = self.flash_rdsr(1)
if status & 0x02 != 0:
break
if blocksize == 4096:
self.flash_se4b(addr + erased)
else:
self.flash_be4b(addr + erased)
erased += blocksize
while (self.flash_rdsr(1) & 0x01) != 0:
pass
result = self.flash_rdscur()
if result & 0x60 != 0:
print("E_FAIL/P_FAIL set on erase, programming may fail, but trying anyways...")
if self.flash_rdsr(1) & 0x02 != 0:
self.flash_wrdi()
while (self.flash_rdsr(1) & 0x02) != 0:
pass
if erased < len(data):
progress.update(erased)
progress.finish()
print("Erase finished")
# program
# pad out to the nearest word length
if len(data) % 4 != 0:
data += bytearray([0xff] * (4 - (len(data) % 4)))
written = 0
progress = ProgressBar(min_value=0, max_value=len(data), prefix='Writing ').start()
while written < len(data):
self.ping_wdt()
if len(data) - written > 256:
chunklen = 256
else:
chunklen = len(data) - written
while True:
self.flash_wren()
status = self.flash_rdsr(1)
if status & 0x02 != 0:
break
self.burst_write(flash_region, data[written:(written+chunklen)])
self.flash_pp4b(addr + written, chunklen)
written += chunklen
if written < len(data):
progress.update(written)
progress.finish()
print("Write finished")
if self.flash_rdsr(1) & 0x02 != 0:
self.flash_wrdi()
while (self.flash_rdsr(1) & 0x02) != 0:
pass
# dummy reads to clear the "read lock" bit
self.flash_rdsr(0)
# verify
self.ping_wdt()
if verify:
print("Performing readback for verification...")
self.ping_wdt()
rbk_data = self.burst_read(addr + flash_region, len(data))
if rbk_data != data:
print("Errors were found in verification, programming failed")
exit(1)
else:
print("Verification passed.")
else:
print("Skipped verification at user request")
self.ping_wdt()