def data_prepare(x, verbous=True):
X = x.copy()
#Пока так
if verbous:
print('Склеиваем строки...')
X['Text'] = X['Txt'].apply(lambda x: ' '.join(x))
if verbous:
print('Готово\nНормализиция текстов...')
X['Words'] = pd.Series([[]]*len(X))
X['WrdCnt'] = pd.Series([0]*len(X))
X['Tokens'] = pd.Series([[]]*len(X))
X['TokCnt'] = pd.Series([0]*len(X))
if verbous:
pb = ProgressBar(max_value = len(X))
pb.start()
for i in range(0,len(X)):
if verbous:
pb.update(i)
w,t = text_preprocess(X.loc[i,'Text'])
X.at[i,'Words'] = w
X.at[i,'WrdCnt'] = len(w)
X.at[i,'Tokens'] = t
X.at[i,'TokCnt'] = len(t)
if verbous:
pb.finish()
if verbous:
X['TokCnt'].hist(bins=100)
X['WrdCnt'].hist(bins=100)
return X
def run(self, target, *args_iter, verbose=False):
tasks = list(zip(*args_iter))
n_tasks = len(tasks)
if verbose:
pbar = ProgressBar(max_value=n_tasks)
for i, task in enumerate(tasks):
target(*task)
if verbose:
pbar.update(i + 1)
class SimulationProgressBar(object):
""" Simulation progress bar
Shows the progress of a simulation towards the time it is scheduled to end, in simulation time
"""
def __init__(self, use=False):
""" Create a simulation progress bar
A `SimulationProgressBar` does nothing by default, so that it can be used without an
`if` statement and configured at run-time.
Args:
use (:obj:`bool`): whether to use a progress bar
"""
self.use = use
def start(self, time_max):
""" Start the simulation's progress bar
Args:
time_max (:obj:`float`): the simulation's end time
"""
if self.use:
self.bar = ProgressBar(widgets=[
widgets.Percentage(),
' ',
widgets.SimpleProgress(
format='%(value)d/%(max_value)d (time_max)'),
' ',
widgets.Bar(),
' ',
widgets.Timer(),
' ',
widgets.AdaptiveETA(),
],
max_value=time_max).start()
def progress(self, sim_time):
""" Advance the simulation's progress bar
Args:
sim_time (:obj:`float`): the simulation time
"""
if self.use:
self.bar.update(sim_time)
def end(self):
""" End the simulation's progress bar
"""
if self.use:
self.bar.finish()
def download(self, images, name):
clear_name = re.sub(r'[^a-zA-Z\s]+', '-', name)
dst = self.destination.joinpath(clear_name)
if dst.is_dir():
print('Destination exists. Skip')
return
dst.mkdir(parents=True)
progress = ProgressBar(max_value=len(images))
for idx, image in enumerate(images):
url = self.sticker_url(image)
with open(str(dst.joinpath('{}.png'.format(idx + 1))),
'wb') as file:
file.write(get(url).content)
progress.update(idx + 1)
def run(self, target, *args_iter, verbose=False):
workers_idle = [False] * self.num_workers
tasks = list(zip(*args_iter))
n_tasks = len(tasks)
if verbose:
pbar = ProgressBar(max_value=n_tasks)
while not all(workers_idle):
for i in range(self.num_workers):
if not self._pool[i].is_alive():
self._pool[i].terminate()
if len(tasks) > 0:
if verbose:
pbar.update(n_tasks - len(tasks))
#print(n_tasks-len(tasks))
next_task = tasks.pop(0)
self._pool[i] = _start_process(target, next_task)
else:
workers_idle[i] = True
if verbose:
pbar.finish()
def migrate(self):
for proj in iter(self.projects.keys()):
print('Creating issue for proj {}...'.format(proj))
bar = ProgressBar(max_value=len(self.projects[proj]['Issues']))
self.migration_errors = {
'milestone': [],
'github': [],
}
for index, issue in enumerate(self.projects[proj]['Issues']):
# Check if this issue has already been created on github
if proj in self.cached_data and issue[
'key'] in self.cached_data[proj]:
bar.update(index)
continue
# Check for milestone
if 'milestone_name' in issue:
issue['milestone'] = self.projects[proj]['Milestones'][
issue['milestone_name']]
if issue['milestone'] is None:
self.migration_errors['milestone'].append(
issue['title'])
continue
del issue['milestone_name']
if len(self.aliases) != 0:
result = []
for i, label in enumerate(issue['labels']):
if label in self.aliases:
if self.aliases[label] == 'DELETED':
continue
if self.aliases[label] != 'same':
result.append(self.aliases[label])
continue
result.append(label)
issue['labels'] = result
comments = issue['comments']
del issue['comments']
result = self._save_issue(proj, issue, comments)
if result is not True:
self.migration_errors['github'].append({
'issue':
issue,
'result':
result.json(),
'status':
result.status_code,
})
if result.status_code == 403:
raise StopIteration('Could not continue')
bar.update(index)
bar.update(len(self.projects[proj]['Issues']))
texts = []
codes = []
quotes = []
pb = ProgressBar(max_value=len(data))
pb.start()
for i, rec in enumerate(data):
msg_ids.append(rec['MsgId'])
creators.append(rec['Creator'])
timestamps.append(rec['Time'])
src_links.append(rec['SrcLink'])
del_reasons.append(rec['DelReason'])
texts.append(rec['Txt'])
codes.append(rec['Code'])
quotes.append(rec['Quotes'])
pb.update(i)
pb.finish()
data = None
print('Создаю DataFrame...')
df = pd.DataFrame({
'TopId': tid,
'MsgId': msg_ids,
'Creator': creators,
'Time': timestamps,
'SrcLink': src_links,
'DelReason': del_reasons,
'Txt': texts,
'Code': codes,
'Quotes': quotes
def flash_program(self, addr, data, verify=True):
flash_region = int(self.regions['spiflash'][0], 0)
flash_len = int(self.regions['spiflash'][1], 0)
if (addr + len(data) > flash_len):
print("Write data out of bounds! Aborting.")
exit(1)
# ID code check
code = self.flash_rdid(1)
print("ID code bytes 1-2: 0x{:08x}".format(code))
if code != 0x8080c2c2:
print("ID code mismatch")
exit(1)
code = self.flash_rdid(2)
print("ID code bytes 2-3: 0x{:08x}".format(code))
if code != 0x3b3b8080:
print("ID code mismatch")
exit(1)
# block erase
progress = ProgressBar(min_value=0, max_value=len(data), prefix='Erasing ').start()
erased = 0
while erased < len(data):
self.ping_wdt()
if (len(data) - erased >= 65536) and ((addr & 0xFFFF) == 0):
blocksize = 65536
else:
blocksize = 4096
while True:
self.flash_wren()
status = self.flash_rdsr(1)
if status & 0x02 != 0:
break
if blocksize == 4096:
self.flash_se4b(addr + erased)
else:
self.flash_be4b(addr + erased)
erased += blocksize
while (self.flash_rdsr(1) & 0x01) != 0:
pass
result = self.flash_rdscur()
if result & 0x60 != 0:
print("E_FAIL/P_FAIL set on erase, programming may fail, but trying anyways...")
if self.flash_rdsr(1) & 0x02 != 0:
self.flash_wrdi()
while (self.flash_rdsr(1) & 0x02) != 0:
pass
if erased < len(data):
progress.update(erased)
progress.finish()
print("Erase finished")
# program
# pad out to the nearest word length
if len(data) % 4 != 0:
data += bytearray([0xff] * (4 - (len(data) % 4)))
written = 0
progress = ProgressBar(min_value=0, max_value=len(data), prefix='Writing ').start()
while written < len(data):
self.ping_wdt()
if len(data) - written > 256:
chunklen = 256
else:
chunklen = len(data) - written
while True:
self.flash_wren()
status = self.flash_rdsr(1)
if status & 0x02 != 0:
break
self.burst_write(flash_region, data[written:(written+chunklen)])
self.flash_pp4b(addr + written, chunklen)
written += chunklen
if written < len(data):
progress.update(written)
progress.finish()
print("Write finished")
if self.flash_rdsr(1) & 0x02 != 0:
self.flash_wrdi()
while (self.flash_rdsr(1) & 0x02) != 0:
pass
# dummy reads to clear the "read lock" bit
self.flash_rdsr(0)
# verify
self.ping_wdt()
if verify:
print("Performing readback for verification...")
self.ping_wdt()
rbk_data = self.burst_read(addr + flash_region, len(data))
if rbk_data != data:
print("Errors were found in verification, programming failed")
exit(1)
else:
print("Verification passed.")
else:
print("Skipped verification at user request")
self.ping_wdt()
class DatabaseCache(object):
def __init__(self, file_name, read_only=False):
self.file_name = file_name
self.read_only = read_only
self.bar = None
def __enter__(self):
"""
Opens the database file and makes its copy in memory
"""
# File URI
if self.read_only:
uri = "file:%s?mode=ro" % self.file_name
else:
uri = "file:%s?mode=rwc" % self.file_name
# Open connections
self.memory_connection = sqlite3.connect(":memory:")
self.file_connection = sqlite3.connect(uri, uri=True)
# Load the database
print("Loading database from '{}'".format(self.file_name))
self.file_connection.backup(self.memory_connection,
pages=100,
progress=self._progress)
self.bar.finish()
self.bar = None
# Return the connection
return self.memory_connection
def __exit__(self, exc_type, exc_value, traceback):
"""
Writes back the database to file if the database was open as not read-only
"""
# Write back only if not read-only
if not self.read_only:
print("Dumping database to '{}'".format(self.file_name))
self.memory_connection.backup(self.file_connection,
pages=100,
progress=self._progress)
self.bar.finish()
self.bar = None
# Close connections
self.memory_connection.close()
self.file_connection.close()
def _progress(self, status, remaining, total):
"""
Prints database copy progress.
"""
if self.bar is None:
self.bar = ProgressBar(max_value=total)
else:
self.bar.update(total - remaining)