def data_prepare(x, verbous=True):
X = x.copy()
#Пока так
if verbous:
print('Склеиваем строки...')
X['Text'] = X['Txt'].apply(lambda x: ' '.join(x))
if verbous:
print('Готово\nНормализиция текстов...')
X['Words'] = pd.Series([[]]*len(X))
X['WrdCnt'] = pd.Series([0]*len(X))
X['Tokens'] = pd.Series([[]]*len(X))
X['TokCnt'] = pd.Series([0]*len(X))
if verbous:
pb = ProgressBar(max_value = len(X))
pb.start()
for i in range(0,len(X)):
if verbous:
pb.update(i)
w,t = text_preprocess(X.loc[i,'Text'])
X.at[i,'Words'] = w
X.at[i,'WrdCnt'] = len(w)
X.at[i,'Tokens'] = t
X.at[i,'TokCnt'] = len(t)
if verbous:
pb.finish()
if verbous:
X['TokCnt'].hist(bins=100)
X['WrdCnt'].hist(bins=100)
return X
def _progress(self, status, remaining, total):
"""
Prints database copy progress.
"""
if self.bar is None:
self.bar = ProgressBar(max_value=total)
else:
self.bar.update(total - remaining)
def migrate(self):
for proj in iter(self.projects.keys()):
print('Creating issue for proj {}...'.format(proj))
bar = ProgressBar(max_value=len(self.projects[proj]['Issues']))
self.migration_errors = {
'milestone': [],
'github': [],
}
for index, issue in enumerate(self.projects[proj]['Issues']):
# Check if this issue has already been created on github
if proj in self.cached_data and issue[
'key'] in self.cached_data[proj]:
bar.update(index)
continue
# Check for milestone
if 'milestone_name' in issue:
issue['milestone'] = self.projects[proj]['Milestones'][
issue['milestone_name']]
if issue['milestone'] is None:
self.migration_errors['milestone'].append(
issue['title'])
continue
del issue['milestone_name']
if len(self.aliases) != 0:
result = []
for i, label in enumerate(issue['labels']):
if label in self.aliases:
if self.aliases[label] == 'DELETED':
continue
if self.aliases[label] != 'same':
result.append(self.aliases[label])
continue
result.append(label)
issue['labels'] = result
comments = issue['comments']
del issue['comments']
result = self._save_issue(proj, issue, comments)
if result is not True:
self.migration_errors['github'].append({
'issue':
issue,
'result':
result.json(),
'status':
result.status_code,
})
if result.status_code == 403:
raise StopIteration('Could not continue')
bar.update(index)
bar.update(len(self.projects[proj]['Issues']))
def run(self, target, *args_iter, verbose=False):
tasks = list(zip(*args_iter))
n_tasks = len(tasks)
if verbose:
pbar = ProgressBar(max_value=n_tasks)
for i, task in enumerate(tasks):
target(*task)
if verbose:
pbar.update(i + 1)
def download(self, images, name):
clear_name = re.sub(r'[^a-zA-Z\s]+', '-', name)
dst = self.destination.joinpath(clear_name)
if dst.is_dir():
print('Destination exists. Skip')
return
dst.mkdir(parents=True)
progress = ProgressBar(max_value=len(images))
for idx, image in enumerate(images):
url = self.sticker_url(image)
with open(str(dst.joinpath('{}.png'.format(idx + 1))),
'wb') as file:
file.write(get(url).content)
progress.update(idx + 1)
def start(self, time_max):
""" Start the simulation's progress bar
Args:
time_max (:obj:`float`): the simulation's end time
"""
if self.use:
self.bar = ProgressBar(widgets=[
widgets.Percentage(),
' ',
widgets.SimpleProgress(
format='%(value)d/%(max_value)d (time_max)'),
' ',
widgets.Bar(),
' ',
widgets.Timer(),
' ',
widgets.AdaptiveETA(),
],
max_value=time_max).start()
def run(self, target, *args_iter, verbose=False):
workers_idle = [False] * self.num_workers
tasks = list(zip(*args_iter))
n_tasks = len(tasks)
if verbose:
pbar = ProgressBar(max_value=n_tasks)
while not all(workers_idle):
for i in range(self.num_workers):
if not self._pool[i].is_alive():
self._pool[i].terminate()
if len(tasks) > 0:
if verbose:
pbar.update(n_tasks - len(tasks))
#print(n_tasks-len(tasks))
next_task = tasks.pop(0)
self._pool[i] = _start_process(target, next_task)
else:
workers_idle[i] = True
if verbose:
pbar.finish()
with open('data/all_data.pkl', 'rb') as f:
tid, data = pk.load(f)
if not data:
exit(-1)
msg_ids = []
creators = []
timestamps = []
src_links = []
del_reasons = []
texts = []
codes = []
quotes = []
pb = ProgressBar(max_value=len(data))
pb.start()
for i, rec in enumerate(data):
msg_ids.append(rec['MsgId'])
creators.append(rec['Creator'])
timestamps.append(rec['Time'])
src_links.append(rec['SrcLink'])
del_reasons.append(rec['DelReason'])
texts.append(rec['Txt'])
codes.append(rec['Code'])
quotes.append(rec['Quotes'])
pb.update(i)
pb.finish()
data = None
def flash_program(self, addr, data, verify=True):
flash_region = int(self.regions['spiflash'][0], 0)
flash_len = int(self.regions['spiflash'][1], 0)
if (addr + len(data) > flash_len):
print("Write data out of bounds! Aborting.")
exit(1)
# ID code check
code = self.flash_rdid(1)
print("ID code bytes 1-2: 0x{:08x}".format(code))
if code != 0x8080c2c2:
print("ID code mismatch")
exit(1)
code = self.flash_rdid(2)
print("ID code bytes 2-3: 0x{:08x}".format(code))
if code != 0x3b3b8080:
print("ID code mismatch")
exit(1)
# block erase
progress = ProgressBar(min_value=0, max_value=len(data), prefix='Erasing ').start()
erased = 0
while erased < len(data):
self.ping_wdt()
if (len(data) - erased >= 65536) and ((addr & 0xFFFF) == 0):
blocksize = 65536
else:
blocksize = 4096
while True:
self.flash_wren()
status = self.flash_rdsr(1)
if status & 0x02 != 0:
break
if blocksize == 4096:
self.flash_se4b(addr + erased)
else:
self.flash_be4b(addr + erased)
erased += blocksize
while (self.flash_rdsr(1) & 0x01) != 0:
pass
result = self.flash_rdscur()
if result & 0x60 != 0:
print("E_FAIL/P_FAIL set on erase, programming may fail, but trying anyways...")
if self.flash_rdsr(1) & 0x02 != 0:
self.flash_wrdi()
while (self.flash_rdsr(1) & 0x02) != 0:
pass
if erased < len(data):
progress.update(erased)
progress.finish()
print("Erase finished")
# program
# pad out to the nearest word length
if len(data) % 4 != 0:
data += bytearray([0xff] * (4 - (len(data) % 4)))
written = 0
progress = ProgressBar(min_value=0, max_value=len(data), prefix='Writing ').start()
while written < len(data):
self.ping_wdt()
if len(data) - written > 256:
chunklen = 256
else:
chunklen = len(data) - written
while True:
self.flash_wren()
status = self.flash_rdsr(1)
if status & 0x02 != 0:
break
self.burst_write(flash_region, data[written:(written+chunklen)])
self.flash_pp4b(addr + written, chunklen)
written += chunklen
if written < len(data):
progress.update(written)
progress.finish()
print("Write finished")
if self.flash_rdsr(1) & 0x02 != 0:
self.flash_wrdi()
while (self.flash_rdsr(1) & 0x02) != 0:
pass
# dummy reads to clear the "read lock" bit
self.flash_rdsr(0)
# verify
self.ping_wdt()
if verify:
print("Performing readback for verification...")
self.ping_wdt()
rbk_data = self.burst_read(addr + flash_region, len(data))
if rbk_data != data:
print("Errors were found in verification, programming failed")
exit(1)
else:
print("Verification passed.")
else:
print("Skipped verification at user request")
self.ping_wdt()
along with this program. If not, see <http://www.gnu.org/licenses/>.
Please contact with me by E-mail: [email protected]
"""
import os
import numpy as np
import pandas as pd
from progressbar.bar import ProgressBar
topics = []
drl = os.listdir('data/download')
drl.remove('arch.pkl')
drl.remove('topic.pkl')
drl.remove('topic_num.pkl')
pb = ProgressBar(max_value = len(drl))
pb.start()
for i, name in enumerate(drl):
bname, ext = name.split('.')
if 'topic_' in bname:
topics.append(pd.read_pickle('data/download/'+name))
pb.update(i)
pb.finish()
topic_data = pd.concat(topics, ignore_index=True, copy=True, sort=False)
N = len(topic_data)
deleted = len(topic_data[topic_data['DelReason'] != ''])
code = np.sum(np.array([len(code) for code in topic_data['Code']]))
quotes = np.sum(np.array([len(code) for code in topic_data['Quotes']]))