def ratings_from_ids_2(self, id_iter):
with tpe() as executor:
future_to_rate = {
executor.submit(self.json_from_url, self.r_from_id_url(group)):
(i + 1, group)
for (i, group) in enumerate(self.grouper(id_iter, 100, ''))
}
for future in as_comp(future_to_rate):
i, group = future_to_rate[future]
self.ow.print(f'Getting Player ratings - {i*100}')
try:
ratings = future.result()
if not ratings.get('status') == 'ok':
print(ratings)
except Exception as exc:
print(
f'rating-ing group {i} generated an exception: {exc}')
else:
r_data = ratings.get('data')
if r_data is not None:
for player_id, player in r_data.items():
if not player:
continue
player['id'] = player_id
yield player
def startCrawl(self, toggle_print = True):
def parse_new_urls(): #parse catalog page that returns(gets) urls of hero page
while self.downloader.updatePageSource():
raw_html = self.downloader.downloadPageSource()
new_urls, next_page_tag = self.parser.parse(raw_html, self.downloader.start_url)
self.url_manager.addNewUrl(new_urls)
def parse_detail(in_url): #parse hero page
if toggle_print:
print("%d: gathering data from: %s" % (self.count, in_url))
page_source = self.downloader.getPageSourceViaRequest(in_url)
data = self.parser.parseHero(page_source)
self.outputer.collectData(data)
self.count += 1
raw_html = self.downloader.downloadPageSource()
new_urls, next_page_tag = self.parser.parse(raw_html, self.downloader.start_url)
self.downloader.next_page_tag = next_page_tag
self.url_manager.addNewUrl(new_urls)
gainNewUrlsThread = Thread(target=parse_new_urls)
gainNewUrlsThread.run()
executor = tpe(self.thread_pool_size)
while not self.url_manager.isEmpty():
all_urls = self.url_manager.getUrls()
executor.map(parse_detail, all_urls)
if self.outputer.data_count > self.outputer.buffer_size:
self.buffer_trigger.set()
self.outputer.end_writing = True
self.buffer_trigger.set()
executor.shutdown(wait=True)
def startCrawl(self, root_url, toggle_print=True):
def parse_once(new_url):
try:
if toggle_print:
print("crawling %d : %s" % (self.count, new_url))
self.downloader.url = new_url
html_out = self.downloader.startDownload()
next_url, data = self.parser.startParse(new_url, html_out)
self.urls.addNewUrl(next_url)
self.outputer.collectData(data)
self.count += 1
time.sleep(random.random() * 10)
except:
print("failed\n")
else:
print("successful\n")
assert isinstance(toggle_print, bool)
self.urls.addNewUrl(root_url)
path = str(os.path.dirname(
os.path.abspath(__file__))) + "/SushiReview.txt"
write_thread = Thread(target=self.write_method, args=(path, ))
write_thread.start()
while not self.urls.isEmpty():
executor = tpe(self.thread_pool_size)
all_urls = self.urls.getAllUrls()
executor.map(parse_once, all_urls)
executor.shutdown(wait=True)
if len(self.outputer.data) > self.outputer.buffer_size:
self.buffer_trigger.set()
self.outputer.end_writing = True
self.buffer_trigger.set()
write_thread.join()
print("\n\nwrote %d lines of file in total. file located at: %s" %
(self.outputer.total_data_count, path))
def do_it_lambda(tag_name):
net_tasks = []
with tpe(max_workers=None) as executor, requests.Session() as session:
session.headers.update(request_header)
tasks_with_tag = get_tasks_by_tag_name(session, tag_name)
tasks = [t['gid'] for t in tasks_with_tag]
tasks_details = {
executor.submit(get_json_response,
session,
'tasks/{}'.format(t),
opt_fields=','.join(task_fields)): t
for t in tasks
}
for f in concurrent.futures.as_completed(tasks_details):
fut = tasks_details[f]
try:
t = f.result()
net_tasks.append(t)
except Exception as exc:
print('Error: {}'.format(exc))
records = [task_as_record(t) for t in net_tasks]
df = pd.DataFrame(records)
df.to_csv(csv_file, index=False)
return df
def runner():
urls = [
"https://www.gutenberg.org/cache/epub/376/pg376.txt",
"https://www.gutenberg.org/files/84/84-0.txt",
"https://www.gutenberg.org/cache/epub/844/pg844.txt",
]
with tpe(max_workers=8) as exe:
results = exe.map(download_nd_save, urls)
combine_files()
def ids_from_names_generator_2(self, names):
with tpe() as executor:
future_to_id = {
executor.submit(self.id_from_name, i + 1, len(names), name):
(i + 1, name)
for (i, name) in enumerate(names)
}
for future in as_comp(future_to_id):
i, name = future_to_id[future]
try:
id = future.result()
except Exception as exc:
print(
f'ID-ing {name} generated an exception: {exc}, retrying'
)
# future_to_id[executor.submit(self.id_from_name, i, len(names), name)] = (i, name)
else:
yield id
def main():
global filename, questions
if len(sys.argv) > 1 and path.isfile(sys.argv[1]):
filename = sys.argv[1]
elif path.isfile("sample.mp3"):
filename = "sample.mp3"
elif path.isfile("sample.wav"):
filename = "sample.wav"
else:
return False
if len(sys.argv) > 2 and path.isfile(sys.argv[2] + '.csv'):
rows = csv.reader(open(sys.argv[2] + '.csv', 'r'))
questions = []
for row in rows:
tmp_ans = row[1:]
answers = []
for a in tmp_ans:
answers.append(a.strip())
questions.append({"question": row[0], "answer": answers})
with tpe(max_workers=2) as e:
e.submit(playMusic)
e.submit(showQuestions)
def allLinks(root):
ans = []
for e in root.getchildren():
if e.tag == 'a':
ans += [e.attrib['href']]
else:
ans += allLinks(e)
return set(ans)
def download(url):
path = os.path.join(dest, url.split('/')[-1])
data = requests.get(url)
with open(path, 'wb') as f:
f.write(data.content)
links = allLinks(root)
pool = tpe(max_workers=4)
t = time.clock()
fs = []
for li in links:
print("downloading ...", li)
f = pool.submit(download, site + li)
fs.append(f)
print("Done")
wait(fs)
print(time.clock() - t)
def mt_func(*args, **kwargs):
return tpe().submit(func, *args, **kwargs)
def by_thread(func, items, callback, workers):
with tpe(max_workers=workers) as executor:
futures = (executor.submit(func, item) for item in items)
for future in comp(futures):
callback(future.result())
def rewrite_files(n):
for k in range(n):
write_files('rewritten_files\\' + str(k) + '.txt',
get_file_content('given_files\\' + str(k) + '.txt'))
# запис до одного файлу
def write_one_file(k):
write_files('rewritten_files\\' + str(k) + '.txt',
get_file_content('given_files\\' + str(k) + '.txt'))
if __name__ == '__main__':
start = time.time()
with tpe(8) as executor:
executor.map(write_one_file, range(1000))
print('Час виконання ф-ції паралельно за допомогою потоків ThreadPoolExecutor: ',
time.time() - start, 'c.')
start = time.time()
with ppe(8) as executor:
executor.map(write_one_file, range(1000))
print('Час виконання ф-ції паралельно за допомогою процесів ProcessPoolExecutor: ',
time.time() - start, 'c.')
start = time.time()
rewrite_files(1000)
print('Час виконання ф-ції послідовно: ', time.time() - start, 'c.')
pr = Predators('Вовк', parent=None)
producer = Producers(f)
consumer = Consumers(f)
consumer.phytophages = [pp]
consumer.predators = [pr]
detr1 = Insects('Insect', f)
detr2 = Worms('Worms', f)
detrio = Detriophages(f, [detr1, detr2])
abio = AbioticFactors(f)
# відтворення життєвих циклів
# на основі Sequence diagram
# запуск в паралельному потоці життєвого циклу консументу
th = Thread(target=pr.main_life_cycle, args=(pr, ))
th.start()
# відтворення життєвих циклів
# на основі activity diagram
with tpe(max_workers=3) as executor:
executor.submit(pr.breath)
executor.submit(pr.main_life_cycle)
executor.submit(pr.sleep)
async def run(self, queue, ctx, bot, search):
opts = {
'format': 'bestaudio/best',
'outtmpl':
f'{ctx.guild.id}/{self.outtmpl_seed()}%(extractor)s_%(id)s.%(ext)s',
'restrictfilenames': True,
'noplaylist': False,
'nocheckcertificate': True,
'ignoreerrors': False,
'logtostderr': False,
'quiet': True,
'no_warnings': True,
'default_search': 'auto',
'source_address': '0.0.0.0',
'playlistend': 50,
}
ytdl = youtube_dl.YoutubeDL(opts)
ytdl.params['extract_flat'] = True
ef_info = ytdl.extract_info(download=False, url=search)
ytdl.params['extract_flat'] = False
if 'entries' in ef_info:
length = len(ef_info['entries'])
else:
length = 1
for v in range(1, length + 1):
try:
ytdl.params.update({'playlistend': v, 'playliststart': v})
tdl = functools.partial(ytdl.extract_info,
download=True,
url=search)
info = await bot.loop.run_in_executor(tpe(max_workers=4), tdl)
except Exception as e:
self._ytdl_error = e
if length <= 1:
return await ctx.send(
f'**There was an error processing your song.** ```css\n[{e}]\n```'
)
else:
continue
if 'entries' in info:
info = info['entries'][0]
duration = info.get('duration') or self.get_duration(
info.get('url'))
song_info = {
'title': info.get('title'),
'weburl': info.get('webpage_url'),
'duration': duration,
'views': info.get('view_count'),
'thumb': info.get('thumbnail'),
'requester': ctx.author,
'upload_date': info.get('upload_date', '\uFEFF')
}
if length == 1:
await ctx.send(
f'```ini\n[Added {song_info["title"]} to the queue.]\n```',
delete_after=15)
try:
await queue.put({
'source': ytdl.prepare_filename(info),
'info': song_info,
'channel': ctx.channel
})
except Exception as e:
self._ytdl_error = e
import tkinter
from easydict import EasyDict as edict
from tkinter.filedialog import askdirectory, askopenfilename, asksaveasfilename
import os
from PIL import Image, ImageTk
import numpy as np
from concurrent.futures import ThreadPoolExecutor as tpe
from functools import lru_cache
pool = tpe(4)
tk = tkinter
shape = (1024, 768)
args = edict({
'input': None,
'output': '',
'cls_list': ['0', '1', '2', '3', '4'],
'filelist': [],
'filelabel': {},
'curlabel': -1,
'curselect': 0,
'im': None,
'imname': None,
'imidx': -1,
'clsidx': -1
})
class labeled(object):
num = 0
