def creat_keypair_mulitthreading(how_many_pairs, paths):
#created file and a+
the_path = "keypair_file" + time.ctime().replace(":", ".") + ".pem"
keypair_file = open(the_path, "w+")
Pools = thread_pool(16)
Pools.map(creat_keypair, [(2048, str(the_path))] * int(how_many_pairs))
Pools.close()
def download_using_thread_pool(urls):
pool = thread_pool(100)
# 第一个参数为函数名，第二个参数一个可迭代对象，为函数所需的参数列表
resps = pool.map(session.get,urls)
pool.close()
pool.join()
return resps
def tutorialspoint_all():
url = "https://www.tutorialspoint.com/tutorialslibrary.htm"
print("Connecting to Tutorialspoint")
while True:
try:
page_response = requests.get(url,
headers={'User-Agent': 'Chrome'},
timeout=5)
soup = BeautifulSoup(page_response.content, "html.parser")
str_soup = str(soup)
except:
# raise
print("Could not connect, trying again in 1 seconds!")
time.sleep(1)
continue
else:
break
links = []
for ul in soup.find_all("ul", attrs={"class": "menu"}):
for li in ul.find_all("li"):
for a in li.find_all("a"):
links.append(url[:url.rfind("/")] + a['href'])
with thread_pool(processes=3) as pool:
pool.map(tutorialspoint, links)
def __async_analyze(self, target_list: list, interrupt=False):
def analyze(post):
def _anl_():
post_i = {}
if self.schema['post_type'] == 'url':
post_i = self.__post_analyze(post, self.schema['domain'])
else:
post_i = self.__post_analyze(post)
if not 'name' in post_i or post_i['name'] in [
None, ''
]: # invalid datapack
print('error: datapack name cannot be null (may be 404)')
return
self.info_list.append(post_i)
if interrupt:
_anl_()
else:
try:
_anl_()
except Exception as e:
print(post, ':', 'got error :', e)
self.retry_list.append(post)
self.current += 1
print('done', self.current, '/', str(self.total))
self.current = 0
self.total = target_list.__len__()
pool = thread_pool()
pool.map(analyze, target_list)
pool.close()
pool.join()
del self.current
del self.total
def __init__(self, bus_name):
try:
super().__init__(bus_name, "/InstallPkgs")
except dbus.DBusException:
raise PkgException("Exception in install pkgs")
self._thread_pool = thread_pool(10)
self._mp_pool = mp_pool(10)
self._result = None
self._lock = threading.Lock()
config_dir = str(Path.home()) + '/' + CONFIG_DIR
self.cfg = ConfigObj(config_dir + CONFIG_FILE)
def process_fn(f):
thread_count = int(os.getenv("BRO_PDNS_THREADS", "1"))
processed = 0
aggregated = list(aggregate_file(f))
batches = window(aggregated, 10000)
pool = thread_pool(thread_count)
processed = sum(pool.imap(load_records, batches, chunksize=1))
print "%d" % processed
def process_fn(f):
thread_count = int(os.getenv("BRO_PDNS_THREADS", "1"))
processed = 0
aggregated = list(aggregate_file(f))
batches = window(aggregated, 10000)
pool = thread_pool(thread_count)
processed = sum(pool.imap(load_records, batches, chunksize=1))
print "%d" % processed
def __init__(self, team, arch):
super().__init__()
self._lp_team = team
self._launchpad = None
self._lp_arch = arch
self._lp_ppa = ""
self._pkgs = []
self.debs = []
self._thread_pool = thread_pool(10)
self._mp_pool = mp_pool(10)
self._result = None
self._lock = threading.Lock()
def mulit_threading_search_path(start_path):
SPpool = thread_pool(8)
templist = []
templist2 = []
sysinfo_path(start_path)
while not diffQ.empty() or not diffQA.empty():
templist.append(diffQ.get())
try:
SPpool.map(sysinfo_path, templist)
except Exception, e:
pass
# if you pool.close() too early something mistakes will happen for example, assertionerror.
#Because you close before it complete,in reality we do not need close.
#Close() -- Prevents any more tasks from being submitted to the pool. Once all the tasks have been completed the worker processes will exit.
templist = []
templist2.append(diffQA.get())
def tutorialspoint(url):
tutorial = url.split("/")[-2]
domain_name = url.split(tutorial)[0][:-1]
while True:
try:
page_response = requests.get(url,
headers={'User-Agent': 'Chrome'},
timeout=1)
soup = BeautifulSoup(page_response.content, "html.parser")
str_soup = str(soup)
except:
# raise
print(
"Could not connect to tutorialspoint, trying again in 1 seconds!"
)
time.sleep(1)
continue
else:
break
print("Downloading " + tutorial)
links = []
for ul in soup.find_all("ul", attrs={"class": "toc chapters"}):
for li in ul.find_all("li"):
for a in li.find_all("a"):
if ".htm" in a['href']:
links.append(domain_name + a['href'])
pages = []
with thread_pool(processes=2 * mp.cpu_count()) as pool:
pages = pool.map(tutorialspoint_get_page, links)
head = str_soup[:str_soup.find("<body")] + '\n<body>\n'
head = head.replace(
"<style>",
'<style>\n.prettyprint{\nbackground-color:#D3D3D3;\nfont-size: 12px;}\n'
)
end = '\n</body>\n</html>'
page = head + "".join(pages) + end
with open('..' + os.sep + 'temp' + os.sep + tutorial + ".html", "w") as f:
f.write(page)
print(tutorial + " download completed")
return
def javatpoint_all():
url = "https://www.javatpoint.com/"
print("Connecting to Javatpoint")
while True:
try:
page_response = requests.get(url,
headers={'User-Agent': 'Chrome'},
timeout=5)
soup = BeautifulSoup(page_response.content, "html.parser")
str_soup = str(soup)
except:
# raise
print("Could not connect, trying again in 1 seconds!")
time.sleep(1)
continue
else:
break
tutorials = []
links = []
break_condition = False
for div in soup.find_all("div", attrs={"class": "firsthomecontent"}):
if break_condition:
break
for a in div.find_all("a"):
if "forum" in a["href"]:
break_condition = True
break
if "http" in a["href"]:
links.append(a["href"])
else:
links.append("https://www.javatpoint.com/" + a["href"])
for tutorial_name in a.find_all("p"):
tutorials.append(tutorial_name.contents[0])
with thread_pool(processes=3) as pool:
pool.starmap(javatpoint, zip(links, tutorials))
return
def javatpoint(url, tutorial=None):
if tutorial is None:
tutorial = url.split("/")[-1]
print("Downloading " + tutorial)
while True:
try:
page_response = requests.get(url,
headers={'User-Agent': 'Chrome'},
timeout=5)
soup = BeautifulSoup(page_response.content, "html.parser")
str_soup = str(soup)
except:
# raise
print(
"Could not connect to javatpoint, trying again in 1 seconds!")
time.sleep(1)
continue
else:
break
links = []
for div in soup.find_all("div", attrs={"class": "leftmenu"}):
for a in div.find_all("a"):
links.append(url[:url.rfind("/") + 1] + a["href"])
pages = []
with thread_pool(processes=2 * mp.cpu_count()) as pool:
pages = pool.map(javatpoint_get_page, links)
page = str_soup[:str_soup.find("<body")] + "\n<body>\n" + "".join(
pages) + "\n</body>\n</html>"
with open('..' + os.sep + 'temp' + os.sep + tutorial + ".html", "w") as f:
f.write(page)
# os.system('xdg-open page.html')
# os.system('xdg-open page.pdf')
print(tutorial + " download completed")
return
def restore_secondary_indexes():
"""Restores dropped secondary indexes.
This is done to replicate the original databases' index configuration. Indexes
commands are stored in files by table name so that each table's indexes can be
loaded in parallel.
Raises:
RuntimeError: If restoring any secondary index failed.
"""
logging.info("Restoring secondary indexes")
restore_files = []
commands_and_files = {}
for restore_table_indexes_file in os.listdir(
cloudsql_importer_constants.RESTORE_INDEXES_COMMANDS_FOLDER):
restore_path = os.path.join(
cloudsql_importer_constants.RESTORE_INDEXES_COMMANDS_FOLDER,
restore_table_indexes_file)
logging.debug("Reading restore commands from file '%s'", restore_path)
restore_file = open(restore_path, "r")
commands = restore_file.read()
restore_files += [restore_file]
commands_and_files[commands] = restore_path
pool = thread_pool(len(commands_and_files.keys()))
failed_commands = pool.map(restore, commands_and_files.keys())
failed_commands = [command for command in failed_commands if command]
for restore_file in restore_files:
restore_file.close()
if failed_commands:
handle_restore_secondary_indexes_failures(failed_commands,
commands_and_files)
raise RuntimeError("Not all secondary indexes were properly restored.")
def javatpoint_start():
with thread_pool(processes=3) as pool:
pool.map(javatpoint, javatpoint_list)
return
from multiprocessing.dummy import Pool as thread_pool
import datetime
import time
def fun(tem):
print(tem)
time.sleep(1)
if __name__ == '__main__':
begin = datetime.datetime.now()
print(begin)
async_pool = thread_pool(processes=5)
results = []
for i in range(10):
result = async_pool.apply_async(fun, (i, ))
print(result)
results.append(result)
for i in results:
i.wait()
end = datetime.datetime.now()
used_time = (end - begin)
print(used_time)
def mulit_type_gather(filepath):
typepool = thread_pool(16)
try:
typepool.map(file_type_gather, filepath)
except Exception, e:
pass
def tutorialspoint_start():
with thread_pool(processes=3) as pool:
pool.map(tutorialspoint, tutorialspoint_list)
return
def run_main(names):
pool = thread_pool(3)
pool.apply_async(get_http, names)
pool.close()
pool.join()
def threadedFunc(iterable):
pool = thread_pool(threads)
for x in pool.map(func, iterable):
yield x