def yahoo_main():
codes = _get_stock_codes()
proxy_queue = queue.Queue()
for proxy in load_proxy():
proxy_queue.put((0, proxy))
args_rows = [(code, proxy_queue) for code in codes]
result_list = map_multithreading(job_func_yahoo, args_rows=args_rows)
save_as_txt("yahoo_finace.txt", result_list)
def wikipedia_main():
urls = _get_wiki_urls()
proxy_queue = queue.PriorityQueue()
for proxy in load_proxy():
proxy_queue.put((1, proxy))
# urls = list(urls)[:10]
jobs = [(job_func_wiki, url) for url in urls]
result = multithreading(proxies, jobs)
save_as_txt("wiki.txt", result, mode='w')
def seve_proxy(self, proxies):
save_as_txt("proxy.txt", [self.last_proxy_refilled_time, proxies])
def save_stock_metadata_local():
stock_metadata = get_stock_metadata()
save_as_txt("stock_metadata.txt", stock_metadata, mode='w')
def save_proxy_local():
proxys = get_anonymous_proxy()
save_as_txt("proxys.txt", proxys, mode='w')
from umihico_commons.xlsx_wrapper import load_xlsx
from umihico_commons.functools import save_as_txt
filenames = ["raw1gram.xlsx", "raw2gram.xlsx"]
new_filenames = ["dict1gram.txt", "dict2gram.txt"]
for filename, new_filename in zip(filenames, new_filenames):
rows = load_xlsx(filename)
dict_ = {
key: int(count.replace(".0", ''))
for count, key, gramtype in rows
}
print(new_filename)
save_as_txt(new_filename, dict_)