def test2(): with PixelSpinner('Communicating with server...') as bar: for i in range(100): sleep(0.06) bar.next() print('Processing server data...') for i in progress.mill(range(100)): sleep(0.02) print('Resolving dependencies...') for i in progress.mill(range(100)): sleep(0.02) print("action is complete.")
def do_download_db_v8(database_name=None, master_pwd=None, base_url=None, backup_format='zip', filename=None): # get the token: token_req = requests.get('%s/web/database/manager' % base_url, params={'db': database_name}, allow_redirects=True) token_req.raise_for_status() parser = etree.XMLParser(recover=True) response = etree.parse(StringIO(token_req.content), parser) token = False for elem in response.xpath( "//form[@name='backup_db_form']/input[@name='token']"): token = elem.text params = { 'token': token, 'backup_pwd': master_pwd, 'backup_db': database_name, 'backup_format': backup_format } backup_request = requests.get( '%s/web/database/backup' % base_url, params, stream=True, timeout=900, ) if not backup_request.headers.get( 'Content-Type', '').startswith('application/octet-stream;'): # get error text from content template is: # <div class="alert alert-danger">{{ error }}</div> print "Error:" print backup_request.content return 0 total_size = int(backup_request.headers.get('content-length', 0)) size = 0 with open(filename, 'wb') as archive_file: for chunk in progress.mill( backup_request.iter_content(chunk_size=1024), label='downloading %s: ' % filename, expected_size=total_size): if not chunk: continue archive_file.write(chunk) size += len(chunk) return size
def list3(): "This is a list of album articles without infoboxes." global wiki FORCE_CACHE_RELOAD = False if (not os.path.exists(INFOBOX_PAGE_CACHE)) or FORCE_CACHE_RELOAD: keyed_pages = {} for template_name in ["Template:WikiProject Albums", "Template:Infobox album"]: template = pywikibot.Page(wiki, template_name) logging.info("Initialized an object for %s" % template_name) template_pages = template.getReferences(onlyTemplateInclusion=True) logging.info("Got a generator of pages transcluding %s" % template_name) template_page_titles = [] for page in progress.mill(template_pages, expected_size=200000, label="Converting "): template_page_titles.append(page.title(withNamespace=False)) logging.info("Turned that list into a list of %d titles." % len(template_page_titles)) template_page_titles = key_on_first_letter(template_page_titles) logging.info("Keyed that list on title, forming a %d-key dict." % len(template_page_titles)) keyed_pages[template_name] = template_page_titles with open(template_name[-1], "w") as cache: json.dump(template_page_titles, cache) print album_pages = keyed_pages["Template:WikiProject Albums"] infoboxed_pages = keyed_pages["Template:Infobox album"] else: with open(ALBUM_PAGE_CACHE, "r") as cache: album_pages = json.load(cache) with open(INFOBOX_PAGE_CACHE, "r") as cache: infoboxed_pages = json.load(cache) logging.info("Removing album pages that already have infoboxes...") for letter in progress.bar(album_pages): if letter not in infoboxed_pages: continue for album in album_pages[letter]: if album in infoboxed_pages[letter]: album_pages[letter].remove(album) infoboxed_pages[letter].remove(album) logging.info("Done removing album pages with infoboxes!") return album_pages
def do_download_db_v9(database_name=None, master_pwd=None, base_url=None, backup_format='zip', filename=None): # get the cookie (creates new session_id): cookie_params = { 'db': database_name, } cookie_req = requests.get('%s/web/login' % base_url, params=cookie_params, allow_redirects=True) params = { 'master_pwd': master_pwd, 'name': database_name, 'backup_format': backup_format } backup_request = requests.post('%s/web/database/backup' % base_url, params, stream=True, timeout=900, cookies=cookie_req.cookies) if not backup_request.headers.get( 'Content-Type', '').startswith('application/octet-stream;'): # get error text from content template is: # <div class="alert alert-danger">{{ error }}</div> print "Error:" parser = etree.XMLParser(recover=True) response = etree.parse(StringIO(backup_request.content), parser) for elem in response.xpath("//div[@class='alert alert-danger']"): print elem.text return 0 total_size = int(backup_request.headers.get('content-length', 0)) size = 0 with open(filename, 'wb') as file: for chunk in progress.mill( backup_request.iter_content(chunk_size=1024), label='downloading %s: ' % filename, expected_size=total_size): if not chunk: continue file.write(chunk) size += len(chunk) return size
def main(): args = docopt(__doc__) c = get_config() e = create_engine(c.db_url) Base.metadata.drop_all(e) Base.metadata.create_all(e) Session = sessionmaker(e) s = Session() with gzip.open(os.path.join(c.textdb_dir, 'acta.csv.gz'), 'rb') as csv_file: reader = csv.reader(csv_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_ALL) reader.next() # Skip header for row in progress.mill(reader, label='Writing to DB ', expected_size=10000, every=1000): sp = Speech(date=datetime.strptime(row[3], '%Y-%m-%d'), speaker_uri=unicode(row[1], 'utf-8'), hansard=unicode(row[0], 'utf-8'), speech=unicode(row[2], 'utf-8')) s.add(sp) print "Committing..." s.commit() print "Done"
def handle(self, *args, **options): in_file = options.get('in_file') out_file = options.get('out_file') n_bits = options.get('n_bits') verbosity = options.get('verbosity') radius = options.get('radius') fp_type = options.get('fp_type') if verbosity > 1: print self.help inp = self.getFileHandle(in_file, 'rb') out = self.getFileHandle(out_file, 'wb') suppl = Chem.ForwardSDMolSupplier(inp) out.write("#FPS1\n#num_bits=%s\n#software=RDKit/%s\n" % (n_bits, rdBase.rdkitVersion)) for i, mol in progress.mill(enumerate(suppl), expected_size=5): if mol: idx = i if mol.HasProp('chembl_id'): idx = mol.GetProp('chembl_id') else: try: idx = Chem.InchiToInchiKey(Chem.MolToInchi(mol)) except: pass if fp_type == 'morgan': fp = rdMolDescriptors.GetMorganFingerprintAsBitVect(mol,radius,nBits=n_bits) elif fp_type == 'pair': fp = Pairs.GetAtomPairFingerprintAsBitVect(mol) elif fp_type == 'maccs': fp = MACCSkeys.GenMACCSKeys(mol) out.write("%s\t%s\n" % (DataStructs.BitVectToFPSText(fp), idx)) if verbosity > 1: print "Conversion completed." #------------------------------------------------------------------------------
def list_regex(expression): 'Gets a list of album pages with whose names match the regex in their name' global wiki wikiproject_template = pywikibot.Page(wiki, "Template:WikiProject Albums") album_pages = wikiproject_template.getReferences(onlyTemplateInclusion=True) logging.info("Got a generator of album pages.") album_titles = [] for page in progress.mill(album_pages, expected_size=200000, label="Converting "): album_titles.append(page.title(withNamespace=True)) logging.info("Turned that into a list of %d titles." % len(album_titles)) INCORRECT = re.compile(expression) incorrect_pages = [] for page_title in progress.bar(album_titles, label="Filtering "): if INCORRECT.match(page_title): incorrect_pages.append(page_title) to_be_removed = [] for page_title in progress.bar(incorrect_pages, label="-Internal "): page = pywikibot.Page(wiki, page_title) category_titles = [x.title() for x in page.categories()] if "Category:Project-Class Album articles" in category_titles: to_be_removed.append(page_title) for page_title in to_be_removed: incorrect_pages.remove(page_title) to_be_removed = [] for page_title in progress.bar(incorrect_pages, label="Unicode "): try: page_title.decode("ascii") except UnicodeEncodeError: to_be_removed.append(page_title) for page_title in to_be_removed: incorrect_pages.remove(page_title) return incorrect_pages
assertRaises(progress.mill, TypeError) with progress.bar(expected_size=200) as bar: for i in range(200): bar.update(i + 1) sleep(0.05) for i in progress.bar(range(100)): sleep(random() * 0.2) for i in progress.dots(range(100)): sleep(random() * 0.2) with progress.dots() as bar: for i in range(200): bar.update() sleep(0.05) for i in progress.mill(range(100)): sleep(random() * 0.2) with progress.mill(expected_size=200) as bar: for i in range(200): bar.update(i + 1) sleep(0.05) # Override the expected_size, for iterables that don't support len() D = dict(zip(range(100), range(100))) for k, v in progress.bar(D.iteritems(), expected_size=len(D)): sleep(random() * 0.2)
from random import random from clint.textui import progress if __name__ == '__main__': with progress.Bar(expected_size=100.) as bar: for i in range(3): for i in range(100): sleep(0.01) bar.show(i) bar = bar.done(i is not 3) for i in progress.bar(range(100)): sleep(random() * 0.2) with progress.Bar(label="nonlinear", expected_size=10) as bar: last_val = 0 for val in (1, 2, 3, 9, 10): sleep(2 * (val - last_val)) bar.show(val) last_val = val for i in progress.dots(range(100)): sleep(random() * 0.2) for i in progress.mill(range(100)): sleep(random() * 0.2) # Override the expected_size, for iterables that don't support len() D = dict(zip(range(100), range(100))) for k, v in progress.bar(D.items(), expected_size=len(D)): sleep(random() * 0.2)
def _save_bytes(self, response, dest_file, resource, decompress=False, process_bytes=None, md5_check=True): # requests automatically decompresses the data. # Tell it to do it only if it had to total_length = 0 hash_md5 = hashlib.md5() class RawRange(object): """ Wrapper class to make response.raw.read work as iterator and behave the same way as the corresponding response.iter_content """ def __init__(self, rsp, decode_content): """ :param rsp: HTTP response object """ self.decode_content = decode_content self._read = rsp.raw.read self._decode = rsp.raw._decode self.block_size = 0 def __iter__(self): return self def __next__(self): return self.next() def next(self): # reads the next raw block # Hack warning: # Not using decode_content argument of the Response.read # method in order to get access to the raw bytes and do the # md5 checksum before decoding (decompressing) them. # Unfortunately, that forces us to use the hidden _decode # method of the urllib3 Response class. data = self._read(self.block_size) if len(data) > 0: if md5_check and\ (response.headers.get('content-MD5', 0) != 0): hash_md5.update(data) if self.decode_content: data = self._decode(data, True, False) return data else: raise StopIteration() def get_instance(self, block_size): self.block_size = block_size return self try: total_length = int(response.headers.get('content-length', 0)) except ValueError: pass rr = RawRange(response, decompress) reader = rr.get_instance if self.logger.isEnabledFor(logging.INFO): if total_length != 0: chunks = progress.bar( reader(READ_BLOCK_SIZE), expected_size=((total_length / READ_BLOCK_SIZE) + 1)) else: chunks = progress.mill(reader(READ_BLOCK_SIZE), expected_size=0) else: chunks = reader(READ_BLOCK_SIZE) start = time.time() for chunk in chunks: if process_bytes is not None: process_bytes(chunk) dest_file.write(chunk) if md5_check and (response.headers.get('content-MD5', 0) != 0): md5sum = hash_md5.hexdigest() if md5sum and response.headers.get('content-MD5', 0) != md5sum: raise DownloadError( 'Downloaded file is corrupted: ' 'expected md5({}) != actual md5({})'.format( response.headers.get('content-MD5', 0), md5sum)) duration = time.time() - start self.logger.info( 'Successfully downloaded file {} as {} in {}s (avg. speed: {}MB/s)' ''.format(resource, dest_file.name, round(duration, 2), round(total_length / 1024 / 1024 / duration, 2)))
#!/usr/bin/env python # -*- coding: utf-8 -*- import sys import os sys.path.insert(0, os.path.abspath('..')) from time import sleep from random import random from clint.textui import progress if __name__ == '__main__': for i in progress.bar(range(100)): sleep(random() * 0.2) for i in progress.dots(range(100)): sleep(random() * 0.2) for i in progress.mill(range(100)): sleep(random() * 0.2)
url = getDirectLink(url) + "asca" print(("downloading: " + out_file_name + " " + url).strip()) headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:70.0) Gecko/20100101 Firefox/70.0' } res = requests.get(url, headers=headers, stream=True) get_ipython().run_line_magic('debug', '') total_size = int(res.headers["Content-Length"]) chunkSize = 1024 * 1024 bars = int(total_size / chunkSize) with open(out_file_name, 'wb') as out_file: for chunk in progress.mill( res.iter_content(chunk_size=chunkSize), label="Downloading in progress: ", expected_size=bars + 1): if chunk: out_file.write(chunk) out_file.flush() print("{0}: In folder: \"{1}\" file \"{2}\" was downloaded". format(time.ctime(), os.getcwd(), out_file_name)) with open(log_file_name, 'a') as f: f.writelines( "{0}: In folder: \"{1}\" file \"{2}\" was downloaded\n" .format(time.ctime(), os.getcwd(), out_file_name)) except Exception as e: print(e) os.system('pause')
def test1(): print('Processing client request...') for i in progress.mill(range(100)): sleep(0.02)
def search_fortune(searchquery, fortunefilep=None): """ @type searchquery: str @type fortunefilep: str, None @return: None """ results = {} for fortune_file in mill(os.listdir("."), label="- Searching quote files", every=4): if fortunefilep is not None: fortune_file = os.path.basename(fortunefilep) + ".dat" if fortune_file.endswith(".dat"): fortune_index_file = "./" + fortune_file if not os.path.exists(fortune_index_file): raise ValueError('Can\'t find file "%s"' % fortune_index_file) fortune_index = open(fortune_index_file, "rb") data = pickle.load(fortune_index) fortune_index.close() fortune_file = "./" + fortune_file f = open(fortune_file.replace(".dat", ""), 'rt') qwords = searchquery.split(" ") qwords = [w.lower() for w in qwords if w] for cnt in range(0, len(data) - 1): (start, length) = data[cnt] f.seek(start) fortune_cookie = f.read(length) spaces = " " content = fortune_cookie.strip().replace("\t", spaces) cnt = 0 nc = spaces for c in content: nc += c cnt += 1 if cnt > 80 and c == " ": nc += "\n" + spaces cnt = 0 nc = nc.replace("\n" + spaces + "\n" + spaces + "--", "\n" + spaces + "--") if len(nc.split("--")) > 1: nc = spaces + nc.split("--")[0].strip( ) + "\n" + spaces + "-- " + nc.split("--")[1].replace( "\n", "").replace(spaces, " ") for i in range(2, 6): sp = i * " " nc = nc.replace(sp, " ") ncs = nc.split("--") quote = nc author = "" if len(ncs) > 1: quote = ncs[0] author = "--" + ncs[1] score = 0 quotesplit = [x for x in quote.lower().split(" ") if x] for word in qwords: if word in quotesplit: score += 1 for word in qwords: if word in author.lower(): if score not in results: results[score] = [] results[score].append( (100, fortune_file.replace(".dat", ""), quote, author)) elif score > 0: if score == len(quotesplit): if score not in results: results[score] = [] results[score].append( (score, fortune_file.replace(".dat", ""), quote, author)) f.close() if fortunefilep is not None: return results return results
manga_name = prompt.query("Manga Name?:").lower().strip() chaps = prompt.query("Which chapters?").lower().strip() parsed_chaps = parse_chaps_user_input(manga_name, chaps) if not os.path.exists(manga_name): os.makedirs(manga_name) puts(colored.green("OKAY!\nDownload Starting Now!\n\n")) for c in parsed_chaps: puts(colored.green("Finding chapter {} image links...".format(c))) page_links = list(get_manga_whole_chapter_links(manga_name, c)) page_contents = map(get_page_from_url, page_links) page_data = list( filter(None.__ne__, map(get_data_from_page, page_contents))) if not os.path.exists('{}/chap_{}'.format(manga_name, c)): os.makedirs('{}/chap_{}'.format(manga_name, c)) for page_info in progress.mill( page_data, label="Downloading chapter {}... ".format(c)): img_url = page_info['img_url'] save_fname = '{}/chap_{}/{}'.format(manga_name, c, page_info['nice_img_fname']) download_img_to_file(save_fname, img_url) time.sleep(0.25) puts( colored.cyan( "\n\nFinished All Downloads, thanks for using MangaDLer!"))
assertRaises(progress.mill, TypeError) with progress.bar(expected_size=200) as bar: for i in range(200): bar.update(i+1) sleep(0.05) for i in progress.bar(range(100)): sleep(random() * 0.2) for i in progress.dots(range(100)): sleep(random() * 0.2) with progress.dots() as bar: for i in range(200): bar.update() sleep(0.05) for i in progress.mill(range(100)): sleep(random() * 0.2) with progress.mill(expected_size=200) as bar: for i in range(200): bar.update(i+1) sleep(0.05) # Override the expected_size, for iterables that don't support len() D = dict(zip(range(100), range(100))) for k, v in progress.bar(D.iteritems(), expected_size=len(D)): sleep(random() * 0.2)
def search_fortune(searchquery, fortunefilep=None): """ @type searchquery: str @type fortunefilep: str, None @return: None """ results = {} for fortune_file in mill(os.listdir("."), label="- Searching quote files", every=4): if fortunefilep is not None: fortune_file = os.path.basename(fortunefilep) + ".dat" if fortune_file.endswith(".dat"): fortune_index_file = "./" + fortune_file if not os.path.exists(fortune_index_file): raise ValueError('Can\'t find file "%s"' % fortune_index_file) fortune_index = open(fortune_index_file, "rb") data = pickle.load(fortune_index) fortune_index.close() fortune_file = "./" + fortune_file f = open(fortune_file.replace(".dat", ""), 'rt') qwords = searchquery.split(" ") qwords = [w.lower() for w in qwords if w] for cnt in range(0, len(data) - 1): (start, length) = data[cnt] f.seek(start) fortune_cookie = f.read(length) spaces = " " content = fortune_cookie.strip().replace("\t", spaces) cnt = 0 nc = spaces for c in content: nc += c cnt += 1 if cnt > 80 and c == " ": nc += "\n" + spaces cnt = 0 nc = nc.replace("\n" + spaces + "\n" + spaces + "--", "\n" + spaces + "--") if len(nc.split("--")) > 1: nc = spaces + nc.split("--")[0].strip() + "\n" + spaces + "-- " + nc.split("--")[1].replace("\n", "").replace(spaces, " ") for i in range(2, 6): sp = i * " " nc = nc.replace(sp, " ") ncs = nc.split("--") quote = nc author = "" if len(ncs) > 1: quote = ncs[0] author = "--" + ncs[1] score = 0 quotesplit = [x for x in quote.lower().split(" ") if x] for word in qwords: if word in quotesplit: score += 1 for word in qwords: if word in author.lower(): if score not in results: results[score] = [] results[score].append((100, fortune_file.replace(".dat", ""), quote, author)) elif score > 0: if score == len(quotesplit): if score not in results: results[score] = [] results[score].append((score, fortune_file.replace(".dat", ""), quote, author)) f.close() if fortunefilep is not None: return results return results
#use start > 0 and end > 0 and start <= end start = int(input("start\t: ")) end = int(input("end\t: ")) file_name = input("name\t: ") if(start > 0 and end > 0 and start <= end): for i in range(start,end+1): no = str(i) if(len(no) == 1): final_url = url + "0" + no else: final_url = url + no print("\naccessing url : " + final_url) r = session.get(final_url) soup = BeautifulSoup(r.text, 'html.parser') source = soup.find_all('source') if(source != []): img_link = source[0].get('src') print("starting download - episode : " + no) raw_image = session.get(img_link,stream=True) total_length = int(raw_image.headers['Content-Length']) with open(file_name + no + '.mp4', 'wb') as f: for c in progress.mill(raw_image.iter_content(chunk_size=1024),expected_size=(total_length/1024) +1): if c: f.write(c) f.flush() else: print("episode not found") print("\nfinished\n") else: print("\nprovide meaningful values\n")