def test2():
with PixelSpinner('Communicating with server...') as bar:
for i in range(100):
sleep(0.06)
bar.next()
print('Processing server data...')
for i in progress.mill(range(100)):
sleep(0.02)
print('Resolving dependencies...')
for i in progress.mill(range(100)):
sleep(0.02)
print("action is complete.")
def do_download_db_v8(database_name=None,
master_pwd=None,
base_url=None,
backup_format='zip',
filename=None):
# get the token:
token_req = requests.get('%s/web/database/manager' % base_url,
params={'db': database_name},
allow_redirects=True)
token_req.raise_for_status()
parser = etree.XMLParser(recover=True)
response = etree.parse(StringIO(token_req.content), parser)
token = False
for elem in response.xpath(
"//form[@name='backup_db_form']/input[@name='token']"):
token = elem.text
params = {
'token': token,
'backup_pwd': master_pwd,
'backup_db': database_name,
'backup_format': backup_format
}
backup_request = requests.get(
'%s/web/database/backup' % base_url,
params,
stream=True,
timeout=900,
)
if not backup_request.headers.get(
'Content-Type', '').startswith('application/octet-stream;'):
# get error text from content template is:
# <div class="alert alert-danger">{{ error }}</div>
print "Error:"
print backup_request.content
return 0
total_size = int(backup_request.headers.get('content-length', 0))
size = 0
with open(filename, 'wb') as archive_file:
for chunk in progress.mill(
backup_request.iter_content(chunk_size=1024),
label='downloading %s: ' % filename,
expected_size=total_size):
if not chunk:
continue
archive_file.write(chunk)
size += len(chunk)
return size
def list3():
"This is a list of album articles without infoboxes."
global wiki
FORCE_CACHE_RELOAD = False
if (not os.path.exists(INFOBOX_PAGE_CACHE)) or FORCE_CACHE_RELOAD:
keyed_pages = {}
for template_name in ["Template:WikiProject Albums",
"Template:Infobox album"]:
template = pywikibot.Page(wiki, template_name)
logging.info("Initialized an object for %s" % template_name)
template_pages = template.getReferences(onlyTemplateInclusion=True)
logging.info("Got a generator of pages transcluding %s" % template_name)
template_page_titles = []
for page in progress.mill(template_pages, expected_size=200000,
label="Converting "):
template_page_titles.append(page.title(withNamespace=False))
logging.info("Turned that list into a list of %d titles." % len(template_page_titles))
template_page_titles = key_on_first_letter(template_page_titles)
logging.info("Keyed that list on title, forming a %d-key dict."
% len(template_page_titles))
keyed_pages[template_name] = template_page_titles
with open(template_name[-1], "w") as cache:
json.dump(template_page_titles, cache)
print
album_pages = keyed_pages["Template:WikiProject Albums"]
infoboxed_pages = keyed_pages["Template:Infobox album"]
else:
with open(ALBUM_PAGE_CACHE, "r") as cache:
album_pages = json.load(cache)
with open(INFOBOX_PAGE_CACHE, "r") as cache:
infoboxed_pages = json.load(cache)
logging.info("Removing album pages that already have infoboxes...")
for letter in progress.bar(album_pages):
if letter not in infoboxed_pages:
continue
for album in album_pages[letter]:
if album in infoboxed_pages[letter]:
album_pages[letter].remove(album)
infoboxed_pages[letter].remove(album)
logging.info("Done removing album pages with infoboxes!")
return album_pages
def do_download_db_v9(database_name=None,
master_pwd=None,
base_url=None,
backup_format='zip',
filename=None):
# get the cookie (creates new session_id):
cookie_params = {
'db': database_name,
}
cookie_req = requests.get('%s/web/login' % base_url,
params=cookie_params,
allow_redirects=True)
params = {
'master_pwd': master_pwd,
'name': database_name,
'backup_format': backup_format
}
backup_request = requests.post('%s/web/database/backup' % base_url,
params,
stream=True,
timeout=900,
cookies=cookie_req.cookies)
if not backup_request.headers.get(
'Content-Type', '').startswith('application/octet-stream;'):
# get error text from content template is:
# <div class="alert alert-danger">{{ error }}</div>
print "Error:"
parser = etree.XMLParser(recover=True)
response = etree.parse(StringIO(backup_request.content), parser)
for elem in response.xpath("//div[@class='alert alert-danger']"):
print elem.text
return 0
total_size = int(backup_request.headers.get('content-length', 0))
size = 0
with open(filename, 'wb') as file:
for chunk in progress.mill(
backup_request.iter_content(chunk_size=1024),
label='downloading %s: ' % filename,
expected_size=total_size):
if not chunk:
continue
file.write(chunk)
size += len(chunk)
return size
def main():
args = docopt(__doc__)
c = get_config()
e = create_engine(c.db_url)
Base.metadata.drop_all(e)
Base.metadata.create_all(e)
Session = sessionmaker(e)
s = Session()
with gzip.open(os.path.join(c.textdb_dir, 'acta.csv.gz'), 'rb') as csv_file:
reader = csv.reader(csv_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_ALL)
reader.next() # Skip header
for row in progress.mill(reader, label='Writing to DB ', expected_size=10000, every=1000):
sp = Speech(date=datetime.strptime(row[3], '%Y-%m-%d'),
speaker_uri=unicode(row[1], 'utf-8'),
hansard=unicode(row[0], 'utf-8'),
speech=unicode(row[2], 'utf-8'))
s.add(sp)
print "Committing..."
s.commit()
print "Done"
def handle(self, *args, **options):
in_file = options.get('in_file')
out_file = options.get('out_file')
n_bits = options.get('n_bits')
verbosity = options.get('verbosity')
radius = options.get('radius')
fp_type = options.get('fp_type')
if verbosity > 1:
print self.help
inp = self.getFileHandle(in_file, 'rb')
out = self.getFileHandle(out_file, 'wb')
suppl = Chem.ForwardSDMolSupplier(inp)
out.write("#FPS1\n#num_bits=%s\n#software=RDKit/%s\n" % (n_bits, rdBase.rdkitVersion))
for i, mol in progress.mill(enumerate(suppl), expected_size=5):
if mol:
idx = i
if mol.HasProp('chembl_id'):
idx = mol.GetProp('chembl_id')
else:
try:
idx = Chem.InchiToInchiKey(Chem.MolToInchi(mol))
except:
pass
if fp_type == 'morgan':
fp = rdMolDescriptors.GetMorganFingerprintAsBitVect(mol,radius,nBits=n_bits)
elif fp_type == 'pair':
fp = Pairs.GetAtomPairFingerprintAsBitVect(mol)
elif fp_type == 'maccs':
fp = MACCSkeys.GenMACCSKeys(mol)
out.write("%s\t%s\n" % (DataStructs.BitVectToFPSText(fp), idx))
if verbosity > 1:
print "Conversion completed."
#------------------------------------------------------------------------------
def list_regex(expression):
'Gets a list of album pages with whose names match the regex in their name'
global wiki
wikiproject_template = pywikibot.Page(wiki, "Template:WikiProject Albums")
album_pages = wikiproject_template.getReferences(onlyTemplateInclusion=True)
logging.info("Got a generator of album pages.")
album_titles = []
for page in progress.mill(album_pages, expected_size=200000,
label="Converting "):
album_titles.append(page.title(withNamespace=True))
logging.info("Turned that into a list of %d titles." % len(album_titles))
INCORRECT = re.compile(expression)
incorrect_pages = []
for page_title in progress.bar(album_titles, label="Filtering "):
if INCORRECT.match(page_title):
incorrect_pages.append(page_title)
to_be_removed = []
for page_title in progress.bar(incorrect_pages, label="-Internal "):
page = pywikibot.Page(wiki, page_title)
category_titles = [x.title() for x in page.categories()]
if "Category:Project-Class Album articles" in category_titles:
to_be_removed.append(page_title)
for page_title in to_be_removed:
incorrect_pages.remove(page_title)
to_be_removed = []
for page_title in progress.bar(incorrect_pages, label="Unicode "):
try:
page_title.decode("ascii")
except UnicodeEncodeError:
to_be_removed.append(page_title)
for page_title in to_be_removed:
incorrect_pages.remove(page_title)
return incorrect_pages
assertRaises(progress.mill, TypeError)
with progress.bar(expected_size=200) as bar:
for i in range(200):
bar.update(i + 1)
sleep(0.05)
for i in progress.bar(range(100)):
sleep(random() * 0.2)
for i in progress.dots(range(100)):
sleep(random() * 0.2)
with progress.dots() as bar:
for i in range(200):
bar.update()
sleep(0.05)
for i in progress.mill(range(100)):
sleep(random() * 0.2)
with progress.mill(expected_size=200) as bar:
for i in range(200):
bar.update(i + 1)
sleep(0.05)
# Override the expected_size, for iterables that don't support len()
D = dict(zip(range(100), range(100)))
for k, v in progress.bar(D.iteritems(), expected_size=len(D)):
sleep(random() * 0.2)
from random import random
from clint.textui import progress
if __name__ == '__main__':
with progress.Bar(expected_size=100.) as bar:
for i in range(3):
for i in range(100):
sleep(0.01)
bar.show(i)
bar = bar.done(i is not 3)
for i in progress.bar(range(100)):
sleep(random() * 0.2)
with progress.Bar(label="nonlinear", expected_size=10) as bar:
last_val = 0
for val in (1, 2, 3, 9, 10):
sleep(2 * (val - last_val))
bar.show(val)
last_val = val
for i in progress.dots(range(100)):
sleep(random() * 0.2)
for i in progress.mill(range(100)):
sleep(random() * 0.2)
# Override the expected_size, for iterables that don't support len()
D = dict(zip(range(100), range(100)))
for k, v in progress.bar(D.items(), expected_size=len(D)):
sleep(random() * 0.2)
def _save_bytes(self,
response,
dest_file,
resource,
decompress=False,
process_bytes=None,
md5_check=True):
# requests automatically decompresses the data.
# Tell it to do it only if it had to
total_length = 0
hash_md5 = hashlib.md5()
class RawRange(object):
"""
Wrapper class to make response.raw.read work as iterator and behave
the same way as the corresponding response.iter_content
"""
def __init__(self, rsp, decode_content):
"""
:param rsp: HTTP response object
"""
self.decode_content = decode_content
self._read = rsp.raw.read
self._decode = rsp.raw._decode
self.block_size = 0
def __iter__(self):
return self
def __next__(self):
return self.next()
def next(self):
# reads the next raw block
# Hack warning:
# Not using decode_content argument of the Response.read
# method in order to get access to the raw bytes and do the
# md5 checksum before decoding (decompressing) them.
# Unfortunately, that forces us to use the hidden _decode
# method of the urllib3 Response class.
data = self._read(self.block_size)
if len(data) > 0:
if md5_check and\
(response.headers.get('content-MD5', 0) != 0):
hash_md5.update(data)
if self.decode_content:
data = self._decode(data, True, False)
return data
else:
raise StopIteration()
def get_instance(self, block_size):
self.block_size = block_size
return self
try:
total_length = int(response.headers.get('content-length', 0))
except ValueError:
pass
rr = RawRange(response, decompress)
reader = rr.get_instance
if self.logger.isEnabledFor(logging.INFO):
if total_length != 0:
chunks = progress.bar(
reader(READ_BLOCK_SIZE),
expected_size=((total_length / READ_BLOCK_SIZE) + 1))
else:
chunks = progress.mill(reader(READ_BLOCK_SIZE),
expected_size=0)
else:
chunks = reader(READ_BLOCK_SIZE)
start = time.time()
for chunk in chunks:
if process_bytes is not None:
process_bytes(chunk)
dest_file.write(chunk)
if md5_check and (response.headers.get('content-MD5', 0) != 0):
md5sum = hash_md5.hexdigest()
if md5sum and response.headers.get('content-MD5', 0) != md5sum:
raise DownloadError(
'Downloaded file is corrupted: '
'expected md5({}) != actual md5({})'.format(
response.headers.get('content-MD5', 0), md5sum))
duration = time.time() - start
self.logger.info(
'Successfully downloaded file {} as {} in {}s (avg. speed: {}MB/s)'
''.format(resource, dest_file.name, round(duration, 2),
round(total_length / 1024 / 1024 / duration, 2)))
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import sys
import os
sys.path.insert(0, os.path.abspath('..'))
from time import sleep
from random import random
from clint.textui import progress
if __name__ == '__main__':
for i in progress.bar(range(100)):
sleep(random() * 0.2)
for i in progress.dots(range(100)):
sleep(random() * 0.2)
for i in progress.mill(range(100)):
sleep(random() * 0.2)
url = getDirectLink(url) + "asca"
print(("downloading: " + out_file_name + " " + url).strip())
headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:70.0) Gecko/20100101 Firefox/70.0'
}
res = requests.get(url, headers=headers, stream=True)
get_ipython().run_line_magic('debug', '')
total_size = int(res.headers["Content-Length"])
chunkSize = 1024 * 1024
bars = int(total_size / chunkSize)
with open(out_file_name, 'wb') as out_file:
for chunk in progress.mill(
res.iter_content(chunk_size=chunkSize),
label="Downloading in progress: ",
expected_size=bars + 1):
if chunk:
out_file.write(chunk)
out_file.flush()
print("{0}: In folder: \"{1}\" file \"{2}\" was downloaded".
format(time.ctime(), os.getcwd(), out_file_name))
with open(log_file_name, 'a') as f:
f.writelines(
"{0}: In folder: \"{1}\" file \"{2}\" was downloaded\n"
.format(time.ctime(), os.getcwd(), out_file_name))
except Exception as e:
print(e)
os.system('pause')
def test1():
print('Processing client request...')
for i in progress.mill(range(100)):
sleep(0.02)
def search_fortune(searchquery, fortunefilep=None):
"""
@type searchquery: str
@type fortunefilep: str, None
@return: None
"""
results = {}
for fortune_file in mill(os.listdir("."),
label="- Searching quote files",
every=4):
if fortunefilep is not None:
fortune_file = os.path.basename(fortunefilep) + ".dat"
if fortune_file.endswith(".dat"):
fortune_index_file = "./" + fortune_file
if not os.path.exists(fortune_index_file):
raise ValueError('Can\'t find file "%s"' % fortune_index_file)
fortune_index = open(fortune_index_file, "rb")
data = pickle.load(fortune_index)
fortune_index.close()
fortune_file = "./" + fortune_file
f = open(fortune_file.replace(".dat", ""), 'rt')
qwords = searchquery.split(" ")
qwords = [w.lower() for w in qwords if w]
for cnt in range(0, len(data) - 1):
(start, length) = data[cnt]
f.seek(start)
fortune_cookie = f.read(length)
spaces = " "
content = fortune_cookie.strip().replace("\t", spaces)
cnt = 0
nc = spaces
for c in content:
nc += c
cnt += 1
if cnt > 80 and c == " ":
nc += "\n" + spaces
cnt = 0
nc = nc.replace("\n" + spaces + "\n" + spaces + "--",
"\n" + spaces + "--")
if len(nc.split("--")) > 1:
nc = spaces + nc.split("--")[0].strip(
) + "\n" + spaces + "-- " + nc.split("--")[1].replace(
"\n", "").replace(spaces, " ")
for i in range(2, 6):
sp = i * " "
nc = nc.replace(sp, " ")
ncs = nc.split("--")
quote = nc
author = ""
if len(ncs) > 1:
quote = ncs[0]
author = "--" + ncs[1]
score = 0
quotesplit = [x for x in quote.lower().split(" ") if x]
for word in qwords:
if word in quotesplit:
score += 1
for word in qwords:
if word in author.lower():
if score not in results:
results[score] = []
results[score].append(
(100, fortune_file.replace(".dat",
""), quote, author))
elif score > 0:
if score == len(quotesplit):
if score not in results:
results[score] = []
results[score].append(
(score, fortune_file.replace(".dat", ""),
quote, author))
f.close()
if fortunefilep is not None:
return results
return results
manga_name = prompt.query("Manga Name?:").lower().strip()
chaps = prompt.query("Which chapters?").lower().strip()
parsed_chaps = parse_chaps_user_input(manga_name, chaps)
if not os.path.exists(manga_name):
os.makedirs(manga_name)
puts(colored.green("OKAY!\nDownload Starting Now!\n\n"))
for c in parsed_chaps:
puts(colored.green("Finding chapter {} image links...".format(c)))
page_links = list(get_manga_whole_chapter_links(manga_name, c))
page_contents = map(get_page_from_url, page_links)
page_data = list(
filter(None.__ne__, map(get_data_from_page, page_contents)))
if not os.path.exists('{}/chap_{}'.format(manga_name, c)):
os.makedirs('{}/chap_{}'.format(manga_name, c))
for page_info in progress.mill(
page_data, label="Downloading chapter {}... ".format(c)):
img_url = page_info['img_url']
save_fname = '{}/chap_{}/{}'.format(manga_name, c,
page_info['nice_img_fname'])
download_img_to_file(save_fname, img_url)
time.sleep(0.25)
puts(
colored.cyan(
"\n\nFinished All Downloads, thanks for using MangaDLer!"))
assertRaises(progress.mill, TypeError)
with progress.bar(expected_size=200) as bar:
for i in range(200):
bar.update(i+1)
sleep(0.05)
for i in progress.bar(range(100)):
sleep(random() * 0.2)
for i in progress.dots(range(100)):
sleep(random() * 0.2)
with progress.dots() as bar:
for i in range(200):
bar.update()
sleep(0.05)
for i in progress.mill(range(100)):
sleep(random() * 0.2)
with progress.mill(expected_size=200) as bar:
for i in range(200):
bar.update(i+1)
sleep(0.05)
# Override the expected_size, for iterables that don't support len()
D = dict(zip(range(100), range(100)))
for k, v in progress.bar(D.iteritems(), expected_size=len(D)):
sleep(random() * 0.2)
def search_fortune(searchquery, fortunefilep=None):
"""
@type searchquery: str
@type fortunefilep: str, None
@return: None
"""
results = {}
for fortune_file in mill(os.listdir("."), label="- Searching quote files", every=4):
if fortunefilep is not None:
fortune_file = os.path.basename(fortunefilep) + ".dat"
if fortune_file.endswith(".dat"):
fortune_index_file = "./" + fortune_file
if not os.path.exists(fortune_index_file):
raise ValueError('Can\'t find file "%s"' % fortune_index_file)
fortune_index = open(fortune_index_file, "rb")
data = pickle.load(fortune_index)
fortune_index.close()
fortune_file = "./" + fortune_file
f = open(fortune_file.replace(".dat", ""), 'rt')
qwords = searchquery.split(" ")
qwords = [w.lower() for w in qwords if w]
for cnt in range(0, len(data) - 1):
(start, length) = data[cnt]
f.seek(start)
fortune_cookie = f.read(length)
spaces = " "
content = fortune_cookie.strip().replace("\t", spaces)
cnt = 0
nc = spaces
for c in content:
nc += c
cnt += 1
if cnt > 80 and c == " ":
nc += "\n" + spaces
cnt = 0
nc = nc.replace("\n" + spaces + "\n" + spaces + "--", "\n" + spaces + "--")
if len(nc.split("--")) > 1:
nc = spaces + nc.split("--")[0].strip() + "\n" + spaces + "-- " + nc.split("--")[1].replace("\n", "").replace(spaces, " ")
for i in range(2, 6):
sp = i * " "
nc = nc.replace(sp, " ")
ncs = nc.split("--")
quote = nc
author = ""
if len(ncs) > 1:
quote = ncs[0]
author = "--" + ncs[1]
score = 0
quotesplit = [x for x in quote.lower().split(" ") if x]
for word in qwords:
if word in quotesplit:
score += 1
for word in qwords:
if word in author.lower():
if score not in results:
results[score] = []
results[score].append((100, fortune_file.replace(".dat", ""), quote, author))
elif score > 0:
if score == len(quotesplit):
if score not in results:
results[score] = []
results[score].append((score, fortune_file.replace(".dat", ""), quote, author))
f.close()
if fortunefilep is not None:
return results
return results
#use start > 0 and end > 0 and start <= end
start = int(input("start\t: "))
end = int(input("end\t: "))
file_name = input("name\t: ")
if(start > 0 and end > 0 and start <= end):
for i in range(start,end+1):
no = str(i)
if(len(no) == 1):
final_url = url + "0" + no
else:
final_url = url + no
print("\naccessing url : " + final_url)
r = session.get(final_url)
soup = BeautifulSoup(r.text, 'html.parser')
source = soup.find_all('source')
if(source != []):
img_link = source[0].get('src')
print("starting download - episode : " + no)
raw_image = session.get(img_link,stream=True)
total_length = int(raw_image.headers['Content-Length'])
with open(file_name + no + '.mp4', 'wb') as f:
for c in progress.mill(raw_image.iter_content(chunk_size=1024),expected_size=(total_length/1024) +1):
if c:
f.write(c)
f.flush()
else:
print("episode not found")
print("\nfinished\n")
else:
print("\nprovide meaningful values\n")
