def main(): global config config = common.load_config() # Permit write of UTF-8 characters to stderr (required when piping output) if sys.stderr.encoding == None: sys.stderr = codecs.getwriter("UTF-8")(sys.stderr) common.mkdirs(config["pt-base"]) os.chdir(config["pt-base"]) _lock = common.Lock(".lock") tls.s = requests.Session() login() for i in range(int(config["pt-threads"])): t = threading.Thread(target=worker, name=i, args=[tls.s.cookies]) t.daemon = True t.start() read_project_list() while q.unfinished_tasks: time.sleep(1) q.join() cleanup() common.write_sync_done()
def main(username, password): global config config = common.load_config() common.mkdirs(config["jive-base"]) os.chdir(config["jive-base"]) l = common.Lock(".lock") global max_index try: with open(".max-index") as f: max_index = int(f.read()) except IOError: pass tls.s = requests.Session() login(username, password) threads = int(config["jive-threads"]) for i in range(threads): t = threading.Thread(target=worker, name=i, args=(tls.s.cookies, )) t.daemon = True t.start() for c in contents(): q.put((iter_content, c)) q.join() cleanup() common.write_sync_done() global index with open(".max-index", "w") as f: print(index, file=f)
def main(): global config config = common.load_config() # Permit write of UTF-8 characters to stderr (required when piping output) if sys.stderr.encoding == None: sys.stderr = codecs.getwriter("UTF-8")(sys.stderr) common.mkdirs(config["pt-base"]) os.chdir(config["pt-base"]) _lock = common.Lock(".lock") tls.s = requests.Session() login() for i in range(int(config["pt-threads"])): t = threading.Thread(target = worker, name = i, args = [tls.s.cookies]) t.daemon = True t.start() read_project_list() while q.unfinished_tasks: time.sleep(1) q.join() cleanup() common.write_sync_done()
def main(username, password): global config config = common.load_config() common.mkdirs(config["jive-base"]) os.chdir(config["jive-base"]) l = common.Lock(".lock") global max_index try: with open(".max-index") as f: max_index = int(f.read()) except IOError: pass tls.s = requests.Session() login(username, password) threads = int(config["jive-threads"]) for i in range(threads): t = threading.Thread(target = worker, name = i, args = (tls.s.cookies, )) t.daemon = True t.start() for c in contents(): q.put((iter_content, c)) q.join() cleanup() common.write_sync_done() global index with open(".max-index", "w") as f: print(index, file = f)
def main(): global config config = common.load_config() common.mkdirs(config["product-docs-base"]) os.chdir(config["product-docs-base"]) l = common.Lock(".lock") get_dump() valid_files = set([".lock", ".sync-done"]) pool = multiprocessing.Pool(processes=int(config["product-docs-threads"])) for x in iter_dump(): x["product_"] = x["product"].replace("_", " ") url = "https://access.redhat.com/documentation/%(language)s/" \ "%(product)s/%(version)s/pdf/%(name)s/" \ "%(product)s-%(version)s-%(name)s-%(language)s.pdf" % x f = "%(product_)s/%(version)s/" \ "%(product)s-%(version)s-%(name)s-%(language)s.pdf" % x pool.apply_async(download, (url, f)) valid_files.add(f) pool.close() pool.join() remove_invalid_files(valid_files) common.write_sync_done()
def sync_webdav(url, dest, username, password, odponly, alfresco): common.mkdirs(dest) os.chdir(dest) lock = common.Lock(".lock") urlp = urllib.parse.urlparse(url) global fileset fileset = FileSet(urlp.path[1:]) global downloadq downloadq = LockedList() global q q = queue.Queue() threads = threads_create(int(config["webdav-threads"]), (urlp.netloc, username, password)) q.put((sync_dir, url, urlp.path.rstrip("/"), username, password, int(odponly), int(alfresco))) q.join() msg("INFO: will download %u files" % len(downloadq.list)) for d in downloadq.list: q.put(d) threads_destroy(threads) if fileset.dirty: cleanup() common.write_sync_done() lock.unlock()
def sync_webdav(url, dest, username, password, odponly, alfresco): common.mkdirs(dest) os.chdir(dest) lock = common.Lock(".lock") urlp = urlparse.urlparse(url) global fileset fileset = FileSet(urlp.path[1:]) global downloadq downloadq = LockedList() global q q = Queue.Queue() threads = threads_create(int(config["webdav-threads"]), (urlp.netloc, username, password)) q.put((sync_dir, url, urlp.path.rstrip("/"), username, password, int(odponly), int(alfresco))) q.join() msg("INFO: will download %u files" % len(downloadq.list)) for d in downloadq.list: q.put(d) threads_destroy(threads) if fileset.dirty: cleanup() common.write_sync_done() lock.unlock()
def main(username, password): global config config = common.load_config() common.mkdirs(config["jive-base"]) os.chdir(config["jive-base"]) l = common.Lock(".lock") tls.s = requests.Session() login(username, password) threads = int(config["jive-threads"]) for i in range(threads): t = threading.Thread(target = worker, name = i, args = (tls.s.cookies, )) t.daemon = True t.start() for c in contents(): q.put((iter_content, c)) q.join() cleanup() common.write_sync_done()
if args["list"]: for iso in get_isos(): print("[%c] %s" % ([" ", "*"][iso.match()], iso.name)) sys.exit(0) common.mkdirs(config["rhn-dumps-base"]) os.chdir(config["rhn-dumps-base"]) lock = common.Lock(".lock") threads = [] for i in range(int(config["rhn-dumps-threads"])): t = threading.Thread(target = worker, name = i) t.daemon = True t.start() threads.append(t) for t in threads: # join/sleep so CTRL+C works correctly while t.isAlive(): t.join(1) if fileset.s: # protect against wiping the directory when login fails for f in os.listdir("."): if f not in fileset.s and not os.path.isdir(f) \ and not f.startswith("."): os.unlink(f) common.write_sync_done()