def main():
global config
config = common.load_config()
# Permit write of UTF-8 characters to stderr (required when piping output)
if sys.stderr.encoding == None:
sys.stderr = codecs.getwriter("UTF-8")(sys.stderr)
common.mkdirs(config["pt-base"])
os.chdir(config["pt-base"])
_lock = common.Lock(".lock")
tls.s = requests.Session()
login()
for i in range(int(config["pt-threads"])):
t = threading.Thread(target=worker, name=i, args=[tls.s.cookies])
t.daemon = True
t.start()
read_project_list()
while q.unfinished_tasks:
time.sleep(1)
q.join()
cleanup()
common.write_sync_done()
def main(username, password):
global config
config = common.load_config()
common.mkdirs(config["jive-base"])
os.chdir(config["jive-base"])
l = common.Lock(".lock")
global max_index
try:
with open(".max-index") as f:
max_index = int(f.read())
except IOError:
pass
tls.s = requests.Session()
login(username, password)
threads = int(config["jive-threads"])
for i in range(threads):
t = threading.Thread(target=worker, name=i, args=(tls.s.cookies, ))
t.daemon = True
t.start()
for c in contents():
q.put((iter_content, c))
q.join()
cleanup()
common.write_sync_done()
global index
with open(".max-index", "w") as f:
print(index, file=f)
def main():
global config
config = common.load_config()
# Permit write of UTF-8 characters to stderr (required when piping output)
if sys.stderr.encoding == None:
sys.stderr = codecs.getwriter("UTF-8")(sys.stderr)
common.mkdirs(config["pt-base"])
os.chdir(config["pt-base"])
_lock = common.Lock(".lock")
tls.s = requests.Session()
login()
for i in range(int(config["pt-threads"])):
t = threading.Thread(target = worker, name = i, args = [tls.s.cookies])
t.daemon = True
t.start()
read_project_list()
while q.unfinished_tasks:
time.sleep(1)
q.join()
cleanup()
common.write_sync_done()
def main(username, password):
global config
config = common.load_config()
common.mkdirs(config["jive-base"])
os.chdir(config["jive-base"])
l = common.Lock(".lock")
global max_index
try:
with open(".max-index") as f:
max_index = int(f.read())
except IOError:
pass
tls.s = requests.Session()
login(username, password)
threads = int(config["jive-threads"])
for i in range(threads):
t = threading.Thread(target = worker, name = i, args = (tls.s.cookies, ))
t.daemon = True
t.start()
for c in contents():
q.put((iter_content, c))
q.join()
cleanup()
common.write_sync_done()
global index
with open(".max-index", "w") as f:
print(index, file = f)
def main():
global config
config = common.load_config()
common.mkdirs(config["product-docs-base"])
os.chdir(config["product-docs-base"])
l = common.Lock(".lock")
get_dump()
valid_files = set([".lock", ".sync-done"])
pool = multiprocessing.Pool(processes=int(config["product-docs-threads"]))
for x in iter_dump():
x["product_"] = x["product"].replace("_", " ")
url = "https://access.redhat.com/documentation/%(language)s/" \
"%(product)s/%(version)s/pdf/%(name)s/" \
"%(product)s-%(version)s-%(name)s-%(language)s.pdf" % x
f = "%(product_)s/%(version)s/" \
"%(product)s-%(version)s-%(name)s-%(language)s.pdf" % x
pool.apply_async(download, (url, f))
valid_files.add(f)
pool.close()
pool.join()
remove_invalid_files(valid_files)
common.write_sync_done()
def sync_webdav(url, dest, username, password, odponly, alfresco):
common.mkdirs(dest)
os.chdir(dest)
lock = common.Lock(".lock")
urlp = urllib.parse.urlparse(url)
global fileset
fileset = FileSet(urlp.path[1:])
global downloadq
downloadq = LockedList()
global q
q = queue.Queue()
threads = threads_create(int(config["webdav-threads"]),
(urlp.netloc, username, password))
q.put((sync_dir, url, urlp.path.rstrip("/"), username, password,
int(odponly), int(alfresco)))
q.join()
msg("INFO: will download %u files" % len(downloadq.list))
for d in downloadq.list:
q.put(d)
threads_destroy(threads)
if fileset.dirty:
cleanup()
common.write_sync_done()
lock.unlock()
def sync_webdav(url, dest, username, password, odponly, alfresco):
common.mkdirs(dest)
os.chdir(dest)
lock = common.Lock(".lock")
urlp = urlparse.urlparse(url)
global fileset
fileset = FileSet(urlp.path[1:])
global downloadq
downloadq = LockedList()
global q
q = Queue.Queue()
threads = threads_create(int(config["webdav-threads"]),
(urlp.netloc, username, password))
q.put((sync_dir, url, urlp.path.rstrip("/"), username, password,
int(odponly), int(alfresco)))
q.join()
msg("INFO: will download %u files" % len(downloadq.list))
for d in downloadq.list:
q.put(d)
threads_destroy(threads)
if fileset.dirty:
cleanup()
common.write_sync_done()
lock.unlock()
def main(username, password):
global config
config = common.load_config()
common.mkdirs(config["jive-base"])
os.chdir(config["jive-base"])
l = common.Lock(".lock")
tls.s = requests.Session()
login(username, password)
threads = int(config["jive-threads"])
for i in range(threads):
t = threading.Thread(target = worker, name = i, args = (tls.s.cookies, ))
t.daemon = True
t.start()
for c in contents():
q.put((iter_content, c))
q.join()
cleanup()
common.write_sync_done()
if args["list"]:
for iso in get_isos():
print("[%c] %s" % ([" ", "*"][iso.match()], iso.name))
sys.exit(0)
common.mkdirs(config["rhn-dumps-base"])
os.chdir(config["rhn-dumps-base"])
lock = common.Lock(".lock")
threads = []
for i in range(int(config["rhn-dumps-threads"])):
t = threading.Thread(target = worker, name = i)
t.daemon = True
t.start()
threads.append(t)
for t in threads:
# join/sleep so CTRL+C works correctly
while t.isAlive():
t.join(1)
if fileset.s:
# protect against wiping the directory when login fails
for f in os.listdir("."):
if f not in fileset.s and not os.path.isdir(f) \
and not f.startswith("."):
os.unlink(f)
common.write_sync_done()
