def folder_list(folder, status, results, args, should_exit):
fold = folder["folder"]
path = folder["path"]
depth = folder["depth"]
if should_exit.is_set():
log.debug("Stopping folder list")
return
elif path and not args.silentqueuer:
log.info(path)
fulldest = os.path.abspath(os.path.join(args.dst, path))
remainingtries = 3
#Create temp dir and dest dir if needed
while not args.dryrun and fulldest and remainingtries > 0:
if should_exit.is_set():
log.debug("Stopping folder list")
return
try:
try:
os.makedirs(fulldest)
except (OSError, IOError, WindowsError):
if not os.path.isdir(fulldest):
raise
except:
remainingtries -= 1
log.exception("Couldn't create folder %s",fulldest)
if remainingtries > 0:
log.error("Will retry to create folder %s more times", remainingtries)
time.sleep(2)
else:
results.writeErrorDir(fold.name, fulldest, fold.path, traceback.format_exc())
return
else:
remainingtries = 0
if not args.silentqueuer:
log.debug(fulldest)
folderitems = get_folder_items(fold, should_exit)
if should_exit.is_set():
log.debug("Stopping folder list")
return
elif folderitems is None:
log.error("Error downloading at folder %s", path)
results.writeError(folder["folder"].name, path, folder["folder"].path, "")
return
for item in folderitems:
if should_exit.is_set():
log.debug("Stopping folder list")
return
try:
nm = item.name
try:
nm = utils.get_decoded_name(nm)
except:
log.warn("Error encoding to utf-8. Will parse anyway")
base64_path = item.path
filesize = None
tfd = os.path.join(fulldest, nm)
fexists = False
if isinstance(item, BitcasaFile):
filesize = item.size
try:
# it is possible that an error here could tell us
# downloading later won't be possible but there are
# too many errno's to try and catch that right now
fexists = os.path.getsize(tfd) >= filesize
except OSError:
pass
if should_exit.is_set():
log.debug("Stopping folder list")
return
elif fexists:
results.writeSkipped(tfd, base64_path, nm)
else:
if args.dryrun:
if not args.silentqueuer:
log.debug("%s %s", nm, filesize)
results.writeSuccess(tfd, base64_path)
else:
if not args.silentqueuer:
log.debug("Queuing file download for %s", nm)
filedownload = {
"filename": nm,
"filepath": base64_path,
"filesize": filesize,
"fullpath": tfd,
"filedir": fulldest
}
status.queue_down(filedownload)
elif isinstance(item, BitcasaFolder):
if should_exit.is_set():
log.debug("Stopping folder list")
return
elif args.rec and (not args.depth or args.depth > depth):
if not args.silentqueuer:
log.debug("Queuing folder listing for %s", nm)
folder = {
"folder": item,
"path": os.path.join(path, nm),
"depth": (depth+1)
}
status.queue(folder)
except: #Hopefully this won't get called
results.writeError(nm, tfd, base64_path, traceback.format_exc())
def folder_list_gdrive(folder, status, results, args, should_exit, g):
fold = folder["folder"]
path = folder["path"]
folder_id = folder["folder_id"]
depth = folder["depth"]
if should_exit.is_set():
log.debug("Stopping folder list")
return
if not args.silentqueuer and path:
log.info(path)
if args.local:
folderitems = get_local_items(fold, should_exit, results)
else:
folderitems = get_folder_items(fold, should_exit)
if folderitems is None:
log.error("Error downloading at folder %s", path)
if not args.local:
results.writeError(folder["folder"].name, path, folder_id, "")
else:
results.writeError(folder["folder"].name, path, folder["folder"].path, "")
return
for item in folderitems:
if should_exit.is_set():
log.debug("Stopping folder list")
return
try:
nm = item.name
try:
nm = utils.get_decoded_name(nm)
except:
log.warn("Error removing special characters. Will try and parse anyway")
base64_path = item.path
filesize = None
needtoupload = False
tfd = os.path.join(path, nm)
if isinstance(item, BitcasaFile):
filesize = item.size
retriesleft = 10
apiratecount = 1
while not should_exit.is_set() and retriesleft > 0:
try:
if args.dryrun and not folder_id:
needtoupload = True
else:
needtoupload = g.need_to_upload(nm, folder_id, filesize)
except HttpError as e:
retriesleft -= 1
if e.resp.status == 403:
apiratecount += 1
retriesleft += 1
log.warn("Google API rate limit reached. Will retry")
else:
log.exception("Error checking if %s exists will retry %s more times", nm, retriesleft)
if retriesleft > 0:
time.sleep(10 * apiratecount)
else:
results.writeError(nm, tfd, base64_path, "Error queuing file %s" % nm)
except:
retriesleft -= 1
log.exception("Error checking if %s exists will retry %s more times", nm, retriesleft)
if retriesleft > 0:
time.sleep(10 * apiratecount)
else:
results.writeError(nm, tfd, base64_path, "Error queuing file %s" % nm)
else:
retriesleft = 0
if should_exit.is_set():
log.debug("Stopping folder list")
return
if needtoupload:
if args.dryrun:
if not args.silentqueuer:
log.debug("%s %s", nm, filesize)
results.writeSuccess(tfd, base64_path)
else:
filedownload = {
"filename": nm,
"filepath": base64_path,
"filesize": filesize,
"fullpath": tfd,
"filedir": folder_id
}
if args.local:
if not args.silentqueuer:
log.debug("Queuing file upload for %s", nm)
filedownload["temppath"] = base64_path
status.queue_up(filedownload)
else:
if not args.silentqueuer:
log.debug("Queuing file download for %s", nm)
status.queue_down(filedownload)
else:
results.writeSkipped(tfd, base64_path, nm)
elif isinstance(item, BitcasaFolder):
cnf = not args.dryrun
if should_exit.is_set():
log.debug("Stopping folder list")
return
if not args.rec or ( args.depth and args.depth <= depth ):
continue
retriesleft = 10
apiratecount = 1
while not should_exit.is_set() and retriesleft > 0:
try:
if args.dryrun and not folder_id:
g_fold = False
else:
g_fold = g.get_folder_byname(nm, parent=folder_id, createnotfound=cnf)
except HttpError as e:
retriesleft -= 1
if e.resp.status == 403:
apiratecount += 1
retriesleft += 1
log.warn("Google API rate limit reached. Will retry")
else:
log.exception("Will retry to get/create %s %s more times", nm, retriesleft)
if retriesleft > 0:
time.sleep(10 * apiratecount)
else:
results.writeError(nm, tfd, base64_path, "Failed to get/create folder %s" % nm)
continue
except:
retriesleft -= 1
log.error("Will retry to get/create %s %s more times", nm, retriesleft)
if retriesleft > 0:
time.sleep(10 * apiratecount)
else:
results.writeError(nm, tfd, base64_path, "Failed to get/create folder %s" % nm)
continue
else:
retriesleft = 0
if should_exit.is_set():
log.debug("Stopping folder list")
return
folder = {
"folder": item,
"depth": (depth+1),
"path": tfd
}
if args.dryrun and not g_fold:
folder["folder_id"] = None
elif not g_fold:
results.writeError(nm, tfd, base64_path, "Failed to get/create folder %s" % nm)
continue
else:
folder["folder_id"] = g_fold["id"]
if not args.silentqueuer:
log.debug("Queuing folder listing for %s", nm)
status.queue(folder)
except:
results.writeError(nm, tfd, base64_path, traceback.format_exc())
