def main():
global log, utils, bitc
args = Args()
args.parse()
log = logger.create("BitcasaFileFetcher", args)
from helpers import utils
from bitcasadownload import BitcasaDownload
from lib import BitcasaUtils
from lib.gdrive import GoogleDrive
if args.run_level == Args.RUN_LEVEL_MAIN:
client = None
if not args.args.upload or not args.args.local:
bitcasa_utils = BitcasaUtils()
if bitcasa_utils.test_auth():
args.args.token = bitcasa_utils.token
else:
log.error("Bitcasa Access token not set or invalid. Use the following commands to get one.")
log.info("python BitcasaFileLister")
return
client = bitcasa_utils.create_client()
if args.args.upload:
if args.args.provider == "gdrive":
g = GoogleDrive()
if not g.test_auth():
log.error("Google Drive Access token not set or invalid. Use the following command to get one.")
log.info("python BitcasaFileFetcher oauth --provider gdrive")
return
log.debug("Initializing download")
bitc = BitcasaDownload(args.args, client, should_exit)
if should_exit.is_set():
log.info("Exiting")
return
input_thread = threading.Thread(target=handle_input, name="Handle Exit")
input_thread.daemon = True
input_thread.start()
if args.args.single:
bitc.process_single()
else:
bitc.process()
elif args.run_level == Args.RUN_LEVEL_OAUTH:
if args.args.provider == "bitcasa":
run_server(args.args.nolaunch)
elif args.args.provider == "gdrive":
g = GoogleDrive()
try:
g.get_service(True, not args.args.nolaunch)
except:
log.exception("Error authenticating to Google drive")
elif args.run_level == Args.RUN_LEVEL_TEST:
if args.args.provider == "bitcasa":
bitcasa_utils = BitcasaUtils()
if bitcasa_utils.test_auth():
log.info("Connected to Bitcasa successfully")
else:
log.info("Error connecting to Bitcasa")
elif args.args.provider == "gdrive":
g = GoogleDrive()
if g.test_auth():
log.info("Connected to Google Drive successfully")
else:
log.info("Error connecting to Google drive")
else:
log.error("An error occurred processing your command. Please check the syntax and try again")
log.info("Done")
def upload(status, should_exit, results, args):
log.debug("Starting up")
g = GoogleDrive()
while not should_exit.is_set():
apiratecount = 1
try:
item = status.up()
except EmptyException:
if status.queuers_active or status.down_active:
continue
else:
log.debug("Nothing left to upload. Shutting down")
if item is None:
continue
filename = item["filename"]
size_bytes = item["filesize"]
size_str = utils.convert_size(size_bytes)
temp_file = item["temppath"]
parent_id = item["filedir"]
log.info("Uploading %s %s", filename, size_str)
retriesleft = 10
up_failed=True
while retriesleft > 0 and not should_exit.is_set():
if apiratecount > 5:
apiratecount = 5
try:
g.get_service()
needtoupload = g.need_to_upload(filename, parent_id, size_bytes)
if needtoupload:
st = time.time()
timespan = 0
log.debug("Uploading file %s to parent %s", filename, parent_id)
result = g.upload_file(temp_file, filename, parent=parent_id)
if not result:
raise UploadError("Upload failed")
timespan = (time.time()-st)
except HttpError as e:
retriesleft -= 1
if e.resp.status == 403:
apiratecount += 1
retriesleft += 1
log.warn("Google API rate limit reached. Will retry")
else:
log.exception("Error uploading file will retry %s more times", retriesleft)
if retriesleft > 0:
time.sleep(10 * apiratecount)
else:
results.writeError(item["filename"], item["fullpath"], item["filepath"], "Error %s could not be uploaded" % filename)
except:
retriesleft -= 1
if retriesleft > 0:
log.exception("Error uploading file will retry %s more times", retriesleft)
time.sleep(10 * apiratecount)
else:
results.writeError(item["filename"], item["fullpath"], item["filepath"], "Error %s could not be uploaded" % filename)
else:
retriesleft = 0
up_failed = False
if not args.local:
try:
os.remove(temp_file)
except:
log.exception("Failed cleaning up temp file %s", temp_file)
status.up({
"timespan": timespan,
"size_uploaded": size_bytes,
"temppath": temp_file
})
if args.progress:
speed = utils.get_speed(size_bytes, timespan)
log.info("%s uploaded at %s", size_str, speed)
log.info("Finished uploading %s", filename)
results.writeSuccess(item["fullpath"], result["id"])
if up_failed:
status.up_fail(item)
log.debug("End of thread")
log.debug("Shutting down")