Exemplo n.º 1
0
def main():
    global log, utils, bitc
    args = Args()
    args.parse()
    log = logger.create("BitcasaFileFetcher", args)
    from helpers import utils
    from bitcasadownload import BitcasaDownload
    from lib import BitcasaUtils
    from lib.gdrive import GoogleDrive

    if args.run_level == Args.RUN_LEVEL_MAIN:
        client = None
        if not args.args.upload or not args.args.local:
            bitcasa_utils = BitcasaUtils()
            if bitcasa_utils.test_auth():
                args.args.token = bitcasa_utils.token
            else:
                log.error("Bitcasa Access token not set or invalid. Use the following commands to get one.")
                log.info("python BitcasaFileLister")
                return
            client = bitcasa_utils.create_client()
        if args.args.upload:
            if args.args.provider == "gdrive":
                g = GoogleDrive()
                if not g.test_auth():
                    log.error("Google Drive Access token not set or invalid. Use the following command to get one.")
                    log.info("python BitcasaFileFetcher oauth --provider gdrive")
                    return
        log.debug("Initializing download")
        bitc = BitcasaDownload(args.args, client, should_exit)
        if should_exit.is_set():
            log.info("Exiting")
            return
        input_thread = threading.Thread(target=handle_input, name="Handle Exit")
        input_thread.daemon = True
        input_thread.start()
        if args.args.single:
            bitc.process_single()
        else:
            bitc.process()
    elif args.run_level == Args.RUN_LEVEL_OAUTH:
        if args.args.provider == "bitcasa":
            run_server(args.args.nolaunch)
        elif args.args.provider == "gdrive":
            g = GoogleDrive()
            try:
                g.get_service(True, not args.args.nolaunch)
            except:
                log.exception("Error authenticating to Google drive")

    elif args.run_level == Args.RUN_LEVEL_TEST:
        if args.args.provider == "bitcasa":
            bitcasa_utils = BitcasaUtils()
            if bitcasa_utils.test_auth():
                log.info("Connected to Bitcasa successfully")
            else:
                log.info("Error connecting to Bitcasa")
        elif args.args.provider == "gdrive":
            g = GoogleDrive()
            if g.test_auth():
                log.info("Connected to Google Drive successfully")
            else:
                log.info("Error connecting to Google drive")
    else:
        log.error("An error occurred processing your command. Please check the syntax and try again")
            
     
    log.info("Done")
Exemplo n.º 2
0
def upload(status, should_exit, results, args):
    log.debug("Starting up")
    g = GoogleDrive()
    while not should_exit.is_set():
        apiratecount = 1
        try:
            item = status.up()
        except EmptyException:
            if status.queuers_active or status.down_active:
                continue
            else:
                log.debug("Nothing left to upload. Shutting down")
        if item is None:
            continue
        filename = item["filename"]
        size_bytes = item["filesize"]
        size_str = utils.convert_size(size_bytes)
        temp_file = item["temppath"]
        parent_id = item["filedir"]

        log.info("Uploading %s %s", filename, size_str)
        retriesleft = 10
        up_failed=True
        while retriesleft > 0 and not should_exit.is_set():
            if apiratecount > 5:
                apiratecount = 5
            try:
                g.get_service()
                needtoupload = g.need_to_upload(filename, parent_id, size_bytes)
                if needtoupload:
                    st = time.time()
                    timespan = 0
                    log.debug("Uploading file %s to parent %s", filename, parent_id)
                    result = g.upload_file(temp_file, filename, parent=parent_id)
                    if not result:
                        raise UploadError("Upload failed")
                    timespan = (time.time()-st)
            except HttpError as e:
                retriesleft -= 1
                if e.resp.status == 403:
                    apiratecount += 1
                    retriesleft += 1
                    log.warn("Google API rate limit reached. Will retry")
                else:
                    log.exception("Error uploading file will retry %s more times", retriesleft)

                if retriesleft > 0:
                    time.sleep(10 * apiratecount)
                else:
                    results.writeError(item["filename"], item["fullpath"], item["filepath"], "Error %s could not be uploaded" % filename)
            except:
                retriesleft -= 1
                if retriesleft > 0:
                    log.exception("Error uploading file will retry %s more times", retriesleft)
                    time.sleep(10 * apiratecount)
                else:
                    results.writeError(item["filename"], item["fullpath"], item["filepath"], "Error %s could not be uploaded" % filename)
            else:
                retriesleft = 0
                up_failed = False
                if not args.local:
                    try:
                        os.remove(temp_file)
                    except:
                        log.exception("Failed cleaning up temp file %s", temp_file)
                status.up({
                    "timespan": timespan,
                    "size_uploaded": size_bytes,
                    "temppath": temp_file
                })
                if args.progress:
                    speed = utils.get_speed(size_bytes, timespan)
                    log.info("%s uploaded at %s", size_str, speed)
                log.info("Finished uploading %s", filename)
                results.writeSuccess(item["fullpath"], result["id"])

        if up_failed:
            status.up_fail(item)
        log.debug("End of thread")

    log.debug("Shutting down")