Exemple #1
0
def main(argv):
    args = argv[1:]

    # Defaults
    key_id = get_s3_info('key_id')
    secret = get_s3_info('key')
    size = DEFAULT_SIZE

    bucket = None
    prefix = None

    while args:
	a = args.pop(0)

        if a == '-C' and args:
            filename = args.pop(0)
            args = file(filename).read().split() + args

        elif a == '-a' and args:
            key_id = args.pop(0)

        elif a == '-s' and args:
            secret = args.pop(0)

        elif a == '-l' and args:
            size = parse_size(args.pop(0))

        elif a == '-mcproperties':
            import s3options
            bucket = s3options.get_bucket()
            key_id = s3options.get_key_id()
            secret = s3options.get_secret()

        elif a.startswith('-'):
            usage()

        elif bucket is None:
            bucket = a

        elif prefix is None:
            prefix = a

        else:
            usage()

    if not prefix or not bucket:
        usage()

    if not key_id:
        print >>stderr, "Must either specify -a {key_id} or put key_id into ~/.s3/key_id"
        exit(1)

    if not secret:
        print >>stderr, "Must either specify -s {secret} or put secret into ~/.s3/key"
        exit(1)

    s3.init0(key_id = key_id, secret = secret)

    segment = 0
    while True:
        block = read_fully(stdin, size)
        if not block:
            break

        object = prefix + '.%03d' % segment

        s3.put_data(bucket, object, block)
        print object, len(block)

        segment += 1
Exemple #2
0
def main(argv):
    args = argv[1:]

    # Defaults
    key_id = get_s3_info('key_id')
    secret = get_s3_info('key')

    bucket = None
    object = None
    output = dup(1)

    while args:
	a = args.pop(0)

        if a == '-C' and args:
            filename = args.pop(0)
            args = file(filename).read().split() + args

        elif a == '-a' and args:
            key_id = args.pop(0)

        elif a == '-s' and args:
            secret = args.pop(0)

        elif a == '-mcproperties':
            import s3options
            bucket = s3options.get_bucket()
            key_id = s3options.get_key_id()
            secret = s3options.get_secret()

        elif a == '-o':
            close(output)
            output = open(args.pop(0), O_WRONLY | O_CREAT | O_TRUNC, 0600)

        elif a.startswith('-'):
            usage()

        elif bucket is None:
            bucket = a

        elif object is None:
            object = a

        else:
            usage()

    if not bucket or not object:
        usage()

    if not key_id:
        print >>stderr, "Must either specify -a {key_id} or put key_id into ~/.s3/key_id"
        exit(1)

    if not secret:
        print >>stderr, "Must either specify -s {secret} or put secret into ~/.s3/key"
        exit(1)

    s3.init0(key_id = key_id, secret = secret)

    if object.count('%%') == object.count('%') * 2:
        # No strftime formatting instructions
        download(bucket, object.replace('%%', '%'), output)

    else:
        for target_date in date_range(date.today()):
            if download(bucket, target_date.strftime(object), output):
                break

    close(output)
Exemple #3
0
def main(argv):
    args = argv[1:]

    # Defaults
    key_id = get_s3_info('key_id')
    secret = get_s3_info('key')
    size = DEFAULT_SIZE

    bucket = None
    prefix = None

    while args:
        a = args.pop(0)

        if a == '-C' and args:
            filename = args.pop(0)
            args = file(filename).read().split() + args

        elif a == '-a' and args:
            key_id = args.pop(0)

        elif a == '-s' and args:
            secret = args.pop(0)

        elif a == '-l' and args:
            size = parse_size(args.pop(0))

        elif a == '-mcproperties':
            import s3options
            bucket = s3options.get_bucket()
            key_id = s3options.get_key_id()
            secret = s3options.get_secret()

        elif a.startswith('-'):
            usage()

        elif bucket is None:
            bucket = a

        elif prefix is None:
            prefix = a

        else:
            usage()

    if not prefix or not bucket:
        usage()

    if not key_id:
        print >> stderr, "Must either specify -a {key_id} or put key_id into ~/.s3/key_id"
        exit(1)

    if not secret:
        print >> stderr, "Must either specify -s {secret} or put secret into ~/.s3/key"
        exit(1)

    s3.init0(key_id=key_id, secret=secret)

    segment = 0
    while True:
        block = read_fully(stdin, size)
        if not block:
            break

        object = prefix + '.%03d' % segment

        s3.put_data(bucket, object, block)
        print object, len(block)

        segment += 1
Exemple #4
0
def main(argv):
    args = argv[1:]

    # Defaults
    key_id = get_s3_info('key_id')
    secret = get_s3_info('key')

    bucket = None
    host = None

    while args:
        a = args.pop(0)

        if a == '-C' and args:
            filename = args.pop(0)
            args = file(filename).read().split() + args

        elif a == '-a' and args:
            key_id = args.pop(0)

        elif a == '-s' and args:
            secret = args.pop(0)

        elif a == '-mcproperties':
            import s3options
            bucket = s3options.get_bucket()
            key_id = s3options.get_key_id()
            secret = s3options.get_secret()

        elif a.startswith('-'):
            usage()

        elif bucket is None:
            bucket = a

        elif host is None:
            host = a

        else:
            usage()

    if not host:
        host = getfqdn()

    if not bucket:
        usage()

    if not key_id:
        print >> stderr, "Must either specify -a {key_id} or put key_id into ~/.s3/key_id"
        exit(1)

    if not secret:
        print >> stderr, "Must either specify -s {secret} or put secret into ~/.s3/key"
        exit(1)

    s3.init0(key_id=key_id, secret=secret)

    today = date.today()

    if today.weekday() == SUNDAY:
        target = today - 5 * WEEK
    else:
        target = today - WEEK

    if target.day != 1:
        # NOT the first of the month, safe to delete
        prefix = '%s-%s' % (host, target.strftime('%Y-%m-%d'))

        delete_all_segments(bucket, prefix + '.sql.bz2.enc')
        delete_all_segments(bucket, prefix + '.tar.bz2.enc')
Exemple #5
0
def main(argv):
    args = argv[1:]

    # Defaults
    key_id = get_s3_info('key_id')
    secret = get_s3_info('key')

    bucket = None
    host = None

    while args:
	a = args.pop(0)

        if a == '-C' and args:
            filename = args.pop(0)
            args = file(filename).read().split() + args

        elif a == '-a' and args:
            key_id = args.pop(0)

        elif a == '-s' and args:
            secret = args.pop(0)

        elif a == '-mcproperties':
            import s3options
            bucket = s3options.get_bucket()
            key_id = s3options.get_key_id()
            secret = s3options.get_secret()

        elif a.startswith('-'):
            usage()

        elif bucket is None:
            bucket = a

        elif host is None:
            host = a

        else:
            usage()

    if not host:
        host = getfqdn()

    if not bucket:
        usage()

    if not key_id:
        print >>stderr, "Must either specify -a {key_id} or put key_id into ~/.s3/key_id"
        exit(1)

    if not secret:
        print >>stderr, "Must either specify -s {secret} or put secret into ~/.s3/key"
        exit(1)

    s3.init0(key_id = key_id, secret = secret)

    today = date.today()

    if today.weekday() == SUNDAY:
        target = today - 5 * WEEK
    else:
        target = today - WEEK

    if target.day != 1:
        # NOT the first of the month, safe to delete
        prefix = '%s-%s' % (host, target.strftime('%Y-%m-%d'))

        delete_all_segments(bucket, prefix + '.sql.bz2.enc')
        delete_all_segments(bucket, prefix + '.tar.bz2.enc')