コード例 #1
0
def list_s3_files(source_url, patterns):
    """List remote s3 files that match one of the patterns."""
    s3_conn = connect_s3()
    for bucket, root, path in generate_s3_sources(s3_conn, source_url, patterns):
        source = join_as_s3_url(bucket, root, path)
        src_key = get_s3_key(s3_conn, source)
        print "%10d %s" % (src_key.size if src_key is not None else -1, path)
コード例 #2
0
def get_s3_files(source_url, dest_root, patterns):
    """Copy remote s3 files that match one of the patterns to a local destination."""
    s3_conn = connect_s3()
    for bucket, root, path in generate_s3_sources(s3_conn, source_url, patterns):
        source = join_as_s3_url(bucket, root, path)
        dest_name = path.replace('/', '_')
        destination = os.path.join(dest_root, dest_name)
        src_key = get_s3_key(s3_conn, source)
        if src_key is not None:
            src_key.get_contents_to_filename(destination)
        else:
            print "No key for source " + source