示例#1
0
文件: s3_to_psv.py 项目: Yelp/mycroft
def bucket_key_exists(bucket_name, key_name, local):
    """
    bucket_key_exists first establishes a connection with S3; from a tron
    server creds are stored in the environment, otherwise, get them from
    the session_file.  Once we have the connection, test if the bucket
    exists, and then if the key exists, and return True if both exist
    otherwise return False.

    Args:
    bucket_name -- the name of the bucket
    key_name -- the name of a key
    run_local -- whether this is run on a dev box (run_local=True) or instance

    Returns:
    True if the bucket key pair exists, otherwise False
    """
    if local:
        conn = S3Connection(**get_boto_creds())
    else:
        conn = S3Connection()
    if conn.lookup(bucket_name) is None:
        return False
    s3_bucket = conn.get_bucket(bucket_name, validate=False)
    exists = (s3_bucket.get_key(key_name) is not None)
    conn.close()
    return exists
示例#2
0
def bucket_key_exists(bucket_name, key_name, local):
    """
    bucket_key_exists first establishes a connection with S3; from a tron
    server creds are stored in the environment, otherwise, get them from
    the session_file.  Once we have the connection, test if the bucket
    exists, and then if the key exists, and return True if both exist
    otherwise return False.

    Args:
    bucket_name -- the name of the bucket
    key_name -- the name of a key
    run_local -- whether this is run on a dev box (run_local=True) or instance

    Returns:
    True if the bucket key pair exists, otherwise False
    """
    if local:
        conn = S3Connection(**get_boto_creds())
    else:
        conn = S3Connection()
    if conn.lookup(bucket_name) is None:
        return False
    s3_bucket = conn.get_bucket(bucket_name, validate=False)
    exists = (s3_bucket.get_key(key_name) is not None)
    conn.close()
    return exists
示例#3
0
def get_sqs_connection():
    '''
    :returns: sqs connection
    '''
    return boto.sqs.connect_to_region(
        read_string('aws_config.region'),
        **get_boto_creds()
    )
示例#4
0
def get_dynamodb_connection():
    '''
    :returns: dynamodb2 connection
    '''
    return boto.dynamodb2.connect_to_region(
        read_string('aws_config.region'),
        **get_boto_creds()
    )
示例#5
0
文件: util.py 项目: wlstyy/mycroft
def load_from_s3_file(s3_uri):
    """Load data from S3
    Useful for loading small config or schema files

    :param s3_uri: path to S3 uri
    :returns: file contents
    """
    _, _, path = s3_uri.partition('://')
    bucket_name, _, key_name = path.partition('/')

    # if region is in a bucket name, put that region first
    def preferred_region(item):
        return item.name not in bucket_name

    boto_creds = get_boto_creds()
    for region in sorted(boto.s3.regions(), key=preferred_region):
        try:
            conn = boto.s3.connect_to_region(region.name, **boto_creds)
            return _load_from_s3_region(conn, bucket_name, key_name)
        except S3ResponseError as e:
            # skip to next region if access is not allowed from this one
            if e.status not in [403, 301]:
                raise
    raise ValueError("{0}: No valid region found".format(s3_uri))
示例#6
0
文件: util.py 项目: Yelp/mycroft
def load_from_s3_file(s3_uri):
    """Load data from S3
    Useful for loading small config or schema files

    :param s3_uri: path to S3 uri
    :returns: file contents
    """
    _, _, path = s3_uri.partition('://')
    bucket_name, _, key_name = path.partition('/')

    # if region is in a bucket name, put that region first
    def preferred_region(item):
        return item.name not in bucket_name

    boto_creds = get_boto_creds()
    for region in sorted(boto.s3.regions(), key=preferred_region):
        try:
            conn = boto.s3.connect_to_region(region.name, **boto_creds)
            return _load_from_s3_region(conn, bucket_name, key_name)
        except S3ResponseError as e:
            # skip to next region if access is not allowed from this one
            if e.status not in [403, 301]:
                raise
    raise ValueError("{0}: No valid region found".format(s3_uri))
示例#7
0
def get_s3_connection():
    '''
    :returns: s3 connection
    '''
    return boto.s3.connect_to_region(read_string('aws_config.region'),
                                     **get_boto_creds())
示例#8
0
def get_dynamodb_connection():
    '''
    :returns: dynamodb2 connection
    '''
    return boto.dynamodb2.connect_to_region(read_string('aws_config.region'),
                                            **get_boto_creds())