def fetch_bucket(bucket_name=None, clear_tables=True):
'''
Download the CSV files in the given bucket and add their
data to the database, optionally clearing the relevant
database tables first.
'''
(client, bucket) = _get_bucket(bucket_name)
if clear_tables:
utils.clear_tables()
for key in bucket.objects.all():
name, ext = os.path.splitext(key.key)
logger.info('object found in bucket: {}.{}'.format(name, ext))
if ext.lstrip('.') not in csvkit.convert.SUPPORTED_FORMATS:
continue
fetch_key(client, bucket.name, key.key)
def fetch_bucket(bucket_name=None, clear_tables=True):
'''
Download the CSV files in the given bucket and add their
data to the database, optionally clearing the relevant
database tables first.
'''
(client, bucket) = _get_bucket(bucket_name)
if clear_tables:
utils.clear_tables()
for key in bucket.objects.all():
name, ext = os.path.splitext(key.key)
logger.info('object found in bucket: {}.{}'.format(name, ext))
if ext.lstrip('.') not in csvkit.convert.SUPPORTED_FORMATS:
continue
fetch_key(client, bucket.name, key.key)
def fetch_bucket(bucket_name=None, clear_tables=True):
(client, bucket) = cf_bucket()
if bucket:
logger.info('Using bound bucket {0}'.format(bucket))
logger.info('bucket name {0}'.format(bucket.name))
# aws.subscribe(bucket.name)
else:
bucket_name = bucket_name or config.BUCKET_NAME
logger.info('Importing bucket {0}'.format(bucket_name))
# aws.subscribe(bucket_name) lacking permissions
s3 = boto3.resource('s3')
client = boto3.client('s3')
bucket = s3.Bucket(bucket_name)
if clear_tables:
utils.clear_tables()
for key in bucket.objects.all():
name, ext = os.path.splitext(key.key)
logger.info('object found in bucket: {}.{}'.format(name, ext))
if ext.lstrip('.') not in csvkit.convert.SUPPORTED_FORMATS:
continue
fetch_key(client, bucket.name, key.key)
def fetch_bucket(bucket_name=None, clear_tables=True):
(client, bucket) = cf_bucket()
if bucket:
logger.info('Using bound bucket {0}'.format(bucket))
logger.info('bucket name {0}'.format(bucket.name))
# aws.subscribe(bucket.name)
else:
bucket_name = bucket_name or config.BUCKET_NAME
logger.info('Importing bucket {0}'.format(bucket_name))
# aws.subscribe(bucket_name) lacking permissions
s3 = boto3.resource('s3')
client = boto3.client('s3')
bucket = s3.Bucket(bucket_name)
if clear_tables:
utils.clear_tables()
for key in bucket.objects.all():
name, ext = os.path.splitext(key.key)
logger.info('object found in bucket: {}.{}'.format(name, ext))
if ext.lstrip('.') not in csvkit.convert.SUPPORTED_FORMATS:
continue
fetch_key(client, bucket.name, key.key)
def clear():
utils.clear_tables()
def clear():
utils.clear_tables()