def main(argv):
"""
This script remove an object from S3 bucket.
"""
# Arguments and description
parser = argparse.ArgumentParser(
description='delete an object from S3 bucket')
parser.add_argument('filename', help='filename to delete in bucket')
parser.add_argument('bucket',
default=None,
help='bucket name. Valid options are: ')
args = parser.parse_args(argv[1:])
# Give names to arguments
filename = args.filename
bucket_name = args.bucket
aws_session = AWSSession()
logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)
if not aws_session.check_bucket_exists(bucket_name):
logger.info(f"Bucket {bucket_name} does not exist")
exit(1)
try:
aws_session.delete_object_in_bucket(filename, bucket_name)
logger.info(f"Object {filename} was deleted successfully!")
except ClientError as e:
# ignore it and continue uploading files
logger.error(e)
def main(argv):
"""
This script remove an object from S3 bucket.
"""
# Arguments and description
parser = argparse.ArgumentParser(description='delete an object from S3 bucket')
parser.add_argument('filename', help='filename to delete in bucket')
parser.add_argument('bucket', default=None, help='bucket name. Valid options are: ')
args = parser.parse_args(argv[1:])
# Give names to arguments
filename = args.filename
bucket_name = args.bucket
aws_session = AWSSession()
if not aws_session.check_bucket_exists(bucket_name):
print('Bucket \'{0}\' does not exist'.format(bucket_name))
exit(1)
try:
aws_session.delete_object_in_bucket(filename, bucket_name)
print('Object {0} was deleted successfully!'.format(filename))
except ClientError as e:
# ignore it and continue uploading files
print(e)
def main(argv):
"""
This script will delete a S3 bucket.
"""
# Arguments and description
parser = argparse.ArgumentParser(description='delete S3 bucket')
parser.add_argument('bucket_name', help='bucket name')
args = parser.parse_args(argv[1:])
# Give names to arguments
bucket_name = args.bucket_name
aws_session = AWSSession()
logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)
if not aws_session.check_bucket_exists(bucket_name):
logger.info(f"Bucket {bucket_name} does not exist")
exit(1)
try:
aws_session.delete_bucket(bucket_name)
except ClientError as e:
logger.error(e)
def send_data_to_s3(path, bucket):
aws_session = AWSSession()
if not aws_session.check_bucket_exists(bucket):
print('Bucket \'{0}\' does not exist'.format(bucket))
exit(1)
filename = ''.join(path.split('/')[-1]).split(".")[0] + '.gz'
print('{0}: uploading file {1}'.format(datetime.now().replace(microsecond=0), path))
aws_session.send_file_to_bucket(path, filename, bucket)
print('{0}: finished load of file {1}'.format(datetime.now().replace(microsecond=0), path))
def main(argv):
"""
This script will download one or more objects from S3 bucket.
"""
# Arguments and description
parser = argparse.ArgumentParser(
description='download one or more objects from S3 bucket')
parser.add_argument('filename', nargs='+', help='one or more filenames')
parser.add_argument('bucket', default=None, help='bucket name')
parser.add_argument(
'--destination-path',
default=None,
help=
'path where files will be saved, if it is not provided we will use current path'
)
args = parser.parse_args(argv[1:])
# Give names to arguments
datafiles = args.filename
bucket_name = args.bucket
destination_path = args.destination_path
logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)
if destination_path is not None and not os.path.isdir(destination_path):
logger.info(f"Path \'{destination_path}\' is not valid")
exit(1)
aws_session = AWSSession()
if not aws_session.check_bucket_exists(bucket_name):
logger.info(f"Bucket \'{bucket_name}\' does not exist")
exit(1)
for datafile in datafiles:
filename = datafile
if destination_path is not None:
filename = os.path.join(destination_path, datafile)
logger.info(f"downloading object {datafile} ...")
try:
aws_session.download_object_from_bucket(datafile, bucket_name,
filename)
except ClientError as e:
logger.error(e)
def main(argv):
"""
This script will download one or more objects from S3 bucket.
"""
# Arguments and description
parser = argparse.ArgumentParser(description='download one or more objects from S3 bucket')
parser.add_argument('filename', nargs='+', help='one or more filenames')
parser.add_argument('bucket', default=None, help='bucket name')
parser.add_argument('--destination-path', default=None,
help='path where files will be saved, if it is not provided we will use current path')
args = parser.parse_args(argv[1:])
# Give names to arguments
datafiles = args.filename
bucket_name = args.bucket
destination_path = args.destination_path
if destination_path is not None and not os.path.isdir(destination_path):
print('Path \'{0}\' is not valid'.format(destination_path))
exit(1)
aws_session = AWSSession()
if not aws_session.check_bucket_exists(bucket_name):
print('Bucket \'{0}\' does not exist'.format(bucket_name))
exit(1)
for datafile in datafiles:
filename = datafile
if destination_path is not None:
filename = os.path.join(destination_path, datafile)
print('downloading object {0} ...'.format(datafile))
try:
aws_session.download_object_from_bucket(datafile, bucket_name, filename)
except ClientError as e:
print(e)
def main(argv):
"""
This script will move one or more objects from S3 bucket to another S3 bucket.
"""
# Arguments and description
parser = argparse.ArgumentParser(description='move one or more objects from source S3 bucket to target S3 bucket')
parser.add_argument('source_bucket', help='source bucket name')
parser.add_argument('target_bucket', help='target bucket name')
parser.add_argument('-f', '--filename', dest='filename', default=None, nargs='*', help='one or more filenames')
parser.add_argument('-e', '--extension', dest='extension_filter', default=None, nargs='*',
help='only files with this extension will be moved')
args = parser.parse_args(argv[1:])
# Give names to arguments
source_bucket_name = args.source_bucket
target_bucket_name = args.target_bucket
datafiles = args.filename
extension = args.extension_filter
aws_session = AWSSession()
logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)
if not aws_session.check_bucket_exists(source_bucket_name):
logger.info(f"Bucket {source_bucket_name} does not exist")
exit(1)
if not aws_session.check_bucket_exists(target_bucket_name):
logger.info(f"Bucket {target_bucket_name} does not exist")
exit(1)
try:
aws_session.move_files_from_bucket_to_bucket(source_bucket_name, target_bucket_name, datafiles, extension)
except ClientError as e:
logger.error(e)
def main(argv):
"""
This script will move a file to S3 bucket automatically.
Check bucket, file name and
"""
# Arguments and description
parser = argparse.ArgumentParser(description='move document to S3 bucket')
parser.add_argument('file', nargs='+',
help='data file path. It can be a pattern, e.g. /path/to/file or /path/to/file*.zip')
parser.add_argument('bucket', default=None, help='bucket name. Valid options are: ')
parser.add_argument('--omit-filename-check', action='store_true',
help='It Accepts filenames with distinct format to YYYY-mm-dd.*')
parser.add_argument('--replace', action='store_true',
help='It replaces file if exists in bucket, default behavior ask to user a confirmation')
parser.add_argument('--ignore-if-exists', action='store_true',
help='It does not upload file if already exist in the bucket')
args = parser.parse_args(argv[1:])
# Give names to arguments
datafiles = args.file
bucket_name = args.bucket
omit_filename_check = args.omit_filename_check
replace = args.replace
ignore_if_exists = args.ignore_if_exists
aws_session = AWSSession()
if replace and ignore_if_exists:
print('replace and ignore-if-exists options are incompatible')
exit(1)
if not aws_session.check_bucket_exists(bucket_name):
print('Bucket \'{0}\' does not exist'.format(bucket_name))
exit(1)
def send_file_to_s3(matched_file, filename):
print('{0}: uploading file {1}'.format(datetime.now().replace(microsecond=0), matched_file))
aws_session.send_file_to_bucket(matched_file, filename, bucket_name)
print('{0}: finished load of file {1}'.format(datetime.now().replace(microsecond=0), matched_file))
for datafile in datafiles:
matched_files = glob.glob(datafile)
if len(matched_files) == 0:
print('path "{0}" does not match with any file'.format(datafile))
continue
for matched_file in matched_files:
filename = matched_file.split(os.sep)[-1]
if not omit_filename_check:
filename_date_part = filename.split('.')[0]
try:
datetime.strptime(filename_date_part, "%Y-%m-%d")
except ValueError:
print('\'{0}\' does not have a valid format name'.format(filename))
continue
try:
file_exists = aws_session.check_file_exists(bucket_name, filename)
if not file_exists:
send_file_to_s3(matched_file, filename)
continue
if replace:
send_file_to_s3(matched_file, filename)
elif ignore_if_exists:
continue
else:
answer = input('file \'{0}\' exists in bucket. Do you want to replace it? (y/n): '.format(filename))
if answer not in ['y', 'Y']:
print('file {0} was not replaced'.format(filename))
continue
send_file_to_s3(matched_file, filename)
except ClientError as e:
# ignore it and continue uploading files
print(e)
def main(argv):
"""
This script will create visualization of bip! transaction by stop for each day.
"""
f = Figlet()
logger.info(f.renderText('Welcome DTPM'))
# Arguments and description
parser = argparse.ArgumentParser(
description=
'create visualization of bip! transaction by stop for each day.')
parser.add_argument('start_date',
help='Lower bound time. For instance 2020-01-01')
parser.add_argument('end_date',
help='Upper bound time. For instance 2020-12-31')
parser.add_argument('output_filename',
help='filename of html file created by the process')
args = parser.parse_args(argv[1:])
start_date = datetime.strptime(args.start_date, "%Y-%m-%d")
end_date = datetime.strptime(args.end_date, "%Y-%m-%d")
output_filename = args.output_filename
aws_session = AWSSession()
mapbox_key = config('MAPBOX_KEY')
# check available days
dates_in_range = check_available_days(aws_session, start_date, end_date)
if not dates_in_range:
logger.error('There is not data between {0} and {1}'.format(
start_date, end_date))
exit(1)
logger.info('dates found in period: {0}'.format(len(dates_in_range)))
# get available files
available_files = get_available_files(dates_in_range, aws_session,
DATA_PATH)
# create output dict
output, metro_stations, metrotren_stations = get_output_dict(
available_files)
# add location to stop data
output = add_location_to_stop_data(INPUTS_PATH, output, dates_in_range)
# add location to metro data
output = add_location_to_metro_station_data(INPUTS_PATH, output,
metro_stations, dates_in_range)
# add location to metrotren data
output = add_location_to_metrotren_station_data(INPUTS_PATH, output,
dates_in_range)
# save csv data
csv_data = create_csv_data(OUTPUTS_PATH, output_filename, output)
# write mapbox_id to kepler file
write_info_to_kepler_file(TEMPLATE_PATH, OUTPUTS_PATH, output_filename,
mapbox_key, csv_data)
logger.info('{0} successfully created!'.format(output_filename))