def test_http_https_endpoint(self):
# Test that HTTP endpoint is parsed properly and defaults to port 80
http_walker = S3Walker('http://endpoint.com', 'accesskey', 'secretkey')
assert http_walker.host == 'endpoint.com'
assert http_walker.port == 80
# Test that HTTPS endpoint is parsed properly and defaults to port 443
https_walker = S3Walker('https://endpoint.com', 'accesskey',
'secretkey')
assert https_walker.host == 'endpoint.com'
assert https_walker.port == 443
def test_invalid_endpoint(self):
# Validate exception is thrown for invalid endpoint (no http:// or https://)
try:
S3Walker('endpoint.com', 'accesskey', 'secretkey')
except ValueError:
return
assert False
def setUpClass(cls):
if import_failed:
test_utils.skipTest(
'Could not import s3_walker, most likely cause is Boto not installed'
)
cls.walker = S3Walker('http://endpoint.com', 'accesskey', 'secretkey')
cls.walker.connect()
cls.walker.conn.get_bucket = mock_get_bucket
default=1,
help=
("What is the minimum allowable number of images per category? "
"(categories which don't meet this criteria will be ignored) [default=2]"
))
parser.add_argument(
'--max',
type=int,
metavar='MAX_PER_CATEGORY',
help=
("What is the maximum limit of images per category? "
"(categories which exceed this limit will be trimmed down) [default=None]"
))
args = vars(parser.parse_args())
walker = S3Walker(args['endpoint'], args['accesskey'], args['secretkey'])
for valid in [
validate_s3(walker, args['bucket'], args['path']),
validate_range(args['percent_train'],
min_value=0,
max_value=100,
allow_none=True),
validate_output_file(args['train_file']),
validate_range(args['percent_val'],
min_value=0,
max_value=100,
allow_none=True),
validate_output_file(args['val_file']),
validate_range(args['percent_test'],
min_value=0,
max_value=100,
def test_port(self):
# Validate port is parsed properly
walker = S3Walker('http://endpoint.com:81', 'accesskey', 'secretkey')
assert walker.port == 81
def test_valid_endpoint(self):
walker = S3Walker('http://endpoint.com', 'accesskey', 'secretkey')
assert walker.host == 'endpoint.com'
assert walker.accesskey == 'accesskey'
assert walker.secretkey == 'secretkey'
bucket_name = config.get('S3 Config', 'bucket')
path_prefix = config.get('S3 Config', 'prefix')
if not (path_prefix.endswith('/')):
path_prefix += '/'
# mnist
# - train
# -- 0 ... 9
# --- XXX.png
try:
mnist_folder = sys.argv[1]
except IndexError:
print('mnist folder should be passed')
sys.exit(1)
walker = S3Walker(endpoint, accesskey, secretkey)
walker.connect()
# Create bucket
print('Creating bucket')
walker.create_bucket(bucket_name)
mnist_train_folder = os.path.join(mnist_folder, 'train')
digits = os.listdir(mnist_train_folder)
for digit in digits:
digit_folder = os.path.join(mnist_train_folder, digit)
if os.path.isfile(digit_folder):
continue
files = os.listdir(digit_folder)
for f in files:
if not f.endswith('.png'):