def test_auth_error(self):
print "bad s3plz connections should throw an S3AuthError"
try:
s3plz.connect(MY_TEST_BUCKET, key=None, secret=None)
except s3plz.S3AuthError:
assert True
else:
assert False
def test_auth_error(self):
print "bad s3plz connections should throw an S3AuthError"
try:
s3plz.connect(MY_TEST_BUCKET, key=None, secret=None)
except s3plz.S3AuthError:
assert True
else:
assert False
def poll():
"""
Get estimates for all trips concurrently
"""
# parser
parser = argparse.ArgumentParser('ubernyc')
parser.add_argument('-g', '--geojson', type=str)
parser.add_argument('-w', '--workers', type=int, default=1)
parser.add_argument('-c', '--config', type=str)
opts = parser.parse_args()
# setup
config = yaml.safe_load(open(opts.config))
s3 = s3plz.connect('s3://brianabelson',
serializer='json.gz',
key=config['AWS_ACCESS_KEY_ID'],
secret=config['AWS_SECRET_ACCESS_KEY'])
# compute trips
shp = geo.read_geojson(opts.geojson)
neighborhoods = list(geo.compute_centroids(shp))
trips = list(compute_trips(neighborhoods))
# scrape
while True:
random.shuffle(trips)
msg = "INFO: Fetching prices for {} trips".format(len(trips))
print(msg)
msg = ':thumbsup:\n {}'.format(msg)
slack_msg(msg, config)
if opts.workers > 1:
pool = Pool(CONCURRENT)
wrkr = partial(worker, config=config, s3=s3)
for trip in pool.imap_unordered(wrkr, trips):
pass
else:
for trip in trips:
worker(trip, config=config, s3=s3)
msg = ':thumbsup:\n Finished estimating *{}* trips'.format(len(trips))
slack_msg(msg)
def poll():
"""
Get estimates for all trips concurrently
"""
# parser
parser = argparse.ArgumentParser('ubernyc')
parser.add_argument('-g', '--geojson', type=str)
parser.add_argument('-w', '--workers', type=int, default=1)
parser.add_argument('-c', '--config', type=str)
opts = parser.parse_args()
# setup
config = yaml.safe_load(open(opts.config))
s3 = s3plz.connect('s3://brianabelson',
serializer='json.gz',
key=config['AWS_ACCESS_KEY_ID'],
secret=config['AWS_SECRET_ACCESS_KEY'])
# compute trips
shp = geo.read_geojson(opts.geojson)
neighborhoods = list(geo.compute_centroids(shp))
trips = list(compute_trips(neighborhoods))
# scrape
while True:
random.shuffle(trips)
msg = "INFO: Fetching prices for {} trips".format(len(trips))
print(msg)
msg = ':thumbsup:\n {}'.format(msg)
slack_msg(msg, config)
if opts.workers > 1:
pool = Pool(CONCURRENT)
wrkr = partial(worker, config=config, s3=s3)
for trip in pool.imap_unordered(wrkr, trips): pass
else:
for trip in trips:
worker(trip, config=config, s3=s3)
msg = ':thumbsup:\n Finished estimating *{}* trips'.format(len(trips))
slack_msg(msg)
def test_s3plz(self):
"""
Simple workflow which addresses
all methods and use cases.
"""
# try:
# connect to s3
plz = s3plz.connect(MY_TEST_BUCKET, public=False)
# create an object and formatstring
obj1 = {"key": "value"}
formatstring = 's3plztest-0.1.2/{@date_path}/{key}/{@uid}.json.gz'
# put the object
fp1 = plz.put(obj1, formatstring, serializer="json.gz", **obj1)
# check exists / put method (did the object make it there?)
assert (plz.exists(fp1) is not False)
# check get method
obj2 = plz.get(
fp1,
serializer="json.gz",
)
# check whether serialization / deserialization works
assert (obj1 == obj2)
# check ls / filepath formatting.
for fp in plz.ls('s3plztest-0.1.2/'):
print fp
assert ('value' in fp)
# check streaming method / deserialization
for fp, obj in plz.stream('s3plztest-0.1.2/', serializer="json.gz"):
print obj
assert ("key" in obj)
assert (isinstance(obj, dict))
assert ("value" in fp)
# check upsert method / whether
# updates to contextual time variables
# are reflected in formatted filepaths.
fp2 = plz.upsert(obj1, formatstring, serializer="json.gz", **obj1)
assert (fp2 is not False)
assert (fp1 != fp2)
# check on-the-fly serialization
obj1 = {"foo": "value"}
fp = plz.put(obj1,
"s3plztest-0.1.2/{foo}.json.gz",
serializer="json.gz",
**obj1)
obj2 = plz.get(fp, serializer="json.gz")
assert (obj1 == obj2)
string1 = "hello world"
fp = plz.put(string1, "s3plztest-0.1.2/string.zip", serializer="zip")
string2 = plz.get(fp, serializer="zip")
assert (string1 == string2)
# check metadata
meta = plz.get_meta(fp)
assert (meta['last_modified'] is not None)
# check age
age = plz.get_age(fp)
assert (isinstance(age, timedelta))
# check whether seri
# check whether delete method works
for fp in plz.ls('s3plztest-0.1.2/'):
plz.delete(fp)
assert (len(list(plz.ls('s3plztest-0.1.2/'))) == 0)
def test_s3plz(self):
"""
Simple workflow which addresses
all methods and use cases.
"""
# try:
# connect to s3
plz = s3plz.connect(MY_TEST_BUCKET,
public = False
)
# create an object and formatstring
obj1 = {"key": "value"}
formatstring = 's3plztest-0.1.2/{@date_path}/{key}/{@uid}.json.gz'
# put the object
fp1 = plz.put(obj1, formatstring, serializer = "json.gz", **obj1)
# check exists / put method (did the object make it there?)
assert(plz.exists(fp1) is not False)
# check get method
obj2 = plz.get(fp1, serializer = "json.gz",)
# check whether serialization / deserialization works
assert(obj1 == obj2)
# check ls / filepath formatting.
for fp in plz.ls('s3plztest-0.1.2/'):
print fp
assert('value' in fp)
# check streaming method / deserialization
for fp, obj in plz.stream('s3plztest-0.1.2/', serializer="json.gz"):
print obj
assert("key" in obj)
assert(isinstance(obj, dict))
assert("value" in fp)
# check upsert method / whether
# updates to contextual time variables
# are reflected in formatted filepaths.
fp2 = plz.upsert(obj1, formatstring, serializer="json.gz", **obj1)
assert(fp2 is not False)
assert(fp1 != fp2)
# check on-the-fly serialization
obj1 = {"foo":"value"}
fp = plz.put(obj1, "s3plztest-0.1.2/{foo}.json.gz", serializer="json.gz", **obj1)
obj2 = plz.get(fp, serializer="json.gz")
assert(obj1 == obj2)
string1 = "hello world"
fp = plz.put(string1, "s3plztest-0.1.2/string.zip", serializer="zip")
string2 = plz.get(fp, serializer="zip")
assert(string1 == string2)
# check metadata
meta = plz.get_meta(fp)
assert(meta['last_modified'] is not None)
# check age
age = plz.get_age(fp)
assert(isinstance(age, timedelta))
# check whether seri
# check whether delete method works
for fp in plz.ls('s3plztest-0.1.2/'):
plz.delete(fp)
assert(len(list(plz.ls('s3plztest-0.1.2/'))) == 0)
