Esempio n. 1
0
 def test_auth_error(self):
     print "bad s3plz connections should throw an S3AuthError"
     try:
         s3plz.connect(MY_TEST_BUCKET, key=None, secret=None)
     except s3plz.S3AuthError:
         assert True
     else:
         assert False
Esempio n. 2
0
 def test_auth_error(self):
     print "bad s3plz connections should throw an S3AuthError"
     try:
         s3plz.connect(MY_TEST_BUCKET, key=None, secret=None)
     except s3plz.S3AuthError:
         assert True 
     else:
         assert False
Esempio n. 3
0
def poll():
    """
    Get estimates for all trips concurrently
    """

    # parser
    parser = argparse.ArgumentParser('ubernyc')
    parser.add_argument('-g', '--geojson', type=str)
    parser.add_argument('-w', '--workers', type=int, default=1)
    parser.add_argument('-c', '--config', type=str)
    opts = parser.parse_args()

    # setup
    config = yaml.safe_load(open(opts.config))
    s3 = s3plz.connect('s3://brianabelson',
                       serializer='json.gz',
                       key=config['AWS_ACCESS_KEY_ID'],
                       secret=config['AWS_SECRET_ACCESS_KEY'])

    # compute trips
    shp = geo.read_geojson(opts.geojson)
    neighborhoods = list(geo.compute_centroids(shp))
    trips = list(compute_trips(neighborhoods))

    # scrape
    while True:
        random.shuffle(trips)
        msg = "INFO: Fetching prices for {} trips".format(len(trips))
        print(msg)
        msg = ':thumbsup:\n {}'.format(msg)
        slack_msg(msg, config)
        if opts.workers > 1:
            pool = Pool(CONCURRENT)
            wrkr = partial(worker, config=config, s3=s3)
            for trip in pool.imap_unordered(wrkr, trips):
                pass
        else:
            for trip in trips:
                worker(trip, config=config, s3=s3)
        msg = ':thumbsup:\n Finished estimating *{}* trips'.format(len(trips))
        slack_msg(msg)
Esempio n. 4
0
def poll():
    """
    Get estimates for all trips concurrently
    """

    # parser
    parser = argparse.ArgumentParser('ubernyc')
    parser.add_argument('-g', '--geojson', type=str)
    parser.add_argument('-w', '--workers', type=int, default=1)
    parser.add_argument('-c', '--config', type=str)
    opts = parser.parse_args()

    # setup
    config = yaml.safe_load(open(opts.config))
    s3 = s3plz.connect('s3://brianabelson', 
        serializer='json.gz',
        key=config['AWS_ACCESS_KEY_ID'],
        secret=config['AWS_SECRET_ACCESS_KEY'])

    # compute trips
    shp = geo.read_geojson(opts.geojson)
    neighborhoods = list(geo.compute_centroids(shp))
    trips = list(compute_trips(neighborhoods))

    # scrape
    while True:
        random.shuffle(trips)
        msg = "INFO: Fetching prices for {} trips".format(len(trips))
        print(msg)
        msg = ':thumbsup:\n {}'.format(msg)
        slack_msg(msg, config)
        if opts.workers > 1:
            pool = Pool(CONCURRENT)
            wrkr = partial(worker, config=config, s3=s3)
            for trip in pool.imap_unordered(wrkr, trips): pass
        else:
            for trip in trips:
                worker(trip, config=config, s3=s3)
        msg = ':thumbsup:\n Finished estimating *{}* trips'.format(len(trips))
        slack_msg(msg)
Esempio n. 5
0
    def test_s3plz(self):
        """
        Simple workflow which addresses 
        all methods and use cases.
        """
        # try:
        # connect to s3
        plz = s3plz.connect(MY_TEST_BUCKET, public=False)

        # create an object and formatstring
        obj1 = {"key": "value"}
        formatstring = 's3plztest-0.1.2/{@date_path}/{key}/{@uid}.json.gz'

        # put the object
        fp1 = plz.put(obj1, formatstring, serializer="json.gz", **obj1)

        # check exists / put method (did the object make it there?)
        assert (plz.exists(fp1) is not False)

        # check get method
        obj2 = plz.get(
            fp1,
            serializer="json.gz",
        )

        # check whether serialization / deserialization works
        assert (obj1 == obj2)

        # check ls / filepath formatting.
        for fp in plz.ls('s3plztest-0.1.2/'):
            print fp
            assert ('value' in fp)

        # check streaming method / deserialization
        for fp, obj in plz.stream('s3plztest-0.1.2/', serializer="json.gz"):
            print obj
            assert ("key" in obj)
            assert (isinstance(obj, dict))
            assert ("value" in fp)

        # check upsert method / whether
        # updates to contextual time variables
        # are reflected in formatted filepaths.
        fp2 = plz.upsert(obj1, formatstring, serializer="json.gz", **obj1)
        assert (fp2 is not False)
        assert (fp1 != fp2)

        # check on-the-fly serialization
        obj1 = {"foo": "value"}
        fp = plz.put(obj1,
                     "s3plztest-0.1.2/{foo}.json.gz",
                     serializer="json.gz",
                     **obj1)
        obj2 = plz.get(fp, serializer="json.gz")
        assert (obj1 == obj2)

        string1 = "hello world"
        fp = plz.put(string1, "s3plztest-0.1.2/string.zip", serializer="zip")
        string2 = plz.get(fp, serializer="zip")
        assert (string1 == string2)

        # check metadata
        meta = plz.get_meta(fp)
        assert (meta['last_modified'] is not None)

        # check age
        age = plz.get_age(fp)
        assert (isinstance(age, timedelta))

        # check whether seri

        # check whether delete method works
        for fp in plz.ls('s3plztest-0.1.2/'):
            plz.delete(fp)
        assert (len(list(plz.ls('s3plztest-0.1.2/'))) == 0)
Esempio n. 6
0
    def test_s3plz(self):
        """
        Simple workflow which addresses 
        all methods and use cases.
        """
        # try:
        # connect to s3
        plz = s3plz.connect(MY_TEST_BUCKET, 
            public = False
        )   

        # create an object and formatstring
        obj1 = {"key": "value"}
        formatstring = 's3plztest-0.1.2/{@date_path}/{key}/{@uid}.json.gz'

        # put the object
        fp1 = plz.put(obj1, formatstring, serializer = "json.gz", **obj1)

        # check exists / put method (did the object make it there?)
        assert(plz.exists(fp1) is not False)

        # check get method
        obj2 =  plz.get(fp1, serializer = "json.gz",)
        
        # check whether serialization / deserialization works
        assert(obj1 == obj2)

        # check ls / filepath formatting.
        for fp in plz.ls('s3plztest-0.1.2/'):
            print fp
            assert('value' in fp)

        # check streaming method / deserialization
        for fp, obj in plz.stream('s3plztest-0.1.2/', serializer="json.gz"):
            print obj
            assert("key" in obj)
            assert(isinstance(obj, dict))
            assert("value" in fp)

        # check upsert method / whether 
        # updates to contextual time variables
        # are reflected in formatted filepaths.
        fp2 = plz.upsert(obj1, formatstring,  serializer="json.gz", **obj1)
        assert(fp2 is not False)
        assert(fp1 != fp2)

        # check on-the-fly serialization
        obj1 = {"foo":"value"}
        fp = plz.put(obj1, "s3plztest-0.1.2/{foo}.json.gz", serializer="json.gz", **obj1)
        obj2 = plz.get(fp, serializer="json.gz")
        assert(obj1 == obj2)

        string1 = "hello world"
        fp = plz.put(string1, "s3plztest-0.1.2/string.zip", serializer="zip")
        string2 = plz.get(fp, serializer="zip")
        assert(string1 == string2)

        # check metadata
        meta = plz.get_meta(fp)
        assert(meta['last_modified'] is not None)

        # check age 
        age = plz.get_age(fp)
        assert(isinstance(age, timedelta))

        # check whether seri

        # check whether delete method works 
        for fp in plz.ls('s3plztest-0.1.2/'):
            plz.delete(fp)
        assert(len(list(plz.ls('s3plztest-0.1.2/'))) == 0)