def _tearDownCallback(self, data):
s3 = AmazonS3(self.aws_access_key_id, self.aws_secret_access_key)
sdb = AmazonSDB(self.aws_access_key_id, self.aws_secret_access_key)
deferreds = []
deferreds.append(s3.deleteBucket(self.aws_s3_http_cache_bucket))
deferreds.append(s3.deleteBucket(self.aws_s3_storage_bucket))
deferreds.append(sdb.deleteDomain(self.aws_sdb_reservation_domain))
deferreds.append(sdb.deleteDomain(self.aws_sdb_coordination_domain))
d = DeferredList(deferreds)
return d
def tearDown(self):
self.s3 = AmazonS3(self.aws_access_key_id, self.aws_secret_access_key)
self.sdb = AmazonSDB(self.aws_access_key_id, self.aws_secret_access_key)
deferreds = []
deferreds.append(self.spider.pg.clearCache())
deferreds.append(self.spider.clearStorage())
deferreds.append(self.sdb.deleteDomain(self.aws_sdb_reservation_domain))
deferreds.append(self.sdb.deleteDomain(self.aws_sdb_coordination_domain))
d = DeferredList(deferreds)
d.addCallback( self._tearDownCallback )
return d
def setUp(self):
config_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "config.yaml"))
if not os.path.isfile(config_path):
self.raiseConfigException(config_path)
config = yaml.load(open(config_path, 'r').read())
if not "aws_access_key_id" in config or "aws_secret_access_key" not in config:
self.raiseConfigException(config_path)
self.sdb = AmazonSDB(config["aws_access_key_id"], config["aws_secret_access_key"])
self.uuid = hashlib.sha256(config["aws_access_key_id"] + config["aws_secret_access_key"] + self.__class__.__name__).hexdigest()
class AWSpiderTestCase(unittest.TestCase):
def setUp(self):
self.mini_web_server = MiniWebServer()
config_path = os.path.abspath( os.path.join( os.path.dirname(__file__), "config.yaml" ) )
if not os.path.isfile( config_path ):
self.raiseConfigException( config_path )
config = yaml.load( open( config_path, 'r').read() )
if not "aws_access_key_id" in config or "aws_secret_access_key" not in config:
self.raiseConfigException( config_path )
self.uuid = hashlib.sha256( config["aws_access_key_id"] + config["aws_secret_access_key"] + self.__class__.__name__ ).hexdigest()
self.aws_access_key_id = config["aws_access_key_id"]
self.aws_secret_access_key = config["aws_secret_access_key"]
self.aws_s3_cache_bucket = "%s_cache" % self.uuid
self.aws_s3_storage_bucket = "%s_storage" % self.uuid
self.aws_sdb_reservation_domain = "%s_reservation" % self.uuid
self.aws_sdb_coordination_domain = "%s_coordination" % self.uuid
self.spider = AWSpider(
aws_access_key_id = self.aws_access_key_id,
aws_secret_access_key = self.aws_secret_access_key,
aws_s3_cache_bucket = self.aws_s3_cache_bucket,
aws_s3_storage_bucket = self.aws_s3_storage_bucket,
aws_sdb_reservation_domain = self.aws_sdb_reservation_domain,
aws_sdb_coordination_domain = self.aws_sdb_coordination_domain,
port = 5000 )
self.s3 = AmazonS3( config["aws_access_key_id"], config["aws_secret_access_key"])
return self.spider.start()
def tearDown(self):
deferreds = []
deferreds.append(self.spider.shutdown())
deferreds.append(self.mini_web_server.shutdown())
d = DeferredList(deferreds)
d.addCallback(self._tearDownCallback)
return d
def _tearDownCallback(self, data):
self.s3 = AmazonS3(self.aws_access_key_id, self.aws_secret_access_key)
self.sdb = AmazonSDB(self.aws_access_key_id, self.aws_secret_access_key)
deferreds = []
deferreds.append(self.spider.pg.clearCache())
deferreds.append(self.spider.clearStorage())
deferreds.append(self.sdb.deleteDomain(self.aws_sdb_reservation_domain))
deferreds.append(self.sdb.deleteDomain(self.aws_sdb_coordination_domain))
d = DeferredList(deferreds)
d.addCallback( self._tearDownCallback2 )
return d
def _tearDownCallback2( self, data ):
deferreds = []
deferreds.append(self.s3.deleteBucket(self.aws_s3_cache_bucket))
deferreds.append(self.s3.deleteBucket(self.aws_s3_storage_bucket))
d = DeferredList(deferreds)
return d
def testPageGetter(self):
d = self.spider.getPage("http://127.0.0.1:8080", timeout=5)
return d
def testClearStorage(self):
d = self.spider.clearStorage()
return d
def testGetServerData(self):
server_data = self.spider.getServerData()
self.failUnlessEqual( isinstance(server_data, dict), True )
def testExpose(self):
self.spider.expose( foo )
d = self.spider.getPage("http://127.0.0.1:5000/function/foo", timeout=5)
return d
class AWSpiderStartTestCase(unittest.TestCase):
def setUp(self):
config_path = os.path.abspath( os.path.join( os.path.dirname(__file__), "config.yaml" ) )
if not os.path.isfile( config_path ):
self.raiseConfigException( config_path )
config = yaml.load( open( config_path, 'r').read() )
if not "aws_access_key_id" in config or "aws_secret_access_key" not in config:
self.raiseConfigException( config_path )
self.uuid = hashlib.sha256( config["aws_access_key_id"] + config["aws_secret_access_key"] + self.__class__.__name__ ).hexdigest()
self.aws_access_key_id = config["aws_access_key_id"]
self.aws_secret_access_key = config["aws_secret_access_key"]
self.aws_s3_cache_bucket = "%s_cache" % self.uuid
self.aws_s3_storage_bucket = "%s_storage" % self.uuid
self.aws_sdb_reservation_domain = "%s_reservation" % self.uuid
self.aws_sdb_coordination_domain = "%s_coordination" % self.uuid
self.spider = AWSpider(
aws_access_key_id = self.aws_access_key_id,
aws_secret_access_key = self.aws_secret_access_key,
aws_s3_cache_bucket = self.aws_s3_cache_bucket,
aws_s3_storage_bucket = self.aws_s3_storage_bucket,
aws_sdb_reservation_domain = self.aws_sdb_reservation_domain,
aws_sdb_coordination_domain = self.aws_sdb_coordination_domain,
port = 5000
)
def tearDown(self):
self.s3 = AmazonS3(self.aws_access_key_id, self.aws_secret_access_key)
self.sdb = AmazonSDB(self.aws_access_key_id, self.aws_secret_access_key)
deferreds = []
deferreds.append(self.spider.pg.clearCache())
deferreds.append(self.spider.clearStorage())
deferreds.append(self.sdb.deleteDomain(self.aws_sdb_reservation_domain))
deferreds.append(self.sdb.deleteDomain(self.aws_sdb_coordination_domain))
d = DeferredList(deferreds)
d.addCallback( self._tearDownCallback )
return d
def _tearDownCallback( self, data ):
deferreds = []
deferreds.append(self.s3.deleteBucket(self.aws_s3_cache_bucket))
deferreds.append(self.s3.deleteBucket(self.aws_s3_storage_bucket))
d = DeferredList(deferreds)
return d
def testStart( self ):
d = self.spider.start()
d.addCallback( self._startCallback )
return d
def _startCallback( self, data ):
d = self.spider.shutdown()
return d
class EncodingTestCase(unittest.TestCase):
def setUp(self):
self.filenames = glob.glob( os.path.join(os.path.dirname(__file__), 'data', '**', '*.xml') )
config_path = os.path.abspath( os.path.join( os.path.dirname(__file__), "config.yaml" ) )
if not os.path.isfile( config_path ):
self.raiseConfigException( config_path )
config = yaml.load( open( config_path, 'r').read() )
if not "aws_access_key_id" in config or "aws_secret_access_key" not in config:
self.raiseConfigException( config_path )
self.uuid = hashlib.sha256( config["aws_access_key_id"] + config["aws_secret_access_key"] + self.__class__.__name__ ).hexdigest()
self.aws_access_key_id = config["aws_access_key_id"]
self.aws_secret_access_key = config["aws_secret_access_key"]
self.aws_s3_cache_bucket = "%s_cache" % self.uuid
self.aws_s3_storage_bucket = "%s_storage" % self.uuid
self.aws_sdb_reservation_domain = "%s_reservation" % self.uuid
#self.aws_sdb_coordination_domain = "%s_coordination" % self.uuid
self.spider = AWSpider(
aws_access_key_id = self.aws_access_key_id,
aws_secret_access_key = self.aws_secret_access_key,
aws_s3_cache_bucket = self.aws_s3_cache_bucket,
aws_s3_storage_bucket = self.aws_s3_storage_bucket,
aws_sdb_reservation_domain = self.aws_sdb_reservation_domain,
#aws_sdb_coordination_domain = self.aws_sdb_coordination_domain,
port = 5000,
log_level="debug" )
self.s3 = AmazonS3( config["aws_access_key_id"], config["aws_secret_access_key"])
self.spider.expose( load_data )
self.spider.expose( load_data, interval=60*60*24, name="load_data_stored" )
return self.spider.start()
def tearDown(self):
deferreds = []
deferreds.append(self.spider.shutdown())
d = DeferredList(deferreds)
d.addCallback(self._tearDownCallback)
return d
def _tearDownCallback(self, data):
self.s3 = AmazonS3(self.aws_access_key_id, self.aws_secret_access_key)
self.sdb = AmazonSDB(self.aws_access_key_id, self.aws_secret_access_key)
deferreds = []
deferreds.append(self.spider.pg.clearCache())
deferreds.append(self.spider.clearStorage())
deferreds.append(self.sdb.deleteDomain(self.aws_sdb_reservation_domain))
#deferreds.append(self.sdb.deleteDomain(self.aws_sdb_coordination_domain))
d = DeferredList(deferreds)
d.addCallback( self._tearDownCallback2 )
return d
def _tearDownCallback2( self, data ):
deferreds = []
deferreds.append(self.s3.deleteBucket(self.aws_s3_cache_bucket))
deferreds.append(self.s3.deleteBucket(self.aws_s3_storage_bucket))
d = DeferredList(deferreds)
return d
# def testImmediateReturn(self):
# deferreds = []
# for filename in self.filenames[0:5]:
# d = self.spider.rq.getPage("http://127.0.0.1:5000/function/load_data", method="POST", postdata={"filename":filename})
# d.addCallback( self._processRequestCallback, filename )
# deferreds.append(d)
# d = DeferredList(deferreds, consumeErrors=True )
# d.addCallback(self._testImmediateReturnCallback)
# d.addErrback( self._testImmediateReturnErrback )
# return d
#
# def _processRequestCallback(self, data, filename):
# processed_data = simplejson.loads( data["response"] )
# original_data = load_data( filename )
# if processed_data == original_data:
# #print "%s passed comparison test." % filename
# return True
# print "%s failed comparison test." % filename
# return False
#
# def _testImmediateReturnCallback(self, data):
# for row in data:
# if row[0] == False:
# raise row[1]
# else:
# self.failUnlessEqual( row[1], True )
#
# def _testImmediateReturnErrback(self, error):
# return error
def testStoredReturn(self):
deferreds = []
for filename in self.filenames:
d = self.spider.rq.getPage("http://127.0.0.1:5000/function/load_data_stored", method="POST", postdata={"filename":filename})
d.addCallback( self._processStoredRequestCallback, filename )
deferreds.append(d)
d = DeferredList(deferreds, consumeErrors=True )
d.addCallback(self._testStoredReturnCallback)
d.addErrback( self._testStoredReturnErrback )
return d
def _processStoredRequestCallback(self, data, filename):
processed_data = simplejson.loads( data["response"] )
uuid = processed_data.keys()[0]
d = self.spider.rq.getPage("http://127.0.0.1:5000/data/get?uuid=%s" % uuid)
d.addCallback( self._processStoredRequestCallback2, filename )
return d
def _processStoredRequestCallback2(self, data, filename):
processed_data = simplejson.loads( data["response"] )
original_data = load_data( filename )
if processed_data == original_data:
#print "%s passed comparison test." % filename
return True
print "%s failed comparison test." % filename
return False
def _testStoredReturnCallback(self, data):
for row in data:
if row[0] == False:
raise row[1]
else:
self.failUnlessEqual( row[1], True )
def _testStoredReturnErrback(self, error):
return error
class AmazonSDBTestCase(unittest.TestCase):
def setUp(self):
config_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "config.yaml"))
if not os.path.isfile(config_path):
self.raiseConfigException(config_path)
config = yaml.load(open(config_path, 'r').read())
if not "aws_access_key_id" in config or "aws_secret_access_key" not in config:
self.raiseConfigException(config_path)
self.sdb = AmazonSDB(config["aws_access_key_id"], config["aws_secret_access_key"])
self.uuid = hashlib.sha256(config["aws_access_key_id"] + config["aws_secret_access_key"] + self.__class__.__name__).hexdigest()
def raiseConfigException(self, filename):
raise Exception("Please create a YAML config file at %s with 'aws_access_key_id' and 'aws_secret_access_key'." % filename)
def tearDown(self):
pass
def test_01_CreateDomain(self):
d = self.sdb.createDomain(self.uuid)
return d
def test_02_ListDomains(self):
d = self.sdb.listDomains()
return d
def test_03_DomainMetadata(self):
d = self.sdb.domainMetadata(self.uuid)
return d
def test_04_PutAttributes(self):
d = self.sdb.putAttributes(self.uuid, "test", {"a":[1,3], "b":2})
return d
def test_04a_BatchPutAttributes(self):
attributes_by_item_name = {
"test_a":{"a":1},
"test_b":{"b":2},
"test_c":{"c":[3,4], "d":5}}
d = self.sdb.batchPutAttributes(self.uuid, attributes_by_item_name)
d.addCallback(self._batchPutAttributesCallback)
return d
def _batchPutAttributesCallback(self, data):
d = deferLater(reactor, 10, self._batchPutAttributesCallback2)
return d
def _batchPutAttributesCallback2(self):
deferreds = []
deferreds.append(self.sdb.getAttributes(self.uuid, "test_a"))
deferreds.append(self.sdb.getAttributes(self.uuid, "test_b"))
deferreds.append(self.sdb.getAttributes(self.uuid, "test_c"))
d = DeferredList(deferreds, consumeErrors=True)
d.addCallback(self._batchPutAttributesCallback3)
return d
def _batchPutAttributesCallback3(self, data):
for row in data:
if row[0] == False:
raise row[1]
if "a" not in data[0][1]:
raise Exception("GetAttributes 1 (a) failed on BatchPutAttributes")
if "1" not in data[0][1]["a"]:
raise Exception("GetAttributes 1 (b) failed on BatchPutAttributes")
if "b" not in data[1][1]:
raise Exception("GetAttributes 2 (a) failed on BatchPutAttributes")
if "2" not in data[1][1]["b"]:
raise Exception("GetAttributes 2 (b) failed on BatchPutAttributes")
if "c" not in data[2][1]:
raise Exception("GetAttributes 3 (a) failed on BatchPutAttributes")
if "4" not in data[2][1]["c"] or "4" not in data[2][1]["c"]:
raise Exception("GetAttributes 3 (b) failed on BatchPutAttributes")
if "5" not in data[2][1]["d"]:
raise Exception("GetAttributes 3 (c) failed on BatchPutAttributes")
def test_04b_BatchPutAttributesMass(self):
attributes_by_item_name = {}
for i in range(1,25):
attributes = {}
for j in range(0,10):
attributes[uuid4().hex] = uuid4().hex
attributes_by_item_name[uuid4().hex] = attributes
attributes_by_item_name["test_d"] = {"e":1}
d = self.sdb.batchPutAttributes(self.uuid, attributes_by_item_name)
d.addCallback(self._batchPutAttributesMassCallback)
return d
def _batchPutAttributesMassCallback(self, data):
d = deferLater(reactor, 10, self._batchPutAttributesMassCallback2)
return d
def _batchPutAttributesMassCallback2(self):
d = self.sdb.getAttributes(self.uuid, "test_d")
d.addCallback(self._batchPutAttributesMassCallback3)
return d
def _batchPutAttributesMassCallback3(self, data):
if "e" not in data:
raise Exception("GetAttributes failed on BatchPutAttributesMass")
if "1" not in data["e"]:
raise Exception("GetAttributes failed on BatchPutAttributesMass")
def test_05_GetAttributes(self):
d = self.sdb.getAttributes(self.uuid, "test")
return d
def test_05a_GetAttributes_Individual(self):
d = self.sdb.getAttributes(self.uuid, "test", "a")
return d
def test_06_Select(self):
d = self.sdb.select("SELECT * FROM `%s` WHERE `a`='1'" % self.uuid)
return d
def test_08_DeleteAttributes_NameAndValue(self):
d = self.sdb.deleteAttributes(self.uuid, "test", {"a":1})
return d
def test_08a_DeleteAttributes_Name(self):
d = self.sdb.deleteAttributes(self.uuid, "test", ["a"])
return d
def test_09_Delete(self):
d = self.sdb.delete(self.uuid, "test")
return d
def test_10_DeleteDomain(self):
d = self.sdb.deleteDomain(self.uuid)
return d
def test_11_CheckAndCreateDomain(self):
d = self.sdb.checkAndCreateDomain(self.uuid)
d.addCallback(self._checkAndCreateDomainCallback)
return d
def _checkAndCreateDomainCallback(self, data):
d = self.sdb.checkAndCreateDomain(self.uuid)
d.addCallback(self._checkAndCreateDomainCallback2)
return d
def _checkAndCreateDomainCallback2(self, data):
d = self.sdb.deleteDomain(self.uuid)
return d
def test_12_CopyDomain(self):
d = self.sdb.createDomain(self.uuid)
d.addCallback(self._copyDomainCallback)
return d
def _copyDomainCallback(self, data):
deferreds = []
for letter in "abcdefghijklmnopqrstuvwxyz":
d = self.sdb.putAttributes(self.uuid, letter, {letter:[letter]})
deferreds.append(d)
d = DeferredList(deferreds, consumeErrors=True)
d.addCallback(self._copyDomainCallback2)
return d
def _copyDomainCallback2(self, data):
for row in data:
if row[0] == False:
raise row[1]
destination_domain = "%s_destination" % self.uuid
d = self.sdb.copyDomain(self.uuid, destination_domain)
d.addCallback(self._copyDomainCallback3, destination_domain)
return d
def _copyDomainCallback3(self, data, destination_domain):
deferreds = []
for letter in "abcdefghijklmnopqrstuvwxyz":
d = self.sdb.getAttributes(destination_domain, letter)
deferreds.append(d)
d = DeferredList(deferreds, consumeErrors=True)
d.addCallback(self._copyDomainCallback4, destination_domain)
return d
def _copyDomainCallback4(self, data, destination_domain):
for row in data:
if row[0] == False:
raise row[1]
for row in data:
letter = row[1].keys()[0]
if row[1][letter][0] != letter:
raise Exception("CopyDomain did not properly execute.")
deferreds = []
deferreds.append(self.sdb.deleteDomain(self.uuid))
deferreds.append(self.sdb.deleteDomain(destination_domain))
d = DeferredList(deferreds)
return d