def test_source_lambda_handler(self):
controller = Controller(self.config)
controller.create_resources()
event = {
'controller_config': json.dumps(self.config),
'source_config': json.dumps(self.config['sources'][0])
}
source_handler(event, None)
def source_handler(event, context):
controller_config = json.loads(event['controller_config'])
source_config = json.loads(event['source_config'])
controller = Controller(controller_config, os.getcwd())
try:
controller.run_source_job(source_config)
return {'status': 'OK'}
except Exception as e:
msg = "Source failed to execute with error: %s" % e
print(msg)
return {'status': 'error', 'message': msg}
def test_deploy(self):
controller = Controller(self.config)
# Create cluster if needed
controller.create_resources()
# Update Lambdas if needed
controller.create_lambda_functions()
# Schedule master lambda
controller.schedule_controller_lambda()
def transformer_handler(event, context):
print("Transformer handler initialized")
controller_config = json.loads(event['controller_config'])
transformer_config = json.loads(event['transformer_config'])
item_dict = json.loads(event['item'])
item = Item(item_type=item_dict['item_type'], payload=item_dict['payload'])
controller = Controller(controller_config, os.getcwd())
#try:
if True:
print("Running transformer job")
controller.run_transformer_job(transformer_config, item, os.getcwd())
return {'status': 'OK'}
try:
pass
except Exception as e:
msg = "Transformer failed to execute with error: %s" % e
print(msg)
return {'status': 'error', 'message': msg}
def test_transformer_lambda_handler(self):
controller = Controller(self.config)
#controller.create_resources()
controller.run_source_job(self.config['sources'][0])[0]
input_queue = controller.get_sqs_queue(item.item_type)
queue_item = None
time.sleep(1)
for message in input_queue.receive_messages():
queue_item = controller.item_from_message_payload(
item.item_type, message, input_queue.url)
print("Message", message)
print(queue_item.payload)
event = {
'controller_config':
json.dumps(self.config),
'transformer_config':
json.dumps(self.config['transformers'][0]),
'item':
json.dumps({
"item_type": queue_item.item_type,
"payload": queue_item.payload
})
}
transformer_handler(event, None)
def test_backfill(self):
source_path = os.path.join(*(["/"] + (__file__.split("/")[:-2]) + ["test_data"]))
controller = Controller(self.config, source_path)
#controller.create_resources()
#item = controller.run_source_job(self.config['sources'][0])[0]
mapper = DataMapper(controller)
stats = mapper.local_backfill("antenna_test_table",
"ArticleReference",
"IdentityTransformer",
required_null_field="source_url",
limit=10
)
self.assertEqual(stats['number_items_backfilled'], 0)
stats = mapper.local_backfill("antenna_test_table",
"ArticleReference",
"IdentityTransformer",
required_null_field=None,
limit=10
)
self.assertTrue(stats['number_items_backfilled'] > 0)
def test_controller(self):
controller = Controller(self.config)
controller.run()
resources = ResourceManager(controller)
cluster = resources.createResourceCluster()
def tearDown(self):
controller = Controller(self.config)
controller.drain_queues()
def test_lambda(self):
controller = Controller(self.config)
controller.create_resources()
controller.create_lambda_functions()
role_arn = controller.get_lambda_role_arn()
client = controller._aws_manager.get_client('lambda')
print("ARN: %s" % role_arn)
function_name = "test%s" % random.randrange(100000000)
res = create_lambda_function(
function_name, role_arn,
controller._aws_manager.get_client('lambda'),
"lambda_handlers.transformer_handler")
controller = Controller(self.config)
item = controller.run_source_job(self.config['sources'][0])[0]
input_queue = controller.get_sqs_queue(item.item_type)
queue_item = None
time.sleep(1)
for message in input_queue.receive_messages():
queue_item = controller.item_from_message_payload(
item.item_type, message, input_queue.url)
break
event = {
'controller_config':
json.dumps(self.config),
'transformer_config':
json.dumps(self.config['transformers'][0]),
'item':
json.dumps({
"item_type": queue_item.item_type,
"payload": queue_item.payload
})
}
response = client.invoke(FunctionName=function_name,
InvocationType='RequestResponse',
Payload=json.dumps(event))
payload = str(response['Payload'].read())
print("Response payload", payload)
self.assertEqual(json.loads(payload)['status'], 'OK')
cleanup_lambda_package()
response = client.delete_function(FunctionName=function_name)
def test_controller_lambda_handler(self):
with open("./antenna.json", 'r') as f:
controller = Controller(json.load(f))
#controller.create_lambda_functions()
controller_handler(None, None)
def test_load_custom_transformer(self):
source_path = os.path.join(*(["/"] + (__file__.split("/")[:-2]) +
["test_data"]))
controller = Controller(self.config, source_path)
module = controller.import_transformer(
"transformers.CustomTestTransformer", source_path)
def controller_handler(event, context):
with open("./antenna.json", 'r') as f:
controller_config = json.load(f)
controller = Controller(controller_config, os.getcwd())
controller.run()