Beispiel #1
0
 def test_source_lambda_handler(self):
     controller = Controller(self.config)
     controller.create_resources()
     event = {
         'controller_config': json.dumps(self.config),
         'source_config': json.dumps(self.config['sources'][0])
     }
     source_handler(event, None)
Beispiel #2
0
def source_handler(event, context):
    controller_config = json.loads(event['controller_config'])
    source_config = json.loads(event['source_config'])
    controller = Controller(controller_config, os.getcwd())

    try:
        controller.run_source_job(source_config)
        return {'status': 'OK'}
    except Exception as e:
        msg = "Source failed to execute with error: %s" % e
        print(msg)
        return {'status': 'error', 'message': msg}
Beispiel #3
0
    def test_deploy(self):
        controller = Controller(self.config)

        # Create cluster if needed
        controller.create_resources()

        # Update Lambdas if needed
        controller.create_lambda_functions()

        # Schedule master lambda
        controller.schedule_controller_lambda()
Beispiel #4
0
def transformer_handler(event, context):
    print("Transformer handler initialized")
    controller_config = json.loads(event['controller_config'])
    transformer_config = json.loads(event['transformer_config'])
    item_dict = json.loads(event['item'])
    item = Item(item_type=item_dict['item_type'], payload=item_dict['payload'])
    controller = Controller(controller_config, os.getcwd())

    #try:
    if True:
        print("Running transformer job")
        controller.run_transformer_job(transformer_config, item, os.getcwd())
        return {'status': 'OK'}
    try:
        pass
    except Exception as e:
        msg = "Transformer failed to execute with error: %s" % e
        print(msg)
        return {'status': 'error', 'message': msg}
Beispiel #5
0
    def test_transformer_lambda_handler(self):
        controller = Controller(self.config)
        #controller.create_resources()
        controller.run_source_job(self.config['sources'][0])[0]
        input_queue = controller.get_sqs_queue(item.item_type)
        queue_item = None

        time.sleep(1)
        for message in input_queue.receive_messages():
            queue_item = controller.item_from_message_payload(
                item.item_type, message, input_queue.url)
            print("Message", message)
            print(queue_item.payload)
            event = {
                'controller_config':
                json.dumps(self.config),
                'transformer_config':
                json.dumps(self.config['transformers'][0]),
                'item':
                json.dumps({
                    "item_type": queue_item.item_type,
                    "payload": queue_item.payload
                })
            }
            transformer_handler(event, None)
Beispiel #6
0
    def test_backfill(self):
        source_path = os.path.join(*(["/"] + (__file__.split("/")[:-2]) + ["test_data"]))
        controller = Controller(self.config, source_path)
        #controller.create_resources()
        #item = controller.run_source_job(self.config['sources'][0])[0]
        mapper = DataMapper(controller)

        stats = mapper.local_backfill("antenna_test_table",
                                      "ArticleReference",
                                      "IdentityTransformer",
                                      required_null_field="source_url",
                                      limit=10
        )
        self.assertEqual(stats['number_items_backfilled'], 0)

        stats = mapper.local_backfill("antenna_test_table",
                                      "ArticleReference",
                                      "IdentityTransformer",
                                      required_null_field=None,
                                      limit=10
        )
        self.assertTrue(stats['number_items_backfilled'] > 0)
Beispiel #7
0
    def test_controller(self):
        controller = Controller(self.config)
        controller.run()

        resources = ResourceManager(controller)
        cluster = resources.createResourceCluster()
Beispiel #8
0
 def tearDown(self):
     controller = Controller(self.config)
     controller.drain_queues()
Beispiel #9
0
    def test_lambda(self):
        controller = Controller(self.config)
        controller.create_resources()
        controller.create_lambda_functions()

        role_arn = controller.get_lambda_role_arn()
        client = controller._aws_manager.get_client('lambda')
        print("ARN: %s" % role_arn)

        function_name = "test%s" % random.randrange(100000000)

        res = create_lambda_function(
            function_name, role_arn,
            controller._aws_manager.get_client('lambda'),
            "lambda_handlers.transformer_handler")

        controller = Controller(self.config)
        item = controller.run_source_job(self.config['sources'][0])[0]
        input_queue = controller.get_sqs_queue(item.item_type)
        queue_item = None
        time.sleep(1)
        for message in input_queue.receive_messages():
            queue_item = controller.item_from_message_payload(
                item.item_type, message, input_queue.url)
            break

        event = {
            'controller_config':
            json.dumps(self.config),
            'transformer_config':
            json.dumps(self.config['transformers'][0]),
            'item':
            json.dumps({
                "item_type": queue_item.item_type,
                "payload": queue_item.payload
            })
        }

        response = client.invoke(FunctionName=function_name,
                                 InvocationType='RequestResponse',
                                 Payload=json.dumps(event))
        payload = str(response['Payload'].read())
        print("Response payload", payload)
        self.assertEqual(json.loads(payload)['status'], 'OK')

        cleanup_lambda_package()
        response = client.delete_function(FunctionName=function_name)
Beispiel #10
0
 def test_controller_lambda_handler(self):
     with open("./antenna.json", 'r') as f:
         controller = Controller(json.load(f))
     #controller.create_lambda_functions()
     controller_handler(None, None)
Beispiel #11
0
 def test_load_custom_transformer(self):
     source_path = os.path.join(*(["/"] + (__file__.split("/")[:-2]) +
                                  ["test_data"]))
     controller = Controller(self.config, source_path)
     module = controller.import_transformer(
         "transformers.CustomTestTransformer", source_path)
Beispiel #12
0
def controller_handler(event, context):
    with open("./antenna.json", 'r') as f:
        controller_config = json.load(f)
    controller = Controller(controller_config, os.getcwd())
    controller.run()