def test_parse(self, mocker): arg_parser = TransformerArgumentParser(description="Test Transformer") sys.argv = ["foo", "--path", "/foo/bar", "--brokerlist", "kafka1.org", "--topic", "mytopic", "--chunks", "100", "--tree", "Events", "--limit", "10", '--result-destination', 'kafka', '--result-format', 'arrow', "--max-message-size", "42", '--rabbit-uri', "http://rabbit.org", '--request-id', "123-45-678" ] args = arg_parser.parse_args() assert args.path == '/foo/bar' assert args.brokerlist == "kafka1.org" assert args.topic == 'mytopic' assert args.chunks == 100 assert args.tree == 'Events' assert args.limit == 10 assert args.result_destination == 'kafka' assert args.result_format == 'arrow' assert args.max_message_size == 42 assert args.rabbit_uri == "http://rabbit.org" assert args.request_id == "123-45-678"
def compile_code(): # Have to use bash as the file runner.sh does not execute properly, despite its 'x' # bit set. This seems to be some vagary of a ConfigMap from k8, which is how we usually get # this file. r = os.system('bash /generated/runner.sh -c | tee log.txt') if r != 0: with open('log.txt', 'r') as f: errors = f.read() raise RuntimeError("Unable to compile the code - error return: " + str(r) + 'errors: \n' + errors) if __name__ == "__main__": parser = TransformerArgumentParser(description="xAOD CPP Transformer") args = parser.parse_args() kafka_brokers = TransformerArgumentParser.extract_kafka_brokers( args.brokerlist) if args.result_destination == 'kafka': messaging = KafkaMessaging(kafka_brokers, args.max_message_size) object_store = None elif not args.output_dir and args.result_destination == 'object-store': messaging = None object_store = ObjectStoreManager() compile_code() if args.request_id and not args.path:
def test_extract_attr_list(self): attrs = TransformerArgumentParser.extract_attr_list("a,b,c") assert len(attrs) == 3 assert attrs == ['a', 'b', 'c']
def test_extract_broker_list(self): brokers = TransformerArgumentParser.extract_kafka_brokers("kafka1,kafka2") assert len(brokers) == 2 assert brokers[0] == 'kafka1' assert brokers[1] == 'kafka2'
def test_init(self, mocker): argparse.ArgumentParser.exit = mocker.Mock() arg_parser = TransformerArgumentParser(description="Test Transformer") sys.argv = ["foo", "bar"] arg_parser.parse_args() argparse.ArgumentParser.exit.assert_called()
# Put your transformer code here! r = -1 reason_bad = None if r != 0: reason_bad = "Error return from transformer: " + str(r) if (reason_bad is None) and not os.path.exists(output_path): reason_bad = "Output file " + output_path + " was not found" if reason_bad is not None: with open('log.txt', 'r') as f: errors = f.read() raise RuntimeError("Failed to transform input file " + file_path + ": " + reason_bad + ' -- errors: \n' + errors) if __name__ == "__main__": parser = TransformerArgumentParser( description="ServiceX Transformer Template") args = parser.parse_args() assert args.result_format == 'root-file', 'We only know how to create root file output' assert args.result_destination != 'kafka', 'Kafka not yet supported' if not args.output_dir and args.result_destination == 'object-store': messaging = None object_store = ObjectStoreManager() if args.request_id and not args.path: rabbitmq = RabbitMQManager(args.rabbit_uri, args.request_id, callback) if args.path: transform_single_file(args.path, args.output_dir)