def test_parse(self, mocker):
arg_parser = TransformerArgumentParser(description="Test Transformer")
sys.argv = ["foo",
"--path", "/foo/bar",
"--brokerlist", "kafka1.org",
"--topic", "mytopic",
"--chunks", "100",
"--tree", "Events",
"--limit", "10",
'--result-destination', 'kafka',
'--result-format', 'arrow',
"--max-message-size", "42",
'--rabbit-uri', "http://rabbit.org",
'--request-id', "123-45-678"
]
args = arg_parser.parse_args()
assert args.path == '/foo/bar'
assert args.brokerlist == "kafka1.org"
assert args.topic == 'mytopic'
assert args.chunks == 100
assert args.tree == 'Events'
assert args.limit == 10
assert args.result_destination == 'kafka'
assert args.result_format == 'arrow'
assert args.max_message_size == 42
assert args.rabbit_uri == "http://rabbit.org"
assert args.request_id == "123-45-678"
def compile_code():
# Have to use bash as the file runner.sh does not execute properly, despite its 'x'
# bit set. This seems to be some vagary of a ConfigMap from k8, which is how we usually get
# this file.
r = os.system('bash /generated/runner.sh -c | tee log.txt')
if r != 0:
with open('log.txt', 'r') as f:
errors = f.read()
raise RuntimeError("Unable to compile the code - error return: " +
str(r) + 'errors: \n' + errors)
if __name__ == "__main__":
parser = TransformerArgumentParser(description="xAOD CPP Transformer")
args = parser.parse_args()
kafka_brokers = TransformerArgumentParser.extract_kafka_brokers(
args.brokerlist)
if args.result_destination == 'kafka':
messaging = KafkaMessaging(kafka_brokers, args.max_message_size)
object_store = None
elif not args.output_dir and args.result_destination == 'object-store':
messaging = None
object_store = ObjectStoreManager()
compile_code()
if args.request_id and not args.path:
# Put your transformer code here!
r = -1
reason_bad = None
if r != 0:
reason_bad = "Error return from transformer: " + str(r)
if (reason_bad is None) and not os.path.exists(output_path):
reason_bad = "Output file " + output_path + " was not found"
if reason_bad is not None:
with open('log.txt', 'r') as f:
errors = f.read()
raise RuntimeError("Failed to transform input file " + file_path +
": " + reason_bad + ' -- errors: \n' + errors)
if __name__ == "__main__":
parser = TransformerArgumentParser(
description="ServiceX Transformer Template")
args = parser.parse_args()
assert args.result_format == 'root-file', 'We only know how to create root file output'
assert args.result_destination != 'kafka', 'Kafka not yet supported'
if not args.output_dir and args.result_destination == 'object-store':
messaging = None
object_store = ObjectStoreManager()
if args.request_id and not args.path:
rabbitmq = RabbitMQManager(args.rabbit_uri, args.request_id, callback)
if args.path:
transform_single_file(args.path, args.output_dir)
def test_init(self, mocker):
argparse.ArgumentParser.exit = mocker.Mock()
arg_parser = TransformerArgumentParser(description="Test Transformer")
sys.argv = ["foo", "bar"]
arg_parser.parse_args()
argparse.ArgumentParser.exit.assert_called()
transformer = ArrowIterator(arrow,
chunk_size=1000,
file_path=file_path)
arrow_writer.write_branches_to_arrow(transformer=transformer,
topic_name=args.request_id,
file_id=None,
request_id=args.request_id)
def compile_code():
import generated_transformer
pass
if __name__ == "__main__":
parser = TransformerArgumentParser(description="Uproot Transformer")
args = parser.parse_args()
print("-----", sys.path)
kafka_brokers = TransformerArgumentParser.extract_kafka_brokers(
args.brokerlist)
print(args.result_destination, args.output_dir)
if args.output_dir:
messaging = None
object_store = None
elif args.result_destination == 'kafka':
messaging = KafkaMessaging(kafka_brokers, args.max_message_size)
object_store = None
elif args.result_destination == 'object-store':
messaging = None