def test_put_file_complete_retry(self, mocker): import requests mock_session = mocker.MagicMock(requests.session) mock_session.mount = mocker.Mock() mock_session.put = mocker.Mock(side_effect=[requests.exceptions.ConnectionError, 200]) mocker.patch('requests.session', return_value=mock_session) adapter = ServiceXAdapter("http://foo.com") adapter.put_file_complete("my-root.root", 42, "testing", 1, 2, 3, 4) assert mock_session.put.call_count == 2
def callback(channel, method, properties, body): transform_request = json.loads(body) _request_id = transform_request['request-id'] _file_path = transform_request['file-path'] _file_id = transform_request['file-id'] _server_endpoint = transform_request['service-endpoint'] servicex = ServiceXAdapter(_server_endpoint) tick = time.time() try: # Do the transform root_file = _file_path.replace('/', ':') output_path = '/home/atlas/' + root_file transform_single_file(_file_path, output_path) tock = time.time() if object_store: object_store.upload_file(_request_id, root_file, output_path) os.remove(output_path) servicex.post_status_update("File " + _file_path + " complete") servicex.put_file_complete(_file_path, _file_id, "success", num_messages=0, total_time=round(tock - tick, 2), total_events=0, total_bytes=0) except Exception as error: transform_request['error'] = str(error) channel.basic_publish(exchange='transformation_failures', routing_key=_request_id + '_errors', body=json.dumps(transform_request)) servicex.put_file_complete(file_path=_file_path, file_id=_file_id, status='failure', num_messages=0, total_time=0, total_events=0, total_bytes=0) finally: channel.basic_ack(delivery_tag=method.delivery_tag)
def test_put_file_complete(self, mocker): import requests mock_session = mocker.MagicMock(requests.session) mock_session.mount = mocker.Mock() mock_session.put = mocker.Mock() mocker.patch('requests.session', return_value=mock_session) adapter = ServiceXAdapter("http://foo.com") adapter.put_file_complete("my-root.root", 42, "testing", 1, 2, 3, 4) mock_session.put.assert_called() args = mock_session.put.call_args assert args[0][0] == 'http://foo.com/file-complete' doc = args[1]['json'] assert doc['status'] == 'testing' assert doc['total-events'] == 3 assert doc['total-time'] == 2 assert doc['file-path'] == 'my-root.root' assert doc['num-messages'] == 1 assert doc['file-id'] == 42 assert doc['avg-rate'] == 1
def callback(channel, method, properties, body): transform_request = json.loads(body) _request_id = transform_request['request-id'] _file_path = transform_request['file-path'].encode('ascii', 'ignore') _file_id = transform_request['file-id'] _server_endpoint = transform_request['service-endpoint'] _chunks = transform_request['chunk-size'] servicex = ServiceXAdapter(_server_endpoint) servicex.post_status_update(file_id=_file_id, status_code="start", info="xAOD Transformer") tick = time.time() file_done = False file_retries = 0 while not file_done: try: # Do the transform root_file = _file_path.replace('/', ':') output_path = '/home/atlas/' + root_file transform_single_file(_file_path, output_path, _chunks, servicex) tock = time.time() if object_store: object_store.upload_file(_request_id, root_file, output_path) os.remove(output_path) servicex.post_status_update(file_id=_file_id, status_code="complete", info="Total time " + str(round(tock - tick, 2))) servicex.put_file_complete(_file_path, _file_id, "success", num_messages=0, total_time=round(tock - tick, 2), total_events=0, total_bytes=0) file_done = True except Exception as error: file_retries += 1 if file_retries == MAX_RETRIES: transform_request['error'] = str(error) channel.basic_publish(exchange='transformation_failures', routing_key=_request_id + '_errors', body=json.dumps(transform_request)) servicex.put_file_complete(file_path=_file_path, file_id=_file_id, status='failure', num_messages=0, total_time=0, total_events=0, total_bytes=0) servicex.post_status_update(file_id=_file_id, status_code="failure", info=" error: " + str(error)[0:1024]) file_done = True exc_type, exc_value, exc_traceback = sys.exc_info() traceback.print_tb(exc_traceback, limit=20, file=sys.stdout) print(exc_value) else: servicex.post_status_update(file_id=_file_id, status_code="retry", info="Try: " + str(file_retries) + " error: " + str(error)[0:1024]) channel.basic_ack(delivery_tag=method.delivery_tag)
def callback(channel, method, properties, body): transform_request = json.loads(body) _request_id = transform_request['request-id'] _file_path = transform_request['file-path'] _file_id = transform_request['file-id'] _server_endpoint = transform_request['service-endpoint'] _tree_name = transform_request['tree-name'] # _chunks = transform_request['chunks'] servicex = ServiceXAdapter(_server_endpoint) tick = time.time() try: # Do the transform servicex.post_status_update(file_id=_file_id, status_code="start", info="tree-name: " + _tree_name) root_file = _file_path.replace('/', ':') output_path = '/home/atlas/' + root_file transform_single_file(_file_path, output_path + ".parquet", servicex, tree_name=_tree_name) tock = time.time() if object_store: object_store.upload_file(_request_id, root_file + ".parquet", output_path + ".parquet") os.remove(output_path + ".parquet") servicex.post_status_update(file_id=_file_id, status_code="complete", info="Success") servicex.put_file_complete(_file_path, _file_id, "success", num_messages=0, total_time=round(tock - tick, 2), total_events=0, total_bytes=0) except Exception as error: exc_type, exc_value, exc_traceback = sys.exc_info() traceback.print_tb(exc_traceback, limit=20, file=sys.stdout) print(exc_value) transform_request['error'] = str(error) channel.basic_publish(exchange='transformation_failures', routing_key=_request_id + '_errors', body=json.dumps(transform_request)) servicex.post_status_update(file_id=_file_id, status_code="failure", info="error: " + str(exc_value)[0:1024]) servicex.put_file_complete(file_path=_file_path, file_id=_file_id, status='failure', num_messages=0, total_time=0, total_events=0, total_bytes=0) finally: channel.basic_ack(delivery_tag=method.delivery_tag)