def test_put_file_complete_retry(self, mocker):
import requests
mock_session = mocker.MagicMock(requests.session)
mock_session.mount = mocker.Mock()
mock_session.put = mocker.Mock(side_effect=[requests.exceptions.ConnectionError, 200])
mocker.patch('requests.session', return_value=mock_session)
adapter = ServiceXAdapter("http://foo.com")
adapter.put_file_complete("my-root.root", 42, "testing", 1, 2, 3, 4)
assert mock_session.put.call_count == 2
def test_post_status_update_retry(self, mocker):
import requests
import os
mocker.patch.dict(os.environ, {"POD_NAME": "my-pod"})
mock_session = mocker.MagicMock(requests.session)
mock_session.mount = mocker.Mock()
mock_session.post = mocker.Mock(
side_effect=[requests.exceptions.ConnectionError, 200])
mocker.patch('requests.session', return_value=mock_session)
adapter = ServiceXAdapter("http://foo.com")
adapter.post_status_update(42, "testing", "this is a test")
assert mock_session.post.call_count == 2
def test_init(self, mocker):
import requests
mock_session = mocker.MagicMock(requests.session)
mocker.patch('requests.session', return_value=mock_session)
mock_session.mount = mocker.Mock()
ServiceXAdapter("http://foo.com")
retries = mock_session.mount.mock_calls[0][1][1].max_retries
assert retries.total == 5
assert retries.connect == 3
def test_post_status_update(self, mocker):
import requests
import os
mocker.patch.dict(os.environ, {"POD_NAME": "my-pod"})
mock_session = mocker.MagicMock(requests.session)
mock_session.mount = mocker.Mock()
mock_session.post = mocker.Mock()
mocker.patch('requests.session', return_value=mock_session)
adapter = ServiceXAdapter("http://foo.com")
adapter.post_status_update(42, "testing", "this is a test")
mock_session.post.assert_called()
args = mock_session.post.call_args
assert args[0][0] == 'http://foo.com/42/status'
doc = args[1]['data']
assert doc['status-code'] == 'testing'
assert doc['info'] == 'this is a test'
assert doc['pod-name'] == 'my-pod'
def test_put_file_complete(self, mocker):
import requests
mock_session = mocker.MagicMock(requests.session)
mock_session.mount = mocker.Mock()
mock_session.put = mocker.Mock()
mocker.patch('requests.session', return_value=mock_session)
adapter = ServiceXAdapter("http://foo.com")
adapter.put_file_complete("my-root.root", 42, "testing", 1, 2, 3, 4)
mock_session.put.assert_called()
args = mock_session.put.call_args
assert args[0][0] == 'http://foo.com/file-complete'
doc = args[1]['json']
assert doc['status'] == 'testing'
assert doc['total-events'] == 3
assert doc['total-time'] == 2
assert doc['file-path'] == 'my-root.root'
assert doc['num-messages'] == 1
assert doc['file-id'] == 42
assert doc['avg-rate'] == 1
def callback(channel, method, properties, body):
transform_request = json.loads(body)
_request_id = transform_request['request-id']
_file_path = transform_request['file-path']
_file_id = transform_request['file-id']
_server_endpoint = transform_request['service-endpoint']
servicex = ServiceXAdapter(_server_endpoint)
tick = time.time()
try:
# Do the transform
root_file = _file_path.replace('/', ':')
output_path = '/home/atlas/' + root_file
transform_single_file(_file_path, output_path)
tock = time.time()
if object_store:
object_store.upload_file(_request_id, root_file, output_path)
os.remove(output_path)
servicex.post_status_update("File " + _file_path + " complete")
servicex.put_file_complete(_file_path,
_file_id,
"success",
num_messages=0,
total_time=round(tock - tick, 2),
total_events=0,
total_bytes=0)
except Exception as error:
transform_request['error'] = str(error)
channel.basic_publish(exchange='transformation_failures',
routing_key=_request_id + '_errors',
body=json.dumps(transform_request))
servicex.put_file_complete(file_path=_file_path,
file_id=_file_id,
status='failure',
num_messages=0,
total_time=0,
total_events=0,
total_bytes=0)
finally:
channel.basic_ack(delivery_tag=method.delivery_tag)
def callback(channel, method, properties, body):
transform_request = json.loads(body)
_request_id = transform_request['request-id']
_file_path = transform_request['file-path'].encode('ascii', 'ignore')
_file_id = transform_request['file-id']
_server_endpoint = transform_request['service-endpoint']
_chunks = transform_request['chunk-size']
servicex = ServiceXAdapter(_server_endpoint)
servicex.post_status_update(file_id=_file_id,
status_code="start",
info="xAOD Transformer")
tick = time.time()
file_done = False
file_retries = 0
while not file_done:
try:
# Do the transform
root_file = _file_path.replace('/', ':')
output_path = '/home/atlas/' + root_file
transform_single_file(_file_path, output_path, _chunks, servicex)
tock = time.time()
if object_store:
object_store.upload_file(_request_id, root_file, output_path)
os.remove(output_path)
servicex.post_status_update(file_id=_file_id,
status_code="complete",
info="Total time " +
str(round(tock - tick, 2)))
servicex.put_file_complete(_file_path,
_file_id,
"success",
num_messages=0,
total_time=round(tock - tick, 2),
total_events=0,
total_bytes=0)
file_done = True
except Exception as error:
file_retries += 1
if file_retries == MAX_RETRIES:
transform_request['error'] = str(error)
channel.basic_publish(exchange='transformation_failures',
routing_key=_request_id + '_errors',
body=json.dumps(transform_request))
servicex.put_file_complete(file_path=_file_path,
file_id=_file_id,
status='failure',
num_messages=0,
total_time=0,
total_events=0,
total_bytes=0)
servicex.post_status_update(file_id=_file_id,
status_code="failure",
info=" error: " +
str(error)[0:1024])
file_done = True
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_tb(exc_traceback, limit=20, file=sys.stdout)
print(exc_value)
else:
servicex.post_status_update(file_id=_file_id,
status_code="retry",
info="Try: " + str(file_retries) +
" error: " + str(error)[0:1024])
channel.basic_ack(delivery_tag=method.delivery_tag)
def callback(channel, method, properties, body):
transform_request = json.loads(body)
_request_id = transform_request['request-id']
_file_path = transform_request['file-path']
_file_id = transform_request['file-id']
_server_endpoint = transform_request['service-endpoint']
_tree_name = transform_request['tree-name']
# _chunks = transform_request['chunks']
servicex = ServiceXAdapter(_server_endpoint)
tick = time.time()
try:
# Do the transform
servicex.post_status_update(file_id=_file_id,
status_code="start",
info="tree-name: " + _tree_name)
root_file = _file_path.replace('/', ':')
output_path = '/home/atlas/' + root_file
transform_single_file(_file_path,
output_path + ".parquet",
servicex,
tree_name=_tree_name)
tock = time.time()
if object_store:
object_store.upload_file(_request_id, root_file + ".parquet",
output_path + ".parquet")
os.remove(output_path + ".parquet")
servicex.post_status_update(file_id=_file_id,
status_code="complete",
info="Success")
servicex.put_file_complete(_file_path,
_file_id,
"success",
num_messages=0,
total_time=round(tock - tick, 2),
total_events=0,
total_bytes=0)
except Exception as error:
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_tb(exc_traceback, limit=20, file=sys.stdout)
print(exc_value)
transform_request['error'] = str(error)
channel.basic_publish(exchange='transformation_failures',
routing_key=_request_id + '_errors',
body=json.dumps(transform_request))
servicex.post_status_update(file_id=_file_id,
status_code="failure",
info="error: " + str(exc_value)[0:1024])
servicex.put_file_complete(file_path=_file_path,
file_id=_file_id,
status='failure',
num_messages=0,
total_time=0,
total_events=0,
total_bytes=0)
finally:
channel.basic_ack(delivery_tag=method.delivery_tag)