def read_file_section(self, target, offset, length, gzipped):
"""
Reads length bytes of the file at the given path in the bundle.
The result is gzipped if gzipped is True.
"""
if self._is_available_locally(target):
file_path = self._get_target_path(target)
bytestring = file_util.read_file_section(file_path, offset, length)
if gzipped:
bytestring = file_util.gzip_bytestring(bytestring)
return bytestring
else:
worker = self._bundle_model.get_bundle_worker(target.bundle_uuid)
response_socket_id = self._worker_model.allocate_socket(
worker['user_id'], worker['worker_id']
)
try:
read_args = {'type': 'read_file_section', 'offset': offset, 'length': length}
self._send_read_message(worker, response_socket_id, target, read_args)
bytestring = self._get_read_response(response_socket_id)
finally:
self._worker_model.deallocate_socket(response_socket_id)
# Note: all data from the worker is gzipped (see `local_reader.py`).
if not gzipped:
bytestring = file_util.un_gzip_bytestring(bytestring)
return bytestring
def summarize_file_thread(final_path):
bytestring = gzip_bytestring(
summarize_file(
final_path,
args['num_head_lines'],
args['num_tail_lines'],
args['max_line_length'],
args['truncation_text'],
).encode())
reply_fn(None, {}, bytestring)
def summarize_file(
self, target, num_head_lines, num_tail_lines, max_line_length, truncation_text, gzipped
):
"""
Summarizes the file at the given path in the bundle, returning bytes
containing the given numbers of lines from beginning and end of the file.
If the file needs to be truncated, places truncation_text at the
truncation point.
The return value is gzipped if gzipped is True.
"""
if self._is_available_locally(target):
file_path = self._get_target_path(target)
# Note: summarize_file returns string, but make it bytes for consistency.
bytestring = file_util.summarize_file(
file_path, num_head_lines, num_tail_lines, max_line_length, truncation_text
).encode()
if gzipped:
bytestring = file_util.gzip_bytestring(bytestring)
return bytestring
else:
worker = self._bundle_model.get_bundle_worker(target.bundle_uuid)
response_socket_id = self._worker_model.allocate_socket(
worker['user_id'], worker['worker_id']
)
try:
read_args = {
'type': 'summarize_file',
'num_head_lines': num_head_lines,
'num_tail_lines': num_tail_lines,
'max_line_length': max_line_length,
'truncation_text': truncation_text,
}
self._send_read_message(worker, response_socket_id, target, read_args)
bytestring = self._get_read_response(response_socket_id)
finally:
self._worker_model.deallocate_socket(response_socket_id)
# Note: all data from the worker is gzipped (see `local_reader.py`).
if not gzipped:
bytestring = file_util.un_gzip_bytestring(bytestring)
return bytestring
def read_file_section_thread(final_path):
bytestring = gzip_bytestring(
read_file_section(final_path, args['offset'], args['length']))
reply_fn(None, {}, bytestring)
def test_fileobj_gz(self):
self.do_upload(('source.gz', BytesIO(gzip_bytestring(b'testing'))))
self.check_file_equals_string('', 'testing')
def test_gzip_bytestring(self):
self.assertEqual(un_gzip_bytestring(gzip_bytestring(b'contents')),
b'contents')
def test_single_local_gzip_path(self):
source = os.path.join(self.temp_dir, 'filename.gz')
self.write_bytes_to_file(gzip_bytestring(b'testing'), source)
self.do_upload([source], unpack=True)
self.assertTrue(os.path.exists(source))
self.check_file_contains_string(self.bundle_location, 'testing')
def test_single_local_gzip_path_remove_sources(self):
source = os.path.join(self.temp_dir, 'filename.gz')
self.write_bytes_to_file(gzip_bytestring(b'testing'), source)
self.do_upload([source], remove_sources=True)
self.assertFalse(os.path.exists(source))
