def _read_file_section(self, uuid, path, offset, length, gzipped):
"""
Reads length bytes of the file at the given path in the bundle.
The result is gzipped if gzipped is True.
"""
if self._is_available_locally(uuid):
file_path = self._get_target_path(uuid, path)
string = file_util.read_file_section(file_path, offset, length)
if gzipped:
string = file_util.gzip_string(string)
return string
else:
worker = self.get_bundle_worker(uuid)
response_socket_id = self._worker_model.allocate_socket(
worker['user_id'], worker['worker_id'])
try:
read_args = {
'type': 'read_file_section',
'offset': offset,
'length': length,
}
self._send_read_message(worker, response_socket_id, uuid, path,
read_args)
string = self._get_read_response_string(response_socket_id)
finally:
self._worker_model.deallocate_socket(response_socket_id)
if not gzipped:
string = file_util.un_gzip_string(string)
return string
def read_file_section(self, uuid, path, offset, length, gzipped):
"""
Reads length bytes of the file at the given path in the bundle.
The result is gzipped if gzipped is True.
"""
if self._is_available_locally(uuid):
file_path = self._get_target_path(uuid, path)
string = file_util.read_file_section(file_path, offset, length)
if gzipped:
string = file_util.gzip_string(string)
return string
else:
worker = self._worker_model.get_bundle_worker(uuid)
response_socket_id = self._worker_model.allocate_socket(
worker['user_id'], worker['worker_id']
)
try:
read_args = {'type': 'read_file_section', 'offset': offset, 'length': length}
self._send_read_message(worker, response_socket_id, uuid, path, read_args)
string = self._get_read_response_string(response_socket_id)
finally:
self._worker_model.deallocate_socket(response_socket_id)
if not gzipped:
string = file_util.un_gzip_string(string)
return string
def summarize_file(self, uuid, path, num_head_lines, num_tail_lines, max_line_length, truncation_text, gzipped):
"""
Summarizes the file at the given path in the bundle, returning a string
containing the given numbers of lines from beginning and end of the file.
If the file needs to be truncated, places truncation_text at the
truncation point.
This string is gzipped if gzipped is True.
"""
if self._is_available_locally(uuid):
file_path = self._get_target_path(uuid, path)
string = file_util.summarize_file(file_path, num_head_lines, num_tail_lines, max_line_length, truncation_text)
if gzipped:
string = file_util.gzip_string(string)
return string
else:
worker = self._worker_model.get_bundle_worker(uuid)
response_socket_id = self._worker_model.allocate_socket(worker['user_id'], worker['worker_id'])
try:
read_args = {
'type': 'summarize_file',
'num_head_lines': num_head_lines,
'num_tail_lines': num_tail_lines,
'max_line_length': max_line_length,
'truncation_text': truncation_text,
}
self._send_read_message(worker, response_socket_id, uuid, path, read_args)
string = self._get_read_response_string(response_socket_id)
finally:
self._worker_model.deallocate_socket(response_socket_id)
if not gzipped:
string = file_util.un_gzip_string(string)
return string
def summarize_file_thread(final_path):
string = gzip_string(
summarize_file(
final_path,
args['num_head_lines'],
args['num_tail_lines'],
args['max_line_length'],
args['truncation_text'],
))
reply_fn(None, {}, string)
def summarize_file(
self, uuid, path, num_head_lines, num_tail_lines, max_line_length, truncation_text, gzipped
):
"""
Summarizes the file at the given path in the bundle, returning a string
containing the given numbers of lines from beginning and end of the file.
If the file needs to be truncated, places truncation_text at the
truncation point.
This string is gzipped if gzipped is True.
"""
if self._is_available_locally(uuid):
file_path = self._get_target_path(uuid, path)
string = file_util.summarize_file(
file_path, num_head_lines, num_tail_lines, max_line_length, truncation_text
)
if gzipped:
string = file_util.gzip_string(string)
return string
else:
worker = self._worker_model.get_bundle_worker(uuid)
response_socket_id = self._worker_model.allocate_socket(
worker['user_id'], worker['worker_id']
)
try:
read_args = {
'type': 'summarize_file',
'num_head_lines': num_head_lines,
'num_tail_lines': num_tail_lines,
'max_line_length': max_line_length,
'truncation_text': truncation_text,
}
self._send_read_message(worker, response_socket_id, uuid, path, read_args)
string = self._get_read_response_string(response_socket_id)
finally:
self._worker_model.deallocate_socket(response_socket_id)
if not gzipped:
string = file_util.un_gzip_string(string)
return string
def test_gzip_string(self):
self.assertEqual(un_gzip_string(gzip_string('contents')), 'contents')
def test_gzip_string(self):
self.assertEqual(un_gzip_string(gzip_string('contents')), 'contents')
def read_file_section_thread(final_path):
string = gzip_string(
read_file_section(final_path, args['offset'], args['length']))
reply_fn(None, {}, string)
def test_single_local_gzip_path_remove_sources(self):
source = os.path.join(self.temp_dir, 'filename.gz')
self.write_bytes_to_file(gzip_string('testing'), source)
self.do_upload([source], remove_sources=True)
self.assertFalse(os.path.exists(source))
def test_single_local_gzip_path(self):
source = os.path.join(self.temp_dir, 'filename.gz')
self.write_bytes_to_file(gzip_string('testing'), source)
self.do_upload([source], unpack=True)
self.assertTrue(os.path.exists(source))
self.check_file_contains_string(self.bundle_location, 'testing')
