def _get_status_string(self):
"""See super class."""
scaled_processed_bytes = scaled_integer.FormatBinaryNumber(
self._processed_bytes, decimal_places=1)
if self._total_files_estimation:
file_progress_string = '{}/{}'.format(self._completed_files,
self._total_files_estimation)
else:
file_progress_string = self._completed_files
if self._total_bytes_estimation:
scaled_total_bytes_estimation = scaled_integer.FormatBinaryNumber(
self._total_bytes_estimation, decimal_places=1)
bytes_progress_string = '{}/{}'.format(scaled_processed_bytes,
scaled_total_bytes_estimation)
else:
bytes_progress_string = scaled_processed_bytes
if self._window_throughput:
throughput_addendum_string = ' | ' + self._window_throughput
else:
throughput_addendum_string = ''
return 'Completed files {} | {}{}\r'.format(file_progress_string,
bytes_progress_string,
throughput_addendum_string)
def _call_api_get_and_print_progress(name):
"""Gets operation from API and prints its progress updating in-place."""
operation = api_get(name)
metadata = encoding.MessageToDict(operation.metadata)
if ('counters' in metadata and 'bytesCopiedToSink' in metadata['counters']
and 'bytesFoundFromSource' in metadata['counters']):
copied_bytes = metadata['counters']['bytesCopiedToSink']
total_bytes = metadata['counters']['bytesFoundFromSource']
progress_percent = int(round(copied_bytes / total_bytes, 2) * 100)
progress_string = '{}% ({} of {})'.format(
progress_percent,
scaled_integer.FormatBinaryNumber(copied_bytes, decimal_places=2),
scaled_integer.FormatBinaryNumber(total_bytes, decimal_places=2))
else:
progress_string = 'UNKNOWN'
if 'errorBreakdowns' in metadata:
error_count = sum(
[error['errorCount'] for error in metadata['errorBreakdowns']])
else:
error_count = 0
log.status.Print(('Status: {} | Progress: {} | Errors: {}\r').format(
metadata['status'], progress_string, error_count))
return operation
def _print_row_list(self, resource_wrappers):
"""Prints ResourceWrapper objects in list with custom row formatting."""
object_count = total_bytes = 0
for i, resource_wrapper in enumerate(resource_wrappers):
if i == 0 and resource_wrapper and str(
resource_wrapper)[0] == '\n':
# First print should not begin with a line break, which can happen
# for headers.
print(str(resource_wrapper)[1:])
else:
print(str(resource_wrapper))
if isinstance(resource_wrapper.resource,
resource_reference.ObjectResource):
# For printing long listing data summary.
object_count += 1
total_bytes += resource_wrapper.resource.size or 0
if (self._display_detail in (DisplayDetail.LONG, DisplayDetail.FULL)
and not self._cloud_url.is_provider()):
# Long listing needs summary line.
print('TOTAL: {} objects, {} bytes ({})'.format(
object_count, int(total_bytes),
scaled_integer.FormatBinaryNumber(total_bytes,
decimal_places=2)))
def _print_progress(operation, retryer_state):
"""Gets operation from API and prints its progress updating in-place."""
metadata = encoding.MessageToDict(operation.metadata)
if 'counters' in metadata:
skipped_bytes = int(metadata['counters'].get('bytesFromSourceSkippedBySync',
0))
skipped_string = scaled_integer.FormatBinaryNumber(
skipped_bytes, decimal_places=1)
copied_bytes = int(metadata['counters'].get('bytesCopiedToSink', 0))
total_bytes = int(metadata['counters'].get('bytesFoundFromSource', 0))
if total_bytes:
progress_percent = int(round(copied_bytes / total_bytes, 2) * 100)
else:
progress_percent = 0
progress_string = '{}% ({} of {})'.format(
progress_percent,
scaled_integer.FormatBinaryNumber(copied_bytes, decimal_places=1),
scaled_integer.FormatBinaryNumber(total_bytes, decimal_places=1))
else:
progress_string = 'Progress: {}'.format(_UNKNOWN_VALUE)
skipped_string = _UNKNOWN_VALUE
if 'errorBreakdowns' in metadata:
error_count = sum(
[int(error['errorCount']) for error in metadata['errorBreakdowns']])
else:
error_count = 0
spin_marks = console_attr.ProgressTrackerSymbolsAscii().spin_marks
if retryer_state.retrial == _LAST_RETRIAL:
spin_mark = ''
else:
spin_mark = spin_marks[retryer_state.retrial % len(spin_marks)]
log.status.write(('{} | {} | Skipped: {} | Errors: {} {}\r').format(
metadata['status'], progress_string, skipped_string, error_count,
spin_mark))
def _format_for_list_long(self):
"""Returns string of select properties from resource."""
if isinstance(self.resource, resource_reference.PrefixResource):
# Align PrefixResource URLs with ObjectResource URLs.
return LONG_LIST_ROW_FORMAT.format(
size='',
creation_time='',
url=self.resource.storage_url.url_string,
metageneration='',
etag='')
creation_time = resource_util.get_formatted_timestamp_in_utc(
self.resource.creation_time)
if self._all_versions:
url_string = self.resource.storage_url.url_string
metageneration_string = ' metageneration={}'.format(
str(self.resource.metageneration))
else:
url_string = self.resource.storage_url.versionless_url_string
metageneration_string = ''
if self._include_etag:
etag_string = ' etag={}'.format(str(self.resource.etag))
else:
etag_string = ''
if self._readable_sizes and self.resource.size is not None:
size = scaled_integer.FormatBinaryNumber(self.resource.size,
decimal_places=2)
else:
# Also handles None values.
size = str(self.resource.size)
# Full example (add 9 spaces of padding to the left):
# 8 2020-07-27T20:58:25Z gs://b/o metageneration=4 etag=CJqt6aup7uoCEAQ=
return LONG_LIST_ROW_FORMAT.format(
size=size,
creation_time=creation_time,
url=url_string,
metageneration=metageneration_string,
etag=etag_string)
def _get_status_string(self):
# TODO(b/180047352) Avoid having other output print on the same line.
return '\rCopied files {} | {}'.format(
self._completed_files,
scaled_integer.FormatBinaryNumber(self._processed_bytes,
decimal_places=1))
def _get_formatted_throughput(bytes_processed, time_delta):
throughput_bytes = max(bytes_processed / time_delta, 0)
return scaled_integer.FormatBinaryNumber(
throughput_bytes, decimal_places=1) + '/s'