def read_or_create_download_tracker_file(source_object_resource,
destination_url,
slice_start_byte=0,
existing_file_size=0,
component_number=None,
create=True):
"""Checks for a download tracker file and creates one if it does not exist.
For normal downloads, if the tracker file exists, the existing_file_size
in bytes is presumed to downloaded from the server. Therefore,
existing_file_size becomes the download start point.
For sliced downloads, the number of bytes previously retrieved from the server
cannot be determined from existing_file_size. Therefore, it is retrieved
from the tracker file.
Args:
source_object_resource (resource_reference.ObjectResource): Needed for
object etag and generation.
destination_url (storage_url.StorageUrl): Destination URL for tracker file.
slice_start_byte (int): Start byte to use if we cannot find a
matching tracker file for a download slice.
existing_file_size (int): Amount of file on disk that already exists.
component_number (int?): The download component number to find the start
point for.
create (bool): Creates tracker file if one could not be found.
Returns:
tracker_file_path (str?): The path to the tracker file, if one was used.
download_start_byte (int): The first byte that still needs to be downloaded.
Raises:
ValueCannotBeDeterminedError: Source object resource does not have
necessary metadata to decide on download start byte.
"""
if not source_object_resource.etag:
raise errors.ValueCannotBeDeterminedError(
'Source object resource is missing etag.')
tracker_file_path = None
if (not source_object_resource.size
or (source_object_resource.size <
properties.VALUES.storage.resumable_threshold.GetInt())):
# There is no tracker file for small downloads, so start from scratch.
return tracker_file_path, slice_start_byte
if component_number is None:
tracker_file_type = TrackerFileType.DOWNLOAD
download_name_for_logger = destination_url.object_name
else:
tracker_file_type = TrackerFileType.DOWNLOAD_COMPONENT
download_name_for_logger = '{} component {}'.format(
destination_url.object_name, component_number)
tracker_file_path = get_tracker_file_path(
destination_url, tracker_file_type, component_number=component_number)
tracker_file = None
# Check to see if we already have a matching tracker file.
try:
tracker_file = files.FileReader(tracker_file_path)
if tracker_file_type is TrackerFileType.DOWNLOAD:
etag_value = tracker_file.readline().rstrip('\n')
if etag_value == source_object_resource.etag:
log.debug(
'Found tracker file starting at byte {} for {}.'.format(
existing_file_size, download_name_for_logger))
return tracker_file_path, existing_file_size
elif tracker_file_type is TrackerFileType.DOWNLOAD_COMPONENT:
component_data = json.loads(tracker_file.read())
if (component_data['etag'] == source_object_resource.etag
and component_data['generation']
== source_object_resource.generation):
start_byte = int(component_data['download_start_byte'])
log.debug(
'Found tracker file starting at byte {} for {}.'.format(
start_byte, download_name_for_logger))
return tracker_file_path, start_byte
except files.MissingFileError:
# Cannot read from file.
pass
finally:
if tracker_file:
tracker_file.close()
if create:
log.debug('No matching tracker file for {}.'.format(
download_name_for_logger))
if tracker_file_type is TrackerFileType.DOWNLOAD:
_write_tracker_file(tracker_file_path,
source_object_resource.etag + '\n')
elif tracker_file_type is TrackerFileType.DOWNLOAD_COMPONENT:
write_download_component_tracker_file(tracker_file_path,
source_object_resource,
slice_start_byte)
# No matching tracker file, so starting point is slice_start_byte.
return tracker_file_path, slice_start_byte
def is_container(self):
raise errors.ValueCannotBeDeterminedError(
'Unknown whether or not UnknownResource is a container.')
def read_or_create_download_tracker_file(source_object_resource,
destination_url,
slice_start_byte=None,
component_number=None,
total_components=None):
"""Checks for a download tracker file and creates one if it does not exist.
Args:
source_object_resource (resource_reference.ObjectResource): Needed for
object etag and generation.
destination_url (storage_url.StorageUrl): Destination URL for tracker file.
slice_start_byte (int|None): Start byte to use if we cannot find a
matching tracker file for a download slice.
component_number (int|None): The download component number to find the start
point for. Indicates part of a multi-component download.
total_components (int|None): The number of components in a sliced download.
Indicates this is the parent tracker for a multi-component operation.
Returns:
tracker_file_path (str): The path to the tracker file (found or created).
found_tracker_file (bool): False if tracker file had to be created.
Raises:
ValueCannotBeDeterminedError: Source object resource does not have
necessary metadata to decide on download start byte.
"""
if not source_object_resource.etag:
raise errors.ValueCannotBeDeterminedError(
'Source object resource is missing etag.')
if total_components and (slice_start_byte is not None
or component_number is not None):
raise ValueError(
'total_components indicates this is the parent tracker file for a'
' multi-component operation. slice_start_byte and component_number'
' cannot be present since this is not for an individual component.'
)
if component_number is not None:
download_name_for_logger = '{} component {}'.format(
destination_url.object_name, component_number)
tracker_file_type = TrackerFileType.DOWNLOAD_COMPONENT
else:
download_name_for_logger = destination_url.object_name
if total_components is not None:
tracker_file_type = TrackerFileType.SLICED_DOWNLOAD
else:
tracker_file_type = TrackerFileType.DOWNLOAD
tracker_file_path = get_tracker_file_path(
destination_url, tracker_file_type, component_number=component_number)
log.debug('Searching for tracker file at {}.'.format(tracker_file_path))
tracker_file = None
does_tracker_file_match = False
# Check to see if we already have a matching tracker file.
try:
tracker_file = files.FileReader(tracker_file_path)
if tracker_file_type is TrackerFileType.DOWNLOAD:
etag_value = tracker_file.readline().rstrip('\n')
if etag_value == source_object_resource.etag:
does_tracker_file_match = True
else:
component_data = json.loads(tracker_file.read())
if (component_data['etag'] == source_object_resource.etag
and component_data['generation']
== source_object_resource.generation):
if (tracker_file_type is TrackerFileType.SLICED_DOWNLOAD
and component_data['total_components']
== total_components):
does_tracker_file_match = True
elif tracker_file_type is TrackerFileType.DOWNLOAD_COMPONENT and component_data[
'slice_start_byte'] == slice_start_byte:
does_tracker_file_match = True
if does_tracker_file_match:
log.debug(
'Found tracker file for {}.'.format(download_name_for_logger))
return tracker_file_path, True
except files.MissingFileError:
# Cannot read from file.
pass
finally:
if tracker_file:
tracker_file.close()
if tracker_file:
# The tracker file exists, but it's not valid.
delete_download_tracker_files(destination_url)
log.debug(
'No matching tracker file for {}.'.format(download_name_for_logger))
if tracker_file_type is TrackerFileType.DOWNLOAD:
_write_tracker_file(tracker_file_path,
source_object_resource.etag + '\n')
elif tracker_file_type is TrackerFileType.DOWNLOAD_COMPONENT:
write_tracker_file_with_component_data(
tracker_file_path,
source_object_resource,
slice_start_byte=slice_start_byte)
elif tracker_file_type is TrackerFileType.SLICED_DOWNLOAD:
write_tracker_file_with_component_data(
tracker_file_path,
source_object_resource,
total_components=total_components)
return tracker_file_path, False
def read_or_create_download_tracker_file(source_object_resource,
destination_url,
existing_file_size=None,
slice_start_byte=None,
component_number=None,
total_components=None,
create=True):
"""Checks for a download tracker file and creates one if it does not exist.
For normal downloads, if the tracker file exists, the existing_file_size
in bytes is presumed to downloaded from the server. Therefore,
existing_file_size becomes the download start point.
For sliced downloads, the number of bytes previously retrieved from the server
cannot be determined from existing_file_size. Therefore, it is retrieved
from the tracker file.
Args:
source_object_resource (resource_reference.ObjectResource): Needed for
object etag and generation.
destination_url (storage_url.StorageUrl): Destination URL for tracker file.
existing_file_size (int): Amount of file on disk that already exists.
slice_start_byte (int|None): Start byte to use if we cannot find a
matching tracker file for a download slice.
component_number (int|None): The download component number to find the start
point for. Indicates part of a multi-component download.
total_components (int|None): The number of components in a sliced download.
Indicates this is the master tracker for a multi-component operation.
create (bool): Creates tracker file if one could not be found.
Returns:
tracker_file_path (str|None): The path to the tracker file, if one was used.
download_start_byte (int|None): The first byte that still needs to be
downloaded, if not a sliced download.
Raises:
ValueCannotBeDeterminedError: Source object resource does not have
necessary metadata to decide on download start byte.
"""
if not source_object_resource.etag:
raise errors.ValueCannotBeDeterminedError(
'Source object resource is missing etag.')
if total_components and (slice_start_byte is not None
or component_number is not None):
raise ValueError(
'total_components indicates this is the master tracker file for a'
' multi-component operation. slice_start_byte and component_number'
' cannot be present since this is not for an individual component.'
)
if component_number:
download_name_for_logger = '{} component {}'.format(
destination_url.object_name, component_number)
tracker_file_type = TrackerFileType.DOWNLOAD_COMPONENT
else:
download_name_for_logger = destination_url.object_name
if total_components:
tracker_file_type = TrackerFileType.SLICED_DOWNLOAD
else:
tracker_file_type = TrackerFileType.DOWNLOAD
tracker_file_path = get_tracker_file_path(
destination_url, tracker_file_type, component_number=component_number)
tracker_file = None
# Check to see if we already have a matching tracker file.
try:
tracker_file = files.FileReader(tracker_file_path)
if tracker_file_type is TrackerFileType.DOWNLOAD:
etag_value = tracker_file.readline().rstrip('\n')
if etag_value == source_object_resource.etag:
log.debug(
'Found tracker file starting at byte {} for {}.'.format(
existing_file_size, download_name_for_logger))
return tracker_file_path, existing_file_size
else:
component_data = json.loads(tracker_file.read())
if (component_data['etag'] == source_object_resource.etag
and component_data['generation']
== source_object_resource.generation):
if (tracker_file_type is TrackerFileType.SLICED_DOWNLOAD
and component_data['total_components']
== total_components):
log.debug(
'Found tracker file for sliced download {}.'.format(
download_name_for_logger))
return tracker_file_path, None
elif tracker_file_type is TrackerFileType.DOWNLOAD_COMPONENT:
# Normal resumable download.
start_byte = int(component_data['download_start_byte'])
log.debug('Found tracker file starting at byte {} for {}.'.
format(start_byte, download_name_for_logger))
return tracker_file_path, start_byte
except files.MissingFileError:
# Cannot read from file.
pass
finally:
if tracker_file:
tracker_file.close()
log.debug(
'No matching tracker file for {}.'.format(download_name_for_logger))
start_byte = 0
if create:
if tracker_file_type is TrackerFileType.DOWNLOAD:
_write_tracker_file(tracker_file_path,
source_object_resource.etag + '\n')
elif tracker_file_type is TrackerFileType.DOWNLOAD_COMPONENT:
write_tracker_file_with_component_data(
tracker_file_path,
source_object_resource,
download_start_byte=slice_start_byte)
start_byte = slice_start_byte
elif tracker_file_type is TrackerFileType.SLICED_DOWNLOAD:
# Delete component tracker files to reset full sliced download.
delete_download_tracker_files(destination_url)
write_tracker_file_with_component_data(
tracker_file_path,
source_object_resource,
total_components=total_components)
start_byte = None
return tracker_file_path, start_byte