def __init__(self, url, file_path,
part_size=PartSize.DOWNLOAD_MINIMUM_PART_SIZE, retry_count=5,
timeout=60, api=None):
"""
File multipart downloader.
:param url: URL of the file.
:param file_path: Local file path.
:param retry_count: Number of times to retry on error.
:param timeout: Connection timeout in seconds.
:param part_size: Size of the parts in bytes.
:param api: Api instance.
"""
threading.Thread.__init__(self)
self.daemon = True
if api is None:
raise SbgError('Api instance missing.')
if part_size and part_size < PartSize.DOWNLOAD_MINIMUM_PART_SIZE:
self._status = TransferState.FAILED
raise SbgError(
'Part size is too small! Minimum get_parts size is {}'.format(
PartSize.DOWNLOAD_MINIMUM_PART_SIZE)
)
# initializes the session
self.url = url
self._file_path = file_path
# append unique suffix to the file
self._temp_file = self._file_path + '.' + hashlib.sha1(
self._file_path.encode('utf-8')).hexdigest()[:10]
self._retry_count = retry_count
self._timeout = timeout
if part_size:
self._part_size = part_size
else:
self._part_size = PartSize.DOWNLOAD_MINIMUM_PART_SIZE
self._api = api
self._bytes_done = 0
self._running = threading.Event()
self._callback = None
self._errorback = None
self._progress_callback = None
self._time_started = 0
self._session = generate_session(self._api.session.proxies)
try:
self._file_size = self._get_file_size()
except SbgError as error:
if self._errorback:
self._errorback(error)
else:
raise error
self._status = TransferState.PREPARING
self._stop_signal = False
def __init__(self,
file_path,
project=None,
parent=None,
file_name=None,
overwrite=False,
part_size=None,
retry_count=5,
timeout=60,
api=None):
"""
Multipart File uploader.
:param file_path: File path on the disc.
:param project: Target project identifier.
:param file_name: Optional file name.
:param overwrite: If true will overwrite file on the server.
:param part_size: Size of part in bytes.
:param retry_count: Retry count.
:param timeout: Timeout for s3/google session.
:param api: Api instance.
"""
threading.Thread.__init__(self)
self.daemon = True
self._validate_project_parent(parent, project)
if not file_path:
raise SbgError('File path is not valid.')
if not os.path.isfile(file_path):
raise SbgError(
'File path {} is not a path to a valid file.'.format(
file_path))
if not api:
raise SbgError('Api instance not provided!')
if not file_name:
self._file_name = os.path.basename(file_path)
else:
self._file_name = file_name
self._part_size = part_size # or PartSize.UPLOAD_MINIMUM_PART_SIZE
self._project = project
self._parent = parent
self._file_path = file_path
self._file_size = os.path.getsize(self._file_path)
if self._file_size == 0:
raise SbgError('File size must not be 0.')
self._verify_file_size()
self._overwrite = overwrite
self._retry = retry_count
self._timeout = timeout
self._api = api
self._bytes_done = 0
self._time_started = 0
self._running = threading.Event()
self._status = TransferState.PREPARING
self._callback = None
self._errorback = None
self._progress_callback = None
self._stop_signal = False
self._result = None
self.session = generate_session(self._api.pool_connections,
self._api.pool_maxsize,
self._api.pool_block,
self._api.session.proxies)
def __init__(self,
url,
file_path,
part_size=PartSize.DOWNLOAD_MINIMUM_PART_SIZE,
retry_count=5,
timeout=60,
api=None):
"""
File multipart downloader.
:param url: URL of the file.
:param file_path: Local file path.
:param retry_count: Number of times to retry on error.
:param timeout: Connection timeout in seconds.
:param part_size: Size of the parts in bytes.
:param api: Api instance.
"""
threading.Thread.__init__(self)
self.daemon = True
if api is None:
raise SbgError('Api instance missing.')
if part_size and part_size < PartSize.DOWNLOAD_MINIMUM_PART_SIZE:
self._status = TransferState.FAILED
raise SbgError(
'Part size is too small! Minimum get_parts size is {}'.format(
PartSize.DOWNLOAD_MINIMUM_PART_SIZE))
# initializes the session
self.url = url
self._file_path = file_path
# append unique suffix to the file
self._temp_file = self._file_path + '.' + hashlib.sha1(
self._file_path.encode('utf-8')).hexdigest()[:10]
self._retry_count = retry_count
self._timeout = timeout
if part_size:
self._part_size = part_size
else:
self._part_size = PartSize.DOWNLOAD_MINIMUM_PART_SIZE
self._api = api
self._bytes_done = 0
self._running = threading.Event()
self._callback = None
self._errorback = None
self._progress_callback = None
self._time_started = 0
self._session = generate_session(self._api.pool_connections,
self._api.pool_maxsize,
self._api.pool_block,
self._api.session.proxies)
try:
self._file_size = self._get_file_size()
except SbgError as error:
if self._errorback:
self._errorback(error)
else:
raise error
self._status = TransferState.PREPARING
self._stop_signal = False
def __init__(self,
file_path,
project,
file_name=None,
overwrite=False,
part_size=PartSize.UPLOAD_MINIMUM_PART_SIZE,
retry_count=5,
timeout=60,
api=None):
"""
Multipart File uploader.
:param file_path: File path on the disc.
:param project: Target project identifier.
:param file_name: Optional file name.
:param overwrite: If true will overwrite file on the server.
:param part_size: Size of part in bytes.
:param retry_count: Retry count.
:param timeout: Timeout for s3/google session.
:param api: Api instance.
"""
threading.Thread.__init__(self)
self.daemon = True
if not project:
raise SbgError('Project identifier is not valid.')
if not file_path:
raise SbgError('File path is not valid.')
if not os.path.isfile(file_path):
raise SbgError(
'File path {} is not a path to a valid file.'.format(
file_path))
if not api:
raise SbgError('Api instance not provided!')
if not file_name:
self._file_name = os.path.basename(file_path)
else:
self._file_name = file_name
if part_size and part_size < PartSize.UPLOAD_MINIMUM_PART_SIZE:
raise SbgError(
'Chunk size is too small! Minimum get_parts size is {}'.format(
PartSize.UPLOAD_MINIMUM_PART_SIZE))
self._part_size = part_size
self._project = project
self._file_path = file_path
self._file_size = os.path.getsize(self._file_path)
self._verify_file_size()
self._verify_part_size()
self._verify_part_number()
self._overwrite = overwrite
self._retry = retry_count
self._timeout = timeout
self._api = api
self._bytes_done = 0
self._time_started = 0
self._running = threading.Event()
self._status = TransferState.PREPARING
self._callback = None
self._errorback = None
self._progress_callback = None
self._stop_signal = False
self._result = None
adapter = HTTPAdapter(max_retries=Retry(total=self._retry,
status_forcelist=[500, 503],
backoff_factor=0.1))
self.session = generate_session('https://', adapter,
self._api.session.proxies)
def __init__(self,
url,
file_path,
part_size=None,
retry_count=None,
timeout=None,
api=None):
"""
File multipart downloader.
:param url: URL of the file.
:param file_path: Local file path.
:param retry_count: Number of times to retry on error.
:param timeout: Connection timeout in seconds.
:param part_size: Size of the parts in bytes.
:param api: Api instance.
"""
threading.Thread.__init__(self)
self.daemon = True
if api is None:
raise SbgError('Api instance missing.')
if part_size and part_size < PartSize.DOWNLOAD_MINIMUM_PART_SIZE:
self._status = TransferState.FAILED
raise SbgError(f'Part size is too small! Minimum get_parts size '
f'is {PartSize.DOWNLOAD_MINIMUM_PART_SIZE}')
self.url = url
self._file_path = file_path
# append unique suffix to the file
suffix = hashlib.sha1(self._file_path.encode('utf-8')).hexdigest()[:10]
self._temp_file = f'{self._file_path}.{suffix}'
self._retry_count = (retry_count
or RequestParameters.DEFAULT_RETRY_COUNT)
self._timeout = timeout or RequestParameters.DEFAULT_TIMEOUT
self._part_size = part_size or PartSize.DOWNLOAD_MINIMUM_PART_SIZE
self._api = api
self._bytes_done = 0
self._running = threading.Event()
self._callback = None
self._errorback = None
self._progress_callback = None
self._time_started = 0
self._session = generate_session(
pool_connections=self._api.pool_connections,
pool_maxsize=self._api.pool_maxsize,
pool_block=self._api.pool_block,
proxies=self._api.session.proxies,
retry_count=self._retry_count,
)
try:
self._file_size = self._get_file_size()
except SbgError as error:
if self._errorback:
self._errorback(error)
else:
raise error
self._status = TransferState.PREPARING
self._stop_signal = False
