def _downloader(self, url, rpath, fheaders, lfile, source, skip=False):
"""Download a specified object in the container.
:param url:
:param rpath:
:param fheaders:
:param lfile:
:param skip:
"""
resp = None
if source is None:
local_f = lfile
else:
local_f = basic.jpath(root=source, inode=lfile)
if self._checker(url, rpath, local_f, fheaders, skip) is True:
report.reporter(
msg='Downloading remote %s to local file %s' % (rpath, lfile),
prt=False,
lvl='debug',
)
# Perform Object GET
resp = http.get_request(url=url,
rpath=rpath,
headers=fheaders,
stream=True)
self.resp_exception(resp=resp)
local_f = basic.collision_rename(file_name=local_f)
# Open our source file and write it
with open(local_f, 'wb') as f_name:
for chunk in resp.iter_content(chunk_size=2048):
if chunk:
f_name.write(chunk)
f_name.flush()
resp.close()
if ARGS.get('restore_perms') is not None:
# Make a connection
if resp is None:
resp = self._header_getter(url=url,
rpath=rpath,
fheaders=fheaders)
all_headers = resp.headers
if all([
'x-object-meta-group' in all_headers, 'x-object-meta-owner'
in all_headers, 'x-object-meta-perms' in all_headers
]):
basic.restor_perms(local_file=local_f, headers=all_headers)
else:
report.reporter(
msg=('No Permissions were restored, because none were'
' saved on the object "%s"' % rpath),
lvl='warn',
log=True)
def _downloader(self, url, rpath, fheaders, lfile, source,
skip=False):
"""Download a specified object in the container.
:param url:
:param rpath:
:param fheaders:
:param lfile:
:param skip:
"""
resp = None
if source is None:
local_f = lfile
else:
local_f = basic.jpath(root=source, inode=lfile)
if self._checker(url, rpath, local_f, fheaders, skip) is True:
report.reporter(
msg='Downloading remote %s to local file %s' % (rpath, lfile),
prt=False,
lvl='debug',
)
# Perform Object GET
resp = http.get_request(
url=url, rpath=rpath, headers=fheaders, stream=True
)
self.resp_exception(resp=resp)
local_f = basic.collision_rename(file_name=local_f)
# Open our source file and write it
with open(local_f, 'wb') as f_name:
for chunk in resp.iter_content(chunk_size=2048):
if chunk:
f_name.write(chunk)
f_name.flush()
resp.close()
if ARGS.get('restore_perms') is not None:
# Make a connection
if resp is None:
resp = self._header_getter(
url=url, rpath=rpath, fheaders=fheaders
)
all_headers = resp.headers
if all(['x-object-meta-group' in all_headers,
'x-object-meta-owner' in all_headers,
'x-object-meta-perms' in all_headers]):
basic.restor_perms(local_file=local_f, headers=all_headers)
else:
report.reporter(
msg=('No Permissions were restored, because none were'
' saved on the object "%s"' % rpath),
lvl='warn',
log=True
)
def _obj_index(b_path, m_path):
f_list = []
l_obj = None
while True:
resp = http.get_request(url=url, rpath=m_path, headers=fheaders)
self.resp_exception(resp=resp)
return_list = resp.json()
for obj in return_list:
time_offset = ARGS.get("time_offset")
if time_offset is not None:
# Get the last_modified data from the Object.
if cloud.time_delta(lmobj=time_offset) is True:
f_list.append(obj)
else:
f_list.append(obj)
last_obj_in_list = f_list[-1].get("name")
if ARGS.get("max_jobs", ARGS.get("object_index")) is not None:
max_jobs = ARGS.get("max_jobs", ARGS.get("object_index"))
if max_jobs <= len(f_list):
return f_list[:max_jobs]
elif l_obj is last_obj_in_list:
return f_list
else:
l_obj = last_obj_in_list
m_path = _marker_type(base=b_path, last=last_obj_in_list)
else:
if l_obj is last_obj_in_list:
return f_list
else:
l_obj = last_obj_in_list
m_path = _marker_type(base=b_path, last=last_obj_in_list)
def auth_request(url, headers=None, body=None):
"""Perform auth request for token."""
headers = headers or {}
body = body or json.dumps({})
if get_authversion() == 'v1.0':
# shouldnt need to allow 'stream'
return http.get_request(url, headers, None)
else:
# shouldnt need to allow 'rpath'
return http.post_request(url, headers, body=body)
def auth_request(url, headers=None, body=None):
"""Perform auth request for token."""
headers = headers or {}
body = body or json.dumps({})
if get_authversion() == 'v1.0':
# shouldnt need to allow 'stream'
return http.get_request(url, headers, None)
else:
# shouldnt need to allow 'rpath'
return http.post_request(url, headers, body=body)
def _obj_index(b_path, m_path):
f_list = []
l_obj = None
while True:
resp = http.get_request(url=url,
rpath=m_path,
headers=fheaders)
self.resp_exception(resp=resp)
return_list = resp.json()
for obj in return_list:
time_offset = ARGS.get('time_offset')
if time_offset is not None:
# Get the last_modified data from the Object.
if cloud.time_delta(lmobj=time_offset) is True:
f_list.append(obj)
else:
f_list.append(obj)
last_obj_in_list = f_list[-1].get('name')
if ARGS.get('max_jobs', ARGS.get('object_index')) is not None:
max_jobs = ARGS.get('max_jobs', ARGS.get('object_index'))
if max_jobs <= len(f_list):
return f_list[:max_jobs]
elif l_obj is last_obj_in_list:
return f_list
else:
l_obj = last_obj_in_list
m_path = _marker_type(base=b_path,
last=last_obj_in_list)
else:
if l_obj is last_obj_in_list:
return f_list
else:
l_obj = last_obj_in_list
m_path = _marker_type(base=b_path,
last=last_obj_in_list)
