def quoter(url, cont=None, ufile=None):
"""Return a Quoted URL.
:param url:
:param cont:
:param ufile:
:return:
"""
url = basic.ustr(obj=url)
if cont is not None:
cont = basic.ustr(obj=cont)
if ufile is not None:
ufile = basic.ustr(obj=ufile)
if ufile is not None and cont is not None:
return urllib.quote(
'%s/%s/%s' % (url, cont, ufile)
)
elif cont is not None:
return urllib.quote(
'%s/%s' % (url, cont)
)
else:
return urllib.quote(
'%s' % url
)
def quoter(url, cont=None, ufile=None):
"""Return a Quoted URL.
:param url:
:param cont:
:param ufile:
:return:
"""
url = basic.ustr(obj=url)
if cont is not None:
cont = basic.ustr(obj=cont)
if ufile is not None:
ufile = basic.ustr(obj=ufile)
if ufile is not None and cont is not None:
return urllib.quote('%s/%s/%s' % (url, cont, ufile))
elif cont is not None:
return urllib.quote('%s/%s' % (url, cont))
else:
return urllib.quote('%s' % url)
def _list_getter(self, conn, count, filepath, fheaders, last_obj=None):
"""Get a list of all objects in a container.
:param conn:
:param count:
:param filepath:
:param fheaders:
:return list:
"""
def _marker_type(base, last):
"""Set and return the marker.
:param base:
:param last:
:return str:
"""
if last is None:
return base
else:
return _last_marker(f_path=base, l_obj=last)
def _last_marker(f_path, l_obj):
"""Set Marker.
:param f_path:
:param l_obj:
:return str:
"""
return '%s&marker=%s' % (f_path, http.quoter(url=l_obj))
def _obj_index(b_path, m_path, l_obj, f_list):
conn.request('GET', m_path, headers=fheaders)
resp, read = http.response_get(conn=conn, retry=retry)
self.resp_exception(resp=resp, rty=retry)
return_list = basic.json_encode(read)
for obj in return_list:
time_offset = ARGS.get('time_offset')
if time_offset is not None:
# Get the last_modified data from the Object.
if cloud.time_delta(lmobj=time_offset) is True:
f_list.append(obj)
else:
f_list.append(obj)
last_obj_in_list = f_list[-1].get('name')
if l_obj is last_obj_in_list:
return f_list
else:
marker = _marker_type(base=b_path, last=last_obj_in_list)
_obj_index(b_path, marker, last_obj_in_list, f_list)
# Quote the file path.
base_path = marked_path = ('%s/?limit=10000&format=json' %
basic.ustr(filepath))
if last_obj is not None:
marked_path = _last_marker(f_path=base_path,
l_obj=http.quoter(url=last_obj))
for retry in basic.retryloop(attempts=ARGS.get('error_retry'),
obj='Object List Creation'):
with meth.operation(retry):
file_list = []
_obj_index(base_path, marked_path, last_obj, file_list)
final_list = basic.unique_list_dicts(dlist=file_list,
key='name')
list_count = len(final_list)
report.reporter(msg='INFO: %d object(s) found' %
len(final_list),
log=True)
return final_list, list_count, last_obj
def _list_getter(self, url, filepath, fheaders, last_obj=None):
"""Get a list of all objects in a container.
:param url:
:param filepath:
:param fheaders:
:return list:
"""
def _marker_type(base, last):
"""Set and return the marker.
:param base:
:param last:
:return str:
"""
if last is None:
return base
else:
return _last_marker(f_path=base, l_obj=last)
def _last_marker(f_path, l_obj):
"""Set Marker.
:param f_path:
:param l_obj:
:return str:
"""
return '%s&marker=%s' % (f_path, http.quoter(url=l_obj))
def _obj_index(b_path, m_path):
f_list = []
l_obj = None
while True:
resp = http.get_request(url=url,
rpath=m_path,
headers=fheaders)
self.resp_exception(resp=resp)
return_list = resp.json()
for obj in return_list:
time_offset = ARGS.get('time_offset')
if time_offset is not None:
# Get the last_modified data from the Object.
if cloud.time_delta(lmobj=time_offset) is True:
f_list.append(obj)
else:
f_list.append(obj)
last_obj_in_list = f_list[-1].get('name')
if ARGS.get('max_jobs', ARGS.get('object_index')) is not None:
max_jobs = ARGS.get('max_jobs', ARGS.get('object_index'))
if max_jobs <= len(f_list):
return f_list[:max_jobs]
elif l_obj is last_obj_in_list:
return f_list
else:
l_obj = last_obj_in_list
m_path = _marker_type(base=b_path,
last=last_obj_in_list)
else:
if l_obj is last_obj_in_list:
return f_list
else:
l_obj = last_obj_in_list
m_path = _marker_type(base=b_path,
last=last_obj_in_list)
# Quote the file path.
base_path = marked_path = ('%s/?limit=10000&format=json' %
basic.ustr(filepath))
if last_obj is not None:
marked_path = _last_marker(f_path=base_path,
l_obj=http.quoter(url=last_obj))
for retry in basic.retryloop(attempts=ARGS.get('error_retry'),
obj='Object List Creation'):
with meth.operation(retry, obj='%s %s' % (fheaders, filepath)):
file_list = _obj_index(base_path, marked_path)
final_list = basic.unique_list_dicts(dlist=file_list,
key='name')
list_count = len(final_list)
report.reporter(msg='INFO: %d object(s) found' %
len(final_list),
log=True)
if 'name' in file_list[-1]:
return final_list, list_count, file_list[-1]['name']
else:
return final_list, list_count, file_list[-1]
def _list_getter(self, conn, count, filepath, fheaders, last_obj=None):
"""Get a list of all objects in a container.
:param conn:
:param count:
:param filepath:
:param fheaders:
:return list:
"""
def _marker_type(base, last):
"""Set and return the marker.
:param base:
:param last:
:return str:
"""
if last is None:
return base
else:
return _last_marker(f_path=base, l_obj=last)
def _last_marker(f_path, l_obj):
"""Set Marker.
:param f_path:
:param l_obj:
:return str:
"""
return '%s&marker=%s' % (f_path, http.quoter(url=l_obj))
def _obj_index(b_path, m_path, l_obj, f_list):
conn.request('GET', m_path, headers=fheaders)
resp, read = http.response_get(conn=conn, retry=retry)
self.resp_exception(resp=resp, rty=retry)
return_list = basic.json_encode(read)
for obj in return_list:
time_offset = ARGS.get('time_offset')
if time_offset is not None:
# Get the last_modified data from the Object.
if cloud.time_delta(lmobj=time_offset) is True:
f_list.append(obj)
else:
f_list.append(obj)
last_obj_in_list = f_list[-1].get('name')
if l_obj is last_obj_in_list:
return f_list
else:
marker = _marker_type(base=b_path, last=last_obj_in_list)
_obj_index(
b_path, marker, last_obj_in_list, f_list
)
# Quote the file path.
base_path = marked_path = (
'%s/?limit=10000&format=json' % basic.ustr(filepath)
)
if last_obj is not None:
marked_path = _last_marker(
f_path=base_path,
l_obj=http.quoter(url=last_obj)
)
for retry in basic.retryloop(attempts=ARGS.get('error_retry'),
obj='Object List Creation'):
with meth.operation(retry):
file_list = []
_obj_index(
base_path, marked_path, last_obj, file_list
)
final_list = basic.unique_list_dicts(
dlist=file_list, key='name'
)
list_count = len(final_list)
report.reporter(
msg='INFO: %d object(s) found' % len(final_list),
log=True
)
return final_list, list_count, last_obj
def _list_getter(self, url, filepath, fheaders, last_obj=None):
"""Get a list of all objects in a container.
:param url:
:param filepath:
:param fheaders:
:return list:
"""
def _marker_type(base, last):
"""Set and return the marker.
:param base:
:param last:
:return str:
"""
if last is None:
return base
else:
return _last_marker(f_path=base, l_obj=last)
def _last_marker(f_path, l_obj):
"""Set Marker.
:param f_path:
:param l_obj:
:return str:
"""
return "%s&marker=%s" % (f_path, http.quoter(url=l_obj))
def _obj_index(b_path, m_path):
f_list = []
l_obj = None
while True:
resp = http.get_request(url=url, rpath=m_path, headers=fheaders)
self.resp_exception(resp=resp)
return_list = resp.json()
for obj in return_list:
time_offset = obj.get("last_modified")
if time_offset is not None and ARGS.get("time_offset"):
# Get the last_modified data from the Object.
if cloud.time_delta(lmobj=time_offset) is False:
f_list.append(obj)
else:
f_list.append(obj)
else:
if not f_list:
return list()
last_obj_in_list = f_list[-1].get("name")
if ARGS.get("max_jobs", ARGS.get("object_index")) is not None:
max_jobs = ARGS.get("max_jobs", ARGS.get("object_index"))
if max_jobs <= len(f_list):
return f_list[:max_jobs]
elif l_obj is last_obj_in_list:
return f_list
else:
l_obj = last_obj_in_list
m_path = _marker_type(base=b_path, last=last_obj_in_list)
else:
if l_obj is last_obj_in_list:
return f_list
else:
l_obj = last_obj_in_list
m_path = _marker_type(base=b_path, last=last_obj_in_list)
# Quote the file path.
base_path = marked_path = "%s/?limit=10000&format=json" % basic.ustr(filepath)
if last_obj is not None:
marked_path = _last_marker(f_path=base_path, l_obj=http.quoter(url=last_obj))
for retry in basic.retryloop(attempts=ARGS.get("error_retry"), obj="Object List Creation"):
with meth.operation(retry, obj="%s %s" % (fheaders, filepath)):
file_list = _obj_index(base_path, marked_path)
final_list = basic.unique_list_dicts(dlist=file_list, key="name")
list_count = len(final_list)
report.reporter(msg="INFO: %d object(s) found" % len(final_list), log=True)
if not final_list:
return final_list, list_count, None
elif "name" in file_list[-1]:
return final_list, list_count, file_list[-1]["name"]
else:
return final_list, list_count, file_list[-1]