def _downloader(self, url, rpath, fheaders, lfile, source, skip=False): """Download a specified object in the container. :param url: :param rpath: :param fheaders: :param lfile: :param skip: """ resp = None if source is None: local_f = lfile else: local_f = basic.jpath(root=source, inode=lfile) if self._checker(url, rpath, local_f, fheaders, skip) is True: report.reporter( msg='Downloading remote %s to local file %s' % (rpath, lfile), prt=False, lvl='debug', ) # Perform Object GET resp = http.get_request(url=url, rpath=rpath, headers=fheaders, stream=True) self.resp_exception(resp=resp) local_f = basic.collision_rename(file_name=local_f) # Open our source file and write it with open(local_f, 'wb') as f_name: for chunk in resp.iter_content(chunk_size=2048): if chunk: f_name.write(chunk) f_name.flush() resp.close() if ARGS.get('restore_perms') is not None: # Make a connection if resp is None: resp = self._header_getter(url=url, rpath=rpath, fheaders=fheaders) all_headers = resp.headers if all([ 'x-object-meta-group' in all_headers, 'x-object-meta-owner' in all_headers, 'x-object-meta-perms' in all_headers ]): basic.restor_perms(local_file=local_f, headers=all_headers) else: report.reporter( msg=('No Permissions were restored, because none were' ' saved on the object "%s"' % rpath), lvl='warn', log=True)
def _downloader(self, url, rpath, fheaders, lfile, source, skip=False): """Download a specified object in the container. :param url: :param rpath: :param fheaders: :param lfile: :param skip: """ resp = None if source is None: local_f = lfile else: local_f = basic.jpath(root=source, inode=lfile) if self._checker(url, rpath, local_f, fheaders, skip) is True: report.reporter( msg='Downloading remote %s to local file %s' % (rpath, lfile), prt=False, lvl='debug', ) # Perform Object GET resp = http.get_request( url=url, rpath=rpath, headers=fheaders, stream=True ) self.resp_exception(resp=resp) local_f = basic.collision_rename(file_name=local_f) # Open our source file and write it with open(local_f, 'wb') as f_name: for chunk in resp.iter_content(chunk_size=2048): if chunk: f_name.write(chunk) f_name.flush() resp.close() if ARGS.get('restore_perms') is not None: # Make a connection if resp is None: resp = self._header_getter( url=url, rpath=rpath, fheaders=fheaders ) all_headers = resp.headers if all(['x-object-meta-group' in all_headers, 'x-object-meta-owner' in all_headers, 'x-object-meta-perms' in all_headers]): basic.restor_perms(local_file=local_f, headers=all_headers) else: report.reporter( msg=('No Permissions were restored, because none were' ' saved on the object "%s"' % rpath), lvl='warn', log=True )
def _obj_index(b_path, m_path): f_list = [] l_obj = None while True: resp = http.get_request(url=url, rpath=m_path, headers=fheaders) self.resp_exception(resp=resp) return_list = resp.json() for obj in return_list: time_offset = ARGS.get("time_offset") if time_offset is not None: # Get the last_modified data from the Object. if cloud.time_delta(lmobj=time_offset) is True: f_list.append(obj) else: f_list.append(obj) last_obj_in_list = f_list[-1].get("name") if ARGS.get("max_jobs", ARGS.get("object_index")) is not None: max_jobs = ARGS.get("max_jobs", ARGS.get("object_index")) if max_jobs <= len(f_list): return f_list[:max_jobs] elif l_obj is last_obj_in_list: return f_list else: l_obj = last_obj_in_list m_path = _marker_type(base=b_path, last=last_obj_in_list) else: if l_obj is last_obj_in_list: return f_list else: l_obj = last_obj_in_list m_path = _marker_type(base=b_path, last=last_obj_in_list)
def auth_request(url, headers=None, body=None): """Perform auth request for token.""" headers = headers or {} body = body or json.dumps({}) if get_authversion() == 'v1.0': # shouldnt need to allow 'stream' return http.get_request(url, headers, None) else: # shouldnt need to allow 'rpath' return http.post_request(url, headers, body=body)
def _obj_index(b_path, m_path): f_list = [] l_obj = None while True: resp = http.get_request(url=url, rpath=m_path, headers=fheaders) self.resp_exception(resp=resp) return_list = resp.json() for obj in return_list: time_offset = ARGS.get('time_offset') if time_offset is not None: # Get the last_modified data from the Object. if cloud.time_delta(lmobj=time_offset) is True: f_list.append(obj) else: f_list.append(obj) last_obj_in_list = f_list[-1].get('name') if ARGS.get('max_jobs', ARGS.get('object_index')) is not None: max_jobs = ARGS.get('max_jobs', ARGS.get('object_index')) if max_jobs <= len(f_list): return f_list[:max_jobs] elif l_obj is last_obj_in_list: return f_list else: l_obj = last_obj_in_list m_path = _marker_type(base=b_path, last=last_obj_in_list) else: if l_obj is last_obj_in_list: return f_list else: l_obj = last_obj_in_list m_path = _marker_type(base=b_path, last=last_obj_in_list)