def _downloader(self, url, rpath, fheaders, lfile, source, skip=False):
"""Download a specified object in the container.
:param url:
:param rpath:
:param fheaders:
:param lfile:
:param skip:
"""
resp = None
if source is None:
local_f = lfile
else:
local_f = basic.jpath(root=source, inode=lfile)
if self._checker(url, rpath, local_f, fheaders, skip) is True:
report.reporter(
msg='Downloading remote %s to local file %s' % (rpath, lfile),
prt=False,
lvl='debug',
)
# Perform Object GET
resp = http.get_request(url=url,
rpath=rpath,
headers=fheaders,
stream=True)
self.resp_exception(resp=resp)
local_f = basic.collision_rename(file_name=local_f)
# Open our source file and write it
with open(local_f, 'wb') as f_name:
for chunk in resp.iter_content(chunk_size=2048):
if chunk:
f_name.write(chunk)
f_name.flush()
resp.close()
if ARGS.get('restore_perms') is not None:
# Make a connection
if resp is None:
resp = self._header_getter(url=url,
rpath=rpath,
fheaders=fheaders)
all_headers = resp.headers
if all([
'x-object-meta-group' in all_headers, 'x-object-meta-owner'
in all_headers, 'x-object-meta-perms' in all_headers
]):
basic.restor_perms(local_file=local_f, headers=all_headers)
else:
report.reporter(
msg=('No Permissions were restored, because none were'
' saved on the object "%s"' % rpath),
lvl='warn',
log=True)
def remote_delete(self, payload):
"""If Remote Delete was True run.
NOTE: Remote delete will delete ALL Objects in a remote container
which differ from the objects in the SOURCED LOCAL FILESYSTEM.
IE: If this option is used, on one directory and then another directory
and the files were different any difference would be deleted and based
on the index information found in LOCAL FILE SYSTEM on the LAST
command run.
:param payload: ``dict``
"""
report.reporter(msg='Getting file list for REMOTE DELETE')
# From the remote system see if we have differences in the local system
f_indexed = self._index_local_files()
objects = self.go.object_lister(url=payload['url'],
container=payload['c_name'])
source = payload['source']
obj_names = [
basic.jpath(root=source, inode=obj.get('name'))
for obj in objects[0]
]
obj_names = set(obj_names)
# Sort the difference between remote files and local files.
objects = [obj for obj in obj_names if obj not in f_indexed]
if objects:
# Set Basic Data for file delete.
num_files = len(objects)
report.reporter(
msg=('MESSAGE: "%d" Files have been found to be removed'
' from the REMOTE CONTAINER.' % num_files))
concurrency = multi.set_concurrency(args=ARGS,
file_count=num_files)
# Delete the difference in Files.
report.reporter(msg='Performing REMOTE DELETE')
del_objects = [
basic.get_sfile(ufile=obj, source=payload['source'])
for obj in objects if obj is not None
]
kwargs = {
'url': payload['url'],
'container': payload['c_name'],
'cf_job': getattr(self.go, 'object_deleter')
}
multi.job_processer(num_jobs=num_files,
objects=del_objects,
job_action=multi.doerator,
concur=concurrency,
kwargs=kwargs)
else:
report.reporter(
msg='No Difference between REMOTE and LOCAL Directories.')
def indexer(location):
"""Return a list of indexed files.
:param location:
:return:
"""
_location = basic.real_full_path(location.encode("utf8"))
if os.path.isdir(_location):
r_walk = os.walk(_location)
objects = list()
for root_inx, inx in [(root, fls) for root, sfs, fls in r_walk]:
for inode in inx:
object_path = basic.jpath(root=root_inx, inode=inode)
try:
objects.append(object_path.decode("utf-8"))
except UnicodeDecodeError:
objects.append(object_path)
else:
return objects
elif os.path.isfile(_location):
return [_location]
else:
raise turbo.NoFileProvided("No Path was Found for %s" % _location)
def _downloader(self, url, rpath, fheaders, lfile, source,
skip=False):
"""Download a specified object in the container.
:param url:
:param rpath:
:param fheaders:
:param lfile:
:param skip:
"""
resp = None
if source is None:
local_f = lfile
else:
local_f = basic.jpath(root=source, inode=lfile)
if self._checker(url, rpath, local_f, fheaders, skip) is True:
report.reporter(
msg='Downloading remote %s to local file %s' % (rpath, lfile),
prt=False,
lvl='debug',
)
# Perform Object GET
resp = http.get_request(
url=url, rpath=rpath, headers=fheaders, stream=True
)
self.resp_exception(resp=resp)
local_f = basic.collision_rename(file_name=local_f)
# Open our source file and write it
with open(local_f, 'wb') as f_name:
for chunk in resp.iter_content(chunk_size=2048):
if chunk:
f_name.write(chunk)
f_name.flush()
resp.close()
if ARGS.get('restore_perms') is not None:
# Make a connection
if resp is None:
resp = self._header_getter(
url=url, rpath=rpath, fheaders=fheaders
)
all_headers = resp.headers
if all(['x-object-meta-group' in all_headers,
'x-object-meta-owner' in all_headers,
'x-object-meta-perms' in all_headers]):
basic.restor_perms(local_file=local_f, headers=all_headers)
else:
report.reporter(
msg=('No Permissions were restored, because none were'
' saved on the object "%s"' % rpath),
lvl='warn',
log=True
)
def remote_delete(self, payload, f_indexed):
"""If Remote Delete was True run.
NOTE: Remote delete will delete ALL Objects in a remote container
which differ from the objects in the SOURCED LOCAL FILESYSTEM.
IE: If this option is used, on one directory and then another directory
and the files were different any difference would be deleted and based
on the index information found in LOCAL FILE SYSTEM on the LAST
command run.
:return:
"""
report.reporter(msg='Getting file list for REMOTE DELETE')
objects = self.go.object_lister(
url=payload['url'], container=payload['c_name']
)
source = payload['source']
obj_names = [basic.jpath(root=source, inode=obj.get('name'))
for obj in objects[0]]
# From the remote system see if we have differences in the local system
objects = multi.return_diff().difference(target=f_indexed,
source=obj_names)
if objects:
# Set Basic Data for file delete.
num_files = len(objects)
LOG.info('MESSAGE\t: "%s" Files have been found to be removed from'
' the REMOTE CONTAINER.', num_files)
concurrency = multi.set_concurrency(
args=ARGS, file_count=num_files
)
# Delete the difference in Files.
report.reporter(msg='Performing Remote Delete')
objects = [basic.get_sfile(
ufile=obj, source=payload['source']
) for obj in objects]
kwargs = {'url': payload['url'],
'container': payload['c_name'],
'cf_job': getattr(self.go, 'object_deleter')}
multi.job_processer(
num_jobs=num_files,
objects=objects,
job_action=multi.doerator,
concur=concurrency,
kwargs=kwargs
)
else:
report.reporter(
msg='No Difference between REMOTE and LOCAL Directories.'
)
def indexer(location):
"""Return a list of indexed files.
:param location:
:return:
"""
_location = basic.real_full_path(location.encode("utf8"))
if os.path.isdir(_location):
r_walk = os.walk(_location)
indexes = [(root, fls) for root, sfs, fls in r_walk]
return [basic.jpath(root=inx[0], inode=inode) for inx in indexes for inode in inx[1]]
elif os.path.isfile(_location):
return [_location]
else:
raise turbo.NoFileProvided("No Path was Found for %s" % _location)
def indexer(location):
"""Return a list of indexed files.
:param location:
:return:
"""
_location = basic.real_full_path(location.encode('utf8'))
if os.path.isdir(_location):
r_walk = os.walk(_location)
indexes = [(root, fls) for root, sfs, fls in r_walk]
return [
basic.jpath(root=inx[0], inode=inode) for inx in indexes
for inode in inx[1]
]
elif os.path.isfile(_location):
return [_location]
else:
raise turbo.NoFileProvided('No Path was Found for %s' % _location)
def indexer(location):
"""Return a list of indexed files.
:param location:
:return:
"""
_location = basic.real_full_path(location.encode('utf8'))
if os.path.isdir(_location):
r_walk = os.walk(_location)
objects = list()
for root_inx, inx in [(root, fls) for root, sfs, fls in r_walk]:
for inode in inx:
object_path = basic.jpath(root=root_inx, inode=inode)
objects.append(unicode(object_path.decode('utf-8')))
else:
return objects
elif os.path.isfile(_location):
return [_location]
else:
raise turbo.NoFileProvided('No Path was Found for %s' % _location)
def compress_files(file_list):
"""If the archive function is used, create a compressed archive.
:param file_list:
This function allows for multiple sources to be added to the
compressed archive.
"""
tmp_file = None
try:
# Set date and time
date_format = '%a%b%d.%H.%M.%S.%Y'
today = datetime.datetime.today()
_ts = today.strftime(date_format)
# Get Home Directory
home_dir = os.getenv('HOME')
# Set the name of the archive.
set_name = ARGS.get('tar_name', '%s_%s' % ('Archive', _ts))
file_name = '%s.tgz' % set_name
# Set the working File.
tmp_file = basic.jpath(root=home_dir, inode=file_name)
# Begin creating the Archive.
tar = tarfile.open(tmp_file, 'w:gz')
for name in file_list:
if basic.file_exists(name) is True:
tar.add(name)
tar.close()
report.reporter(msg='ARCHIVE CREATED: %s' % tmp_file, prt=False)
if ARGS.get('verify'):
tar_len = tarfile.open(tmp_file, 'r')
ver_array = []
for member_info in tar_len.getmembers():
ver_array.append(member_info.name)
count = len(ver_array)
orig_count = len(file_list)
if orig_count != count:
raise turbo.SystemProblem(
'ARCHIVE NOT VERIFIED: Archive and File List do not Match.'
' Original File Count = %s, Found Archive Contents = %s'
% (orig_count, count)
)
report.reporter(
msg='ARCHIVE CONTENTS VERIFIED: %s files' % count,
)
except KeyboardInterrupt:
if basic.file_exists(tmp_file):
basic.remove_file(tmp_file)
turbo.emergency_exit('I have stopped at your command,'
' I removed Local Copy of the Archive')
except Exception as exp:
if basic.file_exists(tmp_file):
basic.remove_file(tmp_file)
turbo.emergency_exit(
'I am sorry i just don\'t know what you put into me, Removing'
' Local Copy of the Archive.'
)
turbo.emergency_exit(
'Exception while working on the archive. MESSAGE: %s' % exp
)
else:
return tmp_file
def _downloader(self,
conn,
rpath,
fheaders,
lfile,
source,
retry,
skip=False):
"""Download a specified object in the container.
:param conn:
:param rpath:
:param fheaders:
:param lfile:
:param retry:
:param skip:
"""
if source is None:
local_f = lfile
else:
local_f = basic.jpath(root=source, inode=lfile)
if self._checker(conn, rpath, local_f, fheaders, retry, skip) is True:
report.reporter(
msg='Downloading remote %s to local file %s' % (rpath, lfile),
prt=False,
lvl='debug',
)
# Perform Object GET
conn.request('GET', rpath, headers=fheaders)
# Open our source file and write it
with open(local_f, 'ab') as f_name:
resp = http.response_get(conn=conn,
retry=retry,
resp_only=True)
self.resp_exception(resp=resp, rty=retry)
if resp is None:
report.reporter(msg='API Response Was NONE. resp was: %s' %
resp.msg,
prt=True,
lvl='error',
log=True)
retry()
else:
while True:
chunk = resp.read(2048)
if not chunk:
break
else:
f_name.write(chunk)
report.reporter(msg=('OBJECT %s MESSAGE %s %s %s' %
(rpath, resp.status, resp.reason, resp.msg)),
prt=False,
lvl='debug')
if ARGS.get('restore_perms') is not None:
# Make a connection
resp = self._header_getter(conn=conn,
rpath=rpath,
fheaders=fheaders,
retry=retry)
all_headers = dict(resp.getheaders())
if all([
'x-object-meta-group' in all_headers, 'x-object-meta-owner'
in all_headers, 'x-object-meta-perms' in all_headers
]):
basic.restor_perms(local_file=local_f, headers=all_headers)
else:
report.reporter(
msg=('No Permissions were restored, because none were'
' saved on the object "%s"' % rpath),
lvl='warn',
log=True)
def test_jpath(self):
return_path = basic_utils.jpath(root='/test', inode='path/of/test')
self.assertEqual(return_path, '/test/path/of/test')
def compress_files(file_list):
"""If the archive function is used, create a compressed archive.
:param file_list:
This function allows for multiple sources to be added to the
compressed archive.
"""
tmp_file = None
try:
# Set date and time
date_format = '%a%b%d.%H.%M.%S.%Y'
today = datetime.datetime.today()
_ts = today.strftime(date_format)
# Get Home Directory
home_dir = os.getenv('HOME')
# Set the name of the archive.
set_name = ARGS.get('tar_name', '%s_%s' % ('Archive', _ts))
file_name = '%s.tgz' % set_name
# Set the working File.
tmp_file = basic.jpath(root=home_dir, inode=file_name)
# Begin creating the Archive.
tar = tarfile.open(tmp_file, 'w:gz')
for name in file_list:
if basic.file_exists(name) is True:
tar.add(name)
tar.close()
report.reporter(msg='ARCHIVE CREATED: %s' % tmp_file, prt=False)
if ARGS.get('verify'):
tar_len = tarfile.open(tmp_file, 'r')
ver_array = []
for member_info in tar_len.getmembers():
ver_array.append(member_info.name)
count = len(ver_array)
orig_count = len(file_list)
if orig_count != count:
raise turbo.SystemProblem(
'ARCHIVE NOT VERIFIED: Archive and File List do not Match.'
' Original File Count = %s, Found Archive Contents = %s' %
(orig_count, count))
report.reporter(msg='ARCHIVE CONTENTS VERIFIED: %s files' %
count, )
except KeyboardInterrupt:
if basic.file_exists(tmp_file):
basic.remove_file(tmp_file)
turbo.emergency_exit('I have stopped at your command,'
' I removed Local Copy of the Archive')
except Exception as exp:
if basic.file_exists(tmp_file):
basic.remove_file(tmp_file)
turbo.emergency_exit(
'I am sorry i just don\'t know what you put into me, Removing'
' Local Copy of the Archive.')
turbo.emergency_exit(
'Exception while working on the archive. MESSAGE: %s' % exp)
else:
return tmp_file
def test_jpath(self):
return_path = basic_utils.jpath(root='/test', inode='path/of/test')
self.assertEqual(return_path, '/test/path/of/test')
def _downloader(self, conn, rpath, fheaders, lfile, source, retry,
skip=False):
"""Download a specified object in the container.
:param conn:
:param rpath:
:param fheaders:
:param lfile:
:param retry:
:param skip:
"""
if source is None:
local_f = lfile
else:
local_f = basic.jpath(root=source, inode=lfile)
if self._checker(conn, rpath, local_f, fheaders, retry, skip) is True:
report.reporter(
msg='Downloading remote %s to local file %s' % (rpath, lfile),
prt=False,
lvl='debug',
)
# Perform Object GET
conn.request('GET', rpath, headers=fheaders)
local_f = basic.collision_rename(file_name=local_f)
# Open our source file and write it
with open(local_f, 'ab') as f_name:
resp = http.response_get(conn=conn,
retry=retry,
resp_only=True)
self.resp_exception(resp=resp, rty=retry)
if resp is None:
report.reporter(
msg='API Response Was NONE. resp was: %s' % resp.msg,
prt=True,
lvl='error',
log=True
)
retry()
else:
while True:
chunk = resp.read(2048)
if not chunk:
break
else:
f_name.write(chunk)
report.reporter(
msg=('OBJECT %s MESSAGE %s %s %s'
% (rpath, resp.status, resp.reason, resp.msg)),
prt=False,
lvl='debug'
)
if ARGS.get('restore_perms') is not None:
# Make a connection
resp = self._header_getter(conn=conn,
rpath=rpath,
fheaders=fheaders,
retry=retry)
all_headers = dict(resp.getheaders())
if all(['x-object-meta-group' in all_headers,
'x-object-meta-owner' in all_headers,
'x-object-meta-perms' in all_headers]):
basic.restor_perms(local_file=local_f, headers=all_headers)
else:
report.reporter(
msg=('No Permissions were restored, because none were'
' saved on the object "%s"' % rpath),
lvl='warn',
log=True
)
