def test_file_exists(self):
return_file = tempfile.mkstemp()[1]
self.assertEqual(basic_utils.file_exists(filename=return_file), True)
try:
os.remove(return_file)
except OSError:
pass
else:
self.assertEqual(basic_utils.file_exists(filename=return_file),
False)
def test_file_exists(self):
return_file = tempfile.mkstemp()[1]
self.assertEqual(
basic_utils.file_exists(filename=return_file), True
)
try:
os.remove(return_file)
except OSError:
pass
else:
self.assertEqual(
basic_utils.file_exists(filename=return_file), False
)
def _putter(self, conn, fpath, rpath, fheaders, retry, skip=False):
"""Place object into the container.
:param conn:
:param fpath:
:param rpath:
:param fheaders:
:param retry:
"""
if self._checker(conn, rpath, fpath, fheaders, retry, skip) is True:
report.reporter(msg='OBJECT ORIGIN %s RPATH %s' % (fpath, rpath),
prt=False,
lvl='debug')
if basic.file_exists(fpath) is False:
return None
else:
with open(fpath, 'rb') as f_open:
conn.request('PUT', rpath, body=f_open, headers=fheaders)
resp, read = http.response_get(conn=conn, retry=retry)
self.resp_exception(resp=resp, rty=retry)
report.reporter(msg=('MESSAGE %s %s %s' %
(resp.status, resp.reason, resp.msg)),
prt=False,
lvl='debug')
def _putter(self, conn, fpath, rpath, fheaders, retry, skip=False):
"""Place object into the container.
:param conn:
:param fpath:
:param rpath:
:param fheaders:
:param retry:
"""
if self._checker(conn, rpath, fpath, fheaders, retry, skip) is True:
report.reporter(
msg='OBJECT ORIGIN %s RPATH %s' % (fpath, rpath),
prt=False,
lvl='debug'
)
if basic.file_exists(fpath) is False:
return None
else:
with open(fpath, 'rb') as f_open:
conn.request('PUT', rpath, body=f_open, headers=fheaders)
resp, read = http.response_get(conn=conn, retry=retry)
self.resp_exception(resp=resp, rty=retry)
report.reporter(
msg=('MESSAGE %s %s %s'
% (resp.status, resp.reason, resp.msg)),
prt=False,
lvl='debug'
)
def _putter(self, url, fpath, rpath, fheaders, skip=False):
"""Place object into the container.
:param url:
:param fpath:
:param rpath:
:param fheaders:
"""
if self._checker(url, rpath, fpath, fheaders, skip) is True:
report.reporter(
msg='OBJECT ORIGIN %s RPATH %s' % (fpath, rpath),
prt=False,
lvl='debug'
)
if basic.file_exists(fpath) is False:
return None
else:
with open(fpath, 'rb') as f_open:
resp = http.put_request(
url=url, rpath=rpath, body=f_open, headers=fheaders
)
self.resp_exception(resp=resp)
report.reporter(
msg=('MESSAGE %s %s %s'
% (resp.status_code, resp.reason, resp.request)),
prt=False,
lvl='debug'
)
def _putter(self, url, fpath, rpath, fheaders, skip=False):
"""Place object into the container.
:param url:
:param fpath:
:param rpath:
:param fheaders:
"""
if self._checker(url, rpath, fpath, fheaders, skip) is True:
report.reporter(msg="OBJECT ORIGIN %s RPATH %s" % (fpath, rpath), prt=False, lvl="debug")
if basic.file_exists(fpath) is False:
return None
else:
if os.path.islink(fpath):
link = os.readlink(fpath)
lpath = os.path.abspath(os.path.join(os.path.dirname(fpath), link))
rpath_reversed = rpath.split("/")[::-1]
fpath_reversed = fpath.split("/")[::-1]
a, b = sorted((rpath_reversed, fpath_reversed), key=len)
for i, j in enumerate(a):
if j != b[i]:
index = i
break
container = rpath_reversed[index]
container_dir = fpath.replace("/".join(rpath_reversed[:index][::-1]), "")
if container_dir in lpath:
manafest = container + "/" + lpath.replace(container_dir, "")
fheaders["X-Object-Manifest"] = manafest
resp = http.put_request(url=url, rpath=rpath, body=None, headers=fheaders)
self.resp_exception(resp=resp)
else:
report.reporter(
msg="symlink %s points to location %s which is"
" outside uploading directory" % (fpath, lpath),
lvl="warning",
)
else:
with open(fpath, "rb") as f_open:
resp = http.put_request(url=url, rpath=rpath, body=f_open, headers=fheaders)
self.resp_exception(resp=resp)
def _putter(self, url, fpath, rpath, fheaders, skip=False):
"""Place object into the container.
:param url:
:param fpath:
:param rpath:
:param fheaders:
"""
if self._checker(url, rpath, fpath, fheaders, skip) is True:
report.reporter(msg="OBJECT ORIGIN %s RPATH %s" % (fpath, rpath), prt=False, lvl="debug")
if basic.file_exists(fpath) is False:
return None
else:
with open(fpath, "rb") as f_open:
resp = http.put_request(url=url, rpath=rpath, body=f_open, headers=fheaders)
self.resp_exception(resp=resp)
def _putter(self, url, fpath, rpath, fheaders, skip=False):
"""Place object into the container.
:param url:
:param fpath:
:param rpath:
:param fheaders:
"""
if self._checker(url, rpath, fpath, fheaders, skip) is True:
report.reporter(msg='OBJECT ORIGIN %s RPATH %s' % (fpath, rpath),
prt=False,
lvl='debug')
if basic.file_exists(fpath) is False:
return None
else:
with open(fpath, 'rb') as f_open:
resp = http.put_request(url=url,
rpath=rpath,
body=f_open,
headers=fheaders)
self.resp_exception(resp=resp)
def compress_files(file_list):
"""If the archive function is used, create a compressed archive.
:param file_list:
This function allows for multiple sources to be added to the
compressed archive.
"""
tmp_file = None
try:
# Set date and time
date_format = '%a%b%d.%H.%M.%S.%Y'
today = datetime.datetime.today()
_ts = today.strftime(date_format)
# Get Home Directory
home_dir = os.getenv('HOME')
# Set the name of the archive.
set_name = ARGS.get('tar_name', '%s_%s' % ('Archive', _ts))
file_name = '%s.tgz' % set_name
# Set the working File.
tmp_file = basic.jpath(root=home_dir, inode=file_name)
# Begin creating the Archive.
tar = tarfile.open(tmp_file, 'w:gz')
for name in file_list:
if basic.file_exists(name) is True:
tar.add(name)
tar.close()
report.reporter(msg='ARCHIVE CREATED: %s' % tmp_file, prt=False)
if ARGS.get('verify'):
tar_len = tarfile.open(tmp_file, 'r')
ver_array = []
for member_info in tar_len.getmembers():
ver_array.append(member_info.name)
count = len(ver_array)
orig_count = len(file_list)
if orig_count != count:
raise turbo.SystemProblem(
'ARCHIVE NOT VERIFIED: Archive and File List do not Match.'
' Original File Count = %s, Found Archive Contents = %s'
% (orig_count, count)
)
report.reporter(
msg='ARCHIVE CONTENTS VERIFIED: %s files' % count,
)
except KeyboardInterrupt:
if basic.file_exists(tmp_file):
basic.remove_file(tmp_file)
turbo.emergency_exit('I have stopped at your command,'
' I removed Local Copy of the Archive')
except Exception as exp:
if basic.file_exists(tmp_file):
basic.remove_file(tmp_file)
turbo.emergency_exit(
'I am sorry i just don\'t know what you put into me, Removing'
' Local Copy of the Archive.'
)
turbo.emergency_exit(
'Exception while working on the archive. MESSAGE: %s' % exp
)
else:
return tmp_file
def _putter(self, url, fpath, rpath, fheaders, skip=False):
"""Place object into the container.
:param url:
:param fpath:
:param rpath:
:param fheaders:
"""
if self._checker(url, rpath, fpath, fheaders, skip) is True:
report.reporter(msg='OBJECT ORIGIN %s RPATH %s' % (fpath, rpath),
prt=False,
lvl='debug')
if basic.file_exists(fpath) is False:
return None
else:
fheaders['content-type'] = "application/x-bzip2"
opener = self._dispatch_open(fpath)
if opener is None:
return None
with opener(fpath, 'rb') as f_open:
compressor = bz2.BZ2Compressor()
buf = ""
buf_len = 0
chunk_size = 2**20 * 100
line_number = 1
last_line_number = 1
resp = self._header_getter(url=url,
rpath="%s-%s.bz2" %
(rpath, last_line_number),
fheaders=fheaders)
skip = resp.status_code >= 200 \
and resp.status_code < 300
if skip:
report.reporter(
msg="Skipping upload of part %s in %s" % \
(last_line_number, rpath),
lvl="info",
prt=False,
log=True)
while True:
line = f_open.readline()
line_number += 1
if line == "":
if buf_len != 0:
data = compressor.flush()
if data:
buf += data
if not skip:
resp = http.put_request(
url=url,
rpath="%s-%s.bz2" %
(rpath, last_line_number),
body=buf,
headers=fheaders)
self.resp_exception(resp=resp)
break
if buf_len + len(line) > chunk_size:
if line_number == last_line_number + 1:
report.reporter(
msg=
"Skipping upload due to %d size lines: %s"
% (chunk_size, fpath),
lvl="info",
prt=False,
log=True)
break
data = compressor.flush()
if data is not None:
buf += data
if not skip:
resp = http.put_request(
url=url,
rpath="%s-%s.bz2" %
(rpath, last_line_number),
body=buf,
headers=fheaders)
self.resp_exception(resp=resp)
last_line_number = line_number - 1
buf = ""
buf_len = 0
resp = self._header_getter(
url=url,
rpath="%s-%s.bz2" % (rpath, last_line_number),
fheaders=fheaders)
skip = resp.status_code >= 200 \
and resp.status_code < 300
if skip:
report.reporter(
msg="Skipping upload of part %s in %s" % \
(last_line_number, rpath),
lvl="info",
prt=False,
log=True)
compressor = bz2.BZ2Compressor()
buf_len += len(line)
data = compressor.compress(line if not skip else "")
if data is not None:
buf += data
def compress_files(file_list):
"""If the archive function is used, create a compressed archive.
:param file_list:
This function allows for multiple sources to be added to the
compressed archive.
"""
tmp_file = None
try:
# Set date and time
date_format = '%a%b%d.%H.%M.%S.%Y'
today = datetime.datetime.today()
_ts = today.strftime(date_format)
# Get Home Directory
home_dir = os.getenv('HOME')
# Set the name of the archive.
set_name = ARGS.get('tar_name', '%s_%s' % ('Archive', _ts))
file_name = '%s.tgz' % set_name
# Set the working File.
tmp_file = basic.jpath(root=home_dir, inode=file_name)
# Begin creating the Archive.
tar = tarfile.open(tmp_file, 'w:gz')
for name in file_list:
if basic.file_exists(name) is True:
tar.add(name)
tar.close()
report.reporter(msg='ARCHIVE CREATED: %s' % tmp_file, prt=False)
if ARGS.get('verify'):
tar_len = tarfile.open(tmp_file, 'r')
ver_array = []
for member_info in tar_len.getmembers():
ver_array.append(member_info.name)
count = len(ver_array)
orig_count = len(file_list)
if orig_count != count:
raise turbo.SystemProblem(
'ARCHIVE NOT VERIFIED: Archive and File List do not Match.'
' Original File Count = %s, Found Archive Contents = %s' %
(orig_count, count))
report.reporter(msg='ARCHIVE CONTENTS VERIFIED: %s files' %
count, )
except KeyboardInterrupt:
if basic.file_exists(tmp_file):
basic.remove_file(tmp_file)
turbo.emergency_exit('I have stopped at your command,'
' I removed Local Copy of the Archive')
except Exception as exp:
if basic.file_exists(tmp_file):
basic.remove_file(tmp_file)
turbo.emergency_exit(
'I am sorry i just don\'t know what you put into me, Removing'
' Local Copy of the Archive.')
turbo.emergency_exit(
'Exception while working on the archive. MESSAGE: %s' % exp)
else:
return tmp_file
