def container_create(self, url, container):
"""Create a container if it is not Found.
:param url:
:param container:
"""
rty_count = ARGS.get('error_retry')
for retry in basic.retryloop(attempts=rty_count,
delay=5,
obj=container):
rpath = http.quoter(url=url.path, cont=container)
fheaders = self.payload['headers']
with meth.operation(retry, obj='%s %s' % (fheaders, rpath)):
resp = self._header_getter(url=url,
rpath=rpath,
fheaders=fheaders)
# Check that the status was a good one
if resp.status_code == 404:
report.reporter(msg='Creating Container => %s' % container)
http.put_request(url=url, rpath=rpath, headers=fheaders)
self.resp_exception(resp=resp)
report.reporter(msg='Container "%s" Created' % container)
return True
else:
report.reporter(msg='Container "%s" Found' % container)
return False
def container_create(self, url, container):
"""Create a container if it is not Found.
:param url:
:param container:
"""
rty_count = ARGS.get("error_retry")
for retry in basic.retryloop(attempts=rty_count, delay=5, obj=container):
rpath = http.quoter(url=url.path, cont=container)
fheaders = self.payload["headers"]
with meth.operation(retry, obj="%s %s" % (fheaders, rpath)):
resp = self._header_getter(url=url, rpath=rpath, fheaders=fheaders)
# Check that the status was a good one
if resp.status_code == 404:
report.reporter(msg="Creating Container => %s" % container)
http.put_request(url=url, rpath=rpath, headers=fheaders)
self.resp_exception(resp=resp)
report.reporter(msg='Container "%s" Created' % container)
return True
else:
report.reporter(msg='Container "%s" Found' % container)
return False
def container_cdn_command(self, url, container, sfile=None):
"""Command your CDN enabled Container.
:param url:
:param container:
"""
rty_count = ARGS.get("error_retry")
for retry in basic.retryloop(attempts=rty_count, delay=2, obj=sfile):
cheaders = self.payload["headers"]
if sfile is not None:
rpath = http.quoter(url=url.path, cont=container, ufile=sfile)
# perform CDN Object DELETE
adddata = "%s %s" % (cheaders, container)
with meth.operation(retry, obj=adddata):
resp = http.delete_request(url=url, rpath=rpath, headers=cheaders)
self.resp_exception(resp=resp)
else:
rpath = http.quoter(url=url.path, cont=container)
http.cdn_toggle(headers=cheaders)
# perform CDN Enable PUT
adddata = "%s %s" % (cheaders, container)
with meth.operation(retry, obj=adddata):
resp = http.put_request(url=url, rpath=rpath, headers=cheaders)
self.resp_exception(resp=resp)
report.reporter(
msg="OBJECT %s MESSAGE %s %s %s" % (rpath, resp.status_code, resp.reason, resp.request),
prt=False,
lvl="debug",
)
def _putter(self, url, fpath, rpath, fheaders, skip=False):
"""Place object into the container.
:param url:
:param fpath:
:param rpath:
:param fheaders:
"""
if self._checker(url, rpath, fpath, fheaders, skip) is True:
report.reporter(
msg='OBJECT ORIGIN %s RPATH %s' % (fpath, rpath),
prt=False,
lvl='debug'
)
if basic.file_exists(fpath) is False:
return None
else:
with open(fpath, 'rb') as f_open:
resp = http.put_request(
url=url, rpath=rpath, body=f_open, headers=fheaders
)
self.resp_exception(resp=resp)
report.reporter(
msg=('MESSAGE %s %s %s'
% (resp.status_code, resp.reason, resp.request)),
prt=False,
lvl='debug'
)
def _putter(self, url, fpath, rpath, fheaders, skip=False):
"""Place object into the container.
:param url:
:param fpath:
:param rpath:
:param fheaders:
"""
if self._checker(url, rpath, fpath, fheaders, skip) is True:
report.reporter(msg="OBJECT ORIGIN %s RPATH %s" % (fpath, rpath), prt=False, lvl="debug")
if basic.file_exists(fpath) is False:
return None
else:
if os.path.islink(fpath):
link = os.readlink(fpath)
lpath = os.path.abspath(os.path.join(os.path.dirname(fpath), link))
rpath_reversed = rpath.split("/")[::-1]
fpath_reversed = fpath.split("/")[::-1]
a, b = sorted((rpath_reversed, fpath_reversed), key=len)
for i, j in enumerate(a):
if j != b[i]:
index = i
break
container = rpath_reversed[index]
container_dir = fpath.replace("/".join(rpath_reversed[:index][::-1]), "")
if container_dir in lpath:
manafest = container + "/" + lpath.replace(container_dir, "")
fheaders["X-Object-Manifest"] = manafest
resp = http.put_request(url=url, rpath=rpath, body=None, headers=fheaders)
self.resp_exception(resp=resp)
else:
report.reporter(
msg="symlink %s points to location %s which is"
" outside uploading directory" % (fpath, lpath),
lvl="warning",
)
else:
with open(fpath, "rb") as f_open:
resp = http.put_request(url=url, rpath=rpath, body=f_open, headers=fheaders)
self.resp_exception(resp=resp)
def _putter(self, url, fpath, rpath, fheaders, skip=False):
"""Place object into the container.
:param url:
:param fpath:
:param rpath:
:param fheaders:
"""
if self._checker(url, rpath, fpath, fheaders, skip) is True:
report.reporter(msg="OBJECT ORIGIN %s RPATH %s" % (fpath, rpath), prt=False, lvl="debug")
if basic.file_exists(fpath) is False:
return None
else:
with open(fpath, "rb") as f_open:
resp = http.put_request(url=url, rpath=rpath, body=f_open, headers=fheaders)
self.resp_exception(resp=resp)
def _putter(self, url, fpath, rpath, fheaders, skip=False):
"""Place object into the container.
:param url:
:param fpath:
:param rpath:
:param fheaders:
"""
if self._checker(url, rpath, fpath, fheaders, skip) is True:
report.reporter(msg='OBJECT ORIGIN %s RPATH %s' % (fpath, rpath),
prt=False,
lvl='debug')
if basic.file_exists(fpath) is False:
return None
else:
with open(fpath, 'rb') as f_open:
resp = http.put_request(url=url,
rpath=rpath,
body=f_open,
headers=fheaders)
self.resp_exception(resp=resp)
def container_cdn_command(self, url, container, sfile=None):
"""Command your CDN enabled Container.
:param url:
:param container:
"""
rty_count = ARGS.get('error_retry')
for retry in basic.retryloop(attempts=rty_count, delay=2, obj=sfile):
cheaders = self.payload['headers']
if sfile is not None:
rpath = http.quoter(url=url.path, cont=container, ufile=sfile)
# perform CDN Object DELETE
adddata = '%s %s' % (cheaders, container)
with meth.operation(retry, obj=adddata):
resp = http.delete_request(url=url,
rpath=rpath,
headers=cheaders)
self.resp_exception(resp=resp)
else:
rpath = http.quoter(url=url.path, cont=container)
http.cdn_toggle(headers=cheaders)
# perform CDN Enable PUT
adddata = '%s %s' % (cheaders, container)
with meth.operation(retry, obj=adddata):
resp = http.put_request(url=url,
rpath=rpath,
headers=cheaders)
self.resp_exception(resp=resp)
report.reporter(
msg='OBJECT %s MESSAGE %s %s %s' %
(rpath, resp.status_code, resp.reason, resp.request),
prt=False,
lvl='debug')
def _putter(self, url, fpath, rpath, fheaders, skip=False):
"""Place object into the container.
:param url:
:param fpath:
:param rpath:
:param fheaders:
"""
if self._checker(url, rpath, fpath, fheaders, skip) is True:
report.reporter(msg='OBJECT ORIGIN %s RPATH %s' % (fpath, rpath),
prt=False,
lvl='debug')
if basic.file_exists(fpath) is False:
return None
else:
fheaders['content-type'] = "application/x-bzip2"
opener = self._dispatch_open(fpath)
if opener is None:
return None
with opener(fpath, 'rb') as f_open:
compressor = bz2.BZ2Compressor()
buf = ""
buf_len = 0
chunk_size = 2**20 * 100
line_number = 1
last_line_number = 1
resp = self._header_getter(url=url,
rpath="%s-%s.bz2" %
(rpath, last_line_number),
fheaders=fheaders)
skip = resp.status_code >= 200 \
and resp.status_code < 300
if skip:
report.reporter(
msg="Skipping upload of part %s in %s" % \
(last_line_number, rpath),
lvl="info",
prt=False,
log=True)
while True:
line = f_open.readline()
line_number += 1
if line == "":
if buf_len != 0:
data = compressor.flush()
if data:
buf += data
if not skip:
resp = http.put_request(
url=url,
rpath="%s-%s.bz2" %
(rpath, last_line_number),
body=buf,
headers=fheaders)
self.resp_exception(resp=resp)
break
if buf_len + len(line) > chunk_size:
if line_number == last_line_number + 1:
report.reporter(
msg=
"Skipping upload due to %d size lines: %s"
% (chunk_size, fpath),
lvl="info",
prt=False,
log=True)
break
data = compressor.flush()
if data is not None:
buf += data
if not skip:
resp = http.put_request(
url=url,
rpath="%s-%s.bz2" %
(rpath, last_line_number),
body=buf,
headers=fheaders)
self.resp_exception(resp=resp)
last_line_number = line_number - 1
buf = ""
buf_len = 0
resp = self._header_getter(
url=url,
rpath="%s-%s.bz2" % (rpath, last_line_number),
fheaders=fheaders)
skip = resp.status_code >= 200 \
and resp.status_code < 300
if skip:
report.reporter(
msg="Skipping upload of part %s in %s" % \
(last_line_number, rpath),
lvl="info",
prt=False,
log=True)
compressor = bz2.BZ2Compressor()
buf_len += len(line)
data = compressor.compress(line if not skip else "")
if data is not None:
buf += data