def test_create_tmp(self):
return_file = basic_utils.create_tmp()
if not os.path.exists(return_file):
self.fail('No File was found when creating a temp file')
else:
try:
os.remove(return_file)
except OSError:
pass
def test_create_tmp(self):
return_file = basic_utils.create_tmp()
if not os.path.exists(return_file):
self.fail('No File was found when creating a temp file')
else:
try:
os.remove(return_file)
except OSError:
pass
def object_syncer(self, surl, turl, scontainer, tcontainer, obj):
"""Download an Object from one Container and the upload it to a target.
:param surl:
:param turl:
:param scontainer:
:param tcontainer:
:param obj:
"""
def _cleanup():
"""Ensure that our temp file is removed."""
if locals().get('tfile') is not None:
basic.remove_file(tfile)
def _time_difference(resp, obj):
if ARGS.get('save_newer') is True:
# Get the source object last modified time.
compare_time = resp.getheader('last_modified')
if compare_time is None:
return True
elif cloud.time_delta(compare_time=compare_time,
lmobj=obj['last_modified']) is True:
return False
else:
return True
else:
return True
def _compare(resp, obj):
if resp.status == 404:
report.reporter(msg='Target Object %s not found' % obj['name'],
prt=False)
return True
elif resp.getheader('etag') != obj['hash']:
report.reporter(msg='Checksum Mismatch on Target Object %s' %
obj['name'],
prt=False,
lvl='debug')
return _time_difference(resp, obj)
else:
return False
fheaders = self.payload['headers']
for retry in basic.retryloop(attempts=ARGS.get('error_retry'),
delay=5,
obj=obj['name']):
# Open connection and perform operation
fmt, date, date_delta, now = basic.time_stamp()
spath = http.quoter(url=surl.path,
cont=scontainer,
ufile=obj['name'])
tpath = http.quoter(url=turl.path,
cont=tcontainer,
ufile=obj['name'])
conn = http.open_connection(url=turl)
with meth.operation(retry, conn=conn, obj=obj):
resp = self._header_getter(conn=conn,
rpath=tpath,
fheaders=fheaders,
retry=retry)
# If object comparison is True GET then PUT object
if _compare(resp=resp, obj=obj) is not True:
return None
try:
# Open Connection for source Download
conn = http.open_connection(url=surl)
with meth.operation(retry, conn=conn, obj=obj):
# make a temp file.
tfile = basic.create_tmp()
# Make a connection
resp = self._header_getter(conn=conn,
rpath=spath,
fheaders=fheaders,
retry=retry)
sheaders = dict(resp.getheaders())
# TODO(kevin) add the ability to short upload if timestamp
# TODO(kevin) ... is newer on the target.
# GET remote Object
self._downloader(conn=conn,
rpath=spath,
fheaders=fheaders,
lfile=tfile,
source=None,
retry=retry,
skip=True)
for nretry in basic.retryloop(attempts=ARGS.get('error_retry'),
delay=5,
obj=obj):
# open connection for target upload.
conn = http.open_connection(url=turl)
with meth.operation(retry,
conn=conn,
obj=obj,
cleanup=_cleanup):
resp = self._header_getter(conn=conn,
rpath=tpath,
fheaders=fheaders,
retry=nretry)
self.resp_exception(resp=resp, rty=nretry)
# PUT remote object
self._putter(conn=conn,
fpath=tfile,
rpath=tpath,
fheaders=fheaders,
retry=nretry,
skip=True)
# let the system rest for 3 seconds.
basic.stupid_hack(wait=3)
# With the source headers POST new headers on target
if ARGS.get('clone_headers') is True:
resp = self._header_getter(conn=conn,
rpath=tpath,
fheaders=fheaders,
retry=nretry)
theaders = dict(resp.getheaders())
for key in sheaders.keys():
if key not in theaders:
fheaders.update({key: sheaders[key]})
# Force the SOURCE content Type on the Target.
fheaders.update(
{'content-type': sheaders.get('content-type')})
self._header_poster(conn=conn,
rpath=tpath,
fheaders=fheaders,
retry=nretry)
finally:
_cleanup()
def object_syncer(self, surl, turl, scontainer, tcontainer, u_file):
"""Download an Object from one Container and the upload it to a target.
:param surl:
:param turl:
:param scontainer:
:param tcontainer:
:param u_file:
"""
def _cleanup():
"""Ensure that our temp file is removed."""
if locals().get('tfile') is not None:
basic.remove_file(tfile)
def _time_difference(obj_resp, obj):
if ARGS.get('save_newer') is True:
# Get the source object last modified time.
compare_time = obj_resp.header.get('last_modified')
if compare_time is None:
return True
elif cloud.time_delta(compare_time=compare_time,
lmobj=obj['last_modified']) is True:
return False
else:
return True
else:
return True
def _compare(obj_resp, obj):
if obj_resp.status_code == 404:
report.reporter(msg='Target Object %s not found' % obj['name'],
prt=False)
return True
elif ARGS.get('add_only'):
report.reporter(msg='Target Object %s already exists' %
obj['name'],
prt=True)
return False
elif obj_resp.headers.get('etag') != obj['hash']:
report.reporter(msg=('Checksum Mismatch on Target Object %s' %
u_file['name']),
prt=False,
lvl='debug')
return _time_difference(obj_resp, obj)
else:
return False
fheaders = self.payload['headers']
for retry in basic.retryloop(attempts=ARGS.get('error_retry'),
delay=5,
obj=u_file['name']):
# Open connection and perform operation
spath = http.quoter(url=surl.path,
cont=scontainer,
ufile=u_file['name'])
tpath = http.quoter(url=turl.path,
cont=tcontainer,
ufile=u_file['name'])
with meth.operation(retry, obj='%s %s' % (fheaders, tpath)):
resp = self._header_getter(url=turl,
rpath=tpath,
fheaders=fheaders)
# If object comparison is True GET then PUT object
if _compare(resp, u_file) is not True:
return None
try:
# Open Connection for source Download
with meth.operation(retry, obj='%s %s' % (fheaders, spath)):
# make a temp file.
tfile = basic.create_tmp()
# Make a connection
resp = self._header_getter(url=surl,
rpath=spath,
fheaders=fheaders)
sheaders = resp.headers
self._downloader(url=surl,
rpath=spath,
fheaders=fheaders,
lfile=tfile,
source=None,
skip=True)
for _retry in basic.retryloop(attempts=ARGS.get('error_retry'),
delay=5,
obj=u_file):
# open connection for target upload.
adddata = '%s %s' % (fheaders, u_file)
with meth.operation(_retry, obj=adddata, cleanup=_cleanup):
resp = self._header_getter(url=turl,
rpath=tpath,
fheaders=fheaders)
self.resp_exception(resp=resp)
# PUT remote object
self._putter(url=turl,
fpath=tfile,
rpath=tpath,
fheaders=fheaders,
skip=True)
# let the system rest for 1 seconds.
basic.stupid_hack(wait=1)
# With the source headers POST new headers on target
if ARGS.get('clone_headers') is True:
theaders = resp.headers
for key in sheaders.keys():
if key not in theaders:
fheaders.update({key: sheaders[key]})
# Force the SOURCE content Type on the Target.
fheaders.update(
{'content-type': sheaders.get('content-type')})
self._header_poster(url=turl,
rpath=tpath,
fheaders=fheaders)
finally:
_cleanup()
def object_syncer(self, surl, turl, scontainer, tcontainer, obj):
"""Download an Object from one Container and the upload it to a target.
:param surl:
:param turl:
:param scontainer:
:param tcontainer:
:param obj:
"""
def _cleanup():
"""Ensure that our temp file is removed."""
if locals().get('tfile') is not None:
basic.remove_file(tfile)
def _time_difference(resp, obj):
if ARGS.get('save_newer') is True:
# Get the source object last modified time.
compare_time = resp.getheader('last_modified')
if compare_time is None:
return True
elif cloud.time_delta(compare_time=compare_time,
lmobj=obj['last_modified']) is True:
return False
else:
return True
else:
return True
def _compare(resp, obj):
if resp.status == 404:
report.reporter(
msg='Target Object %s not found' % obj['name'],
prt=False
)
return True
elif resp.getheader('etag') != obj['hash']:
report.reporter(
msg='Checksum Mismatch on Target Object %s' % obj['name'],
prt=False,
lvl='debug'
)
return _time_difference(resp, obj)
else:
return False
fheaders = self.payload['headers']
for retry in basic.retryloop(attempts=ARGS.get('error_retry'),
delay=5,
obj=obj['name']):
# Open connection and perform operation
fmt, date, date_delta, now = basic.time_stamp()
spath = http.quoter(url=surl.path,
cont=scontainer,
ufile=obj['name'])
tpath = http.quoter(url=turl.path,
cont=tcontainer,
ufile=obj['name'])
conn = http.open_connection(url=turl)
with meth.operation(retry, conn=conn, obj=obj):
resp = self._header_getter(conn=conn,
rpath=tpath,
fheaders=fheaders,
retry=retry)
# If object comparison is True GET then PUT object
if _compare(resp=resp, obj=obj) is not True:
return None
try:
# Open Connection for source Download
conn = http.open_connection(url=surl)
with meth.operation(retry,
conn=conn,
obj=obj):
# make a temp file.
tfile = basic.create_tmp()
# Make a connection
resp = self._header_getter(conn=conn,
rpath=spath,
fheaders=fheaders,
retry=retry)
sheaders = dict(resp.getheaders())
# TODO(kevin) add the ability to short upload if timestamp
# TODO(kevin) ... is newer on the target.
# GET remote Object
self._downloader(
conn=conn,
rpath=spath,
fheaders=fheaders,
lfile=tfile,
source=None,
retry=retry,
skip=True
)
for nretry in basic.retryloop(attempts=ARGS.get('error_retry'),
delay=5,
obj=obj):
# open connection for target upload.
conn = http.open_connection(url=turl)
with meth.operation(retry,
conn=conn,
obj=obj,
cleanup=_cleanup):
resp = self._header_getter(conn=conn,
rpath=tpath,
fheaders=fheaders,
retry=nretry)
self.resp_exception(resp=resp, rty=nretry)
# PUT remote object
self._putter(conn=conn,
fpath=tfile,
rpath=tpath,
fheaders=fheaders,
retry=nretry,
skip=True)
# let the system rest for 3 seconds.
basic.stupid_hack(wait=3)
# With the source headers POST new headers on target
if ARGS.get('clone_headers') is True:
resp = self._header_getter(conn=conn,
rpath=tpath,
fheaders=fheaders,
retry=nretry)
theaders = dict(resp.getheaders())
for key in sheaders.keys():
if key not in theaders:
fheaders.update({key: sheaders[key]})
# Force the SOURCE content Type on the Target.
fheaders.update(
{'content-type': sheaders.get('content-type')}
)
self._header_poster(
conn=conn,
rpath=tpath,
fheaders=fheaders,
retry=nretry
)
finally:
_cleanup()
def object_syncer(self, surl, turl, scontainer, tcontainer, u_file):
"""Download an Object from one Container and the upload it to a target.
:param surl:
:param turl:
:param scontainer:
:param tcontainer:
:param u_file:
"""
def _cleanup():
"""Ensure that our temp file is removed."""
if locals().get("tfile") is not None:
basic.remove_file(tfile)
def _time_difference(obj_resp, obj):
if ARGS.get("save_newer") is True:
# Get the source object last modified time.
compare_time = obj_resp.header.get("last_modified")
if compare_time is None:
return True
elif cloud.time_delta(compare_time=compare_time, lmobj=obj["last_modified"]) is True:
return False
else:
return True
else:
return True
def _compare(obj_resp, obj):
if obj_resp.status_code == 404:
report.reporter(msg="Target Object %s not found" % obj["name"], prt=False)
return True
elif ARGS.get("add_only"):
report.reporter(msg="Target Object %s already exists" % obj["name"], prt=True)
return False
elif obj_resp.headers.get("etag") != obj["hash"]:
report.reporter(msg=("Checksum Mismatch on Target Object %s" % u_file["name"]), prt=False, lvl="debug")
return _time_difference(obj_resp, obj)
else:
return False
fheaders = self.payload["headers"]
for retry in basic.retryloop(attempts=ARGS.get("error_retry"), delay=5, obj=u_file["name"]):
# Open connection and perform operation
spath = http.quoter(url=surl.path, cont=scontainer, ufile=u_file["name"])
tpath = http.quoter(url=turl.path, cont=tcontainer, ufile=u_file["name"])
with meth.operation(retry, obj="%s %s" % (fheaders, tpath)):
resp = self._header_getter(url=turl, rpath=tpath, fheaders=fheaders)
# If object comparison is True GET then PUT object
if _compare(resp, u_file) is not True:
return None
try:
# Open Connection for source Download
with meth.operation(retry, obj="%s %s" % (fheaders, spath)):
# make a temp file.
tfile = basic.create_tmp()
# Make a connection
resp = self._header_getter(url=surl, rpath=spath, fheaders=fheaders)
sheaders = resp.headers
self._downloader(url=surl, rpath=spath, fheaders=fheaders, lfile=tfile, source=None, skip=True)
for _retry in basic.retryloop(attempts=ARGS.get("error_retry"), delay=5, obj=u_file):
# open connection for target upload.
adddata = "%s %s" % (fheaders, u_file)
with meth.operation(_retry, obj=adddata, cleanup=_cleanup):
resp = self._header_getter(url=turl, rpath=tpath, fheaders=fheaders)
self.resp_exception(resp=resp)
# PUT remote object
self._putter(url=turl, fpath=tfile, rpath=tpath, fheaders=fheaders, skip=True)
# let the system rest for 1 seconds.
basic.stupid_hack(wait=1)
# With the source headers POST new headers on target
if ARGS.get("clone_headers") is True:
theaders = resp.headers
for key in sheaders.keys():
if key not in theaders:
fheaders.update({key: sheaders[key]})
# Force the SOURCE content Type on the Target.
fheaders.update({"content-type": sheaders.get("content-type")})
self._header_poster(url=turl, rpath=tpath, fheaders=fheaders)
finally:
_cleanup()