def time_delta(lmobj, compare_time=None):
"""Check to see if a date delta exists based on filter for an object.
:param lmobj:
:param compare_time:
:return True|False:
"""
fmt, date, delta, now = basic.time_stamp()
# Set time objects
odate = date.strptime(lmobj, fmt)
if not compare_time:
# Time Options
time_factor = ARGS.get('time_factor', 1)
offset = ARGS.get('time_offset')
if (odate + delta(**{offset: time_factor})) > now:
return False
else:
return True
else:
if date.strptime(compare_time, fmt) > odate:
return True
else:
return False
def time_delta(lmobj, compare_time=None):
"""Check to see if a date delta exists based on filter for an object.
:param lmobj:
:param compare_time:
:return True|False:
"""
def hours(delta, factor):
return delta(hours=factor)
def days(delta, factor):
return delta(days=factor)
def weeks(delta, factor):
return delta(weeks=factor)
fmt, date, delta, now = basic.time_stamp()
# Set time objects
odate = date.strptime(lmobj, fmt)
if compare_time is None:
# Time Options
time_factor = ARGS.get('time_factor', 1)
offset = ARGS.get('time_offset')
offset_method = locals()[offset]
if (odate + offset_method(delta=delta, factor=time_factor)) > now:
return False
else:
return True
else:
if date.strptime(compare_time, fmt) > odate:
return True
else:
return False
def time_delta(lmobj, compare_time=None):
"""Check to see if a date delta exists based on filter for an object.
:param lmobj:
:param compare_time:
:return True|False:
"""
def hours(delta, factor):
return delta(hours=factor)
def days(delta, factor):
return delta(days=factor)
def weeks(delta, factor):
return delta(weeks=factor)
fmt, date, delta, now = basic.time_stamp()
# Set time objects
odate = date.strptime(lmobj, fmt)
if compare_time is None:
# Time Options
time_factor = ARGS.get('time_factor', 1)
offset = ARGS.get('time_offset')
offset_method = locals()[offset]
if (odate + offset_method(delta=delta, factor=time_factor)) > now:
return False
else:
return True
else:
if date.strptime(compare_time, fmt) > odate:
return True
else:
return False
def object_syncer(self, surl, turl, scontainer, tcontainer, obj):
"""Download an Object from one Container and the upload it to a target.
:param surl:
:param turl:
:param scontainer:
:param tcontainer:
:param obj:
"""
def _cleanup():
"""Ensure that our temp file is removed."""
if locals().get('tfile') is not None:
basic.remove_file(tfile)
def _time_difference(resp, obj):
if ARGS.get('save_newer') is True:
# Get the source object last modified time.
compare_time = resp.getheader('last_modified')
if compare_time is None:
return True
elif cloud.time_delta(compare_time=compare_time,
lmobj=obj['last_modified']) is True:
return False
else:
return True
else:
return True
def _compare(resp, obj):
if resp.status == 404:
report.reporter(msg='Target Object %s not found' % obj['name'],
prt=False)
return True
elif resp.getheader('etag') != obj['hash']:
report.reporter(msg='Checksum Mismatch on Target Object %s' %
obj['name'],
prt=False,
lvl='debug')
return _time_difference(resp, obj)
else:
return False
fheaders = self.payload['headers']
for retry in basic.retryloop(attempts=ARGS.get('error_retry'),
delay=5,
obj=obj['name']):
# Open connection and perform operation
fmt, date, date_delta, now = basic.time_stamp()
spath = http.quoter(url=surl.path,
cont=scontainer,
ufile=obj['name'])
tpath = http.quoter(url=turl.path,
cont=tcontainer,
ufile=obj['name'])
conn = http.open_connection(url=turl)
with meth.operation(retry, conn=conn, obj=obj):
resp = self._header_getter(conn=conn,
rpath=tpath,
fheaders=fheaders,
retry=retry)
# If object comparison is True GET then PUT object
if _compare(resp=resp, obj=obj) is not True:
return None
try:
# Open Connection for source Download
conn = http.open_connection(url=surl)
with meth.operation(retry, conn=conn, obj=obj):
# make a temp file.
tfile = basic.create_tmp()
# Make a connection
resp = self._header_getter(conn=conn,
rpath=spath,
fheaders=fheaders,
retry=retry)
sheaders = dict(resp.getheaders())
# TODO(kevin) add the ability to short upload if timestamp
# TODO(kevin) ... is newer on the target.
# GET remote Object
self._downloader(conn=conn,
rpath=spath,
fheaders=fheaders,
lfile=tfile,
source=None,
retry=retry,
skip=True)
for nretry in basic.retryloop(attempts=ARGS.get('error_retry'),
delay=5,
obj=obj):
# open connection for target upload.
conn = http.open_connection(url=turl)
with meth.operation(retry,
conn=conn,
obj=obj,
cleanup=_cleanup):
resp = self._header_getter(conn=conn,
rpath=tpath,
fheaders=fheaders,
retry=nretry)
self.resp_exception(resp=resp, rty=nretry)
# PUT remote object
self._putter(conn=conn,
fpath=tfile,
rpath=tpath,
fheaders=fheaders,
retry=nretry,
skip=True)
# let the system rest for 3 seconds.
basic.stupid_hack(wait=3)
# With the source headers POST new headers on target
if ARGS.get('clone_headers') is True:
resp = self._header_getter(conn=conn,
rpath=tpath,
fheaders=fheaders,
retry=nretry)
theaders = dict(resp.getheaders())
for key in sheaders.keys():
if key not in theaders:
fheaders.update({key: sheaders[key]})
# Force the SOURCE content Type on the Target.
fheaders.update(
{'content-type': sheaders.get('content-type')})
self._header_poster(conn=conn,
rpath=tpath,
fheaders=fheaders,
retry=nretry)
finally:
_cleanup()
def test_time_stamp(self):
fmt, date, date_delta, now = basic_utils.time_stamp()
self.assertEqual('%Y-%m-%dT%H:%M:%S.%f', fmt)
self.assertIsInstance(date, type)
self.assertIsInstance(date_delta, type)
self.assertIsInstance(now, datetime.datetime)
def test_time_stamp(self):
fmt, date, date_delta, now = basic_utils.time_stamp()
self.assertEqual('%Y-%m-%dT%H:%M:%S.%f', fmt)
self.assertIsInstance(date, type)
self.assertIsInstance(date_delta, type)
self.assertIsInstance(now, datetime.datetime)
def object_syncer(self, surl, turl, scontainer, tcontainer, obj):
"""Download an Object from one Container and the upload it to a target.
:param surl:
:param turl:
:param scontainer:
:param tcontainer:
:param obj:
"""
def _cleanup():
"""Ensure that our temp file is removed."""
if locals().get('tfile') is not None:
basic.remove_file(tfile)
def _time_difference(resp, obj):
if ARGS.get('save_newer') is True:
# Get the source object last modified time.
compare_time = resp.getheader('last_modified')
if compare_time is None:
return True
elif cloud.time_delta(compare_time=compare_time,
lmobj=obj['last_modified']) is True:
return False
else:
return True
else:
return True
def _compare(resp, obj):
if resp.status == 404:
report.reporter(
msg='Target Object %s not found' % obj['name'],
prt=False
)
return True
elif resp.getheader('etag') != obj['hash']:
report.reporter(
msg='Checksum Mismatch on Target Object %s' % obj['name'],
prt=False,
lvl='debug'
)
return _time_difference(resp, obj)
else:
return False
fheaders = self.payload['headers']
for retry in basic.retryloop(attempts=ARGS.get('error_retry'),
delay=5,
obj=obj['name']):
# Open connection and perform operation
fmt, date, date_delta, now = basic.time_stamp()
spath = http.quoter(url=surl.path,
cont=scontainer,
ufile=obj['name'])
tpath = http.quoter(url=turl.path,
cont=tcontainer,
ufile=obj['name'])
conn = http.open_connection(url=turl)
with meth.operation(retry, conn=conn, obj=obj):
resp = self._header_getter(conn=conn,
rpath=tpath,
fheaders=fheaders,
retry=retry)
# If object comparison is True GET then PUT object
if _compare(resp=resp, obj=obj) is not True:
return None
try:
# Open Connection for source Download
conn = http.open_connection(url=surl)
with meth.operation(retry,
conn=conn,
obj=obj):
# make a temp file.
tfile = basic.create_tmp()
# Make a connection
resp = self._header_getter(conn=conn,
rpath=spath,
fheaders=fheaders,
retry=retry)
sheaders = dict(resp.getheaders())
# TODO(kevin) add the ability to short upload if timestamp
# TODO(kevin) ... is newer on the target.
# GET remote Object
self._downloader(
conn=conn,
rpath=spath,
fheaders=fheaders,
lfile=tfile,
source=None,
retry=retry,
skip=True
)
for nretry in basic.retryloop(attempts=ARGS.get('error_retry'),
delay=5,
obj=obj):
# open connection for target upload.
conn = http.open_connection(url=turl)
with meth.operation(retry,
conn=conn,
obj=obj,
cleanup=_cleanup):
resp = self._header_getter(conn=conn,
rpath=tpath,
fheaders=fheaders,
retry=nretry)
self.resp_exception(resp=resp, rty=nretry)
# PUT remote object
self._putter(conn=conn,
fpath=tfile,
rpath=tpath,
fheaders=fheaders,
retry=nretry,
skip=True)
# let the system rest for 3 seconds.
basic.stupid_hack(wait=3)
# With the source headers POST new headers on target
if ARGS.get('clone_headers') is True:
resp = self._header_getter(conn=conn,
rpath=tpath,
fheaders=fheaders,
retry=nretry)
theaders = dict(resp.getheaders())
for key in sheaders.keys():
if key not in theaders:
fheaders.update({key: sheaders[key]})
# Force the SOURCE content Type on the Target.
fheaders.update(
{'content-type': sheaders.get('content-type')}
)
self._header_poster(
conn=conn,
rpath=tpath,
fheaders=fheaders,
retry=nretry
)
finally:
_cleanup()