def job_processer(num_jobs, objects, job_action, concur, kwargs=None,
opt=None):
"""Process all jobs in batches.
:param num_jobs:
:param objects:
:param job_action:
:param concur:
:param payload:
"""
count = 0
batch_size = basic.batcher(num_files=num_jobs)
for work in basic.batch_gen(data=objects,
batch_size=batch_size,
count=num_jobs):
count += 1
report.reporter(msg='Job Count %s' % count)
work_q = basic_queue(work)
with spinner(work_q=work_q):
worker_proc(job_action=job_action,
concurrency=concur,
queue=work_q,
opt=opt,
kwargs=kwargs)
basic.stupid_hack(wait=1)
work_q.close()
def job_processer(num_jobs, objects, job_action, concur, kwargs=None,
opt=None):
"""Process all jobs in batches.
:param num_jobs:
:param objects:
:param job_action:
:param concur:
:param payload:
"""
count = 0
batch_size = basic.batcher(num_files=num_jobs)
while objects:
count += 1
report.reporter(msg='Job Count %s' % count)
work = [
objects.pop(objects.index(obj)) for obj in objects[0:batch_size]
]
work_q = basic_queue(work)
with spinner(work_q=work_q):
worker_proc(
job_action=job_action,
concurrency=concur,
queue=work_q,
opt=opt,
kwargs=kwargs
)
basic.stupid_hack(wait=.2)
work_q.close()
def resp_exception(self, resp, rty):
"""If we encounter an exception in our upload.
we will look at how we can attempt to resolve the exception.
:param resp:
:param rty:
"""
try:
if resp.status == 401:
report.reporter(
msg='MESSAGE: Forced Re-authentication is happening.',
lvl='error',
log=True
)
basic.stupid_hack()
self.payload['headers']['X-Auth-Token'] = auth.get_new_token()
rty()
elif resp.status == 404:
report.reporter(
msg=('Not found STATUS: %s, REASON: %s, MESSAGE: %s'
% (resp.status, resp.reason, resp.msg)),
prt=False,
lvl='debug'
)
elif resp.status == 413:
_di = dict(resp.getheaders())
basic.stupid_hack(wait=_di.get('retry_after', 10))
raise turbo.SystemProblem(
'The System encountered an API limitation and will'
' continue in "%s" Seconds' % _di.get('retry_after')
)
elif resp.status == 502:
raise turbo.SystemProblem('Failure making Connection')
elif resp.status == 503:
basic.stupid_hack(wait=10)
raise turbo.SystemProblem('SWIFT-API FAILURE')
elif resp.status == 504:
basic.stupid_hack(wait=10)
raise turbo.SystemProblem('Gateway Time-out')
elif resp.status >= 300:
raise turbo.SystemProblem('SWIFT-API FAILURE -> REQUEST')
except turbo.SystemProblem as exp:
report.reporter(
msg=('FAIL-MESSAGE %s FAILURE STATUS %s FAILURE REASON %s '
'TYPE %s MESSAGE %s' % (exp, resp.status, resp.reason,
resp._method, resp.msg)),
prt=True,
lvl='warn',
log=True
)
rty()
def resp_exception(self, resp):
"""If we encounter an exception in our upload.
we will look at how we can attempt to resolve the exception.
:param resp:
"""
# Check to make sure we have all the bits needed
if not hasattr(resp, 'status_code'):
raise turbo.SystemProblem('No Status to check.')
elif resp is None:
raise turbo.SystemProblem('No response information.')
elif resp.status_code == 401:
report.reporter(
msg=('Turbolift experienced an Authentication issue.'
' STATUS %s REASON %s REQUEST %s. Turbolift will retry' %
(resp.status_code, resp.reason, resp.request)),
lvl='warn',
log=True,
prt=False)
# This was done in this manor due to how manager dicts are proxied
# related : http://bugs.python.org/issue6766
headers = self.payload['headers']
headers['X-Auth-Token'] = get_new_token()
self.payload['headers'] = headers
raise turbo.AuthenticationProblem(
'Attempting to resolve the Authentication issue.')
elif resp.status_code == 404:
report.reporter(
msg=('Not found STATUS: %s, REASON: %s, MESSAGE: %s' %
(resp.status_code, resp.reason, resp.request)),
prt=False,
lvl='debug')
elif resp.status_code == 413:
_di = resp.headers
basic.stupid_hack(wait=_di.get('retry_after', 10))
raise turbo.SystemProblem(
'The System encountered an API limitation and will'
' continue in "%s" Seconds' % _di.get('retry_after'))
elif resp.status_code == 502:
raise turbo.SystemProblem('Failure making Connection')
elif resp.status_code == 503:
basic.stupid_hack(wait=10)
raise turbo.SystemProblem('SWIFT-API FAILURE')
elif resp.status_code == 504:
basic.stupid_hack(wait=10)
raise turbo.SystemProblem('Gateway Time-out')
elif resp.status_code >= 300:
raise turbo.SystemProblem(
'SWIFT-API FAILURE -> REASON %s REQUEST %s' %
(resp.reason, resp.request))
else:
report.reporter(
msg=('MESSAGE %s %s %s' %
(resp.status_code, resp.reason, resp.request)),
prt=False,
lvl='debug')
def resp_exception(self, resp, rty):
"""If we encounter an exception in our upload.
we will look at how we can attempt to resolve the exception.
:param resp:
:param rty:
"""
try:
if resp.status == 401:
report.reporter(
msg='MESSAGE: Forced Re-authentication is happening.',
lvl='error',
log=True)
basic.stupid_hack()
self.payload['headers']['X-Auth-Token'] = auth.get_new_token()
rty()
elif resp.status == 404:
report.reporter(
msg=('Not found STATUS: %s, REASON: %s, MESSAGE: %s' %
(resp.status, resp.reason, resp.msg)),
prt=False,
lvl='debug')
elif resp.status == 413:
_di = dict(resp.getheaders())
basic.stupid_hack(wait=_di.get('retry_after', 10))
raise turbo.SystemProblem(
'The System encountered an API limitation and will'
' continue in "%s" Seconds' % _di.get('retry_after'))
elif resp.status == 502:
raise turbo.SystemProblem('Failure making Connection')
elif resp.status == 503:
basic.stupid_hack(wait=10)
raise turbo.SystemProblem('SWIFT-API FAILURE')
elif resp.status == 504:
basic.stupid_hack(wait=10)
raise turbo.SystemProblem('Gateway Time-out')
elif resp.status >= 300:
raise turbo.SystemProblem('SWIFT-API FAILURE -> REQUEST')
except turbo.SystemProblem as exp:
report.reporter(
msg=('FAIL-MESSAGE %s FAILURE STATUS %s FAILURE REASON %s '
'TYPE %s MESSAGE %s' %
(exp, resp.status, resp.reason, resp._method, resp.msg)),
prt=True,
lvl='warn',
log=True)
rty()
def resp_exception(self, resp):
"""If we encounter an exception in our upload.
we will look at how we can attempt to resolve the exception.
:param resp:
"""
# Check to make sure we have all the bits needed
if not hasattr(resp, "status_code"):
raise turbo.SystemProblem("No Status to check.")
elif resp is None:
raise turbo.SystemProblem("No response information.")
elif resp.status_code == 401:
report.reporter(
msg=(
"Turbolift experienced an Authentication issue."
" STATUS %s REASON %s REQUEST %s. Turbolift will retry"
% (resp.status_code, resp.reason, resp.request)
),
lvl="warn",
log=True,
prt=False,
)
# This was done in this manor due to how manager dicts are proxied
# related : http://bugs.python.org/issue6766
headers = self.payload["headers"]
headers["X-Auth-Token"] = get_new_token()
self.payload["headers"] = headers
raise turbo.AuthenticationProblem("Attempting to resolve the Authentication issue.")
elif resp.status_code == 404:
report.reporter(
msg=("Not found STATUS: %s, REASON: %s, MESSAGE: %s" % (resp.status_code, resp.reason, resp.request)),
prt=False,
lvl="debug",
)
elif resp.status_code == 413:
_di = resp.headers
basic.stupid_hack(wait=_di.get("retry_after", 10))
raise turbo.SystemProblem(
"The System encountered an API limitation and will" ' continue in "%s" Seconds' % _di.get("retry_after")
)
elif resp.status_code == 502:
raise turbo.SystemProblem("Failure making Connection")
elif resp.status_code == 503:
basic.stupid_hack(wait=10)
raise turbo.SystemProblem("SWIFT-API FAILURE")
elif resp.status_code == 504:
basic.stupid_hack(wait=10)
raise turbo.SystemProblem("Gateway Time-out")
elif resp.status_code >= 300:
raise turbo.SystemProblem("SWIFT-API FAILURE -> REASON %s REQUEST %s" % (resp.reason, resp.request))
else:
report.reporter(
msg=("MESSAGE %s %s %s" % (resp.status_code, resp.reason, resp.request)), prt=False, lvl="debug"
)
def worker_proc(job_action, concurrency, queue, kwargs, opt):
"""Requires the job_action and num_jobs variables for functionality.
:param job_action: What function will be used
:param concurrency: The number of jobs that will be processed
:param queue: The Queue
:param t_args: Optional
All threads produced by the worker are limited by the number of concurrency
specified by the user. The Threads are all made active prior to them
processing jobs.
"""
arguments = []
for item in [queue, opt, kwargs]:
if item is not None:
arguments.append(item)
jobs = [multiprocessing.Process(target=job_action,
args=tuple(arguments))
for _ in xrange(concurrency)]
report.reporter(
msg='Tread Starting Cycle',
lvl='info',
log=True,
prt=True
)
join_jobs = []
for _job in jobs:
basic.stupid_hack(wait=.01)
join_jobs.append(_job)
_job.Daemon = True
_job.start()
for job in join_jobs:
job.join()
def object_syncer(self, surl, turl, scontainer, tcontainer, obj):
"""Download an Object from one Container and the upload it to a target.
:param surl:
:param turl:
:param scontainer:
:param tcontainer:
:param obj:
"""
def _cleanup():
"""Ensure that our temp file is removed."""
if locals().get('tfile') is not None:
basic.remove_file(tfile)
def _time_difference(resp, obj):
if ARGS.get('save_newer') is True:
# Get the source object last modified time.
compare_time = resp.getheader('last_modified')
if compare_time is None:
return True
elif cloud.time_delta(compare_time=compare_time,
lmobj=obj['last_modified']) is True:
return False
else:
return True
else:
return True
def _compare(resp, obj):
if resp.status == 404:
report.reporter(msg='Target Object %s not found' % obj['name'],
prt=False)
return True
elif resp.getheader('etag') != obj['hash']:
report.reporter(msg='Checksum Mismatch on Target Object %s' %
obj['name'],
prt=False,
lvl='debug')
return _time_difference(resp, obj)
else:
return False
fheaders = self.payload['headers']
for retry in basic.retryloop(attempts=ARGS.get('error_retry'),
delay=5,
obj=obj['name']):
# Open connection and perform operation
fmt, date, date_delta, now = basic.time_stamp()
spath = http.quoter(url=surl.path,
cont=scontainer,
ufile=obj['name'])
tpath = http.quoter(url=turl.path,
cont=tcontainer,
ufile=obj['name'])
conn = http.open_connection(url=turl)
with meth.operation(retry, conn=conn, obj=obj):
resp = self._header_getter(conn=conn,
rpath=tpath,
fheaders=fheaders,
retry=retry)
# If object comparison is True GET then PUT object
if _compare(resp=resp, obj=obj) is not True:
return None
try:
# Open Connection for source Download
conn = http.open_connection(url=surl)
with meth.operation(retry, conn=conn, obj=obj):
# make a temp file.
tfile = basic.create_tmp()
# Make a connection
resp = self._header_getter(conn=conn,
rpath=spath,
fheaders=fheaders,
retry=retry)
sheaders = dict(resp.getheaders())
# TODO(kevin) add the ability to short upload if timestamp
# TODO(kevin) ... is newer on the target.
# GET remote Object
self._downloader(conn=conn,
rpath=spath,
fheaders=fheaders,
lfile=tfile,
source=None,
retry=retry,
skip=True)
for nretry in basic.retryloop(attempts=ARGS.get('error_retry'),
delay=5,
obj=obj):
# open connection for target upload.
conn = http.open_connection(url=turl)
with meth.operation(retry,
conn=conn,
obj=obj,
cleanup=_cleanup):
resp = self._header_getter(conn=conn,
rpath=tpath,
fheaders=fheaders,
retry=nretry)
self.resp_exception(resp=resp, rty=nretry)
# PUT remote object
self._putter(conn=conn,
fpath=tfile,
rpath=tpath,
fheaders=fheaders,
retry=nretry,
skip=True)
# let the system rest for 3 seconds.
basic.stupid_hack(wait=3)
# With the source headers POST new headers on target
if ARGS.get('clone_headers') is True:
resp = self._header_getter(conn=conn,
rpath=tpath,
fheaders=fheaders,
retry=nretry)
theaders = dict(resp.getheaders())
for key in sheaders.keys():
if key not in theaders:
fheaders.update({key: sheaders[key]})
# Force the SOURCE content Type on the Target.
fheaders.update(
{'content-type': sheaders.get('content-type')})
self._header_poster(conn=conn,
rpath=tpath,
fheaders=fheaders,
retry=nretry)
finally:
_cleanup()
def object_syncer(self, surl, turl, scontainer, tcontainer, u_file):
"""Download an Object from one Container and the upload it to a target.
:param surl:
:param turl:
:param scontainer:
:param tcontainer:
:param u_file:
"""
def _cleanup():
"""Ensure that our temp file is removed."""
if locals().get('tfile') is not None:
basic.remove_file(tfile)
def _time_difference(obj_resp, obj):
if ARGS.get('save_newer') is True:
# Get the source object last modified time.
compare_time = obj_resp.header.get('last_modified')
if compare_time is None:
return True
elif cloud.time_delta(compare_time=compare_time,
lmobj=obj['last_modified']) is True:
return False
else:
return True
else:
return True
def _compare(obj_resp, obj):
if obj_resp.status_code == 404:
report.reporter(msg='Target Object %s not found' % obj['name'],
prt=False)
return True
elif ARGS.get('add_only'):
report.reporter(msg='Target Object %s already exists' %
obj['name'],
prt=True)
return False
elif obj_resp.headers.get('etag') != obj['hash']:
report.reporter(msg=('Checksum Mismatch on Target Object %s' %
u_file['name']),
prt=False,
lvl='debug')
return _time_difference(obj_resp, obj)
else:
return False
fheaders = self.payload['headers']
for retry in basic.retryloop(attempts=ARGS.get('error_retry'),
delay=5,
obj=u_file['name']):
# Open connection and perform operation
spath = http.quoter(url=surl.path,
cont=scontainer,
ufile=u_file['name'])
tpath = http.quoter(url=turl.path,
cont=tcontainer,
ufile=u_file['name'])
with meth.operation(retry, obj='%s %s' % (fheaders, tpath)):
resp = self._header_getter(url=turl,
rpath=tpath,
fheaders=fheaders)
# If object comparison is True GET then PUT object
if _compare(resp, u_file) is not True:
return None
try:
# Open Connection for source Download
with meth.operation(retry, obj='%s %s' % (fheaders, spath)):
# make a temp file.
tfile = basic.create_tmp()
# Make a connection
resp = self._header_getter(url=surl,
rpath=spath,
fheaders=fheaders)
sheaders = resp.headers
self._downloader(url=surl,
rpath=spath,
fheaders=fheaders,
lfile=tfile,
source=None,
skip=True)
for _retry in basic.retryloop(attempts=ARGS.get('error_retry'),
delay=5,
obj=u_file):
# open connection for target upload.
adddata = '%s %s' % (fheaders, u_file)
with meth.operation(_retry, obj=adddata, cleanup=_cleanup):
resp = self._header_getter(url=turl,
rpath=tpath,
fheaders=fheaders)
self.resp_exception(resp=resp)
# PUT remote object
self._putter(url=turl,
fpath=tfile,
rpath=tpath,
fheaders=fheaders,
skip=True)
# let the system rest for 1 seconds.
basic.stupid_hack(wait=1)
# With the source headers POST new headers on target
if ARGS.get('clone_headers') is True:
theaders = resp.headers
for key in sheaders.keys():
if key not in theaders:
fheaders.update({key: sheaders[key]})
# Force the SOURCE content Type on the Target.
fheaders.update(
{'content-type': sheaders.get('content-type')})
self._header_poster(url=turl,
rpath=tpath,
fheaders=fheaders)
finally:
_cleanup()
def object_syncer(self, surl, turl, scontainer, tcontainer, obj):
"""Download an Object from one Container and the upload it to a target.
:param surl:
:param turl:
:param scontainer:
:param tcontainer:
:param obj:
"""
def _cleanup():
"""Ensure that our temp file is removed."""
if locals().get('tfile') is not None:
basic.remove_file(tfile)
def _time_difference(resp, obj):
if ARGS.get('save_newer') is True:
# Get the source object last modified time.
compare_time = resp.getheader('last_modified')
if compare_time is None:
return True
elif cloud.time_delta(compare_time=compare_time,
lmobj=obj['last_modified']) is True:
return False
else:
return True
else:
return True
def _compare(resp, obj):
if resp.status == 404:
report.reporter(
msg='Target Object %s not found' % obj['name'],
prt=False
)
return True
elif resp.getheader('etag') != obj['hash']:
report.reporter(
msg='Checksum Mismatch on Target Object %s' % obj['name'],
prt=False,
lvl='debug'
)
return _time_difference(resp, obj)
else:
return False
fheaders = self.payload['headers']
for retry in basic.retryloop(attempts=ARGS.get('error_retry'),
delay=5,
obj=obj['name']):
# Open connection and perform operation
fmt, date, date_delta, now = basic.time_stamp()
spath = http.quoter(url=surl.path,
cont=scontainer,
ufile=obj['name'])
tpath = http.quoter(url=turl.path,
cont=tcontainer,
ufile=obj['name'])
conn = http.open_connection(url=turl)
with meth.operation(retry, conn=conn, obj=obj):
resp = self._header_getter(conn=conn,
rpath=tpath,
fheaders=fheaders,
retry=retry)
# If object comparison is True GET then PUT object
if _compare(resp=resp, obj=obj) is not True:
return None
try:
# Open Connection for source Download
conn = http.open_connection(url=surl)
with meth.operation(retry,
conn=conn,
obj=obj):
# make a temp file.
tfile = basic.create_tmp()
# Make a connection
resp = self._header_getter(conn=conn,
rpath=spath,
fheaders=fheaders,
retry=retry)
sheaders = dict(resp.getheaders())
# TODO(kevin) add the ability to short upload if timestamp
# TODO(kevin) ... is newer on the target.
# GET remote Object
self._downloader(
conn=conn,
rpath=spath,
fheaders=fheaders,
lfile=tfile,
source=None,
retry=retry,
skip=True
)
for nretry in basic.retryloop(attempts=ARGS.get('error_retry'),
delay=5,
obj=obj):
# open connection for target upload.
conn = http.open_connection(url=turl)
with meth.operation(retry,
conn=conn,
obj=obj,
cleanup=_cleanup):
resp = self._header_getter(conn=conn,
rpath=tpath,
fheaders=fheaders,
retry=nretry)
self.resp_exception(resp=resp, rty=nretry)
# PUT remote object
self._putter(conn=conn,
fpath=tfile,
rpath=tpath,
fheaders=fheaders,
retry=nretry,
skip=True)
# let the system rest for 3 seconds.
basic.stupid_hack(wait=3)
# With the source headers POST new headers on target
if ARGS.get('clone_headers') is True:
resp = self._header_getter(conn=conn,
rpath=tpath,
fheaders=fheaders,
retry=nretry)
theaders = dict(resp.getheaders())
for key in sheaders.keys():
if key not in theaders:
fheaders.update({key: sheaders[key]})
# Force the SOURCE content Type on the Target.
fheaders.update(
{'content-type': sheaders.get('content-type')}
)
self._header_poster(
conn=conn,
rpath=tpath,
fheaders=fheaders,
retry=nretry
)
finally:
_cleanup()
def object_syncer(self, surl, turl, scontainer, tcontainer, u_file):
"""Download an Object from one Container and the upload it to a target.
:param surl:
:param turl:
:param scontainer:
:param tcontainer:
:param u_file:
"""
def _cleanup():
"""Ensure that our temp file is removed."""
if locals().get("tfile") is not None:
basic.remove_file(tfile)
def _time_difference(obj_resp, obj):
if ARGS.get("save_newer") is True:
# Get the source object last modified time.
compare_time = obj_resp.header.get("last_modified")
if compare_time is None:
return True
elif cloud.time_delta(compare_time=compare_time, lmobj=obj["last_modified"]) is True:
return False
else:
return True
else:
return True
def _compare(obj_resp, obj):
if obj_resp.status_code == 404:
report.reporter(msg="Target Object %s not found" % obj["name"], prt=False)
return True
elif ARGS.get("add_only"):
report.reporter(msg="Target Object %s already exists" % obj["name"], prt=True)
return False
elif obj_resp.headers.get("etag") != obj["hash"]:
report.reporter(msg=("Checksum Mismatch on Target Object %s" % u_file["name"]), prt=False, lvl="debug")
return _time_difference(obj_resp, obj)
else:
return False
fheaders = self.payload["headers"]
for retry in basic.retryloop(attempts=ARGS.get("error_retry"), delay=5, obj=u_file["name"]):
# Open connection and perform operation
spath = http.quoter(url=surl.path, cont=scontainer, ufile=u_file["name"])
tpath = http.quoter(url=turl.path, cont=tcontainer, ufile=u_file["name"])
with meth.operation(retry, obj="%s %s" % (fheaders, tpath)):
resp = self._header_getter(url=turl, rpath=tpath, fheaders=fheaders)
# If object comparison is True GET then PUT object
if _compare(resp, u_file) is not True:
return None
try:
# Open Connection for source Download
with meth.operation(retry, obj="%s %s" % (fheaders, spath)):
# make a temp file.
tfile = basic.create_tmp()
# Make a connection
resp = self._header_getter(url=surl, rpath=spath, fheaders=fheaders)
sheaders = resp.headers
self._downloader(url=surl, rpath=spath, fheaders=fheaders, lfile=tfile, source=None, skip=True)
for _retry in basic.retryloop(attempts=ARGS.get("error_retry"), delay=5, obj=u_file):
# open connection for target upload.
adddata = "%s %s" % (fheaders, u_file)
with meth.operation(_retry, obj=adddata, cleanup=_cleanup):
resp = self._header_getter(url=turl, rpath=tpath, fheaders=fheaders)
self.resp_exception(resp=resp)
# PUT remote object
self._putter(url=turl, fpath=tfile, rpath=tpath, fheaders=fheaders, skip=True)
# let the system rest for 1 seconds.
basic.stupid_hack(wait=1)
# With the source headers POST new headers on target
if ARGS.get("clone_headers") is True:
theaders = resp.headers
for key in sheaders.keys():
if key not in theaders:
fheaders.update({key: sheaders[key]})
# Force the SOURCE content Type on the Target.
fheaders.update({"content-type": sheaders.get("content-type")})
self._header_poster(url=turl, rpath=tpath, fheaders=fheaders)
finally:
_cleanup()
