def container_check(self, container_name):
"""
container_name = 'The name of the Container'
check a container to see if it exists
"""
self.connection_prep()
r_loc = '%s/%s' % (self.c_path, container_name)
path = quote(r_loc)
for retry in generators.retryloop(attempts=self.tur_arg['error_retry'],
delay=5):
c_headers = self.headers
# Check to see if the container exists
self.conn.request('HEAD', path, headers=c_headers)
resp_info = self.response_type()
if resp_info[1] is not True:
retry()
resp = resp_info[0]
resp.read()
# Check that the status was a good one
if resp.status == 404:
print('Container Not Found')
if resp.status >= 300 or resp.status == None:
self.result_exception(resp=resp,
headers=c_headers,
authurl=self.url,
jsonreq=path)
retry()
return resp
def object_deleter(self, file_path, container):
"""check a container to see if it exists
:param file_path:
:param container:
"""
for retry in gen.retryloop(attempts=self.retry_atmp, delay=5):
try:
conn = self.connection_prep()
r_loc = '%s/%s/%s' % (self.c_path, container, file_path)
remote_path = urllib.quote(r_loc)
f_headers = self.set_headers()
conn.request('DELETE', remote_path, headers=f_headers)
resp = self.response_get(conn=conn, rty=retry)
if self.result_exception(resp=resp,
authurl=self.url,
jsonreq=remote_path,
dfc=True):
raise exceptions.SystemProblem(resp)
except exceptions.SystemProblem:
retry()
else:
# Give us more data if we requested it
if any([self.tur_arg['os_verbose'], self.tur_arg['debug']]):
print('INFO\t: %s %s %s' % (resp.status,
resp.reason,
file_path))
if self.tur_arg['debug']:
print('MESSAGE\t: Delete path = %s ==> %s'
% (file_path, container))
finally:
conn.close()
def container_deleter(self, container):
"""
container_name = 'The name of the Container'
check a container to see if it exists
"""
self.connection_prep()
r_loc = '%s/%s' % (self.c_path, container)
path = quote(r_loc)
for retry in generators.retryloop(attempts=self.tur_arg['error_retry'],
delay=5):
c_headers = self.headers
# Check to see if the container exists
self.conn.request('DELETE', path, headers=c_headers)
resp_info = self.response_type()
if resp_info[1] is not True:
retry()
resp = resp_info[0]
resp.read()
if resp.status >= 300 or resp.status == None:
self.result_exception(resp=resp,
headers=c_headers,
authurl=self.url,
jsonreq=path)
retry()
# Give us more data if we requested it
if self.tur_arg['os_verbose'] or self.tur_arg['debug']:
print 'INFO\t: %s %s %s' % (resp.status, resp.reason, container)
if self.tur_arg['debug']:
print 'MESSAGE\t: Delete path = %s' % (container)
self.connection_prep(conn_close=True)
def container_check(self, container_name):
"""check a container to see if it exists.
:param container_name: The name of the Container
"""
try:
conn = self.connection_prep()
r_loc = '%s/%s' % (self.c_path, container_name)
path = urllib.quote(r_loc)
for retry in gen.retryloop(attempts=self.retry_atmp, delay=5):
c_headers = self.set_headers()
# Check to see if the container exists
conn.request('HEAD', path, headers=c_headers)
resp = self.response_get(conn=conn, rty=retry)
# Check that the status was a good one
if resp.status == 404:
print('Container Not Found')
elif self.result_exception(resp=resp,
authurl=self.url,
jsonreq=path):
raise exceptions.SystemProblem(resp)
except exceptions.SystemProblem:
retry()
else:
return resp
finally:
conn.close()
def enable_cdn(self, container_name):
"""Enable the CDN on a provided Container.
If custom meta data is specified, and the container exists the method
will put the metadata onto the object.
:param container_name: The name of the Container
"""
cdnurl_data = self.tur_arg['simple_cdn_endpoint'].split('/')
cdnurl = cdnurl_data[0]
cdn_path = urllib.quote('/%s' % ('/'.join(cdnurl_data[1:])))
r_loc = '%s/%s' % (cdn_path, container_name)
path = urllib.quote(r_loc)
c_headers = self.set_headers(cdn=True)
for retry in gen.retryloop(attempts=self.retry_atmp, delay=5):
try:
conn = self.connection(url=cdnurl)
conn.request('PUT', path, headers=c_headers)
resp = self.response_get(conn=conn, rty=retry)
if self.tur_arg['os_verbose']:
print('ENABLING CDN ON CONTAINER: %s %s %s'
% (resp.status, resp.reason, container_name))
if self.result_exception(resp=resp,
authurl=cdnurl,
jsonreq=path):
raise exceptions.SystemProblem(resp)
except exceptions.SystemProblem:
retry()
except Exception as exp:
print('ERROR\t: Shits broke son, here comes the'
' stack trace:\t %s' % traceback.format_exc())
print(exp)
finally:
conn.close()
def get_downloader(self, file_path, file_name, container):
"""
This is the simple download Method. There is no file level checking
at the target. The files are simply downloaded.
"""
try:
for retry in generators.retryloop(
attempts=self.tur_arg['error_retry'], delay=5, backoff=2):
try:
# Set the headers if some custom ones were specified
f_headers = self.headers
if self.tur_arg['object_headers']:
f_headers.update(self.tur_arg['object_headers'])
# Get a file list ready for action
r_loc = '%s/%s/%s' % (self.c_path, container, file_path)
filepath = quote(r_loc)
self.conn.request('GET', filepath, headers=f_headers)
resp_info = self.response_type()
if resp_info[1] is not True:
retry()
resp = resp_info[0]
_rr = resp.read()
# Check that the status was a good one
if resp.status >= 300 or resp.status == None:
self.result_exception(resp=resp,
headers=f_headers,
authurl=self.url,
jsonreq=filepath,
file_path=file_path)
retry()
# Open our source file
with open(file_name, 'wb') as f_name:
f_name.write(_rr)
f_name.close()
# Give us more data if we requested it
if self.tur_arg['os_verbose'] or self.tur_arg['debug']:
print 'INFO\t: %s %s %s' % (resp.status,
resp.reason,
file_name)
if self.tur_arg['debug']:
print('MESSAGE\t: Upload path = %s ==> %s'
% (file_path, filepath))
except Exception, exp:
print(exp)
print('Exception from within an Download Action, placing'
' the failed Download back in Queue')
self.work_q.put(file_path)
except IOError:
print('ERROR\t: path "%s" does not exist or is a broken symlink'
% file_name)
except ValueError:
print('ERROR\t: The data for "%s" got all jacked up, so it'
' got skipped' % file_name)
except KeyboardInterrupt:
pass
def enable_cdn(self, container_name):
"""
container_name = 'The name of the Container'
Enable the CDN on a provided Container. If custom meta data is
specified, and the container exists the
method will put the metadata onto the object.
"""
cdnurl_data = self.tur_arg['simple_cdn_endpoint'].split('/')
cdnurl = cdnurl_data[0]
cdn_path = quote('/%s' % ('/'.join(cdnurl_data[1:])))
r_loc = '%s/%s' % (cdn_path, container_name)
path = quote(r_loc)
c_headers = self.headers
c_headers.update({'X-CDN-Enabled': True,
'X-TTL': self.tur_arg['cdn_ttl'],
'X-Log-Retention': self.tur_arg['cdn_logs']})
for retry in generators.retryloop(attempts=self.tur_arg['error_retry'],
delay=5):
try:
conn = httplib.HTTPSConnection(cdnurl)
for retry in generators.retryloop(
attempts=self.tur_arg['error_retry'],
delay=5):
conn.request('PUT', path, headers=c_headers)
resp_info = self.response_type()
if resp_info[1] is not True:
retry()
resp = resp_info[0]
resp.read()
status_codes = (resp.status, resp.reason, container_name)
if self.tur_arg['os_verbose']:
print('ENABLING CDN ON CONTAINER: %s %s %s'
% status_codes)
if resp.status >= 300 or resp.status == None:
print('Failure happened, will retry %s %s %s'
% status_codes)
retry()
continue
except Exception, exp:
print('ERROR\t: Shits broke son, here comes the'
' stack trace:\t %s' % (sys.exc_info()[1]))
print exp
finally:
def sync_uploader(self, file_path, file_name, container):
"""This is the Sync method which uploads files to the swift repository
if they are not already found. If a file "name" is found locally and
in the swift repository an MD5 comparison is done between the two
files. If the MD5 is miss-matched the local file is uploaded to the
repository. If custom meta data is specified, and the object exists the
method will put the metadata onto the object.
:param file_path:
:param file_name:
:param container:
"""
for retry in gen.retryloop(attempts=self.retry_atmp, delay=5):
try:
conn = self.connection_prep()
f_headers = self.set_headers()
# Get the path ready for action
r_loc = '%s/%s/%s' % (self.c_path, container, file_name)
remote_path = urllib.quote(r_loc)
conn.request('HEAD', remote_path, headers=f_headers)
resp = self.response_get(conn=conn, rty=retry)
if any([resp.status == 404,
self.md5_checker(resp=resp,
local_file=file_path) is True]):
# If different or not found, perform Upload.
self.object_putter(fpath=file_path,
rpath=remote_path,
fname=file_name,
fheaders=f_headers,
retry=retry)
except IOError:
print('ERROR\t: path "%s" does not exist or is a broken'
' symlink' % file_path)
except ValueError:
print('ERROR\t: The data for "%s" got all jacked up,'
' so it got skipped' % file_path)
except KeyboardInterrupt:
pass
except Exception as exp:
print('\nFile Failed to be uploaded %s. Error ==> %s\n\n%s'
% (file_path, exp, traceback.format_exc()))
else:
# Put headers on the object if custom headers
if self.tur_arg['object_headers']:
conn.request('POST',
remote_path,
headers=f_headers)
resp = self.response_get(conn=conn, rty=retry)
if self.result_exception(resp=resp,
authurl=self.url,
jsonreq=remote_path):
retry()
finally:
conn.close()
def container_create(self, container_name):
"""Create a container if the container specified on the command.
If custom meta data is specified, and the container exists the method
will put the metadata onto the object.
:param container_name: The name of the Container
"""
for retry in gen.retryloop(attempts=self.retry_atmp, delay=5):
try:
conn = self.connection_prep()
r_loc = '%s/%s' % (self.c_path, container_name)
path = urllib.quote(r_loc)
c_headers = self.set_headers(ctr=True)
resp = self.container_check(container_name)
status_codes = (resp.status, resp.reason, container_name)
# Check that the status was a good one
if resp.status == 404:
print('Creating Container ==> %s' % container_name)
conn.request('PUT', path, headers=c_headers)
resp = self.response_get(conn=conn, rty=retry)
if resp.status == 404:
print('Container Not Found %s' % resp.status)
elif self.result_exception(resp=resp,
authurl=self.url,
jsonreq=path):
raise exceptions.SystemProblem(resp)
status_codes = (resp.status, resp.reason, container_name)
if self.tur_arg['os_verbose']:
print('CREATING CONTAINER: %s %s %s' % status_codes)
elif self.result_exception(resp=resp,
authurl=self.url,
jsonreq=path):
raise exceptions.SystemProblem(resp)
except exceptions.SystemProblem:
retry()
except Exception as exp:
print('ERROR\t: Shits broke son, here comes the'
' stack trace:\t %s -> Exception\t: %s'
% (traceback.format_exc(), exp))
else:
if self.tur_arg['os_verbose']:
print('Container Found %s %s %s' % status_codes)
# Put headers on the object if custom headers used
if self.tur_arg['object_headers']:
conn.request('POST', path, headers=c_headers)
resp = self.response_get(conn=conn, rty=retry)
if self.result_exception(resp=resp,
authurl=self.url,
jsonreq=path):
retry()
finally:
conn.close()
def make_request(self, jsonreq, url):
"""Make an API request.
:param jsonreq:
:param url:
"""
try:
conn = self.connection(url=url)
for retry in gen.retryloop(attempts=self.retry_atmp,
timeout=960,
delay=5):
if self.tur_arg['os_verbose']:
print('JSON REQUEST: %s' % jsonreq)
conn.set_debuglevel(1)
headers = {'Content-Type': 'application/json'}
tokenurl = '/%s/tokens' % self.tur_arg.get('os_version')
conn.request('POST', tokenurl, jsonreq, headers)
resp, resp_read = self.response_get(conn=conn,
rty=retry,
ret_read=True,
mcr=True)
jrp = json.loads(resp_read)
# Check that the status was a good one
if resp.status >= 500:
print('500 Error => Attempting HTTP connection')
conn = self.connection(url=url, http=True)
raise exceptions.SystemProblem(resp)
elif self.result_exception(resp=resp,
authurl=url,
jsonreq=jsonreq):
raise exceptions.SystemProblem(resp)
else:
if self.tur_arg['os_verbose']:
print('JSON decoded and pretty')
print(json.dumps(jrp, indent=2))
except exceptions.SystemProblem:
retry()
else:
# Send Response to Parser
return self.parse_request(json_response=jrp)
finally:
conn.close()
def response_type(self):
for retry in generators.retryloop(attempts=self.tur_arg['error_retry'],
timeout=960,
delay=5):
try:
# Compatibility with Python 2.6
if sys.version_info < (2, 7, 0):
resp = self.conn.getresponse()
else:
resp = self.conn.getresponse(buffering=True)
return resp, True
except httplib.BadStatusLine:
self.conn.close()
self.connection_prep()
retry()
continue
else:
return None, False
def response_type(self, conn, mcr=False):
"""Understand the response type and provide for the connection.
:param mcr:
:param conn:
"""
for retry in gen.retryloop(attempts=self.retry_atmp,
timeout=960,
delay=5):
try:
resp = conn.getresponse()
except httplib.BadStatusLine as exp:
if mcr is False:
retry()
else:
raise exceptions.SystemProblem('Failed to perform Action'
' %s' % exp)
else:
return resp, True
def put_uploader(self, file_path, file_name, container):
"""This is the simple upload Method.
There is no file level checking at the target.
The files are simply uploaded.
:param file_path:
:param file_name:
:param container:
"""
for retry in gen.retryloop(attempts=self.retry_atmp,
delay=5,
backoff=2):
try:
f_headers = self.set_headers()
# Get the path ready for action
r_loc = '%s/%s/%s' % (self.c_path, container, file_name)
remote_path = urllib.quote(r_loc)
self.object_putter(fpath=file_path,
rpath=remote_path,
fname=file_name,
fheaders=f_headers,
retry=retry)
except IOError:
print('ERROR\t: path "%s" does not exist or is a broken'
' symlink' % file_path)
except ValueError:
print('ERROR\t: The data for "%s" got all jacked up,'
' so it got skipped' % file_path)
except KeyboardInterrupt:
pass
except Exception as exp:
print('\nFile Failed to be uploaded %s. Error ==> %s\n\n%s'
% (file_path, exp, traceback.format_exc()))
else:
if self.tur_arg['debug']:
print('MESSAGE\t: Upload path = %s ==> %s'
% (file_path, remote_path))
def sync_uploader(self, file_path, file_name, container):
"""
This is the Sync method which uploads files to the swift repository'
if they are not already found. If a file "name" is found locally and
in the swift repository an MD5 comparison is done between the two files.
If the MD5 is miss-matched the local file is uploaded to the repository.
If custom meta data is specified, and the object exists the method will
put the metadata onto the object.
"""
#noinspection PyBroadException
try:
for retry in generators.retryloop(
attempts=self.tur_arg['error_retry'], delay=5):
try:
# Set the headers if some custom ones were specified
f_headers = self.headers
if self.tur_arg['object_headers']:
f_headers.update(self.tur_arg['object_headers'])
# Get the path ready for action
r_loc = '%s/%s/%s' % (self.c_path, container, file_name)
filepath = quote(r_loc)
self.conn.request('HEAD', filepath, headers=f_headers)
resp_info = self.response_type()
if resp_info[1] is not True:
retry()
resp = resp_info[0]
resp.read()
if resp.status == 404:
with open(file_path, 'rb') as f_path:
self.conn.request('PUT',
filepath,
body=f_path,
headers=f_headers)
f_path.close()
try:
# Compatibility with Python 2.6
if sys.version_info < (2, 7, 0):
resp = self.conn.getresponse()
else:
resp = self.conn.getresponse(buffering=True)
except httplib.BadStatusLine:
self.conn.close()
self.connection_prep()
retry()
resp.read()
if resp.status >= 300 or resp.status == None:
self.result_exception(resp=resp,
headers=f_headers,
authurl=filepath,
jsonreq=r_loc,
file_path=file_path)
retry()
if self.tur_arg['verbose']:
print 'INFO\t: %s %s %s' % (resp.status,
resp.reason,
file_name)
elif resp.status >= 300 or resp.status == None:
self.result_exception(resp=resp,
headers=f_headers,
authurl=filepath,
jsonreq=r_loc,
file_path=file_path)
retry()
else:
remotemd5sum = resp.getheader('etag')
md5 = hashlib.md5()
with open(file_path, 'rb') as f_hash:
for chunk in iter(lambda: f_hash.read(
128 * md5.block_size), ''):
md5.update(chunk)
f_hash.close()
localmd5sum = md5.hexdigest()
if remotemd5sum != localmd5sum:
with open(file_path, 'rb') as f_path:
self.conn.request('PUT',
filepath,
body=f_path,
headers=f_headers)
f_path.close()
resp_info = self.response_type()
if resp_info[1] is not True:
retry()
resp = resp_info[0]
resp.read()
if resp.status >= 300:
self.result_exception(resp=resp,
headers=f_headers,
authurl=filepath,
jsonreq=r_loc,
file_path=file_path)
retry()
if self.tur_arg['verbose']:
proc_dict = {'lmd5': localmd5sum,
'rmd5': remotemd5sum,
'rs': resp.status,
'rr': resp.reason,
'sjf': file_name}
print('MESSAGE\t: CheckSumm Mis-Match %(lmd5)s'
' != %(rmd5)s\n\t File Upload : %(rs)s'
' %(rr)s %(sjf)s' % proc_dict)
else:
if self.tur_arg['verbose']:
print 'MESSAGE\t: CheckSum Match', localmd5sum
# Put headers on the object if custom headers
if self.tur_arg['object_headers']:
self.conn.request('POST',
filepath,
headers=f_headers)
resp_info = self.response_type()
if resp_info[1] is not True:
retry()
resp = resp_info[0]
resp.read()
if resp.status >= 300:
self.result_exception(resp=resp,
headers=self.headers,
authurl=filepath,
jsonreq=r_loc,
file_path=file_path)
retry()
except Exception:
print(traceback.format_exc())
print('Exception from within an upload Action, placing'
' the failed upload back in Queue')
self.work_q.put(file_path)
except IOError:
print('ERROR\t: path "%s" does not exist or is a broken symlink'
% file_path)
except ValueError:
print('ERROR\t: The data for "%s" got all jacked up,'
' so it got skipped' % file_path)
except KeyboardInterrupt:
pass
def get_object_list(self, container_name, lastobj=None):
"""Builds a long list of files found in a container
:param container_name:
:param lastobj:
"""
for retry in gen.retryloop(attempts=self.retry_atmp,
delay=5,
backoff=2):
try:
conn = self.connection_prep()
f_headers = self.set_headers()
# Determine how many files are in the container
r_loc = '%s/%s' % (self.c_path, container_name)
filepath = urllib.quote(r_loc)
conn.request('HEAD', filepath, headers=f_headers)
resp = self.response_get(conn=conn, rty=retry)
if self.result_exception(resp=resp,
authurl=self.url,
jsonreq=filepath):
raise exceptions.SystemProblem(resp)
count = int(resp.getheader('X-Container-Object-Count'))
# Build the List
file_list = []
# Set the number of loops that we are going to do
jobs = count / 10000 + 1
filepath = '%s/?limit=10000&format=json' % filepath
filepath_m = filepath
f_headers.update({'Content-type': 'application/json'})
for _ in xrange(jobs):
for nest_rty in gen.retryloop(attempts=self.retry_atmp,
delay=5):
conn.request('GET',
filepath,
headers=f_headers)
resp, resp_read = self.response_get(conn=conn,
rty=retry,
ret_read=True)
if self.result_exception(resp=resp,
authurl=self.url,
jsonreq=filepath):
raise exceptions.NoSource(resp)
_rr = json.loads(resp_read)
for obj in _rr:
file_list.append(obj['name'])
if count - 10000 > 0:
count = count - 10000
lastobj = file_list[-1]
filepath = (
'%s&marker=%s' % (filepath_m,
urllib.quote(lastobj))
)
except exceptions.NoSource:
nest_rty()
except exceptions.SystemProblem:
retry()
except KeyboardInterrupt:
pass
except Exception as exp:
print('Exception from within an Download Action\n%s\n%s'
% (traceback.format_exc(), exp))
else:
# Give us more data if we requested it
if any([self.tur_arg['os_verbose'],
self.tur_arg['debug']]):
print('INFO\t: %s %s %s' % (resp.status,
resp.reason,
file_list))
if self.tur_arg['debug']:
print('MESSAGE\t: Path => %s\nMESSAGE\t: %s'
% (filepath, _rr))
return file_list
finally:
conn.close()
def get_object_list(self, container_name):
"""
Builds a long list of files found in a container
"""
lastobj = None
try:
for retry in generators.retryloop(
attempts=self.tur_arg['error_retry'], delay=5, backoff=2):
try:
self.connection_prep()
# Set the headers if some custom ones were specified
c_headers = self.headers
if self.tur_arg['object_headers']:
c_headers.update(self.tur_arg['object_headers'])
# Determine how many files are in the container
r_loc = '%s/%s' % (self.c_path, container_name)
filepath = quote(r_loc)
self.conn.request('HEAD', filepath, headers=c_headers)
resp_info = self.response_type()
if resp_info[1] is not True:
retry()
resp = resp_info[0]
resp.read()
if resp.status >= 300:
self.result_exception(resp=resp,
headers=c_headers,
authurl=self.url,
jsonreq=filepath)
count = int(resp.getheader('X-Container-Object-Count'))
# Build the List
file_list = []
# Set the number of loops that we are going to do
jobs = count / 10000 + 1
filepath = '%s/?limit=10000&format=json' % filepath
filepath_m = filepath
f_headers = self.headers
f_headers.update({'Content-type': 'application/json'})
for _ in xrange(jobs):
self.conn.request('GET', filepath, headers=f_headers)
resp_info = self.response_type()
if resp_info[1] is not True:
retry()
resp = resp_info[0]
if resp.status >= 300:
self.result_exception(resp=resp,
headers=c_headers,
authurl=self.url,
jsonreq=filepath)
_rr = json.loads(resp.read())
for obj in _rr:
file_list.append(obj['name'])
if count - 10000 > 0:
count = count - 10000
lastobj = file_list[-1]
filepath = '%s&marker=%s' % (filepath_m,
quote(lastobj))
# Give us more data if we requested it
if self.tur_arg['os_verbose'] or self.tur_arg['debug']:
print 'INFO\t: %s %s %s' % (resp.status,
resp.reason,
file_list)
if self.tur_arg['debug']:
print('MESSAGE\t: Path => %s'
% (filepath))
print(_rr)
self.connection_prep(conn_close=True)
return file_list
except Exception, exp:
print exp
print traceback.format_exc()
print('Exception from within an Download Action')
except KeyboardInterrupt:
pass
def container_create(self, container_name):
"""
container_name = 'The name of the Container'
Create a container if the container specified on the command
line did not already exist. If custom meta data is specified,
and the container exists the method will put the metadata
onto the object.
"""
r_loc = '%s/%s' % (self.c_path, container_name)
path = quote(r_loc)
try:
for retry in generators.retryloop(
attempts=self.tur_arg['error_retry'], delay=5):
c_headers = self.headers
if self.tur_arg['container_headers']:
c_headers.update(self.tur_arg['container_headers'])
resp = self.container_check(container_name)
status_codes = (resp.status, resp.reason, container_name)
# Check that the status was a good one
if resp.status == 404:
self.conn.request('PUT', path, headers=c_headers)
resp_info = self.response_type()
if resp_info[1] is not True:
retry()
resp = resp_info[0]
resp.read()
if resp.status >= 300 or resp.status == None:
self.result_exception(resp=resp,
headers=c_headers,
authurl=self.url,
jsonreq=path)
retry()
status_codes = (resp.status, resp.reason, container_name)
if self.tur_arg['os_verbose']:
print('CREATING CONTAINER: %s %s %s' % status_codes)
elif resp.status >= 300 or resp.status == None:
self.result_exception(resp=resp,
headers=c_headers,
authurl=self.url,
jsonreq=path)
retry()
else:
if self.tur_arg['os_verbose']:
print('Container Found %s %s %s' % status_codes)
# Put headers on the object if custom headers were specified
if self.tur_arg['object_headers']:
self.conn.request('POST', path, headers=c_headers)
resp_info = self.response_type()
if resp_info[1] is not True:
retry()
resp = resp_info[0]
resp.read()
if resp.status >= 300:
self.result_exception(resp=resp,
headers=c_headers,
authurl=self.url,
jsonreq=path)
retry()
except Exception, exp:
print('ERROR\t: Shits broke son, here comes the'
' stack trace:\t %s -> Exception\t: %s'
% (sys.exc_info()[1], exp))
def get_downloader(self, file_path, file_name, container):
"""This is the simple download Method.
There is no file level checking at the target. The files are simply
downloaded.
:param file_path:
:param file_name:
:param container:
"""
def get_action(conn, filepath, local_file, headers, url, retry):
"""Get a target file and save it locally.
:param conn:
:param filepath:
:param file_name:
:param headers:
:param url:
:param retry:
"""
conn.request('GET', filepath, headers=headers)
resp, resp_read = self.response_get(conn=conn,
rty=retry,
ret_read=True)
# Check that the status was a good one
if self.result_exception(resp=resp,
authurl=url,
jsonreq=filepath):
raise exceptions.SystemProblem(resp)
# Open our source file and write it
with open(local_file, 'wb') as f_name:
f_name.write(resp_read)
# Give us more data if we requested it
if self.tur_arg.get('verbose'):
print('INFO\t: %s %s %s' % (resp.status,
resp.reason,
local_file))
if self.tur_arg.get('debug'):
print('MESSAGE\t: Download path = %s ==> %s'
% (file_name, filepath))
for retry in gen.retryloop(attempts=self.retry_atmp,
delay=5,
backoff=1):
try:
conn = self.connection_prep()
f_headers = self.set_headers()
# Get a file list ready for action
remote_path = '%s/%s/%s' % (self.c_path, container, file_path)
filepath = urllib.quote(remote_path)
if self.tur_arg.get('dl_sync') is True:
conn.request('HEAD', remote_path, headers=f_headers)
resp = self.response_get(conn=conn, rty=retry)
if self.md5_checker(resp=resp,
local_file=file_name) is True:
get_action(conn=conn,
filepath=filepath,
local_file=file_name,
headers=f_headers,
url=self.url,
retry=retry)
else:
get_action(conn=conn,
filepath=filepath,
local_file=file_name,
headers=f_headers,
url=self.url,
retry=retry)
except exceptions.SystemProblem:
retry()
except IOError:
print('ERROR\t: path "%s" does not exist or is a broken'
' symlink' % file_name)
except ValueError:
print('ERROR\t: The data for "%s" got all jacked up, so it'
' got skipped' % file_name)
except KeyboardInterrupt:
pass
except Exception as exp:
print traceback.format_exc()
print('ERROR\t: Exception from within an Download Action '
'Message == %s' % exp)
finally:
conn.close()
def osauth(self):
"""
Authentication For Openstack API, Pulls the full Openstack Service
Catalog Credentials are the Users API Username and Key/Password
"osauth" has a Built in Rackspace Method for Authentication
Set a DC Endpoint and Authentication URL for the Open Stack environment
"""
rax_us_dc = (self.tur_arg['os_rax_auth'] == 'DFW',
self.tur_arg['os_rax_auth'] == 'ORD')
rax_eu_dc = (self.tur_arg['os_rax_auth'] == 'LON',
self.tur_arg['os_rax_auth'] == 'NEWDC')
if any(rax_eu_dc):
self.tur_arg['os_region'] = self.tur_arg['os_rax_auth']
if self.tur_arg['os_auth_url']:
authurl = self.tur_arg['os_auth_url']
else:
authurl = 'lon.identity.api.rackspacecloud.com'
elif any(rax_us_dc):
self.tur_arg['os_region'] = self.tur_arg['os_rax_auth']
if self.tur_arg['os_auth_url']:
authurl = self.tur_arg['os_auth_url']
else:
authurl = 'identity.api.rackspacecloud.com'
else:
if not self.tur_arg['os_region']:
sys.exit('FAIL\t: You have to specify '
'a Region along with an Auth URL')
if self.tur_arg['os_auth_url']:
authurl = self.tur_arg['os_auth_url']
else:
sys.exit('FAIL\t: You have to specify an Auth URL'
' along with the Region')
if self.tur_arg['os_apikey'] or self.tur_arg['os_rax_auth']:
jsonreq = json.dumps({'auth': {'RAX-KSKEY:apiKeyCredentials':
{'username': self.tur_arg['os_user'],
'apiKey': self.tur_arg['os_apikey']}}})
else:
jsonreq = json.dumps({'auth': {'passwordCredentials':
{'username': self.tur_arg['os_user'],
'password': self.tur_arg['os_password']}}})
authurl = self.tur_arg['os_auth_url']
# remove the prefix for the Authentication URL
authurl = authurl.strip('http?s://')
url_data = authurl.split('/')
url = url_data[0]
for retry in generators.retryloop(attempts=self.tur_arg['error_retry'],
timeout=960,
delay=5):
conn = httplib.HTTPSConnection(url)
if self.tur_arg['os_verbose']:
print('JSON REQUEST: %s' % jsonreq)
conn.set_debuglevel(1)
headers = {'Content-Type': 'application/json'}
tokenurl = '/%s/tokens' % self.tur_arg['os_version']
conn.request('POST', tokenurl, jsonreq, headers)
try:
# Compatibility with Python 2.6
if sys.version_info < (2, 7, 0):
resp = conn.getresponse()
else:
resp = conn.getresponse(buffering=True)
except httplib.BadStatusLine:
conn.close()
retry()
readresp = resp.read()
json_response = json.loads(readresp)
conn.close()
# Check that the status was a good one
if resp.status >= 500:
print('Attempting HTTP connection')
conn = httplib.HTTPConnection(url)
retry()
elif resp.status >= 300 or resp.status == None:
self.result_exception(resp=resp,
headers=self.headers,
authurl=self.url,
jsonreq=self.c_path)
try:
retry()
except Exception, exp:
print('Authentication has FAILED "%s %s %s %s"'
% (resp.status,
resp.reason,
readresp,
exp))
continue
else:
if self.tur_arg['os_verbose']:
print('JSON decoded and pretty')
print json.dumps(json_response, indent=2)
try:
for service in json_response['access']['serviceCatalog']:
if service['name'] == 'cloudFiles':
for _ep in service['endpoints']:
if _ep['region'] == self.tur_arg['os_region']:
if self.tur_arg['internal']:
endpt = _ep['internalURL']
self.tur_arg['endpoint'] = endpt
else:
self.tur_arg['endpoint'] = _ep['publicURL']
elif service['name'] == 'swift':
for _ep in service['endpoints']:
if _ep['region'] == self.tur_arg['os_region']:
if self.tur_arg['internal']:
endpt = _ep['internalURL']
self.tur_arg['endpoint'] = endpt
else:
self.tur_arg['endpoint'] = _ep['publicURL']
if service['name'] == 'cloudFilesCDN':
for _ep in service['endpoints']:
if _ep['region'] == self.tur_arg['os_region']:
self.tur_arg['CDNendpoint'] = _ep['publicURL']
tenant_id = json_response['access']['token']['tenant']['id']
self.tur_arg['tenantid'] = tenant_id
token = json_response['access']['token']['id']
headers = self.tur_arg['base_headers']
headers.update({'X-Auth-Token': token})
self.tur_arg['base_headers'] = headers
cdn_encode = self.tur_arg['CDNendpoint'].encode('utf8')
cdn_split = cdn_encode.split('//')[1]
self.tur_arg['simple_cdn_endpoint'] = cdn_split
url_encode = self.tur_arg['endpoint'].encode('utf8')
url_split = url_encode.split('//')[1]
self.tur_arg['simple_endpoint'] = url_split
if self.tur_arg['os_verbose']:
print('SimpleURL\t: %s'
'\nPublicURL\t: %s'
'\nTenant\t\t: %s'
'\nCDN_manage\t: %s' % (url_split,
self.tur_arg['endpoint'],
tenant_id,
cdn_encode))
return self.tur_arg
except (KeyError, IndexError):
print('Error while getting answers from auth server.'
' Check the endpoint and auth credentials.')
