def run(innerself):
innerself.starttime = datetime.datetime.utcnow()
innerself.reporttime = innerself.starttime
try:
client = SwiftProvider.getConnection(
innerself.clientParams)
with open(innerself.toFilename, 'w') as openFile:
SwiftProvider.doSwiftDownload(
client,
innerself.container,
innerself.fromFilename,
openFile,
progress=innerself._progressReport)
innerself.succeeded = True
except swiftclient.exceptions.ClientException as e:
os.remove(innerself.toFilename)
if e.http_status == 404 or e.http_status == 409:
self.log.exception(
"Download of object '%s' returned %d",
innerself.fromFilename, e.http_status)
self.log.warning(
"Step will not fail as this file was likely but not yet removed from the container"
)
innerself.succeeded = True
else:
self.log.exception("Download of object '%s' failed")
innerself.httpstatus = e.http_status
except:
os.remove(innerself.toFilename)
self.log.exception("Download of object '%s' failed",
innerself.fromFilename)
innerself.succeeded = False
return innerself.succeeded
def prep(self, options):
(clientParams,
account) = SwiftProvider.getAccount(self.engine, options['account'],
'prep')
client = SwiftProvider.getConnection(clientParams)
if not self._isNoop(options):
SwiftProvider.ensureContainer(client, options['to-container'])
self.log.passed()
return True
def push(self, options):
#Execute the account section to get the account info
(clientParams,
account) = SwiftProvider.getAccount(self.engine, options['account'],
'push')
client = SwiftProvider.getConnection(clientParams)
containerMatcher = ','.join(
options['containers'].split('\n')).split(',')
targetedContainers = SwiftProvider.getAccountListing(
client, containerMatcher)
for ctrname, container in targetedContainers.iteritems():
targetedObjects = SwiftProvider.getContainerListing(
client, ctrname)
self.log.debug("In Container: %s", ctrname)
containerlen = len(targetedObjects)
self.log.debug("Items in container to examine: %d", containerlen)
for objectName, objectShort in targetedObjects.iteritems():
objectDetails = client.head_object(ctrname, objectName)
if 'x-object-manifest' in objectDetails:
#Get the pointer(s) to the real object(s)
manifest = objectDetails['x-object-manifest']
if '/' not in manifest:
self.log.info(
"The object manifest for '%s' is not well formed: %s",
objectName, manifest)
self.log.info(" --- skipping")
continue
realcontainer, realobject = manifest.split('/', 1)
realobjects = SwiftProvider.getContainerListing(
client, realcontainer, ["%s*" % realobject])
if len(realobjects) == 0:
#The object DNE
self.log.debug(
"The object link '%s' from '%s' didn't have a referant - deleting link file",
manifest, objectName)
client.delete_object(ctrname, objectName)
containerlen = containerlen - 1
else:
self.log.debug(
"The object link '%s' from '%s' still has a referant",
manifest, objectName)
else:
self.log.debug("Object '%s' is not a link", objectName)
self.log.devdebug(" %s", str(objectDetails))
if containerlen == 0:
self.log.debug(
"Container: %s is now empty, removing container", ctrname)
client.delete_container(ctrname)
self.log.passed()
return True
def run(innerself):
try:
client = SwiftProvider.getConnection(clientParams)
response = {}
deleteat = 9999999999
if secondsToExpire is not None:
deleteat = secondsToExpire
client.put_object(innerself.toContainer,
innerself.objectTarget,
None,
headers={
'X-Copy-From':
'%s/%s' % (options['from-container'],
innerself.objectTarget),
'X-Object-Meta-CreateTime':
str(time.time()),
'X-Delete-At':
deleteat
},
response_dict=response)
innerself.succeeded = True
except swiftclient.exceptions.ClientException as e:
if e.http_status == 404 or e.http_status == 409:
self.log.warning("Copy of object '%s/%s' returned: %d",
options['from-container'],
innerself.objectTarget, e.http_status)
self.log.warning(
"Step will not fail as this file was likely deleted but not yet removed from the container"
)
innerself.succeeded = True
else:
self.log.exception("Copy of object '%s/%s' failed",
options['from-container'],
innerself.objectTarget)
def passed__push(self, phase, options, instance, instancedict):
#Get just the id of the wrapper
wrapperid = options['wrapper']
if '@' in wrapperid:
wrapperid = wrapperid.split('@')[1]
if '__wrapper__%s' % wrapperid not in self.env.env:
self.log.passed()
self.log.info(
"Wrapper %s is not in use, artefact will not be put in the wrapper",
wrapperid)
return False
result = self.engine.launchStep(options['wrapper'], 'getinfo')
if result is None or not result._didPass():
self.log.failed()
self.log.error("Wrapper '%s' is invalid", self['wrapper'])
return False
toContainer, account = result._getReturnValue('getinfo')
swiftAccount, _ = SwiftProvider.getAccount(self.engine, account,
'push')
client = SwiftProvider.getConnection(swiftAccount)
objects = instance._getPushedObjects()
for fromContainer, objectName in objects:
self.log.debug("Linking '%s'/'%s' to '%s'", fromContainer,
objectName, toContainer)
headers = {
'X-Object-Manifest': '%s/%s' % (fromContainer, objectName),
'X-Object-Meta-CreateTime': str(time.time())
}
putResult = {}
client.put_object(toContainer,
objectName,
None,
headers=headers,
response_dict=putResult)
self.log.passed()
return True
def push(self, options):
(clientParams,
account) = SwiftProvider.getAccount(self.engine, options['account'],
"push")
client = SwiftProvider.getConnection(clientParams)
filematchers = ['*']
if 'files' in options:
filematchers = ','.join(options['files'].split('\n')).split(',')
targetedObjects = SwiftProvider.getContainerListing(
client, options['container'], filematchers,
'latest-only' in options and options['latest-only'] == 'True')
targetPath = self.engine.resultsDir
if 'result-dir' in options:
targetPath = options['result-dir']
threads = []
for filename, (data, _) in targetedObjects.iteritems():
self.objects.append((options['container'], filename))
if not self._isNoop(options):
headers = {
'X-Object-Manifest':
'%s/%s' % (options['container'], filename),
'X-Object-Meta-CreateTime': str(time.time())
}
putResult = {}
client.put_object(options['to-container'],
filename,
None,
headers=headers,
response_dict=putResult)
self.log.passed()
return True
def prep(self, options):
self._dontValidateFiles()
#Execute the account section to get the account info
(clientParams,
account) = SwiftProvider.getAccount(self.engine, options['account'],
'prep')
client = SwiftProvider.getConnection(clientParams)
SwiftProvider.ensureContainer(client, options['backup'])
if 'access' in options:
SwiftProvider.setContainerAccounts(self.engine, 'Read', client,
options['backup'],
options['access'], 'prep')
mycontainer = client.get_container(options['container'])
mycontainer[0]['name'] = options['container']
self.log.devdebug("Container obtained: %s", str(mycontainer))
SwiftProvider.addDetailsToAccountListing(client, [mycontainer])
headers = SwiftProvider.getMetadataFromContainerDict(mycontainer[0])
headers['x-versions-location'] = urllib.quote(options['backup'])
client.post_container(options['container'], headers)
self.log.passed()
return True
def prep(self, options):
#Ensure container exists
connectParams, _ = SwiftProvider.getAccount(self.engine,
options['account'],
self.engine.getPhase())
client = SwiftProvider.getConnection(connectParams)
SwiftProvider.ensureContainer(client, options['container'])
if 'access' in options:
SwiftProvider.setContainerAccounts(self.engine, 'Read', client,
options['container'],
options['access'], 'prep')
self.log.passed()
return self.default(options)
def prep(self, options):
#Execute the account section to get the account info
(clientParams,
account) = SwiftProvider.getAccount(self.engine, options['account'],
'prep')
client = SwiftProvider.getConnection(clientParams)
SwiftProvider.ensureContainer(client, options['container'])
if 'access' in options:
SwiftProvider.setContainerAccounts(self.engine, 'Read', client,
options['container'],
options['access'], 'prep')
self.log.passed()
return True
def prep(self, options):
self._dontValidateFiles()
#Execute the account section to get the account info
(clientParams,
account) = SwiftProvider.getAccount(self.engine, options['account'],
'prep')
client = SwiftProvider.getConnection(clientParams)
for froms, tos in self.mapping.iterspecs():
for container in tos:
containerName, _ = SwiftProvider.getContainerAndFile(
None, container['relLocation'])
SwiftProvider.ensureContainer(client, containerName)
if 'access' in options:
SwiftProvider.setContainerAccounts(self.engine, 'Read',
client, containerName,
options['access'],
'prep')
self.log.passed()
return True
def pull(self, options):
class DownloadThread(threading.Thread):
def _progressReport(innerself, xmitted, total):
if self.reportProgress:
currenttime = datetime.datetime.utcnow()
delta = currenttime - innerself.reporttime
if delta.seconds < 2:
return
deltaXmitted = xmitted - innerself.previousamount
if delta.total_seconds() > 0.0:
rate = float(deltaXmitted) / float(
delta.total_seconds()) / 1000.0
else:
rate = 0.0
rateunit = "KB"
if rate > 1000.0:
rate /= 1000.0
rateunit = "MB"
if rate > 1000.0:
rate /= 1000.0
rateunit = "GB"
innerself.previousamount = xmitted
innerself.reporttime = currenttime
percent = float(xmitted) / float(total) * 100.0
if percent > 100.0:
percent = 100.0
self.log.info(
"[%s:%s] Transferred %2.1f%% (%d out of %d - %2.2f%s/s)",
innerself.container, innerself.toFilename, percent,
xmitted, total, rate, rateunit)
def __init__(innerself, container, fromFilename, toFilename,
clientParams):
threading.Thread.__init__(innerself)
innerself.fromFilename = fromFilename
innerself.toFilename = toFilename
innerself.clientParams = clientParams
innerself.container = container
innerself.succeeded = False
innerself.httpstatus = None
innerself.starttime = None
innerself.reporttime = None
innerself.previousamount = 0
def run(innerself):
innerself.starttime = datetime.datetime.utcnow()
innerself.reporttime = innerself.starttime
try:
client = SwiftProvider.getConnection(
innerself.clientParams)
with open(innerself.toFilename, 'w') as openFile:
SwiftProvider.doSwiftDownload(
client,
innerself.container,
innerself.fromFilename,
openFile,
progress=innerself._progressReport)
innerself.succeeded = True
except swiftclient.exceptions.ClientException as e:
os.remove(innerself.toFilename)
if e.http_status == 404 or e.http_status == 409:
self.log.exception(
"Download of object '%s' returned %d",
innerself.fromFilename, e.http_status)
self.log.warning(
"Step will not fail as this file was likely but not yet removed from the container"
)
innerself.succeeded = True
else:
self.log.exception("Download of object '%s' failed")
innerself.httpstatus = e.http_status
except:
os.remove(innerself.toFilename)
self.log.exception("Download of object '%s' failed",
innerself.fromFilename)
innerself.succeeded = False
return innerself.succeeded
def passed(innerself):
return innerself.succeeded or \
('ignore-fail' in options and options['ignore-fail'] == 'True')
def httpstatus(innerself):
if self.httpstatus is None:
return 200
else:
return self.httpstatus
self.reportProgress = True
if 'progress' in options:
self.reportProgress = options['progress'] == "True"
ignorefail = 'ignore-fail' in options and options[
'ignore-fail'] == 'True'
(clientParams,
account) = SwiftProvider.getAccount(self.engine, options['account'],
"pull")
client = SwiftProvider.getConnection(clientParams)
filematchers = ['*']
if 'files' in options:
filematchers = ','.join(options['files'].split('\n')).split(',')
try:
targetedObjects = SwiftProvider.getContainerListing(
client, options['container'], filematchers,
'latest-only' in options and options['latest-only'] == 'True')
except Exception as e:
self.log.exception("Pull of the container failed")
if not ignorefail:
raise e
else:
self.log.passed()
return False
self.log.devdebug("targeted objects: %s", str(targetedObjects))
targetPath = self.engine.resultsDir
if 'result-dir' in options:
targetPath = options['result-dir']
threads = []
for filename, (data, _) in targetedObjects.iteritems():
targetFile = os.path.join(targetPath, filename)
#Does the path exist?
if os.path.exists(targetFile):
#Yes, get the etag for the object
eTag = data['hash']
#get the md5 for the local file if there is one
md5 = None
with open(targetFile) as openFile:
md5 = self._fileMD5(openFile)
if md5 == eTag:
self.log.info(
"'%s' is already up-to-date, not redownloading",
targetFile)
continue
self._ensureDirectoryExists(targetFile)
threads.append(
DownloadThread(options['container'], filename, targetFile,
clientParams))
threads[-1].start()
for thread in threads:
thread.join()
if not thread.passed() and not ignorefail:
self.log.failed()
return False
self.log.passed()
return True
def push(self, options):
class UploadReportThread(threading.Thread):
def __init__(innerself, total, fileobject, container, filename):
threading.Thread.__init__(innerself)
innerself.stopping = False
innerself.total = total
innerself.fileobject = fileobject
innerself.container = container
innerself.filename = filename
innerself.starttime = None
innerself.reporttime = None
innerself.previousamount = 0
def run(innerself):
innerself.starttime = datetime.datetime.utcnow()
innerself.reporttime = innerself.starttime
try:
counter = 0
while not innerself.stopping:
time.sleep(0.1)
counter += 1
if counter % 20:
continue
current = innerself.fileobject.tell()
percent = float(current) / float(
innerself.total) * 100.0
if not innerself.stopping:
currenttime = datetime.datetime.utcnow()
delta = currenttime - innerself.reporttime
deltaXmitted = current - innerself.previousamount
if delta.total_seconds() > 0.0:
rate = float(deltaXmitted) / float(
delta.total_seconds()) / 1000.0
else:
rate = 0.0
rateunit = "KB"
if rate > 1000.0:
rate /= 1000.0
rateunit = "MB"
if rate > 1000.0:
rate /= 1000.0
rateunit = "GB"
innerself.previousamount = current
innerself.reporttime = currenttime
if percent > 100.0:
percent = 100.0
self.log.info(
"[%s:%s] Transferred %d%% (%d out of %d - %2.2f%s/s)",
innerself.container, innerself.filename,
percent, current, innerself.total, rate,
rateunit)
except:
self.log.exception("Couldn't show progress")
return 0
def stop(innerself):
innerself.stopping = True
self.reportProgress = True
if 'progress' in options:
self.reportProgress = options['progress'] == 'True'
self._dontValidateFiles()
secondsToExpire = None
if 'expires' in options:
secondsToExpire = int(
SwiftProvider.timeDeltaDisgronifier(
options['expires']).total_seconds())
#Execute the account section to get the account info
(clientParams,
account) = SwiftProvider.getAccount(self.engine, options['account'],
'push')
client = SwiftProvider.getConnection(clientParams)
#Upload the requested objects in the mapping
for froms, tos in self.mapping.iterspecs():
for filename in froms:
relfilename = filename['relLocation']
if relfilename.startswith('./'):
relfilename = relfilename[2:]
with open(filename['location'], 'rb', 0) as openFile:
self.log.debug("Using raw buffer, 10240 chunk size")
openFile.seek(0, os.SEEK_END)
fileSize = openFile.tell()
openFile.seek(0)
headers = {'X-Object-Meta-CreateTime': str(time.time())}
if secondsToExpire is not None:
headers.update(
{'X-Delete-After': str(secondsToExpire)})
for container in tos:
containerName, toFileName = SwiftProvider.getContainerAndFile(
relfilename, container['relLocation'])
if self.reportProgress:
reporter = UploadReportThread(
fileSize, openFile, containerName, toFileName)
reporter.start()
client.put_object(containerName,
toFileName,
openFile,
content_length=fileSize,
chunk_size=10240,
headers=headers)
if self.reportProgress:
reporter.stop()
reporter.join()
self.objects.append((containerName, toFileName))
self.log.passed()
return True
def push(self, options):
secondsToExpire = None
if 'expires' in options:
secondsToExpire = int(
SwiftProvider.timeDeltaDisgronifier(
options['expires']).total_seconds())
#Execute the account section to get the account info
(clientParams,
account) = SwiftProvider.getAccount(self.engine, options['account'],
'push')
client = SwiftProvider.getConnection(clientParams)
class CopyThread(threading.Thread):
def __init__(innerself, toContainer, objectTarget, fromContainer,
toObjectName):
threading.Thread.__init__(innerself)
innerself.fromContainer = fromContainer
innerself.toContainer = toContainer
innerself.objectTarget = objectTarget
innerself.toObjectName = toObjectName
innerself.succeeded = False
def run(innerself):
try:
client = SwiftProvider.getConnection(clientParams)
response = {}
deleteat = 9999999999
if secondsToExpire is not None:
deleteat = secondsToExpire
client.put_object(innerself.toContainer,
innerself.objectTarget,
None,
headers={
'X-Copy-From':
'%s/%s' % (options['from-container'],
innerself.objectTarget),
'X-Object-Meta-CreateTime':
str(time.time()),
'X-Delete-At':
deleteat
},
response_dict=response)
innerself.succeeded = True
except swiftclient.exceptions.ClientException as e:
if e.http_status == 404 or e.http_status == 409:
self.log.warning("Copy of object '%s/%s' returned: %d",
options['from-container'],
innerself.objectTarget, e.http_status)
self.log.warning(
"Step will not fail as this file was likely deleted but not yet removed from the container"
)
innerself.succeeded = True
else:
self.log.exception("Copy of object '%s/%s' failed",
options['from-container'],
innerself.objectTarget)
def passed(innerself):
return innerself.succeeded
container = options['container']
filematchers = ['*']
if 'objects' in options:
filematchers = ','.join(options['objects'].split('\n')).split(',')
targetedObjects = SwiftProvider.getContainerListing(
client, options['from-container'], filematchers,
'latest-only' in options and options['latest-only'] == 'True')
threads = []
for objectTarget in targetedObjects.keys():
#TODO: Allow a mapping to another name
self.log.info("Copying object: '%s'", objectTarget)
threads.append(
CopyThread(options['container'], objectTarget,
options['from-container'], objectTarget))
threads[-1].start()
for thread in threads:
thread.join()
if not thread.passed():
self.log.failed()
return False
self.log.passed()
return True
def push(self, options):
#Get the current time
utcnow = datetime.datetime.utcnow()
#Get the timeout policy
expires = datetime.timedelta()
if 'expires' in options:
expires = SwiftProvider.timeDeltaDisgronifier(options['expires'])
self.log.devdebug("UTC: %s, expires: %s", utcnow, expires)
#Execute the account section to get the account info
(clientParams, account) = SwiftProvider.getAccount(self.engine, options['account'], 'push')
client = SwiftProvider.getConnection(clientParams)
containerMatcher = ','.join(options['containers'].split('\n')).split(',')
objectMatcher = '*'
if 'objects' in options:
objectMatcher = ','.join(options['objects'].split('\n')).split(',')
targetedContainers = SwiftProvider.getAccountListing(
client,
containerMatcher)
for ctrname, container in targetedContainers.iteritems():
targetedObjects = SwiftProvider.getContainerListing(
client,
ctrname,
objectMatcher,
details=True )
objectlist = targetedObjects.keys()
detaillist = targetedObjects.values()
if 'leave' in options:
leave = int(options['leave'])
leave = min(leave,len(targetedObjects))
sortedvals = sorted(
targetedObjects.values(),
key=lambda k:k[0]['x-object-meta-createtime'] )
leaves = sortedvals[-leave:]
sortedvals = sortedvals[:-leave]
leaveobjects = []
leavedetail = []
for val in leaves:
leaveobjects.append(val[0]['name'])
leavedetail.append(val)
objectlist = []
detaillist = []
for val in sortedvals:
objectlist.append(val[0]['name'])
detaillist.append(val)
leaveObjects = zip(leaveobjects, leavedetail)
for objectTarget, (details, filterer) in leaveObjects:
if 'x-delete-at' in details:
if details['x-delete-at'] == '9999999999':
continue
#I want to delete the "delete-at", but it doesn't
# appear a way to do that
headers = SwiftProvider.getMetadataFromDict(details)
headers['x-delete-at'] = "9999999999"
client.post_object(
ctrname,
objectTarget,
headers=headers )
self.log.debug("%s/%s: Object preserved by policy", ctrname, objectTarget)
targetedObjects = zip(objectlist, detaillist)
self.log.devdebug("In Container: %s", ctrname)
for objectTarget, (details, filterer) in targetedObjects:
#Check to see if x-object-meta-createtime exists
timestamp = datetime.datetime.utcfromtimestamp(
float(details['x-object-meta-createtime']) )
utcexpire = timestamp + expires
#Is the object expired according to this policy?
if utcexpire < utcnow:
#Yes. Delete it.
client.delete_object(ctrname, objectTarget)
self.log.debug("%s/%s: Object deleted - aged out", ctrname, objectTarget)
continue
#Does the object already have an expiration set?
if 'x-delete-at' in details:
#Yes. Check the time, is it according to this policy?
deltime = datetime.datetime.utcfromtimestamp(
float(details['x-delete-at']))
if utcexpire - deltime <= datetime.timedelta(minutes=1) \
or deltime - utcexpire <= datetime.timedelta(minutes=1):
#Yes. The world is as it should be.
#NOTE: A minute should give plenty of time to account
# for losses and, really, what's one minute
# for archival? The object won't be cleaned
# out off the disk until the "expirer" service
# runs and takes out the trash.
# The goal is to not have to communicate for
# an object that already meets the policy +/-
self.log.debug("%s/%s: File expiry follows policy", ctrname,objectTarget)
continue
#Set the policy.
secondsToExpire = str(int((utcexpire-utcnow).total_seconds()))
headers = SwiftProvider.getMetadataFromDict(details)
headers['x-delete-after'] = secondsToExpire
client.post_object(
ctrname,
objectTarget,
headers=headers )
self.log.debug("%s/%s: File will expire after %s seconds", ctrname, objectTarget, secondsToExpire)
self.log.passed()
return True