def push(self, options):
secondsToExpire = None
if 'expires' in options:
secondsToExpire = int(
SwiftProvider.timeDeltaDisgronifier(
options['expires']).total_seconds())
#Execute the account section to get the account info
(clientParams,
account) = SwiftProvider.getAccount(self.engine, options['account'],
'push')
client = SwiftProvider.getConnection(clientParams)
class CopyThread(threading.Thread):
def __init__(innerself, toContainer, objectTarget, fromContainer,
toObjectName):
threading.Thread.__init__(innerself)
innerself.fromContainer = fromContainer
innerself.toContainer = toContainer
innerself.objectTarget = objectTarget
innerself.toObjectName = toObjectName
innerself.succeeded = False
def run(innerself):
try:
client = SwiftProvider.getConnection(clientParams)
response = {}
deleteat = 9999999999
if secondsToExpire is not None:
deleteat = secondsToExpire
client.put_object(innerself.toContainer,
innerself.objectTarget,
None,
headers={
'X-Copy-From':
'%s/%s' % (options['from-container'],
innerself.objectTarget),
'X-Object-Meta-CreateTime':
str(time.time()),
'X-Delete-At':
deleteat
},
response_dict=response)
innerself.succeeded = True
except swiftclient.exceptions.ClientException as e:
if e.http_status == 404 or e.http_status == 409:
self.log.warning("Copy of object '%s/%s' returned: %d",
options['from-container'],
innerself.objectTarget, e.http_status)
self.log.warning(
"Step will not fail as this file was likely deleted but not yet removed from the container"
)
innerself.succeeded = True
else:
self.log.exception("Copy of object '%s/%s' failed",
options['from-container'],
innerself.objectTarget)
def passed(innerself):
return innerself.succeeded
container = options['container']
filematchers = ['*']
if 'objects' in options:
filematchers = ','.join(options['objects'].split('\n')).split(',')
targetedObjects = SwiftProvider.getContainerListing(
client, options['from-container'], filematchers,
'latest-only' in options and options['latest-only'] == 'True')
threads = []
for objectTarget in targetedObjects.keys():
#TODO: Allow a mapping to another name
self.log.info("Copying object: '%s'", objectTarget)
threads.append(
CopyThread(options['container'], objectTarget,
options['from-container'], objectTarget))
threads[-1].start()
for thread in threads:
thread.join()
if not thread.passed():
self.log.failed()
return False
self.log.passed()
return True
def push(self, options):
class UploadReportThread(threading.Thread):
def __init__(innerself, total, fileobject, container, filename):
threading.Thread.__init__(innerself)
innerself.stopping = False
innerself.total = total
innerself.fileobject = fileobject
innerself.container = container
innerself.filename = filename
innerself.starttime = None
innerself.reporttime = None
innerself.previousamount = 0
def run(innerself):
innerself.starttime = datetime.datetime.utcnow()
innerself.reporttime = innerself.starttime
try:
counter = 0
while not innerself.stopping:
time.sleep(0.1)
counter += 1
if counter % 20:
continue
current = innerself.fileobject.tell()
percent = float(current) / float(
innerself.total) * 100.0
if not innerself.stopping:
currenttime = datetime.datetime.utcnow()
delta = currenttime - innerself.reporttime
deltaXmitted = current - innerself.previousamount
if delta.total_seconds() > 0.0:
rate = float(deltaXmitted) / float(
delta.total_seconds()) / 1000.0
else:
rate = 0.0
rateunit = "KB"
if rate > 1000.0:
rate /= 1000.0
rateunit = "MB"
if rate > 1000.0:
rate /= 1000.0
rateunit = "GB"
innerself.previousamount = current
innerself.reporttime = currenttime
if percent > 100.0:
percent = 100.0
self.log.info(
"[%s:%s] Transferred %d%% (%d out of %d - %2.2f%s/s)",
innerself.container, innerself.filename,
percent, current, innerself.total, rate,
rateunit)
except:
self.log.exception("Couldn't show progress")
return 0
def stop(innerself):
innerself.stopping = True
self.reportProgress = True
if 'progress' in options:
self.reportProgress = options['progress'] == 'True'
self._dontValidateFiles()
secondsToExpire = None
if 'expires' in options:
secondsToExpire = int(
SwiftProvider.timeDeltaDisgronifier(
options['expires']).total_seconds())
#Execute the account section to get the account info
(clientParams,
account) = SwiftProvider.getAccount(self.engine, options['account'],
'push')
client = SwiftProvider.getConnection(clientParams)
#Upload the requested objects in the mapping
for froms, tos in self.mapping.iterspecs():
for filename in froms:
relfilename = filename['relLocation']
if relfilename.startswith('./'):
relfilename = relfilename[2:]
with open(filename['location'], 'rb', 0) as openFile:
self.log.debug("Using raw buffer, 10240 chunk size")
openFile.seek(0, os.SEEK_END)
fileSize = openFile.tell()
openFile.seek(0)
headers = {'X-Object-Meta-CreateTime': str(time.time())}
if secondsToExpire is not None:
headers.update(
{'X-Delete-After': str(secondsToExpire)})
for container in tos:
containerName, toFileName = SwiftProvider.getContainerAndFile(
relfilename, container['relLocation'])
if self.reportProgress:
reporter = UploadReportThread(
fileSize, openFile, containerName, toFileName)
reporter.start()
client.put_object(containerName,
toFileName,
openFile,
content_length=fileSize,
chunk_size=10240,
headers=headers)
if self.reportProgress:
reporter.stop()
reporter.join()
self.objects.append((containerName, toFileName))
self.log.passed()
return True
def push(self, options):
#Get the current time
utcnow = datetime.datetime.utcnow()
#Get the timeout policy
expires = datetime.timedelta()
if 'expires' in options:
expires = SwiftProvider.timeDeltaDisgronifier(options['expires'])
self.log.devdebug("UTC: %s, expires: %s", utcnow, expires)
#Execute the account section to get the account info
(clientParams, account) = SwiftProvider.getAccount(self.engine, options['account'], 'push')
client = SwiftProvider.getConnection(clientParams)
containerMatcher = ','.join(options['containers'].split('\n')).split(',')
objectMatcher = '*'
if 'objects' in options:
objectMatcher = ','.join(options['objects'].split('\n')).split(',')
targetedContainers = SwiftProvider.getAccountListing(
client,
containerMatcher)
for ctrname, container in targetedContainers.iteritems():
targetedObjects = SwiftProvider.getContainerListing(
client,
ctrname,
objectMatcher,
details=True )
objectlist = targetedObjects.keys()
detaillist = targetedObjects.values()
if 'leave' in options:
leave = int(options['leave'])
leave = min(leave,len(targetedObjects))
sortedvals = sorted(
targetedObjects.values(),
key=lambda k:k[0]['x-object-meta-createtime'] )
leaves = sortedvals[-leave:]
sortedvals = sortedvals[:-leave]
leaveobjects = []
leavedetail = []
for val in leaves:
leaveobjects.append(val[0]['name'])
leavedetail.append(val)
objectlist = []
detaillist = []
for val in sortedvals:
objectlist.append(val[0]['name'])
detaillist.append(val)
leaveObjects = zip(leaveobjects, leavedetail)
for objectTarget, (details, filterer) in leaveObjects:
if 'x-delete-at' in details:
if details['x-delete-at'] == '9999999999':
continue
#I want to delete the "delete-at", but it doesn't
# appear a way to do that
headers = SwiftProvider.getMetadataFromDict(details)
headers['x-delete-at'] = "9999999999"
client.post_object(
ctrname,
objectTarget,
headers=headers )
self.log.debug("%s/%s: Object preserved by policy", ctrname, objectTarget)
targetedObjects = zip(objectlist, detaillist)
self.log.devdebug("In Container: %s", ctrname)
for objectTarget, (details, filterer) in targetedObjects:
#Check to see if x-object-meta-createtime exists
timestamp = datetime.datetime.utcfromtimestamp(
float(details['x-object-meta-createtime']) )
utcexpire = timestamp + expires
#Is the object expired according to this policy?
if utcexpire < utcnow:
#Yes. Delete it.
client.delete_object(ctrname, objectTarget)
self.log.debug("%s/%s: Object deleted - aged out", ctrname, objectTarget)
continue
#Does the object already have an expiration set?
if 'x-delete-at' in details:
#Yes. Check the time, is it according to this policy?
deltime = datetime.datetime.utcfromtimestamp(
float(details['x-delete-at']))
if utcexpire - deltime <= datetime.timedelta(minutes=1) \
or deltime - utcexpire <= datetime.timedelta(minutes=1):
#Yes. The world is as it should be.
#NOTE: A minute should give plenty of time to account
# for losses and, really, what's one minute
# for archival? The object won't be cleaned
# out off the disk until the "expirer" service
# runs and takes out the trash.
# The goal is to not have to communicate for
# an object that already meets the policy +/-
self.log.debug("%s/%s: File expiry follows policy", ctrname,objectTarget)
continue
#Set the policy.
secondsToExpire = str(int((utcexpire-utcnow).total_seconds()))
headers = SwiftProvider.getMetadataFromDict(details)
headers['x-delete-after'] = secondsToExpire
client.post_object(
ctrname,
objectTarget,
headers=headers )
self.log.debug("%s/%s: File will expire after %s seconds", ctrname, objectTarget, secondsToExpire)
self.log.passed()
return True