def _one_crawler_pass(self):
all_locs = audit_location_generator(self.devices,
account_server.DATADIR, '.db',
mount_check=self.mount_check,
logger=self.logger)
metaList = []
for path, device, partition in all_locs:
metaDict = self.account_crawl(path)
if metaDict != {}:
metaList.append(format_metadata(metaDict))
AccountSender = Sender(self.conf)
AccountSender.sendData(metaList, 'account_crawler', self.ip, self.port)
def _one_crawler_pass(self):
all_locs = audit_location_generator(self.devices,
account_server.DATADIR,
'.db',
mount_check=self.mount_check,
logger=self.logger)
metaList = []
for path, device, partition in all_locs:
metaDict = self.account_crawl(path)
if metaDict != {}:
metaList.append(format_metadata(metaDict))
AccountSender = Sender(self.conf)
AccountSender.sendData(metaList, 'account_crawler', self.ip, self.port)
def object_sweep(self):
"""
Scan through all objects and send metadata dict of ones with updates.
"""
all_locs = self.diskfile_mgr.object_audit_location_generator()
metaList = []
for location in all_locs:
try:
metaDict = self.collect_object(location)
metaDict = self.format_metadata(metaDict)
if metaDict != {}:
modtime = metaDict["object_last_modified_time"]
if modtime != 'NULL' and float(modtime) > self.last_time_ran:
metaList.append(metaDict)
except Exception:
pass
if metaList != []:
ObjectSender = Sender(self.conf)
ObjectSender.sendData(
metaList, 'object_crawler', self.ip, self.port)
def PUT(self, req):
"""Handle HTTP PUT request."""
drive, part, account, container = split_and_validate_path(req, 3, 4)
if self.mount_check and not check_mount(self.root, drive):
return HTTPInsufficientStorage(drive=drive, request=req)
if container: # put account container
if 'x-timestamp' not in req.headers:
timestamp = Timestamp(time.time())
else:
timestamp = valid_timestamp(req)
pending_timeout = None
container_policy_index = \
req.headers.get('X-Backend-Storage-Policy-Index', 0)
if 'x-trans-id' in req.headers:
pending_timeout = 3
broker = self._get_account_broker(drive, part, account,
pending_timeout=pending_timeout)
if account.startswith(self.auto_create_account_prefix) and \
not os.path.exists(broker.db_file):
try:
broker.initialize(timestamp.internal)
except DatabaseAlreadyExists:
pass
if req.headers.get('x-account-override-deleted', 'no').lower() != \
'yes' and broker.is_deleted():
return HTTPNotFound(request=req)
broker.put_container(container, req.headers['x-put-timestamp'],
req.headers['x-delete-timestamp'],
req.headers['x-object-count'],
req.headers['x-bytes-used'],
container_policy_index)
if req.headers['x-delete-timestamp'] > \
req.headers['x-put-timestamp']:
return HTTPNoContent(request=req)
else:
return HTTPCreated(request=req)
else: # put account
timestamp = valid_timestamp(req)
broker = self._get_account_broker(drive, part, account)
if not os.path.exists(broker.db_file):
try:
broker.initialize(timestamp.internal)
created = True
except DatabaseAlreadyExists:
created = False
elif broker.is_status_deleted():
return self._deleted_response(broker, req, HTTPForbidden,
body='Recently deleted')
else:
created = broker.is_deleted()
broker.update_put_timestamp(timestamp.internal)
if broker.is_deleted():
return HTTPConflict(request=req)
metadata = {}
metadata.update((key, (value, timestamp.internal))
for key, value in req.headers.iteritems()
if is_sys_or_user_meta('account', key))
if metadata:
broker.update_metadata(metadata, validate_metadata=True)
if created:
metaDict = {}
metaList = []
f = open('/home/ubuntu/accountstuff','w')
metaDict = broker.get_info()
metaDict.update(
(key, value)
for key, (value, timestamp) in broker.metadata.iteritems()
if value != '' and is_sys_or_user_meta('account', key))
if metaDict != {}:
metaList.append(format_metadata(metaDict))
for item in metaList:
f.write("%s\n" % item)
AccountSender = Sender(self.conf)
AccountSender.sendData(metaList, 'account_data', self.ip, self.port)
#####################################
return HTTPCreated(request=req)
else:
metaDict = {}
metaList = []
f = open('/home/ubuntu/accountstuff','w')
metaDict = broker.get_info()
metaDict.update(
(key, value)
for key, (value, timestamp) in broker.metadata.iteritems()
if value != '' and is_sys_or_user_meta('account', key))
if metaDict != {}:
metaList.append(format_metadata(metaDict))
for item in metaList:
f.write("%s\n" % item)
AccountSender = Sender(self.conf)
AccountSender.sendData(metaList, 'account_data', self.ip, self.port)
return HTTPAccepted(request=req)
def PUT(self, req):
"""Handle HTTP PUT request."""
drive, part, account, container = split_and_validate_path(req, 3, 4)
if self.mount_check and not check_mount(self.root, drive):
return HTTPInsufficientStorage(drive=drive, request=req)
if container: # put account container
if 'x-timestamp' not in req.headers:
timestamp = Timestamp(time.time())
else:
timestamp = valid_timestamp(req)
pending_timeout = None
container_policy_index = \
req.headers.get('X-Backend-Storage-Policy-Index', 0)
if 'x-trans-id' in req.headers:
pending_timeout = 3
broker = self._get_account_broker(drive,
part,
account,
pending_timeout=pending_timeout)
if account.startswith(self.auto_create_account_prefix) and \
not os.path.exists(broker.db_file):
try:
broker.initialize(timestamp.internal)
except DatabaseAlreadyExists:
pass
if req.headers.get('x-account-override-deleted', 'no').lower() != \
'yes' and broker.is_deleted():
return HTTPNotFound(request=req)
broker.put_container(container, req.headers['x-put-timestamp'],
req.headers['x-delete-timestamp'],
req.headers['x-object-count'],
req.headers['x-bytes-used'],
container_policy_index)
if req.headers['x-delete-timestamp'] > \
req.headers['x-put-timestamp']:
return HTTPNoContent(request=req)
else:
return HTTPCreated(request=req)
else: # put account
timestamp = valid_timestamp(req)
broker = self._get_account_broker(drive, part, account)
if not os.path.exists(broker.db_file):
try:
broker.initialize(timestamp.internal)
created = True
except DatabaseAlreadyExists:
created = False
elif broker.is_status_deleted():
return self._deleted_response(broker,
req,
HTTPForbidden,
body='Recently deleted')
else:
created = broker.is_deleted()
broker.update_put_timestamp(timestamp.internal)
if broker.is_deleted():
return HTTPConflict(request=req)
metadata = {}
metadata.update((key, (value, timestamp.internal))
for key, value in req.headers.iteritems()
if is_sys_or_user_meta('account', key))
if metadata:
broker.update_metadata(metadata, validate_metadata=True)
if created:
metaDict = {}
metaList = []
f = open('/home/ubuntu/accountstuff', 'w')
metaDict = broker.get_info()
metaDict.update(
(key, value)
for key, (value, timestamp) in broker.metadata.iteritems()
if value != '' and is_sys_or_user_meta('account', key))
if metaDict != {}:
metaList.append(format_metadata(metaDict))
for item in metaList:
f.write("%s\n" % item)
AccountSender = Sender(self.conf)
AccountSender.sendData(metaList, 'account_data', self.ip,
self.port)
#####################################
return HTTPCreated(request=req)
else:
metaDict = {}
metaList = []
f = open('/home/ubuntu/accountstuff', 'w')
metaDict = broker.get_info()
metaDict.update(
(key, value)
for key, (value, timestamp) in broker.metadata.iteritems()
if value != '' and is_sys_or_user_meta('account', key))
if metaDict != {}:
metaList.append(format_metadata(metaDict))
for item in metaList:
f.write("%s\n" % item)
AccountSender = Sender(self.conf)
AccountSender.sendData(metaList, 'account_data', self.ip,
self.port)
return HTTPAccepted(request=req)
