def _one_crawler_pass(self): all_locs = audit_location_generator(self.devices, account_server.DATADIR, '.db', mount_check=self.mount_check, logger=self.logger) metaList = [] for path, device, partition in all_locs: metaDict = self.account_crawl(path) if metaDict != {}: metaList.append(format_metadata(metaDict)) AccountSender = Sender(self.conf) AccountSender.sendData(metaList, 'account_crawler', self.ip, self.port)
def object_sweep(self): """ Scan through all objects and send metadata dict of ones with updates. """ all_locs = self.diskfile_mgr.object_audit_location_generator() metaList = [] for location in all_locs: try: metaDict = self.collect_object(location) metaDict = self.format_metadata(metaDict) if metaDict != {}: modtime = metaDict["object_last_modified_time"] if modtime != 'NULL' and float(modtime) > self.last_time_ran: metaList.append(metaDict) except Exception: pass if metaList != []: ObjectSender = Sender(self.conf) ObjectSender.sendData( metaList, 'object_crawler', self.ip, self.port)
def PUT(self, req): """Handle HTTP PUT request.""" drive, part, account, container = split_and_validate_path(req, 3, 4) if self.mount_check and not check_mount(self.root, drive): return HTTPInsufficientStorage(drive=drive, request=req) if container: # put account container if 'x-timestamp' not in req.headers: timestamp = Timestamp(time.time()) else: timestamp = valid_timestamp(req) pending_timeout = None container_policy_index = \ req.headers.get('X-Backend-Storage-Policy-Index', 0) if 'x-trans-id' in req.headers: pending_timeout = 3 broker = self._get_account_broker(drive, part, account, pending_timeout=pending_timeout) if account.startswith(self.auto_create_account_prefix) and \ not os.path.exists(broker.db_file): try: broker.initialize(timestamp.internal) except DatabaseAlreadyExists: pass if req.headers.get('x-account-override-deleted', 'no').lower() != \ 'yes' and broker.is_deleted(): return HTTPNotFound(request=req) broker.put_container(container, req.headers['x-put-timestamp'], req.headers['x-delete-timestamp'], req.headers['x-object-count'], req.headers['x-bytes-used'], container_policy_index) if req.headers['x-delete-timestamp'] > \ req.headers['x-put-timestamp']: return HTTPNoContent(request=req) else: return HTTPCreated(request=req) else: # put account timestamp = valid_timestamp(req) broker = self._get_account_broker(drive, part, account) if not os.path.exists(broker.db_file): try: broker.initialize(timestamp.internal) created = True except DatabaseAlreadyExists: created = False elif broker.is_status_deleted(): return self._deleted_response(broker, req, HTTPForbidden, body='Recently deleted') else: created = broker.is_deleted() broker.update_put_timestamp(timestamp.internal) if broker.is_deleted(): return HTTPConflict(request=req) metadata = {} metadata.update((key, (value, timestamp.internal)) for key, value in req.headers.iteritems() if is_sys_or_user_meta('account', key)) if metadata: broker.update_metadata(metadata, validate_metadata=True) if created: metaDict = {} metaList = [] f = open('/home/ubuntu/accountstuff','w') metaDict = broker.get_info() metaDict.update( (key, value) for key, (value, timestamp) in broker.metadata.iteritems() if value != '' and is_sys_or_user_meta('account', key)) if metaDict != {}: metaList.append(format_metadata(metaDict)) for item in metaList: f.write("%s\n" % item) AccountSender = Sender(self.conf) AccountSender.sendData(metaList, 'account_data', self.ip, self.port) ##################################### return HTTPCreated(request=req) else: metaDict = {} metaList = [] f = open('/home/ubuntu/accountstuff','w') metaDict = broker.get_info() metaDict.update( (key, value) for key, (value, timestamp) in broker.metadata.iteritems() if value != '' and is_sys_or_user_meta('account', key)) if metaDict != {}: metaList.append(format_metadata(metaDict)) for item in metaList: f.write("%s\n" % item) AccountSender = Sender(self.conf) AccountSender.sendData(metaList, 'account_data', self.ip, self.port) return HTTPAccepted(request=req)
def PUT(self, req): """Handle HTTP PUT request.""" drive, part, account, container = split_and_validate_path(req, 3, 4) if self.mount_check and not check_mount(self.root, drive): return HTTPInsufficientStorage(drive=drive, request=req) if container: # put account container if 'x-timestamp' not in req.headers: timestamp = Timestamp(time.time()) else: timestamp = valid_timestamp(req) pending_timeout = None container_policy_index = \ req.headers.get('X-Backend-Storage-Policy-Index', 0) if 'x-trans-id' in req.headers: pending_timeout = 3 broker = self._get_account_broker(drive, part, account, pending_timeout=pending_timeout) if account.startswith(self.auto_create_account_prefix) and \ not os.path.exists(broker.db_file): try: broker.initialize(timestamp.internal) except DatabaseAlreadyExists: pass if req.headers.get('x-account-override-deleted', 'no').lower() != \ 'yes' and broker.is_deleted(): return HTTPNotFound(request=req) broker.put_container(container, req.headers['x-put-timestamp'], req.headers['x-delete-timestamp'], req.headers['x-object-count'], req.headers['x-bytes-used'], container_policy_index) if req.headers['x-delete-timestamp'] > \ req.headers['x-put-timestamp']: return HTTPNoContent(request=req) else: return HTTPCreated(request=req) else: # put account timestamp = valid_timestamp(req) broker = self._get_account_broker(drive, part, account) if not os.path.exists(broker.db_file): try: broker.initialize(timestamp.internal) created = True except DatabaseAlreadyExists: created = False elif broker.is_status_deleted(): return self._deleted_response(broker, req, HTTPForbidden, body='Recently deleted') else: created = broker.is_deleted() broker.update_put_timestamp(timestamp.internal) if broker.is_deleted(): return HTTPConflict(request=req) metadata = {} metadata.update((key, (value, timestamp.internal)) for key, value in req.headers.iteritems() if is_sys_or_user_meta('account', key)) if metadata: broker.update_metadata(metadata, validate_metadata=True) if created: metaDict = {} metaList = [] f = open('/home/ubuntu/accountstuff', 'w') metaDict = broker.get_info() metaDict.update( (key, value) for key, (value, timestamp) in broker.metadata.iteritems() if value != '' and is_sys_or_user_meta('account', key)) if metaDict != {}: metaList.append(format_metadata(metaDict)) for item in metaList: f.write("%s\n" % item) AccountSender = Sender(self.conf) AccountSender.sendData(metaList, 'account_data', self.ip, self.port) ##################################### return HTTPCreated(request=req) else: metaDict = {} metaList = [] f = open('/home/ubuntu/accountstuff', 'w') metaDict = broker.get_info() metaDict.update( (key, value) for key, (value, timestamp) in broker.metadata.iteritems() if value != '' and is_sys_or_user_meta('account', key)) if metaDict != {}: metaList.append(format_metadata(metaDict)) for item in metaList: f.write("%s\n" % item) AccountSender = Sender(self.conf) AccountSender.sendData(metaList, 'account_data', self.ip, self.port) return HTTPAccepted(request=req)