class ObjectCrawler(Daemon): """Update object information in container listings.""" def __init__(self, conf): self.conf = conf self.logger = get_logger(conf, log_route='object-updater') self.devices = conf.get('devices', '/srv/node') self.ip = conf.get('md-server-ip', '127.0.0.1') self.port = conf.get('md-server-port', '6090') self.mount_check = config_true_value(conf.get('mount_check', 'true')) self.swift_dir = conf.get('swift_dir', '/etc/swift') self.interval = int(conf.get('interval', 30)) #self.container_ring = None #self.concurrency = int(conf.get('concurrency', 1)) #self.slowdown = float(conf.get('slowdown', 0.01)) #self.node_timeout = int(conf.get('node_timeout', 10)) #self.conn_timeout = float(conf.get('conn_timeout', .5)) self.last_time_ran = 0 self.diskfile_mgr = DiskFileManager(conf, self.logger) def run_forever(self, *args, **kwargs): """Run the updater continuously.""" time.sleep(random() * self.interval) while True: with open("/opt/swift/OBJLOG.txt", "a+") as f: f.write("START\n") try: self.object_sweep() self.last_time_ran = time.time() except (Exception, Timeout): pass time.sleep(self.interval) def run_once(self, *args, **kwargs): """Run the updater once.""" self.object_sweep() def object_sweep(self): """ Scan through all objects and send metadata dict of ones with updates. """ all_locs = self.diskfile_mgr.object_audit_location_generator() metaList = [] for location in all_locs: try: metaDict = self.collect_object(location) metaDict = self.format_metadata(metaDict) if metaDict != {}: modtime = metaDict["object_last_modified_time"] if modtime != 'NULL' and float(modtime) > self.last_time_ran: metaList.append(metaDict) except Exception: pass if metaList != []: ObjectSender = Sender(self.conf) ObjectSender.sendData( metaList, 'object_crawler', self.ip, self.port) def collect_object(self, location): """ Process the object metadata """ metadata = {} try: df = self.diskfile_mgr.get_diskfile_from_audit_location(location) metadata = df.read_metadata() except DiskFileNotExist: pass return metadata def format_metadata(self, data): metadata = {} uri = data['name'].split("/") metadata['object_uri'] = data['name'] metadata['object_name'] = ("/".join(uri[3:])) metadata['object_account_name'] = uri[1] metadata['object_container_name'] = uri[2] metadata['object_location'] = 'NULL' # Not implemented yet metadata['object_uri_create_time'] = \ data.setdefault('X-Timestamp', 'NULL') metadata['object_last_modified_time'] = \ data.setdefault('X-Timestamp', 'NULL') metadata['object_last_changed_time'] = 'NULL' metadata['object_delete_time'] = 'NULL' metadata['object_last_activity_time'] = \ data.setdefault('X-Timestamp', 'NULL') metadata['object_etag_hash'] = \ data.setdefault('ETag', 'NULL') metadata['object_content_type'] = \ data.setdefault('Content-Type', 'NULL') metadata['object_content_length'] = \ data.setdefault('Content-Length', 'NULL') metadata['object_content_encoding'] = \ data.setdefault('Content-Encoding', 'NULL') metadata['object_content_disposition'] = \ data.setdefault('Content-Disposition', 'NULL') metadata['object_content_language'] = \ data.setdefault('Content-Langauge', 'NULL') metadata['object_cache_control'] = 'NULL' metadata['object_delete_at'] = \ data.setdefault('X-Delete-At', 'NULL') metadata['object_manifest_type'] = 'NULL' metadata['object_manifest'] = 'NULL' metadata['object_access_control_allow_origin'] = 'NULL' metadata['object_access_control_allow_credentials'] = 'NULL' metadata['object_access_control_expose_headers'] = 'NULL' metadata['object_access_control_max_age'] = 'NULL' metadata['object_access_control_allow_methods'] = 'NULL' metadata['object_access_control_allow_headers'] = 'NULL' metadata['object_origin'] = 'NULL' metadata['object_access_control_request_method'] = 'NULL' metadata['object_access_control_request_headers'] = 'NULL' #Insert all Object custom metadata for custom in data: if(custom.startswith("X-Object-Meta")): sanitized_custom = custom[2:13].lower() + custom[13:] sanitized_custom = sanitized_custom.replace('-', '_') metadata[sanitized_custom] = data[custom] return metadata