def _volume_name_by_space_id(): """ The control process creates a pool of worker processes of configurable size (default 2) for each distinct file space. However, if multiple file spaces have the same "volume name" value, then one worker process pool handles read requests to all of the file spaces with that same volume name. In other words, there will be a pool of workers for each non null volume name. Null values are never the same as other null values, so if no volume names are specified for the table spaces, there will be one read worker pool per file space. So we assign a volume name to each space_id, creating a 'null-nn' name if volume is null """ connection = get_node_local_connection() file_space_info = load_file_space_info(connection) connection.close() file_space_sanity_check(file_space_info, _repository_path) volume_name_by_space_id = dict() null_count = 0 for file_space_row_list in file_space_info.values(): for file_space_row in file_space_row_list: if file_space_row.volume is None: null_count += 1 volume_name = "null-{0}".format(null_count) else: volume_name = file_space_row.volume volume_name_by_space_id[file_space_row.space_id] = volume_name return volume_name_by_space_id
def _allocate_output_value_files(connection, repository_path, refs): output_value_file_sizes = defaultdict(list) file_space_info = load_file_space_info(connection) file_space_sanity_check(file_space_info, repository_path) space_id = find_least_volume_space_id("storage", file_space_info) for ref in refs: if len(output_value_file_sizes[ref.collection_id]) == 0: output_value_file_sizes[ref.collection_id].append(0) expected_size = output_value_file_sizes[ref.collection_id][-1] \ + ref.data_size if expected_size > _max_value_file_size: output_value_file_sizes[ref.collection_id].append(0) output_value_file_sizes[ref.collection_id][-1] += ref.data_size output_value_files = defaultdict(list) for collection_id in output_value_file_sizes.keys(): for expected_size in output_value_file_sizes[collection_id]: output_value_files[collection_id].append( OutputValueFile(connection, space_id, repository_path, expected_size=expected_size)) return output_value_files
def _run(self): log = logging.getLogger("WriterThread._run") log.debug("thread starts") file_space_info = load_file_space_info(self._database_connection) file_space_sanity_check(file_space_info, _repository_path) # Ticket #1646 mark output value files as closed at startup mark_value_files_as_closed(self._database_connection) self._writer = Writer(self._database_connection, file_space_info, _repository_path, self._active_segments, self._completions) log.debug("start halt_event loop") while not self._halt_event.is_set(): try: message, data = self._message_queue.get(block=True, timeout=_queue_timeout) except queue.Empty: pass else: self._dispatch_table[message["message-type"]](message, data) log.debug("end halt_event loop") # 2012-03-27 dougfort -- we stop the data writer first because it is # going to sync the value file and run the post_sync operations log.debug("stopping data writer") self._writer.close() log.debug("closing database connection") self._database_connection.close() if len(self._completions) > 0: log.warn("{0} PostSyncCompletion's lost in teardown".format( len(self._completions))) if len(self._active_segments) > 0: log.warn("{0} active-segments at teardown".format( len(self._active_segments)))
def main(): """ main entry point return 0 for success (exit code) """ initialize_logging(_log_path) log = logging.getLogger("main") log.info("program starts") halt_event = Event() set_signal_handler(halt_event) zmq_context = zmq.Context() event_push_client = EventPushClient(zmq_context, "defragger") event_push_client.info("program-start", "defragger starts") connection = None file_space_info = None while not halt_event.is_set(): # if we don't have an open database connection, get one if connection is None: try: connection = get_node_local_connection() except Exception as instance: exctype, value = sys.exc_info()[:2] event_push_client.exception( "database exception", str(value), exctype=exctype.__name__ ) log.exception("Exception connecting to database") halt_event.wait(_database_retry_interval) continue file_space_info = load_file_space_info(connection) file_space_sanity_check(file_space_info, _repository_path) # try one defrag pass bytes_defragged = 0 connection.begin_transaction() try: bytes_defragged = _defrag_pass(connection, file_space_info, event_push_client) except KeyboardInterrupt: halt_event.set() connection.rollback() except Exception as instance: log.exception(str(instance)) event_push_client.exception( unhandled_exception_topic, str(instance), exctype=instance.__class__.__name__ ) connection.rollback() else: connection.commit() log.info("bytes defragged = {0:,}".format(bytes_defragged)) # if we didn't do anything on this pass... if bytes_defragged == 0: # exit if we're done and asked to do single pass if int(os.environ.get('NIMBUSIO_EXIT_WHEN_DONE', '0')): halt_event.set() # close the database connection if connection is not None: connection.close() connection = None # wait and try again try: halt_event.wait(_defrag_check_interval) except KeyboardInterrupt: halt_event.set() if connection is not None: connection.close() event_push_client.close() zmq_context.term() log.info("program terminates normally") return 0
def main(): """ main entry point return 0 for success (exit code) """ initialize_logging(_log_path) log = logging.getLogger("main") log.info("program starts") halt_event = Event() set_signal_handler(halt_event) zmq_context = zmq.Context() event_push_client = EventPushClient(zmq_context, "defragger") event_push_client.info("program-start", "defragger starts") connection = None file_space_info = None while not halt_event.is_set(): # if we don't have an open database connection, get one if connection is None: try: connection = get_node_local_connection() except Exception as instance: exctype, value = sys.exc_info()[:2] event_push_client.exception("database exception", str(value), exctype=exctype.__name__) log.exception("Exception connecting to database") halt_event.wait(_database_retry_interval) continue file_space_info = load_file_space_info(connection) file_space_sanity_check(file_space_info, _repository_path) # try one defrag pass bytes_defragged = 0 connection.begin_transaction() try: bytes_defragged = _defrag_pass(connection, file_space_info, event_push_client) except KeyboardInterrupt: halt_event.set() connection.rollback() except Exception as instance: log.exception(str(instance)) event_push_client.exception(unhandled_exception_topic, str(instance), exctype=instance.__class__.__name__) connection.rollback() else: connection.commit() log.info("bytes defragged = {0:,}".format(bytes_defragged)) # if we didn't do anything on this pass... if bytes_defragged == 0: # exit if we're done and asked to do single pass if int(os.environ.get('NIMBUSIO_EXIT_WHEN_DONE', '0')): halt_event.set() # close the database connection if connection is not None: connection.close() connection = None # wait and try again try: halt_event.wait(_defrag_check_interval) except KeyboardInterrupt: halt_event.set() if connection is not None: connection.close() event_push_client.close() zmq_context.term() log.info("program terminates normally") return 0