def _volume_name_by_space_id():
"""
The control process creates a pool of worker processes of configurable size
(default 2) for each distinct file space. However, if multiple file spaces
have the same "volume name" value, then one worker process pool handles
read requests to all of the file spaces with that same volume name.
In other words, there will be a pool of workers for each non null volume
name. Null values are never the same as other null values, so if no volume
names are specified for the table spaces, there will be one read worker
pool per file space.
So we assign a volume name to each space_id, creating a 'null-nn' name
if volume is null
"""
connection = get_node_local_connection()
file_space_info = load_file_space_info(connection)
connection.close()
file_space_sanity_check(file_space_info, _repository_path)
volume_name_by_space_id = dict()
null_count = 0
for file_space_row_list in file_space_info.values():
for file_space_row in file_space_row_list:
if file_space_row.volume is None:
null_count += 1
volume_name = "null-{0}".format(null_count)
else:
volume_name = file_space_row.volume
volume_name_by_space_id[file_space_row.space_id] = volume_name
return volume_name_by_space_id
def _allocate_output_value_files(connection, repository_path, refs):
output_value_file_sizes = defaultdict(list)
file_space_info = load_file_space_info(connection)
file_space_sanity_check(file_space_info, repository_path)
space_id = find_least_volume_space_id("storage", file_space_info)
for ref in refs:
if len(output_value_file_sizes[ref.collection_id]) == 0:
output_value_file_sizes[ref.collection_id].append(0)
expected_size = output_value_file_sizes[ref.collection_id][-1] \
+ ref.data_size
if expected_size > _max_value_file_size:
output_value_file_sizes[ref.collection_id].append(0)
output_value_file_sizes[ref.collection_id][-1] += ref.data_size
output_value_files = defaultdict(list)
for collection_id in output_value_file_sizes.keys():
for expected_size in output_value_file_sizes[collection_id]:
output_value_files[collection_id].append(
OutputValueFile(connection,
space_id,
repository_path,
expected_size=expected_size))
return output_value_files
def _allocate_output_value_files(connection, repository_path, refs):
output_value_file_sizes = defaultdict(list)
file_space_info = load_file_space_info(connection)
file_space_sanity_check(file_space_info, repository_path)
space_id = find_least_volume_space_id("storage", file_space_info)
for ref in refs:
if len(output_value_file_sizes[ref.collection_id]) == 0:
output_value_file_sizes[ref.collection_id].append(0)
expected_size = output_value_file_sizes[ref.collection_id][-1] \
+ ref.data_size
if expected_size > _max_value_file_size:
output_value_file_sizes[ref.collection_id].append(0)
output_value_file_sizes[ref.collection_id][-1] += ref.data_size
output_value_files = defaultdict(list)
for collection_id in output_value_file_sizes.keys():
for expected_size in output_value_file_sizes[collection_id]:
output_value_files[collection_id].append(
OutputValueFile(connection,
space_id,
repository_path,
expected_size=expected_size))
return output_value_files
def _volume_name_by_space_id():
"""
The control process creates a pool of worker processes of configurable size
(default 2) for each distinct file space. However, if multiple file spaces
have the same "volume name" value, then one worker process pool handles
read requests to all of the file spaces with that same volume name.
In other words, there will be a pool of workers for each non null volume
name. Null values are never the same as other null values, so if no volume
names are specified for the table spaces, there will be one read worker
pool per file space.
So we assign a volume name to each space_id, creating a 'null-nn' name
if volume is null
"""
connection = get_node_local_connection()
file_space_info = load_file_space_info(connection)
connection.close()
file_space_sanity_check(file_space_info, _repository_path)
volume_name_by_space_id = dict()
null_count = 0
for file_space_row_list in file_space_info.values():
for file_space_row in file_space_row_list:
if file_space_row.volume is None:
null_count += 1
volume_name = "null-{0}".format(null_count)
else:
volume_name = file_space_row.volume
volume_name_by_space_id[file_space_row.space_id] = volume_name
return volume_name_by_space_id
def _run(self):
log = logging.getLogger("WriterThread._run")
log.debug("thread starts")
file_space_info = load_file_space_info(self._database_connection)
file_space_sanity_check(file_space_info, _repository_path)
# Ticket #1646 mark output value files as closed at startup
mark_value_files_as_closed(self._database_connection)
self._writer = Writer(self._database_connection,
file_space_info,
_repository_path,
self._active_segments,
self._completions)
log.debug("start halt_event loop")
while not self._halt_event.is_set():
try:
message, data = self._message_queue.get(block=True,
timeout=_queue_timeout)
except queue.Empty:
pass
else:
self._dispatch_table[message["message-type"]](message, data)
log.debug("end halt_event loop")
# 2012-03-27 dougfort -- we stop the data writer first because it is
# going to sync the value file and run the post_sync operations
log.debug("stopping data writer")
self._writer.close()
log.debug("closing database connection")
self._database_connection.close()
if len(self._completions) > 0:
log.warn("{0} PostSyncCompletion's lost in teardown".format(
len(self._completions)))
if len(self._active_segments) > 0:
log.warn("{0} active-segments at teardown".format(
len(self._active_segments)))
def _run(self):
log = logging.getLogger("WriterThread._run")
log.debug("thread starts")
file_space_info = load_file_space_info(self._database_connection)
file_space_sanity_check(file_space_info, _repository_path)
# Ticket #1646 mark output value files as closed at startup
mark_value_files_as_closed(self._database_connection)
self._writer = Writer(self._database_connection, file_space_info,
_repository_path, self._active_segments,
self._completions)
log.debug("start halt_event loop")
while not self._halt_event.is_set():
try:
message, data = self._message_queue.get(block=True,
timeout=_queue_timeout)
except queue.Empty:
pass
else:
self._dispatch_table[message["message-type"]](message, data)
log.debug("end halt_event loop")
# 2012-03-27 dougfort -- we stop the data writer first because it is
# going to sync the value file and run the post_sync operations
log.debug("stopping data writer")
self._writer.close()
log.debug("closing database connection")
self._database_connection.close()
if len(self._completions) > 0:
log.warn("{0} PostSyncCompletion's lost in teardown".format(
len(self._completions)))
if len(self._active_segments) > 0:
log.warn("{0} active-segments at teardown".format(
len(self._active_segments)))
def main():
"""
main entry point
return 0 for success (exit code)
"""
initialize_logging(_log_path)
log = logging.getLogger("main")
log.info("program starts")
halt_event = Event()
set_signal_handler(halt_event)
zmq_context = zmq.Context()
event_push_client = EventPushClient(zmq_context, "defragger")
event_push_client.info("program-start", "defragger starts")
connection = None
file_space_info = None
while not halt_event.is_set():
# if we don't have an open database connection, get one
if connection is None:
try:
connection = get_node_local_connection()
except Exception as instance:
exctype, value = sys.exc_info()[:2]
event_push_client.exception(
"database exception",
str(value),
exctype=exctype.__name__
)
log.exception("Exception connecting to database")
halt_event.wait(_database_retry_interval)
continue
file_space_info = load_file_space_info(connection)
file_space_sanity_check(file_space_info, _repository_path)
# try one defrag pass
bytes_defragged = 0
connection.begin_transaction()
try:
bytes_defragged = _defrag_pass(connection,
file_space_info,
event_push_client)
except KeyboardInterrupt:
halt_event.set()
connection.rollback()
except Exception as instance:
log.exception(str(instance))
event_push_client.exception(
unhandled_exception_topic,
str(instance),
exctype=instance.__class__.__name__
)
connection.rollback()
else:
connection.commit()
log.info("bytes defragged = {0:,}".format(bytes_defragged))
# if we didn't do anything on this pass...
if bytes_defragged == 0:
# exit if we're done and asked to do single pass
if int(os.environ.get('NIMBUSIO_EXIT_WHEN_DONE', '0')):
halt_event.set()
# close the database connection
if connection is not None:
connection.close()
connection = None
# wait and try again
try:
halt_event.wait(_defrag_check_interval)
except KeyboardInterrupt:
halt_event.set()
if connection is not None:
connection.close()
event_push_client.close()
zmq_context.term()
log.info("program terminates normally")
return 0
def main():
"""
main entry point
return 0 for success (exit code)
"""
initialize_logging(_log_path)
log = logging.getLogger("main")
log.info("program starts")
halt_event = Event()
set_signal_handler(halt_event)
zmq_context = zmq.Context()
event_push_client = EventPushClient(zmq_context, "defragger")
event_push_client.info("program-start", "defragger starts")
connection = None
file_space_info = None
while not halt_event.is_set():
# if we don't have an open database connection, get one
if connection is None:
try:
connection = get_node_local_connection()
except Exception as instance:
exctype, value = sys.exc_info()[:2]
event_push_client.exception("database exception",
str(value),
exctype=exctype.__name__)
log.exception("Exception connecting to database")
halt_event.wait(_database_retry_interval)
continue
file_space_info = load_file_space_info(connection)
file_space_sanity_check(file_space_info, _repository_path)
# try one defrag pass
bytes_defragged = 0
connection.begin_transaction()
try:
bytes_defragged = _defrag_pass(connection, file_space_info,
event_push_client)
except KeyboardInterrupt:
halt_event.set()
connection.rollback()
except Exception as instance:
log.exception(str(instance))
event_push_client.exception(unhandled_exception_topic,
str(instance),
exctype=instance.__class__.__name__)
connection.rollback()
else:
connection.commit()
log.info("bytes defragged = {0:,}".format(bytes_defragged))
# if we didn't do anything on this pass...
if bytes_defragged == 0:
# exit if we're done and asked to do single pass
if int(os.environ.get('NIMBUSIO_EXIT_WHEN_DONE', '0')):
halt_event.set()
# close the database connection
if connection is not None:
connection.close()
connection = None
# wait and try again
try:
halt_event.wait(_defrag_check_interval)
except KeyboardInterrupt:
halt_event.set()
if connection is not None:
connection.close()
event_push_client.close()
zmq_context.term()
log.info("program terminates normally")
return 0
