def __handle_validate(self, args):
"""Handle 'validate' command.
Command arguments are the same 'load' except the last one:
'action': callable without any parameter itself, encapsulating
any segment-specific validation logic. It returns
a result of the validation.
This method simply calls the passed action, and returns the result
back to the memmgr with other command arguments. This is run in
the builder thread simply because it may take time.
"""
_, dsrc_info, rrclass, dsrc_name, action = args
logger.debug(logger.DBGLVL_TRACE_BASIC,
LIBMEMMGR_BUILDER_SEGMENT_VALIDATE, dsrc_name, rrclass)
try:
result = action()
except Exception as ex:
logger.error(LIBMEMMGR_BUILDER_SEGMENT_VALIDATE_FAIL, dsrc_name,
rrclass, ex)
result = False
self.__send_response(('validate-completed', dsrc_info, rrclass,
dsrc_name, result))
def __handle_validate(self, args):
"""Handle 'validate' command.
Command arguments are the same 'load' except the last one:
'action': callable without any parameter itself, encapsulating
any segment-specific validation logic. It returns
a result of the validation.
This method simply calls the passed action, and returns the result
back to the memmgr with other command arguments. This is run in
the builder thread simply because it may take time.
"""
_, dsrc_info, rrclass, dsrc_name, action = args
logger.debug(logger.DBGLVL_TRACE_BASIC,
LIBMEMMGR_BUILDER_SEGMENT_VALIDATE, dsrc_name, rrclass)
try:
result = action()
except Exception as ex:
logger.error(LIBMEMMGR_BUILDER_SEGMENT_VALIDATE_FAIL, dsrc_name,
rrclass, ex)
result = False
self.__send_response(('validate-completed', dsrc_info, rrclass,
dsrc_name, result))
def __handle_bad_command(self, bad_command):
# A bad command was received. Raising an exception is not useful
# in this case as we are likely running in a different thread
# from the main thread which would need to be notified. Instead
# return this in the response queue.
logger.error(LIBMEMMGR_BUILDER_BAD_COMMAND_ERROR, bad_command)
self._response_queue.append(('bad_command',))
self._shutdown = True
def __handle_bad_command(self, bad_command):
# A bad command was received. Raising an exception is not useful
# in this case as we are likely running in a different thread
# from the main thread which would need to be notified. Instead
# return this in the response queue.
logger.error(LIBMEMMGR_BUILDER_BAD_COMMAND_ERROR, bad_command)
self.__send_response(('bad_command',))
self._shutdown = True
def _switch_versions(self):
# Swith the versions as noted in the constructor.
self.__reader_ver = 1 - self.__reader_ver
self.__writer_ver = 1 - self.__writer_ver
# Versions should be different
assert(self.__reader_ver != self.__writer_ver)
# Now reader file should be ready
self.__reader_file_validated = True
# write/update versions file
versions = {'reader': self.__reader_ver, 'writer': self.__writer_ver}
try:
with open(self.__map_versions_file, 'w') as f:
f.write(json.dumps(versions) + '\n')
except Exception as ex:
logger.error(LIBMEMMGR_MAPPED_SEGMENT_VERFILE_UPDATE_FAIL,
self.__map_versions_file, ex)
def _switch_versions(self):
# Swith the versions as noted in the constructor.
self.__reader_ver = 1 - self.__reader_ver
self.__writer_ver = 1 - self.__writer_ver
# Versions should be different
assert(self.__reader_ver != self.__writer_ver)
# Now reader file should be ready
self.__reader_file_validated = True
# write/update versions file
versions = {'reader': self.__reader_ver, 'writer': self.__writer_ver}
try:
with open(self.__map_versions_file, 'w') as f:
f.write(json.dumps(versions) + '\n')
except Exception as ex:
logger.error(LIBMEMMGR_MAPPED_SEGMENT_VERFILE_UPDATE_FAIL,
self.__map_versions_file, ex)
def __reset_segment(self, clist, dsrc_name, rrclass, params):
try:
clist.reset_memory_segment(dsrc_name,
ConfigurableClientList.READ_WRITE,
params)
logger.debug(logger.DBGLVL_TRACE_BASIC,
LIBMEMMGR_BUILDER_SEGMENT_RESET, dsrc_name, rrclass)
return self.__RESET_SEGMENT_OK
except Exception as ex:
logger.error(LIBMEMMGR_BUILDER_RESET_SEGMENT_ERROR, dsrc_name,
rrclass, ex)
try:
clist.reset_memory_segment(dsrc_name, ConfigurableClientList.CREATE,
params)
logger.info(LIBMEMMGR_BUILDER_SEGMENT_CREATED, dsrc_name, rrclass)
return self.__RESET_SEGMENT_CREATED
except Exception as ex:
logger.error(LIBMEMMGR_BUILDER_SEGMENT_CREATE_ERROR, dsrc_name,
rrclass, ex)
return self.__RESET_SEGMENT_FAILED
def __reset_segment(self, clist, dsrc_name, rrclass, params):
try:
clist.reset_memory_segment(dsrc_name,
ConfigurableClientList.READ_WRITE,
params)
logger.debug(logger.DBGLVL_TRACE_BASIC,
LIBMEMMGR_BUILDER_SEGMENT_RESET, dsrc_name, rrclass)
return self.__RESET_SEGMENT_OK
except Exception as ex:
logger.error(LIBMEMMGR_BUILDER_RESET_SEGMENT_ERROR, dsrc_name,
rrclass, ex)
try:
clist.reset_memory_segment(dsrc_name, ConfigurableClientList.CREATE,
params)
logger.info(LIBMEMMGR_BUILDER_SEGMENT_CREATED, dsrc_name, rrclass)
return self.__RESET_SEGMENT_CREATED
except Exception as ex:
logger.error(LIBMEMMGR_BUILDER_SEGMENT_CREATE_ERROR, dsrc_name,
rrclass, ex)
return self.__RESET_SEGMENT_FAILED
def __handle_load(self, zone_name, dsrc_info, rrclass, dsrc_name):
# This method is called when handling the 'load' command. The
# following tuple is passed:
#
# ('load', zone_name, dsrc_info, rrclass, dsrc_name)
#
# where:
#
# * zone_name is None or bundy.dns.Name, specifying the zone name
# to load. If it's None, it means all zones to be cached in
# the specified data source (used for initialization).
#
# * dsrc_info is a DataSrcInfo object corresponding to the
# generation ID of the set of data sources for this loading.
#
# * rrclass is an bundy.dns.RRClass object, the RR class of the
# data source.
#
# * dsrc_name is a string, specifying a data source name.
clist = dsrc_info.clients_map[rrclass]
sgmt_info = dsrc_info.segment_info_map[(rrclass, dsrc_name)]
params = json.dumps(sgmt_info.get_reset_param(SegmentInfo.WRITER))
clist.reset_memory_segment(dsrc_name,
ConfigurableClientList.READ_WRITE,
params)
if zone_name is not None:
zones = [(None, zone_name)]
else:
zones = clist.get_zone_table_accessor(dsrc_name, True)
for _, zone_name in zones:
catch_load_error = (zone_name is None) # install empty zone initially
result, writer = clist.get_cached_zone_writer(zone_name, catch_load_error,
dsrc_name)
if result != ConfigurableClientList.CACHE_STATUS_ZONE_SUCCESS:
logger.error(LIBMEMMGR_BUILDER_GET_ZONE_WRITER_ERROR, zone_name, dsrc_name)
continue
try:
error = writer.load()
if error is not None:
logger.error(LIBMEMMGR_BUILDER_ZONE_WRITER_LOAD_1_ERROR, zone_name, dsrc_name, error)
continue
except Exception as e:
logger.error(LIBMEMMGR_BUILDER_ZONE_WRITER_LOAD_2_ERROR, zone_name, dsrc_name, str(e))
continue
writer.install()
writer.cleanup()
# need to reset the segment so readers can read it (note: memmgr
# itself doesn't have to keep it open, but there's currently no
# public API to just clear the segment)
clist.reset_memory_segment(dsrc_name,
ConfigurableClientList.READ_ONLY,
params)
self._response_queue.append(('load-completed', dsrc_info, rrclass,
dsrc_name))
def __init__(self, genid, rrclass, datasrc_name, mgr_config):
super().__init__(genid)
# Something like "/var/bundy/zone-IN-1-sqlite3-mapped"
self.__mapped_file_base = mgr_config['mapped_file_dir'] + os.sep + \
'zone-' + str(rrclass) + '-' + str(genid) + '-' + datasrc_name + \
'-mapped'
# Current versions (suffix of the mapped files) for readers and the
# writer. In this initial implementation we assume that all possible
# readers are waiting for a new version (not using pre-existing one),
# and the writer is expected to build a new segment as version "0".
self.__reader_ver = None
self.__writer_ver = None
# State of mapped file for readers: becomes true if validated
# successfully or switched from successfully updated writer version.
# Unless it's true, we won't tell readers the mapped file as it won't
# be usable. For writer (which is actually the memmgr itself, in
# practice), we'll always try to open/create it for any update attempt.
self.__reader_file_validated = False
self.__map_versions_file = self.__mapped_file_base + '-vers.json'
if os.path.exists(self.__map_versions_file):
try:
with open(self.__map_versions_file) as f:
versions = json.load(f)
# reset both versions at once; if either of the values is
# unavailable, neither variables will be reset.
rver, wver = versions['reader'], versions['writer']
if not ((rver == 0 and wver == 1) or (rver == 1 and wver == 0)):
raise SegmentInfoError('broken segment version')
self.__reader_ver = rver
self.__writer_ver = wver
except Exception as ex:
# This is somewhat unexpected, but not fatal; we could still
# rebuild segments from scratch.
logger.error(LIBMEMMGR_MAPPED_SEGMENT_BADVERFILE,
self.__map_versions_file, ex)
try: # ignore any error on unlink
os.unlink(self.__map_versions_file)
except: pass
# Prepare validate callables. To avoid leaving a reference to
# this object (the callables will be run in a separate thread), we
# now build the file name as string (or None if unknown) and embed
# in the callable closures.
reader_file = None if self.__reader_ver is None else \
'%s.%d' % (self.__mapped_file_base, self.__reader_ver)
writer_file = None if self.__writer_ver is None else \
'%s.%d' % (self.__mapped_file_base, self.__writer_ver)
self.__rvalidate_action = \
lambda: _validate_mapped_segment_file(reader_file)
self.__wvalidate_action = \
lambda: _validate_mapped_segment_file(writer_file)
# If the versions are not yet known, we'll begin with the defaults.
if self.__reader_ver is None and self.__writer_ver is None:
self.__reader_ver = 0
self.__writer_ver = 1
def _handle_load(self, zone_name, dsrc_info, rrclass, dsrc_name):
# This method is called when handling the 'load' command. The
# following tuple is passed:
#
# ('load', zone_name, dsrc_info, rrclass, dsrc_name)
#
# where:
#
# * zone_name is None or bundy.dns.Name, specifying the zone name
# to load. If it's None, it means all zones to be cached in
# the specified data source (used for initialization).
#
# * dsrc_info is a DataSrcInfo object corresponding to the
# generation ID of the set of data sources for this loading.
#
# * rrclass is an bundy.dns.RRClass object, the RR class of the
# data source.
#
# * dsrc_name is a string, specifying a data source name.
#
# This is essentially a 'private' method, but allows tests to call it
# directly; for other purposes shouldn't be called outside of the class.
clist = dsrc_info.clients_map[rrclass]
sgmt_info = dsrc_info.segment_info_map[(rrclass, dsrc_name)]
params = json.dumps(sgmt_info.get_reset_param(SegmentInfo.WRITER))
result = self.__reset_segment(clist, dsrc_name, rrclass, params)
if result == self.__RESET_SEGMENT_FAILED:
self.__send_response(('load-completed', dsrc_info, rrclass,
dsrc_name, False))
return
# If we were told to load a single zone but had to create a new
# segment, we'll need to load all zones, not just this one.
if result == self.__RESET_SEGMENT_CREATED and zone_name is not None:
logger.info(LIBMEMMGR_BUILDER_SEGMENT_LOAD_ALL, zone_name, rrclass,
dsrc_name)
zone_name = None
if zone_name is not None:
zones = [(None, zone_name)]
else:
zones = clist.get_zone_table_accessor(dsrc_name, True)
errors = 0
for _, zname in zones: # note: don't override zone_name here
# install empty zone initially
catch_load_error = (zname is None)
try:
result, writer = clist.get_cached_zone_writer(zname,
catch_load_error,
dsrc_name)
if result != ConfigurableClientList.CACHE_STATUS_ZONE_SUCCESS:
# handle this with other genuine exceptions below
raise bundy.datasrc.Error('result=%d' % result)
except bundy.datasrc.Error as ex:
logger.error(LIBMEMMGR_BUILDER_GET_ZONE_WRITER_ERROR,
zname, dsrc_name, ex)
errors += 1
continue
try:
error = writer.load()
if error is not None:
logger.error(LIBMEMMGR_BUILDER_ZONE_WRITER_LOAD_1_ERROR,
zname, dsrc_name, error)
errors += 1
continue
writer.install()
except Exception as e:
logger.error(LIBMEMMGR_BUILDER_ZONE_WRITER_LOAD_2_ERROR,
zname, dsrc_name, e)
errors += 1
# fall through to cleanup
writer.cleanup()
# need to reset the segment so readers can read it (note: memmgr
# itself doesn't have to keep it open, but there's currently no
# public API to just clear the segment). This 'reset' should succeed,
# so we'll let any exception be propagated.
clist.reset_memory_segment(dsrc_name,
ConfigurableClientList.READ_ONLY,
params)
# At this point, we consider the load a failure only if loading a
# specific zone has failed.
succeeded = True if (zone_name is None or errors == 0) else False
self.__send_response(('load-completed', dsrc_info, rrclass, dsrc_name,
succeeded))
def _handle_load(self, zone_name, dsrc_info, rrclass, dsrc_name):
# This method is called when handling the 'load' command. The
# following tuple is passed:
#
# ('load', zone_name, dsrc_info, rrclass, dsrc_name)
#
# where:
#
# * zone_name is None or bundy.dns.Name, specifying the zone name
# to load. If it's None, it means all zones to be cached in
# the specified data source (used for initialization).
#
# * dsrc_info is a DataSrcInfo object corresponding to the
# generation ID of the set of data sources for this loading.
#
# * rrclass is an bundy.dns.RRClass object, the RR class of the
# data source.
#
# * dsrc_name is a string, specifying a data source name.
#
# This is essentially a 'private' method, but allows tests to call it
# directly; for other purposes shouldn't be called outside of the class.
clist = dsrc_info.clients_map[rrclass]
sgmt_info = dsrc_info.segment_info_map[(rrclass, dsrc_name)]
params = json.dumps(sgmt_info.get_reset_param(SegmentInfo.WRITER))
result = self.__reset_segment(clist, dsrc_name, rrclass, params)
if result == self.__RESET_SEGMENT_FAILED:
self.__send_response(('load-completed', dsrc_info, rrclass,
dsrc_name, False))
return
# If we were told to load a single zone but had to create a new
# segment, we'll need to load all zones, not just this one.
if result == self.__RESET_SEGMENT_CREATED and zone_name is not None:
logger.info(LIBMEMMGR_BUILDER_SEGMENT_LOAD_ALL, zone_name, rrclass,
dsrc_name)
zone_name = None
if zone_name is not None:
zones = [(None, zone_name)]
else:
zones = clist.get_zone_table_accessor(dsrc_name, True)
errors = 0
canceled = False
for _, zname in zones: # note: don't override zone_name here
# install empty zone initially
catch_load_error = (zone_name is None)
try:
result, writer = clist.get_cached_zone_writer(zname,
catch_load_error,
dsrc_name)
if result != ConfigurableClientList.CACHE_STATUS_ZONE_SUCCESS:
# handle this with other genuine exceptions below
raise bundy.datasrc.Error('result=%d' % result)
except bundy.datasrc.Error as ex:
logger.error(LIBMEMMGR_BUILDER_GET_ZONE_WRITER_ERROR,
zname, dsrc_name, ex)
errors += 1
continue
try:
try:
if not self.__do_load_zone(writer, zname, rrclass,
dsrc_name, dsrc_info):
canceled = True
break
except bundy.datasrc.Error as error:
logger.error(LIBMEMMGR_BUILDER_ZONE_WRITER_LOAD_1_ERROR,
zname, dsrc_name, error)
errors += 1
# If this is initial full load, we'll add an empty zone
# for failed zones.
if catch_load_error:
writer.install()
continue
writer.install()
except Exception as e:
logger.error(LIBMEMMGR_BUILDER_ZONE_WRITER_LOAD_2_ERROR,
zname, dsrc_name, e)
errors += 1
# fall through to cleanup
writer.cleanup()
# Make sure the writer is destroyed no matter how we reach here
# befoe resetting the segment; otherwise the temporary resource
# maintained in the writer could cause a disruption.
writer = None
# need to reset the segment so readers can read it (note: memmgr
# itself doesn't have to keep it open, but there's currently no
# public API to just clear the segment). This 'reset' should succeed,
# so we'll let any exception be propagated.
clist.reset_memory_segment(dsrc_name,
ConfigurableClientList.READ_ONLY, params)
# If the load has been canceled, we are not expected to return a
# response. We should return after all cleanups are completed.
if canceled:
return
# At this point, we consider the load a failure only if loading a
# specific zone has failed.
succeeded = (zone_name is None or errors == 0)
self.__send_response(('load-completed', dsrc_info, rrclass, dsrc_name,
succeeded))
def __init__(self, genid, rrclass, datasrc_name, mgr_config):
super().__init__()
# Something like "/var/bundy/zone-IN-1-sqlite3-mapped"
self.__mapped_file_base = mgr_config['mapped_file_dir'] + os.sep + \
'zone-' + str(rrclass) + '-' + str(genid) + '-' + datasrc_name + \
'-mapped'
# Current versions (suffix of the mapped files) for readers and the
# writer. In this initial implementation we assume that all possible
# readers are waiting for a new version (not using pre-existing one),
# and the writer is expected to build a new segment as version "0".
self.__reader_ver = None
self.__writer_ver = None
# State of mapped file for readers: becomes true if validated
# successfully or switched from successfully updated writer version.
# Unless it's true, we won't tell readers the mapped file as it won't
# be usable. For writer (which is actually the memmgr itself, in
# practice), we'll always try to open/create it for any update attempt.
self.__reader_file_validated = False
self.__map_versions_file = self.__mapped_file_base + '-vers.json'
if os.path.exists(self.__map_versions_file):
try:
with open(self.__map_versions_file) as f:
versions = json.load(f)
# reset both versions at once; if either of the values is
# unavailable, neither variables will be reset.
rver, wver = versions['reader'], versions['writer']
if not ((rver == 0 and wver == 1) or (rver == 1 and wver == 0)):
raise SegmentInfoError('broken segment version')
self.__reader_ver = rver
self.__writer_ver = wver
except Exception as ex:
# This is somewhat unexpected, but not fatal; we could still
# rebuild segments from scratch.
logger.error(LIBMEMMGR_MAPPED_SEGMENT_BADVERFILE,
self.__map_versions_file, ex)
try: # ignore any error on unlink
os.unlink(self.__map_versions_file)
except: pass
# Prepare validate callables. To avoid leaving a reference to
# this object (the callables will be run in a separate thread), we
# now build the file name as string (or None if unknown) and embed
# in the callable closures.
reader_file = None if self.__reader_ver is None else \
'%s.%d' % (self.__mapped_file_base, self.__reader_ver)
writer_file = None if self.__writer_ver is None else \
'%s.%d' % (self.__mapped_file_base, self.__writer_ver)
self.__rvalidate_action = \
lambda: _validate_mapped_segment_file(reader_file)
self.__wvalidate_action = \
lambda: _validate_mapped_segment_file(writer_file)
# If the versions are not yet known, we'll begin with the defaults.
if self.__reader_ver is None and self.__writer_ver is None:
self.__reader_ver = 0
self.__writer_ver = 1
