def __handle_validate(self, args): """Handle 'validate' command. Command arguments are the same 'load' except the last one: 'action': callable without any parameter itself, encapsulating any segment-specific validation logic. It returns a result of the validation. This method simply calls the passed action, and returns the result back to the memmgr with other command arguments. This is run in the builder thread simply because it may take time. """ _, dsrc_info, rrclass, dsrc_name, action = args logger.debug(logger.DBGLVL_TRACE_BASIC, LIBMEMMGR_BUILDER_SEGMENT_VALIDATE, dsrc_name, rrclass) try: result = action() except Exception as ex: logger.error(LIBMEMMGR_BUILDER_SEGMENT_VALIDATE_FAIL, dsrc_name, rrclass, ex) result = False self.__send_response(('validate-completed', dsrc_info, rrclass, dsrc_name, result))
def __handle_bad_command(self, bad_command): # A bad command was received. Raising an exception is not useful # in this case as we are likely running in a different thread # from the main thread which would need to be notified. Instead # return this in the response queue. logger.error(LIBMEMMGR_BUILDER_BAD_COMMAND_ERROR, bad_command) self._response_queue.append(('bad_command',)) self._shutdown = True
def __handle_bad_command(self, bad_command): # A bad command was received. Raising an exception is not useful # in this case as we are likely running in a different thread # from the main thread which would need to be notified. Instead # return this in the response queue. logger.error(LIBMEMMGR_BUILDER_BAD_COMMAND_ERROR, bad_command) self.__send_response(('bad_command',)) self._shutdown = True
def _switch_versions(self): # Swith the versions as noted in the constructor. self.__reader_ver = 1 - self.__reader_ver self.__writer_ver = 1 - self.__writer_ver # Versions should be different assert(self.__reader_ver != self.__writer_ver) # Now reader file should be ready self.__reader_file_validated = True # write/update versions file versions = {'reader': self.__reader_ver, 'writer': self.__writer_ver} try: with open(self.__map_versions_file, 'w') as f: f.write(json.dumps(versions) + '\n') except Exception as ex: logger.error(LIBMEMMGR_MAPPED_SEGMENT_VERFILE_UPDATE_FAIL, self.__map_versions_file, ex)
def __reset_segment(self, clist, dsrc_name, rrclass, params): try: clist.reset_memory_segment(dsrc_name, ConfigurableClientList.READ_WRITE, params) logger.debug(logger.DBGLVL_TRACE_BASIC, LIBMEMMGR_BUILDER_SEGMENT_RESET, dsrc_name, rrclass) return self.__RESET_SEGMENT_OK except Exception as ex: logger.error(LIBMEMMGR_BUILDER_RESET_SEGMENT_ERROR, dsrc_name, rrclass, ex) try: clist.reset_memory_segment(dsrc_name, ConfigurableClientList.CREATE, params) logger.info(LIBMEMMGR_BUILDER_SEGMENT_CREATED, dsrc_name, rrclass) return self.__RESET_SEGMENT_CREATED except Exception as ex: logger.error(LIBMEMMGR_BUILDER_SEGMENT_CREATE_ERROR, dsrc_name, rrclass, ex) return self.__RESET_SEGMENT_FAILED
def __handle_load(self, zone_name, dsrc_info, rrclass, dsrc_name): # This method is called when handling the 'load' command. The # following tuple is passed: # # ('load', zone_name, dsrc_info, rrclass, dsrc_name) # # where: # # * zone_name is None or bundy.dns.Name, specifying the zone name # to load. If it's None, it means all zones to be cached in # the specified data source (used for initialization). # # * dsrc_info is a DataSrcInfo object corresponding to the # generation ID of the set of data sources for this loading. # # * rrclass is an bundy.dns.RRClass object, the RR class of the # data source. # # * dsrc_name is a string, specifying a data source name. clist = dsrc_info.clients_map[rrclass] sgmt_info = dsrc_info.segment_info_map[(rrclass, dsrc_name)] params = json.dumps(sgmt_info.get_reset_param(SegmentInfo.WRITER)) clist.reset_memory_segment(dsrc_name, ConfigurableClientList.READ_WRITE, params) if zone_name is not None: zones = [(None, zone_name)] else: zones = clist.get_zone_table_accessor(dsrc_name, True) for _, zone_name in zones: catch_load_error = (zone_name is None) # install empty zone initially result, writer = clist.get_cached_zone_writer(zone_name, catch_load_error, dsrc_name) if result != ConfigurableClientList.CACHE_STATUS_ZONE_SUCCESS: logger.error(LIBMEMMGR_BUILDER_GET_ZONE_WRITER_ERROR, zone_name, dsrc_name) continue try: error = writer.load() if error is not None: logger.error(LIBMEMMGR_BUILDER_ZONE_WRITER_LOAD_1_ERROR, zone_name, dsrc_name, error) continue except Exception as e: logger.error(LIBMEMMGR_BUILDER_ZONE_WRITER_LOAD_2_ERROR, zone_name, dsrc_name, str(e)) continue writer.install() writer.cleanup() # need to reset the segment so readers can read it (note: memmgr # itself doesn't have to keep it open, but there's currently no # public API to just clear the segment) clist.reset_memory_segment(dsrc_name, ConfigurableClientList.READ_ONLY, params) self._response_queue.append(('load-completed', dsrc_info, rrclass, dsrc_name))
def __init__(self, genid, rrclass, datasrc_name, mgr_config): super().__init__(genid) # Something like "/var/bundy/zone-IN-1-sqlite3-mapped" self.__mapped_file_base = mgr_config['mapped_file_dir'] + os.sep + \ 'zone-' + str(rrclass) + '-' + str(genid) + '-' + datasrc_name + \ '-mapped' # Current versions (suffix of the mapped files) for readers and the # writer. In this initial implementation we assume that all possible # readers are waiting for a new version (not using pre-existing one), # and the writer is expected to build a new segment as version "0". self.__reader_ver = None self.__writer_ver = None # State of mapped file for readers: becomes true if validated # successfully or switched from successfully updated writer version. # Unless it's true, we won't tell readers the mapped file as it won't # be usable. For writer (which is actually the memmgr itself, in # practice), we'll always try to open/create it for any update attempt. self.__reader_file_validated = False self.__map_versions_file = self.__mapped_file_base + '-vers.json' if os.path.exists(self.__map_versions_file): try: with open(self.__map_versions_file) as f: versions = json.load(f) # reset both versions at once; if either of the values is # unavailable, neither variables will be reset. rver, wver = versions['reader'], versions['writer'] if not ((rver == 0 and wver == 1) or (rver == 1 and wver == 0)): raise SegmentInfoError('broken segment version') self.__reader_ver = rver self.__writer_ver = wver except Exception as ex: # This is somewhat unexpected, but not fatal; we could still # rebuild segments from scratch. logger.error(LIBMEMMGR_MAPPED_SEGMENT_BADVERFILE, self.__map_versions_file, ex) try: # ignore any error on unlink os.unlink(self.__map_versions_file) except: pass # Prepare validate callables. To avoid leaving a reference to # this object (the callables will be run in a separate thread), we # now build the file name as string (or None if unknown) and embed # in the callable closures. reader_file = None if self.__reader_ver is None else \ '%s.%d' % (self.__mapped_file_base, self.__reader_ver) writer_file = None if self.__writer_ver is None else \ '%s.%d' % (self.__mapped_file_base, self.__writer_ver) self.__rvalidate_action = \ lambda: _validate_mapped_segment_file(reader_file) self.__wvalidate_action = \ lambda: _validate_mapped_segment_file(writer_file) # If the versions are not yet known, we'll begin with the defaults. if self.__reader_ver is None and self.__writer_ver is None: self.__reader_ver = 0 self.__writer_ver = 1
def _handle_load(self, zone_name, dsrc_info, rrclass, dsrc_name): # This method is called when handling the 'load' command. The # following tuple is passed: # # ('load', zone_name, dsrc_info, rrclass, dsrc_name) # # where: # # * zone_name is None or bundy.dns.Name, specifying the zone name # to load. If it's None, it means all zones to be cached in # the specified data source (used for initialization). # # * dsrc_info is a DataSrcInfo object corresponding to the # generation ID of the set of data sources for this loading. # # * rrclass is an bundy.dns.RRClass object, the RR class of the # data source. # # * dsrc_name is a string, specifying a data source name. # # This is essentially a 'private' method, but allows tests to call it # directly; for other purposes shouldn't be called outside of the class. clist = dsrc_info.clients_map[rrclass] sgmt_info = dsrc_info.segment_info_map[(rrclass, dsrc_name)] params = json.dumps(sgmt_info.get_reset_param(SegmentInfo.WRITER)) result = self.__reset_segment(clist, dsrc_name, rrclass, params) if result == self.__RESET_SEGMENT_FAILED: self.__send_response(('load-completed', dsrc_info, rrclass, dsrc_name, False)) return # If we were told to load a single zone but had to create a new # segment, we'll need to load all zones, not just this one. if result == self.__RESET_SEGMENT_CREATED and zone_name is not None: logger.info(LIBMEMMGR_BUILDER_SEGMENT_LOAD_ALL, zone_name, rrclass, dsrc_name) zone_name = None if zone_name is not None: zones = [(None, zone_name)] else: zones = clist.get_zone_table_accessor(dsrc_name, True) errors = 0 for _, zname in zones: # note: don't override zone_name here # install empty zone initially catch_load_error = (zname is None) try: result, writer = clist.get_cached_zone_writer(zname, catch_load_error, dsrc_name) if result != ConfigurableClientList.CACHE_STATUS_ZONE_SUCCESS: # handle this with other genuine exceptions below raise bundy.datasrc.Error('result=%d' % result) except bundy.datasrc.Error as ex: logger.error(LIBMEMMGR_BUILDER_GET_ZONE_WRITER_ERROR, zname, dsrc_name, ex) errors += 1 continue try: error = writer.load() if error is not None: logger.error(LIBMEMMGR_BUILDER_ZONE_WRITER_LOAD_1_ERROR, zname, dsrc_name, error) errors += 1 continue writer.install() except Exception as e: logger.error(LIBMEMMGR_BUILDER_ZONE_WRITER_LOAD_2_ERROR, zname, dsrc_name, e) errors += 1 # fall through to cleanup writer.cleanup() # need to reset the segment so readers can read it (note: memmgr # itself doesn't have to keep it open, but there's currently no # public API to just clear the segment). This 'reset' should succeed, # so we'll let any exception be propagated. clist.reset_memory_segment(dsrc_name, ConfigurableClientList.READ_ONLY, params) # At this point, we consider the load a failure only if loading a # specific zone has failed. succeeded = True if (zone_name is None or errors == 0) else False self.__send_response(('load-completed', dsrc_info, rrclass, dsrc_name, succeeded))
def _handle_load(self, zone_name, dsrc_info, rrclass, dsrc_name): # This method is called when handling the 'load' command. The # following tuple is passed: # # ('load', zone_name, dsrc_info, rrclass, dsrc_name) # # where: # # * zone_name is None or bundy.dns.Name, specifying the zone name # to load. If it's None, it means all zones to be cached in # the specified data source (used for initialization). # # * dsrc_info is a DataSrcInfo object corresponding to the # generation ID of the set of data sources for this loading. # # * rrclass is an bundy.dns.RRClass object, the RR class of the # data source. # # * dsrc_name is a string, specifying a data source name. # # This is essentially a 'private' method, but allows tests to call it # directly; for other purposes shouldn't be called outside of the class. clist = dsrc_info.clients_map[rrclass] sgmt_info = dsrc_info.segment_info_map[(rrclass, dsrc_name)] params = json.dumps(sgmt_info.get_reset_param(SegmentInfo.WRITER)) result = self.__reset_segment(clist, dsrc_name, rrclass, params) if result == self.__RESET_SEGMENT_FAILED: self.__send_response(('load-completed', dsrc_info, rrclass, dsrc_name, False)) return # If we were told to load a single zone but had to create a new # segment, we'll need to load all zones, not just this one. if result == self.__RESET_SEGMENT_CREATED and zone_name is not None: logger.info(LIBMEMMGR_BUILDER_SEGMENT_LOAD_ALL, zone_name, rrclass, dsrc_name) zone_name = None if zone_name is not None: zones = [(None, zone_name)] else: zones = clist.get_zone_table_accessor(dsrc_name, True) errors = 0 canceled = False for _, zname in zones: # note: don't override zone_name here # install empty zone initially catch_load_error = (zone_name is None) try: result, writer = clist.get_cached_zone_writer(zname, catch_load_error, dsrc_name) if result != ConfigurableClientList.CACHE_STATUS_ZONE_SUCCESS: # handle this with other genuine exceptions below raise bundy.datasrc.Error('result=%d' % result) except bundy.datasrc.Error as ex: logger.error(LIBMEMMGR_BUILDER_GET_ZONE_WRITER_ERROR, zname, dsrc_name, ex) errors += 1 continue try: try: if not self.__do_load_zone(writer, zname, rrclass, dsrc_name, dsrc_info): canceled = True break except bundy.datasrc.Error as error: logger.error(LIBMEMMGR_BUILDER_ZONE_WRITER_LOAD_1_ERROR, zname, dsrc_name, error) errors += 1 # If this is initial full load, we'll add an empty zone # for failed zones. if catch_load_error: writer.install() continue writer.install() except Exception as e: logger.error(LIBMEMMGR_BUILDER_ZONE_WRITER_LOAD_2_ERROR, zname, dsrc_name, e) errors += 1 # fall through to cleanup writer.cleanup() # Make sure the writer is destroyed no matter how we reach here # befoe resetting the segment; otherwise the temporary resource # maintained in the writer could cause a disruption. writer = None # need to reset the segment so readers can read it (note: memmgr # itself doesn't have to keep it open, but there's currently no # public API to just clear the segment). This 'reset' should succeed, # so we'll let any exception be propagated. clist.reset_memory_segment(dsrc_name, ConfigurableClientList.READ_ONLY, params) # If the load has been canceled, we are not expected to return a # response. We should return after all cleanups are completed. if canceled: return # At this point, we consider the load a failure only if loading a # specific zone has failed. succeeded = (zone_name is None or errors == 0) self.__send_response(('load-completed', dsrc_info, rrclass, dsrc_name, succeeded))
def __init__(self, genid, rrclass, datasrc_name, mgr_config): super().__init__() # Something like "/var/bundy/zone-IN-1-sqlite3-mapped" self.__mapped_file_base = mgr_config['mapped_file_dir'] + os.sep + \ 'zone-' + str(rrclass) + '-' + str(genid) + '-' + datasrc_name + \ '-mapped' # Current versions (suffix of the mapped files) for readers and the # writer. In this initial implementation we assume that all possible # readers are waiting for a new version (not using pre-existing one), # and the writer is expected to build a new segment as version "0". self.__reader_ver = None self.__writer_ver = None # State of mapped file for readers: becomes true if validated # successfully or switched from successfully updated writer version. # Unless it's true, we won't tell readers the mapped file as it won't # be usable. For writer (which is actually the memmgr itself, in # practice), we'll always try to open/create it for any update attempt. self.__reader_file_validated = False self.__map_versions_file = self.__mapped_file_base + '-vers.json' if os.path.exists(self.__map_versions_file): try: with open(self.__map_versions_file) as f: versions = json.load(f) # reset both versions at once; if either of the values is # unavailable, neither variables will be reset. rver, wver = versions['reader'], versions['writer'] if not ((rver == 0 and wver == 1) or (rver == 1 and wver == 0)): raise SegmentInfoError('broken segment version') self.__reader_ver = rver self.__writer_ver = wver except Exception as ex: # This is somewhat unexpected, but not fatal; we could still # rebuild segments from scratch. logger.error(LIBMEMMGR_MAPPED_SEGMENT_BADVERFILE, self.__map_versions_file, ex) try: # ignore any error on unlink os.unlink(self.__map_versions_file) except: pass # Prepare validate callables. To avoid leaving a reference to # this object (the callables will be run in a separate thread), we # now build the file name as string (or None if unknown) and embed # in the callable closures. reader_file = None if self.__reader_ver is None else \ '%s.%d' % (self.__mapped_file_base, self.__reader_ver) writer_file = None if self.__writer_ver is None else \ '%s.%d' % (self.__mapped_file_base, self.__writer_ver) self.__rvalidate_action = \ lambda: _validate_mapped_segment_file(reader_file) self.__wvalidate_action = \ lambda: _validate_mapped_segment_file(writer_file) # If the versions are not yet known, we'll begin with the defaults. if self.__reader_ver is None and self.__writer_ver is None: self.__reader_ver = 0 self.__writer_ver = 1