def setup(self): """Seeds the index directory with empty stubs if the directory is consistently empty. Does not overwrite existing indexes.""" absent = False present = False makedirs(self._index_dir) for d in self._data_dict.values(): file_path = os.path.join(self._index_dir, d.get_file_name()) if os.path.exists(file_path): present = True else: absent = True if absent and present: raise search_errors.InconsistentIndexException( self._index_dir) if present: return if self.file_version_number: raise RuntimeError("Got file_version_number other than " "None in setup.") self.file_version_number = INITIAL_VERSION_NUMBER for d in self._data_dict.values(): d.write_dict_file(self._index_dir, self.file_version_number)
def consistent_open(data_list, directory, timeout = 1): """Opens all data holders in data_list and ensures that the versions are consistent among all of them. It retries several times in case a race condition between file migration and open is encountered. Note: Do not set timeout to be 0. It will cause an exception to be immediately raised. """ missing = None cur_version = None start_time = time.time() while cur_version == None and missing != True: # The assignments to cur_version and missing cannot be # placed here. They must be reset prior to breaking out of the # for loop so that the while loop condition will be true. They # cannot be placed after the for loop since that path is taken # when all files are missing or opened successfully. if timeout != None and ((time.time() - start_time) > timeout): raise search_errors.InconsistentIndexException( directory) for d in data_list: # All indexes must have the same version and all must # either be present or absent for a successful return. # If one of these conditions is not met, the function # tries again until it succeeds or the time spent in # in the function is greater than timeout. try: f = os.path.join(directory, d.get_file_name()) fh = open(f, 'r', encoding='UTF-8') # If we get here, then the current index file # is present. if missing == None: missing = False elif missing: for dl in data_list: dl.close_file_handle() missing = None cur_version = None break d.set_file_handle(fh, f) version_tmp = fh.readline() version_num = \ int(version_tmp.split(' ')[1].rstrip('\n')) # Read the version. If this is the first file, # set the expected version otherwise check that # the version matches the expected version. if cur_version == None: cur_version = version_num elif not (cur_version == version_num): # Got inconsistent versions, so close # all files and try again. for d in data_list: d.close_file_handle() missing = None cur_version = None break except IOError as e: if e.errno == errno.ENOENT: # If the index file is missing, ensure # that previous files were missing as # well. If not, try again. if missing == False: for d in data_list: d.close_file_handle() missing = None cur_version = None break missing = True else: for d in data_list: d.close_file_handle() raise if missing: assert cur_version == None # The index is missing (ie, no files were present). return None else: assert cur_version is not None return cur_version