def build_sources(self): self.sources = {} # { source_context: hostname:source_path, ... } self.paths = {} # { source_context: local_path, ... } self.inventory = {} # { source_context: source:inventory() } # { filename: (size, ncopies), } self.backups = {} # local backups (intended or actual) # { source_context: {filename: size,}, } self.scanners = {} # my local storage (actual) self.claims = {} # { source_context: { filename : time() }, } self.random_source_list = [] # [ list, of, sources ] self.datagrams = {} # internal storage of connections self.metadata = {} # internal storage of server metadata lazy_write = get_interval(self.config, "LAZY WRITE", (self.context, )) source_contexts = self.config.get_contexts_for_key("source") self.prune_sources(source_contexts) for source_context, source in source_contexts.items(): self.sources[source_context] = source path = f"{self.path}/{source_context}" self.paths[source_context] = path self.scanners[source_context] = \ scanner.ScannerLite(source_context, path, pd_path=self.path, loglevel=logging.INFO, name=f"{self.context}:{source_context}") claims = f"{self.path}/claims-{self.context}:{source_context}.bz2" self.claims[source_context] = PersistentDict(claims, lazy_write=lazy_write) self.backups[source_context] = {} self.random_source_list.append(source_context) random.shuffle(self.random_source_list)
def __init__(self, context): super().__init__() self.context = context self.config = config.Config.instance() self.logger = logging.getLogger(logger_str(__class__) + " " + context) self.logger.info(f"Creating clientlet {self.context}") self.path = config.path_for(self.config.get(self.context, "backup")) assert os.path.exists(self.path), f"{self.path} does not exist!" # ALL source contexts (we care a lot) self.sources = {} self.scanners = {} self.random_source_list = [] self.build_sources() lazy_write = utils.str_to_duration( self.config.get(context, "LAZY WRITE", 5)) # TODO: my cache of claims should expire in rescan/2 self.rescan = self.get_interval("rescan") // 2 self.claims = PersistentDict(f"/tmp/cb.c{context}.json.bz2", lazy_write=5, expiry=self.rescan) self.drops = 0 # count the number of times I drop a file self.stats = stats.Stats() self.update_allocation() self.bailing = False self.datagrams = {}
def initialiseCachedParams(self): ''' Initialises default parameters from JSON file-backed store, or creates them if not existing in file from coded defaults ''' self.param_cache_file = self.default_config_path + "/lpdFemGui_config.json" self.cached_params = PersistentDict(self.param_cache_file, 'c', format='json') default_params = { 'connectionTimeout' : 5.0, 'femAddr' : '192.168.2.2', 'femPort' : 6969, 'readoutParamFile' : self.default_config_path + '/superModuleReadout.xml', 'cmdSequenceFile' : self.default_config_path + '/Command_LongExposure_V2.xml', 'setupParamFile' : self.default_config_path + '/Setup_LowPower.xml', 'dataFilePath' : '/tmp', 'analysisPdfPath' : '/tmp', 'hvBiasVolts' : 50.0, 'numTrains' : 8, 'externalTrigger' : True, 'triggerDelay' : 0, 'pwrAutoUpdate' : False, 'pwrUpdateInterval' : 0, 'runNumber' : 0, 'fileWriteEnable' : True, 'liveViewEnable' : False, 'liveViewDivisor' : 1, 'liveViewOffset' : 0, 'evrMcastGroup' : '239.255.16.17', 'evrMcastPort' : 10151, 'evrMcastInterface' : '172.21.22.69', 'evrRecordEnable' : True, 'femAsicGainOverride' : 8, 'femAsicPixelFeedbackOverride' : 0, 'asicModuleType' : 0, 'multiRunEnable' : True, 'multiRunNumRuns' : 123, 'receiveDataInternally': False, # Data received either internally or from ODIN 'odinFrCtrlChannel' : 'tcp://127.0.0.1:5000', 'odinFpCtrlChannel' : 'tcp://127.0.0.1:5004', # TODO: Plan file structure for odinDataConfigFile 'odinDataConfigFile' : self.default_config_path + '/odin_data_lpd_config.json' } # List of parameter names that don't need to force a system reconfigure self.non_volatile_params = ('fileWriteEnable', 'liveViewEnable', 'liveViewDivisor', 'liveViewOffset', 'pwrAutoUpdate', 'pwrUpdateInterval', 'dataFilePath', 'hvBiasBolts', 'multiRunEnable', 'multiNumRuns') # Load default parameters into cache if not already existing for param in default_params: if param not in self.cached_params: self.cached_params[param] = default_params[param] # Sync cached parameters back to file self.cached_params.sync()
def __init__(self, context): super().__init__() self.context = context logger_str = f"{utils.logger_str(__class__)} {context}" self.logger = logging.getLogger(logger_str) # self.logger.setLevel(logging.INFO) self.config = config.Config.instance() self.copies = int(self.config.get(self.context, "copies", 2)) self.path = config.path_for(self.config.get(self.context, "source")) self.scanner = scanner.ScannerLite(self.context, self.path) self.rescan = utils.get_interval(self.config, "rescan", self.context) lazy_write = self.config.get(context, "LAZY WRITE", 5) lazy_write = utils.str_to_duration(lazy_write) # self.clients: { filename : { client: expiry_time, } } clients_state = f"/tmp/cb.{context}-clients.json.bz2" self.clients = PersistentDict(clients_state, lazy_write=5) self.stats = stats.Stats() self.handling = False
def test_read_from_file(tmpfile, test_data): """Should properly read from existing file""" with open(tmpfile, 'wb') as fh: fh.write(pickle.dumps(test_data)) test_dict = PersistentDict(location=tmpfile) assert test_dict == test_data
def test_clear(tmpfile): """Clear must also clear storage file""" test_dict = PersistentDict({'a': 42, 'b': 1464}, location=tmpfile) test_dict.clear() assert test_dict == {} assert not os.path.exists(tmpfile)