def regenerate(self, directory=None): """See `IMailTransportAgentLifecycle`.""" # Acquire a lock file to prevent other processes from racing us here. if directory is None: directory = config.DATA_DIR lock_file = os.path.join(config.LOCK_DIR, 'mta') with Lock(lock_file): lmtp_path = os.path.join(directory, 'postfix_lmtp') lmtp_path_new = lmtp_path + '.new' with open(lmtp_path_new, 'w') as fp: self._generate_lmtp_file(fp) # Atomically rename to the intended path. os.rename(lmtp_path_new, lmtp_path) domains_path = os.path.join(directory, 'postfix_domains') domains_path_new = domains_path + '.new' with open(domains_path_new, 'w') as fp: self._generate_domains_file(fp) # Atomically rename to the intended path. os.rename(domains_path_new, domains_path) # If the transport_file_type is 'hash' then run the postmap command # on newly generated file to convert them in to hash table like # Postfix wants. if self.transport_file_type == 'hash': errors = [] for path in (lmtp_path, domains_path): command = self.postmap_command + ' ' + path status = (os.system(command) >> 8) & 0xff if status: msg = 'command failure: %s, %s, %s' errstr = os.strerror(status) log.error(msg, command, status, errstr) errors.append(msg % (command, status, errstr)) if errors: raise RuntimeError(NL.join(errors))
def get_next_apk(apks_dir): """ Gets the first available (not-locked) apk files and locks it :param apks_dir: directory to scan :return: a tuple containing the apk's path and the locked lock :rtype: (str, lockfile.LockFile) """ try: files = os.listdir(apks_dir) except FileNotFoundError: # folder doesn't exist return None, None for f in files: if not f.endswith(".apk"): continue f = os.path.join(apks_dir, f) try: # lock file should not exist filename = f + LOCK_PREFIX lock = Lock(filename, lifetime=datetime.timedelta( seconds=6000)) # expires in 10 minutes if not lock.is_locked: lock.lock(timeout=datetime.timedelta(milliseconds=350)) if os.path.isfile( f ): # the original file could be deleted in the meantime return f, lock if lock.is_locked: lock.unlock() except (AlreadyLockedError, TimeOutError): # some other process is analyzing the file; go ahead and look for another file pass return None, None
def dump_to_file(entries, package, version_code, version_name): lock_acquired = False while not lock_acquired: try: filename = dest + LOCK_PREFIX lock = Lock(filename, lifetime=datetime.timedelta(seconds=6000)) # expires in 10 minutes if not lock.is_locked: lock.lock(timeout=datetime.timedelta(milliseconds=350)) lock_acquired = True with open(dest, 'a') as f: first = True if os.path.exists(dest) and os.path.getsize(dest) > 0: first = False for entry in entries: entry_dict = entry.__dict__ entry_dict['package'] = package entry_dict['versionCode'] = version_code entry_dict['versionName'] = version_name if first: first = False else: f.write(",\n") json.dump(entry_dict, f, indent=4) if lock.is_locked: lock.unlock() except (AlreadyLockedError, TimeOutError): # some other process is analyzing the file; go ahead and look for another file pass
def run_capsule(self, run_info, gpu_idx): lock_path = os.path.join(run_info['trial_dir'], 'lock') lock = Lock(lock_path, datetime.timedelta(days=365)) lock.lock(timeout=datetime.timedelta(seconds=1)) if not lock.is_locked: self.print_log('locking failed for', run_info['trial_id']) return None # run capsule env = {'CUDA_VISIBLE_DEVICES': str(gpu_idx), 'INFR_TRIAL': run_info['trial_id'], 'INFR_EXP_PATH': self.experiment_dir, 'INFR_MODE': self.mode, 'INFR_REDIRECT_IO': '1', 'INFR_START_STATE': os.path.join(run_info['trial_dir'], 'start_state.json')} if self.cuda_sync: env['CUDA_LAUNCH_BLOCKING'] = '1' proc = subprocess.Popen([sys.executable, '-m', run_info['start_state']['module_name']], env=env) self.print_log('started worker', proc.pid, 'for', run_info['trial_id'], self.mode) return {'trial_dir': run_info['trial_dir'], 'trial_id': run_info['trial_id'], 'start_at': time.time(), 'lock': lock, 'gpu_idx': gpu_idx, 'proc': proc, 'pid': proc.pid, 'ret_code': None}
def dump_strings(self, entries): lock_acquired = False while not lock_acquired: try: filename = self.strings_dest + LOCK_PREFIX lock = Lock(filename, lifetime=datetime.timedelta( seconds=6000)) # expires in 10 minutes if not lock.is_locked: lock.lock(timeout=datetime.timedelta(milliseconds=350)) lock_acquired = True with open(self.strings_dest, 'a') as f: first = True if os.path.exists( self.strings_dest) and os.path.getsize( self.strings_dest) > 0: first = False for entry in entries: entry_dict = entry.__dict__ if first: first = False else: f.write("\n") json.dump(entry_dict, f) if lock.is_locked: lock.unlock() except (AlreadyLockedError, TimeOutError): # some other process is analyzing the file; go ahead and look for another file pass
def acquire_lock_1(force, lock_file=None): """Try to acquire the master lock. :param force: Flag that controls whether to force acquisition of the lock. :type force: bool :param lock_file: Path to the lock file, otherwise `config.LOCK_FILE`. :type lock_file: str :return: The master lock. :raises: `TimeOutError` if the lock could not be acquired. """ if lock_file is None: lock_file = config.LOCK_FILE lock = Lock(lock_file, LOCK_LIFETIME) try: lock.lock(timedelta(seconds=0.1)) return lock except TimeOutError: if not force: raise # Force removal of lock first. hostname, pid, tempfile = lock.details os.unlink(lock_file) # Also remove any stale claim files. dname = os.path.dirname(lock_file) for fname in os.listdir(dname): fpath = os.path.join(dname, fname) if fpath.startswith(lock_file): os.unlink(fpath) return acquire_lock_1(force=False)
def load_json(path, require_exclusive=True, lock_path=None): """ Shortcut for loading json from a file path. :param path: The full path to the file :type: path: str :param require_exclusive: lock file for exclusive read :type require_exclusive: bool :param lock_path: path for the lock file to use :type lock_path: string :returns: loaded json :rtype: dict :raises: IOError, ValueError """ lock = None if require_exclusive: if not lock_path: lock_path = get_lock_path(path) lock = Lock(lock_path) lock.lock() try: with open(path) as f: return json.load(f) finally: if lock: lock.unlock(unconditionally=True)
def master_state(lock_file=None): """Get the state of the master watcher. :param lock_file: Path to the lock file, otherwise `config.LOCK_FILE`. :type lock_file: str :return: 2-tuple of the WatcherState describing the state of the lock file, and the lock object. """ if lock_file is None: lock_file = config.LOCK_FILE # We'll never acquire the lock, so the lifetime doesn't matter. lock = Lock(lock_file) try: hostname, pid, tempfile = lock.details except NotLockedError: return WatcherState.none, lock if hostname != socket.getfqdn(): return WatcherState.host_mismatch, lock # Find out if the process exists by calling kill with a signal 0. try: os.kill(pid, 0) return WatcherState.conflict, lock except ProcessLookupError: # No matching process id. return WatcherState.stale_lock, lock
def test_archive_lock_used(self): # Test that locking the maildir when adding works as a failure here # could mean we lose mail. lock_file = os.path.join( config.LOCK_DIR, '{0}-maildir.lock'.format( self._mlist.fqdn_listname)) with Lock(lock_file): # Acquire the archiver lock, then make sure the archiver logs the # fact that it could not acquire the lock. archive_thread = threading.Thread( target=Prototype.archive_message, args=(self._mlist, self._msg)) mark = LogFileMark('mailman.error') archive_thread.run() # Test that the archiver output the correct error. line = mark.readline() # XXX 2012-03-15 BAW: we really should remove timestamp prefixes # from the loggers when under test. self.assertTrue(line.endswith( 'Unable to acquire prototype archiver lock for {0}, ' 'discarding: {1}\n'.format( self._mlist.fqdn_listname, self._msg.get('message-id')))) # Check that the message didn't get archived. created_files = self._find(config.ARCHIVE_DIR) self.assertEqual(self._expected_dir_structure, created_files)
def load_store_matrix(self, metadata, as_of_dates, return_matrix=True): """ Calls get_dataset to return a pandas DataFrame for the as_of_dates selected Args: list as_of_dates: as_of_dates to use Returns: matrix: dataframe with the features and the last column as the label (called: outcome) """ uuid = metta.metta_io.generate_uuid(metadata) matrix_filename = self.matrices_path + '/' + uuid with Lock(matrix_filename + '.lock', lifetime=datetime.timedelta(minutes=20)): if os.path.isfile(matrix_filename + '.h5'): log.debug(' Matrix {} already stored'.format(uuid)) if return_matrix: df = metta.metta_io.recover_matrix(metadata, self.matrices_path) return df, uuid else: df = self.feature_loader.get_dataset(as_of_dates) log.debug('storing matrix {}'.format(uuid)) metta.metta_io.archive_matrix(matrix_config=metadata, df_matrix=df, directory=self.matrices_path, format='hd5') if return_matrix: return df, uuid
def write_json(path, data, lock_path=None, merge_func=None): """ Shortcut for writing a structure as json to the file system. merge_func is a callable that takes two dict and merges them together. :param path: The full path to the file to write :type: path: str :param data: structure to write out as json :type data: dict or list :param lock_path: path for the lock file to use :type lock_path: string :raises: ValueError, OSError """ # lock before moving if not lock_path: lock_path = get_lock_path(path) with Lock(lock_path): if callable(merge_func): try: disk_data = load_json(path, require_exclusive=False) except FileNotFoundError: disk_data = {} mem_data = data.copy() data = merge_func(disk_data, mem_data) # we could probably write directly to the file, # but set the permissions to RO dn = os.path.dirname(path) f = tempfile.NamedTemporaryFile(mode='w', dir=dn, delete=False) json.dump(data, f, indent=4) os.fchmod(f.file.fileno(), 0o644) shutil.move(f.name, path)
def child_locker(filename, lifetime, queue): # First, acquire the file lock. with Lock(filename, lifetime): # Now inform the parent that we've acquired the lock. queue.put(True) # Keep the file lock for a while. time.sleep(lifetime.seconds - 1)
def __init__(self, lifetime=LOCK_LIFETIME, timeout=LOCK_TIMEOUT): """ Create a SafeUpdater with the given lock lifetime and timeout (see flufl.lock documentation). The defaults are the lifetime and timeout found in the config. """ self.lock = Lock(LOCK_FILE, lifetime=lifetime) self.timeout = timeout
def test_acquire_lock_1_force(self): # Create the lock and lock it. my_lock = Lock(self.lock_file) my_lock.lock(timedelta(seconds=60)) # Try to aquire it again with force. lock = master.acquire_lock_1(True, self.lock_file) self.assertTrue(lock.is_locked) lock.unlock()
def write_job(job_fullpath, job_spec): """ TODO: Docstring """ # Write the contents to a job with Lock(lock_file): with open(job_fullpath, "w") as ofp: json.dump(job_spec, ofp, sort_keys=True)
def write_env(job_fullpath, env): """TODO: write""" env_fullpath = job_fullpath.replace(".job", ".env") # write env to env_fullpath with Lock(lock_file): with open(env_fullpath, "w") as ofp: json.dump(env, ofp, sort_keys=True)
def remove_job(job_fullpath): """ TODO: writeme """ with Lock(lock_file): if os.path.exists(job_fullpath): os.remove(job_fullpath) if os.path.exists(job_fullpath.replace(".job", ".env")): os.remove(job_fullpath.replace(".job", ".env"))
def _lock(self): if self._lockobj is None: # These will get automatically cleaned up by the test # infrastructure. self._uid_file = os.path.join(config.VAR_DIR, '.uid') if self._context is not None: self._uid_file += '.' + self._context self._lock_file = self._uid_file + '.lock' self._lockobj = Lock(self._lock_file) return self._lockobj
def create(): """See `IDatabaseFactory`.""" with Lock(os.path.join(config.LOCK_DIR, 'dbcreate.lck')): database_class = config.database['class'] database = call_name(database_class) verifyObject(IDatabase, database) database.initialize() SchemaManager(database).setup_database() database.commit() return database
def read_job(job_fullpath): """ TODO: Docstring """ # Parse the contents of the job with Lock(lock_file): with open(job_fullpath, "r") as ifp: job_spec = json.load(ifp) return job_spec
def read_env(job_fullpath): """TODO: write""" env_fullpath = job_fullpath.replace(".job", ".env") # read env from env_fullpath with Lock(lock_file): with open(env_fullpath, "r") as ifp: env = json.load(ifp) return env
def ensure_directories_exist(self): """Create all the paths if the directories do not exist.""" if self.create_paths: for path_name, path in self.paths.items(): makedirs(path) # Create a paper-git.cfg file if it already doesn't exist. lock_file = os.path.join(self.VAR_DIR, 'paper-git-cfg.lck') paper_git_cfg = os.path.join(self.ETC_DIR, 'paper-git.cfg') with Lock(lock_file): if not os.path.exists(paper_git_cfg): with open(paper_git_cfg, 'w') as fp: print(CFG_TEMPLATE, file=fp)
def setupTest(self, param, testDataName): # Lock to prevent race condition lock_file = "lift/lib/.locks/setup.lock" lock = Lock(lock_file) lock.lifetime = timedelta(days=2) lock.lock() # --------------------------------------------------------------------- # Base path # for dataset self.dataset = os.getenv('PROJ_DATA_DIR', '') if self.dataset == '': self.dataset = os.path.expanduser("~/Datasets") self.dataset += "/" + testDataName # for temp self.temp = os.getenv('PROJ_TEMP_DIR', '') if self.temp == '': self.temp = os.path.expanduser("~/Temp") self.temp += "/" + testDataName # for volatile temp self.volatile_temp = os.getenv('PROJ_VOLTEMP_DIR', '') if self.volatile_temp == '': self.volatile_temp = "/scratch/" + os.getenv('USER') + "/Temp" self.volatile_temp += "/" + testDataName # --------------------------------------------------------------------- # Path for data loading self.train_data = None # block these as they should not be used self.train_mask = None # block these as they should not be used self.debug = self.dataset + "/debug/" + self.prefix_dataset(param) # --------------------------------------------------------------------- # Path for the model learning resdir = os.getenv('PROJ_RES_DIR', '') if resdir == '': resdir = os.path.expanduser("~/Results") self.result = (resdir + "/" + self.getResPrefix(param) + self.prefix_dataset(param) + self.prefix_patch(param) + self.prefix_learning(param)) # Check if the un-sorted prefix exists unsorted_hash_path = (resdir + "/" + self.getResPrefix(param, do_sort=False) + self.prefix_dataset(param, do_sort=False) + self.prefix_patch(param, do_sort=False) + self.prefix_learning(param, do_sort=False)) if os.path.exists(unsorted_hash_path): shutil.copytree(unsorted_hash_path, self.result) shutil.rmtree(unsorted_hash_path) lock.unlock()
def test_master_state(self): my_lock = Lock(self.lock_file) # Mailman is not running. state, lock = master.master_state(self.lock_file) self.assertEqual(state, master.WatcherState.none) # Acquire the lock as if another process had already started the # master. my_lock.lock() try: state, lock = master.master_state(self.lock_file) finally: my_lock.unlock() self.assertEqual(state, master.WatcherState.conflict)
def test_master_state(self): my_lock = Lock(self.lock_file) # Mailman is not running. state, lock = master.master_state(self.lock_file) self.assertEqual(state, master.WatcherState.none) # Acquire the lock as if another process had already started the # master. Use a timeout to avoid this test deadlocking. my_lock.lock(timedelta(seconds=60)) try: state, lock = master.master_state(self.lock_file) finally: my_lock.unlock() self.assertEqual(state, master.WatcherState.conflict)
def run_with_lock(fn, *args, **kwargs): lock = Lock( getattr( settings, "HYPERKITTY_JOBS_UPDATE_INDEX_LOCKFILE", os.path.join(gettempdir(), "hyperkitty-jobs-update-index.lock"))) if lock.is_locked: log.warning( "Update index lock is accquited by: {}".format(*lock.details)) return with lock: try: fn(*args, **kwargs) except Exception as e: log.exception("Failed to update the fulltext index: %s", e)
def append_to_csv(self, dataframe, file_name): '''Append a pandas dataframe to a csv file. This function is thread save.''' # Make a lock lock_name = os.path.join(file_name + '.lock') lock = Lock(lock_name) lock.lifetime = timedelta(minutes=10) # Write to the result file with a lock with lock: with open(file_name, 'a+') as f: dataframe.to_csv(f, header=False, index=False)
def __init__(self, path): # Specify the path to a file that will be used to synchronize the lock. # Per the flufl.lock documentation, use a file that does not exist. self._lock = Lock(path) # Locks have a lifetime (default 15 seconds) which is the period of time that the process expects # to keep the lock once it has been acquired. We set the lifetime to be 5 minutes as we expect # all operations that require locks to be completed within that time. self._lock.lifetime = timedelta(minutes=5) # Ensure multiple threads within a process run NFSLock operations one at a time. # We must acquire the reentrant lock before acquiring the flufl lock and only release after # the flufl lock is released. self._r_lock = threading.RLock()
def append_to_csv(data_list, file_name): '''Append a list of tuples to a csv file. This function is thread safe.''' # Make a lock lock_name = os.path.join(file_name + '.lock') lock = Lock(lock_name) lock.lifetime = timedelta(minutes=10) # Write to the result file with a lock with lock: with open(file_name, 'a+') as f: for t in data_list: f.write(','.join([str(x) for x in t]) + '\n')
def safe_append(file_name, string): '''Append a string to a file. This function is thread safe. ''' # Make a lock lock_name = os.path.join(file_name + '.lock') lock = Lock(lock_name) lock.lifetime = timedelta(minutes=10) # Write to the result file with a lock with lock: with open(file_name, 'a+') as f: f.write(string)