def test_cannot_acquire_lock_twice(): """Test timeout on Lock.acquire().""" lock = Lock('lockfile', timeout=0.3) with lock: with pytest.raises(TimeoutError): with lock: ...
def __init__(self, filename=None, create_indices=True, use_lock_file=False): self.filename = filename self.create_indices = create_indices if use_lock_file and isinstance(filename, str): self.lock = Lock(filename + '.lock', world=DummyMPI()) else: self.lock = None
def test_lock_close_file_descriptor(): """Test that lock file descriptor is properly closed.""" # The choice of timeout=1.0 is arbitrary but we don't want to use # something that is too large since it could mean that the test # takes long to fail. lock = Lock('lockfile', timeout=1.0) with lock: assert not lock.fd.closed assert lock.fd.closed
def test_lock(): """Test timeout on Lock.acquire().""" from ase.utils import Lock from ase.test import must_raise lock = Lock('lockfile', timeout=0.3) with lock: with must_raise(TimeoutError): with lock: ...
def __init__(self, filename=None, create_indices=True, use_lock_file=False): self.filename = filename self.create_indices = create_indices if use_lock_file: self.lock = Lock(filename + '.lock', world=DummyMPI()) else: self.lock = None self.timestamp = None # timestamp form last write
def test_lock_close_file_descriptor(): """Test that lock file descriptor is properly closed.""" # The choice of timeout=1.0 is arbitrary but we don't want to use # something that is too large since it could mean that the test # takes long to fail. lock = Lock('lockfile', timeout=1.0) with lock: pass # If fstat raises OSError this means that fd.close() was # not called. with pytest.raises(OSError): os.fstat(lock.fd.name)
def create_figures(self, row, prefix, tmpdir, functions): with Lock('ase.db.web.lock'): for func, filenames in functions: for filename in filenames: try: os.remove(filename) except OSError: # Python 3 only: FileNotFoundError pass func(row) for filename in filenames: path = os.path.join(tmpdir, prefix + filename) if os.path.isfile(filename): shutil.move(filename, path) else: # Create an empty file: with open(path, 'w'): pass
def __init__(self, filename=None, create_indices=True, use_lock_file=False, serial=False): """Database object. serial: bool Let someone else handle parallelization. Default behavior is to interact with the database on the master only and then distribute results to all slaves. """ if isinstance(filename, basestring): filename = os.path.expanduser(filename) self.filename = filename self.create_indices = create_indices if use_lock_file and isinstance(filename, basestring): self.lock = Lock(filename + '.lock', world=DummyMPI()) else: self.lock = None self.serial = serial self._metadata = None # decription of columns and other stuff
class Task: taskname = 'generic-task' def __init__(self, calcfactory='emt', tag=None, magmoms=None, gui=False, write_summary=False, use_lock_files=False, write_to_file=None, slice=slice(None), logfile='-', collection=None): """Generic task object. This task will do a single of the energy and forces for the configurations that subcalsses define in their build_system() methods. calcfactory: CalculatorFactory object or str A calculator factory or the name of a calculator. collection: dict Collection of systems. For the meaning of the other arguments, see the add_options() and parse_args() methods.""" self.set_calculator_factory(calcfactory) self.tag = tag self.magmoms = magmoms self.gui = gui self.write_summary = write_summary self.use_lock_files = use_lock_files self.write_to_file = write_to_file self.slice = slice if world.rank == 0: if logfile is None: logfile = devnull elif isinstance(logfile, str): if logfile == '-': logfile = sys.stdout else: logfile = open(logfile, 'w') else: logfile = devnull self.logfile = logfile self.data = {} self.interactive_python_session = False self.contains = None self.modify = None self.after = None self.clean = False self.write_funcs = [] self.lock = None self.summary_keys = ['energy'] self.collection = collection def set_calculator_factory(self, calcfactory): if isinstance(calcfactory, str): calcfactory = calculator_factory(calcfactory) self.calcfactory = calcfactory def log(self, *args, **kwargs): prnt(file=self.logfile, *args, **kwargs) def get_filename(self, name=None, ext=None): filename = self.taskname if self.tag: filename += '-' + self.tag if name: filename = name + '-' + filename if ext: filename += '.' + ext return filename def expand(self, names): """Expand ranges like H-Li to H, He, Li.""" if isinstance(names, str): names = [names] newnames = [] for name in names: if name.count('-') == 1: s1, s2 = name.split('-') Z1 = atomic_numbers.get(s1) Z2 = atomic_numbers.get(s2) if Z1 is None or Z2 is None: newnames.append(name) else: newnames.extend(chemical_symbols[Z1:Z2 + 1]) else: newnames.append(name) return newnames def exclude(self, names): newnames = [] for name in names: atoms = self.create_system(name) if (self.contains is None or self.contains in atoms.get_chemical_symbols()): newnames.append(name) return newnames def run(self, names=None): """Run task for all names. The task will be one of these four: * Open ASE's GUI * Write configuration to file * Write summary * Do the actual calculation """ if self.lock is None: # Create lock object: self.lock = Lock(self.get_filename(ext='lock')) if self.clean: self.clean_json_file(names) return if names is None or len(names) == 0: names = self.collection.keys() names = self.expand(names) names = names[self.slice] names = self.exclude(names) if self.gui: for name in names: view(self.create_system(name)) return if self.write_to_file: if self.write_to_file[0] == '.': for name in names: filename = self.get_filename(name, self.write_to_file) write(filename, self.create_system(name)) else: assert len(names) == 1 write(self.write_to_file, self.create_system(names[0])) return if self.write_summary: self.read() self.analyse() self.summarize(names) return atoms = None for name in names: if self.use_lock_files: try: filename = self.get_filename(ext='json') self.lock.acquire() if os.path.isfile(filename): data = read_json(filename) if name not in data: data[name] = {} write_json(filename, data) else: self.log('Skipping', name) continue else: write_json(filename, {name: {}}) finally: self.lock.release() if atoms is not None: del atoms.calc atoms = self.run_single(name) return atoms def run_single(self, name): self.log('Running', name) try: atoms = self.create_system(name) except Exception: self.log(name, 'FAILED') traceback.print_exc(file=self.logfile) return atoms.calc = self.calcfactory(self.get_filename(name), atoms) tstart = time() try: data = self.calculate(name, atoms) if self.after: exec self.after except KeyboardInterrupt: raise except Exception: self.log(name, 'FAILED') traceback.print_exc(file=self.logfile) return tstop = time() data['time'] = tstop - tstart filename = self.get_filename(ext='json') try: self.lock.acquire() if os.path.isfile(filename): alldata = read_json(filename) else: alldata = {} alldata[name] = data write_json(filename, alldata) finally: self.lock.release() for write in self.write_funcs: filenamebase = self.get_filename(name) write(filenamebase, atoms, data) return atoms def create_system(self, name): if '.' in name: system = read(name) else: if self.collection is None: system = self.build_system(name) else: system = self.collection[name] if self.magmoms is not None: system.set_initial_magnetic_moments( np.tile(self.magmoms, len(system) // len(self.magmoms))) if self.modify: exec self.modify return system def calculate(self, name, atoms): e = atoms.get_potential_energy() data = {'energy': e} return data def read(self, skipempty=True): self.data = read_json(self.get_filename(ext='json')) if skipempty: self.data = dict((key.encode('ascii'), value) for key, value in self.data.items() if value) def clean_json_file(self, names=None): self.read(skipempty=False) n = len(self.data) if names: for name in names: del self.data[name] else: self.data = dict((key, value) for key, value in self.data.items() if value) filename = self.get_filename(ext='json') try: self.lock.acquire() write_json(filename, self.data) finally: self.lock.release() n -= len(self.data) self.log('Cleaned', n, ['tasks', 'task'][n == 1]) def summarize(self, names): lines = [['name'] + self.summary_keys] for name in names: if name not in self.data: continue d = self.data[name] line = [name] for key in self.summary_keys: if key in d: line.append('%.6f' % d[key]) else: line.append('') lines.append(line) lengths = [max(len(line[i]) for line in lines) for i in range(len(lines[0]))] for line in lines: for txt, l in zip(line, lengths): self.log('%*s' % (l, txt), end=' ') self.log() def create_parser(self): calcname = self.calcfactory.name parser = optparse.OptionParser( usage='%prog [options] system(s)', description='Run %s calculation.' % calcname) self.add_options(parser) return parser def add_options(self, parser): general = optparse.OptionGroup(parser, 'General') general.add_option('-t', '--tag', help='String tag added to filenames.') general.add_option('-M', '--magnetic-moment', metavar='M1,M2,...', help='Magnetic moment(s). ' + 'Use "-M 1" or "-M 2.3,-2.3".') general.add_option('-G', '--gui', action='store_true', help="Pop up ASE's GUI.") general.add_option('-s', '--write-summary', action='store_true', help='Write summary.') general.add_option('--slice', metavar='start:stop:step', help='Select subset of calculations using ' + 'Python slice syntax. ' + 'Use "::2" to do every second calculation and ' + '"-5:" to do the last five.') general.add_option('-w', '--write-to-file', metavar='FILENAME', help='Write configuration to file.') general.add_option('-i', '--interactive-python-session', action='store_true', help='Run calculation inside interactive Python ' + 'session. A possible $PYTHONSTARTUP script ' + 'will be imported and the "atoms" variable ' + 'refers to the Atoms object.') general.add_option('-l', '--use-lock-files', action='store_true', help='Skip calculations where the json ' + 'lock-file or result file already exists.') general.add_option('--contains', metavar='ELEMENT', help='Run only systems containing specific ' + 'element.') general.add_option('--modify', metavar='...', help='Modify system with Python statement. ' + 'Example: --modify="system.positions[-1,2]+=0.1".') general.add_option('--after', metavar='...', help='Execute Python statement after calculation.') general.add_option('--clean', action='store_true', help='Remove unfinished tasks from json file.') parser.add_option_group(general) def parse_args(self, args=None): if args is None: args = sys.argv[1:] parser = self.create_parser() self.calcfactory.add_options(parser) opts, args = parser.parse_args(args) if len(args) == 0 and self.collection is None: parser.error('incorrect number of arguments') self.parse(opts, args) self.calcfactory.parse(opts, args) return args def parse(self, opts, args): if opts.tag: self.tag = opts.tag if opts.magnetic_moment: self.magmoms = np.array( [float(m) for m in opts.magnetic_moment.split(',')]) self.gui = opts.gui self.write_summary = opts.write_summary self.write_to_file = opts.write_to_file self.use_lock_files = opts.use_lock_files self.interactive_python_session = opts.interactive_python_session self.contains = opts.contains self.modify = opts.modify if self.modify is not None: try: assert not isinstance(eval(self.modify), str), \ '--modify option received a string argument. Check quoting.' except SyntaxError: pass self.after = opts.after self.clean = opts.clean if opts.slice: self.slice = string2index(opts.slice)
def run(self, names=None): """Run task for all names. The task will be one of these four: * Open ASE's GUI * Write configuration to file * Write summary * Do the actual calculation """ if self.lock is None: # Create lock object: self.lock = Lock(self.get_filename(ext='lock')) if self.clean: self.clean_json_file(names) return if names is None or len(names) == 0: names = self.collection.keys() names = self.expand(names) names = names[self.slice] names = self.exclude(names) if self.gui: for name in names: view(self.create_system(name)) return if self.write_to_file: if self.write_to_file[0] == '.': for name in names: filename = self.get_filename(name, self.write_to_file) write(filename, self.create_system(name)) else: assert len(names) == 1 write(self.write_to_file, self.create_system(names[0])) return if self.write_summary: self.read() self.analyse() self.summarize(names) return atoms = None for name in names: if self.use_lock_files: try: filename = self.get_filename(ext='json') self.lock.acquire() if os.path.isfile(filename): data = read_json(filename) if name not in data: data[name] = {} write_json(filename, data) else: self.log('Skipping', name) continue else: write_json(filename, {name: {}}) finally: self.lock.release() if atoms is not None: del atoms.calc atoms = self.run_single(name) return atoms
"""Test timeout on Lock.acquire().""" from ase.utils import Lock from ase.test import must_raise lock = Lock('lockfile', timeout=0.3) with lock: with must_raise(TimeoutError): with lock: ...
class Task: taskname = 'generic-task' def __init__(self, calcfactory='emt', tag=None, magmoms=None, gui=False, write_summary=False, use_lock_files=False, write_to_file=None, slice=slice(None), logfile='-', collection=None): """Generic task object. This task will do a single of the energy and forces for the configurations that subcalsses define in their build_system() methods. calcfactory: CalculatorFactory object or str A calculator factory or the name of a calculator. collection: dict Collection of systems. For the meaning of the other arguments, see the add_options() and parse_args() methods.""" self.set_calculator_factory(calcfactory) self.tag = tag self.magmoms = magmoms self.gui = gui self.write_summary = write_summary self.use_lock_files = use_lock_files self.write_to_file = write_to_file self.slice = slice if world.rank == 0: if logfile is None: logfile = devnull elif isinstance(logfile, str): if logfile == '-': logfile = sys.stdout else: logfile = open(logfile, 'w') else: logfile = devnull self.logfile = logfile self.data = {} self.interactive_python_session = False self.contains = None self.modify = None self.after = None self.clean = False self.write_funcs = [] self.lock = None self.summary_keys = ['energy'] self.collection = collection def set_calculator_factory(self, calcfactory): if isinstance(calcfactory, str): calcfactory = calculator_factory(calcfactory) self.calcfactory = calcfactory def log(self, *args, **kwargs): prnt(file=self.logfile, *args, **kwargs) def get_filename(self, name=None, ext=None): filename = self.taskname if self.tag: filename += '-' + self.tag if name: filename = name + '-' + filename if ext: filename += '.' + ext return filename def expand(self, names): """Expand ranges like H-Li to H, He, Li.""" if isinstance(names, str): names = [names] newnames = [] for name in names: if name.count('-') == 1: s1, s2 = name.split('-') Z1 = atomic_numbers.get(s1) Z2 = atomic_numbers.get(s2) if Z1 is None or Z2 is None: newnames.append(name) else: newnames.extend(chemical_symbols[Z1:Z2 + 1]) else: newnames.append(name) return newnames def exclude(self, names): newnames = [] for name in names: atoms = self.create_system(name) if (self.contains is None or self.contains in atoms.get_chemical_symbols()): newnames.append(name) return newnames def run(self, names=None): """Run task for all names. The task will be one of these four: * Open ASE's GUI * Write configuration to file * Write summary * Do the actual calculation """ if self.lock is None: # Create lock object: self.lock = Lock(self.get_filename(ext='lock')) if self.clean: self.clean_json_file(names) return if names is None or len(names) == 0: names = self.collection.keys() names = self.expand(names) names = names[self.slice] names = self.exclude(names) if self.gui: for name in names: view(self.create_system(name)) return if self.write_to_file: if self.write_to_file[0] == '.': for name in names: filename = self.get_filename(name, self.write_to_file) write(filename, self.create_system(name)) else: assert len(names) == 1 write(self.write_to_file, self.create_system(names[0])) return if self.write_summary: self.read() self.analyse() self.summarize(names) return atoms = None for name in names: if self.use_lock_files: try: filename = self.get_filename(ext='json') self.lock.acquire() if os.path.isfile(filename): data = read_json(filename) if name not in data: data[name] = {} write_json(filename, data) else: self.log('Skipping', name) continue else: write_json(filename, {name: {}}) finally: self.lock.release() if atoms is not None: del atoms.calc atoms = self.run_single(name) return atoms def run_single(self, name): self.log('Running', name) try: atoms = self.create_system(name) except Exception: self.log(name, 'FAILED') traceback.print_exc(file=self.logfile) return atoms.calc = self.calcfactory(self.get_filename(name), atoms) tstart = time() try: data = self.calculate(name, atoms) if self.after: exec self.after except KeyboardInterrupt: raise except Exception: self.log(name, 'FAILED') traceback.print_exc(file=self.logfile) return tstop = time() data['time'] = tstop - tstart filename = self.get_filename(ext='json') try: self.lock.acquire() if os.path.isfile(filename): alldata = read_json(filename) else: alldata = {} alldata[name] = data write_json(filename, alldata) finally: self.lock.release() for write in self.write_funcs: filenamebase = self.get_filename(name) write(filenamebase, atoms, data) return atoms def create_system(self, name): if '.' in name: system = read(name) else: if self.collection is None: system = self.build_system(name) else: system = self.collection[name] if self.magmoms is not None: system.set_initial_magnetic_moments( np.tile(self.magmoms, len(system) // len(self.magmoms))) if self.modify: exec self.modify return system def calculate(self, name, atoms): e = atoms.get_potential_energy() data = {'energy': e} return data def read(self, skipempty=True): self.data = read_json(self.get_filename(ext='json')) if skipempty: self.data = dict((key.encode('ascii'), value) for key, value in self.data.items() if value) def clean_json_file(self, names=None): self.read(skipempty=False) n = len(self.data) if names: for name in names: del self.data[name] else: self.data = dict( (key, value) for key, value in self.data.items() if value) filename = self.get_filename(ext='json') try: self.lock.acquire() write_json(filename, self.data) finally: self.lock.release() n -= len(self.data) self.log('Cleaned', n, ['tasks', 'task'][n == 1]) def summarize(self, names): lines = [['name'] + self.summary_keys] for name in names: if name not in self.data: continue d = self.data[name] line = [name] for key in self.summary_keys: if key in d: line.append('%.6f' % d[key]) else: line.append('') lines.append(line) lengths = [ max(len(line[i]) for line in lines) for i in range(len(lines[0])) ] for line in lines: for txt, l in zip(line, lengths): self.log('%*s' % (l, txt), end=' ') self.log() def create_parser(self): calcname = self.calcfactory.name parser = optparse.OptionParser(usage='%prog [options] system(s)', description='Run %s calculation.' % calcname) self.add_options(parser) return parser def add_options(self, parser): general = optparse.OptionGroup(parser, 'General') general.add_option('-t', '--tag', help='String tag added to filenames.') general.add_option('-M', '--magnetic-moment', metavar='M1,M2,...', help='Magnetic moment(s). ' + 'Use "-M 1" or "-M 2.3,-2.3".') general.add_option('-G', '--gui', action='store_true', help="Pop up ASE's GUI.") general.add_option('-s', '--write-summary', action='store_true', help='Write summary.') general.add_option('--slice', metavar='start:stop:step', help='Select subset of calculations using ' + 'Python slice syntax. ' + 'Use "::2" to do every second calculation and ' + '"-5:" to do the last five.') general.add_option('-w', '--write-to-file', metavar='FILENAME', help='Write configuration to file.') general.add_option('-i', '--interactive-python-session', action='store_true', help='Run calculation inside interactive Python ' + 'session. A possible $PYTHONSTARTUP script ' + 'will be imported and the "atoms" variable ' + 'refers to the Atoms object.') general.add_option('-l', '--use-lock-files', action='store_true', help='Skip calculations where the json ' + 'lock-file or result file already exists.') general.add_option('--contains', metavar='ELEMENT', help='Run only systems containing specific ' + 'element.') general.add_option('--modify', metavar='...', help='Modify system with Python statement. ' + 'Example: --modify="system.positions[-1,2]+=0.1".') general.add_option('--after', metavar='...', help='Execute Python statement after calculation.') general.add_option('--clean', action='store_true', help='Remove unfinished tasks from json file.') parser.add_option_group(general) def parse_args(self, args=None): if args is None: args = sys.argv[1:] parser = self.create_parser() self.calcfactory.add_options(parser) opts, args = parser.parse_args(args) if len(args) == 0 and self.collection is None: parser.error('incorrect number of arguments') self.parse(opts, args) self.calcfactory.parse(opts, args) return args def parse(self, opts, args): if opts.tag: self.tag = opts.tag if opts.magnetic_moment: self.magmoms = np.array( [float(m) for m in opts.magnetic_moment.split(',')]) self.gui = opts.gui self.write_summary = opts.write_summary self.write_to_file = opts.write_to_file self.use_lock_files = opts.use_lock_files self.interactive_python_session = opts.interactive_python_session self.contains = opts.contains self.modify = opts.modify if self.modify is not None: try: assert not isinstance(eval(self.modify), str), \ '--modify option received a string argument. Check quoting.' except SyntaxError: pass self.after = opts.after self.clean = opts.clean if opts.slice: self.slice = string2index(opts.slice)