def run(self): """ Continuously search for suitable candidates to evaluation among a list of databases. :return: """ procs = [] ids_running = [] # Creating a list to store the 'nconcurrent' running jobs for i in range(self.nconcurrent): procs.append(None) ids_running.append(None) # Main loop looking permanently for candidates for evaluation while True: to_evaluate = self.get_list_candidates() index = 0 currently_evaluating = 0 for j in range(self.nconcurrent): if procs[j] is not None and procs[j].is_alive(): currently_evaluating += 1 print('Candidates to evaluate: %d Candidates in evaluation: %d' % (len(to_evaluate), currently_evaluating)) while index < len(to_evaluate): db_settings = dict(self.db_settings) # The first component of each pair in to_evaluate is the name of the database db_settings['name'] = self.dbname pcdb = get_database(db_settings) slot = None while True: for j in range(self.nconcurrent): if procs[j] is None or not procs[j].is_alive(): slot = j if slot is None: time.sleep(self.sleeping_time) else: break # The function is_evaluated needs two arguments, the database object and entry identifier and # must return a boolean to decide if the candidate should be evaluated. if self.is_evaluated(pcdb, to_evaluate[index]): ids_running[slot] = to_evaluate[index] # This is the actual call to the worker, it must be a function with 4 arguments: # The database settings, the entry identifier, the working directory and arguments for the worker procs[j] = Process(target=self.worker, args=(db_settings, to_evaluate[index])) procs[j].start() else: pcm_log.debug('Not evaluable: %s' % to_evaluate[index]) index += 1 time.sleep(self.sleeping_time)
def __init__(self, dbname, source_dir=None, source_file=None, output_names=None, output_file=None, db_settings=None, nconcurrent=1): """ FireballCollector is a class collect Fireball executions into one PyChemia Database. The name of the database will be 'dbname' and executions will be processed from a give 'source_dir', a path to search for Fireball calculations or 'source_file', one file with one directory per line where Fireball executions are stored. The variable 'output_names' sets possible output files for Fireball. Typically Fireball output is the standard output and the user has to redirect the output to a file with an arbitrary name. The processing of directories happens in parallel using 'nconcurrent' python processes. By default the database is created on the local machine assuming no authorization or authentication, otherwise use the dictionary 'db_settings' to get more control on the database location and access. :param dbname: (str) Name of the database that will contain the Fireball executions :param source_dir: (str) Path for a directory that will be explored recursevely for suitable Fireball executions :param source_file: (str) Path to a file describing directories with Fireball executions, one per line :param output_names: (list) List of possible output filenames :param output_file: (str) Path to a file with possible output names, one per line :param db_settings: (dict) Extra parameters for locating and accessing the mongo server. :param nconcurrent: Number of concurrent python executions to fill the database (default: 1) """ self.db_settings = db_settings self.dbname = dbname self.source_dir = source_dir self.source_file = source_file self.output_names = output_names self.output_file = output_file self.nconcurrent = nconcurrent self.sleeping_time = 2 self.database = None if self.output_file is not None and os.path.exists(self.output_file): rf = open(self.output_file) self.output_names = [x.strip() for x in rf.readlines()] if self.source_dir is None and self.source_file is None: raise ValueError( "One of the variables source_dir or source_file need to be defined" ) if self.db_settings is None: self.db_settings = {'name': self.dbname} else: self.db_settings['name'] = self.dbname self.db_settings = dict(self.db_settings) # The first component of each pair in to_evaluate is the name of the database self.db_settings['name'] = self.dbname self.database = get_database(self.db_settings)
def run(self): """ Continuously search for suitable candidates to evaluation among a list of databases. :return: """ procs = [] ids_running = [] # Creating a list to store the 'nconcurrent' running jobs for i in range(self.nconcurrent): procs.append(None) ids_running.append(None) # Main loop looking permanently for candidates for evaluation while True: to_evaluate = self.get_list_candidates() index = 0 currently_evaluating = 0 for j in range(self.nconcurrent): if procs[j] is not None and procs[j].is_alive(): currently_evaluating += 1 print('Candidates to evaluate: %d Candidates in evaluation: %d' % (len(to_evaluate), currently_evaluating)) while index < len(to_evaluate): db_settings = dict(self.db_settings) # The first component of each pair in to_evaluate is the name of the database db_settings['name'] = self.dbname pcdb = get_database(db_settings) slot = None while True: for j in range(self.nconcurrent): if procs[j] is None or not procs[j].is_alive(): slot = j if slot is None: time.sleep(self.sleeping_time) else: break # The function is_evaluated needs two arguments, the database object and entry identifier and # must return a boolean to decide if the candidate should be evaluated. if self.is_evaluated(pcdb, to_evaluate[index]): ids_running[slot] = to_evaluate[index] # This is the actual call to the worker, it must be a function with 4 arguments: # The database settings, the entry identifier, the working directory and arguments for the worker procs[j] = Process(target=self.worker, args=(db_settings, to_evaluate[index])) procs[j].start() else: pcm_log.debug('Not evaluable: %s' % to_evaluate[index]) index += 1 time.sleep(self.sleeping_time)
def __init__(self, dbname, source_dir=None, source_file=None, output_names=None, output_file=None, db_settings=None, nconcurrent=1): """ FireballCollector is a class collect Fireball executions into one PyChemia Database. The name of the database will be 'dbname' and executions will be processed from a give 'source_dir', a path to search for Fireball calculations or 'source_file', one file with one directory per line where Fireball executions are stored. The variable 'output_names' sets possible output files for Fireball. Typically Fireball output is the standard output and the user has to redirect the output to a file with an arbitrary name. The processing of directories happens in parallel using 'nconcurrent' python processes. By default the database is created on the local machine assuming no authorization or authentication, otherwise use the dictionary 'db_settings' to get more control on the database location and access. :param dbname: (str) Name of the database that will contain the Fireball executions :param source_dir: (str) Path for a directory that will be explored recursevely for suitable Fireball executions :param source_file: (str) Path to a file describing directories with Fireball executions, one per line :param output_names: (list) List of possible output filenames :param output_file: (str) Path to a file with possible output names, one per line :param db_settings: (dict) Extra parameters for locating and accessing the mongo server. :param nconcurrent: Number of concurrent python executions to fill the database (default: 1) """ self.db_settings = db_settings self.dbname = dbname self.source_dir = source_dir self.source_file = source_file self.output_names = output_names self.output_file = output_file self.nconcurrent = nconcurrent self.sleeping_time = 2 self.database = None if self.output_file is not None and os.path.exists(self.output_file): rf = open(self.output_file) self.output_names = [x.strip() for x in rf.readlines()] if self.source_dir is None and self.source_file is None: raise ValueError("One of the variables source_dir or source_file need to be defined") if self.db_settings is None: self.db_settings = {'name': self.dbname} else: self.db_settings['name'] = self.dbname self.db_settings = dict(self.db_settings) # The first component of each pair in to_evaluate is the name of the database self.db_settings['name'] = self.dbname self.database = get_database(self.db_settings)
def add_from_db(self, db_settings, sizemax=1): if self.composition is None: raise ValueError('No composition associated to this population') comp = Composition(self.composition) readdb = get_database(db_settings) index = 0 for entry in readdb.entries.find({'structure.formula': comp.formula, 'structure.natom': {'$lte': self.min_comp_mult * comp.natom, '$gte': self.max_comp_mult * comp.natom}}): if index < sizemax: print('Adding entry ' + str(entry['_id']) + ' from ' + readdb.name) self.new_entry(readdb.get_structure(entry['_id'])) index += 1
def add_from_db(self, db_settings, sizemax=1): if self.composition is None: raise ValueError('No composition associated to this population') comp = Composition(self.composition) readdb = get_database(db_settings) index = 0 for entry in readdb.entries.find({'structure.formula': comp.formula, 'structure.natom': {'$lte': self.min_comp_mult * comp.natom, '$gte': self.max_comp_mult * comp.natom}}): if index < sizemax: print('Adding entry ' + str(entry['_id']) + ' from ' + readdb.name) self.new_entry(readdb.get_structure(entry['_id'])) index += 1
def unlock_all(self): """ Checking all databases and unlocking all entries. :return: None """ for idb in self.dbnames: db_settings = dict(self.db_settings) db_settings['name'] = idb pcdb = get_database(db_settings) print('Database contains: %d entries' % pcdb.entries.count()) for entry in pcdb.entries.find({}): pcdb.unlock(entry['_id'], name=socket.gethostname()) print('Number of entries: ', pcdb.entries.count())
def unlock_all(self): """ Checking all databases and unlocking all entries. :return: None """ for idb in self.dbnames: db_settings = dict(self.db_settings) db_settings['name'] = idb pcdb = get_database(db_settings) print('Database contains: %d entries' % pcdb.entries.count()) for entry in pcdb.entries.find({}): pcdb.unlock(entry['_id'], name=socket.gethostname()) print('Number of entries: ', pcdb.entries.count())
def searcher(dbsettings, popu_settings, searcher_settings, composition, max_formulas): pcdb = get_database(dbsettings) if 'target_forces' in popu_settings: target_forces = popu_settings['target_forces'] else: target_forces = 1E-3 if 'value_tol' in popu_settings: value_tol = popu_settings['value_tol'] else: value_tol = 1E-2 if 'distance_tolerance' in popu_settings: distance_tolerance = popu_settings['distance_tolerance'] else: distance_tolerance = 0.3 # Population popu = pychemia.population.RelaxStructures( pcdb, composition=composition, tag=composition, target_forces=target_forces, value_tol=value_tol, distance_tolerance=distance_tolerance, min_comp_mult=1, max_comp_mult=max_formulas) # Global Searcher if 'target_value' in searcher_settings: target_value = searcher_settings['target_value'] else: target_value = None method = searcher_settings['method'] if method == 'FireFly': globsearcher = pychemia.searcher.FireFly( popu, params=searcher_settings['params'], generation_size=searcher_settings['generation_size'], stabilization_limit=searcher_settings['stabilization_limit'], target_value=target_value) globsearcher.run()
def search(db_settings, search_settings, abipath): pcdb = get_database(db_settings) print("[%s] Path for abinit.in: %s" % (pcdb.name, abipath)) popu = OrbitalDFTU(pcdb, abipath + os.sep + 'abinit.in') if 'generation_size' in search_settings: generation_size = search_settings.pop('generation_size') else: generation_size = 32 if 'stabilization_limit' in search_settings: stabilization_limit = search_settings.pop('stabilization_limit') else: stabilization_limit = 10 fire = FireFly(popu, params=search_settings, generation_size=generation_size, stabilization_limit=stabilization_limit) fire.run()
def search(db_settings, search_settings, abipath): pcdb = get_database(db_settings) print("[%s] Path for abinit.in: %s" % (pcdb.name, abipath)) popu = OrbitalDFTU(pcdb, abipath + os.sep + 'abinit.in') if 'generation_size' in search_settings: generation_size = search_settings.pop('generation_size') else: generation_size = 32 if 'stabilization_limit' in search_settings: stabilization_limit = search_settings.pop('stabilization_limit') else: stabilization_limit = 10 fire = FireFly(popu, params=search_settings, generation_size=generation_size, stabilization_limit=stabilization_limit) fire.run()
def get_list_candidates(self): """ Scan all databases looking for candidates for evaluation :return: A list of pairs, each pair contains the name of the database and candidate identifier. """ ret = [] for idb in self.dbnames: print(idb) db_settings = dict(self.db_settings) db_settings['name'] = idb pcdb = get_database(db_settings) for entry in pcdb.entries.find({}, {'_id': 1}): entry_id = entry['_id'] if not self.is_evaluated(pcdb, entry_id, self.worker_args) or self.evaluate_all: pcm_log.debug('Adding entry %s from db %s' % (str(entry_id), pcdb.name)) ret.append([idb, entry_id]) print('Found %d entries to evaluate' % len(ret)) return ret
def get_list_candidates(self): """ Scan all databases looking for candidates for evaluation :return: A list of pairs, each pair contains the name of the database and candidate identifier. """ ret = [] for idb in self.dbnames: print(idb) db_settings = dict(self.db_settings) db_settings['name'] = idb pcdb = get_database(db_settings) for entry in pcdb.entries.find({}, {'_id': 1}): entry_id = entry['_id'] if not self.is_evaluated( pcdb, entry_id, self.worker_args) or self.evaluate_all: pcm_log.debug('Adding entry %s from db %s' % (str(entry_id), pcdb.name)) ret.append([idb, entry_id]) print('Found %d entries to evaluate' % len(ret)) return ret
def evaluate(db_settings, queue_settings, abipath): pcdb = get_database(db_settings) print("[%s] Path for abinit.in: %s" % (pcdb.name, abipath)) popu = OrbitalDFTU(pcdb, abipath + os.sep + 'abinit.in') popu.evaluator(queue_settings, abipath)
def run(self): """ Continuously search for suitable candidates to evaluation among a list of databases. :return: """ procs = [] ids_running = [] # Creating a list to store the 'nconcurrent' running jobs for i in range(self.nconcurrent): procs.append(None) ids_running.append(None) self.unlock_all() # Main loop looking permanently for candidates for evaluation while True: to_evaluate = self.get_list_candidates() index = 0 currently_evaluating = 0 for j in range(self.nconcurrent): if procs[j] is not None and procs[j].is_alive(): currently_evaluating += 1 print('Candidates to evaluate: %d Candidates in evaluation: %d' % (len(to_evaluate), currently_evaluating)) while index < len(to_evaluate): db_settings = dict(self.db_settings) # The first component of each pair in to_evaluate is the name of the database dbname = to_evaluate[index][0] db_settings['name'] = dbname pcdb = get_database(db_settings) # The second component of each pair in to_evaluate is the entry_id entry_id = to_evaluate[index][1] for j in range(self.nconcurrent): if procs[j] is None or not procs[j].is_alive(): ids_running[j] = None if entry_id in ids_running: print('Already executing: %s' % entry_id) index += 1 continue else: print('DB: %10s Entry: %s' % (dbname, entry_id)) if not os.path.exists(self.source_dir + os.sep + dbname): os.mkdir(self.source_dir + os.sep + dbname) slot = None while True: for j in range(self.nconcurrent): if procs[j] is None or not procs[j].is_alive(): slot = j break if slot is None: time.sleep(self.sleeping_time) else: break # The function is_evaluated needs two arguments, the database object and entry identifier and # must return a boolean to decide if the candidate should be evaluated. if not self.is_evaluated(pcdb, entry_id, self.worker_args) or self.evaluate_all: pcm_log.debug('Evaluable: %s:%s. Relaxing entry %d of %d Slot: %d' % (dbname, str(entry_id), index, len(to_evaluate), slot)) ids_running[slot] = entry_id workdir = self.source_dir + os.sep + dbname + os.sep + str(entry_id) if not os.path.exists(workdir): os.mkdir(workdir) pcm_log.debug('Launching for %s id: %s' % (pcdb.name, str(entry_id))) # This is the actual call to the worker, it must be a function with 4 arguments: # The database settings, the entry identifier, the working directory and arguments for the worker procs[slot] = Process(target=self.worker, args=(db_settings, entry_id, workdir, self.worker_args)) procs[slot].start() time.sleep(1) else: pcm_log.debug('Not evaluable: %s' % str(entry_id)) index += 1 time.sleep(self.sleeping_time)
def run(self): procs = [] ids_running = [] for i in range(self.nparal): procs.append(None) ids_running.append(None) pcdb = get_database(self.database_settings) print('Database contains: %d entries' % pcdb.entries.count()) for entry in pcdb.entries.find({}): pcdb.unlock(entry['_id'], name=socket.gethostname()) print('Number of entries: ', pcdb.entries.count()) while True: to_relax = [] for entry in pcdb.entries.find({}): if self.is_evaluable(entry): pcm_log.debug('Adding entry %s from db %s' % (str(entry['_id']), pcdb.name)) to_relax.append(entry['_id']) if len(to_relax) == 0: pcm_log.debug( 'No more entries to evaluate, sleeping %d seconds' % self.sleeping_time) time.sleep(self.sleeping_time) else: sortlist = np.array([ pcdb.get_structure(i).natom for i in to_relax ]).argsort()[::-1] to_relax = list(np.array(to_relax)[sortlist]) pcm_log.debug('Number of entries to evaluate: %d' % len(to_relax)) index = 0 relaxing = np.array([ True for j in range(self.nparal) if procs[j] is not None and procs[j].is_alive() ]) print('[%s]-> To Relax: %d Relaxing: %d' % (pcdb.name, len(to_relax), sum(relaxing))) while index < len(to_relax): entry_id = to_relax[index] entry = pcdb.entries.find_one({'_id': entry_id}) if not os.path.exists(self.basedir + os.sep + pcdb.name): os.mkdir(self.basedir + os.sep + pcdb.name) for j in range(self.nparal): if procs[j] is None or not procs[j].is_alive(): if ids_running[j] is not None: pcm_log.debug( '%s is not alive. Exit code: %2d. Locked: %s' % (str(ids_running[j]), procs[j].exitcode, str(pcdb.is_locked(ids_running[j])))) if self.is_evaluable( pcdb.entries.find_one({'_id': entry_id})): pcm_log.debug( 'Evaluable: %s. Relaxing entry %d of %d' % (str(entry_id), index, len(to_relax))) ids_running[j] = entry_id workdir = self.basedir + os.sep + pcdb.name + os.sep + str( entry_id) if not os.path.exists(workdir): os.mkdir(workdir) db_settings = self.database_settings.copy() pcm_log.debug('Launching for %s id: %s' % (pcdb.name, str(entry_id))) # Relax lowering the target forces by one order of magnitude each time current_status = self.get_current_status(entry) pcm_log.debug('Current max forces-stress: %7.3e' % current_status) step_target = max(0.1 * current_status, self.target_forces) pcm_log.debug('New target forces-stress: %7.3e' % step_target) procs[j] = Process(target=self.worker, args=(db_settings, entry_id, workdir, step_target, self.relaxator_params)) procs[j].start() else: pcm_log.debug('Not evaluable: %s' % str(entry_id)) index += 1 break time.sleep(3) time.sleep(self.sleeping_time)
# check all settings in args.p dbs = [] for dbi_file in args.p: dbi = safe_read_json(dbi_file) print("DB settings: %s" % dbi) assert ('name' in dbi) assert ('u' in dbi) assert ('j' in dbi) dbs.append(dbi) if args.subparser_name == 'create': pcdbs = [] for dbi in dbs: pcdb = get_database(dbi) pcdbs.append(pcdb) print(pcdb) print(pcdb.entries.count()) if args.subparser_name == 'run': queue_settings = safe_read_json(args.queue_settings) print("PBS settings: %s" % queue_settings) # General actions for 'populate', 'run', 'search' and 'plot' if args.subparser_name in ['run', 'plot', 'populate', 'search']: if not os.path.isdir( args.basepath) or not os.path.isfile(args.basepath + os.sep + 'abinit.in'): print(
def worker(self, path): """ From a given 'path', the worker will collect information from several files such as: fireball.in eigen.dat param.dat answer.bas output collected from the standard output of fireball.x After collecting data into dictionaries, the data is stored in a PyChemia database """ pcdb = get_database(self.db_settings) files = os.listdir(path) properties = {} geo = None if os.path.lexists(path+os.sep+'fireball.in'): properties['path'] = path score = 'input' try: invars = pychemia.code.fireball.read_fireball_in(path + os.sep + 'fireball.in') properties['input'] = invars except: print('Bad fireball.in on %s' % path) if os.path.exists(path+os.sep+'param.dat'): try: score += '+param' param = pychemia.code.fireball.read_param(path + os.sep + 'param.dat') properties['param_dat'] = param except: print('Bad param.dat') if os.path.lexists(path+os.sep+'eigen.dat'): try: score += '+eigen' eigen = pychemia.code.fireball.read_eigen(path + os.sep + 'eigen.dat') properties['eigen_dat'] = eigen except: print('bad eigen.dat') if os.path.lexists(path+os.sep+'answer.bas'): try: score += '+answer' geo = pychemia.code.fireball.read_geometry_bas(path + os.sep + 'answer.bas') periodic = False except: print('Bad answer.bas') for ioutput in self.output_names: if os.path.isfile(path + os.sep + ioutput): rf = open(path + os.sep + ioutput) line = rf.readline() rf.close() if "Welcome to FIREBALL" in line: try: output = pychemia.code.fireball.read_final_fireball_relax(path + os.sep + ioutput) properties['output'] = output break except: print('Bad output %s on %s' % (ioutput, path)) for ifile in files: if ifile[-3:] == 'lvs': try: cell = pychemia.code.fireball.read_lvs(path + os.sep + ifile) periodic = True lat = pychemia.crystral.Lattice(cell) if lat.volume > 1E7: print('Lattice too big, assuming non-periodic structure') periodic = False except: print('Bad %s' % ifile) if geo is not None: print('DB: %d entries, Path: %s' % (pcdb.entries.count(), path)) if periodic: st = pychemia.Structure(symbols=geo.symbols, positions=geo.positions, cell=cell) else: st = pychemia.Structure(symbols=geo.symbols, positions=geo.positions, periodicity=False) pcdb.insert(st, properties) return properties
def worker(db_settings, entry_id, workdir, target_forces, relaxator_params): pcdb = get_database(db_settings) pcm_log.info('[%s]: Starting relaxation. Target forces: %7.3e' % (str(entry_id), target_forces)) if pcdb.is_locked(entry_id): return else: pcdb.lock(entry_id) structure = pcdb.get_structure(entry_id) structure = structure.scale() print('relaxator_params', relaxator_params) relaxer = IonRelaxation(structure, workdir=workdir, target_forces=target_forces, waiting=False, binary=relaxator_params['binary'], encut=1.3, kp_grid=None, kp_density=1E4, relax_cell=True, max_calls=10) print('relaxing on:', relaxer.workdir) relaxer.run(relaxator_params['nmpiparal']) pcm_log.info('[%s]: Finished relaxation. Target forces: %7.3e' % (str(entry_id), target_forces)) filename = workdir + os.sep + 'OUTCAR' if os.path.isfile(filename): forces, stress, total_energy = relaxer.get_forces_stress_energy() if forces is not None: magnitude_forces = np.apply_along_axis(np.linalg.norm, 1, forces) print('Forces: Max: %9.3e Avg: %9.3e' % (np.max(magnitude_forces), np.average(magnitude_forces))) print('Stress: ', np.max(np.abs(stress.flatten()))) if forces is None: pcm_log.error('No forces found on %s' % filename) if stress is None: pcm_log.error('No stress found on %s' % filename) if total_energy is None: pcm_log.error('No total_energy found on %s' % filename) new_structure = relaxer.get_final_geometry() if forces is not None and stress is not None and total_energy is not None and new_structure is not None: pcm_log.info('[%s]: Updating properties' % str(entry_id)) pcdb.update(entry_id, structure=new_structure) te = total_energy pcdb.entries.update({'_id': entry_id}, { '$set': { 'status.relaxation': 'succeed', 'status.target_forces': target_forces, 'properties.forces': generic_serializer(forces), 'properties.stress': generic_serializer(stress), 'properties.energy': te, 'properties.energy_pa': te / new_structure.natom, 'properties.energy_pf': te / new_structure.get_composition().gcd } }) # Fingerprint # Update the fingerprints only if the two structures are really different diffnatom = structure.natom != new_structure.natom diffcell = np.max( np.abs((structure.cell - new_structure.cell).flatten())) diffreduced = np.max( np.abs((structure.reduced - new_structure.reduced).flatten())) if diffnatom != 0 or diffcell > 1E-7 or diffreduced > 1E-7: analysis = StructureAnalysis(new_structure, radius=50) x, ys = analysis.fp_oganov(delta=0.01, sigma=0.01) fingerprint = {'_id': entry_id} for k in ys: atomic_number1 = atomic_number(new_structure.species[k[0]]) atomic_number2 = atomic_number(new_structure.species[k[1]]) pair = '%06d' % min(atomic_number1 * 1000 + atomic_number2, atomic_number2 * 1000 + atomic_number1) fingerprint[pair] = list(ys[k]) if pcdb.db.fingerprints.find_one({'_id': entry_id}) is None: pcdb.db.fingerprints.insert(fingerprint) else: pcdb.db.fingerprints.update({'_id': entry_id}, fingerprint) else: pcm_log.debug('Original and new structures are very similar.') pcm_log.debug('Max diff cell: %10.3e' % np.max( np.absolute( (structure.cell - new_structure.cell).flatten()))) if structure.natom == new_structure.natom: pcm_log.debug( 'Max diff reduced coordinates: %10.3e' % np.max( np.absolute((structure.reduced - new_structure.reduced).flatten()))) else: pcdb.entries.update({'_id': entry_id}, {'$set': { 'status.relaxation': 'failed' }}) pcm_log.error( 'Bad data after relaxation. Tagging relaxation as failed') else: pcm_log.error('ERROR: File not found %s' % filename) pcm_log.info('[%s]: Unlocking the entry' % str(entry_id)) pcdb.unlock(entry_id)
def worker_maise(db_settings, entry_id, workdir, relaxator_params): """ Relax and return evaluate the energy of the structure stored with identifier 'entry_id' using the MAISE code :param db_settings: (dict) Dictionary of DB parameters needed to create a PyChemiaDB object :param entry_id: MongoDB identifier of one entry of the database created from db_settings :param workdir: (str) Working directory where input and output from MAISE is written :param relaxator_params: (dict) Arguments needed to control the relaxation using MAISE Arguments are store as keys and they include: 'target_forces' : Used to defined the tolerance to consider one candidate as relaxed. 'source_dir': Directory with executable maise and directory INI :return: """ max_ncalls = 6 pcdb = get_database(db_settings) target_forces = relaxator_params['target_forces'] source_dir = relaxator_params['source_dir'] pcm_log.info('[%s]: Starting relaxation. Target forces: %7.3e' % (str(entry_id), target_forces)) if pcdb.is_locked(entry_id): return else: pcdb.lock(entry_id) structure = pcdb.get_structure(entry_id) status = pcdb.get_dicts(entry_id)[2] if 'ncalls' in status and status['ncalls'] > 0: ncalls = status['ncalls'] + 1 print('ncalls = ', status['ncalls']) else: ncalls = 1 print('Verifing initial structure...') while np.min(structure.distance_matrix() + (np.eye(structure.natom) * 5)) < 1.9: print( 'ERROR: Bad initial guess, two atoms are to close. Creating new random structure for id: %s' % str(entry_id)) write_poscar(structure, workdir + os.sep + 'Fail_initial_POSCAR') # WIH structure = Structure.random_cell(structure.composition) write_poscar(structure, workdir + os.sep + 'POSCAR') if not os.path.exists(workdir + os.sep + 'setup') and ncalls == 1: # WIH print('First run.') # WIH # print('Verifying that everything runs smoothly') # WIH print(workdir + os.sep + 'setup') shutil.copy2(source_dir + os.sep + 'setup_1', workdir + os.sep + 'setup') # WIH elif ncalls > 1: # WIH shutil.copy2(source_dir + os.sep + 'setup_2', workdir + os.sep + 'setup') # WIH if not os.path.exists(workdir + os.sep + 'INI'): os.symlink(source_dir + os.sep + 'INI', workdir + os.sep + 'INI') if not os.path.exists(workdir + os.sep + 'maise'): os.symlink(source_dir + os.sep + 'maise', workdir + os.sep + 'maise') # Get the Current Working Directory # cwd = os.getcwd() # Move to the actual directory where maise will run os.chdir(workdir) wf = open('maise.stdout', 'w') subprocess.call(['./maise'], stdout=wf) wf.close() if os.path.isfile('OSZICAR'): energies = np.loadtxt('OSZICAR') else: energies = None forces = None stress = None stress_kb = None if os.path.isfile('OUTCAR'): rf = open('OUTCAR', 'r') data = rf.read() pos_forces = re.findall( r'TOTAL-FORCE \(eV/Angst\)\s*-*\s*([-.\d\s]+)\s+-{2}', data) pos_forces = np.array([x.split() for x in pos_forces], dtype=float) if len(pos_forces) > 0 and len(pos_forces[-1]) % 7 == 0: pos_forces.shape = (len(pos_forces), -1, 7) forces = pos_forces[:, :, 3:6] # positions = pos_forces[:, :, :3] else: print('Forces and Positions could not be parsed : ', pos_forces.shape) print('pos_forces =\n%s ' % pos_forces) str_stress = re.findall('Total([.\d\s-]*)in', data) if len(str_stress) == 2: stress = np.array([[float(y) for y in x.split()] for x in str_stress]) str_stress = re.findall('in kB([.\d\s-]*)energy', data) if len(str_stress) == 2: stress_kb = np.array([[float(y) for y in x.split()] for x in str_stress]) create_new = False if not os.path.isfile('CONTCAR') or os.path.getsize("CONTCAR") == 0: create_new = True print('CONTCAR not found in entry: %s' % str(entry_id)) i = 1 while True: if not os.path.isfile('POSCAR-failed-%03s' % str(i)): os.rename('POSCAR', 'POSCAR-failed-%03s' % str(i)) break else: i += 1 else: new_structure = read_poscar('CONTCAR') # min_dist = np.min(new_structure.distance_matrix+np.ones((new_structure.natom,new_structure.natom))) min_dist = np.min(new_structure.distance_matrix() + (np.eye(new_structure.natom) * 5)) # WIH print('Minimal distance= %8.7f' % min_dist) # WIH if min_dist < 2.0: print( 'ERROR: MAISE finished with and structure with distances too close:', entry_id) # WIH write_poscar(new_structure, workdir + os.sep + 'Collapsed_CONTCAR') # WIH create_new = True # WIH if create_new: new_structure = Structure.random_cell(structure.composition) ncalls = 0 # WIH if ncalls > max_ncalls: print( 'WARNING: Too many calls to MAISE and no relaxation succeeded, replacing structure: ', entry_id) # WIH new_structure = Structure.random_cell(structure.composition) pcdb.entries.update({'_id': entry_id}, {'$set': {'status.ncalls': 0}}) create_new = True else: pcdb.entries.update({'_id': entry_id}, {'$set': { 'status.ncalls': ncalls }}) pcdb.update(entry_id, structure=new_structure, properties={}) # if not create_new and energies is not None and forces is not None and stress is not None: if energies is not None and forces is not None and stress is not None: te = energies[1] pcdb.entries.update({'_id': entry_id}, { '$set': { 'status.relaxation': 'succeed', 'status.target_forces': target_forces, 'properties.initial_forces': generic_serializer(forces[0]), 'properties.initial_stress': generic_serializer(stress[0]), 'properties.initial_stress_kB': generic_serializer( stress_kb[0]), 'properties.forces': generic_serializer(forces[1]), 'properties.stress': generic_serializer(stress[1]), 'properties.stress_kB': generic_serializer(stress_kb[1]), 'properties.energy': te, 'properties.energy_pa': te / new_structure.natom, 'properties.energy_pf': te / new_structure.get_composition().gcd } }) for ifile in [ 'POSCAR', 'CONTCAR', 'setup', 'OUTCAR', 'maise.stdout', 'list.dat' ]: if not os.path.exists(ifile): wf = open(ifile, 'w') wf.write('') wf.close() n = 1 while True: if os.path.exists(ifile + ('_%03d' % n)): n += 1 else: break os.rename(ifile, ifile + ('_%03d' % n)) pcm_log.info('[%s]: Unlocking the entry' % str(entry_id)) pcdb.unlock(entry_id)
def evaluate(db_settings, queue_settings, abipath): pcdb = get_database(db_settings) print("[%s] Path for abinit.in: %s" % (pcdb.name, abipath)) popu = OrbitalDFTU(pcdb, abipath + os.sep + 'abinit.in') popu.evaluator(queue_settings, abipath)
def worker(db_settings, entry_id, workdir, target_forces, relaxator_params): pcdb = get_database(db_settings) pcm_log.info('[%s]: Starting relaxation. Target forces: %7.3e' % (str(entry_id), target_forces)) if pcdb.is_locked(entry_id): return else: pcdb.lock(entry_id) structure = pcdb.get_structure(entry_id) structure = structure.scale() print('relaxator_params', relaxator_params) relaxer = IonRelaxation2(structure, workdir=workdir, target_forces=target_forces, waiting=False, binary=relaxator_params['binary'], encut=1.3, kp_grid=None, kp_density=1E4, relax_cell=True) print('relaxing on:', relaxer.workdir) relaxer.run(relaxator_params['nmpiparal']) pcm_log.info('[%s]: Finished relaxation. Target forces: %7.3e' % (str(entry_id), target_forces)) filename = workdir + os.sep + 'OUTCAR' if os.path.isfile(filename): forces, stress, total_energy = relaxer.get_forces_stress_energy() if forces is not None: magnitude_forces = np.apply_along_axis(np.linalg.norm, 1, forces) print('Forces: Max: %9.3e Avg: %9.3e' % (np.max(magnitude_forces), np.average(magnitude_forces))) print('Stress: ', np.max(np.abs(stress.flatten()))) if forces is None: pcm_log.error('No forces found on %s' % filename) if stress is None: pcm_log.error('No stress found on %s' % filename) if total_energy is None: pcm_log.error('No total_energy found on %s' % filename) new_structure = relaxer.get_final_geometry() if forces is not None and stress is not None and total_energy is not None and new_structure is not None: pcm_log.info('[%s]: Updating properties' % str(entry_id)) pcdb.update(entry_id, structure=new_structure) te = total_energy pcdb.entries.update({'_id': entry_id}, {'$set': {'status.relaxation': 'succeed', 'status.target_forces': target_forces, 'properties.forces': generic_serializer(forces), 'properties.stress': generic_serializer(stress), 'properties.energy': te, 'properties.energy_pa': te / new_structure.natom, 'properties.energy_pf': te / new_structure.get_composition().gcd}}) # Fingerprint # Update the fingerprints only if the two structures are really different diffnatom = structure.natom != new_structure.natom diffcell = np.max(np.abs((structure.cell - new_structure.cell).flatten())) diffreduced = np.max(np.abs((structure.reduced - new_structure.reduced).flatten())) if diffnatom != 0 or diffcell > 1E-7 or diffreduced > 1E-7: analysis = StructureAnalysis(new_structure, radius=50) x, ys = analysis.fp_oganov(delta=0.01, sigma=0.01) fingerprint = {'_id': entry_id} for k in ys: atomic_number1 = atomic_number(new_structure.species[k[0]]) atomic_number2 = atomic_number(new_structure.species[k[1]]) pair = '%06d' % min(atomic_number1 * 1000 + atomic_number2, atomic_number2 * 1000 + atomic_number1) fingerprint[pair] = list(ys[k]) if pcdb.db.fingerprints.find_one({'_id': entry_id}) is None: pcdb.db.fingerprints.insert(fingerprint) else: pcdb.db.fingerprints.update({'_id': entry_id}, fingerprint) else: pcm_log.debug('Original and new structures are very similar.') pcm_log.debug('Max diff cell: %10.3e' % np.max(np.absolute((structure.cell - new_structure.cell).flatten()))) if structure.natom == new_structure.natom: pcm_log.debug('Max diff reduced coordinates: %10.3e' % np.max(np.absolute((structure.reduced - new_structure.reduced).flatten()))) else: pcdb.entries.update({'_id': entry_id}, {'$set': {'status.relaxation': 'failed'}}) pcm_log.error('Bad data after relaxation. Tagging relaxation as failed') else: pcm_log.error('ERROR: File not found %s' % filename) pcm_log.info('[%s]: Unlocking the entry' % str(entry_id)) pcdb.unlock(entry_id)
def run(self): procs = [] ids_running = [] for i in range(self.nparal): procs.append(None) ids_running.append(None) pcdb = get_database(self.database_settings) print('Database contains: %d entries' % pcdb.entries.count()) for entry in pcdb.entries.find({}): pcdb.unlock(entry['_id'], name=socket.gethostname()) print('Number of entries: ', pcdb.entries.count()) while True: to_relax = [] for entry in pcdb.entries.find({}): if self.is_evaluable(entry): pcm_log.debug('Adding entry %s from db %s' % (str(entry['_id']), pcdb.name)) to_relax.append(entry['_id']) if len(to_relax) == 0: pcm_log.debug('No more entries to evaluate, sleeping %d seconds' % self.sleeping_time) time.sleep(self.sleeping_time) else: sortlist = np.array([pcdb.get_structure(i).natom for i in to_relax]).argsort()[::-1] to_relax = list(np.array(to_relax)[sortlist]) pcm_log.debug('Number of entries to evaluate: %d' % len(to_relax)) index = 0 relaxing = np.array([True for j in range(self.nparal) if procs[j] is not None and procs[j].is_alive()]) print('[%s]-> To Relax: %d Relaxing: %d' % (pcdb.name, len(to_relax), sum(relaxing))) while index < len(to_relax): entry_id = to_relax[index] entry = pcdb.entries.find_one({'_id': entry_id}) if not os.path.exists(self.basedir + os.sep + pcdb.name): os.mkdir(self.basedir + os.sep + pcdb.name) for j in range(self.nparal): if procs[j] is None or not procs[j].is_alive(): if ids_running[j] is not None: pcm_log.debug('%s is not alive. Exit code: %2d. Locked: %s' % (str(ids_running[j]), procs[j].exitcode, str(pcdb.is_locked(ids_running[j])))) if self.is_evaluable(pcdb.entries.find_one({'_id': entry_id})): pcm_log.debug('Evaluable: %s. Relaxing entry %d of %d' % (str(entry_id), index, len(to_relax))) ids_running[j] = entry_id workdir = self.basedir + os.sep + pcdb.name + os.sep + str(entry_id) if not os.path.exists(workdir): os.mkdir(workdir) db_settings = self.database_settings.copy() pcm_log.debug('Launching for %s id: %s' % (pcdb.name, str(entry_id))) # Relax lowering the target forces by one order of magnitude each time current_status = self.get_current_status(entry) pcm_log.debug('Current max forces-stress: %7.3e' % current_status) step_target = max(0.1 * current_status, self.target_forces) pcm_log.debug('New target forces-stress: %7.3e' % step_target) procs[j] = Process(target=self.worker, args=(db_settings, entry_id, workdir, step_target, self.relaxator_params)) procs[j].start() else: pcm_log.debug('Not evaluable: %s' % str(entry_id)) index += 1 break time.sleep(3) time.sleep(self.sleeping_time)
def worker_maise(db_settings, entry_id, workdir, relaxator_params): """ Relax and return evaluate the energy of the structure stored with identifier 'entry_id' using the MAISE code :param db_settings: (dict) Dictionary of DB parameters needed to create a PyChemiaDB object :param entry_id: MongoDB identifier of one entry of the database created from db_settings :param workdir: (str) Working directory where input and output from MAISE is written :param relaxator_params: (dict) Arguments needed to control the relaxation using MAISE Arguments are store as keys and they include: 'target_forces' : Used to defined the tolerance to consider one candidate as relaxed. 'source_dir': Directory with executable maise and directory INI :return: """ max_ncalls = 6 pcdb = get_database(db_settings) target_forces = relaxator_params['target_forces'] source_dir = relaxator_params['source_dir'] pcm_log.info('[%s]: Starting relaxation. Target forces: %7.3e' % (str(entry_id), target_forces)) if pcdb.is_locked(entry_id): return else: pcdb.lock(entry_id) structure = pcdb.get_structure(entry_id) status = pcdb.get_dicts(entry_id)[2] if 'ncalls' in status and status['ncalls'] > 0: ncalls = status['ncalls'] + 1 print('ncalls = ', status['ncalls']) else: ncalls = 1 print('Verifing initial structure...') while np.min(structure.distance_matrix()+(np.eye(structure.natom)*5)) < 1.9: print('ERROR: Bad initial guess, two atoms are to close. Creating new random structure for id: %s' % str(entry_id)) write_poscar(structure, workdir + os.sep + 'Fail_initial_POSCAR') # WIH structure = Structure.random_cell(structure.composition) write_poscar(structure, workdir + os.sep + 'POSCAR') if not os.path.exists(workdir + os.sep + 'setup') and ncalls == 1: # WIH print('First run.') # WIH # print('Verifying that everything runs smoothly') # WIH print(workdir + os.sep + 'setup') shutil.copy2(source_dir + os.sep + 'setup_1', workdir + os.sep + 'setup') # WIH elif ncalls > 1: # WIH shutil.copy2(source_dir + os.sep + 'setup_2', workdir + os.sep + 'setup') # WIH if not os.path.exists(workdir + os.sep + 'INI'): os.symlink(source_dir + os.sep + 'INI', workdir + os.sep + 'INI') if not os.path.exists(workdir + os.sep + 'maise'): os.symlink(source_dir + os.sep + 'maise', workdir + os.sep + 'maise') # Get the Current Working Directory # cwd = os.getcwd() # Move to the actual directory where maise will run os.chdir(workdir) wf = open('maise.stdout', 'w') subprocess.call(['./maise'], stdout=wf) wf.close() if os.path.isfile('OSZICAR'): energies = np.loadtxt('OSZICAR') else: energies = None forces = None stress = None stress_kb = None if os.path.isfile('OUTCAR'): rf = open('OUTCAR', 'r') data = rf.read() pos_forces = re.findall(r'TOTAL-FORCE \(eV/Angst\)\s*-*\s*([-.\d\s]+)\s+-{2}', data) pos_forces = np.array([x.split() for x in pos_forces], dtype=float) if len(pos_forces) > 0 and len(pos_forces[-1]) % 7 == 0: pos_forces.shape = (len(pos_forces), -1, 7) forces = pos_forces[:, :, 3:6] # positions = pos_forces[:, :, :3] else: print('Forces and Positions could not be parsed : ', pos_forces.shape) print('pos_forces =\n%s ' % pos_forces) str_stress = re.findall('Total([.\d\s-]*)in', data) if len(str_stress) == 2: stress = np.array([[float(y) for y in x.split()] for x in str_stress]) str_stress = re.findall('in kB([.\d\s-]*)energy', data) if len(str_stress) == 2: stress_kb = np.array([[float(y) for y in x.split()] for x in str_stress]) create_new = False if not os.path.isfile('CONTCAR') or os.path.getsize("CONTCAR") == 0: create_new = True print('CONTCAR not found in entry: %s' % str(entry_id)) i = 1 while True: if not os.path.isfile('POSCAR-failed-%03s' % str(i)): os.rename('POSCAR', 'POSCAR-failed-%03s' % str(i)) break else: i += 1 else: new_structure = read_poscar('CONTCAR') # min_dist = np.min(new_structure.distance_matrix+np.ones((new_structure.natom,new_structure.natom))) min_dist = np.min(new_structure.distance_matrix()+(np.eye(new_structure.natom)*5)) # WIH print('Minimal distance= %8.7f' % min_dist) # WIH if min_dist < 2.0: print('ERROR: MAISE finished with and structure with distances too close:', entry_id) # WIH write_poscar(new_structure, workdir + os.sep + 'Collapsed_CONTCAR') # WIH create_new = True # WIH if create_new: new_structure = Structure.random_cell(structure.composition) ncalls = 0 # WIH if ncalls > max_ncalls: print('WARNING: Too many calls to MAISE and no relaxation succeeded, replacing structure: ', entry_id) # WIH new_structure = Structure.random_cell(structure.composition) pcdb.entries.update({'_id': entry_id}, {'$set': {'status.ncalls': 0}}) create_new = True else: pcdb.entries.update({'_id': entry_id}, {'$set': {'status.ncalls': ncalls}}) pcdb.update(entry_id, structure=new_structure, properties={}) # if not create_new and energies is not None and forces is not None and stress is not None: if energies is not None and forces is not None and stress is not None: te = energies[1] pcdb.entries.update({'_id': entry_id}, {'$set': {'status.relaxation': 'succeed', 'status.target_forces': target_forces, 'properties.initial_forces': generic_serializer(forces[0]), 'properties.initial_stress': generic_serializer(stress[0]), 'properties.initial_stress_kB': generic_serializer(stress_kb[0]), 'properties.forces': generic_serializer(forces[1]), 'properties.stress': generic_serializer(stress[1]), 'properties.stress_kB': generic_serializer(stress_kb[1]), 'properties.energy': te, 'properties.energy_pa': te / new_structure.natom, 'properties.energy_pf': te / new_structure.get_composition().gcd}}) for ifile in ['POSCAR', 'CONTCAR', 'setup', 'OUTCAR', 'maise.stdout', 'list.dat']: if not os.path.exists(ifile): wf = open(ifile, 'w') wf.write('') wf.close() n = 1 while True: if os.path.exists(ifile + ('_%03d' % n)): n += 1 else: break os.rename(ifile, ifile+('_%03d' % n)) pcm_log.info('[%s]: Unlocking the entry' % str(entry_id)) pcdb.unlock(entry_id)
def worker_maise(db_settings, entry_id, workdir, target_forces, relaxator_params): max_ncalls = 6 pcdb = get_database(db_settings) pcm_log.info('[%s]: Starting relaxation. Target forces: %7.3e' % (str(entry_id), target_forces)) if pcdb.is_locked(entry_id): return else: pcdb.lock(entry_id) structure = pcdb.get_structure(entry_id) status = pcdb.get_dicts(entry_id)[2] if 'ncalls' in status: ncalls = status['ncalls'] + 1 else: ncalls = 1 #print('Current directory: '+os.getcwd() ) #print('Working directory: '+workdir) write_poscar(structure,workdir+os.sep+'POSCAR') if not os.path.exists(workdir+os.sep+'setup'): shutil.copy2('setup', workdir) if not os.path.exists(workdir+os.sep+'INI'): os.symlink(os.getcwd()+os.sep+'INI', workdir+os.sep+'INI') if not os.path.exists(workdir+os.sep+'maise'): os.symlink(os.getcwd()+os.sep+'maise', workdir+os.sep+'maise') cwd=os.getcwd() os.chdir(workdir) wf=open('maise.stdout','w') subprocess.call(['./maise'], stdout=wf) wf.close() if os.path.isfile('OSZICAR'): energies=np.loadtxt('OSZICAR') else: energies=None if os.path.isfile('OUTCAR'): rf = open('OUTCAR', 'r') data = rf.read() pos_forces = re.findall(r'TOTAL-FORCE \(eV/Angst\)\s*-*\s*([-.\d\s]+)\s+-{2}', data) pos_forces = np.array([x.split() for x in pos_forces], dtype=float) if len(pos_forces) > 0 and len(pos_forces[-1]) % 7 == 0: pos_forces.shape = (len(pos_forces), -1, 7) forces = pos_forces[:, :, 3:6] positions = pos_forces[:, :, :3] else: print('Forces and Positions could not be parsed : ', pos_forces.shape) print('pos_forces =\n%s ' % pos_forces) str_stress=re.findall('Total([\.\d\s-]*)in',data) if len(str_stress)==2: stress = np.array([[float(y) for y in x.split()] for x in str_stress]) str_stress=re.findall('in kB([\.\d\s-]*)energy',data) if len(str_stress)==2: stress_kB = np.array([[float(y) for y in x.split()] for x in str_stress]) else: forces=None stress=None stress_kB=None new_structure=read_poscar('CONTCAR') if np.min(new_structure.distance_matrix()+np.eye(new_structure.natom))<0.23: print('WARNING: Structure collapse 2 atoms, creating a new random structure') new_structure=Structure.random_cell(new_structure.composition) if ncalls > max_ncalls: print('WARNING: Too many calls to MAISE and no relaxation succeeded, replacing structure') new_structure=Structure.random_cell(new_structure.composition) pcdb.entries.update({'_id': entry_id}, {'$set': {'status.ncalls': 0}}) else: pcdb.entries.update({'_id': entry_id}, {'$set': {'status.ncalls': ncalls}}) pcdb.update(entry_id, structure=new_structure) if energies is not None and forces is not None and stress is not None: te = energies[1] pcdb.entries.update({'_id': entry_id}, {'$set': {'status.relaxation': 'succeed', 'status.target_forces': target_forces, 'properties.initial_forces': generic_serializer(forces[0]), 'properties.initial_stress': generic_serializer(stress[0]), 'properties.initial_stress_kB': generic_serializer(stress_kB[0]), 'properties.forces': generic_serializer(forces[1]), 'properties.stress': generic_serializer(stress[1]), 'properties.stress_kB': generic_serializer(stress_kB[1]), 'properties.energy': te, 'properties.energy_pa': te / new_structure.natom, 'properties.energy_pf': te / new_structure.get_composition().gcd}}) for ifile in ['POSCAR', 'CONTCAR', 'setup', 'OUTCAR', 'maise.stdout', 'list.dat']: if not os.path.exists(ifile): wf = open(ifile, 'w') wf.write('') wf.close() n=1 while True: if os.path.exists(ifile+ ('_%03d' % n)): n+=1 else: break os.rename(ifile,ifile+('_%03d' % n)) pcm_log.info('[%s]: Unlocking the entry' % str(entry_id)) pcdb.unlock(entry_id)
def run(self): """ Continuously search for suitable candidates to evaluation among a list of databases. :return: """ procs = [] ids_running = [] # Creating a list to store the 'nconcurrent' running jobs for i in range(self.nconcurrent): procs.append(None) ids_running.append(None) self.unlock_all() # Main loop looking permanently for candidates for evaluation while True: to_evaluate = self.get_list_candidates() index = 0 currently_evaluating = 0 for j in range(self.nconcurrent): if procs[j] is not None and procs[j].is_alive(): currently_evaluating += 1 print('Candidates to evaluate: %d Candidates in evaluation: %d' % (len(to_evaluate), currently_evaluating)) while index < len(to_evaluate): db_settings = dict(self.db_settings) # The first component of each pair in to_evaluate is the name of the database dbname = to_evaluate[index][0] db_settings['name'] = dbname pcdb = get_database(db_settings) # The second component of each pair in to_evaluate is the entry_id entry_id = to_evaluate[index][1] for j in range(self.nconcurrent): if procs[j] is None or not procs[j].is_alive(): ids_running[j] = None if entry_id in ids_running: print('Already executing: %s' % entry_id) index += 1 continue else: print('DB: %10s Entry: %s' % (dbname, entry_id)) if not os.path.exists(self.source_dir + os.sep + dbname): os.mkdir(self.source_dir + os.sep + dbname) slot = None while True: for j in range(self.nconcurrent): if procs[j] is None or not procs[j].is_alive(): slot = j break if slot is None: time.sleep(self.sleeping_time) else: break # The function is_evaluated needs two arguments, the database object and entry identifier and # must return a boolean to decide if the candidate should be evaluated. if not self.is_evaluated( pcdb, entry_id, self.worker_args) or self.evaluate_all: pcm_log.debug( 'Evaluable: %s:%s. Relaxing entry %d of %d Slot: %d' % (dbname, str(entry_id), index, len(to_evaluate), slot)) ids_running[slot] = entry_id workdir = self.source_dir + os.sep + dbname + os.sep + str( entry_id) if not os.path.exists(workdir): os.mkdir(workdir) pcm_log.debug('Launching for %s id: %s' % (pcdb.name, str(entry_id))) # This is the actual call to the worker, it must be a function with 4 arguments: # The database settings, the entry identifier, the working directory and arguments for the worker procs[slot] = Process(target=self.worker, args=(db_settings, entry_id, workdir, self.worker_args)) procs[slot].start() time.sleep(1) else: pcm_log.debug('Not evaluable: %s' % str(entry_id)) index += 1 time.sleep(self.sleeping_time)
# check all settings in args.p dbs = [] for dbi_file in args.p: dbi = safe_read_json(dbi_file) print("DB settings: %s" % dbi) assert('name' in dbi) assert('u' in dbi) assert('j' in dbi) dbs.append(dbi) if args.subparser_name == 'create': pcdbs = [] for dbi in dbs: pcdb = get_database(dbi) pcdbs.append(pcdb) print(pcdb) print(pcdb.entries.count()) if args.subparser_name == 'run': queue_settings = safe_read_json(args.queue_settings) print("PBS settings: %s" % queue_settings) # General actions for 'populate', 'run', 'search' and 'plot' if args.subparser_name in ['run', 'plot', 'populate', 'search']: if not os.path.isdir(args.basepath) or not os.path.isfile(args.basepath + os.sep + 'abinit.in'): print('ERROR: Wrong basepath %s, directory must exist and contain a abinit.in file' % args.basepath) parser.print_help() sys.exit(1)
type=str, help='ID of candidate to relax', required=True) worker_parser.add_argument('-basepath', type=str, help='Path where calculations are performed', required=False, default='.') args = parser.parse_args() print(args) if args.subparser_name in ['searcher', 'manager']: db = safe_read_json(args.db_settings) pcdb = get_database(db) print("") print("Database Settings") print("-----------------\n") print(pcdb) print("") popu_settings = safe_read_json(args.population_settings) print("Population Settings") print("-------------------\n") for i in popu_settings: print(" %30s : %s" % (i.ljust(30), popu_settings[i])) print("") if args.subparser_name == 'searcher':