def get_flags_and_db(self, c,init_db = True): flags, args = self.parse_command_line(c) # if len(args)== 1: # utils.newline_msg("ERR", "a single file was expected or could not parse flags") # return flags, args, None db_name = args[-1] args = args[:-1] if db_name.isdigit(): id = int(db_name) rdb = self.master_db.result_dbs filtered = [x for x in rdb if rdb[x]['id'] == id] if filtered: db_name = filtered[0] else: utils.newline_msg("ERR", "database with id '%s' doesn't exist." % db_name) return flags, args.append(db_name), None else: full_name, path, base_name, extension = utils.translate_name(db_name) # print "do_init::: ",self.translate_name(i_arg) db_name = "%s/%s.spgql" % (path, base_name) # sim_name = "%s/%s.spg" % (path, base_name) if not os.path.exists(db_name): utils.newline_msg("ERR", "database with name '%s' doesn't exist." % utils.shorten_name(db_name)) return flags, args.append(db_name), None return flags, args, self.EnsembleConstructor(db_name, init_db)
def __init__(self, db_name, timeout=5): full_name, self.path, self.base_name, extension = utils.translate_name( db_name) self.db_name = "%s/%s.spgql" % (self.path, self.base_name) sim_name = "%s/%s.spg" % (self.path, self.base_name) MultIteratorParser.__init__(self, open(sim_name)) # print self.data # self.path, foo = os.path.split( os.path.abspath(db_name) ) # self.base_name, foo = os.path.splitext( foo ) # print check_params.consistency(self.command, self) if not check_params.consistency(self.command, self): utils.newline_msg("ERR", "simulation configuration is not consistent.") sys.exit(1) all_outputs = check_params.read_output_configuration(self.command) self.stdout_contents = dict() for k in all_outputs: table = k.output_table if table not in self.stdout_contents.leys(): self.stdout_contents[table] = k self.connection = sql.connect(self.db_name, timeout=timeout, check_same_thread=False) self.cursor = self.connection.cursor()
def __init__(self, arg, command=None): full_name, self.path, self.base_name, extension = utils.translate_name( arg) MultIteratorParser.__init__(self, open(f"{self.base_name}.spg")) if command is not None: self.command = command self.command, ext = os.path.splitext(self.command) if not utils.check_params.consistency(self.command, self): utils.newline_msg("ERR", "simulation configuration is not consistent.") sys.exit(1) self.input_configuration = utils.read_input_configuration( f"{self.command}.input") self.default_parameters = utils.SPGSettings() for k, v in self.input_configuration.items(): if v.var_type != "str": self.default_parameters[k] = eval(f"{v.var_type}({v.default})") else: self.default_parameters[k] = v.default self.stdout_configuration = utils.read_output_configuration( self.command)
def get_flags_and_db(self, c, init_db=True): flags, args = self.parse_command_line(c) # if len(args)== 1: # utils.newline_msg("ERR", "a single file was expected or could not parse flags") # return flags, args, None db_name = args[-1] args = args[:-1] if db_name.isdigit(): id = int(db_name) rdb = self.master_db.result_dbs filtered = [x for x in rdb if rdb[x]['id'] == id] if filtered: db_name = filtered[0] else: utils.newline_msg( "ERR", "database with id '%s' doesn't exist." % db_name) return flags, args.append(db_name), None else: full_name, path, base_name, extension = utils.translate_name( db_name) # print "do_init::: ",self.translate_name(i_arg) db_name = "%s/%s.spgql" % (path, base_name) # sim_name = "%s/%s.spg" % (path, base_name) if not os.path.exists(db_name): utils.newline_msg( "ERR", "database with name '%s' doesn't exist." % utils.shorten_name(db_name)) return flags, args.append(db_name), None return flags, args, self.EnsembleConstructor(db_name, init_db)
def __init__(self, simulation_filename): full_name, self.path, self.base_name, extension = spgu.translate_name( simulation_filename) self.simulation = spgb.MultIteratorParser( open(f"{self.base_name}.spg")) self.datafile_name = "%s.csv" % (self.base_name) self.full_dataframe = pd.read_csv(self.datafile_name) settings_output = self.simulation.stdout_configuration self.output_columns = list(settings_output.keys()) self.constants = {} self.variables = [] for vn in list(self.full_dataframe.keys()): if vn in self.output_columns: continue all_values = self.full_dataframe[vn].unique() if len(all_values) > 1: self.variables.append(vn) else: self.constants[vn] = self.full_dataframe[vn].unique()[0] self.settings = self.simulation.input_configuration self.settings.update(settings_output) self.data = self.full_dataframe[self.variables + self.output_columns] print("[__init__] constants: %s" % list(self.constants.keys())) print("[__init__] independent variables:", self.separated_vars, self.coalesced_vars, self.independent_var) print("[__init__] output columns: %s" % self.output_columns)
def do_init(self, c): """init [--flag ...] PARAMETERS_NAME|DB_NAME [VAR1=VALUE1[:VAR2=VALUE2]] Generates a new database out of a simulation.dat FLAGS::: --purge: deletes the spgql database, if it already exists --repeat=REPEAT repeats the parameter generation REPEAT times """ flags, db_arg = self.parse_command_line(c) if len(db_arg) != 1: utils.newline_msg("WRN", "init must be called with a database", 2) return db_arg = db_arg[0] # i_arg = c[0] full_name, path, base_name, extension = utils.translate_name(db_arg) # print "do_init::: ",self.translate_name(i_arg) full_db_name = "%s/%s.spgql" % (path, base_name) sim_name = "%s/%s.spg" % (path, base_name) if os.path.exists(full_db_name) and "purge" not in flags: utils.newline_msg( "ERR", "database '%s' already exists. Cannot init it twice" % utils.shorten_name(full_db_name), 2) return if not os.path.exists(sim_name): utils.newline_msg( "ERR", "configuration '%s' doesn't exist. Cannot init it" % utils.shorten_name(sim_name), 2) return if "purge" in flags: try: self.do_deregister(db_arg) os.remove(full_db_name) except: utils.newline_msg( "WRN", "database '%s' could not be removed... skipping" % full_db_name) if 'repeat' in flags: repeat = int(flags['repeat']) else: repeat = 1 parser = MultIteratorDBBuilder(db_name=full_db_name) parser.init_db() parser.fill_status(repeat=repeat) current_param_db = ParameterEnsemble(full_db_name, init_db=True) current_param_db.repeat = repeat # if len(c) > 1: self.do_set(":".join(c[1:])) self.master_db.write_ensemble_to_master(current_param_db) self.master_db.update_list_ensemble_dbs() print(" **-- init - %d: '%s' " % (current_param_db.id, utils.shorten_name(current_param_db.full_name)))
def do_init(self, c): """init [--flag ...] PARAMETERS_NAME|DB_NAME [VAR1=VALUE1[:VAR2=VALUE2]] Generates a new database out of a simulation.dat FLAGS::: --purge: deletes the spgql database, if it already exists --repeat=REPEAT repeats the parameter generation REPEAT times """ flags, db_arg = self.parse_command_line(c) if len(db_arg) != 1: utils.newline_msg("WRN", "init must be called with a database", 2) return db_arg = db_arg[0] # i_arg = c[0] full_name, path, base_name, extension = utils.translate_name(db_arg) # print "do_init::: ",self.translate_name(i_arg) full_db_name = "%s/%s.spgql" % (path, base_name) sim_name = "%s/%s.spg" % (path, base_name) if os.path.exists(full_db_name) and "purge" not in flags: utils.newline_msg( "ERR", "database '%s' already exists. Cannot init it twice" % utils.shorten_name(full_db_name), 2 ) return if not os.path.exists(sim_name): utils.newline_msg( "ERR", "configuration '%s' doesn't exist. Cannot init it" % utils.shorten_name(sim_name), 2 ) return if "purge" in flags: self.do_deregister(db_arg) try: os.remove(full_db_name) except: utils.newline_msg("WRN", "database '%s' could not be removed... skipping" % full_db_name) if "repeat" in flags: repeat = int(flags["repeat"]) else: repeat = 1 parser = MultIteratorDBBuilder(db_name=full_db_name) parser.init_db() parser.fill_status(repeat=repeat) current_param_db = ParameterEnsemble(full_db_name, init_db=True) current_param_db.repeat = repeat # if len(c) > 1: self.do_set(":".join(c[1:])) self.master_db.write_ensemble_to_master(current_param_db) self.master_db.update_list_ensemble_dbs() print " **-- init - %d: '%s' " % (current_param_db.id, utils.shorten_name(current_param_db.full_name))
def get_db(self, c): """it returns the db name (or None) of a database identified either from its id or """ try: full_name, path, base_name, extension = utils.translate_name(c.strip()) db_name = "%s/%s.spgql" % (path, base_name) sim_name = "%s/%s.spg" % (path, base_name) except: utils.newline_msg("ERR", "results db '%s' doesn't exist." % c) return None return self.EnsembleConstructor(db_name, init_db=True)
def get_db(self, c): """it returns the db name (or None) of a database identified either from its id or """ try: full_name, path, base_name, extension = utils.translate_name( c.strip()) db_name = "%s/%s.spgql" % (path, base_name) sim_name = "%s/%s.spg" % (path, base_name) except: utils.newline_msg("ERR", "results db '%s' doesn't exist." % c) return None return self.EnsembleConstructor(db_name, init_db=True)
def __init__(self, db_name , timeout = 5): full_name, self.path, self.base_name, extension = utils.translate_name( db_name ) self.db_name = "%s/%s.spgql"%(self.path, self.base_name) sim_name = "%s/%s.spg"%(self.path, self.base_name) MultIteratorParser.__init__(self, open(sim_name) ) # self.path, foo = os.path.split( os.path.abspath(db_name) ) # self.base_name, foo = os.path.splitext( foo ) # print check_params.consistency(self.command, self) if not check_params.consistency(self.command, self): utils.newline_msg("ERR","simulation configuration is not consistent.") sys.exit(1) self.stdout_contents = check_params.contents_in_output(self.command) self.connection = sql.connect(self.db_name, timeout = timeout, check_same_thread = False) self.cursor = self.connection.cursor()
def __init__(self, full_name = "", id=-1, weight=1., queue = '*', status = 'R', repeat = 1, init_db = False): self.full_name, self.path, self.base_name, ext = utils.translate_name(full_name) self.db_name = "%s.spgql"%self.base_name self.full_name = "%s/%s.spgql"%(self.path,self.base_name) self.id = id self.values = {} # self.directory_vars = None self.weight = weight self.current_spg_uid = 0 self.queue = queue self.status = status self.repeat = repeat if init_db : self.init_db()
def do_ls(self, c): full_ls = fnmatch.filter(os.listdir("."), "*.spgql") full_ls.extend( fnmatch.filter( os.listdir("."), "*.spg" ) ) ls_res_db = set() ls_keys = set() for el in full_ls: full_name, path, base_name, extension = utils.translate_name(el) full_base = "%s/%s"%(path, base_name) if full_base not in ls_keys: ls_res_db.add(utils.shorten_name(full_name)) ls_keys.add( full_base ) if c: ls_res_db = fnmatch.filter(ls_res_db, c ) if ls_res_db: print " --- cwd matches found" for i in sorted( ls_res_db ): print " : %s "% i
def do_ls(self, c): full_ls = fnmatch.filter(os.listdir("."), "*.spgql") full_ls.extend(fnmatch.filter(os.listdir("."), "*.spg")) ls_res_db = set() ls_keys = set() for el in full_ls: full_name, path, base_name, extension = utils.translate_name(el) full_base = "%s/%s" % (path, base_name) if full_base not in ls_keys: ls_res_db.add(utils.shorten_name(full_name)) ls_keys.add(full_base) if c: ls_res_db = fnmatch.filter(ls_res_db, c) if ls_res_db: print(" --- cwd matches found") for i in sorted(ls_res_db): print(" : %s " % i)
parser.add_option("--verbose", action='store_true', dest="verbose", help="more verbose output") parser.add_option("--test-run", action='store_true', dest="test_run", help="runs once and preserves the temporary files") # parser.add_option("-d","--directory-var", action='store', type = "string", dest="directory_vars", # default = False, help = "which variables to store as directories, only if tree" ) options, args = parser.parse_args() for i_arg in args: full_name, path, base_name, extension = utils.translate_name(i_arg) db_name = "%s/%s.spgql" % (path, base_name) # sim_name = "%s/%s.spg" % (path, base_name) if options.purge and os.path.exists( db_name ): os.remove( db_name ) if options.initialise or not os.path.exists( db_name ): utils.newline_msg("MSG", "initialising database") parser = MultIteratorDBBuilder(db_name=db_name) parser.init_db() parser.fill_status(repeat=options.repeat) del parser utils.newline_msg("MSG", "running simulation") # if options.dummy: # executor = ParameterEnsembleInputFilesGenerator( db_name )
action='store_true', dest="verbose", help="more verbose output") parser.add_option("--test-run", action='store_true', dest="test_run", help="runs once and preserves the temporary files") # parser.add_option("-d","--directory-var", action='store', type = "string", dest="directory_vars", # default = False, help = "which variables to store as directories, only if tree" ) options, args = parser.parse_args() for i_arg in args: full_name, path, base_name, extension = utils.translate_name(i_arg) db_name = "%s/%s.spgql" % (path, base_name) # sim_name = "%s/%s.spg" % (path, base_name) if options.purge and os.path.exists(db_name): os.remove(db_name) if options.initialise or not os.path.exists(db_name): utils.newline_msg("MSG", "initialising database") parser = MultIteratorDBBuilder(db_name=db_name) parser.init_db() parser.fill_status(repeat=options.repeat) del parser utils.newline_msg("MSG", "running simulation") # if options.dummy: # executor = ParameterEnsembleInputFilesGenerator( db_name )