Exemple #1
0
    def get_flags_and_db(self, c,init_db = True):

        flags, args =  self.parse_command_line(c)
        # if len(args)== 1:
        #     utils.newline_msg("ERR", "a single file was expected or could not parse flags")
        #     return flags, args, None

        db_name = args[-1]
        args = args[:-1]


        if db_name.isdigit():
            id = int(db_name)
            rdb = self.master_db.result_dbs
            filtered = [x for x in rdb if rdb[x]['id'] == id]
            if filtered:
                db_name = filtered[0]
            else:
                utils.newline_msg("ERR", "database with id '%s' doesn't exist." % db_name)
                return flags, args.append(db_name), None
        else:
                full_name, path, base_name, extension = utils.translate_name(db_name)
                # print "do_init::: ",self.translate_name(i_arg)
                db_name = "%s/%s.spgql" % (path, base_name)
                # sim_name = "%s/%s.spg" % (path, base_name)
                if not os.path.exists(db_name):
                    utils.newline_msg("ERR", "database with name '%s' doesn't exist." % utils.shorten_name(db_name))
                    return flags, args.append(db_name), None

        return flags, args, self.EnsembleConstructor(db_name, init_db)
Exemple #2
0
    def __init__(self, db_name, timeout=5):
        full_name, self.path, self.base_name, extension = utils.translate_name(
            db_name)

        self.db_name = "%s/%s.spgql" % (self.path, self.base_name)
        sim_name = "%s/%s.spg" % (self.path, self.base_name)

        MultIteratorParser.__init__(self, open(sim_name))
        #   print self.data
        # self.path, foo = os.path.split( os.path.abspath(db_name) )
        # self.base_name, foo = os.path.splitext( foo )
        #    print  check_params.consistency(self.command, self)
        if not check_params.consistency(self.command, self):
            utils.newline_msg("ERR",
                              "simulation configuration is not consistent.")
            sys.exit(1)

        all_outputs = check_params.read_output_configuration(self.command)

        self.stdout_contents = dict()
        for k in all_outputs:
            table = k.output_table
            if table not in self.stdout_contents.leys():
                self.stdout_contents[table] = k

        self.connection = sql.connect(self.db_name,
                                      timeout=timeout,
                                      check_same_thread=False)
        self.cursor = self.connection.cursor()
Exemple #3
0
    def __init__(self, arg, command=None):

        full_name, self.path, self.base_name, extension = utils.translate_name(
            arg)

        MultIteratorParser.__init__(self, open(f"{self.base_name}.spg"))
        if command is not None:
            self.command = command

        self.command, ext = os.path.splitext(self.command)

        if not utils.check_params.consistency(self.command, self):
            utils.newline_msg("ERR",
                              "simulation configuration is not consistent.")
            sys.exit(1)

        self.input_configuration = utils.read_input_configuration(
            f"{self.command}.input")

        self.default_parameters = utils.SPGSettings()

        for k, v in self.input_configuration.items():

            if v.var_type != "str":
                self.default_parameters[k] = eval(f"{v.var_type}({v.default})")
            else:
                self.default_parameters[k] = v.default

        self.stdout_configuration = utils.read_output_configuration(
            self.command)
Exemple #4
0
    def get_flags_and_db(self, c, init_db=True):

        flags, args = self.parse_command_line(c)
        # if len(args)== 1:
        #     utils.newline_msg("ERR", "a single file was expected or could not parse flags")
        #     return flags, args, None

        db_name = args[-1]
        args = args[:-1]

        if db_name.isdigit():
            id = int(db_name)
            rdb = self.master_db.result_dbs
            filtered = [x for x in rdb if rdb[x]['id'] == id]
            if filtered:
                db_name = filtered[0]
            else:
                utils.newline_msg(
                    "ERR", "database with id '%s' doesn't exist." % db_name)
                return flags, args.append(db_name), None
        else:
            full_name, path, base_name, extension = utils.translate_name(
                db_name)
            # print "do_init::: ",self.translate_name(i_arg)
            db_name = "%s/%s.spgql" % (path, base_name)
            # sim_name = "%s/%s.spg" % (path, base_name)
            if not os.path.exists(db_name):
                utils.newline_msg(
                    "ERR", "database with name '%s' doesn't exist." %
                    utils.shorten_name(db_name))
                return flags, args.append(db_name), None

        return flags, args, self.EnsembleConstructor(db_name, init_db)
Exemple #5
0
    def __init__(self, simulation_filename):

        full_name, self.path, self.base_name, extension = spgu.translate_name(
            simulation_filename)
        self.simulation = spgb.MultIteratorParser(
            open(f"{self.base_name}.spg"))
        self.datafile_name = "%s.csv" % (self.base_name)

        self.full_dataframe = pd.read_csv(self.datafile_name)

        settings_output = self.simulation.stdout_configuration
        self.output_columns = list(settings_output.keys())

        self.constants = {}
        self.variables = []
        for vn in list(self.full_dataframe.keys()):
            if vn in self.output_columns:
                continue
            all_values = self.full_dataframe[vn].unique()
            if len(all_values) > 1:
                self.variables.append(vn)
            else:
                self.constants[vn] = self.full_dataframe[vn].unique()[0]

        self.settings = self.simulation.input_configuration

        self.settings.update(settings_output)

        self.data = self.full_dataframe[self.variables + self.output_columns]

        print("[__init__] constants: %s" % list(self.constants.keys()))
        print("[__init__] independent variables:", self.separated_vars,
              self.coalesced_vars, self.independent_var)
        print("[__init__] output columns: %s" % self.output_columns)
Exemple #6
0
    def do_init(self, c):
        """init [--flag ...] PARAMETERS_NAME|DB_NAME [VAR1=VALUE1[:VAR2=VALUE2]]
        Generates a new database out of a simulation.dat
        FLAGS::: --purge:         deletes the spgql database, if it already exists
                 --repeat=REPEAT  repeats the parameter generation REPEAT times
        """

        flags, db_arg = self.parse_command_line(c)
        if len(db_arg) != 1:
            utils.newline_msg("WRN", "init must be called with a database", 2)
            return
        db_arg = db_arg[0]
        # i_arg = c[0]

        full_name, path, base_name, extension = utils.translate_name(db_arg)
        # print "do_init::: ",self.translate_name(i_arg)
        full_db_name = "%s/%s.spgql" % (path, base_name)
        sim_name = "%s/%s.spg" % (path, base_name)
        if os.path.exists(full_db_name) and "purge" not in flags:
            utils.newline_msg(
                "ERR", "database '%s' already exists. Cannot init it twice" %
                utils.shorten_name(full_db_name), 2)
            return
        if not os.path.exists(sim_name):
            utils.newline_msg(
                "ERR", "configuration '%s' doesn't exist. Cannot init it" %
                utils.shorten_name(sim_name), 2)
            return

        if "purge" in flags:
            try:
                self.do_deregister(db_arg)
                os.remove(full_db_name)
            except:
                utils.newline_msg(
                    "WRN", "database '%s' could not be removed... skipping" %
                    full_db_name)

        if 'repeat' in flags:
            repeat = int(flags['repeat'])
        else:
            repeat = 1

        parser = MultIteratorDBBuilder(db_name=full_db_name)
        parser.init_db()
        parser.fill_status(repeat=repeat)

        current_param_db = ParameterEnsemble(full_db_name, init_db=True)
        current_param_db.repeat = repeat

        # if len(c) > 1: self.do_set(":".join(c[1:]))
        self.master_db.write_ensemble_to_master(current_param_db)

        self.master_db.update_list_ensemble_dbs()

        print(" **-- init       - %d: '%s'   " %
              (current_param_db.id,
               utils.shorten_name(current_param_db.full_name)))
Exemple #7
0
    def do_init(self, c):
        """init [--flag ...] PARAMETERS_NAME|DB_NAME [VAR1=VALUE1[:VAR2=VALUE2]]
        Generates a new database out of a simulation.dat
        FLAGS::: --purge:         deletes the spgql database, if it already exists
                 --repeat=REPEAT  repeats the parameter generation REPEAT times
        """

        flags, db_arg = self.parse_command_line(c)
        if len(db_arg) != 1:
            utils.newline_msg("WRN", "init must be called with a database", 2)
            return
        db_arg = db_arg[0]
        # i_arg = c[0]

        full_name, path, base_name, extension = utils.translate_name(db_arg)
        # print "do_init::: ",self.translate_name(i_arg)
        full_db_name = "%s/%s.spgql" % (path, base_name)
        sim_name = "%s/%s.spg" % (path, base_name)
        if os.path.exists(full_db_name) and "purge" not in flags:
            utils.newline_msg(
                "ERR", "database '%s' already exists. Cannot init it twice" % utils.shorten_name(full_db_name), 2
            )
            return
        if not os.path.exists(sim_name):
            utils.newline_msg(
                "ERR", "configuration '%s' doesn't exist. Cannot init it" % utils.shorten_name(sim_name), 2
            )
            return

        if "purge" in flags:
            self.do_deregister(db_arg)
            try:
                os.remove(full_db_name)
            except:
                utils.newline_msg("WRN", "database '%s' could not be removed... skipping" % full_db_name)

        if "repeat" in flags:
            repeat = int(flags["repeat"])
        else:
            repeat = 1

        parser = MultIteratorDBBuilder(db_name=full_db_name)
        parser.init_db()
        parser.fill_status(repeat=repeat)

        current_param_db = ParameterEnsemble(full_db_name, init_db=True)
        current_param_db.repeat = repeat

        # if len(c) > 1: self.do_set(":".join(c[1:]))
        self.master_db.write_ensemble_to_master(current_param_db)

        self.master_db.update_list_ensemble_dbs()

        print " **-- init       - %d: '%s'   " % (current_param_db.id, utils.shorten_name(current_param_db.full_name))
Exemple #8
0
    def get_db(self, c):
        """it returns the db name (or None) of a database identified either from its id or """

        try:
                full_name, path, base_name, extension = utils.translate_name(c.strip())
                db_name = "%s/%s.spgql" % (path, base_name)
                sim_name = "%s/%s.spg" % (path, base_name)
        except:
                utils.newline_msg("ERR", "results db '%s' doesn't exist." % c)
                return None

        return self.EnsembleConstructor(db_name, init_db=True)
Exemple #9
0
    def get_db(self, c):
        """it returns the db name (or None) of a database identified either from its id or """

        try:
            full_name, path, base_name, extension = utils.translate_name(
                c.strip())
            db_name = "%s/%s.spgql" % (path, base_name)
            sim_name = "%s/%s.spg" % (path, base_name)
        except:
            utils.newline_msg("ERR", "results db '%s' doesn't exist." % c)
            return None

        return self.EnsembleConstructor(db_name, init_db=True)
Exemple #10
0
    def __init__(self, db_name , timeout = 5):
        full_name, self.path, self.base_name, extension = utils.translate_name( db_name )

        self.db_name = "%s/%s.spgql"%(self.path, self.base_name)
        sim_name = "%s/%s.spg"%(self.path, self.base_name)

        MultIteratorParser.__init__(self, open(sim_name) )
        # self.path, foo = os.path.split( os.path.abspath(db_name) )
        # self.base_name, foo = os.path.splitext( foo )
    #    print  check_params.consistency(self.command, self)
        if not check_params.consistency(self.command, self):
            utils.newline_msg("ERR","simulation configuration is not consistent.")
            sys.exit(1)

        self.stdout_contents = check_params.contents_in_output(self.command)
                
        self.connection =  sql.connect(self.db_name, timeout = timeout, check_same_thread = False)
        self.cursor = self.connection.cursor()
Exemple #11
0
    def __init__(self, full_name = "", id=-1, weight=1., queue = '*',
                 status = 'R', repeat = 1, init_db = False):

        self.full_name, self.path, self.base_name, ext = utils.translate_name(full_name)

        self.db_name = "%s.spgql"%self.base_name
        self.full_name = "%s/%s.spgql"%(self.path,self.base_name)
        self.id = id
        self.values = {}
        # self.directory_vars = None

        self.weight = weight
        self.current_spg_uid = 0
        self.queue = queue
        self.status = status
        self.repeat = repeat

        if init_db  :
            self.init_db()
Exemple #12
0
    def __init__(self, full_name = "", id=-1, weight=1., queue = '*',
                 status = 'R', repeat = 1, init_db = False):

        self.full_name, self.path, self.base_name, ext = utils.translate_name(full_name)

        self.db_name = "%s.spgql"%self.base_name
        self.full_name = "%s/%s.spgql"%(self.path,self.base_name)
        self.id = id
        self.values = {}
        # self.directory_vars = None

        self.weight = weight
        self.current_spg_uid = 0
        self.queue = queue
        self.status = status
        self.repeat = repeat

        if init_db  :
            self.init_db()
Exemple #13
0
    def do_ls(self, c):

        full_ls = fnmatch.filter(os.listdir("."), "*.spgql")
        full_ls.extend( fnmatch.filter( os.listdir("."), "*.spg" ) )

        ls_res_db = set()
        ls_keys = set()

        for el in full_ls:
            full_name, path, base_name, extension = utils.translate_name(el)
            full_base = "%s/%s"%(path, base_name)
            if full_base not in ls_keys:
                ls_res_db.add(utils.shorten_name(full_name))
                ls_keys.add( full_base )
        if c:
            ls_res_db = fnmatch.filter(ls_res_db, c )
        if ls_res_db:
            print " --- cwd matches found"
            for i in sorted( ls_res_db  ):
                print "     : %s "% i
Exemple #14
0
    def do_ls(self, c):

        full_ls = fnmatch.filter(os.listdir("."), "*.spgql")
        full_ls.extend(fnmatch.filter(os.listdir("."), "*.spg"))

        ls_res_db = set()
        ls_keys = set()

        for el in full_ls:
            full_name, path, base_name, extension = utils.translate_name(el)
            full_base = "%s/%s" % (path, base_name)
            if full_base not in ls_keys:
                ls_res_db.add(utils.shorten_name(full_name))
                ls_keys.add(full_base)
        if c:
            ls_res_db = fnmatch.filter(ls_res_db, c)
        if ls_res_db:
            print(" --- cwd matches found")
            for i in sorted(ls_res_db):
                print("     : %s " % i)
Exemple #15
0
    parser.add_option("--verbose", action='store_true', dest="verbose",
                      help="more verbose output")

    parser.add_option("--test-run", action='store_true', dest="test_run",
                      help="runs once and preserves the temporary files")

#    parser.add_option("-d","--directory-var", action='store', type = "string", dest="directory_vars",
#                       default = False, help = "which variables to store as directories, only if tree" )


    options, args = parser.parse_args()
    

    for i_arg in args:
      full_name, path, base_name, extension = utils.translate_name(i_arg)

      db_name = "%s/%s.spgql" % (path, base_name)
     # sim_name = "%s/%s.spg" % (path, base_name)
      if options.purge and os.path.exists( db_name ):
          os.remove( db_name )
      if options.initialise or not os.path.exists( db_name ):
          utils.newline_msg("MSG", "initialising database")
          parser = MultIteratorDBBuilder(db_name=db_name)
          parser.init_db()
          parser.fill_status(repeat=options.repeat)
          del parser

      utils.newline_msg("MSG", "running simulation")
 #     if options.dummy:
 #         executor = ParameterEnsembleInputFilesGenerator( db_name )
Exemple #16
0
                      action='store_true',
                      dest="verbose",
                      help="more verbose output")

    parser.add_option("--test-run",
                      action='store_true',
                      dest="test_run",
                      help="runs once and preserves the temporary files")

    #    parser.add_option("-d","--directory-var", action='store', type = "string", dest="directory_vars",
    #                       default = False, help = "which variables to store as directories, only if tree" )

    options, args = parser.parse_args()

    for i_arg in args:
        full_name, path, base_name, extension = utils.translate_name(i_arg)

        db_name = "%s/%s.spgql" % (path, base_name)
        # sim_name = "%s/%s.spg" % (path, base_name)
        if options.purge and os.path.exists(db_name):
            os.remove(db_name)
        if options.initialise or not os.path.exists(db_name):
            utils.newline_msg("MSG", "initialising database")
            parser = MultIteratorDBBuilder(db_name=db_name)
            parser.init_db()
            parser.fill_status(repeat=options.repeat)
            del parser

        utils.newline_msg("MSG", "running simulation")
        #     if options.dummy:
        #         executor = ParameterEnsembleInputFilesGenerator( db_name )