Example #1
0
	def __init__(self):
		config.__init__(self)
		self.fp = FieldProcessor()

		df = None
		if os.path.exists(self.new_data_file):
			df = pandas.read_csv(self.new_data_file)
		else:
			df = pandas.read_excel(self.data_file)
			self.update_year_values(df)
			self.replace_column_by_binary_columns(df)
		self.df = df
		self.fp.set_fields(self.df)
		return
		#self.update_nan_values()
		#self.df.to_csv(self.new_data_file, sep=',')
		max_score = 0
		opt_flds = []
		opt_c = []
		while True:
			score,c,flds = self.get_opt_fields(opt_flds,opt_c)
			if score > max_score:
				self.fp.pop_fields(c)
				opt_flds = flds
				opt_c += c
				max_score = score
			else:
				break
			print(max_score,c)

		print(opt_c,max_score)
Example #2
0
    def __init__(self, *args):
        """Class Constructor ... called when object instance created.
    
           Arguments:
               self - object being created
        """
        config.__init__(self, args)

        self.logger = logging.getLogger(self.__class__.__name__)
        self.setLogging()

        # Figure out the details of the MQTT Server to talk to
        self.setdefault("Mqtt.Server", default="localhost")
        self.setdefault("Mqtt.Port", default="1883")
        self.setdefault("Mqtt.Identity", default="xxx")
        for key in self["Mqtt"]:
            child = self["Mqtt"][key]
            if isinstance(child, dict):
                child.setdefault("Topic", default=key)
                strMatch = child["Topic"].replace("#", "*")
                child.outputRegex = re.compile(strMatch, re.IGNORECASE)
                self.logger.info("Listening for MQQT messages:  %s",
                                 child["Topic"])

        # setup as MQQT client
        self.mos = paho.Client(self["Mqtt.Identity"])
        self.mos.on_connect = self.on_connect
        self.mos.on_disconnect = self.on_disconnect
        self.mos.on_message = self.on_message  # register for callback

        while (self.connect() == False):
            self.logger.info("Trying to connect again ...")
            time.sleep(5)
Example #3
0
    def __init__(self, datestring):
        config.__init__(self)
        dt = utils.convert_cylc_time2(datestring)
        wrfout_time = datetime.datetime.strftime(dt, '%Y-%m-%d_%H:%M:%S')
        max_dom = utils.get_max_dom()
        rundir = self.config['filesystem']['wrf_run_dir']
        archivedir = self.config['filesystem']['archive_dir']
        for dom in range(1, max_dom + 1):
            wrfout = os.path.join(rundir,
                                  'wrfout_d0' + str(dom) + '_' + wrfout_time)
            archived = os.path.join(archivedir,
                                    'wrfout_d0' + str(dom) + '_' + wrfout_time)
            os.system('nc3tonc4 ' + wrfout + ' ' + archived)
            plot_archive = os.path.join(archivedir, 'plot', wrfout_time)
            utils._create_directory(plot_archive)
            #utils._create_directory(os.path.join(plot_archive, trim))
            os.system(
                'ncl /home/haren/cylc-suites/forecast/bin/wrf_Surface3.ncl inputfile='
                + r'\"' + archived + r'\" outputfile=\"' + plot_archive +
                r'/surface_d0' + str(dom) + '.png' + r'\"')

#iname = 'surface_d0' + str(dom) + '.png')
#os.system('convert ' + os.path.join(plot_archive, iname) + ' -fuzz 1% -trim +repage ' + os.path.join(plot_archive, 'trim', iname))
        plot_latest = os.path.join(archivedir, 'plot', 'latest')
        try:
            os.symlink(plot_archive, plot_latest)
        except OSError, e:
            if e.errno == errno.EEXIST:
                os.remove(plot_latest)
                os.symlink(plot_archive, plot_latest)
Example #4
0
 def __init__(self, namelist):
   config.__init__(self)
   self.namelist = namelist
   self.read_namelist()
   self.create_namelist_copies()
   self.modify_coarse_namelist()
   self.modify_fine_namelist()
Example #5
0
    def __init__(self, *args):
        """Class Constructor ... called when object instance created.
    
           Arguments:
               self - object being created
        """
        config.__init__(self, args)

        self.logger = logging.getLogger(self.__class__.__name__)
        self.setLogging()

        # Figure out the details of the MQTT Server to talk to
        self.setdefault("Mqtt.Server", default = "localhost") 
        self.setdefault("Mqtt.Port", default = "1883")
        self.setdefault("Mqtt.Identity", default = "xxx")
        for key in self["Mqtt"]:
            child = self["Mqtt"][key]
            if isinstance(child, dict):
                child.setdefault("Topic", default = key)
                strMatch = child["Topic"].replace("#", "*")
                child.outputRegex = re.compile(strMatch, re.IGNORECASE)
	        self.logger.info("Listening for MQQT messages:  %s", child["Topic"])
        
        # setup as MQQT client
        self.mos = paho.Client(self["Mqtt.Identity"])
        self.mos.on_connect = self.on_connect
        self.mos.on_disconnect = self.on_disconnect
        self.mos.on_message = self.on_message # register for callback

        while (self.connect() == False ):
            self.logger.info ("Trying to connect again ...")
            time.sleep(5)
Example #6
0
 def __init__(self):
     config.__init__(self)
     rundir = self.config['filesystem']['wrf_run_dir']
     wpsdir = os.path.join(self.config['filesystem']['work_dir'], 'wps')
     ## wrf run dir
     # cleanup old met_em files
     # create list of files to remove
     files = [glob.glob(os.path.join(rundir, ext))
              for ext in ['met_em*']]
     # flatten list
     files_flat = [item for sublist in files for item in sublist] 
     # remove files silently
     [ utils.silentremove(filename) for filename in files_flat ]
     # copy new met_em files
     # create list of files to copy
     files = [glob.glob(os.path.join(wpsdir, ext))
              for ext in ['met_em*']]
     # flatten list
     files_flat = [item for sublist in files for item in sublist]
     [ shutil.copyfile(filename, os.path.join(rundir, os.path.basename(filename))) for filename in files_flat ]
     ## wps workdir
     # create list of files to remove
     files = [glob.glob(os.path.join(wpsdir, ext))
              for ext in ['met_em*', 'FILE*', 'PFILE*', 'GRIBFILE*']]
     # flatten list
     files_flat = [item for sublist in files for item in sublist]
     # remove files silently
     [ utils.silentremove(filename) for filename in files_flat ]
Example #7
0
 def _create_cylc_config(self, suitename, basedir):
   '''
   Create cylc suite.rc configuration file based on config.json
   '''
   config.__init__(self, os.path.join(
                   basedir, suitename, 'config.json'))
   suiterc = self._header()
   suiterc += self._scheduling()
   suiterc += self._runtime()
   suiterc += self._visualization()
   self._write(suiterc, os.path.join(basedir, suitename, 'suite.rc'))
Example #8
0
 def __init__(self):
   config.__init__(self)  # load config
   self.rundir = self.config['filesystem']['wrf_run_dir']
   self.wrfda_workdir = os.path.join(self.config['filesystem']['work_dir'],
                                     'wrfda')
   self.max_dom = utils.get_max_dom()  # get maximum domain number
   # copy default 3dvar obsproc namelist to namelist.obsproc
   self.obsproc_dir = os.path.join(self.config['filesystem']['wrfda_dir'],
                                   'var/obsproc')
   # get dictionary with workdir/obs filename per domain
   self.obs = self.get_obsproc_dirs()
Example #9
0
def create_app(config_name):
    app = Flask(__name__)
    app.config.from_object(config[config_name])
    config.__init__(app)

    bootstrap.init_app(app)
    mail.init_app(app)
    moment.init_app(app)
    db.init_app(app)
    login_manager.init_app(app)

    return app
Example #10
0
def create_app(config_name):
    app = Flask(__name__)
    app.config.from_object(config[config_name])
    config.__init__(app)

    bootstrap.init_app(app)
    mail.init_app(app)
    moment.init_app(app)
    db.init_app(app)
    login_manager.init_app(app)

    return app
Example #11
0
 def __init__(self):
     config.__init__(self)
     ifile = open(self.data_doc_file, encoding="utf8", errors='ignore')
     ftypes = ['Discrete', 'Nominal', 'Ordinal', 'Continuous']
     field_type_map = {}
     for l in ifile:
         for t in ftypes:
             te = '(' + t + ')'
             if te in l:
                 l = l.split(te)[0].strip()
                 field_type_map[l] = t
     self.field_type_map = field_type_map
     self.year_fields = [
         'Garage Yr Blt', 'Yr Sold', 'Year Built', 'Year Remod/Add'
     ]
Example #12
0
 def _create_directory_structure(self, suitename, basedir=None):
   '''
   Create directory structure for the Cylc configuration
   '''
   # set basedir to users home directory if not supplied
   if not basedir:
     basedir = os.path.join(os.path.expanduser("~"), 'cylc-suites')
   # subdirectories to create
   subdirs = ['bin' , 'control', 'doc', 'inc']
   # create subdirectories
   [utils._create_directory(
    os.path.join(basedir, suitename, subdir))
    for subdir in subdirs]
   # create empty json config file in suite directory
   # this does not overwrite an existing config file
   config.__init__(self, os.path.join(
                   basedir, suitename, 'config.json'))
Example #13
0
 def __init__(self):
   config.__init__(self)
   # read WRF namelist in WRF work_dir
   wrf_nml = f90nml.read(self.config['options_wrf']['namelist.input'])
   # get number of domains
   ndoms = wrf_nml['domains']['max_dom']
   # check if ndoms is an integer and >0
   if not (isinstance(ndoms, int) and ndoms>0):
     raise ValueError("'domains_max_dom' namelist variable should be an " \
                     "integer>0")
   for dom in range(1, ndoms):
     cdomain = dom + 1
     self.read_init(dom, cdomain)
     self.method="boundary"
     self.fix_2d_field('PSFC')
     self.fix_3d_field('P', 'T')
     self.cleanup()
Example #14
0
 def __init__(self):
     config.__init__(self)
     # read WRF namelist in WRF work_dir
     wrf_nml = f90nml.read(self.config['options_wrf']['namelist.input'])
     # get number of domains
     ndoms = wrf_nml['domains']['max_dom']
     # check if ndoms is an integer and >0
     if not (isinstance(ndoms, int) and ndoms > 0):
         raise ValueError("'domains_max_dom' namelist variable should be an " \
                         "integer>0")
     for dom in range(1, ndoms):
         cdomain = dom + 1
         self.read_init(dom, cdomain)
         self.method = "boundary"
         self.fix_2d_field('PSFC')
         self.fix_3d_field('P', 'T')
         self.cleanup()
Example #15
0
    def __init__(self, datestring):
        config.__init__(self)
        dt = utils.convert_cylc_time2(datestring)
        wrfout_time = datetime.datetime.strftime(dt, '%Y-%m-%d_%H:%M:%S')
        max_dom = utils.get_max_dom()
        rundir = self.config['filesystem']['wrf_run_dir']
        archivedir = self.config['filesystem']['archive_dir']
        for dom in range(1,max_dom+1):
            wrfout = os.path.join(rundir, 'wrfout_d0' + str(dom) + '_' + wrfout_time)
            archived = os.path.join(archivedir, 'wrfout_d0' + str(dom) + '_' + wrfout_time)
            os.system('nc3tonc4 ' + wrfout + ' ' + archived)
            plot_archive = os.path.join(archivedir, 'plot', wrfout_time)
	    utils._create_directory(plot_archive)
            #utils._create_directory(os.path.join(plot_archive, trim))
	    os.system('ncl /home/haren/cylc-suites/forecast/bin/wrf_Surface3.ncl inputfile=' + r'\"' + archived + r'\" outputfile=\"' + plot_archive + r'/surface_d0' + str(dom) + '.png' + r'\"')
	    #iname = 'surface_d0' + str(dom) + '.png')
            #os.system('convert ' + os.path.join(plot_archive, iname) + ' -fuzz 1% -trim +repage ' + os.path.join(plot_archive, 'trim', iname))
        plot_latest = os.path.join(archivedir, 'plot', 'latest')
        try:
	    os.symlink(plot_archive, plot_latest)
        except OSError, e:
            if e.errno == errno.EEXIST:
                 os.remove(plot_latest)
                 os.symlink(plot_archive, plot_latest)
Example #16
0
	def __init__(self):
		config.__init__(self)
Example #17
0
 def __init__(self):
   config.__init__(self)  # load config
   wrf_namelist = self.config['options_wrf']['namelist.input']
   split_nml_shared.__init__(self, wrf_namelist)
   self._save_namelists()
Example #18
0
 def __init__(self):
   config.__init__(self)  # load config
   wps_namelist = self.config['options_wps']['namelist.wps']
   split_nml_shared.__init__(self, wps_namelist)
   self._modify_fine_namelist_wps()
   self._save_namelists()
Example #19
0
 def __init__(self, wrfpy_dir, datestart, dateend):
   config.__init__(self, wrfpy_dir)  # load config
   # define and create wps working directory
   self.wps_workdir = os.path.join(self.config['filesystem']['work_dir'],
                                   'wps')
   utils._create_directory(self.wps_workdir)
Example #20
0
 def __init__(self):
     config.__init__(self)
     self.config["dbase_dir"] = "./"
     self.config["dbase_name"] = "wrfpy.db"
     self.database = os.path.join(self.config["dbase_dir"], self.config["dbase_name"])
Example #21
0
 def __init__(self):
   config.__init__(self)
   self.config['dbase_dir'] = './'
   self.config['dbase_name'] = 'wrfpy.db'
   self.database = os.path.join(self.config['dbase_dir'],
                                self.config['dbase_name'])