def __init__(self): config.__init__(self) self.fp = FieldProcessor() df = None if os.path.exists(self.new_data_file): df = pandas.read_csv(self.new_data_file) else: df = pandas.read_excel(self.data_file) self.update_year_values(df) self.replace_column_by_binary_columns(df) self.df = df self.fp.set_fields(self.df) return #self.update_nan_values() #self.df.to_csv(self.new_data_file, sep=',') max_score = 0 opt_flds = [] opt_c = [] while True: score,c,flds = self.get_opt_fields(opt_flds,opt_c) if score > max_score: self.fp.pop_fields(c) opt_flds = flds opt_c += c max_score = score else: break print(max_score,c) print(opt_c,max_score)
def __init__(self, *args):
"""Class Constructor ... called when object instance created.
Arguments:
self - object being created
"""
config.__init__(self, args)
self.logger = logging.getLogger(self.__class__.__name__)
self.setLogging()
# Figure out the details of the MQTT Server to talk to
self.setdefault("Mqtt.Server", default="localhost")
self.setdefault("Mqtt.Port", default="1883")
self.setdefault("Mqtt.Identity", default="xxx")
for key in self["Mqtt"]:
child = self["Mqtt"][key]
if isinstance(child, dict):
child.setdefault("Topic", default=key)
strMatch = child["Topic"].replace("#", "*")
child.outputRegex = re.compile(strMatch, re.IGNORECASE)
self.logger.info("Listening for MQQT messages: %s",
child["Topic"])
# setup as MQQT client
self.mos = paho.Client(self["Mqtt.Identity"])
self.mos.on_connect = self.on_connect
self.mos.on_disconnect = self.on_disconnect
self.mos.on_message = self.on_message # register for callback
while (self.connect() == False):
self.logger.info("Trying to connect again ...")
time.sleep(5)
def __init__(self, datestring):
config.__init__(self)
dt = utils.convert_cylc_time2(datestring)
wrfout_time = datetime.datetime.strftime(dt, '%Y-%m-%d_%H:%M:%S')
max_dom = utils.get_max_dom()
rundir = self.config['filesystem']['wrf_run_dir']
archivedir = self.config['filesystem']['archive_dir']
for dom in range(1, max_dom + 1):
wrfout = os.path.join(rundir,
'wrfout_d0' + str(dom) + '_' + wrfout_time)
archived = os.path.join(archivedir,
'wrfout_d0' + str(dom) + '_' + wrfout_time)
os.system('nc3tonc4 ' + wrfout + ' ' + archived)
plot_archive = os.path.join(archivedir, 'plot', wrfout_time)
utils._create_directory(plot_archive)
#utils._create_directory(os.path.join(plot_archive, trim))
os.system(
'ncl /home/haren/cylc-suites/forecast/bin/wrf_Surface3.ncl inputfile='
+ r'\"' + archived + r'\" outputfile=\"' + plot_archive +
r'/surface_d0' + str(dom) + '.png' + r'\"')
#iname = 'surface_d0' + str(dom) + '.png')
#os.system('convert ' + os.path.join(plot_archive, iname) + ' -fuzz 1% -trim +repage ' + os.path.join(plot_archive, 'trim', iname))
plot_latest = os.path.join(archivedir, 'plot', 'latest')
try:
os.symlink(plot_archive, plot_latest)
except OSError, e:
if e.errno == errno.EEXIST:
os.remove(plot_latest)
os.symlink(plot_archive, plot_latest)
def __init__(self, namelist): config.__init__(self) self.namelist = namelist self.read_namelist() self.create_namelist_copies() self.modify_coarse_namelist() self.modify_fine_namelist()
def __init__(self, *args):
"""Class Constructor ... called when object instance created.
Arguments:
self - object being created
"""
config.__init__(self, args)
self.logger = logging.getLogger(self.__class__.__name__)
self.setLogging()
# Figure out the details of the MQTT Server to talk to
self.setdefault("Mqtt.Server", default = "localhost")
self.setdefault("Mqtt.Port", default = "1883")
self.setdefault("Mqtt.Identity", default = "xxx")
for key in self["Mqtt"]:
child = self["Mqtt"][key]
if isinstance(child, dict):
child.setdefault("Topic", default = key)
strMatch = child["Topic"].replace("#", "*")
child.outputRegex = re.compile(strMatch, re.IGNORECASE)
self.logger.info("Listening for MQQT messages: %s", child["Topic"])
# setup as MQQT client
self.mos = paho.Client(self["Mqtt.Identity"])
self.mos.on_connect = self.on_connect
self.mos.on_disconnect = self.on_disconnect
self.mos.on_message = self.on_message # register for callback
while (self.connect() == False ):
self.logger.info ("Trying to connect again ...")
time.sleep(5)
def __init__(self):
config.__init__(self)
rundir = self.config['filesystem']['wrf_run_dir']
wpsdir = os.path.join(self.config['filesystem']['work_dir'], 'wps')
## wrf run dir
# cleanup old met_em files
# create list of files to remove
files = [glob.glob(os.path.join(rundir, ext))
for ext in ['met_em*']]
# flatten list
files_flat = [item for sublist in files for item in sublist]
# remove files silently
[ utils.silentremove(filename) for filename in files_flat ]
# copy new met_em files
# create list of files to copy
files = [glob.glob(os.path.join(wpsdir, ext))
for ext in ['met_em*']]
# flatten list
files_flat = [item for sublist in files for item in sublist]
[ shutil.copyfile(filename, os.path.join(rundir, os.path.basename(filename))) for filename in files_flat ]
## wps workdir
# create list of files to remove
files = [glob.glob(os.path.join(wpsdir, ext))
for ext in ['met_em*', 'FILE*', 'PFILE*', 'GRIBFILE*']]
# flatten list
files_flat = [item for sublist in files for item in sublist]
# remove files silently
[ utils.silentremove(filename) for filename in files_flat ]
def _create_cylc_config(self, suitename, basedir):
'''
Create cylc suite.rc configuration file based on config.json
'''
config.__init__(self, os.path.join(
basedir, suitename, 'config.json'))
suiterc = self._header()
suiterc += self._scheduling()
suiterc += self._runtime()
suiterc += self._visualization()
self._write(suiterc, os.path.join(basedir, suitename, 'suite.rc'))
def __init__(self):
config.__init__(self) # load config
self.rundir = self.config['filesystem']['wrf_run_dir']
self.wrfda_workdir = os.path.join(self.config['filesystem']['work_dir'],
'wrfda')
self.max_dom = utils.get_max_dom() # get maximum domain number
# copy default 3dvar obsproc namelist to namelist.obsproc
self.obsproc_dir = os.path.join(self.config['filesystem']['wrfda_dir'],
'var/obsproc')
# get dictionary with workdir/obs filename per domain
self.obs = self.get_obsproc_dirs()
def create_app(config_name):
app = Flask(__name__)
app.config.from_object(config[config_name])
config.__init__(app)
bootstrap.init_app(app)
mail.init_app(app)
moment.init_app(app)
db.init_app(app)
login_manager.init_app(app)
return app
def create_app(config_name):
app = Flask(__name__)
app.config.from_object(config[config_name])
config.__init__(app)
bootstrap.init_app(app)
mail.init_app(app)
moment.init_app(app)
db.init_app(app)
login_manager.init_app(app)
return app
def __init__(self):
config.__init__(self)
ifile = open(self.data_doc_file, encoding="utf8", errors='ignore')
ftypes = ['Discrete', 'Nominal', 'Ordinal', 'Continuous']
field_type_map = {}
for l in ifile:
for t in ftypes:
te = '(' + t + ')'
if te in l:
l = l.split(te)[0].strip()
field_type_map[l] = t
self.field_type_map = field_type_map
self.year_fields = [
'Garage Yr Blt', 'Yr Sold', 'Year Built', 'Year Remod/Add'
]
def _create_directory_structure(self, suitename, basedir=None):
'''
Create directory structure for the Cylc configuration
'''
# set basedir to users home directory if not supplied
if not basedir:
basedir = os.path.join(os.path.expanduser("~"), 'cylc-suites')
# subdirectories to create
subdirs = ['bin' , 'control', 'doc', 'inc']
# create subdirectories
[utils._create_directory(
os.path.join(basedir, suitename, subdir))
for subdir in subdirs]
# create empty json config file in suite directory
# this does not overwrite an existing config file
config.__init__(self, os.path.join(
basedir, suitename, 'config.json'))
def __init__(self):
config.__init__(self)
# read WRF namelist in WRF work_dir
wrf_nml = f90nml.read(self.config['options_wrf']['namelist.input'])
# get number of domains
ndoms = wrf_nml['domains']['max_dom']
# check if ndoms is an integer and >0
if not (isinstance(ndoms, int) and ndoms>0):
raise ValueError("'domains_max_dom' namelist variable should be an " \
"integer>0")
for dom in range(1, ndoms):
cdomain = dom + 1
self.read_init(dom, cdomain)
self.method="boundary"
self.fix_2d_field('PSFC')
self.fix_3d_field('P', 'T')
self.cleanup()
def __init__(self):
config.__init__(self)
# read WRF namelist in WRF work_dir
wrf_nml = f90nml.read(self.config['options_wrf']['namelist.input'])
# get number of domains
ndoms = wrf_nml['domains']['max_dom']
# check if ndoms is an integer and >0
if not (isinstance(ndoms, int) and ndoms > 0):
raise ValueError("'domains_max_dom' namelist variable should be an " \
"integer>0")
for dom in range(1, ndoms):
cdomain = dom + 1
self.read_init(dom, cdomain)
self.method = "boundary"
self.fix_2d_field('PSFC')
self.fix_3d_field('P', 'T')
self.cleanup()
def __init__(self, datestring):
config.__init__(self)
dt = utils.convert_cylc_time2(datestring)
wrfout_time = datetime.datetime.strftime(dt, '%Y-%m-%d_%H:%M:%S')
max_dom = utils.get_max_dom()
rundir = self.config['filesystem']['wrf_run_dir']
archivedir = self.config['filesystem']['archive_dir']
for dom in range(1,max_dom+1):
wrfout = os.path.join(rundir, 'wrfout_d0' + str(dom) + '_' + wrfout_time)
archived = os.path.join(archivedir, 'wrfout_d0' + str(dom) + '_' + wrfout_time)
os.system('nc3tonc4 ' + wrfout + ' ' + archived)
plot_archive = os.path.join(archivedir, 'plot', wrfout_time)
utils._create_directory(plot_archive)
#utils._create_directory(os.path.join(plot_archive, trim))
os.system('ncl /home/haren/cylc-suites/forecast/bin/wrf_Surface3.ncl inputfile=' + r'\"' + archived + r'\" outputfile=\"' + plot_archive + r'/surface_d0' + str(dom) + '.png' + r'\"')
#iname = 'surface_d0' + str(dom) + '.png')
#os.system('convert ' + os.path.join(plot_archive, iname) + ' -fuzz 1% -trim +repage ' + os.path.join(plot_archive, 'trim', iname))
plot_latest = os.path.join(archivedir, 'plot', 'latest')
try:
os.symlink(plot_archive, plot_latest)
except OSError, e:
if e.errno == errno.EEXIST:
os.remove(plot_latest)
os.symlink(plot_archive, plot_latest)
def __init__(self): config.__init__(self)
def __init__(self): config.__init__(self) # load config wrf_namelist = self.config['options_wrf']['namelist.input'] split_nml_shared.__init__(self, wrf_namelist) self._save_namelists()
def __init__(self): config.__init__(self) # load config wps_namelist = self.config['options_wps']['namelist.wps'] split_nml_shared.__init__(self, wps_namelist) self._modify_fine_namelist_wps() self._save_namelists()
def __init__(self, wrfpy_dir, datestart, dateend):
config.__init__(self, wrfpy_dir) # load config
# define and create wps working directory
self.wps_workdir = os.path.join(self.config['filesystem']['work_dir'],
'wps')
utils._create_directory(self.wps_workdir)
def __init__(self):
config.__init__(self)
self.config["dbase_dir"] = "./"
self.config["dbase_name"] = "wrfpy.db"
self.database = os.path.join(self.config["dbase_dir"], self.config["dbase_name"])
def __init__(self):
config.__init__(self)
self.config['dbase_dir'] = './'
self.config['dbase_name'] = 'wrfpy.db'
self.database = os.path.join(self.config['dbase_dir'],
self.config['dbase_name'])