def lade_konfiguration(self, dateiname):
cfg = ConfigParser()
cfg.read([dateiname])
self.speichergröße = cfg.getint("rechner", "speichergröße")
self.pio_speichergröße = cfg.getint("rechner", "pio_speichergröße")
self.view.max_taktrate = cfg.getint("gui", "max_taktrate")
self.view.setze_schriftart(cfg.get("gui", "schriftart"), cfg.getint("gui", "schriftgröße"))
class BoardParameters(object):
"""
Loads device channel settings and parameters from an INI file.
"""
def __init__(self, ini_file):
self.log = logging.getLogger(".".join([__name__, self.__class__.__name__]))
self.log.setLevel(logging.DEBUG)
self._ini_filename = find_file(ini_file)
self.conf = None
def load_ini(self):
"""
Loads and parses the data from INI file.
"""
self.conf = SafeConfigParser(dict_type=OrderedDict)
self.conf.read(self._ini_filename)
self.log.debug("Read Board Parameters INI file %s:", self._ini_filename)
self.log.debug(" sections: %s", self.conf.sections())
@property
def board_name(self):
if "Board_header" not in self.conf.sections():
raise_with_traceback(RuntimeError("Board_header section not found in ini file %s" % str(self._ini_filename)))
return self.conf.get("Board_header", "Board_name")
@property
def board_type(self):
if "Board_header" not in self.conf.sections():
raise_with_traceback(RuntimeError("Board_header section not found in ini file %s" % str(self._ini_filename)))
return BoardTypes(self.conf.getint("Board_header", "Board_type"))
@property
def board_revision(self):
if "Board_header" not in self.conf.sections():
raise_with_traceback(RuntimeError("Board_header section not found in ini file %s" % str(self._ini_filename)))
return self.conf.getint("Board_header", "Board_revision_number")
@property
def control_channels_count(self):
if "Entry_counts" not in self.conf.sections():
raise_with_traceback(RuntimeError("Entry_counts section not found in ini file %s" % str(self._ini_filename)))
return self.conf.getint("Entry_counts", "Control_channels_count")
@property
def monitoring_channels_count(self):
if "Entry_counts" not in self.conf.sections():
raise_with_traceback(RuntimeError("Entry_counts section not found in ini file %s" % str(self._ini_filename)))
return self.conf.getint("Entry_counts", "Monitoring_channels_count")
class Main:
config = None
db = None
hosts = []
def __init__(self, config_path):
self.config = SafeConfigParser(allow_no_value=False)
self.config.read(config_path + os.sep + 'communicator.cfg')
self.db = oursql.Connection(
host=self.config.get('database', 'host'),
port=self.config.getint('database', 'port'),
user=self.config.get('database', 'username'),
passwd=self.config.get('database', 'password'),
db=self.config.get('database', 'dbname'),
charset="utf8")
def getHosts(self):
with self.db.cursor() as cur:
cur.execute(b'SELECT `id`, `name`, `address`, `port`, `key`, `uuid`' +
b' FROM `host` WHERE `active`', plain_query=True)
for row in cur:
self.hosts.append(Host(
row[0],
row[1],
(row[2], row[3]),
row[4],
row[5],
self.db)
)
def run(self):
self.getHosts()
def loadConfig(cls):
config = SafeConfigParser()
if os.path.isfile(CONFIG_FILE):
config.read_file(codecs.open(CONFIG_FILE, encoding='utf-8'))
# The default config
cls.config = {}
cls.config['command_open'] = None
cls.config['player'] = 1
cls.config['debug'] = 0
# Load the options
if config.has_section('options'):
for key, value in config.items('options'):
cls.config[key] = value
# Load the series
cls.series = []
for section in config.sections():
if section != 'options':
title = config.get(section, 'title')
videos = config.get(section, 'videos')
theTvDb = config.getint(section, 'theTvDb')
lang = config.get(section, 'lang')
cls.series.append([section, title, videos, theTvDb, lang])
def main():
try:
config = SafeConfigParser()
if config.read('config.ini') != ['config.ini']:
raise OSError('no config.ini file present')
if config.getboolean('Generate CSVs', 'autogen-q'):
if config.get('Generate CSVs', 'type') == 'logarithmic':
minQ = config.getint('Generate CSVs', 'min-q')
maxQ = config.getint('Generate CSVs', 'max-q')
numQ = config.getint('Generate CSVs', 'num-q')
pd.Series(np.logspace(round(math.log(minQ, 10)), round(math.log(maxQ,10)), numQ, dtype=int)).to_csv('DataIN/Quantity.csv', index=False)
if config.get('Generate CSVs', 'type') == 'linear':
minQ = config.getint('Generate CSVs', 'min-q')
maxQ = config.getint('Generate CSVs', 'max-q')
numQ = config.getint('Generate CSVs', 'num-q')
pd.Series(np.linspace(minQ, maxQ, numQ, dtype=int)).to_csv('DataIN/Quantity.csv', index=False)
if config.get('Generate CSVs', 'type') == 'digikey':
pd.Series([1,5,10,25,50,100,250,500,1000,5000,10000]).to_csv('DataIN/Quantity.csv', index=False)
qs = pd.read_csv('DataIN/Quantity.csv', squeeze=True)
titlesVarCost = ['Name']
titlesDigiKey = ['Name', 'URL', 'Quantity']
for i in range(0, len(qs)):
titlesVarCost.append('Cost @' + str(qs.iloc[i]) + "pcs")
if config.getboolean('Generate CSVs', 'gen-fc'):
pd.DataFrame(columns = ['Name', 'Cost']).to_csv('DataIN/FixedCosts.csv', index=False)
if config.getboolean('Generate CSVs', 'gen-dkp'):
pd.DataFrame(columns = titlesDigiKey).to_csv('DataIN/DigiKeyParts.csv', index=False)
if config.getboolean('Generate CSVs', 'gen-vc'):
pd.DataFrame(columns = titlesVarCost).to_csv('DataIN/VarCosts.csv', index=False)
if config.getboolean('Generate CSVs', 'gen-p'):
pd.DataFrame(columns = ['Price']).to_csv('DataIN/Price.csv', index=False)
except OSError as e:
print(e)
except:
print("something went wrong, most likely your config.ini file is not properly configured")
def fromConfigFile(filename):
print('loading config from file: %s' % filename)
config = SafeConfigParser()
config.read(filename)
ret = ServerConfig()
ret.server_host = config.get('server', 'host')
ret.server_port = config.getint('server', 'port')
ret.whole_world_coords = [ config.getfloat('server', 'min_lon'),
config.getfloat('server', 'min_lat'),
config.getfloat('server', 'max_lon'),
config.getfloat('server', 'max_lat') ]
ret.db_host = config.get('db', 'host')
ret.db_port = config.getint('db', 'port')
ret.db_name = config.get('db', 'name')
ret.verbose = config.get('debug', 'verbose')
return ret
def Init():
global t
global last_update
global config
t = Bot(token)
if(t.CheckSettings()):
config = SafeConfigParser()
config.read(ini_path)
last_update = config.getint('main', 'last_update')
logger.info("-- Init -- " + str(last_update))
UpdatesLoop()
else:
return
def run():
config = SafeConfigParser()
config.read("config/defaults.cfg")
config.read("~/.config/mcdu.cfg")
config.read("config/mcdu.cfg")
sim = config.get("General", "sim")
if sim == "fsx":
from mcdu.fsx import FSXReceiver
receiver = FSXReceiver()
elif sim == "xplane":
from mcdu.xplane import XPlaneReceiver
receiver = XPlaneReceiver()
else:
print("no simulator set");
return 1
receiver.start()
api = ACARS_API(config.get("ACARS", "logon"))
acars = ACARS(api)
atc = ATC(api)
mcdu = MCDU()
mcdu.subsystem_register(acars)
mcdu.subsystem_register(atc)
mcdu.menu()
application = tornado.web.Application([
(r"^/socket", WebSocket, dict(mcdu=mcdu)),
(r"^/(.*)$", tornado.web.StaticFileHandler, {"path": "res/", "default_filename": "index.html"}),
], debug=False)
port = config.getint("General", "port")
application.listen(port)
try:
print("running on port %i" % port)
webbrowser.open_new("http://localhost:%i" % port)
tornado.ioloop.IOLoop.instance().start()
except KeyboardInterrupt:
print("quitting...")
except Exception as e:
import traceback
traceback.print_exc()
print("quitting...")
finally:
receiver.stop()
acars.stop()
atc.stop()
return 0
def get_connection(database, hostname=None, port=None, login=None, password=None):
parser = SafeConfigParser()
filename = os.path.join(os.path.dirname(__file__), 'config.ini')
parser.read(filename)
if hostname is None:
hostname = parser.get(database, 'host')
if port is None:
port = parser.getint(database, 'port')
if login is None:
login = parser.get(database, 'username')
if password is None:
password = parser.get(database, 'password')
return openerplib.get_connection(hostname=hostname, port=port, database=database, login=login, password=password)
def getConnectionInfo():
config_filename = os.path.join(os.path.dirname(__file__), os.path.pardir, 'connection.ini')
cp = SafeConfigParser()
cp.read(config_filename)
info = {
'server_name': cp.get('server', 'name'),
'server_ip': cp.get('server', 'ip'),
'server_port': cp.getint('server', 'port'),
'client_name': cp.get('client', 'name'),
'user': cp.get('user', 'name'),
'password': cp.get('user', 'password'),
}
return info
class SensorConfigurationParameters(object):
"""
Loads sensor configuration parameters from an INI file
"""
def __init__(self, ini_file):
self.log = logging.getLogger(".".join([__name__, self.__class__.__name__]))
self._ini_filename = None
self._ini_buffer = None
self._conf = None
try:
self._ini_filename = find_file(ini_file)
except:
# If we catch any kind of exception here then treat the parameter as the configuration
self._ini_buffer = StringIO(unicode(ini_file))
def load_ini(self):
"""
Loads and parses the data from INI file. The data is stored internally in the object and can be retrieved
through the property methods
For the system settings all parameter names <section>_<name>
"""
self._conf = SafeConfigParser(dict_type=OrderedDict)
self._conf.optionxform = str
if self._ini_filename:
self._conf.read(self._ini_filename)
self.log.info("Read Sensor Configuration Settings INI file: %s", self._ini_filename)
else:
self._conf.readfp(self._ini_buffer)
self.log.info("Read Sensor Configuration Settings INI object %s", self._ini_buffer)
self.log.info(" sections: %s", self._conf.sections())
@property
def value_map(self):
# Read out the section General that describes the rest of the file
values = {}
desc = {}
for item in self._conf.items('General'):
match = re.match(r'^.*Cols<(\w*)>$', item[0])
if match:
desc[match.group(1)] = int(item[1])
# Now loop over each defined group adding the values
self.log.info("Ini description: %s", desc)
for item in desc:
val_list = []
for index in range(desc[item]):
item_name = "Col<{}>".format(index)
val_list.append(self._conf.getint(item, item_name))
values[item] = val_list
return values
class Config(object):
def __init__(self):
self._configFileName = "rover.cfg"
self._readConfig()
def _readConfig(self):
self._config = SafeConfigParser({
"db_host": "localhost",
"db_port": "27017",
"db_name": "rover",
"serial_path": "/dev/ttymxc3",
"serial_baud": "115200",
"serial_timeout": "0.5",
"serial_encoding": "windows-1252"})
self._config.read(self._configFileName)
self._validateConfig()
def _writeConfig(self):
self._config.write(open(self._configFileName, "w"))
def _validateConfig(self):
changed = False
if not self._config.has_section("db"):
self._config.add_section("db")
changed = True
if not self._config.has_section("arduino_serial"):
self._config.add_section("arduino_serial")
changed = True
if changed:
self._writeConfig()
def get(self, section, key):
return self._config.get(section, key)
def getInt(self, section, key):
return self._config.getint(section, key)
def getFloat(self, section, key):
return self._config.getfloat(section, key)
def getBoolean(self, section, key):
return self._config.getboolean(section, key)
def set(self, section, key, value):
self._config.set(section, key, value)
self._writeConfig()
def _load_config(self):
"Load and parse config file, pass options to livestreamer"
config = SafeConfigParser()
config_file = os.path.join(self.config_path, 'settings.ini')
config.read(config_file)
for option, type in list(AVAILABLE_OPTIONS.items()):
if config.has_option('DEFAULT', option):
if type == 'int':
value = config.getint('DEFAULT', option)
if type == 'float':
value = config.getfloat('DEFAULT', option)
if type == 'bool':
value = config.getboolean('DEFAULT', option)
if type == 'str':
value = config.get('DEFAULT', option)
self.livestreamer.set_option(option, value)
class Configuration(object):
def __init__(self):
self.parser = SafeConfigParser(defaults={
'theme' : 'default',
'theme.mobile' : 'touch',
'hostname' : '0.0.0.0',
'port' : '7000',
'log.file' : '/tmp/rb-serve.log',
'log.level' : 'INFO',
'log.format' : '%(levelname)s %(asctime)s %(name)s: %(message)s',
'debug' : 'False',
})
self.parser.add_section('server')
self.parser.read(path.expanduser('~/.rhythmweb'))
self.configure_logger()
def get_string(self, key):
return self.parser.get('server', key, raw=True)
def get_int(self, key):
return self.parser.getint('server', key)
def get_boolean(self, key):
return self.parser.getboolean('server', key)
def configure_logger(self):
root = logging.getLogger()
root.setLevel(self.get_string('log.level'))
handler = logging.handlers.RotatingFileHandler(
self.get_string('log.file'),
backupCount=5,
maxBytes=1024*1024)
handler.setFormatter(logging.Formatter(fmt=self.get_string('log.format')))
root.addHandler(handler)
if root.isEnabledFor(logging.DEBUG):
root.debug('Logger configured')
root.debug('Showing app configuration:')
for section_name in self.parser.sections():
root.debug('Section: %s' % section_name)
for key in self.parser.options(section_name):
root.debug(' %s = %s' % (key, self.parser.get(section_name, key, raw=True)))
def parse_config(self,config,section=None):
if section is None: section = self.section
if isinstance(config,basestring):
configfile = config
config = SafeConfigParser()
config.read(configfile)
# Set the options first (otherwise optional positional arguments disappear)
if OPTIONS in config._sections[section].keys():
self.parse_args(config.get(section,OPTIONS))
# Don't grab the default keys
dests = [a.dest for a in self.parser._actions]
for key in config._sections[section].keys():
if key == '__name__': continue
if key == OPTIONS: continue
try:
index = dests.index(key)
action = self.parser._actions[index]
except ValueError:
msg = "Unrecognized argument in config: %s\n"%key
msg += self.get_help()
raise ValueError(msg)
if isinstance(action,argparse._StoreTrueAction):
value = config.getboolean(section,key)
if isinstance(action,argparse._StoreFalseAction):
value = config.getboolean(section,key)
if isinstance(action,argparse._CountAction):
try:
value = int(config.getboolean(section,key))
except ValueError:
value = config.getint(section,key)
else:
value = config.get(section,key)
setattr(self.namespace,key,value)
return self.namespace
def __init__(self, str_config, modelname):
config = SafeConfigParser()
config.read(str_config)
self.sections = config.sections()
if modelname in self.sections:
try:
self.modelname = modelname
self.arch = config.get(modelname, "ARCH")
self.process_fun = 'default'
if 'PROCESS_FUN' in config[modelname] is not None:
self.process_fun = config[modelname]['PROCESS_FUN']
self.number_of_classes = config.getint(modelname,
"NUM_CLASSES")
self.number_of_iterations = config.getint(
modelname, "NUM_ITERATIONS")
#self.dataset_size = config.getint(modelname, "DATASET_SIZE")
#self.test_size = config.getint(modelname, "TEST_SIZE")
self.batch_size = config.getint(modelname, "BATCH_SIZE")
#self.estimated_number_of_batches = int ( float(self.dataset_size) / float(self.batch_size) )
#self.estimated_number_of_batches_test = int ( float(self.test_size) / float(self.batch_size) )
#snapshot time sets when temporal weights are saved (in steps)
self.snapshot_steps = config.getint(modelname,
"SNAPSHOT_STEPS")
#test time sets when test is run (in seconds)
self.test_time = config.getint(modelname, "TEST_TIME")
self.lr = config.getfloat(modelname, "LEARNING_RATE")
#snapshot folder, where training data will be saved
self.snapshot_prefix = config.get(modelname, "SNAPSHOT_DIR")
#number of estimated epochs
#self.number_of_epochs = int ( float(self.number_of_iterations) / float(self.estimated_number_of_batches) )
#folder where tf data is saved. Used for training and testing
self.data_dir = config.get(modelname, "DATA_DIR")
self.channels = config.getint(modelname, "CHANNELS")
assert (self.channels == 1 or self.channels == 3)
except Exception:
raise ValueError("something wrong with configuration file " +
str_config)
else:
raise ValueError(" {} is not a valid section".format(modelname))
def _read_config(self):
"""Read config file."""
conf = {}
config = SafeConfigParser({
'title': 'BAT:', 'order': '0', 'interval': '0', 'threshold': '15',
'format': '{bar} {percentage}%% {time}', 'alt_format': '{time}'})
config.read([path.expanduser('~/.i3/py3status/modules.ini')])
try:
conf['title'] = config.get('batterystatus', 'title')
conf['order'] = config.getint('batterystatus', 'order')
conf['interval'] = config.getint('batterystatus', 'interval')
conf['threshold'] = config.getint('batterystatus', 'threshold')
conf['format'] = config.get('batterystatus', 'format')
conf['alt_format'] = config.get('batterystatus', 'alt_format')
except NoSectionError:
conf['title'] = config.get('DEFAULT', 'title')
conf['order'] = config.getint('DEFAULT', 'order')
conf['interval'] = config.getint('DEFAULT', 'interval')
conf['threshold'] = config.getint('DEFAULT', 'threshold')
conf['format'] = config.get('DEFAULT', 'format')
conf['alt_format'] = config.get('DEFAULT', 'alt_format')
return conf
def run(fh=None):
print """\
pagerprinter v0.1.3+
Copyright 2010 - 2015 Michael Farrell <http://micolous.id.au/>
"""
# parse config
c = SafeConfigParser()
c.read_dict({
'pagerprinter': {
'update-freq': '30',
'backend': 'sacfs',
'browser': 'firefox',
'browser-exec': 'firefox',
'browser-wait': '20',
'trigger': 'RESPOND',
'trigger-end': 'MAP',
'mapper': 'google',
'print-copies': '1',
'unit': 'all',
'home': '141 King William Street, Adelaide SA 5000',
},
})
if fh is not None:
c.readfp(fh)
# get a scraper instance
scraper = get_scraper(
c.get('pagerprinter', 'backend')
)(
c.getint('pagerprinter', 'update-freq')
)
# get a browser helper instance
browser = get_browser(
c.get('pagerprinter', 'browser')
)(
c.get('pagerprinter', 'browser-exec'),
c.getint('pagerprinter', 'browser-wait')
)
trigger = c.get('pagerprinter', 'trigger').lower().strip()
trigger_end = c.get('pagerprinter', 'trigger-end').lower().strip()
my_unit = c.get('pagerprinter', 'unit').lower().strip()
try:
printer = c.get('pagerprinter', 'printer')
except NoOptionError:
printer = None
print_copies = c.getint('pagerprinter', 'print-copies')
if print_copies < 1:
print "ERROR: print-copies is set to less than one. You probably don't want this."
return
mapper = c.get('pagerprinter', 'mapper')
plugins = []
if c.has_option('pagerprinter', 'plugins'):
plugins = [
get_plugin(x.strip())
for x
in c.get('pagerprinter', 'plugins').lower().split(',')
]
for plugin in plugins:
plugin.configure(c)
mapper = get_map(mapper)
# special case: all units.
# may result in dupe printouts
if my_unit == 'all':
my_unit = ''
# home
home = c.get('pagerprinter', 'home')
# now, lets setup a handler for these events.
def page_handler(good_parse, msg, date=None, unit=None):
if good_parse:
print "Good parse! %s - %s" % (repr(msg), unit)
# filter for unit
if my_unit in unit.lower():
# this is for me!!!
print "- This is a message for my unit!"
# check for trigger
lmsg = msg.lower()
if trigger in lmsg:
# trigger found
# split by trigger and find address nicely.
addr = lmsg.split(trigger)[1]
if trigger_end in lmsg:
addr = addr.split(trigger_end)[0]
# Remove the @ symbols in the message, and the ASE device number (#nnn/nnn)
addr = re.sub(r'#\d{3}/\d{3}|@|\s:\s', '', addr)
# now split that up into parts, discarding the first
# which is a description of the event
addr_p = addr.split(',')[-2:]
# clone the list for iteration as we well modify in this operation as well
for i, part in enumerate(list(addr_p)):
if 'alarm level' in part:
del addr_p[i]
break
# reassemble the address
addr = ','.join(addr_p)
del addr_p
# we have an address. feed it to the mapping engine
url = mapper.get_url(home, addr)
print "- Address: %s" % addr
print "- URL for directions: %s" % url
# sending to browser
browser.print_url(url, printer, print_copies)
# now, send to plugins
for plugin in plugins:
try:
plugin.execute(msg, unit, addr, date, printer, print_copies)
except Exception, e:
print "Exception caught in plugin %s" % type(plugin)
print e
else:
print "- WARNING: End trigger not found! Skipping..."
else:
print "- Trigger not found. Skipping..."
else:
print "- This isn't for my unit. Skipping..."
else:
class ZFind:
def __init__(self, num_z=5, inifile=None, dest=None, overwrite=True):
self.num_z = num_z
self.inifile = inifile
self.dest = dest
self.overwrite = overwrite
if self.inifile: self.set_templates_from_inifile()
def set_templates_from_inifile(self):
self.labels = []
self.templates = []
self.zmin = []
self.zmax = []
self.npoly = []
self.npixstep = []
if exists(self.inifile):
self.option = SafeConfigParser()
self.option.optionxform = str
r = self.option.read(self.inifile)
if len(r) == 1:
for section in self.option.sections():
self.labels.append(section)
if self.option.has_option(section, 'template'):
self.templates.append(
self.option.get(section, 'template'))
if self.option.has_option(section, 'zmin'):
self.zmin.append(self.option.getfloat(section, 'zmin'))
if self.option.has_option(section, 'zmax'):
self.zmax.append(self.option.getfloat(section, 'zmax'))
if self.option.has_option(section, 'npoly'):
self.npoly.append(self.option.getint(section, 'npoly'))
if self.option.has_option(section, 'npixstep'):
self.npixstep.append(
self.option.getint(section, 'npixstep'))
else:
print("Cannot parse ini file %r" % self.inifile)
if not self.labels: self.labels = None
if not self.templates: self.templates = None
if not self.zmin: self.zmin = None
if not self.zmax: self.zmax = None
if not self.npoly: self.npoly = None
if not self.npixstep: self.npixstep = None
self.set_templates()
else:
print("WARNING: %r does not exist" % self.inifile)
def set_templates(self,
templates=None,
zmin=None,
zmax=None,
npoly=None,
npixstep=None):
if templates: self.templates = templates
if zmin: self.zmin = zmin
if zmax: self.zmax = zmax
if npoly: self.npoly = npoly
if npixstep: self.npixstep = npixstep
if type(self.templates) is str:
try:
self.templates = [self.templates]
except:
print('Templates not a list and unable to convert to list!')
sys.exit(1)
if type(self.templates) is list:
self.templates = list(map(str, self.templates))
if self.zmin is not None:
if type(self.zmin) is not list:
try:
self.zmin = [self.zmin]
except:
try:
self.zmin = self.zmin.tolist()
except:
print(
'Can\'t convert zmin to list - defaulting to full zrange!'
)
self.zmin = None
self.zmax = None
if type(self.zmin) is list:
if len(self.zmin) != len(self.templates):
print(
'Length of zmin doesn\'t match length of templates - defaulting to full zrange!'
)
self.zmin = None
self.zmax = None
if self.zmax is None:
print('zmax not given - defaulting to full zrange!')
self.zmin = None
self.zmax = None
else:
if type(self.zmax) is not list:
try:
self.zmax = [self.zmax]
except:
try:
self.zmax = self.zmax.tolist()
except:
print(
'Can\'t convert zmax to list - defaulting to full zrange!'
)
self.zmin = None
self.zmax = None
if len(self.zmin) != len(self.zmax):
print(
'Length of zmin and zmax don\'t match - defaulting to full zrange!'
)
self.zmin = None
self.zmax = None
#import pdb; pdb.set_trace()
if self.npoly is None:
self.npoly = [4] * len(self.templates)
else:
if type(self.npoly) is not list:
try:
self.npoly = [self.npoly]
except:
try:
self.npoly = self.npoly.tolist()
except:
print('npoly not a list and unable to convert to \
list - defaulting to npoly=4 for all templates!'
)
self.npoly = [4] * len(self.templates)
else:
self.npoly = list(map(int, self.npoly))
if self.npixstep is None:
self.npixstep = [1] * len(self.templates)
else:
if type(self.npixstep) is not list:
try:
self.npixstep = [self.npixstep]
except:
try:
self.npixstep = self.npixstep.tolist()
except:
print('npixstep not a list and unable to convert to \
list - defaulting to npixstep=1 for all \
templates!')
self.npixstep = [1] * len(self.templates)
else:
self.npixstep = list(map(int, self.npixstep))
def reduce_plate_mjd(self, plate, mjd, fiberid=None, chi2file=False):
self.chi2file = chi2file
# Check types and try to convert to proper types if necessary
if fiberid is None: fiberid = [i for i in range(1000)]
else:
if type(fiberid) is not list:
try:
fiberid = [fiberid]
fiberid = list(map(int, fiberid))
except:
try:
fiberid = fiberid.tolist()
fiberid = list(map(int, fiberid))
except:
print('fiberid not set properly - running full plate!')
fiberid = [i for i in range(1000)]
else:
fiberid = list(map(int, fiberid))
# Spec
specs = spec.Spec(plate=plate, mjd=mjd, fiberid=fiberid)
# ZFinder, ZFitter
zfindobjs = []
zfitobjs = []
if (self.zmin is not None) & (self.zmax is not None):
for i in range(len(self.templates)):
zfindobjs.append(
zfinder.ZFinder(fname=self.templates[i],
npoly=self.npoly[i],
zmin=self.zmin[i],
zmax=self.zmax[i]))
zfindobjs[i].zchi2(specs.flux,
specs.loglambda,
specs.ivar,
npixstep=self.npixstep[i],
plate=plate,
mjd=mjd,
fiberid=fiberid[0],
chi2file=self.chi2file)
zfitobjs.append(
zfitter.ZFitter(zfindobjs[i].zchi2arr, zfindobjs[i].zbase))
zfitobjs[i].z_refine()
else:
for i in range(len(self.templates)):
zfindobjs.append(
zfinder.ZFinder(fname=self.templates[i],
npoly=self.npoly[i],
npixstep=self.npixstep[i]))
zfindobjs[i].zchi2(specs.flux,
specs.loglambda,
specs.ivar,
npixstep=self.npixstep[i],
plate=plate,
mjd=mjd,
fiberid=fiberid[0],
chi2file=self.chi2file)
zfitobjs.append(
zfitter.ZFitter(zfindobjs[i].zchi2arr, zfindobjs[i].zbase))
zfitobjs[i].z_refine()
# Flags
flags = []
for i in range(len(zfindobjs)):
flags.append(misc.comb_flags(specs, zfindobjs[i], zfitobjs[i]))
# ZPicker
if len(self.templates) == 1:
zpick = zpicker.ZPicker(specs, zfindobjs[0], zfitobjs[0], flags[0])
elif len(self.templates) == 2:
zpick = zpicker.ZPicker(specs, zfindobjs[0], zfitobjs[0], flags[0],
zfindobjs[1], zfitobjs[1], flags[1])
elif len(self.templates) == 3:
zpick = zpicker.ZPicker(specs, zfindobjs[0], zfitobjs[0], flags[0],
zfindobjs[1], zfitobjs[1], flags[1],
zfindobjs[2], zfitobjs[2], flags[2])
elif len(self.templates) == 4:
zpick = zpicker.ZPicker(specs, zfindobjs[0], zfitobjs[0], flags[0],
zfindobjs[1], zfitobjs[1], flags[1],
zfindobjs[2], zfitobjs[2], flags[2],
zfindobjs[3], zfitobjs[3], flags[3])
elif len(self.templates) == 5:
zpick = zpicker.ZPicker(specs, zfindobjs[0], zfitobjs[0], flags[0],
zfindobjs[1], zfitobjs[1], flags[1],
zfindobjs[2], zfitobjs[2], flags[2],
zfindobjs[3], zfitobjs[3], flags[3],
zfindobjs[4], zfitobjs[4], flags[4])
output = None
# Write output
if self.dest is None:
output = io.WriteRedmonster(zpick, overwrite=self.overwrite)
else:
if type(self.dest) is str:
output = io.WriteRedmonster(zpick,
dest=self.dest,
overwrite=self.overwrite)
else:
try:
self.dest = str(self.dest)
output = io.WriteRedmonster(zpick,
dest=self.dest,
overwrite=self.overwrite)
except:
print(
'Could not convert dest to string - writing to default directory and NOT clobbering old files!'
)
output = io.WriteRedmonster(zpick, overwrite=True)
if output:
if len(zpick.fiberid) == 1: output.write_fiberid()
else: output.write_plate()
class VideoWall(QtGui.QWidget):
def __init__(self, app, parent):
super(VideoWall, self).__init__()
self.players = {}
#reading config.ini
self.config = SafeConfigParser()
self.config.read('config.ini')
self.number_of_cameras = self.config.getint('videowall',
'number_of_cameras',
fallback=6)
if self.number_of_cameras > 6:
self.number_of_cameras = 6
self.window_aspect_ratio_coeff = self.config.getfloat(
'videowall', 'window_aspect_ratio_coeff', fallback=1.77777777778)
self.audio_mode = self.config.get('videowall',
'audio_mode',
fallback='hdmi')
self.rtsp_sleep_delay = self.config.getint('videowall',
'rtsp_sleep_delay',
fallback=1)
self.hide_window_title = self.config.getint('videowall',
'hide_window_title',
fallback=0)
if self.hide_window_title == 1:
self.setWindowFlags(QtCore.Qt.FramelessWindowHint)
self.timer = None
self.use_sequence_view = self.config.getint('videowall',
'use_sequence_view',
fallback=0)
if self.use_sequence_view == 1:
self.sequence_view = {}
self.sequence_current_index = 0
sequence = self.config.get('videowall',
'sequence_view',
fallback='1').split(',')
for s_ind in range(0, len(sequence)):
self.sequence_view[s_ind] = int(sequence[s_ind])
self.sequence_view_seconds = self.config.getint(
'videowall', 'sequence_view_seconds', fallback=30000)
#reading cameras settings
self.cameras_settings = {}
for cam_number in range(0, self.number_of_cameras):
self.cameras_settings[cam_number] = {
'preview_url':
(self.config.get('camera-' + str(cam_number + 1),
'preview_url',
fallback='')),
'full_url': (self.config.get('camera-' + str(cam_number + 1),
'full_url',
fallback=''))
}
#setting window title
window_title = self.config.get('videowall',
'window_title',
fallback='Video Wall')
self.setWindowTitle(window_title)
#setting cameras viewports
self.stylesheet = self.config.get('videowall',
'camera_stylesheet',
fallback='background-color: black')
self.viewports = {}
for cam_number in range(0, self.number_of_cameras):
self.viewports[cam_number] = QtGui.QLabel(self)
self.viewports[cam_number].setStyleSheet(self.stylesheet)
self.viewports[cam_number].setObjectName('camera-' +
str(cam_number + 1))
self.viewports[cam_number].setText(str(cam_number + 1))
#self.viewports[cam_number].installEventFilter(self)
app.installEventFilter(self)
def eventFilter(self, source, event):
if event.type() == QtCore.QEvent.MouseButtonPress:
if type(source) == QtGui.QLabel:
if source.objectName() != 'camera-1':
clicked_id = int(source.text()) - 1
self.changeMainCamera(clicked_id)
if event.type() == QtCore.QEvent.KeyPress:
current_player_id = int(self.viewports[0].text()) - 1
next_player_id = 0
if event.key() == 16777236: # ->
next_player_id = current_player_id + 1
if next_player_id > (self.number_of_cameras - 1):
next_player_id = 0
if event.key() == 16777234: # <-
next_player_id = current_player_id - 1
if next_player_id < 0:
next_player_id = self.number_of_cameras - 1
self.changeMainCamera(next_player_id)
return super(VideoWall, self).eventFilter(source, event)
def changeMainCamera(self, new_cam_id):
cam_viewport = None
for viewport_id in range(0, self.number_of_cameras):
if self.viewports[viewport_id].text() == str(new_cam_id + 1):
cam_viewport = self.viewports[viewport_id]
current_player_id = int(self.viewports[0].text()) - 1
current_player_atX = self.players[current_player_id].atX
current_player_atY = self.players[current_player_id].atY
current_player_width = self.players[current_player_id].width
current_player_height = self.players[current_player_id].height
current_player_audio_mode = self.players[current_player_id].audio_mode
clicked_id = new_cam_id
clicked_atX = self.players[clicked_id].atX
clicked_atY = self.players[clicked_id].atY
clicked_width = self.players[clicked_id].width
clicked_height = self.players[clicked_id].height
clicked_audio_mode = self.players[clicked_id].audio_mode
#stop main screen player camera-1
self.players[current_player_id].stop()
self.players[current_player_id].process.stdin.write(b'q')
self.players[current_player_id].process.terminate()
#stop and kill clicked player
self.players[clicked_id].stop()
self.players[clicked_id].process.stdin.write(b'q')
self.players[clicked_id].process.terminate()
QtCore.QThread.sleep(self.rtsp_sleep_delay)
#run preview from main camera-1 screen on this place
cam_viewport.setText(str(current_player_id + 1))
self.players[current_player_id] = Player(
self.cameras_settings[current_player_id]['preview_url'],
clicked_atX, clicked_atY, clicked_width, clicked_height,
clicked_audio_mode, current_player_id)
#run clicked on main screen in HD (full_url)
self.viewports[0].setText(str(clicked_id + 1))
self.players[clicked_id] = Player(
self.cameras_settings[clicked_id]['full_url'], current_player_atX,
current_player_atY, current_player_width, current_player_height,
current_player_audio_mode, clicked_id)
QtCore.QThread.sleep(self.rtsp_sleep_delay)
def redrawViewPorts(self, width, height, width_difference):
sc = 1 / 3
lc = 2 / 3
w_offset = width_difference / 2
for cam_number in range(0, self.number_of_cameras):
if cam_number == 0:
self.viewports[cam_number].resize(width * lc, height * lc)
self.viewports[cam_number].move(w_offset, 0)
else:
if cam_number > 3:
self.viewports[cam_number].resize(width * sc, height * sc)
self.viewports[cam_number].move(
w_offset + ((cam_number - 4) * width * sc),
height * lc)
else:
self.viewports[cam_number].resize(width * sc, height * sc)
self.viewports[cam_number].move(w_offset + (width * lc),
(cam_number - 1) * height *
sc)
def redrawPlayers(self):
for cam_number in range(0, self.number_of_cameras):
pos = self.geometry().topLeft() + self.viewports[cam_number].pos()
player_id = int(self.viewports[cam_number].text()) - 1
quality = 'preview_url'
if cam_number == 0:
quality = 'full_url'
if cam_number in self.players:
self.players[player_id].resize(
pos.x(), pos.y(),
self.viewports[cam_number].geometry().width(),
self.viewports[cam_number].geometry().height())
else:
if len(self.cameras_settings[cam_number][quality]) is not 0:
self.players[cam_number] = Player(
self.cameras_settings[cam_number][quality], pos.x(),
pos.y(), self.viewports[cam_number].geometry().width(),
self.viewports[cam_number].geometry().height(),
self.audio_mode, cam_number)
QtCore.QThread.sleep(self.rtsp_sleep_delay)
#start sequence if needed
if self.timer is None and self.use_sequence_view == 1:
self.timer = QtCore.QTimer()
self.timer.timeout.connect(self.timerEvent)
self.timer.start(self.sequence_view_seconds)
def resizeEvent(self, e):
width = e.size().width()
height = e.size().height()
calc_width = height * self.window_aspect_ratio_coeff
width_difference = 0
if calc_width > width:
height = width / self.window_aspect_ratio_coeff
else:
width_difference = width - calc_width
width = calc_width
self.redrawViewPorts(width, height, width_difference)
self.redrawPlayers()
def moveEvent(self, e):
self.redrawPlayers()
def closeEvent(self, e):
for player in self.players:
if self.players[player].process is not None and self.players[
player].process.poll() is None:
self.players[player].stop()
self.players[player].process.stdin.write(b'q')
self.players[player].process.terminate()
e.accept()
def changeEvent(self, event):
if event.type() == QtCore.QEvent.WindowStateChange:
if self.windowState() & QtCore.Qt.WindowMinimized:
for player in self.players:
self.players[player].hide()
elif event.oldState() & QtCore.Qt.WindowMinimized:
for player in self.players:
self.players[player].unhide()
QtGui.QWidget.changeEvent(self, event)
def timerEvent(self):
self.sequence_current_index = self.sequence_current_index + 1
if self.sequence_current_index > len(self.sequence_view) - 1:
self.sequence_current_index = 0
self.changeMainCamera(self.sequence_view[self.sequence_current_index] -
1)
def loadParams(filepath):
config = SafeConfigParser({'color':'0',
'PreTrainNet':'',
'loopRestartFrequency':'-1'})
config.read(filepath)
params = {}
#device
params['randomSeed'] = config.getint('device', 'randomSeed')
#solver
params['SolverType'] = config.get('solver', 'SolverType')
params['lr'] = config.getfloat('solver', 'lr')
params['momentum'] = config.getfloat('solver', 'momentum')
params['lrDecay'] = config.getfloat('solver', 'lrDecay')
params['batchSize'] = config.getint('solver', 'batchSize')
params['weightDecay'] = config.getfloat('solver', 'weightDecay')
#stopping crteria
params['nMaxEpoch'] = config.getint('stopping', 'nMaxEpoch')
params['nMaxIter'] = config.getint('stopping', 'nMaxIter')
#the SA training would alternating between 'normalBatchLength' iteration of normal training and 'loopBatchLength' of self-augment training
params['normalBatchLength'] = config.getint('loop', 'normalBatchLength')
params['loopStartEpoch'] = config.getint('loop', 'loopStartEpoch')
params['loopStartIteration'] = config.getint('loop', 'loopStartIteration') #add loop after this number of normal training.
params['loopBatchLength'] = config.getint('loop', 'loopBatchLength') #how many mini-batch iteration for ever loop optimize
params['loopRestartFrequency'] = config.getint('loop', 'loopRestartFrequency')
#network structure
params['NetworkType'] = config.get('network', 'NetworkType')
params['Channal'] = config.get('network', 'Channal')
params['BN'] = config.getboolean('network', 'BN')
params['color'] = config.getboolean('network', 'color')
params['PreTrainNet'] = config.get('network', 'PreTrainNet')
#dataset
params['NormalizeInput'] = config.getboolean('dataset', 'NormalizeInput')
params['dataset'] = config.get('dataset', 'dataset')
params['testDataset'] = config.get('dataset', 'testDataset')
params['predictDataset'] = config.get('dataset', 'predictDataset')
#display and testing
params['displayStep'] = config.getint('display', 'displayStep')
params['loopdisplayStep'] = config.getint('display', 'loopdisplayStep')
params['checkPointStepIteration'] = config.getint('display', 'checkPointStepIteration')
params['checkPointStepEpoch'] = config.getint('display', 'checkPointStepEpoch')
return params
def main(argv):
"""
Runs evaluation of a prediction technique on a selected evaluation problem
from a selected dataset. Runs the evaluation multiple times and prints stats
to output. Takes as an argument the file path to a configeration file that
is used to set the parameters of the evaluation.
"""
startTime = time.time()
parser = SafeConfigParser()
parser.read(argv[0])
validConfig = validateConfigFile(parser)
if not validConfig:
print('[ERROR] - Config not valid!')
exit()
seed(parser.getint('evaluation_params', 'seedNo'))
# print('Remove stop words: {} Remove punctuation: {} Lemmatize: {}'.format(rmStopwords, rmPunct, lemmatize))
dictionaryDataPath = parser.get('evaluation_params', 'dictionary')
try:
evaluationData = sl.loadDataFromFile('dictionaryData/' +
dictionaryDataPath)
except IOError as err:
print((dictionaryDataPath +
' can not be found in the dictionaryData directory.'))
exit()
evaluationData = ds.selectPoS(evaluationData,
parser.get('evaluation_params', 'pos'))
evaluationData = ds.removeWordsWithTooFewSenses(
evaluationData, parser.getint('evaluation_params', 'numOfSenses'),
parser.getint('evaluation_params', 'numOfExamp'))
evaluationData = ds.examplesToLowerCase(evaluationData)
evaluationData = ds.tokenizeAndLemmatizeExamples(
evaluationData, parser.getboolean('evaluation_params', 'lemmatize'))
evaluationData = ds.removeStopwordsAndPunct(
evaluationData, parser.getboolean('evaluation_params', 'rmStopwords'),
parser.getboolean('evaluation_params', 'rmPunct'))
num_examples_lt_5 = 0
num_examples_lt_10 = 0
num_examples_lt_20 = 0
num_examples_all = 0
base_path = '/Users/thomas/DevSandbox/EpicDataShelf/tag-lab/sense_alloc'
pos = parser.get('evaluation_params', 'pos')
with open(os.path.join(base_path, dictionaryDataPath.split('.')[0], '{}_lt_5.txt'.format(pos)), 'w', encoding='utf-8') as lt_5_file, \
open(os.path.join(base_path, dictionaryDataPath.split('.')[0], '{}_lt_10.txt'.format(pos)), 'w', encoding='utf-8') as lt_10_file, \
open(os.path.join(base_path, dictionaryDataPath.split('.')[0], '{}_lt_20.txt'.format(pos)), 'w', encoding='utf-8') as lt_20_file, \
open(os.path.join(base_path, dictionaryDataPath.split('.')[0], '{}_all.txt'.format(pos)), 'w', encoding='utf-8') as all_file:
for key, vals in list(evaluationData.items()):
print(['### ITEM: {} ###'.format(key)])
# print('\t{}\n'.format(json.dumps(vals, indent=4)))
for val in vals:
ex_lt_5 = []
ex_lt_10 = []
ex_lt_20 = []
ex_all = []
for ex in val['examples']:
if (ex['sent'].count(' ') < 5):
num_examples_lt_5 += 1
ex_lt_5.append((ex['sent'], val['def']))
if (ex['sent'].count(' ') < 10):
num_examples_lt_10 += 1
ex_lt_10.append((ex['sent'], val['def']))
if (ex['sent'].count(' ') < 20):
# print('{}: {}'.format(ex['sent'].count(' '), ex['sent']))
num_examples_lt_20 += 1
ex_lt_20.append((ex['sent'], val['def']))
num_examples_all += 1
ex_all.append((ex['sent'], val['def']))
for l, f in [(ex_lt_5, lt_5_file), (ex_lt_10, lt_10_file),
(ex_lt_20, lt_20_file), (ex_all, all_file)]:
if (len(l) > 2):
for ex, syn in l:
f.write('{}\t{}\t{}\n'.format(key, ex, syn))
print('---------------------------------------------------')
print(('<=5: {}; <=10: {}; <=20: {}'.format(num_examples_lt_5,
num_examples_lt_10,
num_examples_lt_20)))
"""
pyplot.ylabel('Power Normalized Magnetude [dBFS]')
pyplot.xlabel('Sampling Frequency [Hz]')
return pyplot
######################################################################
# Main script
config = SafeConfigParser()
config.read('config.ini')
data_dir = config.get('Test', 'data_dir')
data_file_name = config.get('Test', 'data_file_name')
power_file_name = config.get('Test', 'power_file_name')
power_plot_name = config.get('Test', 'power_plot_name')
num_samples = config.getint('Test', 'num_samples')
fs = config.getfloat('Test', 'fs') # sampling frequency in Hz
adc_resolution_bits = config.getint('Test', 'adc_resolution_bits')
sys.stdout.write("\nRunning test...\n\n")
# Initialize arrays that will receive data in each loop cycle
fund_power_array = []
fsig_array = []
# Load power and frequency data
data = numpy.loadtxt(data_dir + power_file_name, delimiter=',', skiprows=1)
fsig_array = [d[0] for d in data]
fund_power_array = [d[1] for d in data]
power_array = []
def cli():
parser = argparse.ArgumentParser(
description='Check HTTPs rules for validity')
parser.add_argument(
'checker_config', help='an integer for the accumulator')
parser.add_argument('rule_files', nargs="*", default=[],
help="Specific XML rule files")
parser.add_argument('--json_file', default=None,
help='write results in json file')
args = parser.parse_args()
config = SafeConfigParser()
config.read(args.checker_config)
logfile = config.get("log", "logfile")
loglevel = convertLoglevel(config.get("log", "loglevel"))
if logfile == "-":
logging.basicConfig(stream=sys.stderr, level=loglevel,
format="%(levelname)s %(message)s")
else:
logging.basicConfig(filename=logfile, level=loglevel,
format="%(asctime)s %(levelname)s %(message)s [%(pathname)s:%(lineno)d]")
autoDisable = False
if config.has_option("rulesets", "auto_disable"):
autoDisable = config.getboolean("rulesets", "auto_disable")
# Test rules even if they have default_off=...
includeDefaultOff = False
if config.has_option("rulesets", "include_default_off"):
includeDefaultOff = config.getboolean(
"rulesets", "include_default_off")
ruledir = config.get("rulesets", "rulesdir")
checkCoverage = False
if config.has_option("rulesets", "check_coverage"):
checkCoverage = config.getboolean("rulesets", "check_coverage")
checkTargetValidity = False
if config.has_option("rulesets", "check_target_validity"):
checkTargetValidity = config.getboolean(
"rulesets", "check_target_validity")
checkNonmatchGroups = False
if config.has_option("rulesets", "check_nonmatch_groups"):
checkNonmatchGroups = config.getboolean(
"rulesets", "check_nonmatch_groups")
checkTestFormatting = False
if config.has_option("rulesets", "check_test_formatting"):
checkTestFormatting = config.getboolean(
"rulesets", "check_test_formatting")
certdir = config.get("certificates", "basedir")
if config.has_option("rulesets", "skiplist") and config.has_option("rulesets", "skipfield"):
skiplist = config.get("rulesets", "skiplist")
skipfield = config.get("rulesets", "skipfield")
with open(skiplist) as f:
f.readline()
for line in f:
splitLine = line.split(",")
fileHash = splitLine[0]
if splitLine[int(skipfield)] == "1":
skipdict[binascii.unhexlify(fileHash)] = 1
threadCount = config.getint("http", "threads")
httpEnabled = True
if config.has_option("http", "enabled"):
httpEnabled = config.getboolean("http", "enabled")
metricName = config.get("thresholds", "metric")
thresholdDistance = config.getfloat("thresholds", "max_distance")
metricClass = getMetricClass(metricName)
metric = metricClass()
# Debugging options, graphviz dump
dumpGraphvizTrie = False
if config.has_option("debug", "dump_graphviz_trie"):
dumpGraphvizTrie = config.getboolean("debug", "dump_graphviz_trie")
if dumpGraphvizTrie:
graphvizFile = config.get("debug", "graphviz_file")
exitAfterDump = config.getboolean("debug", "exit_after_dump")
if args.rule_files:
xmlFnames = args.rule_files
else:
xmlFnames = glob.glob(os.path.join(ruledir, "*.xml"))
trie = RuleTrie()
rulesets = []
coverageProblemsExist = False
targetValidityProblemExist = False
nonmatchGroupProblemsExist = False
testFormattingProblemsExist = False
for xmlFname in xmlFnames:
logging.debug("Parsing {}".format(xmlFname))
if skipFile(xmlFname):
logging.debug(
"Skipping rule file '{}', matches skiplist.".format(xmlFname))
continue
ruleset = Ruleset(etree.parse(open(xmlFname, "rb")).getroot(), xmlFname)
if ruleset.defaultOff and not includeDefaultOff:
logging.debug("Skipping rule '{}', reason: {}".format(
ruleset.name, ruleset.defaultOff))
continue
# Check whether ruleset coverage by tests was sufficient.
if checkCoverage:
logging.debug("Checking coverage for '{}'.".format(ruleset.name))
problems = ruleset.getCoverageProblems()
for problem in problems:
coverageProblemsExist = True
logging.error(problem)
if checkTargetValidity:
logging.debug("Checking target validity for '{}'.".format(ruleset.name))
problems = ruleset.getTargetValidityProblems()
for problem in problems:
targetValidityProblemExist = True
logging.error(problem)
if checkNonmatchGroups:
logging.debug("Checking non-match groups for '{}'.".format(ruleset.name))
problems = ruleset.getNonmatchGroupProblems()
for problem in problems:
nonmatchGroupProblemsExist = True
logging.error(problem)
if checkTestFormatting:
logging.debug("Checking test formatting for '{}'.".format(ruleset.name))
problems = ruleset.getTestFormattingProblems()
for problem in problems:
testFormattingProblemsExist = True
logging.error(problem)
trie.addRuleset(ruleset)
rulesets.append(ruleset)
# Trie is built now, dump it if it's set in config
if dumpGraphvizTrie:
logging.debug("Dumping graphviz ruleset trie")
graph = trie.generateGraphizGraph()
if graphvizFile == "-":
graph.dot()
else:
with open(graphvizFile, "w") as gvFd:
graph.dot(gvFd)
if exitAfterDump:
sys.exit(0)
fetchOptions = http_client.FetchOptions(config)
fetchers = list()
# Ensure "default" is in the platform dirs
if not os.path.isdir(os.path.join(certdir, "default")):
raise RuntimeError(
"Platform 'default' is missing from certificate directories")
platforms = http_client.CertificatePlatforms(
os.path.join(certdir, "default"))
fetchers.append(http_client.HTTPFetcher(
"default", platforms, fetchOptions, trie))
# fetches pages with unrewritten URLs
fetcherPlain = http_client.HTTPFetcher("default", platforms, fetchOptions)
urlList = []
if config.has_option("http", "url_list"):
with open(config.get("http", "url_list")) as urlFile:
urlList = [line.rstrip() for line in urlFile.readlines()]
if httpEnabled:
taskQueue = queue.Queue(1000)
resQueue = queue.Queue()
startTime = time.time()
testedUrlPairCount = 0
config.getboolean("debug", "exit_after_dump")
for i in range(threadCount):
t = UrlComparisonThread(
taskQueue, metric, thresholdDistance, autoDisable, resQueue)
t.setDaemon(True)
t.start()
# set of main pages to test
mainPages = set(urlList)
# If list of URLs to test/scan was not defined, use the test URL extraction
# methods built into the Ruleset implementation.
if not urlList:
for ruleset in rulesets:
if ruleset.platform != "default" and os.path.isdir(os.path.join(certdir, ruleset.platform)):
theseFetchers = copy.deepcopy(fetchers)
platforms.addPlatform(ruleset.platform, os.path.join(certdir, ruleset.platform))
theseFetchers.append(http_client.HTTPFetcher(
ruleset.platform, platforms, fetchOptions, trie))
else:
theseFetchers = fetchers
testUrls = []
for test in ruleset.tests:
if not ruleset.excludes(test.url):
testedUrlPairCount += 1
testUrls.append(test.url)
else:
# TODO: We should fetch the non-rewritten exclusion URLs to make
# sure they still exist.
logging.debug("Skipping excluded URL {}".format(test.url))
task = ComparisonTask(testUrls, fetcherPlain, theseFetchers, ruleset)
taskQueue.put(task)
taskQueue.join()
logging.info("Finished in {:.2f} seconds. Loaded rulesets: {}, URL pairs: {}.".format(
time.time() - startTime, len(xmlFnames), testedUrlPairCount))
if args.json_file:
json_output(resQueue, args.json_file, problems)
if checkCoverage:
if coverageProblemsExist:
return 1 # exit with error code
if checkTargetValidity:
if targetValidityProblemExist:
return 1 # exit with error code
if checkNonmatchGroups:
if nonmatchGroupProblemsExist:
return 1 # exit with error code
if checkTestFormatting:
if testFormattingProblemsExist:
return 1 # exit with error code
return 0 # exit with success
def run():
config = SafeConfigParser()
config.read("config/defaults.cfg")
config.read("~/.config/mcdu.cfg")
config.read("config/mcdu.cfg")
sim = config.get("General", "sim")
if sim == "fsx":
from mcdu.fsx import FSXReceiver
receiver = FSXReceiver()
elif sim == "xplane":
from mcdu.xplane import XPlaneReceiver
receiver = XPlaneReceiver()
else:
print("no simulator set")
return 1
db = Database()
receiver.start()
api = ACARS_API(config.get("ACARS", "logon"))
acars = ACARS(api)
atc = ATC(api)
data = DATA(api)
init = INIT(api)
fplan = FPLAN(api)
perf = PERF(api)
mcdu = MCDU()
mcdu.subsystem_register(acars)
mcdu.subsystem_register(atc)
mcdu.subsystem_register(data)
mcdu.subsystem_register(init)
mcdu.subsystem_register(fplan)
mcdu.subsystem_register(perf)
mcdu.database_register(db)
mcdu.menu()
application = myDisplay()
port = config.getint("General", "port")
application.initialize(mcdu)
application.open()
try:
print("running on port %i" % port)
# Call my application here
application.mainloop()
# receiver.run()
except KeyboardInterrupt:
print("quitting...")
except Exception:
import traceback
traceback.print_exc()
print("quitting...")
finally:
receiver.stop()
acars.stop()
atc.stop()
return 0
assert np.all(mgrid == mexp_edges)
assert np.all(z_edges == zgrid)
saveId = expName + "_" + gridName + "_" + calName + "_v" + version
from orphics.io import dict_from_section, list_from_config
constDict = dict_from_section(Config, 'constants')
clusterDict = dict_from_section(Config, 'cluster_params')
beam = list_from_config(Config, expName, 'beams')
noise = list_from_config(Config, expName, 'noises')
freq = list_from_config(Config, expName, 'freqs')
lknee = list_from_config(Config, expName, 'lknee')[0]
alpha = list_from_config(Config, expName, 'alpha')[0]
fsky = Config.getfloat(expName, 'fsky')
try:
v3mode = Config.getint(expName, 'V3mode')
except:
v3mode = -1
clttfile = Config.get('general', 'clttfile')
# get s/n q-bins
mus = list_from_config(Config, 'general', 'mubins')
mubin_edges = np.linspace(mus[0], mus[1], int(mus[2]) + 1)
print(mubin_edges)
massMultiplier = Config.getfloat('general', 'mass_calib_factor')
YWLcorrflag = Config.getfloat('general', 'ywl_corr_flag')
if debug: print("Finished rank 0 work.")
if sys.version_info.major == 3:
from configparser import SafeConfigParser
else:
from ConfigParser import SafeConfigParser
from pathlib2 import Path
import pygeoutil.util as util
# Parse config file
parser = SafeConfigParser()
parser.read('config_rotations.txt')
# Get directory path (3 levels up is the parent directory)
dir_prj = str(Path(__file__).parents[3])
FAO_START_YR = parser.getint('PARAMETERS',
'FAO_START_YR') # Starting year of FAO data
FAO_END_YR = parser.getint('PARAMETERS',
'FAO_END_YR') # Ending year of FAO data
TAG = parser.get('PROJECT', 'TAG')
FAO_FILE = parser.get('PROJECT', 'fao_data')
FAO_SHEET = parser.get('PROJECT', 'fao_sheet')
PROJ_NAME = parser.get('PROJECT', 'project_name')
DO_PARALLEL = parser.getboolean('PARAMETERS',
'DO_PARALLEL') # Use multiprocessing or not?
NUM_LATS = 180.0
NUM_LONS = 360.0
PLOT_CNTRS = 10
PLOT_CROPS = 10
DPI = 300
CFT_FRAC_YR = parser.getint('PARAMETERS', 'CFT_FRAC_YR')
GLM_STRT_YR = parser.getint('PARAMETERS', 'GLM_STRT_YR')
def parse_config_file(rcPath, args):
try:
rcConfig = SafeConfigParser()
rv = rcConfig.read(rcPath)
except configparser.ParsingError as err:
sys.stderr.write('RC Parse error: {}\n'.format(err.args[0]))
sys.exit(1) # Abort program. Can't continue with RC error
# OPTIMIZE - LOOP THIS
try:
options['dbpath'] = rcConfig.get('main', 'dbpath')
options['logpath'] = rcConfig.get('main', 'logpath')
options['verbose'] = rcConfig.getint('main', 'verbose')
options['logappend'] = rcConfig.getboolean('main', 'logappend')
options['sizereduce'] = rcConfig.get('main', 'sizereduce')
options['subjectregex'] = rcConfig.get('main', 'subjectregex')
options['srcregex'] = rcConfig.get('main', 'srcregex')
options['destregex'] = rcConfig.get('main', 'destregex')
options['srcdestdelimiter'] = rcConfig.get('main', 'srcdestdelimiter')
options['summarysubject'] = rcConfig.get('main', 'summarysubject')
options['border'] = rcConfig.get('main', 'border')
options['padding'] = rcConfig.get('main', 'padding')
options['intransport'] = rcConfig.get('incoming', 'transport')
options['inserver'] = rcConfig.get('incoming', 'server')
options['inport'] = rcConfig.get('incoming', 'port')
options['inencryption'] = rcConfig.get('incoming', 'encryption')
options['inaccount'] = rcConfig.get('incoming', 'account')
options['inpassword'] = rcConfig.get('incoming', 'password')
options['infolder'] = rcConfig.get('incoming', 'folder')
options['outserver'] = rcConfig.get('outgoing', 'server')
options['outport'] = rcConfig.get('outgoing', 'port')
options['outencryption'] = rcConfig.get('outgoing', 'encryption')
options['outaccount'] = rcConfig.get('outgoing', 'account')
options['outpassword'] = rcConfig.get('outgoing', 'password')
options['outsender'] = rcConfig.get('outgoing', 'sender')
options['outreceiver'] = rcConfig.get('outgoing', 'receiver')
except configparser.NoOptionError as err:
sys.stderr.write('RC Parse error - No Option: {}\n'.format(
err.args[0]))
sys.exit(1) # Abort program. Can't continue with RC error
except configparser.NoSectionError as err:
sys.stderr.write('RC Parse error - No Section: {}\n'.format(
err.args[0]))
sys.exit(1) # Abort program. Can't continue with RC error
# Now, overlay with command line options
# Database Path
if args.dbpath != None: #dbPath specified on command line
options['dbpath'] = '{}/{}'.format(args.dbpath, dbName)
elif options['dbpath'] == '': # No command line & not specified in RC file
options['dbpath'] = '{}/{}'.format(get_script_path(), dbName)
else: # Path specified in rc file. Add dbname for full path
options['dbpath'] = '{}/{}'.format(options['dbpath'], dbName)
# Log file path
if args.logpath != None: #logPath specified on command line
options['logpath'] = '{}/{}'.format(args.logpath, logName)
elif options[
'logpath'] == '': # No command line & not specified in RC file
options['logpath'] = '{}/{}'.format(get_script_path(), logName)
else: # Path specified in rc file. Add dbname for full path
options['logpath'] = '{}/{}'.format(options['logpath'], logName)
options['rcpath'] = rcPath
if args.collect == True:
options['collect'] = True
if args.report == True:
options['report'] = True
if args.verbose != None:
options['verbose'] = args.verbose
if args.append == True:
options['logappend'] = True
if args.initdb == True:
options['initdb'] = True
if args.mega != None:
options['sizereduce'] = args.mega
return
MFD_DATA_DIR = input_dir + os.sep + parser.get('MONFREDA', 'mon_data_dir')
# MIAMI-LU
miami_lu_nc = input_dir + os.sep + parser.get('MIAMI_LU', 'miami_lu_nc')
miami_npp = input_dir + os.sep + parser.get(
'MIAMI_LU', 'miami_npp') # Previous MIAMI-LU NPP estimates
miami_vba = input_dir + os.sep + parser.get(
'MIAMI_LU', 'miami_vba') # Previous MIAMI-LU biomass estimates
# HOTSPOTS
file_hotspots = dir_prj + os.sep + parser.get('HOTSPOTS', 'path_hotspots')
# GCAM Wood harvest file
WOOD_HARVEST = dir_prj + os.sep + parser.get('GCAM', 'WOOD_HARVEST')
FERT_DATA = dir_prj + os.sep + parser.get('GCAM', 'FERT_DATA')
GCAM_START_YR = parser.getint('GCAM', 'GCAM_START_YR')
GCAM_END_YR = parser.getint('GCAM', 'GCAM_END_YR')
GCAM_STEP_YR = parser.getint('GCAM', 'GCAM_STEP_YR')
SKIP_GCAM_COLS = parser.getint('GCAM', 'SKIP_GCAM_COLS')
GCAM_MAPPING = dir_prj + os.sep + parser.get('GCAM', 'GCAM_MAPPING')
# CONSTANTS
FILL_VALUE = 1e20
M2_TO_KM2 = 0.000001
M2_TO_HA = 1e-4
KM2_TO_HA = 100.0
IMG_SIZE = 100.0
BILLION = 1e9
KG_TO_PG = 1e-12
KG_TO_TG = 1e-9
KG_TO_MG = 1e-6
def main():
def ensure_dir(path):
"""Ensure that path is a directory creating it if necessary.
If path already exists and is not a directory, print an error
message and quit with sys.exit().
Parameters:
path String specifying the path to ensure
Return value:
path
"""
if not os.path.exists(path):
os.makedirs(path)
elif not os.path.isdir(path):
print("error: '%s' is not a directory" % path)
sys.exit(1)
return path
def config_default(config, section, option, default):
"""Set default values for options that do not have a value."""
try:
config.get(section, option)
except NoSectionError:
config.add_section(section)
config.set(section, option, default)
except NoOptionError:
config.set(section, option, default)
config_dir = ensure_dir(os.path.expanduser("~/.diceware.py"))
cache_dir = ensure_dir(os.path.join(config_dir, "cache"))
# Parse config file
config_file = os.path.join(config_dir, "config")
config = SafeConfigParser()
config.read(config_file)
config_default(config, "defaults", "lang", "en")
config_default(config, "defaults", "words", "5")
config_default(config, "defaults", "special", "0")
config_default(config, "defaults", "file", "")
config_default(config, "defaults", "separator", " ")
# Sanity checks for config options
if config.get("defaults", "lang") not in WORD_LIST_URLS.keys():
print("error: '%s' is not a valid value for option 'lang'"
% config.get("defaults", "lang"))
sys.exit(1)
try:
config.getint("defaults", "words")
config.getint("defaults", "special")
except ValueError:
print("error: 'words' and 'special' options must have integer values")
sys.exit(1)
# Parse command line arguments
parser = OptionParser()
parser.add_option("-g", "--grid", dest="grid", action="store_true",
help="Instead of a single line, generate NxN grid of "+
"words. This makes eavesdropping harder")
parser.add_option("-n", "--words", dest="words", type="int", metavar="N",
help="generate N words (default: %default)",
default=config.getint("defaults", "words"))
parser.add_option("-s", "--special", dest="special", type="int", metavar="M",
help="insert M special characters (default: %default)",
default=config.getint("defaults", "special"))
parser.add_option("-f", "--file", dest="file", metavar="FILE",
help="override the `lang' option and read the word list " +
"from FILE", default=config.get("defaults", "file"))
parser.add_option("-p", "--separator", dest="separator", type="string", metavar="P",
help="specify the separator between words (default: %default)",
default=config.get("defaults", "separator"))
linguas = sorted(WORD_LIST_URLS.keys())
parser.add_option("-l", "--lang", dest="lang", metavar="LANG",
type="choice", choices=linguas,
help="use the word list for LANG (" + ", ".join(linguas) +
") (default: %default)", default=config.get("defaults", "lang"))
options, args = parser.parse_args()
if args or options.words < 1 or options.special < 0:
parser.print_help()
sys.exit(0)
parser.destroy()
# --file has higher precedence than --lang
if options.file:
try:
fobj = open(options.file)
except IOError:
print("error: unable to open word list file '%s'" % options.file)
sys.exit(1)
try:
word_list = read_word_list(fobj)
except ValueError as e:
print("error: %s" % e)
sys.exit(1)
else:
word_list = get_word_list(cache_dir, options.lang)
if not options.grid:
words, with_specials = generate(word_list, options.words,
options.special)
print("passphrase : %s" % options.separator.join(words))
if options.special > 0:
print("with specials: %s" % options.separator.join(with_specials))
else:
words, length = generate_grid(word_list, options.words,
options.special)
for word_row in words:
print (" ".join([word.ljust(length) for word in word_row]))
class ConfigurationCapsule(object):
"""
Encapsulation class to carry command line args, the logging handler
and config file information through the parse and transport classes.
Also handles reading and and validating the config.ini file.
"""
def __init__(self, options, log, config_path):
self._options = options
self._log = log
self._config = SafeConfigParser(interpolation=EnvInterpolation())
self._config.read(config_path)
# Print active configuration if verbose mode
if options.verbose:
self._print_current_config()
self._validate_config(config_path)
def _validate_config(self, config_path):
"""Validate the config file."""
# makes sure there is a stanza for the transport that
# has been selected
if self.options.transport not in self.config.sections():
msg = '{t} selected as transport, but no [{t}] stanza in config {f}'.format(
t=self.options.transport, f=config_path)
raise TstatConfigException(msg)
# make sure we have the universal bare minimum host and port values
try:
self.get_cfg_val('host')
self.get_cfg_val('port')
except TstatConfigException:
msg = '"port" and "host" are required in [{t}] config stanza'.format(
t=self.options.transport)
raise TstatConfigException(msg)
# is the hostname valid?
if not valid_hostname(self.get_cfg_val('host')):
msg = '{h} is not a valid hostname'.format(h=self.get_cfg_val('host'))
raise TstatConfigException(msg)
# is the port an integer?
try:
self.get_cfg_val('port', as_int=True)
except TstatConfigException:
msg = 'port {0} is not a valid integer'.format(
self._config.get(self.options.transport, 'port'))
raise TstatConfigException(msg)
# pylint: disable=missing-docstring
def get_cfg_val(self, value, as_int=False, as_bool=False):
"""
Return a named value from the config file stanza that matches
the currently selected transport.
"""
try:
if as_int:
return self._config.getint(self.options.transport, value)
elif as_bool:
return self._config.getboolean(self.options.transport, value)
else:
return self._config.get(self.options.transport, value)
except ConfigParser.NoOptionError:
msg = '{0} config value not found'.format(value)
raise TstatConfigException(msg)
except ValueError:
msg = '{0} config value improper type'.format(value)
raise TstatConfigException(msg)
def _print_current_config(self):
for each_section in self._config.sections():
print("[{section}]".format(section=each_section))
for (each_key, each_val) in self._config.items(each_section):
print("{key}={value}".format(key=each_key, value=each_val))
def _config_stanza_to_dict(self, stanza):
opts = dict()
if stanza in self.config.sections() and len(self.config.options(stanza)):
for i in self.config.options(stanza):
opts[i] = self.config.get(stanza, i)
self._log('_config_stanza_to_dict',
'stanza [{s}] to dict: {d}'.format(s=stanza, d=opts))
return opts
def get_ssl_opts(self):
return self._config_stanza_to_dict('ssl_options')
# Some rabbit specific option calls to pass addional kwargs to
# pika methods.
def get_rabbit_queue_opts(self):
return self._config_stanza_to_dict('rabbit_queue_options')
# expose the internals as properties.
@property
def options(self):
return self._options
@property
def log(self):
return self._log
@property
def config(self):
return self._config
from rally_task_analysis import RallyTaskAnalysis
import pika
from syslog import syslog, LOG_ERR, LOG_INFO
from configparser import SafeConfigParser
import subprocess
import threading
import time
import json
syslog(LOG_INFO, 'Starting')
# Config
configparser = SafeConfigParser()
try:
configparser.read('/etc/packer-utils/config.ini')
THREAD_COUNT = configparser.getint('rabbit2packer', 'THREAD_COUNT')
if (THREAD_COUNT < 1):
raise UserWarning(
'A thread count < 1 is defined, no worker threads will run')
PACKER_TEMPLATE_MAP = configparser.get('rabbit2packer',
'PACKER_TEMPLATE_MAP')
LOG_DIR = configparser.get('rabbit2packer', 'LOG_DIR')
BUILD_FILE_DIR = configparser.get('rabbit2packer', 'BUILD_FILE_DIR')
PACKER_AUTH_FILE = configparser.get('rabbit2packer', 'PACKER_AUTH_FILE')
QUEUE = configparser.get('global', 'QUEUE')
IMAGES_CONFIG = configparser.get('rabbit2packer', 'IMAGES_CONFIG')
RABBIT_HOST = configparser.get('global', 'RABBIT_HOST')
RABBIT_PORT = configparser.getint('global', 'RABBIT_PORT')
RABBIT_USER = configparser.get('global', 'RABBIT_USER')
RABBIT_PW = configparser.get('global', 'RABBIT_PW')
except Exception as e:
class ChannelParameters(object):
"""
Loads device channel settings and parameters from an INI file.
"""
def __init__(self, ini_file):
self.log = logging.getLogger(".".join([__name__, self.__class__.__name__]))
self._ini_filename = find_file(ini_file)
self.conf = None
self._control_channels = None
self._monitoring_channels = None
def load_ini(self):
"""
Loads and parses the data from INI file. The data is stored internally in the object and can be retrieved
through the `self.control_channels` and `self.monitoring_channels` properties.
"""
self._control_channels = None
self._monitoring_channels = None
self.conf = SafeConfigParser(dict_type=OrderedDict)
self.conf.read(self._ini_filename)
self.log.debug("Read Board Parameters INI file %s:", self._ini_filename)
self.log.debug(" sections: %s", self.conf.sections())
@staticmethod
def _get_channel_number(section_name):
match = re.match(r'^.*_channel<(\d{4})>$', section_name)
if match:
result = int(match.group(1))
return result
else:
raise_with_traceback(RuntimeError("Unable to detect channel name from \"%s\""%section_name))
def _get_channel_matching(self, regexp):
sections_matching = []
for section in self.conf.sections():
if regexp.match(section):
sections_matching.append(section)
return sections_matching
def _get_channel_name_by_address(self, uart_address, channels):
for ch in channels:
if ch.UART_address == uart_address:
return ch.Channel_name
def _get_channel_name_by_index(self, index, channels):
for ch in channels:
if ch.channel_index == index:
name = ch.Channel_name
if name is None or len(name) == 0:
name = ch.ini_section
return name
def _get_channel_name_by_id_and_board_type(self, channel_id, board_type, channels):
for ch in channels:
if ch.Channel_ID == channel_id and BoardTypes(ch.Board_type) == board_type:
name = ch.Channel_name
if name is None or len(name) == 0:
name = ch.ini_section
return name
def _get_channel_by_address(self, uart_address, channels):
for ch in channels:
if ch.UART_address == uart_address:
return ch
def monitoring_channel_name_by_index(self, index):
return self._get_channel_name_by_index(index, self.monitoring_channels)
def monitoring_channel_name_by_id_and_board_type(self, channel_id, board_type):
return self._get_channel_name_by_id_and_board_type(channel_id, board_type, self.monitoring_channels)
def control_channel_name_by_index(self, index):
return self._get_channel_name_by_index(index, self.control_channels)
def monitoring_channel_name(self, uart_address):
return self._get_channel_name_by_address(uart_address, self.monitoring_channels)
def control_channel_name(self, uart_address):
return self._get_channel_name_by_address(uart_address, self.control_channels)
def control_channel_by_address(self, uart_address):
return self._get_channel_by_address(uart_address, self.control_channels)
def monitoring_channel_by_address(self, uart_address):
return self._get_channel_by_address(uart_address, self.monitoring_channels)
def control_channels_by_name(self, name):
return self._get_channels_by_name(self.control_channels, name)
def monitoring_channels_by_name(self, name):
return self._get_channels_by_name(self.monitoring_channels, name)
@property
def control_channels(self):
"""
List of `ControlChannelIniParameters`
"""
if self._control_channels:
return self._control_channels
self._control_channels = self._get_channels(ControlChannelIniParameters)
return self._control_channels
@property
def monitoring_channels(self):
"""
List of `MonitoringChannelIniParameters`
"""
if self._monitoring_channels:
return self._monitoring_channels
self._monitoring_channels = self._get_channels(MonitoringChannelIniParameters)
return self._monitoring_channels
def _get_channels(self, channel_class):
"""
Loop through all channel sections matching a certain type `channel_class` and parse the parameters of each
section into new `channel_class` instance objects.
:param channel_class: an :class:`IniSectionParameters` derivative class
:return: a list of instances of the channel_class :obj:`IniSectionParameters` derivative.
:rtype list:
"""
channels = []
sections = self._get_channel_matching(channel_class.section_regexp)
for section in sections:
channel_number = self._get_channel_number(section)
channel = channel_class(channel_number)
channel.ini_section = section
for param in channel.parameters():
parameter_type = channel.get_type(param)
if parameter_type == int:
value = self.conf.getint(section, param)
elif parameter_type == str:
value = self.conf.get(section, param)
value = str(value.strip("\"")) # Get rid of any double quotes from the ini file
elif parameter_type == bool:
str_value = self.conf.get(section, param)
if str_value.lower() in positive_configuration:
value = True
elif str_value.lower() in negative_configuration:
value = False
else:
raise_with_traceback(TypeError("Unsupported boolean: %s = \"%s\""%(param, str_value)))
else:
raise_with_traceback(TypeError("Unsupported parameter type %s"%str(parameter_type)))
channel.__setattr__(param, value)
self.log.debug("Appending channel: %s", channel)
channels.append(channel)
return channels
@staticmethod
def _get_channels_by_name(channel_list, name):
result = []
for ch in channel_list:
if re.match(name, ch.Channel_name):
result.append(ch)
if len(result) == 1:
result = result[0]
return result
def __str__(self):
channels = "Control channels: "
if self._control_channels:
channels += str(len(self._control_channels))
else:
channels += "[]"
channels += " Monitoring channels: "
if self._monitoring_channels:
channels += str(len(self._monitoring_channels))
else:
channels += "[]"
s = "<%s: inifile: %s %s>"%(self.__class__.__name__, self._ini_filename, channels)
return s
def __repr__(self):
return self.__str__()
import os
import sys
import numpy as np
import time
from shutil import copyfile
from configparser import SafeConfigParser
timestamp = str(time.time())
iniFile = "output/submission"+timestamp+".ini"
copyfile("input/submission.ini",iniFile)
Config = SafeConfigParser()
Config.optionxform=str
Config.read(iniFile)
numJobs = Config.getint('jobs','numJobs')
numClusters = Config.getint('jobs','numClusters')
snapRange = [int(x) for x in Config.get('jobs','snapRange').split(',')]
saveName = Config.get('jobs','saveName')
for snap in range(snapRange[0],snapRange[1]):
for i in range(numJobs):
cmd = "quick_wq.sh python bin/lensRecon.py "+str(i)+" "+str(numClusters)+" "+str(snap)+" "+saveName+" & "
print(cmd)
os.system(cmd)
time.sleep(0.5)
class TestPlanParser(object):
"""WRTM Test Plan Parser
Accepts INI style files describing tests in separate sections, with each test
described in a separate line of a 'plan' option of a section. Each such line
has a format of:
test-id interface-identifier test-length address/offset mask
Address/offset can either be an immediate value or a difference from the last
returned value (in the form of +x or -x).
Additionally the 'main' section specifies the address of the device under test
('dut') and the address of the N2X probe ('n2x'). Comments are allowed in separate
lines starting with a # (hashtag) or a ; (semicolon).
"""
def __init__(self):
self.parser = SafeConfigParser()
self.loaded = False
self.loadedPath = None
self.testTypes = {}
def load(self, path):
if len(self.parser.read(path)) < 1:
raise RuntimeError("Unable to load the specified file.")
self.loaded = True
self.loadedPath = path
def loadTestTypes(self, fileName):
testsFile = open(fileName)
for testType in testsFile:
splitarray = testType.split(' ')
testId = int(splitarray[0])
testContinuous = int(splitarray[1])
testDescription = ' '.join(splitarray[2:])
self.testTypes[testId] = (testContinuous, testDescription)
def sections(self):
return self.parser.sections()
def getListOfTests(self):
if self.loaded:
return self.parser.sections()
return None
def getNumberOfTestCases(self, name):
if self.parser.has_option(name, 'loop'):
loops = self.parser.getint(name, 'loop')
else:
loops = 1
return len(self.parser[name]['plan'].split('\n')[1:]) * loops
def getTestGenerator(self, name=None):
if self.parser.has_option(name, 'loop'):
loops = self.parser.getint(name, 'loop')
else:
loops = 1
if self.parser.has_option(name, 'offsetStep'):
offsetStep = self.parser.getint(name, 'offsetStep')
else:
offsetStep = 0
tests = self.parser[name]['plan'].split('\n')[1:]
offset = 0
for it in range(0, loops):
for i in range(0, len(tests)):
testTuple = tests[i].split(' ')
if testTuple[3][0] == '+' or testTuple[3][0] == '-':
offset += int(testTuple[3][0])
else:
offset = int(testTuple[3][0]) + it * offsetStep
testTuple[3] = offset
yield self.parseTestLine(i+1, testTuple)
return None
def parseTestLine(self, testIter, testTuple):
return [testIter,
int(testTuple[0]), # testId
testTuple[1], # interfaceName
int(testTuple[2]), # testDuration
int(testTuple[3]), # address/offset
int(testTuple[4])] # mask
'Files and Folders will be visible from root folder: "{}"'.format(
nwd))
chdir(
nwd
) # If files are to be served, change the working directory to the document root folder
meters = dict()
LoadSettings() # Initial load of definition file
Shutdown = Event() # The flag which will shutdown the auto-updating thread
AutoPoll = Thread(
target=RegularUpdate,
args=(meters, config.getfloat('DEFAULT', 'autopollsec'), Shutdown),
name='AutoPollThread',
daemon=True) # Daemon Thread to auto-update certain items
AutoPoll.start()
httpd = ThreadingTCPServer(
(config.get('DEFAULT', 'httphost'), config.getint(
'DEFAULT', 'httpport')), CustomHandler) # Start the HTTP Server
print('Server Running "{}:{}"'.format(config.get('DEFAULT', 'httphost'),
config.get('DEFAULT', 'httpport')))
print('To shut down, visit "/command?{}"'.format(
config.get('DEFAULT', 'shutdowncmd')))
signal.signal(signal.SIGTERM, Killer)
try:
httpd.serve_forever()
except KeyboardInterrupt: # Allow Ctrl+C locally to close it gracefully
Killer(0, None) # Envoke the signal handler
httpd.server_close() # Finally close everything off
chdir(
cwd
) # Change the working directory back to what it was when it started.
raise SystemExit # Ensure explicit termination at this point
from http.server import HTTPServer, CGIHTTPRequestHandler
from configparser import SafeConfigParser
if __name__ == '__main__':
config = SafeConfigParser()
config.read("server.ini")
server_address = (config.get("server","url"),config.getint("server","port"))
httpd = HTTPServer(server_address,CGIHTTPRequestHandler)
httpd.serve_forever()
pyplot.title('Power delivered to ADC')
pyplot.ylabel('Power Normalized Magnetude [dBFS]')
pyplot.xlabel('Sampling Frequency [Hz]')
return pyplot
######################################################################
# Main script
config = SafeConfigParser()
config.read('config.ini')
data_dir = config.get('Test','data_dir')
data_file_name = config.get('Test','data_file_name')
power_file_name = config.get('Test','power_file_name')
power_plot_name = config.get('Test','power_plot_name')
num_samples = config.getint('Test','num_samples')
fs = config.getfloat('Test','fs') # sampling frequency in Hz
adc_resolution_bits = config.getint('Test','adc_resolution_bits')
sys.stdout.write("\nRunning test...\n\n")
# Initialize arrays that will receive data in each loop cycle
fund_power_array = []
fsig_array = []
# Load power and frequency data
data = numpy.loadtxt(data_dir + power_file_name, delimiter = ',', skiprows = 1)
fsig_array = [d[0] for d in data]
fund_power_array = [d[1] for d in data]
class GlobalSettings(GObject.Object):
"""
Pitivi app settings.
The settings object loads settings from different sources, currently:
- the local configuration file,
- environment variables.
Modules declare which settings they wish to access by calling the
addConfigOption() class method during initialization.
@cvar options: A dictionnary of available settings.
@cvar environment: A list of the controlled environment variables.
"""
options = {}
environment = set()
defaults = {}
__gsignals__ = {}
def __init__(self, **unused_kwargs):
GObject.Object.__init__(self)
self._config = SafeConfigParser()
self._readSettingsFromConfigurationFile()
self._readSettingsFromEnvironmentVariables()
def _readSettingsFromConfigurationFile(self):
"""
Read the configuration from the user configuration file.
"""
try:
conf_file_path = os.path.join(xdg_config_home(), "pitivi.conf")
self._config.read(conf_file_path)
except UnicodeDecodeError:
unicode_error_dialog()
return
except ParsingError:
return
for (section, attrname, typ, key, env, value) in self.iterAllOptions():
if not self._config.has_section(section):
continue
if key and self._config.has_option(section, key):
if typ == int or typ == int:
try:
value = self._config.getint(section, key)
except ValueError:
# In previous configurations we incorrectly stored
# ints using float values.
value = int(self._config.getfloat(section, key))
elif typ == float:
value = self._config.getfloat(section, key)
elif typ == bool:
value = self._config.getboolean(section, key)
else:
value = self._config.get(section, key)
setattr(self, attrname, value)
@classmethod
def readSettingSectionFromFile(self, cls, section):
"""
Force reading a particular section of the settings file.
Use this if you dynamically determine settings sections/keys at runtime
(like in tabsmanager.py). Otherwise, the settings file would be read
only once (at the initialization phase of your module) and your config
sections would never be read, and thus values would be reset to defaults
on every startup because GlobalSettings would think they don't exist.
"""
if cls._config.has_section(section):
for option in cls._config.options(section):
# We don't know the value type in advance, just try them all.
try:
value = cls._config.getfloat(section, option)
except:
try:
value = cls._config.getint(section, option)
except:
try:
value = cls._config.getboolean(section, option)
except:
value = cls._config.get(section, option)
setattr(cls, section + option, value)
def _readSettingsFromEnvironmentVariables(self):
"""
Override options values using their registered environment variables.
"""
for section, attrname, typ, key, env, value in self.iterAllOptions():
if not env:
# This option does not have an environment variable name.
continue
var = get_env_by_type(typ, env)
if var is not None:
setattr(self, attrname, var)
def _writeSettingsToConfigurationFile(self):
conf_file_path = os.path.join(xdg_config_home(), "pitivi.conf")
for (section, attrname, typ, key, env_var,
value) in self.iterAllOptions():
if not self._config.has_section(section):
self._config.add_section(section)
if key:
if value is not None:
self._config.set(section, key, str(value))
else:
self._config.remove_option(section, key)
try:
file = open(conf_file_path, 'w')
except IOError as OSError:
return
self._config.write(file)
file.close()
def storeSettings(self):
"""
Write settings to the user's local configuration file. Note that only
those settings which were added with a section and a key value are
stored.
"""
self._writeSettingsToConfigurationFile()
def iterAllOptions(self):
"""
Iterate over all registered options
@return: an iterator which yields a tuple of (attrname, type, key,
environment, value) for each option.
"""
for section, options in list(self.options.items()):
for attrname, (typ, key, environment) in list(options.items()):
yield section, attrname, typ, key, environment, getattr(
self, attrname)
def isDefault(self, attrname):
return getattr(self, attrname) == self.defaults[attrname]
def setDefault(self, attrname):
setattr(self, attrname, self.defaults[attrname])
@classmethod
def addConfigOption(
cls,
attrname,
type_=None,
section=None,
key=None,
environment=None,
default=None,
notify=False,
):
"""
Add a configuration option.
This function should be called during module initialization, before
the config file is actually read. By default, only options registered
beforehand will be loaded.
See mainwindow.py and medialibrary.py for examples of usage.
If you want to add configuration options after initialization,
use the readSettingSectionFromFile method to force reading later on.
See tabsmanager.py for an example of such a scenario.
@param attrname: the attribute of this class which represents the option
@type attrname: C{str}
@param type_: type of the attribute. Unnecessary if default is given.
@type type_: a builtin or class
@param section: The section of the config file under which this option is
saved. This section must have been added with addConfigSection(). Not
necessary if key is not given.
@param key: the key under which this option is to be saved. Can be none if
this option should not be saved.
@type key: C{str}
@param notify: whether or not this attribute should emit notification
signals when modified (default is False).
@type notify: C{boolean}
"""
if section and section not in cls.options:
raise ConfigError("You must add the section \"%s\" first." %
section)
if key and not section:
raise ConfigError("You must specify a section for key \"%s\"" %
key)
if section and key in cls.options[section]:
raise ConfigError("Option \"%s\" is already in use.")
if hasattr(cls, attrname):
raise ConfigError("Settings attribute \"%s\" is already in use.")
if environment and environment in cls.environment:
raise ConfigError("Settings environment varaible \"%s\" is"
"already in use.")
if not type_ and default is None:
raise ConfigError("Settings attribute \"%s\" has must have a"
" type or a default." % attrname)
if not type_:
type_ = type(default)
if notify:
notification = Notification(attrname)
setattr(cls, attrname, notification)
setattr(cls, "_" + attrname, default)
GObject.signal_new(notification.signame, cls,
GObject.SIGNAL_RUN_LAST, None, ())
else:
setattr(cls, attrname, default)
if section and key:
cls.options[section][attrname] = type_, key, environment
cls.environment.add(environment)
cls.defaults[attrname] = default
@classmethod
def addConfigSection(cls, section):
"""
Add a section to the local config file.
@param section: The section name. This section must not already exist.
@type section: C{str}
"""
if section in cls.options:
raise ConfigError("Duplicate Section \"%s\"." % section)
cls.options[section] = {}
@classmethod
def notifiesConfigOption(cls, attrname):
signal_name = Notification.signalName(attrname)
GObject.signal_lookup(signal_name, cls)
# every 60 minutes, start delay = 30 seconds
CONF_DEFAULTS = {
'update_interval': '60',
'update_startup_delay': '30',
'autocheck_updates': '0',
'update_notify': True,
'update_showicon': True
}
CONFIG = SafeConfigParser(CONF_DEFAULTS)
if not CONFIG.has_section('yumex'):
CONFIG.add_section('yumex')
if os.path.exists(CONF_FILE):
CONFIG.read(CONF_FILE)
TIMER_STARTUP_DELAY = CONFIG.getint('yumex', 'update_startup_delay')
UPDATE_INTERVAL = CONFIG.getint('yumex', 'update_interval')
AUTOCHECK_UPDATE = CONFIG.getboolean('yumex', 'autocheck_updates')
NOTIFY = CONFIG.getboolean('yumex', 'update_notify')
SHOWICON = CONFIG.getboolean('yumex', 'update_showicon')
def check_pid(pid):
""" Check For the existence of a unix pid. """
try:
os.kill(pid, 0)
except OSError:
return False
else:
return True
def open(self):
self.noInput = False
try:
self.file = open("/dev/kbdscan","r")
except:
self.noInput = True # No KbdScan device, so use the regular stdin keyboard
self.kb = KBHit()
try:
from configparser import SafeConfigParser
except ImportError:
from ConfigParser import SafeConfigParser
config = SafeConfigParser()
config.read("config/defaults.cfg")
config.read("~/.config/mcdu.cfg")
config.read("config/mcdu.cfg")
self.keymap = {}
self.keymap[config.getint("KeyMap", "KEY_CLR")] = "CLR"
self.keymap[config.getint("KeyMap", "KEY_DEL")] = "DEL"
self.keymap[config.getint("KeyMap", "KEY_DIR")] = "DIR"
self.keymap[config.getint("KeyMap", "KEY_PROG")] = "PROG"
self.keymap[config.getint("KeyMap", "KEY_PERF")] = "PERF"
self.keymap[config.getint("KeyMap", "KEY_INIT")] = "INIT"
self.keymap[config.getint("KeyMap", "KEY_DATA")] = "DATA"
self.keymap[config.getint("KeyMap", "KEY_F_PLN")] = "F_PLN"
self.keymap[config.getint("KeyMap", "KEY_RAD_NAV")] = "RAD_NAV"
self.keymap[config.getint("KeyMap", "KEY_FUEL_PRED")] = "FUEL_PRED"
self.keymap[config.getint("KeyMap", "KEY_SEC_F_PLN")] = "SEC_F_PLN"
self.keymap[config.getint("KeyMap", "KEY_ATC_COMM")] = "ATC_COMM"
self.keymap[config.getint("KeyMap", "KEY_MENU")] = "MENU"
self.keymap[config.getint("KeyMap", "KEY_DIM")] = "DIM"
self.keymap[config.getint("KeyMap", "KEY_AIRPORT")] = "AIRPORT"
self.keymap[config.getint("KeyMap", "KEY_PAGE_UP")] = "PAGE_UP"
self.keymap[config.getint("KeyMap", "KEY_NEXT_PAGE")] = "NEXT_PAGE"
self.keymap[config.getint("KeyMap", "KEY_PAGE_DN")] = "PAGE_DN"
self.keymap[config.getint("KeyMap", "KEY_LSK1L" )] = "LSK0L"
self.keymap[config.getint("KeyMap", "KEY_LSK2L" )] = "LSK1L"
self.keymap[config.getint("KeyMap", "KEY_LSK3L" )] = "LSK2L"
self.keymap[config.getint("KeyMap", "KEY_LSK4L" )] = "LSK3L"
self.keymap[config.getint("KeyMap", "KEY_LSK5L" )] = "LSK4L"
self.keymap[config.getint("KeyMap", "KEY_LSK6L" )] = "LSK5L"
self.keymap[config.getint("KeyMap", "KEY_LSK1R" )] = "LSK0R"
self.keymap[config.getint("KeyMap", "KEY_LSK2R" )] = "LSK1R"
self.keymap[config.getint("KeyMap", "KEY_LSK3R" )] = "LSK2R"
self.keymap[config.getint("KeyMap", "KEY_LSK4R" )] = "LSK3R"
self.keymap[config.getint("KeyMap", "KEY_LSK5R" )] = "LSK4R"
self.keymap[config.getint("KeyMap", "KEY_LSK6R" )] = "LSK5R"
from urllib.error import URLError, HTTPError
from configparser import SafeConfigParser
syslog(LOG_INFO, 'Starting')
# Config
configparser = SafeConfigParser()
try:
configparser.read('/etc/packer-utils/config.ini')
PROFILE_INFO_URL = configparser.get('cdb2rabbit', 'PROFILE_INFO_URL')
PROFILE_DIR_URL = configparser.get('cdb2rabbit', 'PROFILE_DIR_URL')
PROFILE_MATCH = configparser.get('cdb2rabbit', 'PROFILE_MATCH')
CACHE_DIR = configparser.get('cdb2rabbit', 'CACHE_DIR')
QUEUE = configparser.get('global', 'QUEUE')
RABBIT_HOST = configparser.get('global', 'RABBIT_HOST')
RABBIT_PORT = configparser.getint('global', 'RABBIT_PORT')
RABBIT_USER = configparser.get('global', 'RABBIT_USER')
RABBIT_PW = configparser.get('global', 'RABBIT_PW')
except Exception as e:
syslog(LOG_ERR, 'Unable to read from config file')
syslog(LOG_ERR, repr(e))
sys.exit(1)
def updateCachedFile(file_name, contents):
try:
with open(file_name, "wt") as file:
file.write(contents)
except IOError as e:
syslog(LOG_ERR, "Unable to write profile info to file %s" % file_name)
syslog(LOG_ERR, repr(e))
def run(ini_file='TOPKAPI.ini',
verbose=False, quiet=False,
parallel_exec=True, nworkers=int(mp.cpu_count()-1)):
"""Run the model.
Parameters
----------
ini_file : str
The name of the PyTOPKAPI initialization file. This file describes the
locations of the parameter files and model setup options. Default is to
use a file named `TOPKAPI.ini` in the current directory.
verbose : bool
Prints runtime information [default False - don't display runtime
info]. Is independent of the `quiet` keyword argument.
quiet : bool
Toggles whether to display an informational banner at runtime [default
False - display banner]. Is independent of the `verbose` keyword
argument.
nworkers : int
Number of worker processes to spawn for solving each cell's time-series
in parallel. Default is one fewer than CPU count reported by
multiprocessing.
"""
##================================##
## Read the input file (*.ini) ##
##================================##
config = SafeConfigParser()
config.read(ini_file)
##~~~~~~ Numerical_options ~~~~~~##
solve_s = config.getint('numerical_options', 'solve_s')
solve_o = config.getint('numerical_options', 'solve_o')
solve_c = config.getint('numerical_options', 'solve_c')
##~~~~~~~~~~~ input files ~~~~~~~~~~~##
#Param
file_global_param = config.get('input_files', 'file_global_param')
file_cell_param = config.get('input_files', 'file_cell_param')
#Rain
file_rain = config.get('input_files', 'file_rain')
#ETP
file_ET = config.get('input_files', 'file_ET')
#~~~~~~~~~~~ Group (simulated event) ~~~~~~~~~~~##
group_name = config.get('groups', 'group_name')
##~~~~~~ Calibration ~~~~~~##
fac_L = config.getfloat('calib_params', 'fac_L')
fac_Ks = config.getfloat('calib_params', 'fac_Ks')
fac_n_o = config.getfloat('calib_params', 'fac_n_o')
fac_n_c = config.getfloat('calib_params', 'fac_n_c')
##~~~~~~ External flows ~~~~~~##
external_flow = config.getboolean('external_flow', 'external_flow')
if external_flow:
file_Qexternal_flow = config.get('external_flow',
'file_Qexternal_flow')
Xexternal_flow = config.getfloat('external_flow', 'Xexternal_flow')
Yexternal_flow = config.getfloat('external_flow', 'Yexternal_flow')
##~~~~~~~~~~~ output files ~~~~~~~~~~##
file_out = config.get('output_files', 'file_out')
ut.check_file_exist(file_out) #create path_out if it doesn't exist
if os.path.exists(file_out):
first_run = False
else:
first_run = True
append_output = config.getboolean('output_files', 'append_output')
if append_output is True:
fmode = 'a'
else:
fmode = 'w'
##============================##
## Read the forcing data ##
##============================##
if verbose:
print('Read the forcing data')
#~~~~Rainfall
h5_rain = h5py.File(file_rain)
dset_name = '/{}/rainfall'.format(group_name)
rainfall_forcing = h5_rain[dset_name][...]
h5_rain.close()
#~~~~ETr - Reference crop ET
h5_ET = h5py.File(file_ET)
dset_name = '/{}/ETr'.format(group_name)
ETr_forcing = h5_ET[dset_name][...]
#~~~~ETo - Open water potential evap.
dset_name = '/{}/ETo'.format(group_name)
ET0_forcing = h5_ET[dset_name][...]
h5_ET.close()
#~~~~external_flow flows
if external_flow:
external_flow_records = np.loadtxt(file_Qexternal_flow)[:, 5]
else:
external_flow_records = None
##============================##
## Pretreatment of input data ##
##============================##
if verbose:
print('Pretreatment of input data')
#~~~~Read Global parameters file
X, Dt, alpha_s, \
alpha_o, alpha_c, \
A_thres, W_min, W_max = pm.read_global_parameters(file_global_param)
#~~~~Read Cell parameters file
ar_cell_label, ar_coorx, \
ar_coory, channel_flag, \
Xc, ar_dam, \
ar_tan_beta, ar_tan_beta_channel, \
ar_L, Ks, \
ar_theta_r, ar_theta_s, \
ar_n_o, ar_n_c, \
ar_cell_down, ar_pVs_t0, \
ar_Vo_t0, ar_Qc_t0, \
Kc, psi_b, lamda = pm.read_cell_parameters(file_cell_param)
#~~~~Number of cell in the catchment
nb_cell = len(ar_cell_label)
#~~~~Computation of cell order
node_hierarchy = pm.compute_node_hierarchy(ar_cell_label, ar_cell_down)
ar_label_sort = pm.sort_cell(ar_cell_label, ar_cell_down)
#~~~~Computation of upcells
li_cell_up = pm.direct_up_cell(ar_cell_label, ar_cell_down, ar_label_sort)
#~~~~Computation of drained area
ar_A_drained = pm.drained_area(ar_label_sort, li_cell_up, X)
#~~~~Apply calibration factors to the parameter values
ar_L = ar_L*fac_L
Ks = Ks*fac_Ks
ar_n_o = ar_n_o*fac_n_o
ar_n_c = ar_n_c*fac_n_c
if verbose:
print('Max L=', max(ar_L))
print('Max Ks=', max(Ks))
print('Max n_o=', max(ar_n_o))
print('Max n_c=', max(ar_n_c))
#~~~~Computation of model parameters from physical parameters
Vsm, b_s, b_o, \
W, b_c = pm.compute_cell_param(X, Xc, Dt, alpha_s,
alpha_o, alpha_c, nb_cell,
A_thres, W_max, W_min,
channel_flag, ar_tan_beta,
ar_tan_beta_channel, ar_L,
Ks, ar_theta_r, ar_theta_s,
ar_n_o, ar_n_c, ar_A_drained)
#~~~~Look for the cell of external_flow tunnel
if external_flow:
cell_external_flow = ut.find_cell_coordinates(ar_cell_label,
Xexternal_flow,
Yexternal_flow,
ar_coorx,
ar_coory,
channel_flag)
if verbose:
print('external flows will be taken into account for cell no',\
cell_external_flow, ' coordinates ('\
,Xexternal_flow,',',Yexternal_flow,')')
else:
cell_external_flow = None
#~~~~Number of simulation time steps
nb_time_step = rainfall_forcing.shape[0]
##=============================##
## Variable array definition ##
##=============================##
## Initialisation of the reservoirs
#Matrix of soil,overland and channel store at the begining of the time step
if append_output and not first_run:
if verbose:
print('Initialize from simulation file')
h5file_in = h5py.File(file_out)
Vs_t0 = h5file_in['/Soil/V_s'][-1, :]
Vc_t0 = h5file_in['/Channel/V_c'][-1, :]
Vo_t0 = h5file_in['/Overland/V_o'][-1, :]
h5file_in.close()
else:
if verbose:
print('Initialize from parameters')
Vs_t0 = fl.initial_volume_soil(ar_pVs_t0, Vsm)
Vo_t0 = ar_Vo_t0
Vc_t0 = fl.initial_volume_channel(ar_Qc_t0, W, X, ar_n_c)
##=============================##
## HDF5 output file definition ##
##=============================##
h5file, dset_Vs, dset_Vo, dset_Vc, \
dset_Qs_out, dset_Qo_out, dset_Qc_out, \
dset_Q_down, dset_ET_out, dset_Ec_out \
= ut.open_simulation_file(file_out, fmode,
Vs_t0, Vo_t0, Vc_t0, no_data,
nb_cell, nb_time_step,
append_output, first_run)
eff_theta = ar_theta_s - ar_theta_r
##===========================##
## Core of the Model ##
##===========================##
if not quiet:
ut.show_banner(ini_file, nb_cell, nb_time_step)
progress_desc = 'Simulation'
else:
progress_desc = 'PyTOPKAPI v{}'.format(pytopkapi.__version__)
# prepare parameter dict
exec_params = {'nb_cell': nb_cell,
'nb_time_step': nb_time_step,
'progress_desc': progress_desc,
'Dt': Dt,
'rainfall_forcing': rainfall_forcing,
'ETr_forcing': ETr_forcing,
'ET0_forcing': ET0_forcing,
'psi_b': psi_b,
'lamda': lamda,
'eff_theta': eff_theta,
'Ks': Ks,
'X': X,
'b_s': b_s,
'b_o': b_o,
'b_c': b_c,
'alpha_s': alpha_s,
'alpha_o': alpha_o,
'alpha_c': alpha_c,
'Vs_t0': Vs_t0,
'Vo_t0': Vo_t0,
'Vc_t0': Vc_t0,
'Vsm': Vsm,
'dset_Vs': dset_Vs,
'dset_Vo': dset_Vo,
'dset_Vc': dset_Vc,
'dset_Qs_out': dset_Qs_out,
'dset_Qo_out': dset_Qo_out,
'dset_Qc_out': dset_Qc_out,
'dset_Q_down': dset_Q_down,
'dset_ET_out': dset_ET_out,
'dset_Ec_out': dset_Ec_out,
'solve_s': solve_s,
'solve_o': solve_o,
'solve_c': solve_c,
'channel_flag': channel_flag,
'W': W,
'Xc': Xc,
'Kc': Kc,
'cell_external_flow': cell_external_flow,
'external_flow_records': external_flow_records,
'node_hierarchy': node_hierarchy,
'li_cell_up': li_cell_up,
'nworkers': nworkers}
if not parallel_exec:
# Serial execution. Solve by timestep in a single process.
# Outer loop timesteps - inner loop cells
_serial_execute(exec_params)
else:
# Parallel execution. Solve by cell using multiple processes.
# Outer loop cells - inner loop timesteps
_parallel_execute(exec_params)
h5file.close()
class marvelous_config:
def __init__(self, filename=None, cdict=None):
self.config = SafeConfigParser()
if filename is not None:
self.filename = filename
else:
self.filename = os.path.join('.', 'config.ini')
if os.path.isfile(self.filename):
self.config.read(self.filename)
else:
self.config.add_section('general')
self.config.add_section('daligner')
if cdict is not None:
self.set_dict(cdict)
self.save()
def get(self, section, key, default=None):
if not self.config.has_option(section, key):
raise KeyError('key {}:{} does not exist'.format(section, key))
value = self.config.get(section, key, fallback=default)
if value == 'None':
return None
return value
def getboolean(self, section, key, default=None):
return self.config.getboolean(section, key, fallback=default)
def getint(self, section, key, default=None):
if self.get(section, key, default) is None:
return None
return self.config.getint(section, key, fallback=default)
def getfloat(self, section, key, default=None):
return self.config.getfloat(section, key, fallback=default)
def set_dict(self, cdict):
for section, c in cdict.items():
if section not in self.config.sections():
self.config.add_section(section)
for key, value in c.items():
self.config.set(section, key, str(value))
def set(self, section, key, value):
"""Set a config value.
If `section` does not yet exist, it is created.
Parameters
----------
section : str
Name of the section to set the key in.
key : str
Name of the key to set the value for.
value : object
The value to set for the key. This object is
saved as `str(value)`.
"""
if not self.config.has_section(section):
self.config.add_section(section)
self.config.set(section, key, str(value))
self.save()
def update(self, section, key, value, default=None):
"""Update a config value.
If `value` is `None` and the key has not been set to
a non-`None` value already, then the key is set to
the default value. If `value` is not `None` the key
will get this value. If neither of these cases apply
it means the key already has a value and that `value`
is `None`, and nothing will changed.
Note that if the section and/or key does not exist,
it will be created with an inital value of `None`.
Parameters
----------
section : str
Name of the section to set the key in.
key : str
Name of the key to set the value for.
value : object
The value to set for the key. This object is
saved as `str(value)`.
default : object, optional
The value to set if `value` is `None`. This object
is saved as `str(value)`.
"""
if not self.config.has_section(section):
self.config.add_section(section)
if not self.config.has_option(section, key):
self.set(section, key, None)
if value is None and self.get(section, key) is None:
self.set(section, key, default)
elif value is not None:
if self.get(section, key) != value:
self.set(section, key, value)
def save(self):
"""Save the config to file."""
with open(self.filename, 'w') as f:
self.config.write(f)
from peewee import *
import datetime
from configparser import SafeConfigParser
config = SafeConfigParser()
config.read('config.ini')
if config.getboolean('Mysql', 'enable'):
database = MySQLDatabase(
config.get('Mysql', 'scheme'),
host=config.get('Mysql', 'host'),
port=config.getint('Mysql', 'port'),
user=config.get('Mysql', 'user'),
passwd=config.get('Mysql', 'password'),
charset='utf8',
use_unicode=True,
)
elif config.getboolean('Sqlite', 'enable'):
database = SqliteDatabase(config.get('Sqlite', 'dbname'))
elif config.getboolean('Postgresql', 'enable'):
database = PostgresqlDatabase(
config.get('Postgresql', 'scheme'),
user=config.get('Postgresql', 'user'),
password=config.get('Postgresql', 'password'),
host=config.get('Postgresql', 'host'),
charset='utf8',
use_unicode=True,
)
title = 'Spider Search'
host = '0.0.0.0' # 'localhost'
port = 8080
debug = False
datapath_sub = dict()
search_path = ['/etc', '']
for path in search_path:
candidate = os.path.join(path, 'spider.conf')
if os.path.isfile(candidate):
try:
config = SafeConfigParser()
config.read([candidate])
database = config.get('Spider', 'database')
title = config.get('WebUI', 'title')
host = config.get('WebUI', 'host')
port = config.getint('WebUI', 'port')
debug = config.getboolean('WebUI', 'debug')
#datapathsub = list(filter(None, [x.strip() for x in datapath_regexps.splitlines()]))
except:
pass
datapath_sub = dict()
try:
datapath_sub = config.items('datapath_substitutions')
except:
pass
engine = create_engine(database)
Session = sessionmaker(bind=engine)
session = Session()
Base.metadata.create_all(engine)
sys.stderr.write("""
*** HEY CLOWN ***
your file permissions on {0} allow anyone to read your plain text splunk password!
use the --save-config option with --encrypt to save your configuration with an encrypted password or chmod o-rwx this file
so that other people cannot read it
*** END CLOWN MESSAGE ***
""".format(config_path))
password = config.get(args.enviro, 'password')
if config.has_option(args.enviro, 'max_result_count'):
max_result_count = config.getint(args.enviro, 'max_result_count')
except Exception as e:
logging.warning("invalid configuration file {0}: {1}".format(
config_path, str(e)))
# command line options override configuration values
if args.uri is not None:
uri = args.uri
if args.username is not None:
username = args.username
if args.password:
password = getpass("Enter password: ")
if encrypted_password is not None:
password = decrypt_password(encrypted_password)
import time
import cv2
import imutils
import numpy as np
import ffmpeg
import json
from configparser import SafeConfigParser
from serverModule.log import log
config = SafeConfigParser()
config.read('config.ini')
opencv_width = config.getint('opencvVideoRes', 'width') # width
opencv_height = config.getint('opencvVideoRes', 'height') # height
encodingToCompressionRatio = config.get('ffmpeg', 'encodingSpeed')
resize_image = config.getboolean('opencvVideoRes', 'resize')
class CameraStream:
FFMPEG_TIMEOUT = 60
ffmpeg_process = None
def __init__(self, uri, options=None):
self.uri = uri
self.key = self.get_key(uri)
self.ws_list = list()
self.options = options
class Configuration(object):
"""Class storing configuration details for Modula"""
def __init__(self, rootdir=".", cfg=None):
defaults = {
"@paths.modules": "modules",
"@paths.storage": "storage"
}
if cfg:
self.cfg = cfg
else:
cfgfiles = ["modules.cfg", "modula.cfg"]
cfgfiles = [os.path.join(rootdir, file) for file in cfgfiles]
self.cfg = Parser()
self.cfg.read(cfgfiles)
# Add defaults
for k, v in defaults.items():
if k not in self:
self[k] = v
# Make sure all paths are absolute
for k, v in self.items("@paths"):
self["@paths.%s" % k] = os.path.abspath(os.path.expanduser(v))
# Make sure all input files are absolute
for k, v in self.items("@inputs"):
self["@inputs.%s" % k] = os.path.abspath(os.path.expanduser(v))
def _parse_name(self, name):
if "." in name:
section, option = name.split(".", 1)
else:
section = "@global"
option = name
return section, option
def sections(self):
return self.cfg.sections()
def get(self, name):
return self.cfg.get(*self._parse_name(name))
def getInt(self, name):
return self.cfg.getint(*self._parse_name(name))
def getFloat(self, name):
return self.cfg.getfloat(*self._parse_name(name))
def getBoolean(self, name):
return self.cfg.getboolean(*self._parse_name(name))
def items(self, section):
return self.cfg.items(section)
def __contains__(self, name):
return self.cfg.has_option(*self._parse_name(name))
def __getitem__(self, name):
try:
return self.cfg.get(*self._parse_name(name))
except KeyError:
raise KeyError(name)
def __setitem__(self, name, value):
section, option = self._parse_name(name)
if not self.cfg.has_section(section):
self.cfg.add_section(section)
self.cfg.set(section, option, value)
def __delitem__(self, name):
section, option = self._parse_name(name)
try:
ok = self.cfg.remove_option(section, option)
if not ok:
raise NoOptionError()
except NoOptionError:
raise KeyError(name)
class Admin_Agent(object):
"""
Admin agent: initializes the simulation, decomposes space and
assigns space to Subvolume agents. Growth cones are also initialized
here and distributed to the Subvolumes corresponding to the location
of the growth cones.
Modelled as conceptually close to biology but great in load balancing...
"""
def __init__(self, total_processors, cfg_file="test_config.cfg"):
self.cfg_file = cfg_file
self.parser = SafeConfigParser()
self.parser.read(cfg_file)
np.random.seed(
self.parser.getint("system", "seed")
) # randomized locations of the initial parts of the structures to generate.
if self.parser.has_option(
"system", "recursion_limit"
): # Set the maximum depth of the Python interpreter stack to n
sys.setrecursionlimit(
self.parser.getint("system", "recursion_limit"))
self.substrate = {}
# a few variable should be global
self.total_processors = total_processors # until I find a better way
self.processor_ids = range(
1, self.total_processors) # start from 1: skip the Admin
self.summarized_constellations = {
} # store summarized constellations based on their self.num / proc id
# set up communication links for the Admin
self._initialize_communication_links()
self._setup_DBs()
ret = self._initialize_SVs()
time.sleep(
1
) # sleep a second: make sure al SVs are initialized (all messages processed) return 1
if ret < 0:
self._destruction()
return # and break out
ret = self._initialize_GE_beta() # finish return 1
if ret < 0:
self._destruction()
return
# and continue with the real deal if we haven't been kicked out yet
self.main_loop()
self._destruction()
def _initialize_communication_links(self):
""" Set up the communication links to and from the Admin
"""
self.context = zmq.Context()
""" Initialize socket to receive incoming messages. Incoming messages \
are queued "fair" according to 0MQ...
"""
self.socket_pull = self.context.socket(zmq.PULL)
self.socket_pull.bind("tcp://*:%s" %
self.parser.getint("system", "pull_port"))
# SETUP PROXY: to communicate through a proxy; all-to-all
self.psub = self.context.socket(zmq.SUB)
self.psub.connect("tcp://localhost:%s" %
self.parser.getint("system", "proxy_pub_port"))
self.psub.setsockopt(zmq.SUBSCRIBE, "Admin")
self.ppub = self.context.socket(zmq.PUB)
self.ppub.connect("tcp://localhost:%s" %
self.parser.getint("system", "proxy_sub_port"))
self.poller = zmq.Poller()
self.poller.register(self.psub, zmq.POLLIN)
registered = 0
while registered < len(self.processor_ids):
msg = self.socket_pull.recv()
print_with_rank(str(msg))
registered = registered + 1
print("all Subvolumes registered. Proceed!")
def _setup_DBs(self):
self.db_file_name = self.parser.get("system", "out_db")
try:
os.remove(self.db_file_name)
except Exception:
pass
self.conn = sqlite3.connect(self.db_file_name)
self.conn.execute(
'''CREATE TABLE swc_data (id INTEGER PRIMARY KEY AUTOINCREMENT,\
name text,\
swc_type int,\
from_x real, \
from_y real, \
from_z real,\
to_x real,\
to_y real,\
to_z real,\
radius real,\
proc integer)''')
# only if this parameter is set in the config file
if self.parser.has_option("system", "syn_db"):
self.syn_file_name = self.parser.get("system", "syn_db")
try:
os.remove(self.syn_file_name)
except Exception:
pass
self.syn_conn = sqlite3.connect(self.syn_file_name)
self.syn_conn.execute(
'''CREATE TABLE synapses (id INTEGER PRIMARY KEY AUTOINCREMENT,\
pre_syn_entity text,\
pre_x real,\
pre_y real,\
pre_z real,\
post_syn_entity text,\
post_x real,\
post_y real,\
post_z real)''')
def _get_substrate_information(self):
for option in self.parser.options("substrate"):
if not option.startswith("dim_"): # then this is an entity
val = self.parser.get("substrate", option)
if val.endswith("pkl"):
self.substrate[option] = pickle.load(open(val, "r"))
else:
print_with_rank(
"substrate entity (%s) should be given as name of a pickle file"
% option)
time.sleep(0.0)
def _get_sub_substrate(self, boundary): # get subvolume substrate
sub_substrate = {}
b0 = np.array(boundary[0])
b1 = np.array(boundary[1])
for entity in self.substrate:
print("checking entity: ", entity, ", boundary: ", boundary)
# 2014-08-08, make all internals sets of fronts.. bit redundant memory wise, but easy for administration
sub_substrate[entity] = set()
for ppoint in self.substrate[entity]:
point = ppoint[0]
t_front = Front(None, None, point, ppoint[1], 0, 0)
sub_substrate[entity].add(t_front)
#time.sleep(1)
if len(sub_substrate[entity]) == 0:
sub_substrate.pop(entity, None)
return sub_substrate
def _get_virtual_substrate(self):
virtual_substrate = {}
for name, value in self.parser.items("substrate"):
if name.startswith("virtual"):
virtual_name = name.split("_")[1]
print("Found virtual: ", virtual_name)
virtual_substrate.update(
{virtual_name: eval(self.parser.get("substrate", name))})
if virtual_substrate == {}:
print("No virtual substrates found")
#time.sleep(5)
else:
print("virtual_substrate:\n", virtual_substrate)
#time.sleep(5)
return virtual_substrate
def _initialize_SVs(self): # each process assign sv
required = ["xa", "ya", "za"] # slice number x,y,z corrdination
for item in required:
if not self.parser.has_option("sub_volumes", item):
print_with_rank("%s does not exists in the cfg file" % item)
xa,ya,za = self.parser.getint("sub_volumes","xa"), \
self.parser.getint("sub_volumes","ya"), \
self.parser.getint("sub_volumes","za")
# CHECK: do I still need this clause???
if xa * ya * za > (len(self.processor_ids)):
print_with_rank("Not enough processors (%i processors for %i SVs)" \
% (len(self.processor_ids),xa*ya*za))
return -1 # fetch in __init__
substrate = self._get_substrate_information() # volume substrate
dim_xyz = eval(self.parser.get("substrate", "dim_xyz"))
x_space = dim_xyz[0] / xa
y_space = dim_xyz[1] / ya
z_space = dim_xyz[2] / za
self.space_division = {
} # per ZMQ processor what part of the Volume they control
self.assigned_ids = {
} # per chunk of Volume, which processor controls it
count = 0
for i in range(xa):
for j in range(ya):
for k in range(za):
x0, y0, z0 = i * x_space, j * y_space, k * z_space
x1, y1, z1 = (i + 1) * x_space, (j + 1) * y_space, (
k + 1) * z_space
self.space_division[i, j, k] = [[x0, y0, z0], [x1, y1, z1]]
self.assigned_ids[i, j, k] = self.processor_ids[count]
print_with_rank("proc: %i [%i,%i,%i]= %1.f,%1.f,%1.f -> %1.f,%1.f,%1.f" \
% (self.processor_ids[count],i,j,k,x0,y0,z0,x1,y1,z1))
count = count + 1
# for now, pass the global virtual information to each SV (independent of SV boundaries)
virtual_substrate = self._get_virtual_substrate(
) # [substrate] virtual_
for key in sorted(
self.space_division.keys()): # find neighbor sv(assigned_ids)
dest = self.assigned_ids[key]
print_with_rank("key: " + str(key) + "type: " + str(type(key)) +
"assigned to " + str(dest))
boundary = self.space_division[key]
neighbors = []
for i in range(key[0] - 1, key[0] + 2):
for j in range(key[1] - 1, key[1] + 2):
for k in range(key[2] - 1, key[2] + 2):
new_key = (i, j, k)
if new_key in self.space_division and key != new_key:
neighbors.append(self.assigned_ids[i, j, k])
print_with_rank("neighbors of " + str(dest) + ": " +
str(neighbors))
""" get part of the substrate contained in this SV, prepare to include in message.
currently the substrate entities are represented by points
"""
sub_substrate = self._get_sub_substrate(boundary)
print_with_rank("sending Init_SV to %06d" % dest)
message = ("Init_SV", boundary, neighbors, sub_substrate,
virtual_substrate)
self.ppub.send_multipart(["%06d" % dest, pickle.dumps(message)])
# time.sleep(2)
return 1 # positive: all is well
def _initialize_GE_beta(self): # cell_type_
cfg_sections = self.parser.sections(
) # cfg file all sections like[system]
ship_entity_to_proc = {}
for i in self.processor_ids:
ship_entity_to_proc[i] = []
for name in cfg_sections:
if name.startswith("cell_type"):
entity_id = 0
print_with_rank("found cell type: %s" % name)
# retrieve algo to use and how many to deploy
no_seeds = self.parser.getint(name,
"no_seeds") # number of seeds
algorithm_name = self.parser.get(name,
"algorithm") # growth rule
""" Sample the soma position of the entity and assign \
to the correct processor
"""
loc = eval(self.parser.get(name, "location"))
rs = eval(self.parser.get(name, "soma_radius"))
for seed in range(no_seeds):
details = {}
for option in self.parser.options(
name): # section[cell_type_] option
details.update({option: self.parser.get(name, option)
}) # insert into details
soma_xyz = np.array([])
for i in range(3): # 3D
soma_xyz = np.append(
soma_xyz,
(loc[1][i] - loc[0][i]) * np.random.random() +
loc[0][i]) # soma location
# soma_pos = P3D2(soma_xyz,radius=rs)
details["radius"] = rs
details["soma_xyz"] = soma_xyz
dest = self._which_volume_contains_position(
soma_xyz) # contains soma subvolume
if dest == -1:
print_with_rank("Could not find subvolume for soma: " +
str(soma_xyz))
return -1
print_with_rank("sampled s%i:%s (D:%i)" %
(seed, str(soma_xyz), dest))
entity_name = name + "__" + str(
entity_id) # cell_type__0 /cell_type__1
details.update({'entity_name': entity_name})
ship_entity_to_proc[dest].append(details)
entity_id = entity_id + 1
# print_with_rank("ship_entity_to_proc: " + str(ship_entity_to_proc))
for proc in ship_entity_to_proc.keys():
entries = ship_entity_to_proc[proc]
message = ("Initialize_GEs", entries)
# comm.send(message,dest=proc,tag=2)
self.ppub.send_multipart(["%06d" % proc, pickle.dumps(message)])
print_with_rank("to %i: %s" % (proc, str(message)))
return 1 # positive: all is well
def main_loop(self):
""" Perform some update cycles. When the admin publishes an \
"Update" note, the Subvolumes send their current "my_constellation" \
and their summarized constellation to the Admin.
1. wait/gather all the info/constellations from each SV
2. wait for replies
3. proceed to next Update cycle
"""
for i in range(self.parser.getint(
"system",
"no_cycles")): # no_cycle:number of each front extend
for dest in self.processor_ids:
message = ("Update", i, self.summarized_constellations)
self.ppub.send_multipart(
["%06d" % dest, pickle.dumps(message)])
responses = 0
while responses < len(self.processor_ids):
# socks = dict(self.poller.poll())
# if self.psub in socks and socks[self.psub] == zmq.POLLIN:
[address, message] = self.psub.recv_multipart()
message = pickle.loads(message)
print_with_rank("received message on PULL")
if message[0] == "Update_OK":
responses = responses + 1
sender = int(message[1])
changes = message[2]
syn_locs = message[3]
t_constellation = message[4]
# write changes
self._temp_to_db(changes, sender)
# store putative synapse locations
if self.parser.has_option("system", "syn_db"):
self._syn_to_db(syn_locs, sender)
# store internally
self.summarized_constellations[sender] = t_constellation
# ##### TEST
# print_with_rank("Received summ ("+str(sender)+"): " + str(t_constellation.keys()) )
# time.sleep(1)
# #### END OF TEST
elif message[0] == "Migrate_Front":
self._process_migrate_front(message)
time.sleep(0.)
elif message[0] == "Extra_synapses":
print_with_rank("EXTRA_SYNAPSES")
sender = int(message[1])
syn_locs = message[2]
# store putative synapse locations
if self.parser.has_option("system", "syn_db"):
self._syn_to_db(syn_locs, sender)
self.conn.commit()
if self.parser.has_option("system", "syn_db"):
self.syn_conn.commit()
def _temp_to_db(self, changes, sender): #front,c_fronts) :
for front, c_fronts in changes:
pos = front.xyz
conn = sqlite3.connect(self.db_file_name)
cursor = conn.cursor()
for c_front in c_fronts:
cpos = c_front.xyz
name = c_front.entity_name
radius = c_front.radius
swc_type = c_front.swc_type
values = (None, name, swc_type, pos[0], pos[1], pos[2],
cpos[0], cpos[1], cpos[2], radius,
"%06d" % int(sender))
self.conn.execute(
"INSERT into swc_data VALUES (?,?,?,?,?,?,?,?,?,?,?)",
values)
#self.conn.commit()
def _syn_to_db(self, syn_locs, sender):
for syn_loc in syn_locs:
# pre_front = syn_loc[0]
# post_front = syn_loc[1]
# prp = pre_front.xyz
# pop = post_front.xyz
# values = (None,pre_front.entity_name,prp[0],prp[1],prp[2],post_front.entity_name,pop[0],pop[1],pop[2])
pre_name = syn_loc[0]
pre_x = syn_loc[1]
pre_y = syn_loc[2]
pre_z = syn_loc[3]
post_name = syn_loc[4]
post_x = syn_loc[5]
post_y = syn_loc[6]
post_z = syn_loc[7]
# values = (None,pre_front.entity_name,prp[0],prp[1],prp[2],post_front.entity_name,pop[0],pop[1],pop[2])
values = (None, pre_name, pre_x, pre_y, pre_z, post_name, post_x,
post_y, post_z)
print("values: ", values)
self.syn_conn.execute(
"INSERT into synapses VALUES (?,?,?,?,?,?,?,?,?)", values)
def _process_migrate_front(self, message):
new_front = message[1]
parent_front = message[2]
new_dest = self._which_volume_contains_position(new_front.xyz)
print_with_rank("******: new_dest %i (for %s)" %
(new_dest, str(new_front.entity_name)))
if new_dest == -1:
# this front is dead. outside of space
return
message = ("Add_Front", new_front)
#comm.send(message,dest=new_dest,tag=2)
self.ppub.send_multipart(["%06d" % new_dest, pickle.dumps(message)])
def _which_volume_contains_position(self, pos):
for key in self.space_division:
sv = self.space_division[key]
if (pos[0] >= sv[0][0]) and \
(pos[1] >= sv[0][1]) and \
(pos[2] >= sv[0][2]) and \
(pos[0] < sv[1][0]) and \
(pos[1] < sv[1][1]) and \
(pos[2] < sv[1][2]) :
return self.assigned_ids[key]
return -1 # if not found, return -1. Fetch somewhere else
def _destruction(self):
""" Shut down the system. End all threads and all MPI instances
"""
_me(True)
print_with_rank("pickle_msg:" + pickle.dumps("Done"))
self.ppub.send_multipart(["All", pickle.dumps("Done")])
self.context.destroy()
_, max_curr_seq_len = curr_seq.shape
_, max_prev_seq_len = prev_seq.shape
N_data, seq_len, dim = curr_atoms.shape
N_data_val, _, _ = curr_atoms_val.shape
N_masks, max_seq_len, N_interactions, max_atoms_ia = curr_masks.shape
N_atomtypes = 4
cfg = SafeConfigParser()
cfg.read('config.ini')
shift = tf.range(1, 8, 1)
sigma = float(cfg.getint('grid', 'sigma'))
ds = float(cfg.getint('grid', 'length')) / float(
cfg.getint('grid', 'max_resolution'))
#Map coords to grid
def map_to_grid(coord_inp, grid_size, ds, sigma):
grid = tf.range(-int(grid_size / 2), int(grid_size / 2), dtype=tf.float32)
grid = tf.add(grid, 0.5)
grid = tf.scalar_mul(ds, grid)
X, Y, Z = tf.meshgrid(grid, grid, grid, indexing='ij')
grid = tf.stack([X, Y, Z])
grid = tf.expand_dims(grid, 0)
grid = tf.expand_dims(grid, 0)
def main(argv):
"""UBEZ connection test program
Usage: python ubez.py [ip] [port]
Options:
-i ..., --ip=... ip addresss
-h, --help show this help
-p port number
"""
# print 'ARGV :', sys.argv
dirname = module_locator.module_path()
#dirname=os.path.dirname(os.path.abspath(__file__))
path = dirname.replace('\\', '/')
config = SafeConfigParser()
config.read(path + ('/ubez.ini'))
ID = config.get('SMS', 'id')
PWD = config.get('SMS', 'pwd')
SMSTIMER = config.getint('SMS', 'smstimer')
SMSURL = config.get('SMS', 'smsurl')
TELLIST = []
TELLIST.append(config.get('ONCALL', 'tel1'))
TELLIST.append(config.get('ONCALL', 'tel2'))
TELLIST.append(config.get('ONCALL', 'tel3'))
TELLIST.append(config.get('ONCALL', 'tel4'))
TELLIST.append(config.get('ONCALL', 'tel5'))
TRYLIMIT = config.getint('SYSTEM', 'trylimit')
WAITTIME = config.getint('SYSTEM', 'waittime')
OPHOST = config.get('IBM', 'ophost')
OPID = config.get('IBM', 'opid')
OPPWD = config.get('IBM', 'oppwd')
IBMFTP = config.getint('SYSTEM', 'IBMFTP')
SMS = config.getint('SYSTEM', 'SMS')
logger = logging.getLogger(config.get('SYSTEM', 'logname'))
formatter = \
logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
file_handler = RotatingFileHandler((dirname + '\ubez.log'), 'a', 4096, 5)
file_handler.setLevel(logging.DEBUG)
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
serverip = ''
serverport = 4500
try:
opts, args = getopt.getopt(argv, "hi:p:", ["ip=", "port="])
except getopt.GetoptError:
print 'Usage:ubez.py -i <ipaddress> -p <portnumber>'
sys.exit(2)
if len(opts) != 2:
print 'Usage:ubez.py -i <ipaddress> -p <portnumber>'
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print 'ubez.py -i <ipaddress> -p <portnumber>'
sys.exit()
elif opt in ("-i", "--ip"):
serverip = arg
elif opt in ("-p", "--port"):
serverport = int(arg)
msg = '-------------- Begin log -------------------'
logger.warning(msg)
for i in range(1, TRYLIMIT + 1):
LOSTCONNECT = False
sSock = socket(AF_INET, SOCK_STREAM)
try:
#Connect to server
sSock.connect((serverip, serverport))
except:
LOSTCONNECT = True
msg = 'Try %s time Connect IP: %s Port: %s fail' % (str(i), serverip, str(serverport))
print msg
logger.error(msg)
else:
msg = 'Try %s time Connect IP: %s Port: %s OK' % (str(i), serverip, str(serverport))
print msg
logger.error(msg)
finally:
sSock.close()
time.sleep(WAITTIME)
if LOSTCONNECT:
if serverip == '192.168.110.133' and \
(serverport == 4500 or serverport == 4700):
SMS_MSG = "UBEZ Acquirer service down"
elif serverip == '192.168.110.133' and serverport == 4900:
SMS_MSG = "UBEZ Issuer service down"
else:
SMS_MSG = "Service down IP:%s,PORT:%s" % (str(serverip), str(serverport))
#FTP to IBM Command console
if IBMFTP is True:
print 'use IBMFTP function'
f = open(dirname + '\message.txt', 'w+')
f.write(SMS_MSG)
f.close()
try:
ftp = FTP(OPHOST)
ftp.login(OPID, OPPWD)
fmsgname = dirname + 'message.txt'
fwavname = dirname + 'sound.wav'
ftp.retrlines('LIST')
file = open(fwavname, 'rb')
ftp.storbinary('STOR ' + 'sound.wav', file)
file.close()
file = open(fmsgname, 'rb')
ftp.delete('message.txt')
ftp.storbinary('STOR ' + 'message.txt', file)
file.close()
ftp.quit()
ftp.close()
except Exception, err:
print logger.error(err)
if SMS == True:
print 'use SMS function'
for tel in TELLIST:
if tel <> '':
data = OrderedDict();
data['ID'] = str(ID)
data['PWD'] = str(PWD)
data['TEL'] = str(tel)
data['MSG'] = SMS_MSG.encode('hex')
print data['MSG'].decode('hex')
url_values = urllib.parse.urlencode(data)
#url='http://172.28.223.10:9080/SMSer'
#url='http://127.0.0.1:8080'
url = SMSURL
full_url = url + '?' + url_values
print full_url
#try:
response = urllib.request.urlopen(full_url)
data_received = response.read()
msg1 = str(data_received)
#msg=msg1.replace("Big5","utf-8")
#tree=ElementTree.fromstring(str(msg))
logger.error(msg1)
#except Exception,err:
# errmsg=str(err)+':'+SMSURL
# logger.error(errmsg)
#finally:
time.sleep(1)
def init_globals():
"""Initialize global variables."""
# Determine if running directly from source code or as a pkg
src_path = gbls.CONFDIR
# If the default config file is in the "vulnmine/" directory, then
# must be running from source code directly
# Assume running directly from src until proven otherwise
gbls.pkgdir = ('vulnmine/' + src_path)
if os.path.exists(gbls.pkgdir):
run_from_src_code = True
print('=== Appears to be executing source code directly.\n'
'=== In this mode, subdirectories are as follows:\n'
'=== data/ contains data and user .ini config file,\n'
'=== vulnmine/ contains the source code. \n'
'=== vulnmine/vulnmine_data has default configuration data.')
else:
run_from_src_code = False
print('=== Appears to be executing a packaged module.\n'
'=== In this mode:\n'
'=== data/ subdirectory contains data and '
' user .ini config file,\n'
'=== source code and data distributed with pkg are '
'in the pkg install directory.')
if not run_from_src_code:
try:
# Files distributed with vulnmine are installed in the python
# '<sys.prefix>/vulnmine_data' directory
gbls.pkgdir = pkg_resources.resource_filename(
'vulnmine', gbls.CONFDIR)
print(('Utils Pkg directory is: {0}'.format(gbls.pkgdir)))
except Exception as e:
print(('*** Error reading default configuration file: {0} \n'
'*** Default .ini file is not in the'
' module directory or is misconfigured.\n'
'*** Aborting execution'.format(e)))
# No use trying to do anything else
return 200
parser = SafeConfigParser()
try:
default_config_file = gbls.pkgdir + gbls.CONFIG_DEFAUlTS
user_config_file = gbls.DATADIR + gbls.CONF_FILE
print(('Utils: Default .ini config file: {0} \n'
'User .ini config file: {1}'.format(default_config_file,
user_config_file)))
parser.read([default_config_file, user_config_file])
except Exception as e:
print(('*** Error reading configuration file: {0}\n'
'*** Aborting execution'.format(e)))
return 100
try:
gbls.pckdir = (gbls.wkdir + gbls.DATADIR +
parser.get('User', 'Pckdir'))
gbls.csvdir = (gbls.wkdir + gbls.DATADIR +
parser.get('User', 'Csvdir'))
gbls.nvddir = (gbls.wkdir + gbls.DATADIR +
parser.get('User', 'Nvddir'))
gbls.activate_plugins = parser.getboolean('User', 'Activate_plugins')
if gbls.activate_plugins:
plugin_directory = parser.get('User', 'Plugins')
if run_from_src_code:
gbls.plugin_folder = 'vulnmine/' + plugin_directory
else:
gbls.plugin_folder = pkg_resources.resource_filename(
'vulnmine', plugin_directory)
# Create directories if do not exist
if not os.path.isdir(gbls.pckdir):
os.makedirs(gbls.pckdir)
if not os.path.isdir(gbls.nvddir):
os.makedirs(gbls.nvddir)
######
# Vulnmine pkg data files
######
gbls.s_vndr_stop_wds = (gbls.pkgdir +
parser.get('User', 'S_vndr_stop_wds'))
gbls.df_label_software = (gbls.pkgdir +
parser.get('User', 'Df_label_software'))
gbls.df_label_vendors = (gbls.pkgdir +
parser.get('User', 'Df_label_vendors'))
gbls.clf_vendor = gbls.pkgdir + parser.get('User', 'Clf_vendor')
gbls.clf_software = gbls.pkgdir + parser.get('User', 'Clf_software')
gbls.log_conf = gbls.pkgdir + parser.get('User', 'Log_conf')
######
# CSV Input data
######
gbls.v_r_system = (gbls.csvdir + parser.get('User', 'V_r_system'))
gbls.v_gs_add_rem_pgms = (gbls.csvdir +
parser.get('User', 'V_gs_add_rem_pgms'))
gbls.v_gs_add_rem_pgms_64 = (
gbls.csvdir + parser.get('User', 'V_gs_add_rem_pgms_64'))
gbls.ad_vip_grps = gbls.csvdir + parser.get('User', 'Ad_vip_grps')
######
# Saved dataframe names
# 'rf_' - refactor code version
######
gbls.df_sys_pck = gbls.pckdir + parser.get('User', 'Df_sys_pck')
gbls.df_add_rem_g_pck = (gbls.pckdir +
parser.get('User', 'Df_add_rem_g_pck'))
gbls.df_cpe4_pck = gbls.pckdir + parser.get('User', 'Df_cpe4_pck')
gbls.df_cve_pck = gbls.pckdir + parser.get('User', 'Df_cve_pck')
gbls.df_v_R_System_3modified_pck = (
gbls.pckdir + parser.get('User', 'Df_v_R_System_3modified_pck'))
gbls.df_sft_vuln_pck = gbls.pckdir + parser.get(
'User', 'Df_sft_vuln_pck')
gbls.df_match_vendor_publisher_pck = (
gbls.pckdir + parser.get('User', 'Df_match_vendor_publisher_pck'))
gbls.df_match_cpe_sft_pck = (
gbls.pckdir + parser.get('User', 'Df_match_cpe_sft_pck'))
######
# NVD data
######
# https://static.nvd.nist.gov/feeds/xml/cve/2.0/nvdcve-2.0-2016.meta
gbls.url_meta_base = parser.get('User', 'Url_meta_base')
gbls.url_meta_end = parser.get('User', 'Url_meta_end')
#gbls.url_xml_base = parser.get('User', 'Url_xml_base')
gbls.url_json_base = parser.get('User', 'Url_json_base')
#gbls.url_xml_end = parser.get('User', 'Url_xml_end')
gbls.url_json_end = parser.get('User', 'Url_json_end')
gbls.url_cpe = parser.get('User', 'Url_cpe')
gbls.cpe_filename = parser.get('User', 'Cpe_filename')
gbls.cve_filename = parser.get('User', 'Cve_filename')
gbls.cpe_max_age = parser.getint('User', 'Cpe_max_age')
gbls.nvd_meta_filename = parser.get('User', 'Nvd_meta_filename')
gbls.nvdcpe = gbls.nvddir + gbls.cpe_filename
gbls.nvdcve = gbls.nvddir + gbls.cve_filename
except Exception as e:
print(('*** Error in config file: {0}'.format(e)))
return 0
def load(self, filename):
''' load a config file '''
# parser and default value
parser = SafeConfigParser({ \
'LRCatalog' : self.default_lrcat, \
'ProdDirectory' : self.default_prod_directory, \
'RSyncExe' : self.rsync_exe, \
'RsyncMaxLenLine' : self.rsync_max_len_line, \
'FSEncoding' : self.fs_encoding, \
'ArchiveDateFmt' : self.archive_date_fmt, \
'ProductionDateFmt' : self.production_date_fmt, \
'DayFirst' : self.dayfirst, \
'GeoCoder' : self.geocoder, \
})
# config file is located in directory where main script is lauched
_dir, _ = os.path.split(filename)
if _dir:
config_file = filename
else:
config_file = os.path.join(os.path.dirname(sys.argv[0]), filename)
parser.read(config_file)
try:
self.default_lrcat = parser.get(CONFIG_MAIN, 'LRCatalog')
self.default_prod_directory = parser.get(CONFIG_MAIN, 'ProdDirectory')
self.rsync_exe = parser.get(CONFIG_MAIN, 'RSyncExe')
self.rsync_max_len_line = parser.getint(CONFIG_MAIN, 'RsyncMaxLenLine')
self.archive_date_fmt = parser.get(CONFIG_MAIN, 'ArchiveDateFmt')
self.production_date_fmt = parser.get(CONFIG_MAIN, 'ProductionDateFmt')
self.fs_encoding = parser.get(CONFIG_MAIN, 'FSEncoding')
self.dayfirst = parser.get(CONFIG_MAIN, 'DayFirst')
self.geocoder = parser.get(CONFIG_MAIN, 'GeoCoder')
if parser.has_section(CONFIG_ARCHVOL):
self.archive_volumes = []
idvol = 1
while True:
if not parser.has_option(CONFIG_ARCHVOL, 'VOLUME%s' % idvol):
break
archvol = parser.get(CONFIG_ARCHVOL, 'VOLUME%s' % idvol)
#print archvol
try:
volname, dirname, datestart, dateend = archvol.split(',')
volname = volname.strip()
dirname = dirname.strip()
datestart = dateparser.parse(datestart)
dateend = dateparser.parse(dateend)
except ValueError:
print('ERROR section %s invalid' % CONFIG_ARCHVOL)
sys.exit(0)
self.archive_volumes.append((volname, dirname, datestart, dateend))
idvol += 1
except Error as _e:
raise LRConfigException('Failed to read config file %s' % filename)
def process(config_ini, limit=None):
config = SafeConfigParser()
config.read(config_ini)
destination_name = config.get('destination', 'name')
log.info('connecting to source server')
connection = IMAP_CLS[config.getboolean('source', 'ssl')](config.get(
'source', 'server'), config.getint('source', 'port'))
connection.login(config.get('source', 'login'),
config.get('source', 'password'))
mailbox = config.get('source', 'mailbox')
log.debug('selecting mailbox %s', mailbox)
res = connection.select('"{}"'.format(mailbox))
if res[0] != 'OK':
raise Exception(res[1])
log.debug('searching messages')
if config.has_option('source', 'imap_search'):
res = connection.uid('search', None,
config.get('source', 'imap_search'))
else:
res = connection.uid('search', None, 'LARGER',
str(config.getint('source', 'email_min_size')))
message_uids = res[1][0].split()
total_messages = total_process = len(message_uids)
if limit is not None:
total_process = min([total_messages, limit])
parser = BytesParser()
processed_mailbox = config.get('source', 'processed_mailbox')
destination = DESTINATIONS[config.get('destination',
'type')](config['destination'])
log.info('found %s messages to process', total_messages)
for idx, uid in enumerate(message_uids):
if idx >= total_process:
break
log.info('processing %s/%s...', idx + 1, total_process)
log.debug('downloading and parsing message...')
res = connection.uid('fetch', uid, '(FLAGS BODY.PEEK[])')
message = parser.parsebytes(res[1][0][1])
flags = re.findall(r'FLAGS (\(.*?\))', res[1][0][0].decode('utf8'))[0]
payload = message.get_payload()
textmsg = payload[0]
bkp_identifier = str(uuid.uuid4())
content_type = textmsg['Content-Type'].lower()
if 'multipart/related' in content_type:
log.warn('multipart/related not supported, skipping...')
continue
elif 'multipart/alternative' in content_type:
text, html = textmsg.get_payload()
add_saved_notice(text, destination_name, bkp_identifier)
add_saved_notice(html, destination_name, bkp_identifier)
elif 'text/plain' in content_type or 'text/html' in content_type:
add_saved_notice(textmsg, destination_name, bkp_identifier)
else:
raise NotImplementedError
log.debug('backup identifier: %s', bkp_identifier)
destitem = destination.new(bkp_identifier,
format_relevant_headers(message))
attachments = payload[1:]
if not len(attachments):
log.debug('no attachments to save')
continue
for attachment in attachments:
destitem.add(attachment)
del payload[1]
log.debug('saving attachments to destination...')
destitem.save()
log.debug('moving processed message to mailbox %s', processed_mailbox)
res = connection.xatom(
'UID MOVE', '{} "{}"'.format(uid.decode('utf8'),
processed_mailbox))
if res[0] != 'OK':
raise Exception(res[1])
log.debug('storing stripped message')
connection.append(mailbox, flags, datetime.now(timezone.utc),
message.as_bytes())
destination.finalize()
log.debug('logging out')
connection.logout()
log.info('done')
class GlobalSettings(GObject.Object):
"""
Pitivi app settings.
The settings object loads settings from different sources, currently:
- the local configuration file,
- environment variables.
Modules declare which settings they wish to access by calling the
addConfigOption() class method during initialization.
@cvar options: A dictionnary of available settings.
@cvar environment: A list of the controlled environment variables.
"""
options = {}
environment = set()
defaults = {}
__gsignals__ = {}
def __init__(self, **unused_kwargs):
GObject.Object.__init__(self)
self._config = SafeConfigParser()
self._readSettingsFromConfigurationFile()
self._readSettingsFromEnvironmentVariables()
def _readSettingsFromConfigurationFile(self):
"""
Read the configuration from the user configuration file.
"""
try:
conf_file_path = os.path.join(xdg_config_home(), "pitivi.conf")
self._config.read(conf_file_path)
except UnicodeDecodeError:
unicode_error_dialog()
return
except ParsingError:
return
for (section, attrname, typ, key, env, value) in self.iterAllOptions():
if not self._config.has_section(section):
continue
if key and self._config.has_option(section, key):
if typ == int or typ == int:
try:
value = self._config.getint(section, key)
except ValueError:
# In previous configurations we incorrectly stored
# ints using float values.
value = int(self._config.getfloat(section, key))
elif typ == float:
value = self._config.getfloat(section, key)
elif typ == bool:
value = self._config.getboolean(section, key)
else:
value = self._config.get(section, key)
setattr(self, attrname, value)
@classmethod
def readSettingSectionFromFile(self, cls, section):
"""
Force reading a particular section of the settings file.
Use this if you dynamically determine settings sections/keys at runtime
(like in tabsmanager.py). Otherwise, the settings file would be read
only once (at the initialization phase of your module) and your config
sections would never be read, and thus values would be reset to defaults
on every startup because GlobalSettings would think they don't exist.
"""
if cls._config.has_section(section):
for option in cls._config.options(section):
# We don't know the value type in advance, just try them all.
try:
value = cls._config.getfloat(section, option)
except:
try:
value = cls._config.getint(section, option)
except:
try:
value = cls._config.getboolean(section, option)
except:
value = cls._config.get(section, option)
setattr(cls, section + option, value)
def _readSettingsFromEnvironmentVariables(self):
"""
Override options values using their registered environment variables.
"""
for section, attrname, typ, key, env, value in self.iterAllOptions():
if not env:
# This option does not have an environment variable name.
continue
var = get_env_by_type(typ, env)
if var is not None:
setattr(self, attrname, var)
def _writeSettingsToConfigurationFile(self):
conf_file_path = os.path.join(xdg_config_home(), "pitivi.conf")
for (section, attrname, typ, key, env_var, value) in self.iterAllOptions():
if not self._config.has_section(section):
self._config.add_section(section)
if key:
if value is not None:
self._config.set(section, key, str(value))
else:
self._config.remove_option(section, key)
try:
file = open(conf_file_path, 'w')
except IOError as OSError:
return
self._config.write(file)
file.close()
def storeSettings(self):
"""
Write settings to the user's local configuration file. Note that only
those settings which were added with a section and a key value are
stored.
"""
self._writeSettingsToConfigurationFile()
def iterAllOptions(self):
"""
Iterate over all registered options
@return: an iterator which yields a tuple of (attrname, type, key,
environment, value) for each option.
"""
for section, options in list(self.options.items()):
for attrname, (typ, key, environment) in list(options.items()):
yield section, attrname, typ, key, environment, getattr(self, attrname)
def isDefault(self, attrname):
return getattr(self, attrname) == self.defaults[attrname]
def setDefault(self, attrname):
setattr(self, attrname, self.defaults[attrname])
@classmethod
def addConfigOption(cls, attrname, type_=None, section=None, key=None,
environment=None, default=None, notify=False,):
"""
Add a configuration option.
This function should be called during module initialization, before
the config file is actually read. By default, only options registered
beforehand will be loaded.
See mainwindow.py and medialibrary.py for examples of usage.
If you want to add configuration options after initialization,
use the readSettingSectionFromFile method to force reading later on.
See tabsmanager.py for an example of such a scenario.
@param attrname: the attribute of this class which represents the option
@type attrname: C{str}
@param type_: type of the attribute. Unnecessary if default is given.
@type type_: a builtin or class
@param section: The section of the config file under which this option is
saved. This section must have been added with addConfigSection(). Not
necessary if key is not given.
@param key: the key under which this option is to be saved. Can be none if
this option should not be saved.
@type key: C{str}
@param notify: whether or not this attribute should emit notification
signals when modified (default is False).
@type notify: C{boolean}
"""
if section and section not in cls.options:
raise ConfigError(
"You must add the section \"%s\" first." % section)
if key and not section:
raise ConfigError(
"You must specify a section for key \"%s\"" % key)
if section and key in cls.options[section]:
raise ConfigError("Option \"%s\" is already in use.")
if hasattr(cls, attrname):
raise ConfigError("Settings attribute \"%s\" is already in use.")
if environment and environment in cls.environment:
raise ConfigError("Settings environment varaible \"%s\" is"
"already in use.")
if not type_ and default is None:
raise ConfigError("Settings attribute \"%s\" has must have a"
" type or a default." % attrname)
if not type_:
type_ = type(default)
if notify:
notification = Notification(attrname)
setattr(cls, attrname, notification)
setattr(cls, "_" + attrname, default)
GObject.signal_new(notification.signame,
cls,
GObject.SIGNAL_RUN_LAST,
None,
())
else:
setattr(cls, attrname, default)
if section and key:
cls.options[section][attrname] = type_, key, environment
cls.environment.add(environment)
cls.defaults[attrname] = default
@classmethod
def addConfigSection(cls, section):
"""
Add a section to the local config file.
@param section: The section name. This section must not already exist.
@type section: C{str}
"""
if section in cls.options:
raise ConfigError("Duplicate Section \"%s\"." % section)
cls.options[section] = {}
@classmethod
def notifiesConfigOption(cls, attrname):
signal_name = Notification.signalName(attrname)
GObject.signal_lookup(signal_name, cls)
class ZFind:
def __init__(self, num_z=5, inifile=None, dest=None, clobber=True):
self.num_z = num_z
self.inifile = inifile
self.dest = dest
self.clobber = clobber
if self.inifile: self.set_templates_from_inifile()
def set_templates_from_inifile(self):
self.labels = []
self.templates = []
self.zmin = []
self.zmax = []
self.npoly = []
self.npixstep = []
if exists(self.inifile):
self.option = SafeConfigParser()
self.option.optionxform = str
r = self.option.read(self.inifile)
if len(r) == 1:
for section in self.option.sections():
self.labels.append(section)
if self.option.has_option(section,'template'):
self.templates.append(self.option.get(section,
'template'))
if self.option.has_option(section,'zmin'):
self.zmin.append(self.option.getfloat(section,'zmin'))
if self.option.has_option(section,'zmax'):
self.zmax.append(self.option.getfloat(section,'zmax'))
if self.option.has_option(section,'npoly'):
self.npoly.append(self.option.getint(section,'npoly'))
if self.option.has_option(section,'npixstep'):
self.npixstep.append(self.option.getint(section,
'npixstep'))
else: print("Cannot parse ini file %r" % self.inifile)
if not self.labels: self.labels = None
if not self.templates: self.templates = None
if not self.zmin: self.zmin = None
if not self.zmax: self.zmax = None
if not self.npoly: self.npoly = None
if not self.npixstep: self.npixstep = None
self.set_templates()
else: print("WARNING: %r does not exist" % self.inifile)
def set_templates(self, templates=None, zmin=None, zmax=None, npoly=None, npixstep=None):
if templates: self.templates = templates
if zmin: self.zmin = zmin
if zmax: self.zmax = zmax
if npoly: self.npoly = npoly
if npixstep: self.npixstep = npixstep
if type(self.templates) is str:
try: self.templates = [self.templates]
except:
print('Templates not a list and unable to convert to list!')
sys.exit(1)
if type(self.templates) is list: self.templates = list(map(str, self.templates))
if self.zmin is not None:
if type(self.zmin) is not list:
try:
self.zmin = [self.zmin]
except:
try:
self.zmin = self.zmin.tolist()
except:
print('Can\'t convert zmin to list - defaulting to full zrange!')
self.zmin = None
self.zmax = None
if type(self.zmin) is list:
if len(self.zmin) != len(self.templates):
print('Length of zmin doesn\'t match length of templates - defaulting to full zrange!')
self.zmin = None
self.zmax = None
if self.zmax is None:
print('zmax not given - defaulting to full zrange!')
self.zmin = None
self.zmax = None
else:
if type(self.zmax) is not list:
try: self.zmax = [self.zmax]
except:
try:
self.zmax = self.zmax.tolist()
except:
print('Can\'t convert zmax to list - defaulting to full zrange!')
self.zmin = None
self.zmax = None
if len(self.zmin) != len(self.zmax):
print('Length of zmin and zmax don\'t match - defaulting to full zrange!')
self.zmin = None
self.zmax = None
#import pdb; pdb.set_trace()
if self.npoly is None:
self.npoly = [4]*len(self.templates)
else:
if type(self.npoly) is not list:
try:
self.npoly = [self.npoly]
except:
try:
self.npoly = self.npoly.tolist()
except:
print('npoly not a list and unable to convert to \
list - defaulting to npoly=4 for all templates!')
self.npoly = [4]*len(self.templates)
else:
self.npoly = list(map(int, self.npoly))
if self.npixstep is None:
self.npixstep = [1]*len(self.templates)
else:
if type(self.npixstep) is not list:
try:
self.npixstep = [self.npixstep]
except:
try:
self.npixstep = self.npixstep.tolist()
except:
print('npixstep not a list and unable to convert to \
list - defaulting to npixstep=1 for all \
templates!')
self.npixstep = [1]*len(self.templates)
else:
self.npixstep = list(map(int, self.npixstep))
def reduce_plate_mjd(self, plate, mjd, fiberid=None, chi2file=False):
self.chi2file = chi2file
# Check types and try to convert to proper types if necessary
if fiberid is None: fiberid = [i for i in range(1000)]
else:
if type(fiberid) is not list:
try:
fiberid = [fiberid]
fiberid = list(map(int, fiberid))
except:
try:
fiberid = fiberid.tolist()
fiberid = list(map(int, fiberid))
except:
print('fiberid not set properly - running full plate!')
fiberid = [i for i in range(1000)]
else: fiberid = list(map(int, fiberid))
# Spec
specs = spec.Spec(plate=plate, mjd=mjd, fiberid=fiberid)
# ZFinder, ZFitter
zfindobjs = []
zfitobjs = []
if (self.zmin is not None) & (self.zmax is not None):
for i in range(len(self.templates)):
zfindobjs.append( zfinder.ZFinder(fname=self.templates[i],
npoly=self.npoly[i],
zmin=self.zmin[i],
zmax=self.zmax[i]) )
zfindobjs[i].zchi2( specs.flux, specs.loglambda, specs.ivar,
npixstep=self.npixstep[i], plate=plate,
mjd=mjd, fiberid=fiberid[0],
chi2file=self.chi2file )
zfitobjs.append( zfitter.ZFitter(zfindobjs[i].zchi2arr,
zfindobjs[i].zbase) )
zfitobjs[i].z_refine()
else:
for i in range(len(self.templates)):
zfindobjs.append( zfinder.ZFinder(fname=self.templates[i],
npoly=self.npoly[i],
npixstep=self.npixstep[i]) )
zfindobjs[i].zchi2( specs.flux, specs.loglambda, specs.ivar,
npixstep=self.npixstep[i], plate=plate,
mjd=mjd, fiberid=fiberid[0],
chi2file=self.chi2file )
zfitobjs.append( zfitter.ZFitter(zfindobjs[i].zchi2arr,
zfindobjs[i].zbase) )
zfitobjs[i].z_refine()
# Flags
flags = []
for i in range(len(zfindobjs)):
flags.append( misc.comb_flags(specs, zfindobjs[i], zfitobjs[i]) )
# ZPicker
if len(self.templates) == 1:
zpick = zpicker.ZPicker(specs, zfindobjs[0], zfitobjs[0], flags[0])
elif len(self.templates) == 2: zpick = zpicker.ZPicker(specs, zfindobjs[0], zfitobjs[0], flags[0], zfindobjs[1], zfitobjs[1], flags[1])
elif len(self.templates) == 3: zpick = zpicker.ZPicker(specs, zfindobjs[0], zfitobjs[0], flags[0], zfindobjs[1], zfitobjs[1], flags[1],
zfindobjs[2], zfitobjs[2], flags[2])
elif len(self.templates) == 4: zpick = zpicker.ZPicker(specs, zfindobjs[0], zfitobjs[0], flags[0], zfindobjs[1], zfitobjs[1], flags[1],
zfindobjs[2], zfitobjs[2], flags[2], zfindobjs[3], zfitobjs[3], flags[3])
elif len(self.templates) == 5: zpick = zpicker.ZPicker(specs, zfindobjs[0], zfitobjs[0], flags[0], zfindobjs[1], zfitobjs[1], flags[1],
zfindobjs[2], zfitobjs[2], flags[2], zfindobjs[3], zfitobjs[3], flags[3],
zfindobjs[4], zfitobjs[4], flags[4])
output = None
# Write output
if self.dest is None:
output = io.WriteRedmonster(zpick, clobber=self.clobber)
else:
if type(self.dest) is str:
output = io.WriteRedmonster(zpick, dest=self.dest, clobber=self.clobber)
else:
try:
self.dest = str(self.dest)
output = io.WriteRedmonster(zpick, dest=self.dest, clobber=self.clobber)
except:
print('Could not convert dest to string - writing to default directory and NOT clobbering old files!')
output = io.WriteRedmonster(zpick, clobber=True)
if output:
if len(zpick.fiberid) == 1: output.write_fiberid()
else: output.write_plate()
logging.basicConfig(format="%(message)s", level=logging.INFO)
if len(sys.argv) == 1:
logging.critical("No config file specified")
sys.exit(1)
vibrating = False
appliance_active = False
last_vibration_time = time.time()
start_vibration_time = last_vibration_time
config = SafeConfigParser()
config.read(sys.argv[1])
verbose = config.getboolean("main", "VERBOSE")
sensor_pin = config.getint("main", "SENSOR_PIN")
begin_seconds = config.getint("main", "SECONDS_TO_START")
end_seconds = config.getint("main", "SECONDS_TO_END")
mqtt_hostname = config.get("mqtt", "mqtt_hostname")
mqtt_port = int(config.get("mqtt", "mqtt_port"))
mqtt_topic = config.get("mqtt", "mqtt_topic")
mqtt_availability_topic = config.get("mqtt", "mqtt_availability_topic")
mqtt_username = config.get("mqtt", "mqtt_username")
mqtt_password = config.get("mqtt", "mqtt_password")
mqtt_clientid = config.get("mqtt", "mqtt_clientid")
start_message = config.get("main", "START_MESSAGE")
end_message = config.get("main", "END_MESSAGE")
boot_message = config.get("main", "BOOT_MESSAGE")
term_message = config.get("main", "TERM_MESSAGE")
plt.plot(freq, phase)
plt.grid('on')
plt.title('FFT - Phase')
plt.ylabel('Phase [degrees]')
plt.xlabel('Frequency [Hz]')
return plt
#####################################################################
# Main script
# Init
config = SafeConfigParser()
config.read('config.ini')
fs = config.getfloat('Test','fs') # sampling frequency in Hz
adc_resolution_bits = config.getint('Test','adc_resolution_bits')
amp_threshold = config.getfloat('Test','amp_threshold')
data_dir = config.get('Test','data_dir')
data_file_name = config.get('Test','data_file_name')
freq_file_name = config.get('Test','freq_file_name')
power_file_name = config.get('Test','power_file_name')
# Load power and frequency data
data = numpy.loadtxt(data_dir + power_file_name, delimiter = ',', skiprows = 1)
fsig_array = [d[0] for d in data]
fund_power_array = [d[1] for d in data]
# Generate fourier series for each frequency and plot
for i, power in enumerate(fund_power_array):
data = numpy.loadtxt(data_dir + data_file_name + str(int(fsig_array[i])) + '.csv',
