示例#1
0
 def lade_konfiguration(self, dateiname):
     cfg = ConfigParser()
     cfg.read([dateiname])
     self.speichergröße = cfg.getint("rechner", "speichergröße")
     self.pio_speichergröße = cfg.getint("rechner", "pio_speichergröße")
     self.view.max_taktrate = cfg.getint("gui", "max_taktrate")
     self.view.setze_schriftart(cfg.get("gui", "schriftart"), cfg.getint("gui", "schriftgröße"))
class BoardParameters(object):
    """
    Loads device channel settings and parameters from an INI file.
    """
    def __init__(self, ini_file):
        self.log = logging.getLogger(".".join([__name__, self.__class__.__name__]))
        self.log.setLevel(logging.DEBUG)
        self._ini_filename = find_file(ini_file)
        self.conf = None

    def load_ini(self):
        """
        Loads and parses the data from INI file.
        """
        self.conf = SafeConfigParser(dict_type=OrderedDict)
        self.conf.read(self._ini_filename)

        self.log.debug("Read Board Parameters INI file %s:", self._ini_filename)
        self.log.debug("    sections: %s", self.conf.sections())

    @property
    def board_name(self):
        if "Board_header" not in self.conf.sections():
            raise_with_traceback(RuntimeError("Board_header section not found in ini file %s" % str(self._ini_filename)))
        return self.conf.get("Board_header", "Board_name")

    @property
    def board_type(self):
        if "Board_header" not in self.conf.sections():
            raise_with_traceback(RuntimeError("Board_header section not found in ini file %s" % str(self._ini_filename)))
        return BoardTypes(self.conf.getint("Board_header", "Board_type"))


    @property
    def board_revision(self):
        if "Board_header" not in self.conf.sections():
            raise_with_traceback(RuntimeError("Board_header section not found in ini file %s" % str(self._ini_filename)))
        return self.conf.getint("Board_header", "Board_revision_number")

    @property
    def control_channels_count(self):
        if "Entry_counts" not in self.conf.sections():
            raise_with_traceback(RuntimeError("Entry_counts section not found in ini file %s" % str(self._ini_filename)))
        return self.conf.getint("Entry_counts", "Control_channels_count")

    @property
    def monitoring_channels_count(self):
        if "Entry_counts" not in self.conf.sections():
            raise_with_traceback(RuntimeError("Entry_counts section not found in ini file %s" % str(self._ini_filename)))
        return self.conf.getint("Entry_counts", "Monitoring_channels_count")
示例#3
0
class Main:
  config = None
  db     = None
  hosts  = []

  def __init__(self, config_path):

    self.config = SafeConfigParser(allow_no_value=False)
    self.config.read(config_path + os.sep + 'communicator.cfg')

    self.db = oursql.Connection(
                                host=self.config.get('database', 'host'),
                                port=self.config.getint('database', 'port'),
                                user=self.config.get('database', 'username'),
                                passwd=self.config.get('database', 'password'),
                                db=self.config.get('database', 'dbname'),
                                charset="utf8")

  def getHosts(self):
    with self.db.cursor() as cur:
      cur.execute(b'SELECT `id`, `name`, `address`, `port`, `key`, `uuid`' +
                  b' FROM `host` WHERE `active`', plain_query=True)

      for row in cur:
        self.hosts.append(Host(
                          row[0],
                          row[1],
                          (row[2], row[3]),
                          row[4],
                          row[5],
                          self.db)
                    )

  def run(self):
    self.getHosts()
示例#4
0
    def loadConfig(cls):
        config = SafeConfigParser()
        if os.path.isfile(CONFIG_FILE):
            config.read_file(codecs.open(CONFIG_FILE, encoding='utf-8'))

        # The default config
        cls.config = {}
        cls.config['command_open'] = None
        cls.config['player'] = 1
        cls.config['debug'] = 0

        # Load the options
        if config.has_section('options'):
            for key, value in config.items('options'):
                cls.config[key] = value

        # Load the series
        cls.series = []
        for section in config.sections():
            if section != 'options':
                title = config.get(section, 'title')
                videos = config.get(section, 'videos')
                theTvDb = config.getint(section, 'theTvDb')
                lang = config.get(section, 'lang')
                cls.series.append([section, title, videos, theTvDb, lang])
def main():
	try:
		config = SafeConfigParser()
		if config.read('config.ini') != ['config.ini']:
			 raise OSError('no config.ini file present')

		if config.getboolean('Generate CSVs', 'autogen-q'):
			if config.get('Generate CSVs', 'type') == 'logarithmic':
				minQ = config.getint('Generate CSVs', 'min-q')
				maxQ = config.getint('Generate CSVs', 'max-q')
				numQ = config.getint('Generate CSVs', 'num-q')
				pd.Series(np.logspace(round(math.log(minQ, 10)), round(math.log(maxQ,10)), numQ, dtype=int)).to_csv('DataIN/Quantity.csv', index=False)
			if config.get('Generate CSVs', 'type') == 'linear':
				minQ = config.getint('Generate CSVs', 'min-q')
				maxQ = config.getint('Generate CSVs', 'max-q')
				numQ = config.getint('Generate CSVs', 'num-q')
				pd.Series(np.linspace(minQ, maxQ, numQ, dtype=int)).to_csv('DataIN/Quantity.csv', index=False)
			if config.get('Generate CSVs', 'type') == 'digikey':
				pd.Series([1,5,10,25,50,100,250,500,1000,5000,10000]).to_csv('DataIN/Quantity.csv', index=False)

		qs = pd.read_csv('DataIN/Quantity.csv', squeeze=True)

		titlesVarCost = ['Name']
		titlesDigiKey = ['Name', 'URL', 'Quantity']

		for i in range(0, len(qs)):
			titlesVarCost.append('Cost @' + str(qs.iloc[i]) + "pcs")

		if config.getboolean('Generate CSVs', 'gen-fc'):
			pd.DataFrame(columns = ['Name', 'Cost']).to_csv('DataIN/FixedCosts.csv', index=False)

		if config.getboolean('Generate CSVs', 'gen-dkp'):
			pd.DataFrame(columns = titlesDigiKey).to_csv('DataIN/DigiKeyParts.csv', index=False)

		if config.getboolean('Generate CSVs', 'gen-vc'):
			pd.DataFrame(columns = titlesVarCost).to_csv('DataIN/VarCosts.csv', index=False)

		if config.getboolean('Generate CSVs', 'gen-p'):
			pd.DataFrame(columns = ['Price']).to_csv('DataIN/Price.csv', index=False)

	except OSError as e:
		print(e)
	except:
		print("something went wrong, most likely your config.ini file is not properly configured")
示例#6
0
    def fromConfigFile(filename):
        print('loading config from file: %s' % filename)

        config = SafeConfigParser()
        config.read(filename)

        ret = ServerConfig()

        ret.server_host = config.get('server', 'host')
        ret.server_port = config.getint('server', 'port')
        ret.whole_world_coords = [ config.getfloat('server', 'min_lon'),
                                   config.getfloat('server', 'min_lat'),
                                   config.getfloat('server', 'max_lon'),
                                   config.getfloat('server', 'max_lat') ]

        ret.db_host = config.get('db', 'host')
        ret.db_port = config.getint('db', 'port')
        ret.db_name = config.get('db', 'name')

        ret.verbose = config.get('debug', 'verbose')

        return ret
示例#7
0
def Init():
    global t
    global last_update
    global config
    t = Bot(token)
    if(t.CheckSettings()):
        config = SafeConfigParser()
        config.read(ini_path)
        last_update = config.getint('main', 'last_update')
        logger.info("-- Init --  " + str(last_update))
        UpdatesLoop()
    else:
        return
示例#8
0
文件: main.py 项目: 7h0ma5/mcdu
def run():
    config = SafeConfigParser()
    config.read("config/defaults.cfg")
    config.read("~/.config/mcdu.cfg")
    config.read("config/mcdu.cfg")

    sim = config.get("General", "sim")
    if sim == "fsx":
        from mcdu.fsx import FSXReceiver
        receiver = FSXReceiver()
    elif sim == "xplane":
        from mcdu.xplane import XPlaneReceiver
        receiver = XPlaneReceiver()
    else:
        print("no simulator set");
        return 1

    receiver.start()

    api = ACARS_API(config.get("ACARS", "logon"))
    acars = ACARS(api)
    atc = ATC(api)

    mcdu = MCDU()
    mcdu.subsystem_register(acars)
    mcdu.subsystem_register(atc)
    mcdu.menu()

    application = tornado.web.Application([
        (r"^/socket", WebSocket, dict(mcdu=mcdu)),
        (r"^/(.*)$", tornado.web.StaticFileHandler, {"path": "res/", "default_filename": "index.html"}),
    ], debug=False)

    port = config.getint("General", "port")
    application.listen(port)

    try:
        print("running on port %i" % port)
        webbrowser.open_new("http://localhost:%i" % port)
        tornado.ioloop.IOLoop.instance().start()
    except KeyboardInterrupt:
        print("quitting...")
    except Exception as e:
        import traceback
        traceback.print_exc()
        print("quitting...")
    finally:
        receiver.stop()
        acars.stop()
        atc.stop()
        return 0
示例#9
0
def get_connection(database, hostname=None, port=None, login=None, password=None):
    parser = SafeConfigParser()
    filename = os.path.join(os.path.dirname(__file__), 'config.ini')
    parser.read(filename)
    if hostname is None:
        hostname = parser.get(database, 'host')
    if port is None:
        port = parser.getint(database, 'port')
    if login is None:
        login = parser.get(database, 'username')
    if password is None:
        password = parser.get(database, 'password')

    return openerplib.get_connection(hostname=hostname, port=port, database=database, login=login, password=password)
示例#10
0
文件: util.py 项目: 50onRed/pysmb
def getConnectionInfo():
    config_filename = os.path.join(os.path.dirname(__file__), os.path.pardir, 'connection.ini')
    cp = SafeConfigParser()
    cp.read(config_filename)

    info = {
        'server_name': cp.get('server', 'name'),
        'server_ip': cp.get('server', 'ip'),
        'server_port': cp.getint('server', 'port'),
        'client_name': cp.get('client', 'name'),
        'user': cp.get('user', 'name'),
        'password': cp.get('user', 'password'),
    }
    return info
class SensorConfigurationParameters(object):
    """
    Loads sensor configuration parameters from an INI file
    """
    def __init__(self, ini_file):
        self.log = logging.getLogger(".".join([__name__, self.__class__.__name__]))
        self._ini_filename = None
        self._ini_buffer = None
        self._conf = None
        try:
            self._ini_filename = find_file(ini_file)
        except:
            # If we catch any kind of exception here then treat the parameter as the configuration
            self._ini_buffer = StringIO(unicode(ini_file))

    def load_ini(self):
        """
        Loads and parses the data from INI file. The data is stored internally in the object and can be retrieved
        through the property methods
        For the system settings all parameter names <section>_<name>
        """
        self._conf = SafeConfigParser(dict_type=OrderedDict)
        self._conf.optionxform = str
        if self._ini_filename:
            self._conf.read(self._ini_filename)
            self.log.info("Read Sensor Configuration Settings INI file: %s", self._ini_filename)
        else:
            self._conf.readfp(self._ini_buffer)
            self.log.info("Read Sensor Configuration Settings INI object %s", self._ini_buffer)
        self.log.info("    sections: %s", self._conf.sections())

    @property
    def value_map(self):
        # Read out the section General that describes the rest of the file
        values = {}
        desc = {}
        for item in self._conf.items('General'):
            match = re.match(r'^.*Cols<(\w*)>$', item[0])
            if match:
                desc[match.group(1)] = int(item[1])
        # Now loop over each defined group adding the values
        self.log.info("Ini description: %s", desc)
        for item in desc:
            val_list = []
            for index in range(desc[item]):
                item_name = "Col<{}>".format(index)
                val_list.append(self._conf.getint(item, item_name))
            values[item] = val_list
        return values
示例#12
0
文件: config.py 项目: spidermo/Udoo
class Config(object):
	def __init__(self):
		self._configFileName = "rover.cfg"
		self._readConfig()

	def _readConfig(self):
		self._config = SafeConfigParser({
			"db_host": "localhost",
			"db_port": "27017",
			"db_name": "rover",
			"serial_path": "/dev/ttymxc3",
			"serial_baud": "115200",
			"serial_timeout": "0.5",
			"serial_encoding": "windows-1252"})
		self._config.read(self._configFileName)
		self._validateConfig()

	def _writeConfig(self):
		self._config.write(open(self._configFileName, "w"))

	def _validateConfig(self):
		changed = False
		if not self._config.has_section("db"):
			self._config.add_section("db")
			changed = True
		if not self._config.has_section("arduino_serial"):
			self._config.add_section("arduino_serial")
			changed = True
		if changed:
			self._writeConfig()

	def get(self, section, key):
		return self._config.get(section, key)

	def getInt(self, section, key):
		return self._config.getint(section, key)

	def getFloat(self, section, key):
		return self._config.getfloat(section, key)

	def getBoolean(self, section, key):
		return self._config.getboolean(section, key)

	def set(self, section, key, value):
		self._config.set(section, key, value)
		self._writeConfig()
示例#13
0
    def _load_config(self):
        "Load and parse config file, pass options to livestreamer"
        
        config = SafeConfigParser()
        config_file = os.path.join(self.config_path, 'settings.ini')
        config.read(config_file)

        for option, type in list(AVAILABLE_OPTIONS.items()):
            if config.has_option('DEFAULT', option):
                if type == 'int':
                    value = config.getint('DEFAULT', option)
                if type == 'float':
                    value = config.getfloat('DEFAULT', option)
                if type == 'bool':
                    value = config.getboolean('DEFAULT', option)
                if type == 'str':
                    value = config.get('DEFAULT', option)

                self.livestreamer.set_option(option, value)
示例#14
0
class Configuration(object):

    def __init__(self):
        self.parser = SafeConfigParser(defaults={
            'theme' : 'default',
            'theme.mobile' : 'touch',
            'hostname' : '0.0.0.0',
            'port' : '7000',
            'log.file' : '/tmp/rb-serve.log',
            'log.level' : 'INFO',
            'log.format' : '%(levelname)s	%(asctime)s	%(name)s: %(message)s',
            'debug' : 'False',
            })
        self.parser.add_section('server')
        self.parser.read(path.expanduser('~/.rhythmweb'))
        self.configure_logger()

    def get_string(self, key):
        return self.parser.get('server', key, raw=True)

    def get_int(self, key):
        return self.parser.getint('server', key)

    def get_boolean(self, key):
        return self.parser.getboolean('server', key)

    def configure_logger(self):
        root = logging.getLogger()
        root.setLevel(self.get_string('log.level'))
        handler = logging.handlers.RotatingFileHandler(
                self.get_string('log.file'),
                backupCount=5, 
                maxBytes=1024*1024)
        handler.setFormatter(logging.Formatter(fmt=self.get_string('log.format')))
        root.addHandler(handler)

        if root.isEnabledFor(logging.DEBUG):
            root.debug('Logger configured')
            root.debug('Showing app configuration:')
            for section_name in self.parser.sections():
                root.debug('Section: %s' % section_name)
                for key in self.parser.options(section_name):
                    root.debug('  %s = %s' % (key, self.parser.get(section_name, key, raw=True)))
示例#15
0
    def parse_config(self,config,section=None):
        if section is None: section = self.section

        if isinstance(config,basestring):
            configfile = config
            config = SafeConfigParser()
            config.read(configfile)

        # Set the options first (otherwise optional positional arguments disappear)
        if OPTIONS in config._sections[section].keys():
            self.parse_args(config.get(section,OPTIONS))

        # Don't grab the default keys
        dests = [a.dest for a in self.parser._actions]
        for key in config._sections[section].keys():
            if key == '__name__': continue
            if key == OPTIONS: continue
            try: 
                index = dests.index(key)
                action = self.parser._actions[index]
            except ValueError:
                msg = "Unrecognized argument in config: %s\n"%key
                msg += self.get_help()
                raise ValueError(msg)

            if isinstance(action,argparse._StoreTrueAction):
                value = config.getboolean(section,key)
            if isinstance(action,argparse._StoreFalseAction):
                value = config.getboolean(section,key)
            if isinstance(action,argparse._CountAction):
                try:
                    value = int(config.getboolean(section,key))
                except ValueError:
                    value = config.getint(section,key)
            else:
                value = config.get(section,key)
            setattr(self.namespace,key,value)

        return self.namespace
示例#16
0
    def __init__(self, str_config, modelname):
        config = SafeConfigParser()
        config.read(str_config)
        self.sections = config.sections()
        if modelname in self.sections:
            try:
                self.modelname = modelname
                self.arch = config.get(modelname, "ARCH")
                self.process_fun = 'default'
                if 'PROCESS_FUN' in config[modelname] is not None:
                    self.process_fun = config[modelname]['PROCESS_FUN']
                self.number_of_classes = config.getint(modelname,
                                                       "NUM_CLASSES")
                self.number_of_iterations = config.getint(
                    modelname, "NUM_ITERATIONS")
                #self.dataset_size = config.getint(modelname, "DATASET_SIZE")
                #self.test_size = config.getint(modelname, "TEST_SIZE")
                self.batch_size = config.getint(modelname, "BATCH_SIZE")
                #self.estimated_number_of_batches =  int ( float(self.dataset_size) / float(self.batch_size) )
                #self.estimated_number_of_batches_test = int ( float(self.test_size) / float(self.batch_size) )
                #snapshot time sets when temporal weights are saved (in steps)
                self.snapshot_steps = config.getint(modelname,
                                                    "SNAPSHOT_STEPS")
                #test time sets when test is run (in seconds)
                self.test_time = config.getint(modelname, "TEST_TIME")
                self.lr = config.getfloat(modelname, "LEARNING_RATE")
                #snapshot folder, where training data will be saved
                self.snapshot_prefix = config.get(modelname, "SNAPSHOT_DIR")
                #number of estimated epochs
                #self.number_of_epochs = int ( float(self.number_of_iterations) / float(self.estimated_number_of_batches) )
                #folder where tf data is saved. Used for training and testing
                self.data_dir = config.get(modelname, "DATA_DIR")
                self.channels = config.getint(modelname, "CHANNELS")

                assert (self.channels == 1 or self.channels == 3)
            except Exception:
                raise ValueError("something wrong with configuration file " +
                                 str_config)
        else:
            raise ValueError(" {} is not a valid section".format(modelname))
示例#17
0
    def _read_config(self):
        """Read config file."""

        conf = {}
        config = SafeConfigParser({
            'title': 'BAT:', 'order': '0', 'interval': '0', 'threshold': '15',
            'format': '{bar} {percentage}%% {time}', 'alt_format': '{time}'})
        config.read([path.expanduser('~/.i3/py3status/modules.ini')])
        try:
            conf['title'] = config.get('batterystatus', 'title')
            conf['order'] = config.getint('batterystatus', 'order')
            conf['interval'] = config.getint('batterystatus', 'interval')
            conf['threshold'] = config.getint('batterystatus', 'threshold')
            conf['format'] = config.get('batterystatus', 'format')
            conf['alt_format'] = config.get('batterystatus', 'alt_format')
        except NoSectionError:
            conf['title'] = config.get('DEFAULT', 'title')
            conf['order'] = config.getint('DEFAULT', 'order')
            conf['interval'] = config.getint('DEFAULT', 'interval')
            conf['threshold'] = config.getint('DEFAULT', 'threshold')
            conf['format'] = config.get('DEFAULT', 'format')
            conf['alt_format'] = config.get('DEFAULT', 'alt_format')

        return conf
示例#18
0
def run(fh=None):
	print """\
pagerprinter v0.1.3+
Copyright 2010 - 2015 Michael Farrell <http://micolous.id.au/>
"""
	# parse config
	c = SafeConfigParser()
	c.read_dict({
		'pagerprinter': {
			'update-freq': '30',
			'backend': 'sacfs',
			'browser': 'firefox',
			'browser-exec': 'firefox',
			'browser-wait': '20',
			'trigger': 'RESPOND',
			'trigger-end': 'MAP',
			'mapper': 'google',
			'print-copies': '1',
			'unit': 'all',
			'home': '141 King William Street, Adelaide SA 5000',
		},
	})

	if fh is not None:
		c.readfp(fh)

	# get a scraper instance
	scraper = get_scraper(
		c.get('pagerprinter', 'backend')
	)(
		c.getint('pagerprinter', 'update-freq')
	)

	# get a browser helper instance
	browser = get_browser(
		c.get('pagerprinter', 'browser')
	)(
		c.get('pagerprinter', 'browser-exec'),
		c.getint('pagerprinter', 'browser-wait')
	)

	trigger = c.get('pagerprinter', 'trigger').lower().strip()
	trigger_end = c.get('pagerprinter', 'trigger-end').lower().strip()
	my_unit = c.get('pagerprinter', 'unit').lower().strip()

	try:
		printer = c.get('pagerprinter', 'printer')
	except NoOptionError:
		printer = None
	print_copies = c.getint('pagerprinter', 'print-copies')
	if print_copies < 1:
		print "ERROR: print-copies is set to less than one.  You probably don't want this."
		return
	mapper = c.get('pagerprinter', 'mapper')

	plugins = []
	if c.has_option('pagerprinter', 'plugins'):
		plugins = [
			get_plugin(x.strip())
			for x
			in c.get('pagerprinter', 'plugins').lower().split(',')
		]

		for plugin in plugins:
			plugin.configure(c)

	mapper = get_map(mapper)

	# special case: all units.
	# may result in dupe printouts
	if my_unit == 'all':
		my_unit = ''

	# home
	home = c.get('pagerprinter', 'home')

	# now, lets setup a handler for these events.
	def page_handler(good_parse, msg, date=None, unit=None):
		if good_parse:
			print "Good parse! %s - %s" % (repr(msg), unit)
			# filter for unit
			if my_unit in unit.lower():
				# this is for me!!!
				print "- This is a message for my unit!"
				# check for trigger
				lmsg = msg.lower()
				if trigger in lmsg:
					# trigger found
					# split by trigger and find address nicely.
					addr = lmsg.split(trigger)[1]

					if trigger_end in lmsg:
						addr = addr.split(trigger_end)[0]

						# Remove the @ symbols in the message, and the ASE device number (#nnn/nnn)
						addr = re.sub(r'#\d{3}/\d{3}|@|\s:\s', '', addr)

						# now split that up into parts, discarding the first
						# which is a description of the event
						addr_p = addr.split(',')[-2:]

						# clone the list for iteration as we well modify in this operation as well
						for i, part in enumerate(list(addr_p)):
							if 'alarm level' in part:
								del addr_p[i]
								break

						# reassemble the address
						addr = ','.join(addr_p)
						del addr_p

						# we have an address.  feed it to the mapping engine
						url = mapper.get_url(home, addr)

						print "- Address: %s" % addr
						print "- URL for directions: %s" % url

						# sending to browser
						browser.print_url(url, printer, print_copies)

						# now, send to plugins
						for plugin in plugins:
							try:
								plugin.execute(msg, unit, addr, date, printer, print_copies)
							except Exception, e:
								print "Exception caught in plugin %s" % type(plugin)
								print e
					else:
						print "- WARNING: End trigger not found!  Skipping..."
				else:
					print "- Trigger not found.  Skipping..."
			else:
				print "- This isn't for my unit.  Skipping..."
		else:
示例#19
0
class ZFind:
    def __init__(self, num_z=5, inifile=None, dest=None, overwrite=True):
        self.num_z = num_z
        self.inifile = inifile
        self.dest = dest
        self.overwrite = overwrite
        if self.inifile: self.set_templates_from_inifile()

    def set_templates_from_inifile(self):
        self.labels = []
        self.templates = []
        self.zmin = []
        self.zmax = []
        self.npoly = []
        self.npixstep = []

        if exists(self.inifile):
            self.option = SafeConfigParser()
            self.option.optionxform = str
            r = self.option.read(self.inifile)
            if len(r) == 1:
                for section in self.option.sections():
                    self.labels.append(section)
                    if self.option.has_option(section, 'template'):
                        self.templates.append(
                            self.option.get(section, 'template'))
                    if self.option.has_option(section, 'zmin'):
                        self.zmin.append(self.option.getfloat(section, 'zmin'))
                    if self.option.has_option(section, 'zmax'):
                        self.zmax.append(self.option.getfloat(section, 'zmax'))
                    if self.option.has_option(section, 'npoly'):
                        self.npoly.append(self.option.getint(section, 'npoly'))
                    if self.option.has_option(section, 'npixstep'):
                        self.npixstep.append(
                            self.option.getint(section, 'npixstep'))
            else:
                print("Cannot parse ini file %r" % self.inifile)

            if not self.labels: self.labels = None
            if not self.templates: self.templates = None
            if not self.zmin: self.zmin = None
            if not self.zmax: self.zmax = None
            if not self.npoly: self.npoly = None
            if not self.npixstep: self.npixstep = None

            self.set_templates()
        else:
            print("WARNING: %r does not exist" % self.inifile)

    def set_templates(self,
                      templates=None,
                      zmin=None,
                      zmax=None,
                      npoly=None,
                      npixstep=None):
        if templates: self.templates = templates
        if zmin: self.zmin = zmin
        if zmax: self.zmax = zmax
        if npoly: self.npoly = npoly
        if npixstep: self.npixstep = npixstep

        if type(self.templates) is str:
            try:
                self.templates = [self.templates]
            except:
                print('Templates not a list and unable to convert to list!')
                sys.exit(1)
        if type(self.templates) is list:
            self.templates = list(map(str, self.templates))
        if self.zmin is not None:
            if type(self.zmin) is not list:
                try:
                    self.zmin = [self.zmin]
                except:
                    try:
                        self.zmin = self.zmin.tolist()
                    except:
                        print(
                            'Can\'t convert zmin to list - defaulting to full zrange!'
                        )
                        self.zmin = None
                        self.zmax = None
            if type(self.zmin) is list:
                if len(self.zmin) != len(self.templates):
                    print(
                        'Length of zmin doesn\'t match length of templates - defaulting to full zrange!'
                    )
                    self.zmin = None
                    self.zmax = None
                if self.zmax is None:
                    print('zmax not given - defaulting to full zrange!')
                    self.zmin = None
                    self.zmax = None
                else:
                    if type(self.zmax) is not list:
                        try:
                            self.zmax = [self.zmax]
                        except:
                            try:
                                self.zmax = self.zmax.tolist()
                            except:
                                print(
                                    'Can\'t convert zmax to list - defaulting to full zrange!'
                                )
                                self.zmin = None
                                self.zmax = None
                    if len(self.zmin) != len(self.zmax):
                        print(
                            'Length of zmin and zmax don\'t match - defaulting to full zrange!'
                        )
                        self.zmin = None
                        self.zmax = None
        #import pdb; pdb.set_trace()
        if self.npoly is None:
            self.npoly = [4] * len(self.templates)
        else:
            if type(self.npoly) is not list:
                try:
                    self.npoly = [self.npoly]
                except:
                    try:
                        self.npoly = self.npoly.tolist()
                    except:
                        print('npoly not a list and unable to convert to \
                                list - defaulting to npoly=4 for all templates!'
                              )
                        self.npoly = [4] * len(self.templates)
            else:
                self.npoly = list(map(int, self.npoly))
        if self.npixstep is None:
            self.npixstep = [1] * len(self.templates)
        else:
            if type(self.npixstep) is not list:
                try:
                    self.npixstep = [self.npixstep]
                except:
                    try:
                        self.npixstep = self.npixstep.tolist()
                    except:
                        print('npixstep not a list and unable to convert to \
                                list - defaulting to npixstep=1 for all \
                                templates!')
                        self.npixstep = [1] * len(self.templates)
            else:
                self.npixstep = list(map(int, self.npixstep))

    def reduce_plate_mjd(self, plate, mjd, fiberid=None, chi2file=False):
        self.chi2file = chi2file
        # Check types and try to convert to proper types if necessary
        if fiberid is None: fiberid = [i for i in range(1000)]
        else:
            if type(fiberid) is not list:
                try:
                    fiberid = [fiberid]
                    fiberid = list(map(int, fiberid))
                except:
                    try:
                        fiberid = fiberid.tolist()
                        fiberid = list(map(int, fiberid))
                    except:
                        print('fiberid not set properly - running full plate!')
                        fiberid = [i for i in range(1000)]
            else:
                fiberid = list(map(int, fiberid))

        # Spec
        specs = spec.Spec(plate=plate, mjd=mjd, fiberid=fiberid)

        # ZFinder, ZFitter
        zfindobjs = []
        zfitobjs = []
        if (self.zmin is not None) & (self.zmax is not None):
            for i in range(len(self.templates)):
                zfindobjs.append(
                    zfinder.ZFinder(fname=self.templates[i],
                                    npoly=self.npoly[i],
                                    zmin=self.zmin[i],
                                    zmax=self.zmax[i]))
                zfindobjs[i].zchi2(specs.flux,
                                   specs.loglambda,
                                   specs.ivar,
                                   npixstep=self.npixstep[i],
                                   plate=plate,
                                   mjd=mjd,
                                   fiberid=fiberid[0],
                                   chi2file=self.chi2file)
                zfitobjs.append(
                    zfitter.ZFitter(zfindobjs[i].zchi2arr, zfindobjs[i].zbase))
                zfitobjs[i].z_refine()
        else:
            for i in range(len(self.templates)):
                zfindobjs.append(
                    zfinder.ZFinder(fname=self.templates[i],
                                    npoly=self.npoly[i],
                                    npixstep=self.npixstep[i]))
                zfindobjs[i].zchi2(specs.flux,
                                   specs.loglambda,
                                   specs.ivar,
                                   npixstep=self.npixstep[i],
                                   plate=plate,
                                   mjd=mjd,
                                   fiberid=fiberid[0],
                                   chi2file=self.chi2file)
                zfitobjs.append(
                    zfitter.ZFitter(zfindobjs[i].zchi2arr, zfindobjs[i].zbase))
                zfitobjs[i].z_refine()

        # Flags
        flags = []
        for i in range(len(zfindobjs)):
            flags.append(misc.comb_flags(specs, zfindobjs[i], zfitobjs[i]))

        # ZPicker
        if len(self.templates) == 1:
            zpick = zpicker.ZPicker(specs, zfindobjs[0], zfitobjs[0], flags[0])
        elif len(self.templates) == 2:
            zpick = zpicker.ZPicker(specs, zfindobjs[0], zfitobjs[0], flags[0],
                                    zfindobjs[1], zfitobjs[1], flags[1])
        elif len(self.templates) == 3:
            zpick = zpicker.ZPicker(specs, zfindobjs[0], zfitobjs[0], flags[0],
                                    zfindobjs[1], zfitobjs[1], flags[1],
                                    zfindobjs[2], zfitobjs[2], flags[2])
        elif len(self.templates) == 4:
            zpick = zpicker.ZPicker(specs, zfindobjs[0], zfitobjs[0], flags[0],
                                    zfindobjs[1], zfitobjs[1], flags[1],
                                    zfindobjs[2], zfitobjs[2], flags[2],
                                    zfindobjs[3], zfitobjs[3], flags[3])
        elif len(self.templates) == 5:
            zpick = zpicker.ZPicker(specs, zfindobjs[0], zfitobjs[0], flags[0],
                                    zfindobjs[1], zfitobjs[1], flags[1],
                                    zfindobjs[2], zfitobjs[2], flags[2],
                                    zfindobjs[3], zfitobjs[3], flags[3],
                                    zfindobjs[4], zfitobjs[4], flags[4])

        output = None

        # Write output
        if self.dest is None:
            output = io.WriteRedmonster(zpick, overwrite=self.overwrite)
        else:
            if type(self.dest) is str:
                output = io.WriteRedmonster(zpick,
                                            dest=self.dest,
                                            overwrite=self.overwrite)
            else:
                try:
                    self.dest = str(self.dest)
                    output = io.WriteRedmonster(zpick,
                                                dest=self.dest,
                                                overwrite=self.overwrite)
                except:
                    print(
                        'Could not convert dest to string - writing to default directory and NOT clobbering old files!'
                    )
                    output = io.WriteRedmonster(zpick, overwrite=True)

        if output:
            if len(zpick.fiberid) == 1: output.write_fiberid()
            else: output.write_plate()
示例#20
0
class VideoWall(QtGui.QWidget):
    def __init__(self, app, parent):
        super(VideoWall, self).__init__()
        self.players = {}
        #reading config.ini
        self.config = SafeConfigParser()
        self.config.read('config.ini')
        self.number_of_cameras = self.config.getint('videowall',
                                                    'number_of_cameras',
                                                    fallback=6)
        if self.number_of_cameras > 6:
            self.number_of_cameras = 6
        self.window_aspect_ratio_coeff = self.config.getfloat(
            'videowall', 'window_aspect_ratio_coeff', fallback=1.77777777778)
        self.audio_mode = self.config.get('videowall',
                                          'audio_mode',
                                          fallback='hdmi')
        self.rtsp_sleep_delay = self.config.getint('videowall',
                                                   'rtsp_sleep_delay',
                                                   fallback=1)
        self.hide_window_title = self.config.getint('videowall',
                                                    'hide_window_title',
                                                    fallback=0)
        if self.hide_window_title == 1:
            self.setWindowFlags(QtCore.Qt.FramelessWindowHint)
        self.timer = None
        self.use_sequence_view = self.config.getint('videowall',
                                                    'use_sequence_view',
                                                    fallback=0)
        if self.use_sequence_view == 1:
            self.sequence_view = {}
            self.sequence_current_index = 0
            sequence = self.config.get('videowall',
                                       'sequence_view',
                                       fallback='1').split(',')
            for s_ind in range(0, len(sequence)):
                self.sequence_view[s_ind] = int(sequence[s_ind])
            self.sequence_view_seconds = self.config.getint(
                'videowall', 'sequence_view_seconds', fallback=30000)
        #reading cameras settings
        self.cameras_settings = {}
        for cam_number in range(0, self.number_of_cameras):
            self.cameras_settings[cam_number] = {
                'preview_url':
                (self.config.get('camera-' + str(cam_number + 1),
                                 'preview_url',
                                 fallback='')),
                'full_url': (self.config.get('camera-' + str(cam_number + 1),
                                             'full_url',
                                             fallback=''))
            }
        #setting window title
        window_title = self.config.get('videowall',
                                       'window_title',
                                       fallback='Video Wall')
        self.setWindowTitle(window_title)
        #setting cameras viewports
        self.stylesheet = self.config.get('videowall',
                                          'camera_stylesheet',
                                          fallback='background-color: black')
        self.viewports = {}
        for cam_number in range(0, self.number_of_cameras):
            self.viewports[cam_number] = QtGui.QLabel(self)
            self.viewports[cam_number].setStyleSheet(self.stylesheet)
            self.viewports[cam_number].setObjectName('camera-' +
                                                     str(cam_number + 1))
            self.viewports[cam_number].setText(str(cam_number + 1))
            #self.viewports[cam_number].installEventFilter(self)
        app.installEventFilter(self)

    def eventFilter(self, source, event):
        if event.type() == QtCore.QEvent.MouseButtonPress:
            if type(source) == QtGui.QLabel:
                if source.objectName() != 'camera-1':
                    clicked_id = int(source.text()) - 1
                    self.changeMainCamera(clicked_id)

        if event.type() == QtCore.QEvent.KeyPress:
            current_player_id = int(self.viewports[0].text()) - 1
            next_player_id = 0

            if event.key() == 16777236:  # ->
                next_player_id = current_player_id + 1
                if next_player_id > (self.number_of_cameras - 1):
                    next_player_id = 0
            if event.key() == 16777234:  # <-
                next_player_id = current_player_id - 1
                if next_player_id < 0:
                    next_player_id = self.number_of_cameras - 1
            self.changeMainCamera(next_player_id)

        return super(VideoWall, self).eventFilter(source, event)

    def changeMainCamera(self, new_cam_id):
        cam_viewport = None
        for viewport_id in range(0, self.number_of_cameras):
            if self.viewports[viewport_id].text() == str(new_cam_id + 1):
                cam_viewport = self.viewports[viewport_id]

        current_player_id = int(self.viewports[0].text()) - 1
        current_player_atX = self.players[current_player_id].atX
        current_player_atY = self.players[current_player_id].atY
        current_player_width = self.players[current_player_id].width
        current_player_height = self.players[current_player_id].height
        current_player_audio_mode = self.players[current_player_id].audio_mode
        clicked_id = new_cam_id
        clicked_atX = self.players[clicked_id].atX
        clicked_atY = self.players[clicked_id].atY
        clicked_width = self.players[clicked_id].width
        clicked_height = self.players[clicked_id].height
        clicked_audio_mode = self.players[clicked_id].audio_mode
        #stop main screen player camera-1
        self.players[current_player_id].stop()
        self.players[current_player_id].process.stdin.write(b'q')
        self.players[current_player_id].process.terminate()
        #stop and kill clicked player
        self.players[clicked_id].stop()
        self.players[clicked_id].process.stdin.write(b'q')
        self.players[clicked_id].process.terminate()
        QtCore.QThread.sleep(self.rtsp_sleep_delay)
        #run preview from main camera-1 screen on this place
        cam_viewport.setText(str(current_player_id + 1))
        self.players[current_player_id] = Player(
            self.cameras_settings[current_player_id]['preview_url'],
            clicked_atX, clicked_atY, clicked_width, clicked_height,
            clicked_audio_mode, current_player_id)
        #run clicked on main screen in HD (full_url)
        self.viewports[0].setText(str(clicked_id + 1))
        self.players[clicked_id] = Player(
            self.cameras_settings[clicked_id]['full_url'], current_player_atX,
            current_player_atY, current_player_width, current_player_height,
            current_player_audio_mode, clicked_id)
        QtCore.QThread.sleep(self.rtsp_sleep_delay)

    def redrawViewPorts(self, width, height, width_difference):
        sc = 1 / 3
        lc = 2 / 3
        w_offset = width_difference / 2
        for cam_number in range(0, self.number_of_cameras):
            if cam_number == 0:
                self.viewports[cam_number].resize(width * lc, height * lc)
                self.viewports[cam_number].move(w_offset, 0)
            else:
                if cam_number > 3:
                    self.viewports[cam_number].resize(width * sc, height * sc)
                    self.viewports[cam_number].move(
                        w_offset + ((cam_number - 4) * width * sc),
                        height * lc)
                else:
                    self.viewports[cam_number].resize(width * sc, height * sc)
                    self.viewports[cam_number].move(w_offset + (width * lc),
                                                    (cam_number - 1) * height *
                                                    sc)

    def redrawPlayers(self):
        for cam_number in range(0, self.number_of_cameras):
            pos = self.geometry().topLeft() + self.viewports[cam_number].pos()
            player_id = int(self.viewports[cam_number].text()) - 1
            quality = 'preview_url'
            if cam_number == 0:
                quality = 'full_url'
            if cam_number in self.players:
                self.players[player_id].resize(
                    pos.x(), pos.y(),
                    self.viewports[cam_number].geometry().width(),
                    self.viewports[cam_number].geometry().height())
            else:
                if len(self.cameras_settings[cam_number][quality]) is not 0:
                    self.players[cam_number] = Player(
                        self.cameras_settings[cam_number][quality], pos.x(),
                        pos.y(), self.viewports[cam_number].geometry().width(),
                        self.viewports[cam_number].geometry().height(),
                        self.audio_mode, cam_number)
                    QtCore.QThread.sleep(self.rtsp_sleep_delay)
                    #start sequence if needed
                    if self.timer is None and self.use_sequence_view == 1:
                        self.timer = QtCore.QTimer()
                        self.timer.timeout.connect(self.timerEvent)
                        self.timer.start(self.sequence_view_seconds)

    def resizeEvent(self, e):
        width = e.size().width()
        height = e.size().height()
        calc_width = height * self.window_aspect_ratio_coeff
        width_difference = 0
        if calc_width > width:
            height = width / self.window_aspect_ratio_coeff
        else:
            width_difference = width - calc_width
            width = calc_width
        self.redrawViewPorts(width, height, width_difference)
        self.redrawPlayers()

    def moveEvent(self, e):
        self.redrawPlayers()

    def closeEvent(self, e):
        for player in self.players:
            if self.players[player].process is not None and self.players[
                    player].process.poll() is None:
                self.players[player].stop()
                self.players[player].process.stdin.write(b'q')
                self.players[player].process.terminate()
        e.accept()

    def changeEvent(self, event):
        if event.type() == QtCore.QEvent.WindowStateChange:
            if self.windowState() & QtCore.Qt.WindowMinimized:
                for player in self.players:
                    self.players[player].hide()
            elif event.oldState() & QtCore.Qt.WindowMinimized:
                for player in self.players:
                    self.players[player].unhide()
        QtGui.QWidget.changeEvent(self, event)

    def timerEvent(self):
        self.sequence_current_index = self.sequence_current_index + 1
        if self.sequence_current_index > len(self.sequence_view) - 1:
            self.sequence_current_index = 0
        self.changeMainCamera(self.sequence_view[self.sequence_current_index] -
                              1)
示例#21
0
def loadParams(filepath):
    config = SafeConfigParser({'color':'0', 
                               'PreTrainNet':'',
                               'loopRestartFrequency':'-1'})
    config.read(filepath)

    params = {}
    #device
    params['randomSeed'] = config.getint('device', 'randomSeed')
    #solver
    params['SolverType'] = config.get('solver', 'SolverType')
    params['lr'] = config.getfloat('solver', 'lr')
    params['momentum'] = config.getfloat('solver', 'momentum')
    params['lrDecay'] = config.getfloat('solver', 'lrDecay')
    params['batchSize'] = config.getint('solver', 'batchSize')
    params['weightDecay'] = config.getfloat('solver', 'weightDecay')

    #stopping crteria
    params['nMaxEpoch'] = config.getint('stopping', 'nMaxEpoch')
    params['nMaxIter'] = config.getint('stopping', 'nMaxIter')
  
    #the SA training would alternating between 'normalBatchLength' iteration of normal training and 'loopBatchLength' of self-augment training
    params['normalBatchLength'] = config.getint('loop', 'normalBatchLength')     
    params['loopStartEpoch'] = config.getint('loop', 'loopStartEpoch')      
    params['loopStartIteration'] = config.getint('loop', 'loopStartIteration') #add loop after this number of normal training.
    params['loopBatchLength'] = config.getint('loop', 'loopBatchLength') #how many mini-batch iteration for ever loop optimize
    params['loopRestartFrequency'] = config.getint('loop', 'loopRestartFrequency')
    #network structure
    params['NetworkType'] = config.get('network', 'NetworkType')
    params['Channal'] = config.get('network', 'Channal')
    params['BN'] = config.getboolean('network', 'BN')
    params['color'] = config.getboolean('network', 'color')
    params['PreTrainNet'] = config.get('network', 'PreTrainNet')
    #dataset
    params['NormalizeInput'] = config.getboolean('dataset', 'NormalizeInput')
    params['dataset'] = config.get('dataset', 'dataset')
    params['testDataset'] = config.get('dataset', 'testDataset')
    params['predictDataset'] = config.get('dataset', 'predictDataset')
     
    #display and testing
    params['displayStep'] = config.getint('display', 'displayStep')
    params['loopdisplayStep'] = config.getint('display', 'loopdisplayStep')
    params['checkPointStepIteration'] = config.getint('display', 'checkPointStepIteration')
    params['checkPointStepEpoch'] = config.getint('display', 'checkPointStepEpoch')

    return params
def main(argv):
    """
	Runs evaluation of a prediction technique on a selected evaluation problem
	from a selected dataset. Runs the evaluation multiple times and prints stats
	to output. Takes as an argument the file path to a configeration file that
	is used to set the parameters of the evaluation.
	"""
    startTime = time.time()
    parser = SafeConfigParser()
    parser.read(argv[0])

    validConfig = validateConfigFile(parser)
    if not validConfig:
        print('[ERROR] - Config not valid!')
        exit()

    seed(parser.getint('evaluation_params', 'seedNo'))
    # print('Remove stop words: {} Remove punctuation: {} Lemmatize: {}'.format(rmStopwords, rmPunct, lemmatize))
    dictionaryDataPath = parser.get('evaluation_params', 'dictionary')
    try:
        evaluationData = sl.loadDataFromFile('dictionaryData/' +
                                             dictionaryDataPath)
    except IOError as err:
        print((dictionaryDataPath +
               ' can not be found in the dictionaryData directory.'))
        exit()

    evaluationData = ds.selectPoS(evaluationData,
                                  parser.get('evaluation_params', 'pos'))
    evaluationData = ds.removeWordsWithTooFewSenses(
        evaluationData, parser.getint('evaluation_params', 'numOfSenses'),
        parser.getint('evaluation_params', 'numOfExamp'))
    evaluationData = ds.examplesToLowerCase(evaluationData)
    evaluationData = ds.tokenizeAndLemmatizeExamples(
        evaluationData, parser.getboolean('evaluation_params', 'lemmatize'))
    evaluationData = ds.removeStopwordsAndPunct(
        evaluationData, parser.getboolean('evaluation_params', 'rmStopwords'),
        parser.getboolean('evaluation_params', 'rmPunct'))

    num_examples_lt_5 = 0
    num_examples_lt_10 = 0
    num_examples_lt_20 = 0
    num_examples_all = 0
    base_path = '/Users/thomas/DevSandbox/EpicDataShelf/tag-lab/sense_alloc'
    pos = parser.get('evaluation_params', 'pos')
    with open(os.path.join(base_path, dictionaryDataPath.split('.')[0], '{}_lt_5.txt'.format(pos)), 'w', encoding='utf-8') as lt_5_file, \
      open(os.path.join(base_path, dictionaryDataPath.split('.')[0], '{}_lt_10.txt'.format(pos)), 'w', encoding='utf-8') as lt_10_file, \
      open(os.path.join(base_path, dictionaryDataPath.split('.')[0], '{}_lt_20.txt'.format(pos)), 'w', encoding='utf-8') as lt_20_file, \
      open(os.path.join(base_path, dictionaryDataPath.split('.')[0], '{}_all.txt'.format(pos)), 'w', encoding='utf-8') as all_file:
        for key, vals in list(evaluationData.items()):
            print(['### ITEM: {} ###'.format(key)])
            # print('\t{}\n'.format(json.dumps(vals, indent=4)))
            for val in vals:
                ex_lt_5 = []
                ex_lt_10 = []
                ex_lt_20 = []
                ex_all = []
                for ex in val['examples']:
                    if (ex['sent'].count(' ') < 5):
                        num_examples_lt_5 += 1
                        ex_lt_5.append((ex['sent'], val['def']))
                    if (ex['sent'].count(' ') < 10):
                        num_examples_lt_10 += 1
                        ex_lt_10.append((ex['sent'], val['def']))
                    if (ex['sent'].count(' ') < 20):
                        # print('{}: {}'.format(ex['sent'].count(' '), ex['sent']))
                        num_examples_lt_20 += 1
                        ex_lt_20.append((ex['sent'], val['def']))
                    num_examples_all += 1
                    ex_all.append((ex['sent'], val['def']))

                for l, f in [(ex_lt_5, lt_5_file), (ex_lt_10, lt_10_file),
                             (ex_lt_20, lt_20_file), (ex_all, all_file)]:
                    if (len(l) > 2):
                        for ex, syn in l:
                            f.write('{}\t{}\t{}\n'.format(key, ex, syn))

            print('---------------------------------------------------')
    print(('<=5: {}; <=10: {}; <=20: {}'.format(num_examples_lt_5,
                                                num_examples_lt_10,
                                                num_examples_lt_20)))
    """
示例#23
0
    pyplot.ylabel('Power Normalized Magnetude [dBFS]')
    pyplot.xlabel('Sampling Frequency [Hz]')
    return pyplot


######################################################################
# Main script

config = SafeConfigParser()
config.read('config.ini')

data_dir = config.get('Test', 'data_dir')
data_file_name = config.get('Test', 'data_file_name')
power_file_name = config.get('Test', 'power_file_name')
power_plot_name = config.get('Test', 'power_plot_name')
num_samples = config.getint('Test', 'num_samples')
fs = config.getfloat('Test', 'fs')  # sampling frequency in Hz
adc_resolution_bits = config.getint('Test', 'adc_resolution_bits')

sys.stdout.write("\nRunning test...\n\n")

# Initialize arrays that will receive data in each loop cycle
fund_power_array = []
fsig_array = []

# Load power and frequency data
data = numpy.loadtxt(data_dir + power_file_name, delimiter=',', skiprows=1)
fsig_array = [d[0] for d in data]
fund_power_array = [d[1] for d in data]

power_array = []
示例#24
0
def cli():
    parser = argparse.ArgumentParser(
        description='Check HTTPs rules for validity')
    parser.add_argument(
        'checker_config', help='an integer for the accumulator')
    parser.add_argument('rule_files', nargs="*", default=[],
                        help="Specific XML rule files")
    parser.add_argument('--json_file', default=None,
                        help='write results in json file')
    args = parser.parse_args()

    config = SafeConfigParser()
    config.read(args.checker_config)

    logfile = config.get("log", "logfile")
    loglevel = convertLoglevel(config.get("log", "loglevel"))
    if logfile == "-":
        logging.basicConfig(stream=sys.stderr, level=loglevel,
                            format="%(levelname)s %(message)s")
    else:
        logging.basicConfig(filename=logfile, level=loglevel,
                            format="%(asctime)s %(levelname)s %(message)s [%(pathname)s:%(lineno)d]")

    autoDisable = False
    if config.has_option("rulesets", "auto_disable"):
        autoDisable = config.getboolean("rulesets", "auto_disable")
    # Test rules even if they have default_off=...
    includeDefaultOff = False
    if config.has_option("rulesets", "include_default_off"):
        includeDefaultOff = config.getboolean(
            "rulesets", "include_default_off")
    ruledir = config.get("rulesets", "rulesdir")
    checkCoverage = False
    if config.has_option("rulesets", "check_coverage"):
        checkCoverage = config.getboolean("rulesets", "check_coverage")
    checkTargetValidity = False
    if config.has_option("rulesets", "check_target_validity"):
        checkTargetValidity = config.getboolean(
            "rulesets", "check_target_validity")
    checkNonmatchGroups = False
    if config.has_option("rulesets", "check_nonmatch_groups"):
        checkNonmatchGroups = config.getboolean(
            "rulesets", "check_nonmatch_groups")
    checkTestFormatting = False
    if config.has_option("rulesets", "check_test_formatting"):
        checkTestFormatting = config.getboolean(
            "rulesets", "check_test_formatting")
    certdir = config.get("certificates", "basedir")
    if config.has_option("rulesets", "skiplist") and config.has_option("rulesets", "skipfield"):
        skiplist = config.get("rulesets", "skiplist")
        skipfield = config.get("rulesets", "skipfield")
        with open(skiplist) as f:
            f.readline()
            for line in f:
                splitLine = line.split(",")
                fileHash = splitLine[0]
                if splitLine[int(skipfield)] == "1":
                    skipdict[binascii.unhexlify(fileHash)] = 1

    threadCount = config.getint("http", "threads")
    httpEnabled = True
    if config.has_option("http", "enabled"):
        httpEnabled = config.getboolean("http", "enabled")

    metricName = config.get("thresholds", "metric")
    thresholdDistance = config.getfloat("thresholds", "max_distance")
    metricClass = getMetricClass(metricName)
    metric = metricClass()

    # Debugging options, graphviz dump
    dumpGraphvizTrie = False
    if config.has_option("debug", "dump_graphviz_trie"):
        dumpGraphvizTrie = config.getboolean("debug", "dump_graphviz_trie")
    if dumpGraphvizTrie:
        graphvizFile = config.get("debug", "graphviz_file")
        exitAfterDump = config.getboolean("debug", "exit_after_dump")

    if args.rule_files:
        xmlFnames = args.rule_files
    else:
        xmlFnames = glob.glob(os.path.join(ruledir, "*.xml"))
    trie = RuleTrie()

    rulesets = []
    coverageProblemsExist = False
    targetValidityProblemExist = False
    nonmatchGroupProblemsExist = False
    testFormattingProblemsExist = False
    for xmlFname in xmlFnames:
        logging.debug("Parsing {}".format(xmlFname))
        if skipFile(xmlFname):
            logging.debug(
                "Skipping rule file '{}', matches skiplist.".format(xmlFname))
            continue

        ruleset = Ruleset(etree.parse(open(xmlFname, "rb")).getroot(), xmlFname)
        if ruleset.defaultOff and not includeDefaultOff:
            logging.debug("Skipping rule '{}', reason: {}".format(
                          ruleset.name, ruleset.defaultOff))
            continue
        # Check whether ruleset coverage by tests was sufficient.
        if checkCoverage:
            logging.debug("Checking coverage for '{}'.".format(ruleset.name))
            problems = ruleset.getCoverageProblems()
            for problem in problems:
                coverageProblemsExist = True
                logging.error(problem)
        if checkTargetValidity:
            logging.debug("Checking target validity for '{}'.".format(ruleset.name))
            problems = ruleset.getTargetValidityProblems()
            for problem in problems:
                targetValidityProblemExist = True
                logging.error(problem)
        if checkNonmatchGroups:
            logging.debug("Checking non-match groups for '{}'.".format(ruleset.name))
            problems = ruleset.getNonmatchGroupProblems()
            for problem in problems:
                nonmatchGroupProblemsExist = True
                logging.error(problem)
        if checkTestFormatting:
            logging.debug("Checking test formatting for '{}'.".format(ruleset.name))
            problems = ruleset.getTestFormattingProblems()
            for problem in problems:
                testFormattingProblemsExist = True
                logging.error(problem)
        trie.addRuleset(ruleset)
        rulesets.append(ruleset)

    # Trie is built now, dump it if it's set in config
    if dumpGraphvizTrie:
        logging.debug("Dumping graphviz ruleset trie")
        graph = trie.generateGraphizGraph()
        if graphvizFile == "-":
            graph.dot()
        else:
            with open(graphvizFile, "w") as gvFd:
                graph.dot(gvFd)
        if exitAfterDump:
            sys.exit(0)
    fetchOptions = http_client.FetchOptions(config)
    fetchers = list()

    # Ensure "default" is in the platform dirs
    if not os.path.isdir(os.path.join(certdir, "default")):
        raise RuntimeError(
            "Platform 'default' is missing from certificate directories")

    platforms = http_client.CertificatePlatforms(
        os.path.join(certdir, "default"))
    fetchers.append(http_client.HTTPFetcher(
        "default", platforms, fetchOptions, trie))
    # fetches pages with unrewritten URLs
    fetcherPlain = http_client.HTTPFetcher("default", platforms, fetchOptions)

    urlList = []
    if config.has_option("http", "url_list"):
        with open(config.get("http", "url_list")) as urlFile:
            urlList = [line.rstrip() for line in urlFile.readlines()]

    if httpEnabled:
        taskQueue = queue.Queue(1000)
        resQueue = queue.Queue()
        startTime = time.time()
        testedUrlPairCount = 0
        config.getboolean("debug", "exit_after_dump")

        for i in range(threadCount):
            t = UrlComparisonThread(
                taskQueue, metric, thresholdDistance, autoDisable, resQueue)
            t.setDaemon(True)
            t.start()

        # set of main pages to test
        mainPages = set(urlList)
        # If list of URLs to test/scan was not defined, use the test URL extraction
        # methods built into the Ruleset implementation.
        if not urlList:
            for ruleset in rulesets:
                if ruleset.platform != "default" and os.path.isdir(os.path.join(certdir, ruleset.platform)):
                    theseFetchers = copy.deepcopy(fetchers)
                    platforms.addPlatform(ruleset.platform, os.path.join(certdir, ruleset.platform))
                    theseFetchers.append(http_client.HTTPFetcher(
                        ruleset.platform, platforms, fetchOptions, trie))
                else:
                    theseFetchers = fetchers
                testUrls = []
                for test in ruleset.tests:
                    if not ruleset.excludes(test.url):
                        testedUrlPairCount += 1
                        testUrls.append(test.url)
                    else:
                        # TODO: We should fetch the non-rewritten exclusion URLs to make
                        # sure they still exist.
                        logging.debug("Skipping excluded URL {}".format(test.url))
                task = ComparisonTask(testUrls, fetcherPlain, theseFetchers, ruleset)
                taskQueue.put(task)

        taskQueue.join()
        logging.info("Finished in {:.2f} seconds. Loaded rulesets: {}, URL pairs: {}.".format(
                     time.time() - startTime, len(xmlFnames), testedUrlPairCount))
        if args.json_file:
            json_output(resQueue, args.json_file, problems)
    if checkCoverage:
        if coverageProblemsExist:
            return 1  # exit with error code
    if checkTargetValidity:
        if targetValidityProblemExist:
            return 1  # exit with error code
    if checkNonmatchGroups:
        if nonmatchGroupProblemsExist:
            return 1  # exit with error code
    if checkTestFormatting:
        if testFormattingProblemsExist:
            return 1  # exit with error code
    return 0  # exit with success
示例#25
0
def run():
    config = SafeConfigParser()
    config.read("config/defaults.cfg")
    config.read("~/.config/mcdu.cfg")
    config.read("config/mcdu.cfg")

    sim = config.get("General", "sim")
    if sim == "fsx":
        from mcdu.fsx import FSXReceiver
        receiver = FSXReceiver()
    elif sim == "xplane":
        from mcdu.xplane import XPlaneReceiver
        receiver = XPlaneReceiver()
    else:
        print("no simulator set")
        return 1

    db = Database()
    receiver.start()

    api = ACARS_API(config.get("ACARS", "logon"))
    acars = ACARS(api)
    atc = ATC(api)
    data = DATA(api)
    init = INIT(api)
    fplan = FPLAN(api)
    perf = PERF(api)

    mcdu = MCDU()
    mcdu.subsystem_register(acars)
    mcdu.subsystem_register(atc)
    mcdu.subsystem_register(data)
    mcdu.subsystem_register(init)
    mcdu.subsystem_register(fplan)
    mcdu.subsystem_register(perf)
    mcdu.database_register(db)
    mcdu.menu()

    application = myDisplay()

    port = config.getint("General", "port")

    application.initialize(mcdu)
    application.open()

    try:
        print("running on port %i" % port)
        # Call my application here
        application.mainloop()


#        receiver.run()
    except KeyboardInterrupt:
        print("quitting...")
    except Exception:
        import traceback
        traceback.print_exc()
        print("quitting...")
    finally:
        receiver.stop()
        acars.stop()
        atc.stop()
        return 0
示例#26
0
    assert np.all(mgrid == mexp_edges)
    assert np.all(z_edges == zgrid)

    saveId = expName + "_" + gridName + "_" + calName + "_v" + version

    from orphics.io import dict_from_section, list_from_config
    constDict = dict_from_section(Config, 'constants')
    clusterDict = dict_from_section(Config, 'cluster_params')
    beam = list_from_config(Config, expName, 'beams')
    noise = list_from_config(Config, expName, 'noises')
    freq = list_from_config(Config, expName, 'freqs')
    lknee = list_from_config(Config, expName, 'lknee')[0]
    alpha = list_from_config(Config, expName, 'alpha')[0]
    fsky = Config.getfloat(expName, 'fsky')
    try:
        v3mode = Config.getint(expName, 'V3mode')
    except:
        v3mode = -1

    clttfile = Config.get('general', 'clttfile')

    # get s/n q-bins
    mus = list_from_config(Config, 'general', 'mubins')
    mubin_edges = np.linspace(mus[0], mus[1], int(mus[2]) + 1)

    print(mubin_edges)

    massMultiplier = Config.getfloat('general', 'mass_calib_factor')
    YWLcorrflag = Config.getfloat('general', 'ywl_corr_flag')

    if debug: print("Finished rank 0 work.")
示例#27
0
if sys.version_info.major == 3:
    from configparser import SafeConfigParser
else:
    from ConfigParser import SafeConfigParser
from pathlib2 import Path

import pygeoutil.util as util

# Parse config file
parser = SafeConfigParser()
parser.read('config_rotations.txt')

# Get directory path (3 levels up is the parent directory)
dir_prj = str(Path(__file__).parents[3])

FAO_START_YR = parser.getint('PARAMETERS',
                             'FAO_START_YR')  # Starting year of FAO data
FAO_END_YR = parser.getint('PARAMETERS',
                           'FAO_END_YR')  # Ending year of FAO data
TAG = parser.get('PROJECT', 'TAG')
FAO_FILE = parser.get('PROJECT', 'fao_data')
FAO_SHEET = parser.get('PROJECT', 'fao_sheet')
PROJ_NAME = parser.get('PROJECT', 'project_name')
DO_PARALLEL = parser.getboolean('PARAMETERS',
                                'DO_PARALLEL')  # Use multiprocessing or not?
NUM_LATS = 180.0
NUM_LONS = 360.0
PLOT_CNTRS = 10
PLOT_CROPS = 10
DPI = 300
CFT_FRAC_YR = parser.getint('PARAMETERS', 'CFT_FRAC_YR')
GLM_STRT_YR = parser.getint('PARAMETERS', 'GLM_STRT_YR')
示例#28
0
def parse_config_file(rcPath, args):

    try:
        rcConfig = SafeConfigParser()
        rv = rcConfig.read(rcPath)
    except configparser.ParsingError as err:
        sys.stderr.write('RC Parse error: {}\n'.format(err.args[0]))
        sys.exit(1)  # Abort program. Can't continue with RC error

    # OPTIMIZE - LOOP THIS
    try:
        options['dbpath'] = rcConfig.get('main', 'dbpath')
        options['logpath'] = rcConfig.get('main', 'logpath')
        options['verbose'] = rcConfig.getint('main', 'verbose')
        options['logappend'] = rcConfig.getboolean('main', 'logappend')
        options['sizereduce'] = rcConfig.get('main', 'sizereduce')
        options['subjectregex'] = rcConfig.get('main', 'subjectregex')
        options['srcregex'] = rcConfig.get('main', 'srcregex')
        options['destregex'] = rcConfig.get('main', 'destregex')
        options['srcdestdelimiter'] = rcConfig.get('main', 'srcdestdelimiter')
        options['summarysubject'] = rcConfig.get('main', 'summarysubject')
        options['border'] = rcConfig.get('main', 'border')
        options['padding'] = rcConfig.get('main', 'padding')

        options['intransport'] = rcConfig.get('incoming', 'transport')
        options['inserver'] = rcConfig.get('incoming', 'server')
        options['inport'] = rcConfig.get('incoming', 'port')
        options['inencryption'] = rcConfig.get('incoming', 'encryption')
        options['inaccount'] = rcConfig.get('incoming', 'account')
        options['inpassword'] = rcConfig.get('incoming', 'password')
        options['infolder'] = rcConfig.get('incoming', 'folder')

        options['outserver'] = rcConfig.get('outgoing', 'server')
        options['outport'] = rcConfig.get('outgoing', 'port')
        options['outencryption'] = rcConfig.get('outgoing', 'encryption')
        options['outaccount'] = rcConfig.get('outgoing', 'account')
        options['outpassword'] = rcConfig.get('outgoing', 'password')
        options['outsender'] = rcConfig.get('outgoing', 'sender')
        options['outreceiver'] = rcConfig.get('outgoing', 'receiver')
    except configparser.NoOptionError as err:
        sys.stderr.write('RC Parse error - No Option: {}\n'.format(
            err.args[0]))
        sys.exit(1)  # Abort program. Can't continue with RC error
    except configparser.NoSectionError as err:
        sys.stderr.write('RC Parse error - No Section: {}\n'.format(
            err.args[0]))
        sys.exit(1)  # Abort program. Can't continue with RC error

    # Now, overlay with command line options
    # Database Path
    if args.dbpath != None:  #dbPath specified on command line
        options['dbpath'] = '{}/{}'.format(args.dbpath, dbName)
    elif options['dbpath'] == '':  # No command line & not specified in RC file
        options['dbpath'] = '{}/{}'.format(get_script_path(), dbName)
    else:  # Path specified in rc file. Add dbname for full path
        options['dbpath'] = '{}/{}'.format(options['dbpath'], dbName)

    # Log file path
    if args.logpath != None:  #logPath specified on command line
        options['logpath'] = '{}/{}'.format(args.logpath, logName)
    elif options[
            'logpath'] == '':  # No command line & not specified in RC file
        options['logpath'] = '{}/{}'.format(get_script_path(), logName)
    else:  # Path specified in rc file. Add dbname for full path
        options['logpath'] = '{}/{}'.format(options['logpath'], logName)

    options['rcpath'] = rcPath

    if args.collect == True:
        options['collect'] = True
    if args.report == True:
        options['report'] = True
    if args.verbose != None:
        options['verbose'] = args.verbose
    if args.append == True:
        options['logappend'] = True
    if args.initdb == True:
        options['initdb'] = True
    if args.mega != None:
        options['sizereduce'] = args.mega

    return
示例#29
0
MFD_DATA_DIR = input_dir + os.sep + parser.get('MONFREDA', 'mon_data_dir')

# MIAMI-LU
miami_lu_nc = input_dir + os.sep + parser.get('MIAMI_LU', 'miami_lu_nc')
miami_npp = input_dir + os.sep + parser.get(
    'MIAMI_LU', 'miami_npp')  # Previous MIAMI-LU NPP estimates
miami_vba = input_dir + os.sep + parser.get(
    'MIAMI_LU', 'miami_vba')  # Previous MIAMI-LU biomass estimates

# HOTSPOTS
file_hotspots = dir_prj + os.sep + parser.get('HOTSPOTS', 'path_hotspots')

# GCAM Wood harvest file
WOOD_HARVEST = dir_prj + os.sep + parser.get('GCAM', 'WOOD_HARVEST')
FERT_DATA = dir_prj + os.sep + parser.get('GCAM', 'FERT_DATA')
GCAM_START_YR = parser.getint('GCAM', 'GCAM_START_YR')
GCAM_END_YR = parser.getint('GCAM', 'GCAM_END_YR')
GCAM_STEP_YR = parser.getint('GCAM', 'GCAM_STEP_YR')
SKIP_GCAM_COLS = parser.getint('GCAM', 'SKIP_GCAM_COLS')
GCAM_MAPPING = dir_prj + os.sep + parser.get('GCAM', 'GCAM_MAPPING')

# CONSTANTS
FILL_VALUE = 1e20
M2_TO_KM2 = 0.000001
M2_TO_HA = 1e-4
KM2_TO_HA = 100.0
IMG_SIZE = 100.0
BILLION = 1e9
KG_TO_PG = 1e-12
KG_TO_TG = 1e-9
KG_TO_MG = 1e-6
示例#30
0
def main():
    def ensure_dir(path):
        """Ensure that path is a directory creating it if necessary.

        If path already exists and is not a directory, print an error
        message and quit with sys.exit().

        Parameters:
          path   String specifying the path to ensure

        Return value:
          path

        """
        if not os.path.exists(path):
            os.makedirs(path)
        elif not os.path.isdir(path):
            print("error: '%s' is not a directory" % path)
            sys.exit(1)
        return path


    def config_default(config, section, option, default):
        """Set default values for options that do not have a value."""
        try:
            config.get(section, option)
        except NoSectionError:
            config.add_section(section)
            config.set(section, option, default)
        except NoOptionError:
            config.set(section, option, default)


    config_dir = ensure_dir(os.path.expanduser("~/.diceware.py"))
    cache_dir = ensure_dir(os.path.join(config_dir, "cache"))

    # Parse config file
    config_file = os.path.join(config_dir, "config")
    config = SafeConfigParser()
    config.read(config_file)

    config_default(config, "defaults", "lang", "en")
    config_default(config, "defaults", "words", "5")
    config_default(config, "defaults", "special", "0")
    config_default(config, "defaults", "file", "")
    config_default(config, "defaults", "separator", " ")

    # Sanity checks for config options
    if config.get("defaults", "lang") not in WORD_LIST_URLS.keys():
        print("error: '%s' is not a valid value for option 'lang'"
              % config.get("defaults", "lang"))
        sys.exit(1)
    try:
        config.getint("defaults", "words")
        config.getint("defaults", "special")
    except ValueError:
        print("error: 'words' and 'special' options must have integer values")
        sys.exit(1)

    # Parse command line arguments
    parser = OptionParser()
    parser.add_option("-g", "--grid", dest="grid", action="store_true",
                      help="Instead of a single line, generate NxN grid of "+
                      "words. This makes eavesdropping harder")
    parser.add_option("-n", "--words", dest="words", type="int", metavar="N",
                      help="generate N words (default: %default)",
                      default=config.getint("defaults", "words"))
    parser.add_option("-s", "--special", dest="special", type="int", metavar="M",
                      help="insert M special characters (default: %default)",
                      default=config.getint("defaults", "special"))
    parser.add_option("-f", "--file", dest="file", metavar="FILE",
                      help="override the `lang' option and read the word list " +
                      "from FILE", default=config.get("defaults", "file"))
    parser.add_option("-p", "--separator", dest="separator", type="string", metavar="P",
                      help="specify the separator between words (default: %default)",
                      default=config.get("defaults", "separator"))
    linguas = sorted(WORD_LIST_URLS.keys())
    parser.add_option("-l", "--lang", dest="lang", metavar="LANG",
                      type="choice", choices=linguas,
                      help="use the word list for LANG (" + ", ".join(linguas) +
                      ") (default: %default)", default=config.get("defaults", "lang"))

    options, args = parser.parse_args()
    if args or options.words < 1 or options.special < 0:
        parser.print_help()
        sys.exit(0)

    parser.destroy()

    # --file has higher precedence than --lang
    if options.file:
        try:
            fobj = open(options.file)
        except IOError:
            print("error: unable to open word list file '%s'" % options.file)
            sys.exit(1)
        try:
            word_list = read_word_list(fobj)
        except ValueError as e:
            print("error: %s" % e)
            sys.exit(1)
    else:
        word_list = get_word_list(cache_dir, options.lang)

    if not options.grid:
        words, with_specials = generate(word_list, options.words,
                                        options.special)
        print("passphrase   : %s" % options.separator.join(words))
        if options.special > 0:
            print("with specials: %s" % options.separator.join(with_specials))
    else:
        words, length = generate_grid(word_list, options.words,
                                            options.special)
        for word_row in words:
            print (" ".join([word.ljust(length) for word in word_row]))
示例#31
0
class ConfigurationCapsule(object):
    """
    Encapsulation class to carry command line args, the logging handler
    and config file information through the parse and transport classes.

    Also handles reading and and validating the config.ini file.
    """
    def __init__(self, options, log, config_path):
        self._options = options
        self._log = log
        self._config = SafeConfigParser(interpolation=EnvInterpolation())
        self._config.read(config_path)
        # Print active configuration if verbose mode
        if options.verbose:
            self._print_current_config()

        self._validate_config(config_path)

    def _validate_config(self, config_path):
        """Validate the config file."""

        # makes sure there is a stanza for the transport that
        # has been selected
        if self.options.transport not in self.config.sections():
            msg = '{t} selected as transport, but no  [{t}] stanza in config {f}'.format(
                t=self.options.transport, f=config_path)
            raise TstatConfigException(msg)

        # make sure we have the universal bare minimum host and port values
        try:
            self.get_cfg_val('host')
            self.get_cfg_val('port')
        except TstatConfigException:
            msg = '"port" and "host" are required in [{t}] config stanza'.format(
                t=self.options.transport)
            raise TstatConfigException(msg)

        # is the hostname valid?
        if not valid_hostname(self.get_cfg_val('host')):
            msg = '{h} is not a valid hostname'.format(h=self.get_cfg_val('host'))
            raise TstatConfigException(msg)

        # is the port an integer?
        try:
            self.get_cfg_val('port', as_int=True)
        except TstatConfigException:
            msg = 'port {0} is not a valid integer'.format(
                self._config.get(self.options.transport, 'port'))
            raise TstatConfigException(msg)

    # pylint: disable=missing-docstring

    def get_cfg_val(self, value, as_int=False, as_bool=False):
        """
        Return a named value from the config file stanza that matches
        the currently selected transport.
        """
        try:
            if as_int:
                return self._config.getint(self.options.transport, value)
            elif as_bool:
                return self._config.getboolean(self.options.transport, value)
            else:
                return self._config.get(self.options.transport, value)
        except ConfigParser.NoOptionError:
            msg = '{0} config value not found'.format(value)
            raise TstatConfigException(msg)
        except ValueError:
            msg = '{0} config value improper type'.format(value)
            raise TstatConfigException(msg)

    def _print_current_config(self):
        for each_section in self._config.sections():
            print("[{section}]".format(section=each_section))
            for (each_key, each_val) in self._config.items(each_section):
                print("{key}={value}".format(key=each_key, value=each_val))

    def _config_stanza_to_dict(self, stanza):
        opts = dict()

        if stanza in self.config.sections() and len(self.config.options(stanza)):
            for i in self.config.options(stanza):
                opts[i] = self.config.get(stanza, i)
            self._log('_config_stanza_to_dict',
                      'stanza [{s}] to dict: {d}'.format(s=stanza, d=opts))

        return opts

    def get_ssl_opts(self):
        return self._config_stanza_to_dict('ssl_options')

    # Some rabbit specific option calls to pass addional kwargs to
    # pika methods.

    def get_rabbit_queue_opts(self):
        return self._config_stanza_to_dict('rabbit_queue_options')

    # expose the internals as properties.

    @property
    def options(self):
        return self._options

    @property
    def log(self):
        return self._log

    @property
    def config(self):
        return self._config
示例#32
0
from rally_task_analysis import RallyTaskAnalysis
import pika
from syslog import syslog, LOG_ERR, LOG_INFO
from configparser import SafeConfigParser
import subprocess
import threading
import time
import json

syslog(LOG_INFO, 'Starting')

# Config
configparser = SafeConfigParser()
try:
    configparser.read('/etc/packer-utils/config.ini')
    THREAD_COUNT = configparser.getint('rabbit2packer', 'THREAD_COUNT')
    if (THREAD_COUNT < 1):
        raise UserWarning(
            'A thread count < 1 is defined, no worker threads will run')
    PACKER_TEMPLATE_MAP = configparser.get('rabbit2packer',
                                           'PACKER_TEMPLATE_MAP')
    LOG_DIR = configparser.get('rabbit2packer', 'LOG_DIR')
    BUILD_FILE_DIR = configparser.get('rabbit2packer', 'BUILD_FILE_DIR')
    PACKER_AUTH_FILE = configparser.get('rabbit2packer', 'PACKER_AUTH_FILE')
    QUEUE = configparser.get('global', 'QUEUE')
    IMAGES_CONFIG = configparser.get('rabbit2packer', 'IMAGES_CONFIG')
    RABBIT_HOST = configparser.get('global', 'RABBIT_HOST')
    RABBIT_PORT = configparser.getint('global', 'RABBIT_PORT')
    RABBIT_USER = configparser.get('global', 'RABBIT_USER')
    RABBIT_PW = configparser.get('global', 'RABBIT_PW')
except Exception as e:
class ChannelParameters(object):
    """
    Loads device channel settings and parameters from an INI file.
    """
    def __init__(self, ini_file):
        self.log = logging.getLogger(".".join([__name__, self.__class__.__name__]))
        self._ini_filename = find_file(ini_file)
        self.conf = None

        self._control_channels = None
        self._monitoring_channels = None

    def load_ini(self):
        """
        Loads and parses the data from INI file. The data is stored internally in the object and can be retrieved
        through the `self.control_channels` and `self.monitoring_channels` properties.
        """
        self._control_channels = None
        self._monitoring_channels = None
        self.conf = SafeConfigParser(dict_type=OrderedDict)
        self.conf.read(self._ini_filename)
        self.log.debug("Read Board Parameters INI file %s:", self._ini_filename)
        self.log.debug("    sections: %s", self.conf.sections())

    @staticmethod
    def _get_channel_number(section_name):
        match = re.match(r'^.*_channel<(\d{4})>$', section_name)
        if match:
            result = int(match.group(1))
            return result
        else:
            raise_with_traceback(RuntimeError("Unable to detect channel name from \"%s\""%section_name))

    def _get_channel_matching(self, regexp):
        sections_matching = []
        for section in self.conf.sections():
            if regexp.match(section):
                sections_matching.append(section)
        return sections_matching

    def _get_channel_name_by_address(self, uart_address, channels):
        for ch in channels:
            if ch.UART_address == uart_address:
                return ch.Channel_name

    def _get_channel_name_by_index(self, index, channels):
        for ch in channels:
            if ch.channel_index == index:
                name = ch.Channel_name
                if name is None or len(name) == 0:
                    name = ch.ini_section
                return name

    def _get_channel_name_by_id_and_board_type(self, channel_id, board_type, channels):
        for ch in channels:
            if ch.Channel_ID == channel_id and BoardTypes(ch.Board_type) == board_type:
                name = ch.Channel_name
                if name is None or len(name) == 0:
                    name = ch.ini_section
                return name

    def _get_channel_by_address(self, uart_address, channels):
        for ch in channels:
            if ch.UART_address == uart_address:
                return ch

    def monitoring_channel_name_by_index(self, index):
        return self._get_channel_name_by_index(index, self.monitoring_channels)

    def monitoring_channel_name_by_id_and_board_type(self, channel_id, board_type):
        return self._get_channel_name_by_id_and_board_type(channel_id, board_type, self.monitoring_channels)

    def control_channel_name_by_index(self, index):
        return self._get_channel_name_by_index(index, self.control_channels)

    def monitoring_channel_name(self, uart_address):
        return self._get_channel_name_by_address(uart_address, self.monitoring_channels)

    def control_channel_name(self, uart_address):
        return self._get_channel_name_by_address(uart_address, self.control_channels)

    def control_channel_by_address(self, uart_address):
        return self._get_channel_by_address(uart_address, self.control_channels)

    def monitoring_channel_by_address(self, uart_address):
        return self._get_channel_by_address(uart_address, self.monitoring_channels)

    def control_channels_by_name(self, name):
        return self._get_channels_by_name(self.control_channels, name)

    def monitoring_channels_by_name(self, name):
        return self._get_channels_by_name(self.monitoring_channels, name)

    @property
    def control_channels(self):
        """
        List of `ControlChannelIniParameters`
        """
        if self._control_channels:
            return self._control_channels
        self._control_channels = self._get_channels(ControlChannelIniParameters)
        return self._control_channels

    @property
    def monitoring_channels(self):
        """
        List of `MonitoringChannelIniParameters`
        """
        if self._monitoring_channels:
            return self._monitoring_channels
        self._monitoring_channels = self._get_channels(MonitoringChannelIniParameters)
        return self._monitoring_channels

    def _get_channels(self, channel_class):
        """
        Loop through all channel sections matching a certain type `channel_class` and parse the parameters of each
        section into new `channel_class` instance objects.

        :param channel_class: an :class:`IniSectionParameters` derivative class
        :return: a list of instances of the channel_class :obj:`IniSectionParameters` derivative.
        :rtype list:
        """
        channels = []
        sections = self._get_channel_matching(channel_class.section_regexp)
        for section in sections:
            channel_number = self._get_channel_number(section)
            channel = channel_class(channel_number)
            channel.ini_section = section
            for param in channel.parameters():
                parameter_type = channel.get_type(param)
                if parameter_type == int:
                    value = self.conf.getint(section, param)
                elif parameter_type == str:
                    value = self.conf.get(section, param)
                    value = str(value.strip("\""))  # Get rid of any double quotes from the ini file
                elif parameter_type == bool:
                    str_value = self.conf.get(section, param)
                    if str_value.lower() in positive_configuration:
                        value = True
                    elif str_value.lower() in negative_configuration:
                        value = False
                    else:
                        raise_with_traceback(TypeError("Unsupported boolean: %s = \"%s\""%(param, str_value)))
                else:
                    raise_with_traceback(TypeError("Unsupported parameter type %s"%str(parameter_type)))
                channel.__setattr__(param, value)
            self.log.debug("Appending channel: %s", channel)
            channels.append(channel)
        return channels

    @staticmethod
    def _get_channels_by_name(channel_list, name):
        result = []
        for ch in channel_list:
            if re.match(name, ch.Channel_name):
                result.append(ch)
        if len(result) == 1:
            result = result[0]
        return result

    def __str__(self):
        channels = "Control channels: "
        if self._control_channels:
            channels += str(len(self._control_channels))
        else:
            channels += "[]"
        channels += " Monitoring channels: "
        if self._monitoring_channels:
            channels += str(len(self._monitoring_channels))
        else:
            channels += "[]"
        s = "<%s: inifile: %s %s>"%(self.__class__.__name__, self._ini_filename, channels)
        return s

    def __repr__(self):
        return self.__str__()
示例#34
0
文件: submit.py 项目: msyriac/alhazen
import os
import sys
import numpy as np
import time
from shutil import copyfile
from configparser import SafeConfigParser 


timestamp = str(time.time())
iniFile = "output/submission"+timestamp+".ini"
copyfile("input/submission.ini",iniFile) 

Config = SafeConfigParser()
Config.optionxform=str
Config.read(iniFile)

numJobs = Config.getint('jobs','numJobs')
numClusters = Config.getint('jobs','numClusters')
snapRange = [int(x) for x in Config.get('jobs','snapRange').split(',')]
saveName = Config.get('jobs','saveName')

for snap in range(snapRange[0],snapRange[1]):
    for i in range(numJobs):

        cmd = "quick_wq.sh python bin/lensRecon.py "+str(i)+" "+str(numClusters)+" "+str(snap)+" "+saveName+" & "
        print(cmd)
        os.system(cmd)
        time.sleep(0.5)
示例#35
0
class TestPlanParser(object):
    
    """WRTM Test Plan Parser
    
       Accepts INI style files describing tests in separate sections, with each test 
       described in a separate line of a 'plan' option of a section. Each such line 
       has a format of:

           test-id interface-identifier test-length address/offset mask

       Address/offset can either be an immediate value or a difference from the last
       returned value (in the form of +x or -x).

       Additionally the 'main' section specifies the address of the device under test 
       ('dut') and the address of the N2X probe ('n2x'). Comments are allowed in separate 
       lines starting with a # (hashtag) or a ; (semicolon).
    """

    def __init__(self):
        self.parser = SafeConfigParser()
        self.loaded = False
        self.loadedPath = None
        self.testTypes = {}

    def load(self, path):
        if len(self.parser.read(path)) < 1:
            raise RuntimeError("Unable to load the specified file.")
        self.loaded = True
        self.loadedPath = path

    def loadTestTypes(self, fileName):
        testsFile = open(fileName)
        for testType in testsFile:
            splitarray = testType.split(' ')
            testId = int(splitarray[0])
            testContinuous = int(splitarray[1])
            testDescription = ' '.join(splitarray[2:])
            self.testTypes[testId] = (testContinuous, testDescription)

    def sections(self):
        return self.parser.sections()

    def getListOfTests(self):
        if self.loaded:
            return self.parser.sections()
        return None

    def getNumberOfTestCases(self, name):
        if self.parser.has_option(name, 'loop'):
            loops = self.parser.getint(name, 'loop')
        else:
            loops = 1

        return len(self.parser[name]['plan'].split('\n')[1:]) * loops

    def getTestGenerator(self, name=None):
        if self.parser.has_option(name, 'loop'):
            loops = self.parser.getint(name, 'loop')
        else:
            loops = 1

        if self.parser.has_option(name, 'offsetStep'):
            offsetStep = self.parser.getint(name, 'offsetStep')
        else:
            offsetStep = 0

        tests = self.parser[name]['plan'].split('\n')[1:]
        offset = 0

        for it in range(0, loops):
            for i in range(0, len(tests)):
                testTuple = tests[i].split(' ')
                if testTuple[3][0] == '+' or testTuple[3][0] == '-':
                    offset += int(testTuple[3][0])
                else:
                    offset = int(testTuple[3][0]) + it * offsetStep
                
                testTuple[3] = offset

                yield self.parseTestLine(i+1, testTuple)
    
        return None
    
    def parseTestLine(self, testIter, testTuple):
        return [testIter,
                int(testTuple[0]),  # testId 
                testTuple[1],       # interfaceName
                int(testTuple[2]),  # testDuration
                int(testTuple[3]),  # address/offset
                int(testTuple[4])]  # mask
示例#36
0
            'Files and Folders will be visible from root folder: "{}"'.format(
                nwd))
        chdir(
            nwd
        )  # If files are to be served, change the working directory to the document root folder
    meters = dict()
    LoadSettings()  # Initial load of definition file
    Shutdown = Event()  # The flag which will shutdown the auto-updating thread
    AutoPoll = Thread(
        target=RegularUpdate,
        args=(meters, config.getfloat('DEFAULT', 'autopollsec'), Shutdown),
        name='AutoPollThread',
        daemon=True)  # Daemon Thread to auto-update certain items
    AutoPoll.start()
    httpd = ThreadingTCPServer(
        (config.get('DEFAULT', 'httphost'), config.getint(
            'DEFAULT', 'httpport')), CustomHandler)  # Start the HTTP Server
    print('Server Running "{}:{}"'.format(config.get('DEFAULT', 'httphost'),
                                          config.get('DEFAULT', 'httpport')))
    print('To shut down, visit "/command?{}"'.format(
        config.get('DEFAULT', 'shutdowncmd')))
    signal.signal(signal.SIGTERM, Killer)
    try:
        httpd.serve_forever()
    except KeyboardInterrupt:  # Allow Ctrl+C locally to close it gracefully
        Killer(0, None)  # Envoke the signal handler
    httpd.server_close()  # Finally close everything off
    chdir(
        cwd
    )  # Change the working directory back to what it was when it started.
    raise SystemExit  # Ensure explicit termination at this point
示例#37
0
from http.server import HTTPServer, CGIHTTPRequestHandler
from configparser import SafeConfigParser

if __name__ == '__main__':
    config = SafeConfigParser()
    config.read("server.ini")
    server_address = (config.get("server","url"),config.getint("server","port"))
    httpd = HTTPServer(server_address,CGIHTTPRequestHandler)
    httpd.serve_forever()
示例#38
0
    pyplot.title('Power delivered to ADC')
    pyplot.ylabel('Power Normalized Magnetude [dBFS]')
    pyplot.xlabel('Sampling Frequency [Hz]')
    return pyplot

######################################################################
# Main script

config = SafeConfigParser()
config.read('config.ini')

data_dir = config.get('Test','data_dir')
data_file_name = config.get('Test','data_file_name')
power_file_name = config.get('Test','power_file_name')
power_plot_name = config.get('Test','power_plot_name')
num_samples = config.getint('Test','num_samples')
fs = config.getfloat('Test','fs') # sampling frequency in Hz
adc_resolution_bits = config.getint('Test','adc_resolution_bits')

sys.stdout.write("\nRunning test...\n\n")

# Initialize arrays that will receive data in each loop cycle
fund_power_array = []
fsig_array = []


# Load power and frequency data
data = numpy.loadtxt(data_dir + power_file_name, delimiter = ',', skiprows = 1)
fsig_array = [d[0] for d in data]
fund_power_array = [d[1] for d in data]
示例#39
0
class GlobalSettings(GObject.Object):
    """
    Pitivi app settings.

    The settings object loads settings from different sources, currently:
    - the local configuration file,
    - environment variables.

    Modules declare which settings they wish to access by calling the
    addConfigOption() class method during initialization.

    @cvar options: A dictionnary of available settings.
    @cvar environment: A list of the controlled environment variables.
    """

    options = {}
    environment = set()
    defaults = {}

    __gsignals__ = {}

    def __init__(self, **unused_kwargs):
        GObject.Object.__init__(self)
        self._config = SafeConfigParser()
        self._readSettingsFromConfigurationFile()
        self._readSettingsFromEnvironmentVariables()

    def _readSettingsFromConfigurationFile(self):
        """
        Read the configuration from the user configuration file.
        """

        try:
            conf_file_path = os.path.join(xdg_config_home(), "pitivi.conf")
            self._config.read(conf_file_path)
        except UnicodeDecodeError:
            unicode_error_dialog()
            return
        except ParsingError:
            return

        for (section, attrname, typ, key, env, value) in self.iterAllOptions():
            if not self._config.has_section(section):
                continue
            if key and self._config.has_option(section, key):
                if typ == int or typ == int:
                    try:
                        value = self._config.getint(section, key)
                    except ValueError:
                        # In previous configurations we incorrectly stored
                        # ints using float values.
                        value = int(self._config.getfloat(section, key))
                elif typ == float:
                    value = self._config.getfloat(section, key)
                elif typ == bool:
                    value = self._config.getboolean(section, key)
                else:
                    value = self._config.get(section, key)
                setattr(self, attrname, value)

    @classmethod
    def readSettingSectionFromFile(self, cls, section):
        """
        Force reading a particular section of the settings file.

        Use this if you dynamically determine settings sections/keys at runtime
        (like in tabsmanager.py). Otherwise, the settings file would be read
        only once (at the initialization phase of your module) and your config
        sections would never be read, and thus values would be reset to defaults
        on every startup because GlobalSettings would think they don't exist.
        """
        if cls._config.has_section(section):
            for option in cls._config.options(section):
                # We don't know the value type in advance, just try them all.
                try:
                    value = cls._config.getfloat(section, option)
                except:
                    try:
                        value = cls._config.getint(section, option)
                    except:
                        try:
                            value = cls._config.getboolean(section, option)
                        except:
                            value = cls._config.get(section, option)

                setattr(cls, section + option, value)

    def _readSettingsFromEnvironmentVariables(self):
        """
        Override options values using their registered environment variables.
        """
        for section, attrname, typ, key, env, value in self.iterAllOptions():
            if not env:
                # This option does not have an environment variable name.
                continue
            var = get_env_by_type(typ, env)
            if var is not None:
                setattr(self, attrname, var)

    def _writeSettingsToConfigurationFile(self):
        conf_file_path = os.path.join(xdg_config_home(), "pitivi.conf")

        for (section, attrname, typ, key, env_var,
             value) in self.iterAllOptions():
            if not self._config.has_section(section):
                self._config.add_section(section)
            if key:
                if value is not None:
                    self._config.set(section, key, str(value))
                else:
                    self._config.remove_option(section, key)
        try:
            file = open(conf_file_path, 'w')
        except IOError as OSError:
            return
        self._config.write(file)
        file.close()

    def storeSettings(self):
        """
        Write settings to the user's local configuration file. Note that only
        those settings which were added with a section and a key value are
        stored.
        """
        self._writeSettingsToConfigurationFile()

    def iterAllOptions(self):
        """
        Iterate over all registered options

        @return: an iterator which yields a tuple of (attrname, type, key,
        environment, value) for each option.
        """
        for section, options in list(self.options.items()):
            for attrname, (typ, key, environment) in list(options.items()):
                yield section, attrname, typ, key, environment, getattr(
                    self, attrname)

    def isDefault(self, attrname):
        return getattr(self, attrname) == self.defaults[attrname]

    def setDefault(self, attrname):
        setattr(self, attrname, self.defaults[attrname])

    @classmethod
    def addConfigOption(
        cls,
        attrname,
        type_=None,
        section=None,
        key=None,
        environment=None,
        default=None,
        notify=False,
    ):
        """
        Add a configuration option.

        This function should be called during module initialization, before
        the config file is actually read. By default, only options registered
        beforehand will be loaded.
        See mainwindow.py and medialibrary.py for examples of usage.

        If you want to add configuration options after initialization,
        use the readSettingSectionFromFile method to force reading later on.
        See tabsmanager.py for an example of such a scenario.

        @param attrname: the attribute of this class which represents the option
        @type attrname: C{str}
        @param type_: type of the attribute. Unnecessary if default is given.
        @type type_: a builtin or class
        @param section: The section of the config file under which this option is
        saved. This section must have been added with addConfigSection(). Not
        necessary if key is not given.
        @param key: the key under which this option is to be saved. Can be none if
        this option should not be saved.
        @type key: C{str}
        @param notify: whether or not this attribute should emit notification
        signals when modified (default is False).
        @type notify: C{boolean}
        """
        if section and section not in cls.options:
            raise ConfigError("You must add the section \"%s\" first." %
                              section)
        if key and not section:
            raise ConfigError("You must specify a section for key \"%s\"" %
                              key)
        if section and key in cls.options[section]:
            raise ConfigError("Option \"%s\" is already in use.")
        if hasattr(cls, attrname):
            raise ConfigError("Settings attribute \"%s\" is already in use.")
        if environment and environment in cls.environment:
            raise ConfigError("Settings environment varaible \"%s\" is"
                              "already in use.")
        if not type_ and default is None:
            raise ConfigError("Settings attribute \"%s\" has must have a"
                              " type or a default." % attrname)
        if not type_:
            type_ = type(default)
        if notify:
            notification = Notification(attrname)
            setattr(cls, attrname, notification)
            setattr(cls, "_" + attrname, default)
            GObject.signal_new(notification.signame, cls,
                               GObject.SIGNAL_RUN_LAST, None, ())
        else:
            setattr(cls, attrname, default)
        if section and key:
            cls.options[section][attrname] = type_, key, environment
        cls.environment.add(environment)
        cls.defaults[attrname] = default

    @classmethod
    def addConfigSection(cls, section):
        """
        Add a section to the local config file.

        @param section: The section name. This section must not already exist.
        @type section: C{str}
        """
        if section in cls.options:
            raise ConfigError("Duplicate Section \"%s\"." % section)
        cls.options[section] = {}

    @classmethod
    def notifiesConfigOption(cls, attrname):
        signal_name = Notification.signalName(attrname)
        GObject.signal_lookup(signal_name, cls)
示例#40
0
# every 60 minutes, start delay = 30 seconds
CONF_DEFAULTS = {
    'update_interval': '60',
    'update_startup_delay': '30',
    'autocheck_updates': '0',
    'update_notify': True,
    'update_showicon': True
}

CONFIG = SafeConfigParser(CONF_DEFAULTS)
if not CONFIG.has_section('yumex'):
    CONFIG.add_section('yumex')
if os.path.exists(CONF_FILE):
    CONFIG.read(CONF_FILE)

TIMER_STARTUP_DELAY = CONFIG.getint('yumex', 'update_startup_delay')
UPDATE_INTERVAL = CONFIG.getint('yumex', 'update_interval')
AUTOCHECK_UPDATE = CONFIG.getboolean('yumex', 'autocheck_updates')
NOTIFY = CONFIG.getboolean('yumex', 'update_notify')
SHOWICON = CONFIG.getboolean('yumex', 'update_showicon')


def check_pid(pid):
    """ Check For the existence of a unix pid. """
    try:
        os.kill(pid, 0)
    except OSError:
        return False
    else:
        return True
示例#41
0
	def open(self):
		self.noInput = False
		try:
			self.file = open("/dev/kbdscan","r")
		except:
			self.noInput = True					# No KbdScan device, so use the regular stdin keyboard
			self.kb = KBHit()
		try:
			from configparser import SafeConfigParser
		except ImportError:
			from ConfigParser import SafeConfigParser
		config = SafeConfigParser()
		config.read("config/defaults.cfg")
		config.read("~/.config/mcdu.cfg")
		config.read("config/mcdu.cfg")
		self.keymap = {}
		self.keymap[config.getint("KeyMap", "KEY_CLR")] = "CLR"
		self.keymap[config.getint("KeyMap", "KEY_DEL")] = "DEL"
		self.keymap[config.getint("KeyMap", "KEY_DIR")] = "DIR"
		self.keymap[config.getint("KeyMap", "KEY_PROG")] = "PROG"
		self.keymap[config.getint("KeyMap", "KEY_PERF")] = "PERF"
		self.keymap[config.getint("KeyMap", "KEY_INIT")] = "INIT"
		self.keymap[config.getint("KeyMap", "KEY_DATA")] = "DATA"
		self.keymap[config.getint("KeyMap", "KEY_F_PLN")] = "F_PLN"
		self.keymap[config.getint("KeyMap", "KEY_RAD_NAV")] = "RAD_NAV"
		self.keymap[config.getint("KeyMap", "KEY_FUEL_PRED")] = "FUEL_PRED"
		self.keymap[config.getint("KeyMap", "KEY_SEC_F_PLN")] = "SEC_F_PLN"
		self.keymap[config.getint("KeyMap", "KEY_ATC_COMM")] = "ATC_COMM"
		self.keymap[config.getint("KeyMap", "KEY_MENU")] = "MENU"
		self.keymap[config.getint("KeyMap", "KEY_DIM")] = "DIM"
		self.keymap[config.getint("KeyMap", "KEY_AIRPORT")] = "AIRPORT"
		self.keymap[config.getint("KeyMap", "KEY_PAGE_UP")] = "PAGE_UP"
		self.keymap[config.getint("KeyMap", "KEY_NEXT_PAGE")] = "NEXT_PAGE"
		self.keymap[config.getint("KeyMap", "KEY_PAGE_DN")] = "PAGE_DN"
		self.keymap[config.getint("KeyMap", "KEY_LSK1L"  )] = "LSK0L"
		self.keymap[config.getint("KeyMap", "KEY_LSK2L"  )] = "LSK1L"
		self.keymap[config.getint("KeyMap", "KEY_LSK3L"  )] = "LSK2L"
		self.keymap[config.getint("KeyMap", "KEY_LSK4L"  )] = "LSK3L"
		self.keymap[config.getint("KeyMap", "KEY_LSK5L"  )] = "LSK4L"
		self.keymap[config.getint("KeyMap", "KEY_LSK6L"  )] = "LSK5L"
		self.keymap[config.getint("KeyMap", "KEY_LSK1R"  )] = "LSK0R"
		self.keymap[config.getint("KeyMap", "KEY_LSK2R"  )] = "LSK1R"
		self.keymap[config.getint("KeyMap", "KEY_LSK3R"  )] = "LSK2R"
		self.keymap[config.getint("KeyMap", "KEY_LSK4R"  )] = "LSK3R"
		self.keymap[config.getint("KeyMap", "KEY_LSK5R"  )] = "LSK4R"
		self.keymap[config.getint("KeyMap", "KEY_LSK6R"  )] = "LSK5R"
示例#42
0
from urllib.error import URLError, HTTPError
from configparser import SafeConfigParser

syslog(LOG_INFO, 'Starting')

# Config
configparser = SafeConfigParser()
try:
    configparser.read('/etc/packer-utils/config.ini')
    PROFILE_INFO_URL = configparser.get('cdb2rabbit', 'PROFILE_INFO_URL')
    PROFILE_DIR_URL = configparser.get('cdb2rabbit', 'PROFILE_DIR_URL')
    PROFILE_MATCH = configparser.get('cdb2rabbit', 'PROFILE_MATCH')
    CACHE_DIR = configparser.get('cdb2rabbit', 'CACHE_DIR')
    QUEUE = configparser.get('global', 'QUEUE')
    RABBIT_HOST = configparser.get('global', 'RABBIT_HOST')
    RABBIT_PORT = configparser.getint('global', 'RABBIT_PORT')
    RABBIT_USER = configparser.get('global', 'RABBIT_USER')
    RABBIT_PW = configparser.get('global', 'RABBIT_PW')
except Exception as e:
    syslog(LOG_ERR, 'Unable to read from config file')
    syslog(LOG_ERR, repr(e))
    sys.exit(1)


def updateCachedFile(file_name, contents):
    try:
        with open(file_name, "wt") as file:
            file.write(contents)
    except IOError as e:
        syslog(LOG_ERR, "Unable to write profile info to file %s" % file_name)
        syslog(LOG_ERR, repr(e))
示例#43
0
文件: model.py 项目: sahg/PyTOPKAPI
def run(ini_file='TOPKAPI.ini',
        verbose=False, quiet=False,
        parallel_exec=True, nworkers=int(mp.cpu_count()-1)):
    """Run the model.

    Parameters
    ----------
    ini_file : str
       The name of the PyTOPKAPI initialization file. This file describes the
       locations of the parameter files and model setup options. Default is to
       use a file named `TOPKAPI.ini` in the current directory.
    verbose : bool
        Prints runtime information [default False - don't display runtime
        info]. Is independent of the `quiet` keyword argument.
    quiet : bool
        Toggles whether to display an informational banner at runtime [default
        False - display banner]. Is independent of the `verbose` keyword
        argument.
    nworkers : int
        Number of worker processes to spawn for solving each cell's time-series
        in parallel. Default is one fewer than CPU count reported by
        multiprocessing.

    """

    ##================================##
    ##  Read the input file (*.ini)   ##
    ##================================##
    config = SafeConfigParser()
    config.read(ini_file)

    ##~~~~~~ Numerical_options ~~~~~~##
    solve_s = config.getint('numerical_options', 'solve_s')
    solve_o = config.getint('numerical_options', 'solve_o')
    solve_c = config.getint('numerical_options', 'solve_c')

    ##~~~~~~~~~~~ input files ~~~~~~~~~~~##
    #Param
    file_global_param = config.get('input_files', 'file_global_param')
    file_cell_param = config.get('input_files', 'file_cell_param')
    #Rain
    file_rain = config.get('input_files', 'file_rain')
    #ETP
    file_ET = config.get('input_files', 'file_ET')

    #~~~~~~~~~~~ Group (simulated event) ~~~~~~~~~~~##
    group_name = config.get('groups', 'group_name')

    ##~~~~~~ Calibration ~~~~~~##
    fac_L = config.getfloat('calib_params', 'fac_L')
    fac_Ks = config.getfloat('calib_params', 'fac_Ks')
    fac_n_o = config.getfloat('calib_params', 'fac_n_o')
    fac_n_c = config.getfloat('calib_params', 'fac_n_c')

    ##~~~~~~ External flows ~~~~~~##
    external_flow = config.getboolean('external_flow', 'external_flow')
    if external_flow:
        file_Qexternal_flow = config.get('external_flow',
                                         'file_Qexternal_flow')
        Xexternal_flow = config.getfloat('external_flow', 'Xexternal_flow')
        Yexternal_flow = config.getfloat('external_flow', 'Yexternal_flow')

    ##~~~~~~~~~~~ output files ~~~~~~~~~~##
    file_out = config.get('output_files', 'file_out')
    ut.check_file_exist(file_out) #create path_out if it doesn't exist
    if os.path.exists(file_out):
        first_run = False
    else:
        first_run = True

    append_output = config.getboolean('output_files', 'append_output')
    if append_output is True:
        fmode = 'a'
    else:
        fmode = 'w'

    ##============================##
    ##   Read the forcing data    ##
    ##============================##
    if verbose:
        print('Read the forcing data')

    #~~~~Rainfall
    h5_rain = h5py.File(file_rain)
    dset_name = '/{}/rainfall'.format(group_name)
    rainfall_forcing = h5_rain[dset_name][...]
    h5_rain.close()

    #~~~~ETr - Reference crop ET
    h5_ET = h5py.File(file_ET)
    dset_name = '/{}/ETr'.format(group_name)
    ETr_forcing = h5_ET[dset_name][...]

    #~~~~ETo - Open water potential evap.
    dset_name = '/{}/ETo'.format(group_name)
    ET0_forcing = h5_ET[dset_name][...]
    h5_ET.close()

    #~~~~external_flow flows
    if external_flow:
        external_flow_records = np.loadtxt(file_Qexternal_flow)[:, 5]
    else:
        external_flow_records = None


    ##============================##
    ## Pretreatment of input data ##
    ##============================##
    if verbose:
        print('Pretreatment of input data')

    #~~~~Read Global parameters file
    X, Dt, alpha_s, \
    alpha_o, alpha_c, \
    A_thres, W_min, W_max = pm.read_global_parameters(file_global_param)

    #~~~~Read Cell parameters file
    ar_cell_label, ar_coorx, \
    ar_coory, channel_flag, \
    Xc, ar_dam, \
    ar_tan_beta, ar_tan_beta_channel, \
    ar_L, Ks, \
    ar_theta_r, ar_theta_s, \
    ar_n_o, ar_n_c, \
    ar_cell_down, ar_pVs_t0, \
    ar_Vo_t0, ar_Qc_t0, \
    Kc, psi_b, lamda = pm.read_cell_parameters(file_cell_param)

    #~~~~Number of cell in the catchment
    nb_cell = len(ar_cell_label)

    #~~~~Computation of cell order
    node_hierarchy = pm.compute_node_hierarchy(ar_cell_label, ar_cell_down)
    ar_label_sort = pm.sort_cell(ar_cell_label, ar_cell_down)

    #~~~~Computation of upcells
    li_cell_up = pm.direct_up_cell(ar_cell_label, ar_cell_down, ar_label_sort)

    #~~~~Computation of drained area
    ar_A_drained = pm.drained_area(ar_label_sort, li_cell_up, X)

    #~~~~Apply calibration factors to the parameter values
    ar_L = ar_L*fac_L
    Ks = Ks*fac_Ks
    ar_n_o = ar_n_o*fac_n_o
    ar_n_c = ar_n_c*fac_n_c

    if verbose:
        print('Max L=', max(ar_L))
        print('Max Ks=', max(Ks))
        print('Max n_o=', max(ar_n_o))
        print('Max n_c=', max(ar_n_c))

    #~~~~Computation of model parameters from physical parameters
    Vsm, b_s, b_o, \
    W, b_c = pm.compute_cell_param(X, Xc, Dt, alpha_s,
                                         alpha_o, alpha_c, nb_cell,
                                         A_thres, W_max, W_min,
                                         channel_flag, ar_tan_beta,
                                         ar_tan_beta_channel, ar_L,
                                         Ks, ar_theta_r, ar_theta_s,
                                         ar_n_o, ar_n_c, ar_A_drained)

    #~~~~Look for the cell of external_flow tunnel
    if external_flow:
        cell_external_flow = ut.find_cell_coordinates(ar_cell_label,
                                                      Xexternal_flow,
                                                      Yexternal_flow,
                                                      ar_coorx,
                                                      ar_coory,
                                                      channel_flag)

        if verbose:
            print('external flows will be taken into account for cell no',\
                  cell_external_flow, ' coordinates ('\
                  ,Xexternal_flow,',',Yexternal_flow,')')
    else:
        cell_external_flow = None

    #~~~~Number of simulation time steps
    nb_time_step = rainfall_forcing.shape[0]


    ##=============================##
    ##  Variable array definition  ##
    ##=============================##

    ## Initialisation of the reservoirs
    #Matrix of soil,overland and channel store at the begining of the time step
    if append_output and not first_run:
        if verbose:
            print('Initialize from simulation file')

        h5file_in = h5py.File(file_out)

        Vs_t0 = h5file_in['/Soil/V_s'][-1, :]
        Vc_t0 = h5file_in['/Channel/V_c'][-1, :]
        Vo_t0 = h5file_in['/Overland/V_o'][-1, :]

        h5file_in.close()
    else:
        if verbose:
            print('Initialize from parameters')
        Vs_t0 = fl.initial_volume_soil(ar_pVs_t0, Vsm)
        Vo_t0 = ar_Vo_t0
        Vc_t0 = fl.initial_volume_channel(ar_Qc_t0, W, X, ar_n_c)

    ##=============================##
    ## HDF5 output file definition ##
    ##=============================##
    h5file, dset_Vs, dset_Vo, dset_Vc,     \
    dset_Qs_out, dset_Qo_out, dset_Qc_out, \
    dset_Q_down, dset_ET_out, dset_Ec_out  \
                                    = ut.open_simulation_file(file_out, fmode,
                                                   Vs_t0, Vo_t0, Vc_t0, no_data,
                                                   nb_cell, nb_time_step,
                                                   append_output, first_run)

    eff_theta = ar_theta_s - ar_theta_r

    ##===========================##
    ##     Core of the Model     ##
    ##===========================##
    if not quiet:
        ut.show_banner(ini_file, nb_cell, nb_time_step)
        progress_desc = 'Simulation'
    else:
        progress_desc = 'PyTOPKAPI v{}'.format(pytopkapi.__version__)

    # prepare parameter dict
    exec_params = {'nb_cell': nb_cell,
                   'nb_time_step': nb_time_step,
                   'progress_desc': progress_desc,
                   'Dt': Dt,
                   'rainfall_forcing': rainfall_forcing,
                   'ETr_forcing': ETr_forcing,
                   'ET0_forcing': ET0_forcing,
                   'psi_b': psi_b,
                   'lamda': lamda,
                   'eff_theta': eff_theta,
                   'Ks': Ks,
                   'X': X,
                   'b_s': b_s,
                   'b_o': b_o,
                   'b_c': b_c,
                   'alpha_s': alpha_s,
                   'alpha_o': alpha_o,
                   'alpha_c': alpha_c,
                   'Vs_t0': Vs_t0,
                   'Vo_t0': Vo_t0,
                   'Vc_t0': Vc_t0,
                   'Vsm': Vsm,
                   'dset_Vs': dset_Vs,
                   'dset_Vo': dset_Vo,
                   'dset_Vc': dset_Vc,
                   'dset_Qs_out': dset_Qs_out,
                   'dset_Qo_out': dset_Qo_out,
                   'dset_Qc_out': dset_Qc_out,
                   'dset_Q_down': dset_Q_down,
                   'dset_ET_out': dset_ET_out,
                   'dset_Ec_out': dset_Ec_out,
                   'solve_s': solve_s,
                   'solve_o': solve_o,
                   'solve_c': solve_c,
                   'channel_flag': channel_flag,
                   'W': W,
                   'Xc': Xc,
                   'Kc': Kc,
                   'cell_external_flow': cell_external_flow,
                   'external_flow_records': external_flow_records,
                   'node_hierarchy': node_hierarchy,
                   'li_cell_up': li_cell_up,
                   'nworkers': nworkers}

    if not parallel_exec:
        # Serial execution. Solve by timestep in a single process.
        # Outer loop timesteps - inner loop cells
        _serial_execute(exec_params)
    else:
        # Parallel execution. Solve by cell using multiple processes.
        # Outer loop cells - inner loop timesteps
        _parallel_execute(exec_params)

    h5file.close()
示例#44
0
class marvelous_config:
    def __init__(self, filename=None, cdict=None):
        self.config = SafeConfigParser()
        if filename is not None:
            self.filename = filename
        else:
            self.filename = os.path.join('.', 'config.ini')

        if os.path.isfile(self.filename):
            self.config.read(self.filename)
        else:
            self.config.add_section('general')
            self.config.add_section('daligner')

        if cdict is not None:
            self.set_dict(cdict)

        self.save()

    def get(self, section, key, default=None):
        if not self.config.has_option(section, key):
            raise KeyError('key {}:{} does not exist'.format(section, key))
        value = self.config.get(section, key, fallback=default)
        if value == 'None':
            return None
        return value

    def getboolean(self, section, key, default=None):
        return self.config.getboolean(section, key, fallback=default)

    def getint(self, section, key, default=None):
        if self.get(section, key, default) is None:
            return None
        return self.config.getint(section, key, fallback=default)

    def getfloat(self, section, key, default=None):
        return self.config.getfloat(section, key, fallback=default)

    def set_dict(self, cdict):
        for section, c in cdict.items():
            if section not in self.config.sections():
                self.config.add_section(section)
            for key, value in c.items():
                self.config.set(section, key, str(value))

    def set(self, section, key, value):
        """Set a config value.

        If `section` does not yet exist, it is created.

        Parameters
        ----------
        section : str
            Name of the section to set the key in.
        key : str
            Name of the key to set the value for.
        value : object
            The value to set for the key. This object is
            saved as `str(value)`.
        """
        if not self.config.has_section(section):
            self.config.add_section(section)
        self.config.set(section, key, str(value))
        self.save()

    def update(self, section, key, value, default=None):
        """Update a config value.

        If `value` is `None` and the key has not been set to
        a non-`None` value already, then the key is set to
        the default value. If `value` is not `None` the key
        will get this value. If neither of these cases apply
        it means the key already has a value and that `value`
        is `None`, and nothing will changed.

        Note that if the section and/or key does not exist,
        it will be created with an inital value of `None`.

        Parameters
        ----------
        section : str
            Name of the section to set the key in.
        key : str
            Name of the key to set the value for.
        value : object
            The value to set for the key. This object is
            saved as `str(value)`.
        default : object, optional
            The value to set if `value` is `None`. This object
            is saved as `str(value)`.
        """
        if not self.config.has_section(section):
            self.config.add_section(section)
        if not self.config.has_option(section, key):
            self.set(section, key, None)
        if value is None and self.get(section, key) is None:
            self.set(section, key, default)
        elif value is not None:
            if self.get(section, key) != value:
                self.set(section, key, value)

    def save(self):
        """Save the config to file."""
        with open(self.filename, 'w') as f:
            self.config.write(f)
示例#45
0
from peewee import *
import datetime
from configparser import SafeConfigParser

config = SafeConfigParser()
config.read('config.ini')

if config.getboolean('Mysql', 'enable'):
    database = MySQLDatabase(
        config.get('Mysql', 'scheme'),
        host=config.get('Mysql', 'host'),
        port=config.getint('Mysql', 'port'),
        user=config.get('Mysql', 'user'),
        passwd=config.get('Mysql', 'password'),
        charset='utf8',
        use_unicode=True,
    )

elif config.getboolean('Sqlite', 'enable'):
    database = SqliteDatabase(config.get('Sqlite', 'dbname'))

elif config.getboolean('Postgresql', 'enable'):
    database = PostgresqlDatabase(
        config.get('Postgresql', 'scheme'),
        user=config.get('Postgresql', 'user'),
        password=config.get('Postgresql', 'password'),
        host=config.get('Postgresql', 'host'),
        charset='utf8',
        use_unicode=True,
    )
示例#46
0
文件: webui.py 项目: h3ndrik/spider
title = 'Spider Search'
host = '0.0.0.0'  # 'localhost'
port = 8080
debug = False
datapath_sub = dict()
search_path = ['/etc', '']
for path in search_path:
    candidate = os.path.join(path, 'spider.conf')
    if os.path.isfile(candidate):
        try:
            config = SafeConfigParser()
            config.read([candidate])
            database = config.get('Spider', 'database')
            title = config.get('WebUI', 'title')
            host = config.get('WebUI', 'host')
            port = config.getint('WebUI', 'port')
            debug = config.getboolean('WebUI', 'debug')
            #datapathsub = list(filter(None, [x.strip() for x in datapath_regexps.splitlines()]))
        except:
            pass
        datapath_sub = dict()
        try:
            datapath_sub = config.items('datapath_substitutions')
        except:
            pass

engine = create_engine(database)
Session = sessionmaker(bind=engine)
session = Session()
Base.metadata.create_all(engine)
示例#47
0
                    sys.stderr.write("""

*** HEY CLOWN ***

your file permissions on {0} allow anyone to read your plain text splunk password!
use the --save-config option with --encrypt to save your configuration with an encrypted password or chmod o-rwx this file
so that other people cannot read it

*** END CLOWN MESSAGE ***

""".format(config_path))

                password = config.get(args.enviro, 'password')

        if config.has_option(args.enviro, 'max_result_count'):
            max_result_count = config.getint(args.enviro, 'max_result_count')

    except Exception as e:
        logging.warning("invalid configuration file {0}: {1}".format(
            config_path, str(e)))

# command line options override configuration values
if args.uri is not None:
    uri = args.uri
if args.username is not None:
    username = args.username
if args.password:
    password = getpass("Enter password: ")

if encrypted_password is not None:
    password = decrypt_password(encrypted_password)
示例#48
0
import time

import cv2
import imutils
import numpy as np
import ffmpeg
import json
from configparser import SafeConfigParser

from serverModule.log import log

config = SafeConfigParser()

config.read('config.ini')

opencv_width = config.getint('opencvVideoRes', 'width')  # width
opencv_height = config.getint('opencvVideoRes', 'height')  # height
encodingToCompressionRatio = config.get('ffmpeg', 'encodingSpeed')
resize_image = config.getboolean('opencvVideoRes', 'resize')


class CameraStream:
    FFMPEG_TIMEOUT = 60
    ffmpeg_process = None

    def __init__(self, uri, options=None):
        self.uri = uri
        self.key = self.get_key(uri)
        self.ws_list = list()

        self.options = options
示例#49
0
class Configuration(object):
    """Class storing configuration details for Modula"""

    def __init__(self, rootdir=".", cfg=None):
        defaults = {
            "@paths.modules": "modules",
            "@paths.storage": "storage"
        }

        if cfg:
            self.cfg = cfg
        else:
            cfgfiles = ["modules.cfg", "modula.cfg"]
            cfgfiles = [os.path.join(rootdir, file) for file in cfgfiles]

            self.cfg = Parser()
            self.cfg.read(cfgfiles)

        # Add defaults
        for k, v in defaults.items():
            if k not in self:
                self[k] = v

        # Make sure all paths are absolute
        for k, v in self.items("@paths"):
            self["@paths.%s" % k] = os.path.abspath(os.path.expanduser(v))

        # Make sure all input files are absolute
        for k, v in self.items("@inputs"):
            self["@inputs.%s" % k] = os.path.abspath(os.path.expanduser(v))

    def _parse_name(self, name):
        if "." in name:
            section, option = name.split(".", 1)
        else:
            section = "@global"
            option = name
        return section, option

    def sections(self):
        return self.cfg.sections()

    def get(self, name):
        return self.cfg.get(*self._parse_name(name))

    def getInt(self, name):
        return self.cfg.getint(*self._parse_name(name))

    def getFloat(self, name):
        return self.cfg.getfloat(*self._parse_name(name))

    def getBoolean(self, name):
        return self.cfg.getboolean(*self._parse_name(name))

    def items(self, section):
        return self.cfg.items(section)

    def __contains__(self, name):
        return self.cfg.has_option(*self._parse_name(name))

    def __getitem__(self, name):
        try:
            return self.cfg.get(*self._parse_name(name))
        except KeyError:
            raise KeyError(name)

    def __setitem__(self, name, value):
        section, option = self._parse_name(name)
        if not self.cfg.has_section(section):
            self.cfg.add_section(section)
        self.cfg.set(section, option, value)

    def __delitem__(self, name):
        section, option = self._parse_name(name)
        try:
            ok = self.cfg.remove_option(section, option)
            if not ok:
                raise NoOptionError()
        except NoOptionError:
            raise KeyError(name)
示例#50
0
class Admin_Agent(object):
    """
    Admin agent: initializes the simulation, decomposes space and
    assigns space to Subvolume agents. Growth cones are also initialized
    here and distributed to the Subvolumes corresponding to the location
    of the growth cones.

    Modelled as conceptually close to biology but great in load balancing...
    """
    def __init__(self, total_processors, cfg_file="test_config.cfg"):
        self.cfg_file = cfg_file
        self.parser = SafeConfigParser()
        self.parser.read(cfg_file)
        np.random.seed(
            self.parser.getint("system", "seed")
        )  # randomized locations of the initial parts of the structures to generate.
        if self.parser.has_option(
                "system", "recursion_limit"
        ):  # Set the maximum depth of the Python interpreter stack to n
            sys.setrecursionlimit(
                self.parser.getint("system", "recursion_limit"))
        self.substrate = {}
        # a few variable should be global
        self.total_processors = total_processors  # until I find a better way
        self.processor_ids = range(
            1, self.total_processors)  # start from 1: skip the Admin
        self.summarized_constellations = {
        }  # store summarized constellations based on their self.num / proc id
        # set up communication links for the Admin
        self._initialize_communication_links()
        self._setup_DBs()
        ret = self._initialize_SVs()
        time.sleep(
            1
        )  # sleep a second: make sure al SVs are initialized (all messages processed) return 1
        if ret < 0:
            self._destruction()
            return  # and break out
        ret = self._initialize_GE_beta()  # finish return 1
        if ret < 0:
            self._destruction()
            return
        # and continue with the real deal if we haven't been kicked out yet
        self.main_loop()

        self._destruction()

    def _initialize_communication_links(self):
        """ Set up the communication links to and from the Admin
        """
        self.context = zmq.Context()
        """ Initialize socket to receive incoming messages. Incoming messages \
        are queued "fair" according to 0MQ...
        """
        self.socket_pull = self.context.socket(zmq.PULL)
        self.socket_pull.bind("tcp://*:%s" %
                              self.parser.getint("system", "pull_port"))

        # SETUP PROXY: to communicate through a proxy; all-to-all
        self.psub = self.context.socket(zmq.SUB)
        self.psub.connect("tcp://localhost:%s" %
                          self.parser.getint("system", "proxy_pub_port"))
        self.psub.setsockopt(zmq.SUBSCRIBE, "Admin")
        self.ppub = self.context.socket(zmq.PUB)
        self.ppub.connect("tcp://localhost:%s" %
                          self.parser.getint("system", "proxy_sub_port"))

        self.poller = zmq.Poller()
        self.poller.register(self.psub, zmq.POLLIN)

        registered = 0
        while registered < len(self.processor_ids):
            msg = self.socket_pull.recv()
            print_with_rank(str(msg))
            registered = registered + 1
        print("all Subvolumes registered. Proceed!")

    def _setup_DBs(self):
        self.db_file_name = self.parser.get("system", "out_db")
        try:
            os.remove(self.db_file_name)
        except Exception:
            pass
        self.conn = sqlite3.connect(self.db_file_name)
        self.conn.execute(
            '''CREATE TABLE swc_data (id INTEGER PRIMARY KEY AUTOINCREMENT,\
                                            name text,\
                                            swc_type int,\
                                            from_x real, \
                                            from_y real, \
                                            from_z real,\
                                            to_x real,\
                                            to_y real,\
                                            to_z real,\
                                            radius real,\
                                            proc integer)''')

        # only if this parameter is set in the config file
        if self.parser.has_option("system", "syn_db"):
            self.syn_file_name = self.parser.get("system", "syn_db")
            try:
                os.remove(self.syn_file_name)
            except Exception:
                pass
            self.syn_conn = sqlite3.connect(self.syn_file_name)
            self.syn_conn.execute(
                '''CREATE TABLE synapses (id INTEGER PRIMARY KEY AUTOINCREMENT,\
                                                pre_syn_entity text,\
                                                pre_x real,\
                                                pre_y real,\
                                                pre_z real,\
                                                post_syn_entity text,\
                                                post_x real,\
                                                post_y real,\
                                                post_z real)''')

    def _get_substrate_information(self):
        for option in self.parser.options("substrate"):
            if not option.startswith("dim_"):  # then this is an entity
                val = self.parser.get("substrate", option)
                if val.endswith("pkl"):
                    self.substrate[option] = pickle.load(open(val, "r"))
                else:
                    print_with_rank(
                        "substrate entity (%s) should be given as name of a pickle file"
                        % option)
        time.sleep(0.0)

    def _get_sub_substrate(self, boundary):  # get subvolume substrate
        sub_substrate = {}
        b0 = np.array(boundary[0])
        b1 = np.array(boundary[1])
        for entity in self.substrate:
            print("checking entity: ", entity, ", boundary: ", boundary)

            # 2014-08-08, make all internals sets of fronts.. bit redundant memory wise, but easy for administration
            sub_substrate[entity] = set()
            for ppoint in self.substrate[entity]:
                point = ppoint[0]
                t_front = Front(None, None, point, ppoint[1], 0, 0)
                sub_substrate[entity].add(t_front)
                #time.sleep(1)
            if len(sub_substrate[entity]) == 0:
                sub_substrate.pop(entity, None)
        return sub_substrate

    def _get_virtual_substrate(self):
        virtual_substrate = {}
        for name, value in self.parser.items("substrate"):
            if name.startswith("virtual"):
                virtual_name = name.split("_")[1]
                print("Found virtual: ", virtual_name)
                virtual_substrate.update(
                    {virtual_name: eval(self.parser.get("substrate", name))})

        if virtual_substrate == {}:
            print("No virtual substrates found")
            #time.sleep(5)
        else:
            print("virtual_substrate:\n", virtual_substrate)
            #time.sleep(5)
        return virtual_substrate

    def _initialize_SVs(self):  # each process assign  sv
        required = ["xa", "ya", "za"]  # slice  number x,y,z corrdination
        for item in required:
            if not self.parser.has_option("sub_volumes", item):
                print_with_rank("%s does not exists in the cfg file" % item)
        xa,ya,za = self.parser.getint("sub_volumes","xa"), \
          self.parser.getint("sub_volumes","ya"), \
          self.parser.getint("sub_volumes","za")

        # CHECK: do I still need this clause???
        if xa * ya * za > (len(self.processor_ids)):
            print_with_rank("Not enough processors (%i processors for %i SVs)" \
              % (len(self.processor_ids),xa*ya*za))
            return -1  # fetch in __init__

        substrate = self._get_substrate_information()  # volume substrate

        dim_xyz = eval(self.parser.get("substrate", "dim_xyz"))
        x_space = dim_xyz[0] / xa
        y_space = dim_xyz[1] / ya
        z_space = dim_xyz[2] / za
        self.space_division = {
        }  # per ZMQ processor what part of the Volume they control
        self.assigned_ids = {
        }  # per chunk of Volume, which processor controls it
        count = 0
        for i in range(xa):
            for j in range(ya):
                for k in range(za):
                    x0, y0, z0 = i * x_space, j * y_space, k * z_space
                    x1, y1, z1 = (i + 1) * x_space, (j + 1) * y_space, (
                        k + 1) * z_space
                    self.space_division[i, j, k] = [[x0, y0, z0], [x1, y1, z1]]
                    self.assigned_ids[i, j, k] = self.processor_ids[count]
                    print_with_rank("proc: %i [%i,%i,%i]= %1.f,%1.f,%1.f -> %1.f,%1.f,%1.f" \
                                    % (self.processor_ids[count],i,j,k,x0,y0,z0,x1,y1,z1))
                    count = count + 1

        # for now, pass the global virtual information to each SV (independent of SV boundaries)
        virtual_substrate = self._get_virtual_substrate(
        )  # [substrate] virtual_

        for key in sorted(
                self.space_division.keys()):  # find neighbor sv(assigned_ids)
            dest = self.assigned_ids[key]
            print_with_rank("key: " + str(key) + "type: " + str(type(key)) +
                            "assigned to " + str(dest))
            boundary = self.space_division[key]
            neighbors = []
            for i in range(key[0] - 1, key[0] + 2):
                for j in range(key[1] - 1, key[1] + 2):
                    for k in range(key[2] - 1, key[2] + 2):
                        new_key = (i, j, k)
                        if new_key in self.space_division and key != new_key:
                            neighbors.append(self.assigned_ids[i, j, k])
            print_with_rank("neighbors of " + str(dest) + ": " +
                            str(neighbors))
            """ get part of the substrate contained in this SV, prepare to include in message.
            currently the substrate entities are represented by points
            """
            sub_substrate = self._get_sub_substrate(boundary)

            print_with_rank("sending Init_SV to %06d" % dest)
            message = ("Init_SV", boundary, neighbors, sub_substrate,
                       virtual_substrate)
            self.ppub.send_multipart(["%06d" % dest, pickle.dumps(message)])
            # time.sleep(2)
        return 1  # positive: all is well

    def _initialize_GE_beta(self):  # cell_type_
        cfg_sections = self.parser.sections(
        )  # cfg file all sections like[system]
        ship_entity_to_proc = {}
        for i in self.processor_ids:
            ship_entity_to_proc[i] = []
        for name in cfg_sections:
            if name.startswith("cell_type"):
                entity_id = 0
                print_with_rank("found cell type: %s" % name)
                # retrieve algo to use and how many to deploy
                no_seeds = self.parser.getint(name,
                                              "no_seeds")  # number of seeds
                algorithm_name = self.parser.get(name,
                                                 "algorithm")  # growth rule
                """ Sample the soma position of the entity and assign \
                    to the correct processor
                """
                loc = eval(self.parser.get(name, "location"))
                rs = eval(self.parser.get(name, "soma_radius"))

                for seed in range(no_seeds):
                    details = {}
                    for option in self.parser.options(
                            name):  # section[cell_type_]  option
                        details.update({option: self.parser.get(name, option)
                                        })  # insert into details

                    soma_xyz = np.array([])
                    for i in range(3):  # 3D
                        soma_xyz = np.append(
                            soma_xyz,
                            (loc[1][i] - loc[0][i]) * np.random.random() +
                            loc[0][i])  # soma location

                    # soma_pos = P3D2(soma_xyz,radius=rs)
                    details["radius"] = rs
                    details["soma_xyz"] = soma_xyz

                    dest = self._which_volume_contains_position(
                        soma_xyz)  # contains soma subvolume
                    if dest == -1:
                        print_with_rank("Could not find subvolume for soma: " +
                                        str(soma_xyz))
                        return -1
                    print_with_rank("sampled s%i:%s  (D:%i)" %
                                    (seed, str(soma_xyz), dest))

                    entity_name = name + "__" + str(
                        entity_id)  # cell_type__0 /cell_type__1
                    details.update({'entity_name': entity_name})
                    ship_entity_to_proc[dest].append(details)
                    entity_id = entity_id + 1

        # print_with_rank("ship_entity_to_proc: " + str(ship_entity_to_proc))

        for proc in ship_entity_to_proc.keys():
            entries = ship_entity_to_proc[proc]
            message = ("Initialize_GEs", entries)
            # comm.send(message,dest=proc,tag=2)
            self.ppub.send_multipart(["%06d" % proc, pickle.dumps(message)])
            print_with_rank("to %i: %s" % (proc, str(message)))
        return 1  # positive: all is well

    def main_loop(self):
        """ Perform some update cycles. When the admin publishes an \
        "Update" note, the Subvolumes send their current "my_constellation" \
        and their summarized constellation to the Admin.
        1. wait/gather all the info/constellations from each SV
        2. wait for replies
        3. proceed to next Update cycle
        """
        for i in range(self.parser.getint(
                "system",
                "no_cycles")):  # no_cycle:number of each front extend
            for dest in self.processor_ids:
                message = ("Update", i, self.summarized_constellations)
                self.ppub.send_multipart(
                    ["%06d" % dest, pickle.dumps(message)])
            responses = 0
            while responses < len(self.processor_ids):
                # socks = dict(self.poller.poll())

                # if self.psub in socks and socks[self.psub] == zmq.POLLIN:
                [address, message] = self.psub.recv_multipart()
                message = pickle.loads(message)
                print_with_rank("received message on PULL")
                if message[0] == "Update_OK":
                    responses = responses + 1
                    sender = int(message[1])
                    changes = message[2]
                    syn_locs = message[3]
                    t_constellation = message[4]
                    # write changes
                    self._temp_to_db(changes, sender)
                    # store putative synapse locations
                    if self.parser.has_option("system", "syn_db"):
                        self._syn_to_db(syn_locs, sender)
                    # store internally
                    self.summarized_constellations[sender] = t_constellation

                    # ##### TEST
                    # print_with_rank("Received summ ("+str(sender)+"): " + str(t_constellation.keys())  )
                    # time.sleep(1)
                    # #### END OF TEST
                elif message[0] == "Migrate_Front":
                    self._process_migrate_front(message)
                    time.sleep(0.)
                elif message[0] == "Extra_synapses":
                    print_with_rank("EXTRA_SYNAPSES")
                    sender = int(message[1])
                    syn_locs = message[2]
                    # store putative synapse locations
                    if self.parser.has_option("system", "syn_db"):
                        self._syn_to_db(syn_locs, sender)

        self.conn.commit()
        if self.parser.has_option("system", "syn_db"):
            self.syn_conn.commit()

    def _temp_to_db(self, changes, sender):  #front,c_fronts) :
        for front, c_fronts in changes:
            pos = front.xyz
            conn = sqlite3.connect(self.db_file_name)
            cursor = conn.cursor()
            for c_front in c_fronts:
                cpos = c_front.xyz
                name = c_front.entity_name
                radius = c_front.radius
                swc_type = c_front.swc_type
                values = (None, name, swc_type, pos[0], pos[1], pos[2],
                          cpos[0], cpos[1], cpos[2], radius,
                          "%06d" % int(sender))
                self.conn.execute(
                    "INSERT into swc_data VALUES (?,?,?,?,?,?,?,?,?,?,?)",
                    values)
        #self.conn.commit()

    def _syn_to_db(self, syn_locs, sender):
        for syn_loc in syn_locs:
            # pre_front = syn_loc[0]
            # post_front = syn_loc[1]
            # prp = pre_front.xyz
            # pop = post_front.xyz
            # values = (None,pre_front.entity_name,prp[0],prp[1],prp[2],post_front.entity_name,pop[0],pop[1],pop[2])

            pre_name = syn_loc[0]
            pre_x = syn_loc[1]
            pre_y = syn_loc[2]
            pre_z = syn_loc[3]
            post_name = syn_loc[4]
            post_x = syn_loc[5]
            post_y = syn_loc[6]
            post_z = syn_loc[7]

            # values = (None,pre_front.entity_name,prp[0],prp[1],prp[2],post_front.entity_name,pop[0],pop[1],pop[2])
            values = (None, pre_name, pre_x, pre_y, pre_z, post_name, post_x,
                      post_y, post_z)

            print("values: ", values)
            self.syn_conn.execute(
                "INSERT into synapses VALUES (?,?,?,?,?,?,?,?,?)", values)

    def _process_migrate_front(self, message):
        new_front = message[1]
        parent_front = message[2]
        new_dest = self._which_volume_contains_position(new_front.xyz)
        print_with_rank("******: new_dest %i (for %s)" %
                        (new_dest, str(new_front.entity_name)))
        if new_dest == -1:
            # this front is dead. outside of space
            return
        message = ("Add_Front", new_front)
        #comm.send(message,dest=new_dest,tag=2)
        self.ppub.send_multipart(["%06d" % new_dest, pickle.dumps(message)])

    def _which_volume_contains_position(self, pos):
        for key in self.space_division:
            sv = self.space_division[key]
            if (pos[0] >= sv[0][0]) and \
               (pos[1] >= sv[0][1]) and \
               (pos[2] >= sv[0][2]) and \
               (pos[0] < sv[1][0]) and \
               (pos[1] < sv[1][1]) and \
               (pos[2] < sv[1][2]) :
                return self.assigned_ids[key]
        return -1  # if not found, return -1. Fetch somewhere else

    def _destruction(self):
        """ Shut down the system. End all threads and all MPI instances
        """
        _me(True)
        print_with_rank("pickle_msg:" + pickle.dumps("Done"))
        self.ppub.send_multipart(["All", pickle.dumps("Done")])
        self.context.destroy()
示例#51
0
_, max_curr_seq_len = curr_seq.shape
_, max_prev_seq_len = prev_seq.shape

N_data, seq_len, dim = curr_atoms.shape
N_data_val, _, _ = curr_atoms_val.shape

N_masks, max_seq_len, N_interactions, max_atoms_ia = curr_masks.shape

N_atomtypes = 4

cfg = SafeConfigParser()
cfg.read('config.ini')

shift = tf.range(1, 8, 1)

sigma = float(cfg.getint('grid', 'sigma'))
ds = float(cfg.getint('grid', 'length')) / float(
    cfg.getint('grid', 'max_resolution'))


#Map coords to grid
def map_to_grid(coord_inp, grid_size, ds, sigma):
    grid = tf.range(-int(grid_size / 2), int(grid_size / 2), dtype=tf.float32)
    grid = tf.add(grid, 0.5)
    grid = tf.scalar_mul(ds, grid)

    X, Y, Z = tf.meshgrid(grid, grid, grid, indexing='ij')
    grid = tf.stack([X, Y, Z])
    grid = tf.expand_dims(grid, 0)
    grid = tf.expand_dims(grid, 0)
示例#52
0
def main(argv):
    """UBEZ connection test program
    Usage: python ubez.py [ip] [port]
    Options:
      -i ..., --ip=...   ip addresss
      -h, --help         show this help
      -p                 port number
    """
    # print 'ARGV      :', sys.argv
    dirname = module_locator.module_path()
    #dirname=os.path.dirname(os.path.abspath(__file__))
    path = dirname.replace('\\', '/')

    config = SafeConfigParser()
    config.read(path + ('/ubez.ini'))
    ID = config.get('SMS', 'id')
    PWD = config.get('SMS', 'pwd')
    SMSTIMER = config.getint('SMS', 'smstimer')
    SMSURL = config.get('SMS', 'smsurl')

    TELLIST = []
    TELLIST.append(config.get('ONCALL', 'tel1'))
    TELLIST.append(config.get('ONCALL', 'tel2'))
    TELLIST.append(config.get('ONCALL', 'tel3'))
    TELLIST.append(config.get('ONCALL', 'tel4'))
    TELLIST.append(config.get('ONCALL', 'tel5'))

    TRYLIMIT = config.getint('SYSTEM', 'trylimit')
    WAITTIME = config.getint('SYSTEM', 'waittime')

    OPHOST = config.get('IBM', 'ophost')
    OPID = config.get('IBM', 'opid')
    OPPWD = config.get('IBM', 'oppwd')

    IBMFTP = config.getint('SYSTEM', 'IBMFTP')
    SMS = config.getint('SYSTEM', 'SMS')

    logger = logging.getLogger(config.get('SYSTEM', 'logname'))
    formatter = \
        logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    file_handler = RotatingFileHandler((dirname + '\ubez.log'), 'a', 4096, 5)
    file_handler.setLevel(logging.DEBUG)
    file_handler.setFormatter(formatter)
    logger.addHandler(file_handler)

    serverip = ''
    serverport = 4500
    try:
        opts, args = getopt.getopt(argv, "hi:p:", ["ip=", "port="])
    except getopt.GetoptError:
        print 'Usage:ubez.py -i <ipaddress> -p <portnumber>'
        sys.exit(2)

    if len(opts) != 2:
        print 'Usage:ubez.py -i <ipaddress> -p <portnumber>'
        sys.exit(2)

    for opt, arg in opts:
        if opt == '-h':
            print 'ubez.py -i <ipaddress> -p <portnumber>'
            sys.exit()
        elif opt in ("-i", "--ip"):
            serverip = arg
        elif opt in ("-p", "--port"):
            serverport = int(arg)

    msg = '-------------- Begin log -------------------'
    logger.warning(msg)
    for i in range(1, TRYLIMIT + 1):
        LOSTCONNECT = False
        sSock = socket(AF_INET, SOCK_STREAM)
        try:
            #Connect to server
            sSock.connect((serverip, serverport))
        except:
            LOSTCONNECT = True
            msg = 'Try %s time Connect IP: %s Port: %s  fail' % (str(i), serverip, str(serverport))
            print msg
            logger.error(msg)
        else:
            msg = 'Try %s time Connect IP: %s Port: %s OK' % (str(i), serverip, str(serverport))
            print msg
            logger.error(msg)
        finally:
            sSock.close()
        time.sleep(WAITTIME)

    if LOSTCONNECT:
        if serverip == '192.168.110.133' and \
                (serverport == 4500 or serverport == 4700):
            SMS_MSG = "UBEZ Acquirer service down"
        elif serverip == '192.168.110.133' and serverport == 4900:
            SMS_MSG = "UBEZ Issuer service down"
        else:
            SMS_MSG = "Service down IP:%s,PORT:%s" % (str(serverip), str(serverport))

        #FTP to IBM Command console
        if IBMFTP is True:
            print 'use IBMFTP function'
            f = open(dirname + '\message.txt', 'w+')
            f.write(SMS_MSG)
            f.close()
            try:
                ftp = FTP(OPHOST)
                ftp.login(OPID, OPPWD)
                fmsgname = dirname + 'message.txt'
                fwavname = dirname + 'sound.wav'
                ftp.retrlines('LIST')

                file = open(fwavname, 'rb')
                ftp.storbinary('STOR ' + 'sound.wav', file)
                file.close()

                file = open(fmsgname, 'rb')
                ftp.delete('message.txt')
                ftp.storbinary('STOR ' + 'message.txt', file)
                file.close()

                ftp.quit()
                ftp.close()
            except Exception, err:
                print logger.error(err)

        if SMS == True:
            print 'use SMS function'
            for tel in TELLIST:
                if tel <> '':
                    data = OrderedDict();
                    data['ID'] = str(ID)
                    data['PWD'] = str(PWD)
                    data['TEL'] = str(tel)
                    data['MSG'] = SMS_MSG.encode('hex')
                    print data['MSG'].decode('hex')
                    url_values = urllib.parse.urlencode(data)

                    #url='http://172.28.223.10:9080/SMSer'
                    #url='http://127.0.0.1:8080'

                    url = SMSURL

                    full_url = url + '?' + url_values
                    print full_url
                    #try:
                    response = urllib.request.urlopen(full_url)
                    data_received = response.read()
                    msg1 = str(data_received)
                    #msg=msg1.replace("Big5","utf-8")
                    #tree=ElementTree.fromstring(str(msg))
                    logger.error(msg1)
                    #except Exception,err:
                    #    errmsg=str(err)+':'+SMSURL
                    #    logger.error(errmsg)
                    #finally:
                    time.sleep(1)
示例#53
0
文件: utils.py 项目: dsaccon/vulnmine
def init_globals():
    """Initialize global variables."""
    # Determine if running directly from source code or as a pkg
    src_path = gbls.CONFDIR

    # If the default config file is in the "vulnmine/" directory, then
    # must be running from source code directly
    # Assume running directly from src until proven otherwise

    gbls.pkgdir = ('vulnmine/' + src_path)

    if os.path.exists(gbls.pkgdir):
        run_from_src_code = True
        print('=== Appears to be executing source code directly.\n'
              '=== In this mode, subdirectories are as follows:\n'
              '===    data/ contains data and user .ini config file,\n'
              '===    vulnmine/ contains the source code. \n'
              '===    vulnmine/vulnmine_data has default configuration data.')
    else:
        run_from_src_code = False
        print('=== Appears to be executing a packaged module.\n'
              '=== In this mode:\n'
              '===    data/ subdirectory contains data and '
              ' user .ini config file,\n'
              '===    source code and data distributed with pkg are '
              'in the pkg install directory.')
    if not run_from_src_code:
        try:
            # Files distributed with vulnmine are installed in the python
            # '<sys.prefix>/vulnmine_data' directory
            gbls.pkgdir = pkg_resources.resource_filename(
                'vulnmine', gbls.CONFDIR)
            print(('Utils Pkg directory is: {0}'.format(gbls.pkgdir)))

        except Exception as e:
            print(('*** Error reading default configuration file: {0} \n'
                   '*** Default .ini file is not in the'
                   ' module directory or is misconfigured.\n'
                   '*** Aborting execution'.format(e)))

            # No use trying to do anything else
            return 200

    parser = SafeConfigParser()
    try:
        default_config_file = gbls.pkgdir + gbls.CONFIG_DEFAUlTS
        user_config_file = gbls.DATADIR + gbls.CONF_FILE
        print(('Utils: Default .ini config file: {0} \n'
               'User .ini config file: {1}'.format(default_config_file,
                                                   user_config_file)))
        parser.read([default_config_file, user_config_file])

    except Exception as e:
        print(('*** Error reading configuration file: {0}\n'
               '*** Aborting execution'.format(e)))
        return 100

    try:
        gbls.pckdir = (gbls.wkdir + gbls.DATADIR +
                       parser.get('User', 'Pckdir'))
        gbls.csvdir = (gbls.wkdir + gbls.DATADIR +
                       parser.get('User', 'Csvdir'))
        gbls.nvddir = (gbls.wkdir + gbls.DATADIR +
                       parser.get('User', 'Nvddir'))

        gbls.activate_plugins = parser.getboolean('User', 'Activate_plugins')

        if gbls.activate_plugins:

            plugin_directory = parser.get('User', 'Plugins')

            if run_from_src_code:
                gbls.plugin_folder = 'vulnmine/' + plugin_directory
            else:
                gbls.plugin_folder = pkg_resources.resource_filename(
                    'vulnmine', plugin_directory)

        # Create directories if do not exist
        if not os.path.isdir(gbls.pckdir):
            os.makedirs(gbls.pckdir)
        if not os.path.isdir(gbls.nvddir):
            os.makedirs(gbls.nvddir)

        ######
        #   Vulnmine pkg data files
        ######

        gbls.s_vndr_stop_wds = (gbls.pkgdir +
                                parser.get('User', 'S_vndr_stop_wds'))
        gbls.df_label_software = (gbls.pkgdir +
                                  parser.get('User', 'Df_label_software'))
        gbls.df_label_vendors = (gbls.pkgdir +
                                 parser.get('User', 'Df_label_vendors'))

        gbls.clf_vendor = gbls.pkgdir + parser.get('User', 'Clf_vendor')
        gbls.clf_software = gbls.pkgdir + parser.get('User', 'Clf_software')
        gbls.log_conf = gbls.pkgdir + parser.get('User', 'Log_conf')

        ######
        #   CSV Input data
        ######

        gbls.v_r_system = (gbls.csvdir + parser.get('User', 'V_r_system'))
        gbls.v_gs_add_rem_pgms = (gbls.csvdir +
                                  parser.get('User', 'V_gs_add_rem_pgms'))
        gbls.v_gs_add_rem_pgms_64 = (
            gbls.csvdir + parser.get('User', 'V_gs_add_rem_pgms_64'))

        gbls.ad_vip_grps = gbls.csvdir + parser.get('User', 'Ad_vip_grps')

        ######
        #   Saved dataframe names
        #   'rf_' - refactor code version
        ######
        gbls.df_sys_pck = gbls.pckdir + parser.get('User', 'Df_sys_pck')
        gbls.df_add_rem_g_pck = (gbls.pckdir +
                                 parser.get('User', 'Df_add_rem_g_pck'))
        gbls.df_cpe4_pck = gbls.pckdir + parser.get('User', 'Df_cpe4_pck')
        gbls.df_cve_pck = gbls.pckdir + parser.get('User', 'Df_cve_pck')

        gbls.df_v_R_System_3modified_pck = (
            gbls.pckdir + parser.get('User', 'Df_v_R_System_3modified_pck'))

        gbls.df_sft_vuln_pck = gbls.pckdir + parser.get(
            'User', 'Df_sft_vuln_pck')
        gbls.df_match_vendor_publisher_pck = (
            gbls.pckdir + parser.get('User', 'Df_match_vendor_publisher_pck'))
        gbls.df_match_cpe_sft_pck = (
            gbls.pckdir + parser.get('User', 'Df_match_cpe_sft_pck'))

        ######
        #   NVD data
        ######

        # https://static.nvd.nist.gov/feeds/xml/cve/2.0/nvdcve-2.0-2016.meta
        gbls.url_meta_base = parser.get('User', 'Url_meta_base')
        gbls.url_meta_end = parser.get('User', 'Url_meta_end')
        #gbls.url_xml_base = parser.get('User', 'Url_xml_base')
        gbls.url_json_base = parser.get('User', 'Url_json_base')
        #gbls.url_xml_end = parser.get('User', 'Url_xml_end')
        gbls.url_json_end = parser.get('User', 'Url_json_end')
        gbls.url_cpe = parser.get('User', 'Url_cpe')
        gbls.cpe_filename = parser.get('User', 'Cpe_filename')
        gbls.cve_filename = parser.get('User', 'Cve_filename')
        gbls.cpe_max_age = parser.getint('User', 'Cpe_max_age')
        gbls.nvd_meta_filename = parser.get('User', 'Nvd_meta_filename')

        gbls.nvdcpe = gbls.nvddir + gbls.cpe_filename
        gbls.nvdcve = gbls.nvddir + gbls.cve_filename

    except Exception as e:
        print(('*** Error in config file: {0}'.format(e)))

    return 0
    def load(self, filename):
        ''' load a config file '''

        # parser and default value
        parser = SafeConfigParser({ \
                    'LRCatalog' : self.default_lrcat, \
                    'ProdDirectory' : self.default_prod_directory, \
                    'RSyncExe' : self.rsync_exe, \
                    'RsyncMaxLenLine' : self.rsync_max_len_line, \
                    'FSEncoding' : self.fs_encoding, \
                    'ArchiveDateFmt' : self.archive_date_fmt, \
                    'ProductionDateFmt' : self.production_date_fmt, \
                    'DayFirst' : self.dayfirst, \
                    'GeoCoder' : self.geocoder, \
                })

        # config file is located in directory where main script is lauched
        _dir, _ = os.path.split(filename)
        if _dir:
            config_file = filename
        else:
            config_file = os.path.join(os.path.dirname(sys.argv[0]), filename)

        parser.read(config_file)
        try:
            self.default_lrcat = parser.get(CONFIG_MAIN, 'LRCatalog')
            self.default_prod_directory = parser.get(CONFIG_MAIN, 'ProdDirectory')

            self.rsync_exe = parser.get(CONFIG_MAIN, 'RSyncExe')
            self.rsync_max_len_line = parser.getint(CONFIG_MAIN, 'RsyncMaxLenLine')

            self.archive_date_fmt = parser.get(CONFIG_MAIN, 'ArchiveDateFmt')
            self.production_date_fmt = parser.get(CONFIG_MAIN, 'ProductionDateFmt')

            self.fs_encoding = parser.get(CONFIG_MAIN, 'FSEncoding')

            self.dayfirst = parser.get(CONFIG_MAIN, 'DayFirst')

            self.geocoder = parser.get(CONFIG_MAIN, 'GeoCoder')


            if parser.has_section(CONFIG_ARCHVOL):
                self.archive_volumes = []
                idvol = 1
                while True:
                    if not parser.has_option(CONFIG_ARCHVOL, 'VOLUME%s' % idvol):
                        break
                    archvol = parser.get(CONFIG_ARCHVOL, 'VOLUME%s' % idvol)
                    #print archvol
                    try:
                        volname, dirname, datestart, dateend = archvol.split(',')
                        volname = volname.strip()
                        dirname = dirname.strip()
                        datestart = dateparser.parse(datestart)
                        dateend = dateparser.parse(dateend)
                    except ValueError:
                        print('ERROR section %s invalid' % CONFIG_ARCHVOL)
                        sys.exit(0)
                    self.archive_volumes.append((volname, dirname, datestart, dateend))
                    idvol += 1


        except Error as _e:
            raise LRConfigException('Failed to read config file %s' % filename)
def process(config_ini, limit=None):
    config = SafeConfigParser()
    config.read(config_ini)

    destination_name = config.get('destination', 'name')

    log.info('connecting to source server')
    connection = IMAP_CLS[config.getboolean('source', 'ssl')](config.get(
        'source', 'server'), config.getint('source', 'port'))
    connection.login(config.get('source', 'login'),
                     config.get('source', 'password'))

    mailbox = config.get('source', 'mailbox')
    log.debug('selecting mailbox %s', mailbox)
    res = connection.select('"{}"'.format(mailbox))
    if res[0] != 'OK':
        raise Exception(res[1])

    log.debug('searching messages')
    if config.has_option('source', 'imap_search'):
        res = connection.uid('search', None,
                             config.get('source', 'imap_search'))
    else:
        res = connection.uid('search', None, 'LARGER',
                             str(config.getint('source', 'email_min_size')))

    message_uids = res[1][0].split()
    total_messages = total_process = len(message_uids)
    if limit is not None:
        total_process = min([total_messages, limit])

    parser = BytesParser()
    processed_mailbox = config.get('source', 'processed_mailbox')
    destination = DESTINATIONS[config.get('destination',
                                          'type')](config['destination'])

    log.info('found %s messages to process', total_messages)
    for idx, uid in enumerate(message_uids):
        if idx >= total_process:
            break

        log.info('processing %s/%s...', idx + 1, total_process)

        log.debug('downloading and parsing message...')
        res = connection.uid('fetch', uid, '(FLAGS BODY.PEEK[])')
        message = parser.parsebytes(res[1][0][1])
        flags = re.findall(r'FLAGS (\(.*?\))', res[1][0][0].decode('utf8'))[0]

        payload = message.get_payload()
        textmsg = payload[0]

        bkp_identifier = str(uuid.uuid4())

        content_type = textmsg['Content-Type'].lower()
        if 'multipart/related' in content_type:
            log.warn('multipart/related not supported, skipping...')
            continue
        elif 'multipart/alternative' in content_type:
            text, html = textmsg.get_payload()
            add_saved_notice(text, destination_name, bkp_identifier)
            add_saved_notice(html, destination_name, bkp_identifier)
        elif 'text/plain' in content_type or 'text/html' in content_type:
            add_saved_notice(textmsg, destination_name, bkp_identifier)
        else:
            raise NotImplementedError

        log.debug('backup identifier: %s', bkp_identifier)

        destitem = destination.new(bkp_identifier,
                                   format_relevant_headers(message))

        attachments = payload[1:]
        if not len(attachments):
            log.debug('no attachments to save')
            continue

        for attachment in attachments:
            destitem.add(attachment)
            del payload[1]

        log.debug('saving attachments to destination...')
        destitem.save()

        log.debug('moving processed message to mailbox %s', processed_mailbox)
        res = connection.xatom(
            'UID MOVE', '{} "{}"'.format(uid.decode('utf8'),
                                         processed_mailbox))
        if res[0] != 'OK':
            raise Exception(res[1])

        log.debug('storing stripped message')
        connection.append(mailbox, flags, datetime.now(timezone.utc),
                          message.as_bytes())

    destination.finalize()
    log.debug('logging out')
    connection.logout()

    log.info('done')
示例#56
0
文件: settings.py 项目: Jactry/pitivi
class GlobalSettings(GObject.Object):

    """
    Pitivi app settings.

    The settings object loads settings from different sources, currently:
    - the local configuration file,
    - environment variables.

    Modules declare which settings they wish to access by calling the
    addConfigOption() class method during initialization.

    @cvar options: A dictionnary of available settings.
    @cvar environment: A list of the controlled environment variables.
    """

    options = {}
    environment = set()
    defaults = {}

    __gsignals__ = {}

    def __init__(self, **unused_kwargs):
        GObject.Object.__init__(self)
        self._config = SafeConfigParser()
        self._readSettingsFromConfigurationFile()
        self._readSettingsFromEnvironmentVariables()

    def _readSettingsFromConfigurationFile(self):
        """
        Read the configuration from the user configuration file.
        """

        try:
            conf_file_path = os.path.join(xdg_config_home(), "pitivi.conf")
            self._config.read(conf_file_path)
        except UnicodeDecodeError:
            unicode_error_dialog()
            return
        except ParsingError:
            return

        for (section, attrname, typ, key, env, value) in self.iterAllOptions():
            if not self._config.has_section(section):
                continue
            if key and self._config.has_option(section, key):
                if typ == int or typ == int:
                    try:
                        value = self._config.getint(section, key)
                    except ValueError:
                        # In previous configurations we incorrectly stored
                        # ints using float values.
                        value = int(self._config.getfloat(section, key))
                elif typ == float:
                    value = self._config.getfloat(section, key)
                elif typ == bool:
                    value = self._config.getboolean(section, key)
                else:
                    value = self._config.get(section, key)
                setattr(self, attrname, value)

    @classmethod
    def readSettingSectionFromFile(self, cls, section):
        """
        Force reading a particular section of the settings file.

        Use this if you dynamically determine settings sections/keys at runtime
        (like in tabsmanager.py). Otherwise, the settings file would be read
        only once (at the initialization phase of your module) and your config
        sections would never be read, and thus values would be reset to defaults
        on every startup because GlobalSettings would think they don't exist.
        """
        if cls._config.has_section(section):
            for option in cls._config.options(section):
                # We don't know the value type in advance, just try them all.
                try:
                    value = cls._config.getfloat(section, option)
                except:
                    try:
                        value = cls._config.getint(section, option)
                    except:
                        try:
                            value = cls._config.getboolean(section, option)
                        except:
                            value = cls._config.get(section, option)

                setattr(cls, section + option, value)

    def _readSettingsFromEnvironmentVariables(self):
        """
        Override options values using their registered environment variables.
        """
        for section, attrname, typ, key, env, value in self.iterAllOptions():
            if not env:
                # This option does not have an environment variable name.
                continue
            var = get_env_by_type(typ, env)
            if var is not None:
                setattr(self, attrname, var)

    def _writeSettingsToConfigurationFile(self):
        conf_file_path = os.path.join(xdg_config_home(), "pitivi.conf")

        for (section, attrname, typ, key, env_var, value) in self.iterAllOptions():
            if not self._config.has_section(section):
                self._config.add_section(section)
            if key:
                if value is not None:
                    self._config.set(section, key, str(value))
                else:
                    self._config.remove_option(section, key)
        try:
            file = open(conf_file_path, 'w')
        except IOError as OSError:
            return
        self._config.write(file)
        file.close()

    def storeSettings(self):
        """
        Write settings to the user's local configuration file. Note that only
        those settings which were added with a section and a key value are
        stored.
        """
        self._writeSettingsToConfigurationFile()

    def iterAllOptions(self):
        """
        Iterate over all registered options

        @return: an iterator which yields a tuple of (attrname, type, key,
        environment, value) for each option.
        """
        for section, options in list(self.options.items()):
            for attrname, (typ, key, environment) in list(options.items()):
                yield section, attrname, typ, key, environment, getattr(self, attrname)

    def isDefault(self, attrname):
        return getattr(self, attrname) == self.defaults[attrname]

    def setDefault(self, attrname):
        setattr(self, attrname, self.defaults[attrname])

    @classmethod
    def addConfigOption(cls, attrname, type_=None, section=None, key=None,
                        environment=None, default=None, notify=False,):
        """
        Add a configuration option.

        This function should be called during module initialization, before
        the config file is actually read. By default, only options registered
        beforehand will be loaded.
        See mainwindow.py and medialibrary.py for examples of usage.

        If you want to add configuration options after initialization,
        use the readSettingSectionFromFile method to force reading later on.
        See tabsmanager.py for an example of such a scenario.

        @param attrname: the attribute of this class which represents the option
        @type attrname: C{str}
        @param type_: type of the attribute. Unnecessary if default is given.
        @type type_: a builtin or class
        @param section: The section of the config file under which this option is
        saved. This section must have been added with addConfigSection(). Not
        necessary if key is not given.
        @param key: the key under which this option is to be saved. Can be none if
        this option should not be saved.
        @type key: C{str}
        @param notify: whether or not this attribute should emit notification
        signals when modified (default is False).
        @type notify: C{boolean}
        """
        if section and section not in cls.options:
            raise ConfigError(
                "You must add the section \"%s\" first." % section)
        if key and not section:
            raise ConfigError(
                "You must specify a section for key \"%s\"" % key)
        if section and key in cls.options[section]:
            raise ConfigError("Option \"%s\" is already in use.")
        if hasattr(cls, attrname):
            raise ConfigError("Settings attribute \"%s\" is already in use.")
        if environment and environment in cls.environment:
            raise ConfigError("Settings environment varaible \"%s\" is"
                              "already in use.")
        if not type_ and default is None:
            raise ConfigError("Settings attribute \"%s\" has must have a"
                              " type or a default." % attrname)
        if not type_:
            type_ = type(default)
        if notify:
            notification = Notification(attrname)
            setattr(cls, attrname, notification)
            setattr(cls, "_" + attrname, default)
            GObject.signal_new(notification.signame,
                               cls,
                               GObject.SIGNAL_RUN_LAST,
                               None,
                               ())
        else:
            setattr(cls, attrname, default)
        if section and key:
            cls.options[section][attrname] = type_, key, environment
        cls.environment.add(environment)
        cls.defaults[attrname] = default

    @classmethod
    def addConfigSection(cls, section):
        """
        Add a section to the local config file.

        @param section: The section name. This section must not already exist.
        @type section: C{str}
        """
        if section in cls.options:
            raise ConfigError("Duplicate Section \"%s\"." % section)
        cls.options[section] = {}

    @classmethod
    def notifiesConfigOption(cls, attrname):
        signal_name = Notification.signalName(attrname)
        GObject.signal_lookup(signal_name, cls)
示例#57
0
class ZFind:

    def __init__(self, num_z=5, inifile=None, dest=None, clobber=True):
        self.num_z = num_z
        self.inifile = inifile
        self.dest = dest
        self.clobber = clobber
        if self.inifile: self.set_templates_from_inifile()

    def set_templates_from_inifile(self):
        self.labels = []
        self.templates = []
        self.zmin = []
        self.zmax = []
        self.npoly = []
        self.npixstep = []

        if exists(self.inifile):
            self.option = SafeConfigParser()
            self.option.optionxform = str
            r = self.option.read(self.inifile)
            if len(r) == 1:
                for section in self.option.sections():
                    self.labels.append(section)
                    if self.option.has_option(section,'template'):
                        self.templates.append(self.option.get(section,
                                                              'template'))
                    if self.option.has_option(section,'zmin'):
                        self.zmin.append(self.option.getfloat(section,'zmin'))
                    if self.option.has_option(section,'zmax'):
                        self.zmax.append(self.option.getfloat(section,'zmax'))
                    if self.option.has_option(section,'npoly'):
                        self.npoly.append(self.option.getint(section,'npoly'))
                    if self.option.has_option(section,'npixstep'):
                        self.npixstep.append(self.option.getint(section,
                                                                'npixstep'))
            else: print("Cannot parse ini file %r" % self.inifile)

            if not self.labels: self.labels = None
            if not self.templates: self.templates = None
            if not self.zmin: self.zmin = None
            if not self.zmax: self.zmax = None
            if not self.npoly: self.npoly = None
            if not self.npixstep: self.npixstep = None

            self.set_templates()
        else: print("WARNING: %r does not exist" % self.inifile)


    def set_templates(self, templates=None, zmin=None, zmax=None, npoly=None, npixstep=None):
        if templates: self.templates = templates
        if zmin: self.zmin = zmin
        if zmax: self.zmax = zmax
        if npoly: self.npoly = npoly
        if npixstep: self.npixstep = npixstep

        if type(self.templates) is str:
            try: self.templates = [self.templates]
            except:
                print('Templates not a list and unable to convert to list!')
                sys.exit(1)
        if type(self.templates) is list: self.templates = list(map(str, self.templates))
        if self.zmin is not None:
            if type(self.zmin) is not list:
                try:
                    self.zmin = [self.zmin]
                except:
                    try:
                        self.zmin = self.zmin.tolist()
                    except:
                        print('Can\'t convert zmin to list - defaulting to full zrange!')
                        self.zmin = None
                        self.zmax = None
            if type(self.zmin) is list:
                if len(self.zmin) != len(self.templates):
                    print('Length of zmin doesn\'t match length of templates - defaulting to full zrange!')
                    self.zmin = None
                    self.zmax = None
                if self.zmax is None:
                    print('zmax not given - defaulting to full zrange!')
                    self.zmin = None
                    self.zmax = None
                else:
                    if type(self.zmax) is not list:
                        try: self.zmax = [self.zmax]
                        except:
                            try:
                                self.zmax = self.zmax.tolist()
                            except:
                                print('Can\'t convert zmax to list - defaulting to full zrange!')
                                self.zmin = None
                                self.zmax = None
                    if len(self.zmin) != len(self.zmax):
                        print('Length of zmin and zmax don\'t match - defaulting to full zrange!')
                        self.zmin = None
                        self.zmax = None
        #import pdb; pdb.set_trace()
        if self.npoly is None:
            self.npoly = [4]*len(self.templates)
        else:
            if type(self.npoly) is not list:
                try:
                    self.npoly = [self.npoly]
                except:
                    try:
                        self.npoly = self.npoly.tolist()
                    except:
                        print('npoly not a list and unable to convert to \
                                list - defaulting to npoly=4 for all templates!')
                        self.npoly = [4]*len(self.templates)
            else:
                self.npoly = list(map(int, self.npoly))
        if self.npixstep is None:
            self.npixstep = [1]*len(self.templates)
        else:
            if type(self.npixstep) is not list:
                try:
                    self.npixstep = [self.npixstep]
                except:
                    try:
                        self.npixstep = self.npixstep.tolist()
                    except:
                        print('npixstep not a list and unable to convert to \
                                list - defaulting to npixstep=1 for all \
                                templates!')
                        self.npixstep = [1]*len(self.templates)
            else:
                self.npixstep = list(map(int, self.npixstep))

    def reduce_plate_mjd(self, plate, mjd, fiberid=None, chi2file=False):
        self.chi2file = chi2file
        # Check types and try to convert to proper types if necessary
        if fiberid is None: fiberid = [i for i in range(1000)]
        else:
            if type(fiberid) is not list:
                try:
                    fiberid = [fiberid]
                    fiberid = list(map(int, fiberid))
                except:
                    try:
                        fiberid = fiberid.tolist()
                        fiberid = list(map(int, fiberid))
                    except:
                        print('fiberid not set properly - running full plate!')
                        fiberid = [i for i in range(1000)]
            else: fiberid = list(map(int, fiberid))

        # Spec
        specs = spec.Spec(plate=plate, mjd=mjd, fiberid=fiberid)

        # ZFinder, ZFitter
        zfindobjs = []
        zfitobjs = []
        if (self.zmin is not None) & (self.zmax is not None):
            for i in range(len(self.templates)):
                zfindobjs.append( zfinder.ZFinder(fname=self.templates[i],
                                                  npoly=self.npoly[i],
                                                  zmin=self.zmin[i],
                                                  zmax=self.zmax[i]) )
                zfindobjs[i].zchi2( specs.flux, specs.loglambda, specs.ivar,
                                   npixstep=self.npixstep[i], plate=plate,
                                   mjd=mjd, fiberid=fiberid[0],
                                   chi2file=self.chi2file )
                zfitobjs.append( zfitter.ZFitter(zfindobjs[i].zchi2arr,
                                                 zfindobjs[i].zbase) )
                zfitobjs[i].z_refine()
        else:
            for i in range(len(self.templates)):
                zfindobjs.append( zfinder.ZFinder(fname=self.templates[i],
                                                  npoly=self.npoly[i],
                                                  npixstep=self.npixstep[i]) )
                zfindobjs[i].zchi2( specs.flux, specs.loglambda, specs.ivar,
                                   npixstep=self.npixstep[i], plate=plate,
                                   mjd=mjd, fiberid=fiberid[0],
                                   chi2file=self.chi2file )
                zfitobjs.append( zfitter.ZFitter(zfindobjs[i].zchi2arr,
                                                 zfindobjs[i].zbase) )
                zfitobjs[i].z_refine()

        # Flags
        flags = []
        for i in range(len(zfindobjs)):
            flags.append( misc.comb_flags(specs, zfindobjs[i], zfitobjs[i]) )

        # ZPicker
        if len(self.templates) == 1:
            zpick = zpicker.ZPicker(specs, zfindobjs[0], zfitobjs[0], flags[0])
        elif len(self.templates) == 2: zpick = zpicker.ZPicker(specs, zfindobjs[0], zfitobjs[0], flags[0], zfindobjs[1], zfitobjs[1], flags[1])
        elif len(self.templates) == 3: zpick = zpicker.ZPicker(specs, zfindobjs[0], zfitobjs[0], flags[0], zfindobjs[1], zfitobjs[1], flags[1],
                                                          zfindobjs[2], zfitobjs[2], flags[2])
        elif len(self.templates) == 4: zpick = zpicker.ZPicker(specs, zfindobjs[0], zfitobjs[0], flags[0], zfindobjs[1], zfitobjs[1], flags[1],
                                                          zfindobjs[2], zfitobjs[2], flags[2], zfindobjs[3], zfitobjs[3], flags[3])
        elif len(self.templates) == 5: zpick = zpicker.ZPicker(specs, zfindobjs[0], zfitobjs[0], flags[0], zfindobjs[1], zfitobjs[1], flags[1],
                                                          zfindobjs[2], zfitobjs[2], flags[2], zfindobjs[3], zfitobjs[3], flags[3],
                                                          zfindobjs[4], zfitobjs[4], flags[4])

        output = None

        # Write output
        if self.dest is None:
            output = io.WriteRedmonster(zpick, clobber=self.clobber)
        else:
            if type(self.dest) is str:
                output = io.WriteRedmonster(zpick, dest=self.dest, clobber=self.clobber)
            else:
                try:
                    self.dest = str(self.dest)
                    output = io.WriteRedmonster(zpick, dest=self.dest, clobber=self.clobber)
                except:
                    print('Could not convert dest to string - writing to default directory and NOT clobbering old files!')
                    output = io.WriteRedmonster(zpick, clobber=True)

        if output:
            if len(zpick.fiberid) == 1: output.write_fiberid()
            else: output.write_plate()
示例#58
0
logging.basicConfig(format="%(message)s", level=logging.INFO)

if len(sys.argv) == 1:
    logging.critical("No config file specified")
    sys.exit(1)

vibrating = False
appliance_active = False
last_vibration_time = time.time()
start_vibration_time = last_vibration_time

config = SafeConfigParser()
config.read(sys.argv[1])
verbose = config.getboolean("main", "VERBOSE")
sensor_pin = config.getint("main", "SENSOR_PIN")
begin_seconds = config.getint("main", "SECONDS_TO_START")
end_seconds = config.getint("main", "SECONDS_TO_END")

mqtt_hostname = config.get("mqtt", "mqtt_hostname")
mqtt_port = int(config.get("mqtt", "mqtt_port"))
mqtt_topic = config.get("mqtt", "mqtt_topic")
mqtt_availability_topic = config.get("mqtt", "mqtt_availability_topic")
mqtt_username = config.get("mqtt", "mqtt_username")
mqtt_password = config.get("mqtt", "mqtt_password")
mqtt_clientid = config.get("mqtt", "mqtt_clientid")

start_message = config.get("main", "START_MESSAGE")
end_message = config.get("main", "END_MESSAGE")
boot_message = config.get("main", "BOOT_MESSAGE")
term_message = config.get("main", "TERM_MESSAGE")
示例#59
0
    plt.plot(freq, phase)
    plt.grid('on')
    plt.title('FFT - Phase')
    plt.ylabel('Phase [degrees]')
    plt.xlabel('Frequency [Hz]')
    return plt

#####################################################################
# Main script

# Init
config = SafeConfigParser()
config.read('config.ini')

fs = config.getfloat('Test','fs') # sampling frequency in Hz
adc_resolution_bits = config.getint('Test','adc_resolution_bits')
amp_threshold = config.getfloat('Test','amp_threshold')
data_dir = config.get('Test','data_dir')
data_file_name = config.get('Test','data_file_name')
freq_file_name = config.get('Test','freq_file_name')
power_file_name = config.get('Test','power_file_name')

# Load power and frequency data
data = numpy.loadtxt(data_dir + power_file_name, delimiter = ',', skiprows = 1)
fsig_array = [d[0] for d in data]
fund_power_array = [d[1] for d in data]

# Generate fourier series for each frequency and plot
for i, power in enumerate(fund_power_array):

    data = numpy.loadtxt(data_dir + data_file_name + str(int(fsig_array[i])) + '.csv',