class Parse_MitmConfig(): def __init__(self, configFile): self.config = ConfigParser() self.configFile = configFile self.trigger_parsing() ''' trigger_parsing is a seperate function, to reread the configuration file if needed. This feature might be implemented in the future ''' def trigger_parsing(self): # TODO Check if file exists self.config.readfp(open(self.configFile)) self.dst_ip = self.config.get('Destination', 'ip') self.dst_mac = self.config.get('Destination', 'mac') self.dst_port = self.config.get('Destination', 'port') self.src_ip = self.config.get('Source', 'ip') self.src_mac = self.config.get('Source', 'mac') self.src_port = self.config.get('Source', 'port') self.bridge0_interface = self.config.get('Interfaces', 'bridge0') self.bridge1_interface = self.config.get('Interfaces', 'bridge1') self.mitm_interface = self.config.get('Interfaces', 'mitm')
def parseConfigurationFile(self, configFile): # Initialize defaults defaultParams = {} # We have the option of collecting stats during burnin # TODO: RENAME THESE defaultParams["collect_burnin_stats_intvl"] = 100 defaultParams["results_dir"] = "." defaultParams["burnin"] = 10000 defaultParams["samples"] = 1000 defaultParams["thin"] = 10 defaultParams["restarts"] = 1 # Create a config parser object and read in the file cfgParser = ConfigParser(defaultParams) cfgParser.read(configFile) self.params = {} self.params["results_dir"] = cfgParser.get("io", "results_dir") self.params["results_file"] = cfgParser.get("io", "results_file") self.params["samples"] = cfgParser.getint("MCMC", "samples") self.params["samples_thin"] = cfgParser.getint("MCMC", "thin") self.params["burnin"] = cfgParser.getint("MCMC", "burnin") self.params["burnin_thin"] = cfgParser.getint("output", "collect_burnin_stats_intvl") self.params["restarts"] = cfgParser.getint("MCMC", "restarts") # Update N_samples and N_burnin self.params["N_samples"] = self.params["samples"] * self.params["restarts"] self.params["N_burnin"] = np.ceil(self.params["burnin"] / self.params["burnin_thin"]) * self.params["restarts"] # Basic error checking if not os.path.exists(self.params["results_dir"]): log.error("Results directory does not exist!") raise Exception("Invalid parameter: io.results_dir %s" % self.params["results_dir"])
class handlers: def __init__(self, root): self.root = root ## quit out of everything def Quit(self): print "quiting...\n" sys.exit() def Record(self): sys.stdout.write("Cant record yet") sys.Quit() def SaveConfig(self): sys.stdout.write("TBD") sys.Quit() def LoadIniData(self, FileName): self.cp = ConfigParser() try: self.cp.readfp(open(FileName, "r")) # f.close() except IOError: raise Exception, "NoFileError" return
def mf_construct_model_extensions(baseModel, configFile): """ Initialize a model by iterating through the config file """ extensions = {} cfgParser = ConfigParser() cfgParser.read(configFile) # Iterate over each section. If it is one of the recognized types, # call the constructor for that section for section in cfgParser.sections(): params = dict(cfgParser.items(section)) model = None if section == "cluster_model": model = mfConstructClusterModel(baseModel, configFile, params) elif section == "location_model": model = mfConstructLocationModel(baseModel, configFile, params) # Add this model to the extensions dictionary if model != None: if section not in extensions.keys(): extensions[section] = model else: # If there is already an extension with this section name, append this to the list # This is to support multiple clusters or location models extensions[section] = list(extensions[section]) + [model] return extensions
def __init__(self, configFile=None): """ Initialize all values and setup :param configFile: """ if configFile is None: configFile = u"rmq_settings.conf" Config = ConfigParser() Config.read(configFile) self.vhost = ConfigSectionMap(u"Send", Config)[u'vhost'] self.queue = ConfigSectionMap(u"Send", Config)[u'queue'] self.routing_key = ConfigSectionMap(u"Send", Config)[u'routing_key'] self.exchange = ConfigSectionMap(u"Send", Config)[u'exchange'] self.host = ConfigSectionMap(u"Send", Config)[u'host'] self.port = int(ConfigSectionMap(u"Send", Config)[u'port']) username = ConfigSectionMap(u"Send", Config)[u'username'] password = ConfigSectionMap(u"Send", Config)[u'password'] credentials = pika.PlainCredentials(username, password) self.connection = pika.BlockingConnection\ ( pika.ConnectionParameters(host=self.host, port=self.port, virtual_host=self.vhost, credentials=credentials) ) self.channel = self.connection.channel()
def build_packets(self, dp): # config_dir, working_set = ConfigParser.get_working_set(self.CONFIG_FILE) # for filename in working_set: # full_filename = config_dir + '/' + filename print "------------------------------------------------------------" print "processing file: %s" % full_filename print "------------------------------------------------------------" # config = ConfigParser.get_config(full_filename) # for type in ConfigParser.get_config_type(config): # if (type == "flow_mod"): mod_config = ConfigParser.get_flow_mod(config) mod = Mods.create_flow_mod(dp, mod_config) # elif (type == "group_mod"): # mod_config = ConfigParser.get_group_mod(config) mod = Mods.create_group_mod(dp, mod_config) print "mod len: %i" % sys.getsizeof(mod) dp.send_msg(mod) print "message sent"
class PluginManager: def __init__(self, cfgFile): self.mainPlugin = None self.loaded = {} self.config = ConfigParser() self.configFile = cfgFile self.rehash() def getProcesses(self): """Returns a list of all plugins, as well as the main plugin""" return self.loaded.values() + [self.mainPlugin] def launchPlugin(self, name): """Launches a new plugin, returning the plugin if it succeded""" p = Plugin(name, self.config.get(name,"exec")) print "Launched",name if p.isRunning(): return p print "Failed." return None def rehash(self): """reread the config file""" self.config.read(self.configFile) def load(self, name): """Loads a plugin and stores it in the plugin table""" print "load:",name if self.loaded.has_key(name): print "Already loaded, reloading." self.unload(name) plugin = self.launchPlugin(name) if plugin: self.loaded[name] = plugin else: print "Failed to load" def unload(self, name): """Stops a plugin and removes it from the plugin table""" print "unload:",name if not self.loaded.has_key(name): print "Not loaded. Skipping." return self.loaded[name].stop() del self.loaded[name] print "Unloaded" def startMain(self): """Starts the main exec and returns whether it was a successful start""" self.mainPlugin = self.launchPlugin("main") return self.mainPlugin is not None def startPlugins(self): """Starts the (non main) plugins and stores them all in the plugin table""" for plugin in self.config.sections(): if plugin != "main": self.load(plugin)
def python3parser(): ''' Return a ConfigParser (Python 3) ''' from configparser import ConfigParser cp3 = ConfigParser() cp3.optionxform = str return cp3
def load_files(filenames, **kwds): ''' Parse configuration files, return configuration data structure. ''' cfg = ConfigParser() cfg.read(filenames) d = dictify(cfg) return interpolate(d, **kwds)
def send_text(sender, recipient, text_message, config=None): if not config: config = ConfigParser('config/development.cfg') client = TwilioRestClient(config.get('twilio', 'account_sid'), config.get('twilio', 'auth_token')) sms = client.sms.messages.create(body=text_message, to=recipient, from_=sender)
def __init__ (self,filename="timba.conf", system_paths=_default_system_paths, user_path=_default_user_path): self.syscp = ConfigParser(); system_paths = [ os.path.join(path,filename) for path in system_paths ]; self.syscp.read(system_paths); self.usercp = ConfigParser(); self._user_file = os.path.join(user_path,"."+filename); self.usercp.read([self._user_file]);
def read_configuration(section): cp = ConfigParser() cp.read(config_file) #options = {} #for i in cp.items('Parser'): # options[i[0]] = i[1] return dict(cp.items(section))
def get_common_config(): from os.path import expanduser ROOT_PATH = expanduser("~") CONTACT_FILE_LOC = os.path.join(ROOT_PATH, ".contacts.pkl") CONFIG_FILE_LOC = os.path.join(ROOT_PATH, ".config.ini") config = ConfigParser() config.read(CONFIG_FILE_LOC) return config
def update_config(configfile=None, modulename=None): """Update the system configuration from given config file and/or module. 'configfile' is a ConfigObj (INI-style) config file, 'modulename' a module path in dotted notation. The function looks for files with a ".cfg" extension if the given module name refers to a package directory or a file with the base name of the right-most part of the module path and a ".cfg" extension added. If both 'configfile' and 'modulname' are specified, the module is read first, followed by the config file. This means that the config file's options override the options in the module file. """ defaults = config_defaults() configdata = ConfigObj(unrepr=True) if modulename: lastdot = modulename.rfind('.') firstdot = modulename.find('.') packagename = modulename[:lastdot] top_level_package = modulename[:firstdot] modname = modulename[lastdot+1:] modfile = resource_filename(packagename, modname + '.cfg') if not os.path.exists(modfile): modfile = resource_filename(packagename, modname) if os.path.isdir(modfile): configfiles = glob.glob(os.path.join(modfile, '*.cfg')) else: configfiles = [modfile] top_level_dir = os.path.normpath(resource_filename(top_level_package, '')) package_dir = os.path.normpath(resource_filename(packagename, '')) defaults.update(dict(top_level_dir=top_level_dir, package_dir=package_dir)) # Update Python logging config for pathname in configfiles: if 'app.cfg' in pathname: parser = ConfigParser() conf = parser.dict_from_files(pathname, vars=defaults) else: obj = ConfigObj(pathname, unrepr=True) obj.merge(dict(DEFAULT=defaults)) conf = obj.dict() configdata.merge(conf) if configfile: obj = ConfigObj(configfile, unrepr=True) obj.merge(dict(DEFAULT=defaults)) conf = obj.dict() configdata.merge(conf) update(configdata.dict())
def load(section, option, archive=_ConfigFile): """ Load variable """ cfg = ConfigParser() try: cfg.readfp(file(archive)) except Exception, e: sys.stderr.write("%s, %s\n" % (archive, e.strerror)) return
def __init__(self): parser = ConfigParser() parser.read('config.ini') #sets the key bindings keys = parser.get('controls', 'keys') keyslist = keys.split(',') for i in range(len(keyslist)): self.accept(keyslist[i],self.hitcountfunction) print "The key '"+keyslist[i]+"' has been added."
def _tryLoadConfig(): 'load the config from file' c = ConfigParser() if c.read('gateway.ini'): IOT.GatewayId = c.get('general', 'gatewayId') IOT.ClientId = c.get('general', 'clientId') IOT.ClientKey = c.get('general', 'clientKey') return True else: return False
def save(pom, APPPATH, CONFIG): """ Method to save the changes (volume, song,...) in the configuration file (By default pomfig.cfg) @param pom The Pomodoro to set the options @type pom Pomodoro @param APPPATH The path to the apply @type APPPATH String @param CONFIG The name of the config archive (*.cfg) @type CONFIG String """ cfg = ConfigParser() cfg.read(APPPATH + '/' + CONFIG) cfg.set("configuration", "volume", pom.volume) if(pom.filemp3 != ""): cfg.set("configuration", "song", pom.filemp3) f = open(APPPATH + '/' + CONFIG, "w") cfg.write(f) f.close()
def save_config(self, prop_name, prop_value): # TODO make the operation an transaction. config = ConfigParser() if os.path.exists(self.config_file_path): config.read(self.config_file_path) # read values from a section if not config.has_section(self.azure_crypt_config_section): config.add_section(self.azure_crypt_config_section) config.set(self.azure_crypt_config_section, prop_name, prop_value) with open(self.config_file_path, "wb") as configfile: config.write(configfile)
def LoadLabels(self,NEW_LABS = None): ''' Need to comment further. Ambiguous input Capable of loading several different types of labels depending on the NEW_LABS has ARG: NEW_LABS TYPE: None,string,list,np.ndarray DESC: TODO ''' if self.DATA_LOADED == False: self.NewInput() #if called default: a random label is created #This is mostly for testing if NEW_LABS is None: if self.CONFIG_LOADED: self.LABELS = np.random.randint(0,self.NUM_CLASS, (np.size(self.TRNG_DATA,0),1)) self.LABELS = np.transpose(np.sort(self.LABELS)) else: self.LoadStackConfig() self.LABELS = np.random.randint(0,self.NUM_CLASS, (np.size(self.TRNG_DATA,0),1)) self.LABELS = np.transpose(self.LABELS) #if called witha string: function checks to see if that string # says 'load' and if so loads the file using the name in the config # file. If not, it will try to use that string as a label vector #and if that fails, a blank list is returned # Maybe I over thought this whole thing elif type(NEW_LABS) is str: if NEW_LABS == 'load': cp = ConfigParser() LABFILE_NAME = cp.get('ASCII_DATA', 'labels_filename') LABFILE = open(LABFILE_NAME, 'rb') self.LABELS = pickle.load(LABFILE) self.LABELS = np.transpose(self.LABELS) else: try: self.LABELS = np.array(list(NEW_LABS)) self.LABELS = np.transpose(self.LABELS) except ValueError('Not not valid labels'): return [] #if called with a list: the function will convert the list into an array elif type(NEW_LABS) is list: self.LABELS = np.asarray(NEW_LABS) self.LABELS = np.transpose(self.LABELS) #if function is called with an np.ndarray. the array is stored in the #class parameter elif type(NEW_LABS) is np.ndarray: self.LABELS = NEW_LABS #if called with AAAnything else: the function will fail/ else: raise TypeError('Labels can be of type None, strings, lists, or numpy.ndarray')
def writeINI(self, filename): """Write the version map to an INI file. """ cp = ConfigParser() for key, value in self.iteritems(): cp.set('versions', key, value) fo = open(filename, 'wb') try: cp.write(fo) finally: fo.close()
def readINI(self, filename): """Read the version map from an INI file. """ cp = ConfigParser() fi = open(filename, 'rb') try: cp.read(fi) finally: fi.close() self.update(cp.items('versions'))
def load_text(text, **kwds): ''' Parse configuration text, return configuration data structure. ''' cfg = ConfigParser() if hasattr(cfg, 'read_string'): cfg.read_string(text) else: import io cfg.readfp(io.BytesIO(text)) d = dictify(cfg) return interpolate(d, **kwds)
def map_config(configparser, section): dict1 = {} options = configparser.options(section) for option in options: try: dict1[option] = configparser.get(section, option) if dict1[option] == -1: print("skip: %s" % option, f=sys.stderr) except: print("exception on %s!" % option) dict1[option] = None return dict1
def cpm(self, task): global time global timer parser = ConfigParser() parser.read('config.ini') timer = int(parser.get('settings','timer')) time = task.time if task.time < timer: task.time and \ bpmText.setText("BPM: "+str(round((hitcount / ( task.time / 60 ))))) osuText.setText("osu!: "+str(round((hitcount / ( (task.time + .01) / 60 ))/4))) return task.cont
def save_configs(self, key_value_pairs): config = ConfigParser() if os.path.exists(self.config_file_path): config.read(self.config_file_path) # read values from a section if not config.has_section(self.azure_crypt_config_section): config.add_section(self.azure_crypt_config_section) for key_value_pair in key_value_pairs: if key_value_pair.prop_value is not None: config.set(self.azure_crypt_config_section, key_value_pair.prop_name, key_value_pair.prop_value) with open(self.config_file_path, "wb") as configfile: config.write(configfile)
def getContext(self): """Create an SSL context. This will load SSL Public and private certs from the file. """ config = ConfigParser() apppath = os.environ['ZIVIOSAGENTHOME'] config.read(apppath+"/config/ZiviosAgentManager.ini") cafile = config.get("general","sslcert") prvfile = config.get("general","sslprivatekey") ctx = SSL.Context(SSL.SSLv23_METHOD) ctx.use_certificate_file(cafile) ctx.use_privatekey_file(prvfile) return ctx
def getConfigurationFromINI(): """ Return a dictionary of section with the configuration read from the ini file merged with the defaults passed. Various ini location are tried. """ defaults = _getConfigurationDefaults() ini = ConfigParser() x = os.path.join(_ROOT, 'serclient.ini') v = ini.read([x, '/opt/serclient/serclient.ini']) c = dict(ini._sections) for k, v in defaults.items(): if k in c.keys(): v.update(c[k]) return defaults
def DobOpcii(CP, IC, sekcii): """ Обработка ошибки NoOptionError, которая может возникнуть при различии версии модуля и его настроек. DobOpcii(CP, IC, sekcii) IC - Файл из которого будут взяты недостающие настройки sekcii - Список Секций которые будут обрабатыватся """ if os.path.isfile(os.path.abspath(IC)): ICF = ConfigParser(); ICF.readfp(open(IC, "r")) for x in sekcii: Sopcii = CP.options(x) for y in ICF.options(x): if not y in Sopcii: CP.set(x, y, ICF.get(x,y))
def LoadLabels(self, NEW_LABS=None): ''' DESCRPT: PRECOND: POSTCON: IN ARGS: RETURNS: NOTES: ''' if self.DATA_LOADED == False: self.NewInput() if NEW_LABS is None: if self.CONFIG_LOADED: self.TRNG_LABS = np.random.randint(0, self.NUM_CLASS, (np.size(self.TRNG_DATA, 0), 1)) self.TRNG_LABS = np.transpose(np.sort(self.TRNG_LABS)) else: self.LoadStackConfig() self.TRNG_LABS = np.random.randint(0, self.NUM_CLASS, (np.size(self.TRNG_DATA, 0), 1)) self.TRNG_LABS = np.transpose(self.TRNG_LABS) elif type(NEW_LABS) is str: if NEW_LABS == 'load': cp = ConfigParser() LABFILE_NAME = cp.get('ASCII_DATA', 'labels_filename') LABFILE = open(LABFILE_NAME, 'rb') self.TRNG_LABS = pickle.load(LABFILE) self.TRNG_LABS = np.transpose(self.TRNG_LABS) else: try: self.TRNG_LABS = np.array(list(NEW_LABS)) self.TRNG_LABS = np.transpose(self.TRNG_LABS) except ValueError('Not not valid labels'): return [] elif type(NEW_LABS) is list: self.TRNG_LABS = np.asarray(NEW_LABS) self.TRNG_LABS = np.transpose(self.TRNG_LABS) elif type(NEW_LABS) is np.ndarray: self.TRNG_LABS = NEW_LABS else: raise TypeError('Labels can be of type None, strings, lists, or numpy.ndarray') if self.TRNG_LABS is not None: self.CHECKS['labels_loaded'] = True
# basename = os.path.splitext(os.path.basename(options.config_file))[0] log_fh = open(basename + '.pipeline.log', 'w') # FIXME: the following code uses obsolete CVS ID tags. # It should be modified to use git version information. print >> log_fh, "$Id$\nInvoked with arguments:" for name_value in options.__dict__.items(): print >> log_fh, "%s %s" % name_value print >> log_fh # # create the config parser object and read in the ini file # config_parser = ConfigParser.ConfigParser() config_parser.read(options.config_file) # # initialize lalapps.power and lalapps.cosmicstring modules # power.init_job_types(config_parser, job_types=("datafind", "binj", "lladd", "binjfind", "burca", "sqlite")) cosmicstring.init_job_types(config_parser, job_types=("string", "meas_likelihood", "calc_likelihood", "runsqlite")) # # make directories to store the cache files, job logs, and trigger output
def newSession(self): start_time = time() print "Starting a new BitGrail session with Captcha." Config = ConfigParser.ConfigParser() Config.read("./.settings.ini") email = Config.get("BitGrail", 'email') password = Config.get("BitGrail", 'password') API_KEY_2CAPTCHA = Config.get("2captcha", '2CAPCHA_API_KEY') recapcha_sitekey = Config.get("BitGrail", 'recapcha_sitekey') # bitgrail submit_url = "https://bitgrail.com/login" # http://scraping.pro/2captcha-service-to-solve-recaptcha-2/ # send credentials to the service to solve captcha # returns service's captcha_id of captcha to be solved url = "http://2captcha.com/in.php?key=" + API_KEY_2CAPTCHA + "&method=userrecaptcha&googlekey=" + recapcha_sitekey + "&pageurl=" + submit_url resp = requests.get(url) if resp.text[0:2] != 'OK': print(resp.text) quit('Error. Captcha is not received') captcha_id = resp.text[3:] # print("Captcha ID = ",captcha_id) if (captcha_id[0:3] == 'OR_'): quit("Error: " + captcha_id) # fetch ready 'g-recaptcha-response' token for captcha_id fetch_url = "http://2captcha.com/res.php?key=" + API_KEY_2CAPTCHA + "&action=get&id=" + captcha_id print "Waiting for captcha to be solved by mechanical Turk..." captcha_ok = False for i in range(1, 200): sleep(1) resp = requests.get(fetch_url) print ".", sys.stdout.flush() if resp.text[0:2] == 'OK': captcha_ok = True break if (captcha_ok != True): quit("Captcha isn't OK after " + (time() - start_time) + " s: " + resp.text) print('\nDone. Time to solve captcha: ', time() - start_time) headers = { 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36', 'referrer': 'https://bitgrail.com/login', 'x-requested-with': 'XMLHttpRequest' } # POST parameters, might be more, depending on form content # resp = requests.post(submit_url, headers=headers, files=payload) self._session_ = requests.Session() self._session_.get("https://bitgrail.com/") payload = { 'email': (None, email), 'ajax': (None, '1'), 'password': (None, password), 'login': (None, '1'), 'submit': (None, 'Login'), 'g-recaptcha-response': (None, resp.text[3:]) } resp = self._session_.post(submit_url, headers=headers, files=payload) # print(resp.text) if (not resp.text.find('icon-ok-circled')): print "BitGrail: printing full dump or response" # data = dump.dump_response(resp) # print(data.decode('utf-8')) print resp.text quit("No valid response from login.") submit_url = "https://bitgrail.com/login2fa" two_factor_token = raw_input("Enter your 2FA code: ").strip() print("You entered: " + two_factor_token) payload = { 'token2fa': (None, two_factor_token), 'login2fa': (None, '1'), 'ajax': (None, '1') } resp = self._session_.post(submit_url, headers=headers, files=payload) # print(resp.text) if (not resp.text.find('icon-ok-circled')): print "BitGrail: printing full dump or response" # data = dump.dump_response(resp) # print(data.decode('utf-8')) print resp.text quit("No valid response from login.") sdata = self.store_session() with open("bg_session.dump", "w") as f: f.write(sdata) return self._session_
def __init__(self, login_configfile, charset='utf8'): self.charset = charset self.configparser = ConfigParser.RawConfigParser() self.configparser.readfp(open(login_configfile)) self.loginfo = dict(self.configparser.items('database'))
def create_collector_config(self, workdir): """ Telegraf collector config, toml format """ cfg_path = "agent_collector_{}.cfg".format(self.host) if os.path.isfile(cfg_path): logger.info( 'Found agent config file in working directory with the same name as created for host %s.\n' 'Creating new one via tempfile. This will affect predictable filenames for agent artefacts', self.host) handle, cfg_path = tempfile.mkstemp('.cfg', 'agent_collector_') os.close(handle) self.monitoring_data_output = "{remote_folder}/monitoring.rawdata".format( remote_folder=workdir) defaults_old_enabled = ['CPU', 'Memory', 'Disk', 'Net', 'System'] try: config = ConfigParser.RawConfigParser() config.add_section("global_tags") config.add_section("agent") config.set( "agent", "interval", "'{interval}s'".format(interval=self.interval)) config.set("agent", "round_interval", "true") config.set("agent", "flush_interval", "'1s'") config.set("agent", "collection_jitter", "'0s'") config.set("agent", "flush_jitter", "'1s'") for section in self.host_config.keys(): # telegraf-style config if not self.old_style_configs: config.add_section( "{section_name}".format( section_name=self.host_config[section]['name'])) for key, value in iteritems(self.host_config[section]): if key != 'name': config.set( "{section_name}".format( section_name=self.host_config[section][ 'name']), "{key}".format(key=key), "{value}".format(value=value)) # monitoring-style config else: if section in defaults_old_enabled: config.add_section( "{section_name}".format( section_name=self.host_config[section]['name'])) for key, value in iteritems(self.host_config[section]): if key in [ 'fielddrop', 'fieldpass', 'percpu', 'devices', 'interfaces' ]: config.set( "{section_name}".format( section_name=self.host_config[section][ 'name']), "{key}".format(key=key), "{value}".format(value=value)) # outputs config.add_section("[outputs.file]") config.set( "[outputs.file]", "files", "['{config}']".format(config=self.monitoring_data_output)) config.set("[outputs.file]", "data_format", "'json'") with open(cfg_path, 'w') as fds: config.write(fds) # dirty hack, this allow to avoid bash escape quoting, we're pushing shell script w/ arguments # index of argument is index of custom metric in our config inputs = "" for idx, cmd in enumerate(self.custom): inputs += "[[inputs.exec]]\n" inputs += "commands = ['/bin/sh {workdir}/agent_customs.sh -{idx}']\n".format( workdir=workdir, idx=idx) inputs += "data_format = 'value'\n" inputs += "data_type = 'float'\n" inputs += "name_prefix = '{}_'\n\n".format(cmd.get('label')) if cmd['diff']: decoder.diff_metrics['custom'].append( decoder.find_common_names(cmd.get('label'))) with open(cfg_path, 'a') as fds: fds.write(inputs) # telegraf raw configuration into xml telegraf_raw = "" for element in self.telegrafraw: telegraf_raw += element with open(cfg_path, 'a') as fds: fds.write(telegraf_raw) except Exception as exc: logger.error( 'Error trying to create monitoring config. Malformed? %s', exc, exc_info=True) return cfg_path
def send_sms(message): #read parameters that is needed to send sms from_number = config_parser.get("SMS_SECTION", "from_number") to_number = config_parser.get("SMS_SECTION", "to_number") twilio_account_number = config_parser.get("SMS_SECTION", "twilio_account_number") twilio_account_token = config_parser.get("SMS_SECTION", "twilio_account_token") client = TwilioRestClient(twilio_account_number, twilio_account_token) client.messages.create(to=to_number, from_=from_number, body=message) if __name__ == '__main__': #read and initialize from config config_file = os.path.join(os.path.pardir, "config", "Config.txt") config_parser = ConfigParser.RawConfigParser() config_parser.read(config_file) #initialize logger logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) # create a file handler handler = logging.FileHandler(os.path.join(os.path.pardir, "log", "logger.log")) handler.setLevel(logging.DEBUG) # create a logging format formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') handler.setFormatter(formatter) logger.addHandler(handler)
def __init__(self,confFile): self.parser = argparse.ArgumentParser() self.parser.add_argument(confFile,help="Need a configuration file.") self.args = self.parser.parse_args() self.inargs = {} ########### Configuration ############ self.Config = ConfigParser.ConfigParser() self.configFile = self.args.confFile self.Config.read(self.configFile) self.template_file = self.ConfigSectionMap("General")['template'] self.exp_template_file = self.ConfigSectionMap("General")['exp_template'] self.chiplist = self.ConfigSectionMap("General")['chiplist'] self.data_dir = self.ConfigSectionMap("General")['data_dir'] self.data_file = self.ConfigSectionMap("General")['data_file'] self.data_conf = self.ConfigSectionMap("General")['conf_dir'] self.correc_dir = self.ConfigSectionMap("General")['corr_dir'] self.year = self.ConfigSectionMap("General")['year'] self.yearb = self.ConfigSectionMap("General")['yearb'] self.epoch = self.ConfigSectionMap("General")['epoch'] self.epochb = self.ConfigSectionMap("General")['epochb'] self.FILTER = self.ConfigSectionMap("General")['filter'] #--------------------------------------------------------# # crosstalk # #--------------------------------------------------------# self.xtalk_file = self.ConfigSectionMap("crosstalk")['xtalk'] self.xtalk_template = self.ConfigSectionMap("crosstalk")['template'] self.replace_file = self.ConfigSectionMap("crosstalk")['replace'] #-----------------------------------------------------------# # pixcorrect # #-----------------------------------------------------------# self.bias = self.ConfigSectionMap("pixcorrect")['bias'] self.bpm = self.ConfigSectionMap("pixcorrect")['bpm'] self.linearity = self.ConfigSectionMap("pixcorrect")['linearity'] self.bf = self.ConfigSectionMap("pixcorrect")['bf'] self.flat =self.ConfigSectionMap("pixcorrect")['flat'] self.copy_from_Dcache(self.correc_dir+'lin_'+str(self.yearb)+'/'+self.linearity) self.copy_from_Dcache(self.correc_dir+'bf_'+str(self.yearb)+'/'+self.bf) #---------------------------------------# # skyCombine # #---------------------------------------# self.PCFILENAMEPREFIX = self.ConfigSectionMap("skyCombineFit")['pcafileprefix'] self.PCFILENAME = self.PCFILENAMEPREFIX # starflat # self.starflat = self.ConfigSectionMap("starflat")['starflat'] # skySubtract self.pc_filename = self.ConfigSectionMap("skysubtract")['pcfilename'] self.weight = self.ConfigSectionMap("skysubtract")['weight'] #------------------------------------------# # scamp # #------------------------------------------# self.imagflags = self.ConfigSectionMap("scamp")['imagflags'] self.flag_mask = self.ConfigSectionMap("scamp")['flag_mask'] self.flag_astr = self.ConfigSectionMap("scamp")['flag_astr'] self.catalog_ref = self.ConfigSectionMap("scamp")['catalog_ref'] self.default_scamp = self.ConfigSectionMap("scamp")['default_scamp'] self.head_file = self.ConfigSectionMap("scamp")['head'] self.farg = {'filter':self.FILTER} self.head_FILE = self.head_file.format(**self.farg) # self.copy_from_Dcache(self.data_conf+self.default_scamp) self.copy_from_Dcache(self.data_conf+self.head_FILE) #-----------------------------------------# # sextractor # #-----------------------------------------# self.sexnnwFile = self.ConfigSectionMap("sextractor")['starnnw_name'] self.sexconvFile = self.ConfigSectionMap("sextractor")['filter_name'] self.sexparamFile = self.ConfigSectionMap("sextractor")['parameters_name'] self.sexparamPSFEXFile = self.ConfigSectionMap("sextractor")['parameters_name_psfex'] self.configFile = self.ConfigSectionMap("sextractor")['configfile'] self.confPSF = self.ConfigSectionMap("sextractor")['sexforpsfexconfigfile'] self.copy_from_Dcache(self.data_conf+self.sexnnwFile) self.copy_from_Dcache(self.data_conf+self.sexconvFile) self.copy_from_Dcache(self.data_conf+self.sexparamFile) self.copy_from_Dcache(self.data_conf+self.sexparamPSFEXFile) self.copy_from_Dcache(self.data_conf+self.configFile) self.copy_from_Dcache(self.data_conf+'default.psf') self.copy_from_Dcache(self.data_conf+self.confPSF) self.copy_from_Dcache(self.data_conf+'sex.param_bkg') self.sexbkgparamFile='sex.param_bkg' #----------------------------------------# # psfex # #----------------------------------------# self.config_filePSF = self.ConfigSectionMap("psfex")['configfile'] self.copy_from_Dcache(self.data_conf+self.config_filePSF) #-----------------------------------------# # sextractor with psf # #-----------------------------------------# self.sexparamFile_2 = self.ConfigSectionMap("sextractor")['parameters_name2'] self.configFile2 = self.ConfigSectionMap("sextractor")['configfile2'] self.sexconvFile2 = self.ConfigSectionMap("sextractor")['filter_name2'] self.copy_from_Dcache(self.data_conf+self.sexconvFile2) self.copy_from_Dcache(self.data_conf+self.sexparamFile_2) self.copy_from_Dcache(self.data_conf+self.configFile2) self.copy_from_Dcache(self.data_conf+'default.psf')
def load_config(self, config): """ Load the configuration file and parse entries, when encountering a issue change safe defaults """ self.parser = ConfigParser.SafeConfigParser() self.parser.read(config) # Set some safe defaults self.opacity = 50 self.button_theme = "default" self.bgcolor = gtk.gdk.color_parse("black") self.monitor = 0 blist = "" # Check if we're using HAL, and init it as required. if self.parser.has_section("settings"): if self.parser.has_option("settings","backend"): self.backend = self.parser.get("settings","backend") else: self.backend = "" if self.parser.has_option("settings", "monitor"): self.monitor = self.parser.getint("settings", "monitor") if self.backend == "HAL" or self.backend == "ConsoleKit": from dbushandler import DbusController self.dbus = DbusController(self.backend) if self.dbus.check() == False: del self.dbus self.backend = "" else: self.backend = "" # Check the looks section and load the config as required if self.parser.has_section("looks"): if self.parser.has_option("looks", "opacity"): self.opacity = self.parser.getint("looks", "opacity") if self.parser.has_option("looks","buttontheme"): self.button_theme = self.parser.get("looks", "buttontheme") if self.parser.has_option("looks", "bgcolor"): try: self.bgcolor = gtk.gdk.color_parse(self.parser.get("looks", "bgcolor")) except: self.logger.warning(_("Color %s is not a valid color, defaulting to black") % self.parser.get("looks", "bgcolor")) self.bgcolor = gtk.gdk.color_parse("black") if self.parser.has_option("looks", "opacity"): blist = self.parser.get("looks", "buttons") # Parse shortcuts section and load them into a array for later reference. if self.parser.has_section("shortcuts"): self.shortcut_keys = self.parser.items("shortcuts") self.logger.debug("Shortcut Options: %s" % self.shortcut_keys) # Parse in commands section of the configuration file. Check for valid keys and set the attribs on self if self.parser.has_section("commands"): for key in self.parser.items("commands"): self.logger.debug("Setting cmd_%s as %s" % (key[0], key[1])) if key[0] in ['logout', 'restart', 'shutdown', 'suspend', 'hibernate', 'safesuspend', 'lock', 'switch']: if key[0]: setattr(self, "cmd_" + key[0], key[1]) # Load theme information from local directory if local mode is set if self.local_mode: self.theme_prefix = "./data/themes" else: self.theme_prefix = "%s/share/themes" % sys.prefix self.img_path = "%s/%s/oblogout" % (self.theme_prefix, self.button_theme) if os.path.exists("%s/.themes/%s/oblogout" % (os.environ['HOME'], self.button_theme)): # Found a valid theme folder in the userdir, use that self.img_path = "%s/.themes/%s/oblogout" % (os.environ['HOME'], self.button_theme) self.logger.info("Using user theme at %s" % self.img_path) else: if not os.path.exists("%s/%s/oblogout" % (self.theme_prefix, self.button_theme)): self.logger.warning("Button theme %s not found, reverting to default" % self.button_theme) self.button_theme = 'foom' # Parse button list from config file. validbuttons = ['cancel', 'logout', 'restart', 'shutdown', 'suspend', 'hibernate', 'safesuspend', 'lock', 'switch'] buttonname = [_('cancel'), _('logout'), _('restart'), _('shutdown'), _('suspend'), _('hibernate'), _('safesuspend'), _('lock'), _('switch')] if not blist: list = validbuttons elif blist == "default": list = validbuttons else: list = map(lambda button: string.strip(button), blist.split(",")) # Validate the button list for button in list: if not button in validbuttons: self.logger.warning(_("Button %s is not a valid button name, removing") % button) list.remove(button) else: if self.backend: if not self.dbus.check_ability(button): self.logger.warning(_("Can't %s, disabling button" % button)) list.remove(button) if len(list) == 0: self.logger.warning(_("No valid buttons found, resetting to defaults")) self.button_list = validbuttons else: self.logger.debug("Validated Button List: %s" % list) self.button_list = list
class HooksConfig(object): """ Gets and manages the bridge configuration file. """ __instance = None _ = None _config = ConfigParser.ConfigParser() _CONFIG_FILE = "hooks_config" + os.sep + "bridge.cfg" @staticmethod def get_default_locale(): return ('en_US', 'UTF8') @staticmethod def get_translations(): user_locale = locale.getdefaultlocale() if all(x is None for x in user_locale): user_locale = HooksConfig.get_default_locale() t = gettext.translation("Git2CC", "locale", user_locale, fallback=True) if isinstance(t, gettext.NullTranslations): user_locale = HooksConfig.get_default_locale() t = gettext.translation("Git2CC", "locale", user_locale, fallback=True) t.install() return t.ugettext def __new__(cls, *args, **kargs): if cls.__instance is None: cls.__instance = object.__new__(cls, *args, **kargs) return cls.__instance def __init__(self): """ Class constructor that gets and validates the configuration and sets the translation alias """ # Load user messages self._ = HooksConfig.get_translations() # Read and validate configuration file self._config.readfp(open(self._CONFIG_FILE)) self._validate_config() def _validate_config(self): """ This procedure checks the configuration file and ensures every section, field, file and folder existence. """ # Section validation if not self._config.has_section("cc_view"): raise ConfigException(self._("missing_section") + " cc_view") elif not self._config.has_section("cc_config"): raise ConfigException(self._("missing_section") + " cc_config") # cc_view section fields validation if not self._config.has_option("cc_view", "path"): raise ConfigException(self._("missing_field") + " path " + self._("in_section") + " cc_view.") elif not os.path.isdir(self.get_view()): raise ConfigException(self.get_view() + self._("folder_not_exists")) # cc_config section fields validation if not self._config.has_option("cc_config", "cleartool_path"): raise ConfigException(self._("missing_field") + " cleartool_path " + self._("in_section") + " cc_config.") elif not os.path.isfile(self.get_cleartool_path()): raise ConfigException(self.get_cleartool_path() + self._("file_not_exists")) if not self._config.has_option("cc_config", "cc_pusher_user"): raise ConfigException(self._("missing_field") + " cc_pusher_user " + self._("in_section") + " cc_config.") def get_view(self): """ Path to ClearCase view to sync """ return self._config.get("cc_view", "path") def get_cleartool_path(self): """ Path to clear tool command """ return self._config.get("cc_config", "cleartool_path") def get_cc_pusher_user(self): """ Returns the user pushing from the ClearCase view to sync work with Git. When hooks detect this users, they avoid doing any other operation. """ return self._config.get("cc_config", "cc_pusher_user") def get_sync_branches(self): """ Returns the list of Git branches in sync with ClearCase """ branches = [] if not self._config.has_option("git_config", "sync_branches"): branches.append("master") else: branches = self._config.get("git_config", "sync_branches").split(',') branches = [x.strip() for x in branches] return branches def get_vobs(self): """ Return the configured CC vobs """ vobs = [] vobs = self._config.get("cc_view", "vobs").split(' ') return vobs
#####matplotlib############### import matplotlib matplotlib.use('Agg') import matplotlib.pyplot as plt import matplotlib.mlab as mlab #####import for django database#### sys.path.append('./db') import exec_sqlite import query_mongo import ml_util from ml_util import * import zip_feature_util CONF_FILE='../../app.config' # at the base dir of the web config=ConfigParser.ConfigParser() config.read(CONF_FILE) libsvm_filename = config.get("machine_learning","libsvm_alldata_filename") dnn_filename = config.get("machine_learning","dnn_alldata_filename") def main(): parser = ArgumentParser(description=__description__) parser.add_argument("-f", "--folder", type=str, metavar="folder of features", help="hdfs folder contains features", required=False) parser.add_argument("-n", "--data_fname", type=str, metavar="data file name", help="file name for sample dataset", required=False) parser.add_argument("-o", "--out", type=str, metavar="out figure folder", help="folder contains output", required=False) parser.add_argument("-r", "--row_id", type=str, metavar="row id", help="row_id number in the db", required=False) parser.add_argument("-w", "--fromweb", type=str, metavar="flag for web", help="flag for web", required=False) parser.add_argument("-pm", "--parameter", type=str, metavar="parameters in json", help="json string contains learning alg and parameter selection", required=False)
''' Set the config variable. ''' import ConfigParser as cp import json config = cp.RawConfigParser() config.read('../data/config/config.cfg') min_wdw_sz = json.loads(config.get("hog","min_wdw_sz")) step_size = json.loads(config.get("hog", "step_size")) orientations = config.getint("hog", "orientations") pixels_per_cell = json.loads(config.get("hog", "pixels_per_cell")) cells_per_block = json.loads(config.get("hog", "cells_per_block")) visualize = config.getboolean("hog", "visualize") normalize = config.getboolean("hog", "normalize") pos_feat_ph = config.get("paths", "pos_feat_ph") neg_feat_ph = config.get("paths", "neg_feat_ph") model_path = config.get("paths", "model_path") threshold = config.getfloat("nms", "threshold")
def process(keyword, page, website): #后面需要分类型 siteconfs = os.listdir( os.path.dirname(os.path.abspath(__file__)) + '/siteconfs') if not website in siteconfs: print 'siteconf not found' return [] confpath = os.path.dirname( os.path.abspath(__file__)) + '/siteconfs/' + website siteconf = ConfigParser.ConfigParser() siteconf.read(confpath) extractors = siteconf.sections() try: extractors = sorted(extractors, key=lambda d: int(d[-1])) except: pass urlBac = '' for extractor in extractors: url = siteconf.get(extractor, 'searchUrl') url = url.replace('${keyword}', keyword).replace('${page}', str(page)) print url segmentCut = siteconf.get(extractor, 'segment') titleCut = siteconf.get(extractor, 'title') urlCut = siteconf.get(extractor, 'url') infoCuts = siteconf.get(extractor, 'info') urlinfos = [] if urlBac == url: #如果是一样的链接就不重复打开了 pageBuf = ct.crawlerTool.getPage( url ) #print HTMLParser().unescape('·').encode('unicode-escape').decode('string_escape')是乱码 else: urlBac = url pageBuf = ct.crawlerTool.getPage(url) baseurl = '/'.join(url.split('/')[:3]) pageBuf = urlHostParser.make_links_absolute(pageBuf, baseurl) segments = ct.crawlerTool.getXpath(segmentCut, pageBuf) if not segments: print 'no matched segments', website continue for segment in segments: try: urlinfo = {} urlinfo['url'] = ct.crawlerTool.getXpath(urlCut, segment)[0] title = HTMLParser().unescape( ct.crawlerTool.extractorText( ct.crawlerTool.getXpath( titleCut, segment)[0])) #好像不转str格式后面输出是乱码S #print title,HTMLParser().unescape(title) #print ct.crawlerTool.getXpath('//h2/a[1]', segment)#解码后·好像变乱码了 urlinfo['title'] = title #print title urlinfo['info'] = '' for infoCut in infoCuts.split(';'): urlinfo['info'] += ' '.join( ct.crawlerTool.getXpath(infoCut, segment)) #info 作拼接处理 #print urlinfo['url'], urlinfo['title'], urlinfo['info'] urlinfos.append(urlinfo) except Exception, e: traceback.print_exc() return {"urlinfos": urlinfos}
def compare_ini(ini_dirs, ini_name): # list of configuration INI readers configs = list() common_prefix_len = len(os.path.commonprefix(ini_dirs)) # iterate all ini files and make a list of (folder_name, ConfigParser) items for ini_dir in ini_dirs: # extract the first unique item of the path to use as an identifier of ini files set folder_name = ini_dir[common_prefix_len:].split(os.sep)[0] configs.append((folder_name, ConfigParser.RawConfigParser())) configs[-1][1].read(os.path.join(ini_dir, ini_name)) # initialize exceptions from the corresponding exceptions *.ini exceptions = ConfigParser.RawConfigParser() file_name, file_ext = os.path.splitext(ini_name) exception_ini = file_name.strip('1234567890') + file_ext exception_ini = os.path.join('exceptions', exception_ini) if os.path.exists(exception_ini): exceptions.read(exception_ini) # build a united structure of all sections in all configs conf_structure = dict() result_table = Table(ini_name) header = list() header.append('') for folder_name, conf in configs: # compose the header of the result table header.append(folder_name) for section in conf.sections(): if not conf_structure.has_key(section): conf_structure[section] = set() conf_structure[section] |= set(conf.options(section)) header.append('status') result_table.add_header(header) # initializing statistic counters ignored_cnt = 0 default_cnt = 0 unequal_cnt = 0 for section in conf_structure: section_exc = section.strip('1234567890') section_values = list() option_values = list() section_values.append(section) for folder_name, conf in configs: if conf.has_section(section): section_values.append('+') else: section_values.append('-') for option in conf_structure[section]: option_exc = option.strip('1234567890') value_exc = 'unequal' # if current option is marked 'ignore' in exceptions list - don't check it if exceptions.has_section(section_exc): if exceptions.has_option(section_exc, option_exc): value_exc = exceptions.get(section_exc, option_exc) values = list() values.append(' ' + str(option)) for folder_name, conf in configs: if conf.has_section(section): if conf.has_option(section, option): values.append(conf.get(section, option)) else: values.append('-') else: values.append('-') # if at least 2 different values are found - add a row to the table if len(set(values)) > 2: values.append(value_exc) option_values.append(values) # else: # equal_cnt += 1 # add to output: name of the section and its state (present/absent) for each conf if len(set(section_values)) > 2 or len(option_values): result_table.add_row(Row(section_values, align=Alignment.CENTER)) for ov in option_values: value_exc = ov[-1] color = 'Default' if value_exc == 'unequal': color = 'Red' unequal_cnt += 1 else: if value_exc == 'ignore': color = 'Grey' ignored_cnt += 1 else: if value_exc == 'default': color = 'Blue' default_cnt += 1 result_table.add_row(Row(ov, font=color)) result_table.add_sep() if ignored_cnt or default_cnt or unequal_cnt: result_table.add_total('Ignored:', ignored_cnt) result_table.add_total('Default:', default_cnt) result_table.add_total('Unequal:', unequal_cnt) result_table.output() return bool(unequal_cnt)
def pymelLogFileConfig(fname, defaults=None, disable_existing_loggers=False): """ Reads in a file to set up pymel's loggers. In most respects, this function behaves similarly to logging.config.fileConfig - consult it's help for details. In particular, the format of the config file must meet the same requirements - it must have the sections [loggers], [handlers], and [fomatters], and it must have an entry for [logger_root]... even if not options are set for it. It differs from logging.config.fileConfig in the following ways: 1) It will not disable any pre-existing loggers which are not specified in the config file, unless disable_existing_loggers is set to True. 2) Like logging.config.fileConfig, the default behavior for pre-existing handlers on any loggers whose settings are specified in the config file is to remove them; ie, ONLY the handlers explicitly given in the config will be on the configured logger. However, pymelLogFileConfig provides the ability to keep pre-exisiting handlers, by setting the 'remove_existing_handlers' option in the appropriate section to True. """ cp = ConfigParser.ConfigParser(defaults) if hasattr(cp, 'readfp') and hasattr(fname, 'readline'): cp.readfp(fname) else: cp.read(fname) formatters = logging.config._create_formatters(cp) # _install_loggers will remove all existing handlers for the # root logger, and any other handlers specified... to override # this, save the existing handlers first root = logging.root root_logger_level = root.level # restore root level below if NOTSET # make sure you get a COPY of handlers! rootHandlers = root.handlers[:] oldLogHandlers = {} # Don't use getLogger while iterating through loggerDict, as that # may actually create a logger, and change the size of the dict # ...instead, just ignore any PlaceHolder instances we get, as they # won't have any handlers to worry about anyway # thanks to pierre.augeard for pointing this one out for loggerName, logger in root.manager.loggerDict.iteritems(): # Make sure it's not a PlaceHolder if isinstance(logger, logging.Logger): # make sure you get a COPY of handlers! oldLogHandlers[loggerName] = logger.handlers[:] # critical section logging._acquireLock() try: # Handlers add themselves to logging._handlers handlers = logging.config._install_handlers(cp, formatters) if sys.version_info >= (2, 6): logging.config._install_loggers(cp, handlers, disable_existing_loggers=0) else: logging.config._install_loggers(cp, handlers) # The _install_loggers function disables old-loggers, so we need to # re-enable them for k, v in logging.root.manager.loggerDict.iteritems(): if hasattr(v, 'disabled') and v.disabled: v.disabled = 0 # Now re-add any removed handlers, if needed secNames = cp.get('loggers', 'keys').split(',') secNames = ['logger_' + x.strip() for x in secNames] _addOldHandlers(root, rootHandlers, 'logger_root', cp) for secName in secNames: if secName == 'logger_root': logger = root oldHandlers = rootHandlers else: logName = cp.get(secName, 'qualname') logger = logging.getLogger(logName) oldHandlers = oldLogHandlers.get(logName) if oldHandlers: _addOldHandlers(logger, oldHandlers, secName, cp) # if root logger level not explicitly set in the pymel.conf file, # then set it back to the original value. The root logger always # has to have a level. if logging.root.level == logging.NOTSET: logging.root.setLevel(root_logger_level) finally: logging._releaseLock()
import Queue import threading import time import bottle import audiere from bottle import route, view, request from ..dtmf import tonegen as dtmf from ..nvv import core socket.setdefaulttimeout(0.5) parser = ConfigParser.ConfigParser( dict(remote_ip='192.168.1.254', remote_port='1024', debug='0', server_port='8080', callsign='0')) parser.read( ["settings.ini", os.path.join(os.path.dirname(__file__), 'settings.ini')]) if not parser.has_section("Server"): parser.add_section("Server") bottle.debug(parser.getboolean('Server', 'debug')) CALL_FILE = os.path.join(os.path.dirname(__file__), 'call.wav') REMOTE = (parser.get('Server',
def __init__(self, config_file, resume_training=True, resume_epoch=None, predict_length=None, multi_gpu=False): self.config = ConfigParser.ConfigParser(allow_no_value=True) try: self.config.readfp(open(config_file)) except: print('Could not read configuration file {} - exiting.'.format(config_file)) sys.exit(1) # Get General Configuration self.train_multi_gpu = multi_gpu self.resume_training = resume_training self.resume_epoch = resume_epoch self.keras_verbose = self.config.getint('general', 'keras_verbose') self.seed = self.config.getint('general', 'seed') if self.seed is None: self.seed = 42 # Get Model Configuration self.data_dir = self.config.get('model', 'data_dir') self.data_dir_structure = self.config.get('model', 'data_dir_structure') self.model_dir = self.config.get('model', 'model_dir') if len(self.model_dir) == 0: self.model_dir = None self.sample_rate = self.config.getint('model', 'sample_rate') self.debug = self.config.getint('model', 'debug') # Training Configuration self.max_epoch = self.config.getint('training', 'max_epoch') self.test_factor = self.config.getfloat('training', 'test_factor') self.batch_size = self.config.getint('training', 'batch_size') self.output_bins = self.config.getint('training', 'output_bins') self.filters = self.config.getint('training', 'filters') self.dilation_depth = self.config.getint('training', 'dilation_depth') self.stacks = self.config.getint('training', 'stacks') self.use_bias = self.config.getboolean('training', 'use_bias') self.use_ulaw = self.config.getboolean('training', 'use_ulaw') self.res_l2 = self.config.getint('training', 'res_l2') self.final_l2 = self.config.getint('training', 'final_l2') self.initial_fragment_length = self.config.getint('training', 'initial_fragment_length') self.fragment_stride = self.config.getint('training', 'fragment_stride') self.use_skip_connections = self.config.getboolean('training', 'use_skip_connections') self.learn_all_outputs = self.config.getboolean('training', 'learn_all_outputs') self.random_train_batches = self.config.getboolean('training', 'random_train_batches') self.randomize_batch_order = self.config.getboolean('training', 'randomize_batch_order') self.train_only_in_receptive_field = self.config.getboolean('training', 'train_only_in_receptive_field') self.train_with_soft_targets = self.config.getboolean('training', 'train_with_soft_targets') self.soft_target_stdev = self.config.getfloat('training', 'soft_target_stdev') self.optimizer = self.config.get('training', 'optimizer') self.early_stopping_patience = self.config.getint('training', 'early_stopping_patience') # Prediction Configuration self.predict_length = self.config.getfloat('prediction', 'predict_length') # Let's allow the user to overwrite the length via cmd-line, it is more practical :-) if predict_length is not None: self.predict_length = predict_length self.sample_argmax = self.config.getboolean('prediction', 'sample_argmax') self.sample_temperature = self.config.getfloat('prediction', 'sample_temperature') self.predict_initial_input = self.config.get('prediction', 'initial_input') if len(self.predict_initial_input) == 0: self.predict_initial_input = None self.predict_use_softmax_as_input = self.config.getboolean('prediction', 'use_softmax_as_input') self.sample_seed = self.seed np.random.seed(self.seed) self.rnd = np.random.RandomState(self.seed) self.fragment_length = self.initial_fragment_length + self._compute_receptive_field2(self.sample_rate, self.dilation_depth, self.stacks)[0] # Additional Settings self.num_gpus = 1 self.train_rank = 0 if self.train_multi_gpu: self.train_rank = hvd.rank() self.num_gpus = hvd.size() print('rank = {}, num_gpu={}'.format(self.train_rank, self.num_gpus)) self.dataset = DataSet(self.config, self.fragment_length, self.num_gpus, self.train_rank)
def read_plat(env, args, fn): cfig = ConfigParser.ConfigParser(env.Dictionary()) cfig.read(fn) buildvars = [ [ 'CC', 'TIFFINCLUDE', # where to find preinstalled tifflib headers 'TIFFLIB', # where to find a preinstalled tifflib library ], # replace [ 'CPPPATH', 'CPPDEFINES', 'CPPFLAGS', 'CCFLAGS', 'LIBPATH', 'LINKFLAGS', 'EZXML_CPPDEFINES', # build flags specific to ezxml.c ] ] # append vars = [ [ 'install', ['RAD_BASEDIR', 'RAD_BINDIR', 'RAD_RLIBDIR', 'RAD_MANDIR'], [] ], [ 'code', [ # replace ], [ # append 'RAD_COMPAT', # theoretically obsolete (src/common/strcmp.c) 'RAD_MATHCOMPAT', # erf.c floating point error function 'RAD_ARGSCOMPAT', # fixargv0.c for Windows 'RAD_NETCOMPAT', # [win_]netproc.c for ranimate 'RAD_MLIB', # usually 'm', or any fastlib available 'RAD_SOCKETLIB', # ws_2_32 on Windows (VC links it automatically) 'RAD_PROCESS', # our process abstraction and win_popen() 'RAD_PCALLS', # more custom process abstraction ] ], ] if args.get('RAD_DEBUG', 0): vars.insert(0, ['debug'] + buildvars) else: vars.insert(0, ['build'] + buildvars) for section in vars: if cfig.has_section(section[0]): for p in section[1]: # single items to replace try: v = cfig.get(section[0], p) except ConfigParser.NoOptionError: continue env[p] = v #print '%s: %s' % (p, env[p]) for p in section[2]: # multiple items to append try: v = cfig.get(section[0], p) except ConfigParser.NoOptionError: continue apply(env.Append, [], {p: env.Split(v)}) # XXX Check that basedir exists. for k in ['RAD_BINDIR', 'RAD_RLIBDIR', 'RAD_MANDIR']: if (env.has_key('RAD_BASEDIR') and env.has_key(k) and not os.path.isabs(env[k])): env[k] = os.path.join(env['RAD_BASEDIR'], env[k])
sys.exit(0) if '-h' in sys.argv or '--help' in sys.argv: print __helpmsg__ sys.exit(0) options = {'d':'debug', 'f':'full', 'l':'long', 'version':'version', 'q':'q', 'reverse':'reverse'} doptions = {'header':'header', 'sort':'sort', 'u':'user'} (opts, args) = Cobalt.Util.dgetopt_long(sys.argv[1:], options, doptions, __helpmsg__) # check for custom header, first in cobalt.conf, env, then in --header custom_header = None custom_header_full = None try: CP = ConfigParser.ConfigParser() CP.read(Cobalt.CONFIG_FILES) custom_header = CP.get('cqm', 'cqstat_header').split(':') except: pass try: custom_header_full = CP.get('cqm', 'cqstat_header_full').split(':') except: pass if 'QSTAT_HEADER' in os.environ.keys(): custom_header = os.environ['QSTAT_HEADER'].split(':') elif 'CQSTAT_HEADER' in os.environ.keys(): custom_header = os.environ['CQSTAT_HEADER'].split(':') if 'QSTAT_HEADER_FULL' in os.environ.keys(): custom_header_full = os.environ['QSTAT_HEADER_FULL'].split(':')
def run_pants_with_workdir_without_waiting(self, command, workdir, config=None, extra_env=None, build_root=None, print_exception_stacktrace=True, **kwargs): args = [ '--no-pantsrc', '--pants-workdir={}'.format(workdir), '--kill-nailguns', '--print-exception-stacktrace={}'.format(print_exception_stacktrace), ] if self.hermetic(): args.extend(['--pants-config-files=[]', # Turn off cache globally. A hermetic integration test shouldn't rely on cache, # or we have no idea if it's actually testing anything. '--no-cache-read', '--no-cache-write', # Turn cache on just for tool bootstrapping, for performance. '--cache-bootstrap-read', '--cache-bootstrap-write' ]) if config: config_data = config.copy() ini = ConfigParser.ConfigParser(defaults=config_data.pop('DEFAULT', None)) for section, section_config in config_data.items(): ini.add_section(section) for key, value in section_config.items(): ini.set(section, key, value) ini_file_name = os.path.join(workdir, 'pants.ini') with safe_open(ini_file_name, mode='w') as fp: ini.write(fp) args.append('--pants-config-files=' + ini_file_name) pants_script = os.path.join(build_root or get_buildroot(), self.PANTS_SCRIPT_NAME) # Permit usage of shell=True and string-based commands to allow e.g. `./pants | head`. if kwargs.get('shell') is True: assert not isinstance(command, list), 'must pass command as a string when using shell=True' pants_command = ' '.join([pants_script, ' '.join(args), command]) else: pants_command = [pants_script] + args + command # Only whitelisted entries will be included in the environment if hermetic=True. if self.hermetic(): env = dict() for h in self.hermetic_env_whitelist(): env[h] = os.getenv(h) or '' hermetic_env = os.getenv('HERMETIC_ENV') if hermetic_env: for h in hermetic_env.strip(',').split(','): env[h] = os.getenv(h) else: env = os.environ.copy() if extra_env: env.update(extra_env) # Don't overwrite the profile of this process in the called process. # Instead, write the profile into a sibling file. if env.get('PANTS_PROFILE'): prof = '{}.{}'.format(env['PANTS_PROFILE'], self._get_profile_disambiguator()) env['PANTS_PROFILE'] = prof # Make a note the subprocess command, so the user can correctly interpret the profile files. with open('{}.cmd'.format(prof), 'w') as fp: fp.write(b' '.join(pants_command)) return pants_command, subprocess.Popen(pants_command, env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs)
path = c_path else: msg = 'Ignoring config item {!r}:{!r} (section:option) as {!r}' \ ' is not a valid directory path.' warnings.warn(msg.format(section, option, c_path)) return path # Figure out the full path to the "iris" package. ROOT_PATH = os.path.abspath(os.path.dirname(__file__)) # The full path to the configuration directory of the active Iris instance. CONFIG_PATH = os.path.join(ROOT_PATH, 'etc') # Load the optional "site.cfg" file if it exists. config = ConfigParser.SafeConfigParser() config.read([os.path.join(CONFIG_PATH, 'site.cfg')]) ################## # Resource options _RESOURCE_SECTION = 'Resources' SAMPLE_DATA_DIR = get_dir_option( _RESOURCE_SECTION, 'sample_data_dir', default=os.path.join(os.path.dirname(__file__), 'sample_data')) TEST_DATA_DIR = get_dir_option(_RESOURCE_SECTION, 'test_data_dir', default=os.path.join(os.path.dirname(__file__), 'test_data'))
import ConfigParser import re import sys class initAutomation(object): def __init__(self,trafficProfile): #pass self.trafficProfile=trafficProfile print trafficProfile def updateConfigFile(self,trafficProfile,gemuConfigfile): gemuConfigfile.set('Test_Case_Details','traffic_profile',trafficProfile[1]) gemuConfigfile.set('Test_Environment_GEMU','logFolder',trafficProfile[2]) with open('/home/Automation/scripts/testConfiguration.ini', 'w') as configFile: gemuConfigfile.write(configFile) if __name__ == "__main__": gemuConfigfile = ConfigParser.ConfigParser() gemuConfigfile.read('/home/Automation/scripts/testConfiguration.ini') trafficProfile=sys.argv testCase=initAutomation(trafficProfile) testCase.updateConfigFile(trafficProfile,gemuConfigfile)
# @{ ## import time import os from oeqa.runtime.wifi import wifi import string try: import ConfigParser except: import configparser as ConfigParser from oeqa.oetest import oeRuntimeTest from oeqa.utils.helper import shell_cmd_timeout from oeqa.utils.decorators import tag ssid_config = ConfigParser.ConfigParser() config_path = os.path.join(os.path.dirname(__file__), "files/config.ini") ssid_config.readfp(open(config_path)) @tag(TestType="FVT") class CommWiFiConect(oeRuntimeTest): """ @class CommWiFiConect """ def setUp(self): ''' initialize wifi class @fn setUp @param self @return ''' self.wifi = wifi.WiFiFunction(self.target)
def __init__(self, login_configfile, send_info_configfile): """Constructor""" self.configparser = ConfigParser.RawConfigParser() self.configparser.readfp(open(login_configfile)) self.loginfo = dict(self.configparser.items('sender'))
def user_configuration(self, configFile=None): # get a logger logger = logging.getLogger("configuration") # load and parse the provided configFile, if provided if not configFile: logger.warn( 'no user configuration file provided; using only built-in default settings' ) return # load the config file try: configparser = ConfigParser.ConfigParser() configparser.readfp(open(configFile)) logger.debug( 'successfully read and parsed user configuration file %s' % configFile) except: logger.fatal('error reading user configuration file %s' % configFile) raise #work_dir must be provided before initialising other directories self.work_dir = None if self.work_dir == None: try: self.work_dir = configparser.get('Paths', 'work') except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): if self.work_dir == None: logger.critical('Paths:work has no value!') raise Exception # look for those items that are user-configurable, and get their values # sptk_bindir= .... # a list instead of a dict because OrderedDict is not available until 2.7 # and I don't want to import theano here just for that one class # each entry is a tuple of (variable name, default value, section in config file, option name in config file) # # the type of the default value is important and controls the type that the corresponding # variable will have # # to set a default value of 'undefined' use an empty string # or the special value 'impossible', as appropriate # impossible_int = int(-99999) impossible_float = float(-99999.0) user_options = [ ('work_dir', self.work_dir, 'Paths', 'work'), ('data_dir', '', 'Paths', 'data'), ('plot_dir', '', 'Paths', 'plot'), ('plot', False, 'Utility', 'plot'), ('profile', False, 'Utility', 'profile'), ('file_id_scp', os.path.join(self.work_dir, 'data/file_id_list.scp'), 'Paths', 'file_id_list'), ('test_id_scp', os.path.join(self.work_dir, 'data/test_id_list.scp'), 'Paths', 'test_id_list'), ('GV_dir', os.path.join(self.work_dir, 'data/GV'), 'Paths', 'GV_dir'), ('in_stepw_dir', os.path.join(self.work_dir, 'data/stepw'), 'Paths', 'in_stepw_dir'), ('in_mgc_dir', os.path.join(self.work_dir, 'data/mgc'), 'Paths', 'in_mgc_dir'), ('in_lf0_dir', os.path.join(self.work_dir, 'data/lf0'), 'Paths', 'in_lf0_dir'), ('in_bap_dir', os.path.join(self.work_dir, 'data/bap'), 'Paths', 'in_bap_dir'), ('in_sp_dir', os.path.join(self.work_dir, 'data/sp'), 'Paths', 'in_sp_dir'), ('in_seglf0_dir', os.path.join(self.work_dir, 'data/lf03'), 'Paths', 'in_seglf0_dir'), ## for glottHMM ('in_F0_dir', os.path.join(self.work_dir, 'data/F0'), 'Paths', 'in_F0_dir'), ('in_Gain_dir', os.path.join(self.work_dir, 'data/Gain'), 'Paths', 'in_Gain_dir'), ('in_HNR_dir', os.path.join(self.work_dir, 'data/HNR'), 'Paths', 'in_HNR_dir'), ('in_LSF_dir', os.path.join(self.work_dir, 'data/LSF'), 'Paths', 'in_LSF_dir'), ('in_LSFsource_dir', os.path.join(self.work_dir, 'data/LSFsource'), 'Paths', 'in_LSFsource_dir'), ## for joint duration ('in_seq_dur_dir', os.path.join(self.work_dir, 'data/S2S_dur'), 'Paths', 'in_seq_dur_dir'), ('in_dur_dir', os.path.join(self.work_dir, 'data/dur'), 'Paths', 'in_dur_dir'), ('nn_norm_temp_dir', os.path.join(self.work_dir, 'data/step_hidden9'), 'Paths', 'nn_norm_temp_dir'), ('process_labels_in_work_dir', False, 'Labels', 'process_labels_in_work_dir'), ('label_style', 'HTS', 'Labels', 'label_style'), ('label_type', 'state_align', 'Labels', 'label_type'), ('in_label_align_dir', os.path.join(self.work_dir, 'data/label_state_align'), 'Labels', 'label_align'), ('question_file_name', os.path.join(self.work_dir, 'data/questions.hed'), 'Labels', 'question_file_name'), ('silence_pattern', ['*-#+*'], 'Labels', 'silence_pattern'), ('subphone_feats', 'full', 'Labels', 'subphone_feats'), ('xpath_file_name', os.path.join(self.work_dir, 'data/xml_labels/xpaths.txt'), 'Labels', 'xpath_file_name'), ('label_config_file', 'configuration/examplelabelconfigfile.py', 'Labels', 'label_config'), ('add_frame_features', True, 'Labels', 'add_frame_features'), ('fill_missing_values', False, 'Labels', 'fill_missing_values'), ('xpath_label_align_dir', os.path.join(self.work_dir, 'data/label_state_align'), 'Labels', 'xpath_label_align'), ('enforce_silence', False, 'Labels', 'enforce_silence'), ('remove_silence_using_binary_labels', False, 'Labels', 'remove_silence_using_binary_labels'), ('precompile_xpaths', True, 'Labels', 'precompile_xpaths'), ('iterate_over_frames', True, 'Labels', 'iterate_over_frames'), ('appended_input_dim', 0, 'Labels', 'appended_input_dim'), ('buffer_size', 200000, 'Data', 'buffer_size'), ('train_file_number', impossible_int, 'Data', 'train_file_number'), ('valid_file_number', impossible_int, 'Data', 'valid_file_number'), ('test_file_number', impossible_int, 'Data', 'test_file_number'), ('log_path', os.path.join(self.work_dir, 'log'), 'Paths', 'log_path'), ('log_file', '', 'Paths', 'log_file'), ('log_config_file', 'configuration/exampleloggingconfigfile.conf', 'Paths', 'log_config_file'), ('sptk_bindir', 'tools/bin/SPTK-3.9', 'Paths', 'sptk'), ('straight_bindir', 'tools/bin/straight', 'Paths', 'straight'), ('world_bindir', 'tools/bin/WORLD', 'Paths', 'world'), ('network_type', 'RNN', 'Architecture', 'network_type'), ('model_type', 'DNN', 'Architecture', 'model_type'), ('hidden_layer_type', ['TANH', 'TANH', 'TANH', 'TANH', 'TANH', 'TANH'], 'Architecture', 'hidden_layer_type'), ('output_layer_type', 'LINEAR', 'Architecture', 'output_layer_type'), ('sequential_training', False, 'Architecture', 'sequential_training'), ('dropout_rate', 0.0, 'Architecture', 'dropout_rate'), ## some config variables for token projection DNN ('scheme', 'stagewise', 'Architecture', 'scheme'), ('index_to_project', 0, 'Architecture', 'index_to_project'), ('projection_insize', 10000, 'Architecture', 'projection_insize'), ('projection_outsize', 10, 'Architecture', 'projection_outsize'), ('initial_projection_distrib', 'gaussian', 'Architecture', 'initial_projection_distrib'), ('projection_weights_output_dir', 'some_path', 'Architecture', 'projection_weights_output_dir'), ('layers_with_projection_input', [0], 'Architecture', 'layers_with_projection_input'), ('projection_learning_rate_scaling', 1.0, 'Architecture', 'projection_learning_rate_scaling'), ('learning_rate', 0.0002, 'Architecture', 'learning_rate'), ('l2_reg', 0.00001, 'Architecture', 'L2_regularization'), ('l1_reg', 0.0, 'Architecture', 'L1_regularization'), ('batch_size', 16, 'Architecture', 'batch_size'), ('training_epochs', 25, 'Architecture', 'training_epochs'), ('hidden_activation', 'tanh', 'Architecture', 'hidden_activation'), ('output_activation', 'linear', 'Architecture', 'output_activation'), ('hidden_layer_size', [1024, 1024, 1024, 1024, 1024, 1024], 'Architecture', 'hidden_layer_size'), ('private_hidden_sizes', [1024], 'Architecture', 'private_hidden_sizes'), ('stream_weights', [1.0], 'Architecture', 'stream_weights'), ('private_l2_reg', 0.00001, 'Architecture', 'private_l2_reg'), ('warmup_epoch', 5, 'Architecture', 'warmup_epoch'), ('warmup_momentum', 0.3, 'Architecture', 'warmup_momentum'), ('momentum', 0.9, 'Architecture', 'momentum'), ('warmup_epoch', 5, 'Architecture', 'warmup_epoch'), ('mdn_component', 1, 'Architecture', 'mdn_component'), ('var_floor', 0.01, 'Architecture', 'var_floor'), ('beta_opt', False, 'Architecture', 'beta_opt'), ('eff_sample_size', 0.8, 'Architecture', 'eff_sample_size'), ('mean_log_det', -100.0, 'Architecture', 'mean_log_det'), ('start_from_trained_model', '_', 'Architecture', 'start_from_trained_model'), ('use_rprop', 0, 'Architecture', 'use_rprop'), ('mgc_dim', 60, 'Outputs', 'mgc'), ('dmgc_dim', 60 * 3, 'Outputs', 'dmgc'), ('vuv_dim', 1, 'Outputs', 'vuv'), ('lf0_dim', 1, 'Outputs', 'lf0'), ('dlf0_dim', 1 * 3, 'Outputs', 'dlf0'), ('bap_dim', 25, 'Outputs', 'bap'), ('dbap_dim', 25 * 3, 'Outputs', 'dbap'), ('cmp_dim', (60 * 3) + 1 + (1 * 3) + (25 * 3), 'Outputs', 'cmp'), ('stepw_dim', 55, 'Outputs', 'stepw_dim'), ('temp_sp_dim', 1025, 'Outputs', 'temp_sp_dim'), ('seglf0_dim', 7, 'Outputs', 'seglf0_dim'), ('delta_win', [-0.5, 0.0, 0.5], 'Outputs', 'delta_win'), ('acc_win', [1.0, -2.0, 1.0], 'Outputs', 'acc_win'), ('do_MLPG', True, 'Outputs', 'do_MLPG'), ## for GlottHMM ('F0_dim', 1, 'Outputs', 'F0'), ('dF0_dim', 1 * 3, 'Outputs', 'dF0'), ('Gain_dim', 1, 'Outputs', 'Gain'), ('dGain_dim', 1 * 3, 'Outputs', 'dGain'), ('HNR_dim', 5, 'Outputs', 'HNR'), ('dHNR_dim', 5 * 3, 'Outputs', 'dHNR'), ('LSF_dim', 30, 'Outputs', 'LSF'), ('dLSF_dim', 30 * 3, 'Outputs', 'dLSF'), ('LSFsource_dim', 10, 'Outputs', 'LSFsource'), ('dLSFsource_dim', 10 * 3, 'Outputs', 'dLSFsource'), ## for joint dur:- ('seq_dur_dim', 1, 'Outputs', 'seq_dur'), ('remove_silence_from_dur', True, 'Outputs', 'remove_silence_from_dur'), ('dur_dim', 5, 'Outputs', 'dur'), ('dur_feature_type', 'numerical', 'Outputs', 'dur_feature_type'), ('output_feature_normalisation', 'MVN', 'Outputs', 'output_feature_normalisation'), ('multistream_switch', False, 'Streams', 'multistream_switch'), # ('use_private_hidden' , False, 'Streams', 'use_private_hidden'), ('output_features', ['mgc', 'lf0', 'vuv', 'bap'], 'Streams', 'output_features'), ('gen_wav_features', ['mgc', 'bap', 'lf0'], 'Streams', 'gen_wav_features'), # ('stream_mgc_hidden_size' , 192 , 'Streams', 'stream_mgc_hidden_size'), # ('stream_lf0_hidden_size' , 32 , 'Streams', 'stream_lf0_hidden_size'), # ('stream_vuv_hidden_size' , 32 , 'Streams', 'stream_vuv_hidden_size'), # ('stream_bap_hidden_size' , 128 , 'Streams', 'stream_bap_hidden_size'), # ('stream_stepw_hidden_size' , 64 , 'Streams', 'stream_stepw_hidden_size'), # ('stream_seglf0_hidden_size', 64 , 'Streams', 'stream_seglf0_hidden_size'), # ('stream_cmp_hidden_size' , 256 , 'Streams', 'stream_cmp_hidden_size'), #when multi-stream is disabled, use this to indicate the final hidden layer size #if this is also not provided, use the top common hidden layer size ## Glott HMM -- dummy values -- haven't used private streams:-- # ('stream_F0_hidden_size' , 192 , 'Streams', 'stream_F0_hidden_size'), # ('stream_Gain_hidden_size' , 192 , 'Streams', 'stream_Gain_hidden_size'), # ('stream_HNR_hidden_size' , 192 , 'Streams', 'stream_HNR_hidden_size'), # ('stream_LSF_hidden_size' , 192 , 'Streams', 'stream_LSF_hidden_size'), # ('stream_LSFsource_hidden_size' , 192 , 'Streams', 'stream_LSFsource_hidden_size'), ## joint dur -- dummy values -- haven't used private streams:-- # ('stream_dur_hidden_size' , 192 , 'Streams', 'stream_dur_hidden_size'), # ('stream_sp_hidden_size' , 1024, 'Streams', 'stream_sp_hidden_size'), # ('stream_weight_mgc' , 1.0, 'Streams', 'stream_weight_mgc'), # ('stream_weight_lf0' , 3.0, 'Streams', 'stream_weight_lf0'), # ('stream_weight_vuv' , 1.0, 'Streams', 'stream_weight_vuv'), # ('stream_weight_bap' , 1.0, 'Streams', 'stream_weight_bap'), # ('stream_weight_stepw' , 0.0, 'Streams', 'stream_weight_stepw'), # ('stream_weight_seglf0', 1.0, 'Streams', 'stream_weight_seglf0'), # ('stream_weight_sp' , 1.0, 'Streams', 'stream_weight_sp'), ## Glott HMM - unused? # ('stream_weight_F0' , 1.0, 'Streams', 'stream_weight_F0'), # ('stream_weight_Gain' , 1.0, 'Streams', 'stream_weight_Gain'), # ('stream_weight_HNR' , 1.0, 'Streams', 'stream_weight_HNR'), # ('stream_weight_LSF' , 1.0, 'Streams', 'stream_weight_LSF'), # ('stream_weight_LSFsource' , 1.0, 'Streams', 'stream_weight_LSFsource'), ## dur - unused? # ('stream_weight_dur' , 1.0, 'Streams', 'stream_weight_dur'), # ('stream_lf0_lr' , 0.5, 'Streams', 'stream_lf0_lr'), # ('stream_vuv_lr' , 0.5, 'Streams', 'stream_vuv_lr'), ('vocoder_type', 'STRAIGHT', 'Waveform', 'vocoder_type'), ('sr', 48000, 'Waveform', 'samplerate'), ('fl', 4096, 'Waveform', 'framelength'), ('shift', 1000 * 240 / 48000, 'Waveform', 'frameshift'), ('sp_dim', (4096 / 2) + 1, 'Waveform', 'sp_dim'), # fw_alpha: 'Bark' or 'ERB' allowing deduction of alpha, or explicity float value (e.g. 0.77) ('fw_alpha', 0.77, 'Waveform', 'fw_alpha'), ('pf_coef', 1.4, 'Waveform', 'postfilter_coef'), ('co_coef', 2047, 'Waveform', 'minimum_phase_order'), ('use_cep_ap', True, 'Waveform', 'use_cep_ap'), ('do_post_filtering', True, 'Waveform', 'do_post_filtering'), ('apply_GV', False, 'Waveform', 'apply_GV'), ('test_synth_dir', 'test_synthesis/wav', 'Waveform', 'test_synth_dir'), ('DurationModel', False, 'Processes', 'DurationModel'), ('AcousticModel', False, 'Processes', 'AcousticModel'), ('GenTestList', False, 'Processes', 'GenTestList'), ('NORMLAB', False, 'Processes', 'NORMLAB'), ('MAKEDUR', False, 'Processes', 'MAKEDUR'), ('MAKECMP', False, 'Processes', 'MAKECMP'), ('NORMCMP', False, 'Processes', 'NORMCMP'), ('TRAINDNN', False, 'Processes', 'TRAINDNN'), ('DNNGEN', False, 'Processes', 'DNNGEN'), ('GENWAV', False, 'Processes', 'GENWAV'), ('CALMCD', False, 'Processes', 'CALMCD'), ('NORMSTEP', False, 'Processes', 'NORMSTEP'), ('GENBNFEA', False, 'Processes', 'GENBNFEA'), ('mgc_ext', '.mgc', 'Extensions', 'mgc_ext'), ('bap_ext', '.bap', 'Extensions', 'bap_ext'), ('lf0_ext', '.lf0', 'Extensions', 'lf0_ext'), ('cmp_ext', '.cmp', 'Extensions', 'cmp_ext'), ('lab_ext', '.lab', 'Extensions', 'lab_ext'), ('utt_ext', '.utt', 'Extensions', 'utt_ext'), ('stepw_ext', '.stepw', 'Extensions', 'stepw_ext'), ('sp_ext', '.sp', 'Extensions', 'sp_ext'), ## GlottHMM ('F0_ext', '.F0', 'Extensions', 'F0_ext'), ('Gain_ext', '.Gain', 'Extensions', 'Gain_ext'), ('HNR_ext', '.HNR', 'Extensions', 'HNR_ext'), ('LSF_ext', '.LSF', 'Extensions', 'LSF_ext'), ('LSFsource_ext', '.LSFsource', 'Extensions', 'LSFsource_ext'), ## joint dur ('dur_ext', '.dur', 'Extensions', 'dur_ext'), ] # this uses exec(...) which is potentially dangerous since arbitrary code could be executed for (variable, default, section, option) in user_options: value = None try: # first, look for a user-set value for this variable in the config file value = configparser.get(section, option) user_or_default = 'user' except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): # use default value, if there is one if (default == None) or \ (default == '') or \ ((type(default) == int) and (default == impossible_int)) or \ ((type(default) == float) and (default == impossible_float)) : logger.critical('%20s has no value!' % (section + ":" + option)) raise Exception else: value = default user_or_default = 'default' if type(default) == str: exec('self.%s = "%s"' % (variable, value)) elif type(default) == int: exec('self.%s = int(%s)' % (variable, value)) elif type(default) == float: exec('self.%s = float(%s)' % (variable, value)) elif type(default) == bool: exec('self.%s = bool(%s)' % (variable, value)) elif type(default) == list: exec('self.%s = list(%s)' % (variable, value)) else: logger.critical( 'Variable %s has default value of unsupported type %s', variable, type(default)) raise Exception( 'Internal error in configuration settings: unsupported default type' ) logger.info('%20s has %7s value %s' % (section + ":" + option, user_or_default, value)) self.combined_feature_name = '' for feature_name in self.output_features: self.combined_feature_name += '_' self.combined_feature_name += feature_name self.combined_model_name = self.model_type for hidden_type in self.hidden_layer_type: self.combined_model_name += '_' + hidden_type self.combined_model_name += '_' + self.output_layer_type
#############COMMAND CODE IS BELOW ###################### ###########OPTIONS parser = argparse.ArgumentParser(description='Command line parser of skim options') parser.add_argument('--config', dest='cfgfile', help='Name of config file', required = True) parser.add_argument('--sample', dest='samplename', help='Type of sample: 0 signal, 1 bkgmodel', required = True) parser.add_argument('--casename', dest='casename', help='Case name', required = True) args = parser.parse_args() configfilename = args.cfgfile sampletorun = args.samplename case = args.casename ###########Read Config file print "[INFO] Reading skim configuration file . . ." cfgparser = ConfigParser() cfgparser.read('%s'%configfilename) ##########Get skim variables print "[INFO] Getting configuration parameters . . ." directory = ast.literal_eval(cfgparser.get("configuration","directory")) print " -The directory:" print " *",directory signalsamples = ast.literal_eval(cfgparser.get("configuration","mvasigsamples")) print " -The list of signal samples:" for x in range(len(signalsamples)): print " *",signalsamples[x] bdtbkgsamples = ast.literal_eval(cfgparser.get("configuration","mvabkgsamples")) print " -The list of bdtbkg samples:" print " *",bdtbkgsamples[0] tag = ast.literal_eval(cfgparser.get("configuration","tag")) print " -The tag:"
def get_content(toUrl,count): """ Return the content of given url Args: toUrl: aim url count: index of this connect Return: content if success 'Fail' if fail """ cf = ConfigParser.ConfigParser() cf.read("config.ini") cookie = cf.get("cookie", "cookie") headers = { 'Cookie': cookie, 'Host':'www.zhihu.com', 'Referer':'http://www.zhihu.com/', 'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36', # 'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36', 'Accept-Encoding':'gzip' } req = urllib2.Request( url = toUrl, headers = headers ) proxy_all = [ "10.25.170.247:5678", "10.25.171.82:5678", "10.47.114.111:5678", "10.47.54.77:5678", "10.25.60.218:5678", "10.47.54.180:5678", "10.47.54.115:5678", "10.47.106.138:5678" ] current_proxy = random.choice(proxy_all) try: opener = urllib2.build_opener(urllib2.ProxyHandler({"http" : current_proxy})) #urllib2.ProxyHandler() urllib2.install_opener(opener) page = urllib2.urlopen(req,timeout = 15) headers = page.info() content = page.read() # except Exception,e: # if count % 1 == 0: # print str(count) + ", Error: " + str(e) + " URL: " + toUrl # return "FAIL" except urllib2.HTTPError, e: if e.code == 404: if count % 1 == 0: print str(count) + ", Error: " + str(e) + " URL: " + toUrl return "NO FOUND" else: try: page = urlopen_with_retry(req, proxy_all) headers = page.info() content = page.read() except Exception,e: if count % 1 == 0: print str(count) + ", Error: " + str(e) + " URL: " + toUrl + "retry_fail" return "FAIL"
def parse(filename): # Set up defaults defaults = { 'pixel_size_x': 1, 'pixel_size_y': 1, 'get_coordinates': True, 'draw_aspect_ratio': True, 'save_intermediates': False, 'disk_size_opening': 10, 'contrast_adjustment': 3, 'threshold_adjustment': 0, 'disk_size_smoothing': 20, 'downsample': True, 'num_points': 100, 'run3dmorph': False } required_list = ['in_directory', 'input_ext', 'out_directory', 'sampleID'] boolean_list = [ 'get_coordinates', 'draw_aspect_ratio', 'save_intermediates', 'downsample', 'run3dmorph' ] float_list = [ 'pixel_size_x', 'pixel_size_y', 'disk_size_opening', 'contrast_adjustment', 'threshold_adjustment', 'disk_size_smoothing', 'num_points' ] # Parse setting settings = defaults settings['timestamp'] = datetime.now().strftime('%Y-%m-%d at %H:%M:%S') parser = ConfigParser.SafeConfigParser(allow_no_value=True) parser.optionxform = str # Preserve case if os.path.isfile(filename): try: parser.read(filename) except ConfigParser.MissingSectionHeaderError: vfile = StringIO(u'[morph2d]\n%s' % open(filename).read()) parser.readfp(vfile) else: sys.exit("Error: Cannot open settings file " + filename) # Set optional variables for setting in parser.options('morph2d'): if setting in boolean_list: try: settings[setting] = parser.getboolean('morph2d', setting) except: pass elif setting in float_list: try: settings[setting] = parser.getfloat('morph2d', setting) except: pass else: settings[setting] = str(parser.get('morph2d', setting)) # Check for required parameters for required in required_list: if required not in settings.keys(): sys.exit('Error: ' + required + ' must be set in settings file.') # Set default output directory if none specified if not settings['out_directory']: settings[ 'out_directory'] = settings['in_directory'] + os.sep + 'morph2d' # Set up additional global settings if settings['in_directory'].endswith(os.sep): settings['in_directory'] = settings['in_directory'].rstrip(os.sep) return settings
def gather_tests(include=[], exclude=[], benchmarker=None): ''' Given test names as strings, returns a list of FrameworkTest objects. For example, 'aspnet-mysql-raw' turns into a FrameworkTest object with variables for checking the test directory, the test database os, and other useful items. With no arguments, every test in this framework will be returned. With include, only tests with this exact name will be returned. With exclude, all tests but those excluded will be returned. A benchmarker is needed to construct full FrameworkTest objects. If one is not provided, a default Benchmarker will be created. ''' # Avoid setting up a circular import from benchmark import framework_test from benchmark.benchmarker import Benchmarker from setup.linux import setup_util # Help callers out a bit if include is None: include = [] if exclude is None: exclude = [] # Old, hacky method to exclude all tests was to # request a test known to not exist, such as ''. # If test '' was requested, short-circuit and return # nothing immediately if len(include) == 1 and '' in include: return [] # Setup default Benchmarker using example configuration if benchmarker is None: print "Creating Benchmarker from benchmark.cfg.example" default_config = setup_util.get_fwroot() + "/benchmark.cfg.example" config = ConfigParser.SafeConfigParser() config.readfp(open(default_config)) defaults = dict(config.items("Defaults")) # Convert strings into proper python types for k, v in defaults.iteritems(): try: defaults[k] = literal_eval(v) except Exception: pass # Ensure we only run the __init__ method of Benchmarker defaults['install'] = None benchmarker = Benchmarker(defaults) # Search in both old and new directories fwroot = setup_util.get_fwroot() config_files = glob.glob("%s/*/benchmark_config.json" % fwroot) config_files.extend( glob.glob("%s/frameworks/*/*/benchmark_config.json" % fwroot)) tests = [] for config_file_name in config_files: config = None with open(config_file_name, 'r') as config_file: try: config = json.load(config_file) except ValueError: # User-friendly errors print("Error loading '%s'." % config_file_name) raise # Find all tests in the config file config_tests = framework_test.parse_config( config, os.path.dirname(config_file_name), benchmarker) # Filter for test in config_tests: if len(include) is 0 and len(exclude) is 0: # No filters, we are running everything tests.append(test) elif test.name in exclude: continue elif test.name in include: tests.append(test) else: # An include list exists, but this test is # not listed there, so we ignore it pass # Ensure we were able to locate everything that was # explicitly included if 0 != len(include): names = {test.name for test in tests} if 0 != len(set(include) - set(names)): missing = list(set(include) - set(names)) raise Exception("Unable to locate tests %s" % missing) tests.sort(key=lambda x: x.name) return tests
import subprocess import ConfigParser # Read conf file config = ConfigParser.RawConfigParser() configFilePath = 'config.conf' goldConfig = '../goldImage/goldconfig.conf' config.read(configFilePath) lang = config.get('Section', 'lang') dataset = config.get('Section','dataset') script = config.get('Section', 'script') package = config.get('Section', 'package') packagelist = package.split() config.read(goldConfig) goldpackage = config.get('Section', 'package') goldpackagelist = goldpackage.split() delete = [] install = [] for idx, golditem in enumerate(goldpackagelist): if golditem not in packagelist: delete.append(golditem) strdelete = ' '.join(delete) for idx, deliveritem in enumerate(packagelist): if deliveritem not in goldpackagelist: install.append(deliveritem) strinstall = ' '.join(install)
def read_and_set_configuration(config_path): """Reads the worker configuration from the file at config_path and sets the read configuration to the env variable. The configuration is read of the path, put into a dictionary which will be serialized and set in the env variable. Notes: The WORKER_VERSION has to be set manually for now. The COMPONENT has to be set manually at the entry point of each component (worker/sandbox). Args: config_path: string, the configuration file path. """ clear_config() # init and set default values for optional configuration keys config = ConfigParser.SafeConfigParser({ DEBUG_TRACES: DEFAULT_DEBUG_TRACES, BYPASS_CERTIFICATE_VERIFICATION: DEFAUTL_BYPASS_CERTIFICATE_VERIFICATION, ENFORCE_RUNBOOK_SIGNATURE_VALIDATION: DEFAULT_ENFORCE_RUNBOOK_SIGNATURE_VALIDATION, GPG_PUBLIC_KEYRING_PATH: DEFAULT_GPG_PUBLIC_KEYRING_PATH, STATE_DIRECTORY_PATH: DEFAULT_STATE_DIRECTORY_PATH, JRDS_POLLING_FREQUENCY: DEFAULT_JRDS_POLLING_FREQUENCY, PROXY_CONFIGURATION_PATH: DEFAULT_PROXY_CONFIGURATION_PATH }) # load the worker configuration file config.read(config_path) # create the configuration dictionary # read required configuration values configuration = { CERT_PATH: os.path.abspath(config.get(WORKER_REQUIRED_CONFIG_SECTION, CERT_PATH)), KEY_PATH: os.path.abspath(config.get(WORKER_REQUIRED_CONFIG_SECTION, KEY_PATH)), BASE_URI: config.get(WORKER_REQUIRED_CONFIG_SECTION, BASE_URI), ACCOUNT_ID: config.get(WORKER_REQUIRED_CONFIG_SECTION, ACCOUNT_ID), MACHINE_ID: config.get(WORKER_REQUIRED_CONFIG_SECTION, MACHINE_ID), HYBRID_WORKER_GROUP_NAME: config.get(WORKER_REQUIRED_CONFIG_SECTION, HYBRID_WORKER_GROUP_NAME), WORKING_DIRECTORY_PATH: os.path.abspath( config.get(WORKER_REQUIRED_CONFIG_SECTION, WORKING_DIRECTORY_PATH)), SOURCE_DIRECTORY_PATH: os.path.dirname(os.path.realpath(__file__)), WORKER_VERSION: DEFAULT_WORKER_VERSION, COMPONENT: DEFAULT_COMPONENT } # read optional configuration section configuration.update({ DEBUG_TRACES: config.getboolean(WORKER_OPTIONAL_CONFIG_SECTION, DEBUG_TRACES), BYPASS_CERTIFICATE_VERIFICATION: config.getboolean(WORKER_OPTIONAL_CONFIG_SECTION, BYPASS_CERTIFICATE_VERIFICATION), ENFORCE_RUNBOOK_SIGNATURE_VALIDATION: config.getboolean(WORKER_OPTIONAL_CONFIG_SECTION, ENFORCE_RUNBOOK_SIGNATURE_VALIDATION), GPG_PUBLIC_KEYRING_PATH: config.get(WORKER_OPTIONAL_CONFIG_SECTION, GPG_PUBLIC_KEYRING_PATH), STATE_DIRECTORY_PATH: config.get(WORKER_OPTIONAL_CONFIG_SECTION, STATE_DIRECTORY_PATH), JRDS_POLLING_FREQUENCY: config.getint(WORKER_OPTIONAL_CONFIG_SECTION, JRDS_POLLING_FREQUENCY), PROXY_CONFIGURATION_PATH: config.get(WORKER_OPTIONAL_CONFIG_SECTION, PROXY_CONFIGURATION_PATH) }) # set the worker conf to env var set_config(configuration)
def main(): # Load cluster.conf parser = ConfigParser.SafeConfigParser() parser.read(CLUSTER_CONF) # Get default log directory log_directory = parser.get("cluster", "log_directory") parser = argparse.ArgumentParser( description="coordinates the execution of Themis jobs") parser.add_argument("themis_binary", help="path to the Themis binary") parser.add_argument("config", help="a YAML file giving configuration " "options for Themis") parser.add_argument("--log_directory", "-l", help="the directory in which to store coordinator logs " "(default: %(default)s)", default=log_directory) parser.add_argument("--keepalive_refresh", help="the length of time node " "coordinators should wait between refreshing keepalive " "information (default: %(default)s seconds)", type=int, default=2) parser.add_argument("--keepalive_timeout", help="the amount of time that " "must pass without receiving a keepalive message from " "a node coordinator before the cluster coordinator " "considers that node to be dead (default: %(default)s " "seconds)", type=int, default=10) parser.add_argument("--profiler", help="path to the binary of a profiling" "tool to use, for example valgrind or operf") parser.add_argument("--profiler_options", help="options surrounded by " "quotes to pass to the profiler") parser.add_argument("--ld_preload", help="Path to a library to be " "preloaded using LD_PRELOAD.") utils.add_redis_params(parser) utils.add_interfaces_params(parser) args = parser.parse_args() args.config = os.path.abspath(args.config) args.log_directory = create_log_directory(args.log_directory) log.info("Logging to %s" % (args.log_directory)) job_status_gui = None job_status_gui_out_fp = None resource_monitor_gui = None resource_monitor_gui_out_fp = None coordinator = None try: # To make the status GUI port distinct for each user but deterministic # for a single user, use 2000 + (the md5 hash of the user's username # mod 1000) as the web GUI's port number username_md5sum = hashlib.md5() username_md5sum.update(getpass.getuser()) job_status_gui_port = ( (int(username_md5sum.hexdigest(), 16) % 1000 + 2000) / 10) * 10 resource_monitor_gui_port = ( (int(username_md5sum.hexdigest(), 16) % 1000 + 3200) / 10) * 10 print "" # Start the resource monitor web GUI resource_monitor_gui, resource_monitor_gui_out_fp = \ start_resource_monitor_gui(args, resource_monitor_gui_port) # Start the job status web GUI job_status_gui, job_status_gui_out_fp = start_job_status_gui( args, job_status_gui_port) print "" coordinator = ClusterCoordinator(**vars(args)) coordinator.run() finally: if job_status_gui is not None: log.info("Stopping job status GUI (PID %d)" % (job_status_gui.pid)) os.killpg(job_status_gui.pid, signal.SIGTERM) job_status_gui.wait() if job_status_gui_out_fp is not None: job_status_gui_out_fp.flush() job_status_gui_out_fp.close() if resource_monitor_gui is not None: log.info("Stopping resource monitor GUI (PID %d)" % ( resource_monitor_gui.pid)) os.killpg(resource_monitor_gui.pid, signal.SIGTERM) resource_monitor_gui.wait() if resource_monitor_gui_out_fp is not None: resource_monitor_gui_out_fp.flush() resource_monitor_gui_out_fp.close() if coordinator is not None: log.info("Stopping node coordinators") coordinator.stop_node_coordinators()