# import sys if len(sys.argv) < 3: print """ OpenEmbedded source checker script require two arguments: 1. location of conf/checksums.ini 2. path to DL_DIR (without "/" at the end) """ sys.exit(0) import ConfigParser, os, itertools checksums_parser = ConfigParser.ConfigParser() checksums_parser.read(sys.argv[1]) parsespin = itertools.cycle(r'|/-\\') item = 1 files_total = len(checksums_parser.sections()) files_checked = 0 files_good = 0 files_wrong = [] for source in checksums_parser.sections(): archive = source.split("/")[-1] localpath = os.path.join(sys.argv[2], archive) md5 = checksums_parser.get(source, "md5") sha = checksums_parser.get(source, "sha256")
#############COMMAND CODE IS BELOW ###################### ###########OPTIONS parser = argparse.ArgumentParser(description='Command line parser of skim options') parser.add_argument('--config', dest='cfgfile', help='Name of config file', required = True) parser.add_argument('--sample', dest='samplename', help='Type of sample: 0 signal, 1 bkgmodel', required = True) parser.add_argument('--casename', dest='casename', help='Case name', required = True) args = parser.parse_args() configfilename = args.cfgfile sampletorun = args.samplename case = args.casename ###########Read Config file print "[INFO] Reading skim configuration file . . ." cfgparser = ConfigParser() cfgparser.read('%s'%configfilename) ##########Get skim variables print "[INFO] Getting configuration parameters . . ." directory = ast.literal_eval(cfgparser.get("configuration","directory")) print " -The directory:" print " *",directory signalsamples = ast.literal_eval(cfgparser.get("configuration","mvasigsamples")) print " -The list of signal samples:" for x in range(len(signalsamples)): print " *",signalsamples[x] bdtbkgsamples = ast.literal_eval(cfgparser.get("configuration","mvabkgsamples")) print " -The list of bdtbkg samples:" print " *",bdtbkgsamples[0] tag = ast.literal_eval(cfgparser.get("configuration","tag")) print " -The tag:"
#!/usr/bin/env python import re import argparse import sys import pika import imp import os import ConfigParser import logging logging.basicConfig() config = ConfigParser.ConfigParser() config.read("./rabbitMQtests.config") class Receiver(object): def __init__(self): #setup the connection sslOptions = {} sslOptions["ca_certs"] = os.path.abspath(config.get('general', "CA_CERT_FILE")) sslOptions["certfile"] = os.path.abspath(config.get('general', 'CLIENT_CERT_FILE')) sslOptions["keyfile"] = os.path.abspath(config.get('general', 'CLIENT_KEY_FILE')) try: self.connection = pika.BlockingConnection(pika.ConnectionParameters( host=config.get('general', 'HOST'), port=config.getint('general', 'PORT'), ssl=config.getboolean('general', 'USE_SSL'), ssl_options = sslOptions ) ) self.channel = self.connection.channel()
# read an INI (config) file import ConfigParser read_opts = ConfigParser.ConfigParser() read_opts.read('../data/options.ini') # print parameters and values for section in read_opts.sections(): print "[%s]" % section for param in read_opts.items(section): print param
# Import necessary modules import os import tempfile import urllib import csv import pyodbc import ConfigParser print "Read config.ini file..." # User defined variables from .ini file... # User needs to change config path configPath = "D:\Dropbox\NiyaMIT\config.ini" cfgParser = ConfigParser.ConfigParser() cfgParser.read(configPath) url = cfgParser.get("HIFLD OPEN DATA URLS", "CareFlty_URL") url2 = cfgParser.get("HIFLD OPEN DATA URLS", "CareFlty2_URL") userDefinedServer = cfgParser.get("SQL SERVER", "ServerName") UserName = cfgParser.get("SQL SERVER", "UserName") Password = cfgParser.get("SQL SERVER", "Password") possibleDatabaseListRaw = cfgParser.get("DATABASE LIST", "possibleDatabaseList") possibleDatabaseList = [] for database in possibleDatabaseListRaw.split(","): possibleDatabaseList.append(database) userDefinedSqFt = cfgParser.get("HOSPITALS", "BedRoomSqFt") print "Done" print
def read_config(filename): config = ConfigParser.ConfigParser() config.readfp(open(filename)) return config
def main(): cur_platform = '??' llvm_path = '??' ndk_root = _check_ndk_root_env() # del the " in the path ndk_root = re.sub(r"\"", "", ndk_root) python_bin = _check_python_bin_env() platform = sys.platform if platform == 'win32': cur_platform = 'windows' elif platform == 'darwin': cur_platform = platform elif 'linux' in platform: cur_platform = 'linux' else: print 'Your platform is not supported!' sys.exit(1) if platform == 'win32': x86_llvm_path = os.path.abspath( os.path.join(ndk_root, 'toolchains/llvm-3.3/prebuilt', '%s' % cur_platform)) else: x86_llvm_path = os.path.abspath( os.path.join(ndk_root, 'toolchains/llvm-3.3/prebuilt', '%s-%s' % (cur_platform, 'x86'))) x64_llvm_path = os.path.abspath( os.path.join(ndk_root, 'toolchains/llvm-3.3/prebuilt', '%s-%s' % (cur_platform, 'x86_64'))) if os.path.isdir(x86_llvm_path): llvm_path = x86_llvm_path elif os.path.isdir(x64_llvm_path): llvm_path = x64_llvm_path else: print 'llvm toolchain not found!' print 'path: %s or path: %s are not valid! ' % (x86_llvm_path, x64_llvm_path) sys.exit(1) project_root = os.path.abspath( os.path.join(os.path.dirname(__file__), '..', '..')) cocos_root = os.path.abspath(os.path.join(project_root, '')) cxx_generator_root = os.path.abspath( os.path.join(project_root, 'tools/bindings-generator')) # save config to file config = ConfigParser.ConfigParser() config.set('DEFAULT', 'androidndkdir', ndk_root) config.set('DEFAULT', 'clangllvmdir', llvm_path) config.set('DEFAULT', 'cocosdir', cocos_root) config.set('DEFAULT', 'cxxgeneratordir', cxx_generator_root) config.set('DEFAULT', 'extra_flags', '') # To fix parse error on windows, we must difine __WCHAR_MAX__ and undefine __MINGW32__ . if platform == 'win32': config.set('DEFAULT', 'extra_flags', '-D__WCHAR_MAX__=0x7fffffff -U__MINGW32__') conf_ini_file = os.path.abspath( os.path.join(os.path.dirname(__file__), 'userconf.ini')) print 'generating userconf.ini...' with open(conf_ini_file, 'w') as configfile: config.write(configfile) # set proper environment variables if 'linux' in platform or platform == 'darwin': os.putenv('LD_LIBRARY_PATH', '%s/libclang' % cxx_generator_root) if platform == 'win32': path_env = os.environ['PATH'] os.putenv( 'PATH', r'%s;%s\libclang;%s\tools\win32;' % (path_env, cxx_generator_root, cxx_generator_root)) try: tolua_root = '%s/tools/tolua' % project_root output_dir = '%s/../runtime-src/Classes/auto' % project_root cmd_args = {'dragonbones.ini' : ('dragonbones', 'lua_dragonbones_auto'), \ } target = 'lua' generator_py = '%s/generator.py' % cxx_generator_root for key in cmd_args.keys(): args = cmd_args[key] cfg = '%s/%s' % (tolua_root, key) print 'Generating bindings for %s...' % (key[:-4]) command = '%s %s %s -s %s -t %s -o %s -n %s' % ( python_bin, generator_py, cfg, args[0], target, output_dir, args[1]) _run_cmd(command) if platform == 'win32': with _pushd(output_dir): _run_cmd('dos2unix *') print '---------------------------------' print 'Generating lua bindings succeeds.' print '---------------------------------' except Exception as e: if e.__class__.__name__ == 'CmdError': print '---------------------------------' print 'Generating lua bindings fails.' print '---------------------------------' sys.exit(1) else: raise
def read_ini(ini_path): config = configparser.ConfigParser(allow_no_value=True) config.read(ini_path) os.path.isfile(ini_path) BORKIFY_METHOD = config.getint('Borkifier', 'Language')
class Settings(object): c = ConfigParser.ConfigParser() def __init__(self, pair): self._path = "./config/%s.ini" % pair self.settings = self.parseConfig printInfo("Checking config file .....\n") if not os.path.isfile(self._path): printInfo("No config file ..... will create one") self.createConfig() self.settings = self.parseConfig() if not self.validateConfig(): quit() def parseConfig(self): settings = {} self.c.read(self._path) for o in self.c.options("Settings"): try: settings[o] = float(self.c.get("Settings", o)) except: settings[o] = self.c.get("Settings", o) return settings def createConfig(self): with open(self._path, 'w') as f: self.c.add_section('Settings') self.c.set('Settings', 'candlestickPeriod', 86400) self.c.set('Settings', 'candlestickHours', 168) self.c.set('Settings', 'bb_length', 10) self.c.set('Settings', 'bb_std', 2) self.c.set('Settings', 'maxBalance', 0.85) self.c.set('Settings', 'profitMargin', 1.1) self.c.set('Settings', 'longMargin', 0.97) self.c.set('Settings', 'shortMargin', 1.03) self.c.set('Settings', 'delta', 0.9) self.c.set('Settings', 'stopLimit', 0.05) self.c.set('Settings', 'stopLimitTimeout', 2) self.c.set('Settings', 'marginCloseTimeout', 2) self.c.set('Settings', "cycletimeout", 0) self.c.write(f) printSuccess( "Config file generated, please modify the config file and re-run this script" ) quit() def validateConfig(self): ok = True if self.settings["longmargin"] >= 1.0: printError("Long margin cannot be >= 1.00") ok = False if self.settings["shortmargin"] <= 1.0: printError("Short margin cannot be <= 1.00") ok = False if self.settings["profitmargin"] <= 1.0: printError("Profit margin cannot be <= 1.0") ok = False if self.settings["candlestickperiod"] not in [ 300, 900, 1800, 7200, 14400, 86400 ]: printError( "Invalid candle stick period, use 300, 900, 1800, 7200, 14400, or 86400" ) ok = False if self.settings["maxbalance"] <= 0.0: printError("Max balance cannot be < 0.0") ok = False if self.settings["maxbalance"] >= 0.95: printError("Max balance cannot be >= 0.95") ok = False if self.settings["stoplimit"] <= 0.01: printError("Stop limit cannot be <= 0.01") ok = False if self.settings["stoplimittimeout"] < 0.5: printError("Stop limit timeout cannot be less than 0.5 hours") ok = False if self.settings["marginclosetimeout"] < 1: printError("Margin close timeout cannot be < 1") ok = False return ok def refresh(self): oldSettings = self.settings self.settings = self.parseConfig() if not self.validateConfig(): self.settings = oldSettings quit() print "\n" for key in self.settings: if self.settings[key] != oldSettings[key]: printHeader(">> %s changed from %s to %s" % (key, oldSettings[key], self.settings[key]))
def __init__(self): self.file = None self.config = ConfigParser.ConfigParser()
def daemon(): global MyPatching, backup_logger, hutil, run_result, run_status, error_msg, freezer, para_parser, snapshot_done, snapshot_info_array, g_fsfreeze_on, total_used_size #this is using the most recent file timestamp. hutil.do_parse_context('Executing') freezer = FsFreezer(patching=MyPatching, logger=backup_logger) global_error_result = None # precheck freeze_called = False configfile = '/etc/azure/vmbackup.conf' thread_timeout = str(60) #Adding python version to the telemetry try: python_version_info = sys.version_info python_version = str(sys.version_info[0]) + '.' + str( sys.version_info[1]) + '.' + str(sys.version_info[2]) HandlerUtil.HandlerUtility.add_to_telemetery_data( "pythonVersion", python_version) except Exception as e: errMsg = 'Failed to do retrieve python version with error: %s, stack trace: %s' % ( str(e), traceback.format_exc()) backup_logger.log(errMsg, True, 'Error') try: if (freezer.mounts is not None): hutil.partitioncount = len(freezer.mounts.mounts) backup_logger.log(" configfile " + str(configfile), True) config = ConfigParsers.ConfigParser() config.read(configfile) if config.has_option('SnapshotThread', 'timeout'): thread_timeout = config.get('SnapshotThread', 'timeout') except Exception as e: errMsg = 'cannot read config file or file not present' backup_logger.log(errMsg, True, 'Warning') backup_logger.log("final thread timeout" + thread_timeout, True) snapshot_info_array = None try: # we need to freeze the file system first backup_logger.log('starting daemon', True) """ protectedSettings is the privateConfig passed from Powershell. WATCHOUT that, the _context_config are using the most freshest timestamp. if the time sync is alive, this should be right. """ if (hutil.is_prev_in_transition()): backup_logger.log( 'retrieving the previous logs for this again inside daemon', True) backup_logger.set_prev_log() protected_settings = hutil._context._config['runtimeSettings'][0][ 'handlerSettings'].get('protectedSettings') public_settings = hutil._context._config['runtimeSettings'][0][ 'handlerSettings'].get('publicSettings') para_parser = ParameterParser(protected_settings, public_settings) commandToExecute = para_parser.commandToExecute #validate all the required parameter here backup_logger.log(commandToExecute, True) if (CommonVariables.iaas_install_command in commandToExecute.lower()): backup_logger.log('install succeed.', True) run_status = 'success' error_msg = 'Install Succeeded' run_result = CommonVariables.success backup_logger.log(error_msg) elif (CommonVariables.iaas_vmbackup_command in commandToExecute.lower()): if (para_parser.backup_metadata is None or para_parser.public_config_obj is None or para_parser.private_config_obj is None): run_result = CommonVariables.error_parameter hutil.SetExtErrorCode(ExtensionErrorCodeHelper. ExtensionErrorCodeEnum.error_parameter) run_status = 'error' error_msg = 'required field empty or not correct' backup_logger.log(error_msg, True, 'Error') else: backup_logger.log('commandToExecute is ' + commandToExecute, True) """ make sure the log is not doing when the file system is freezed. """ temp_status = 'success' temp_result = CommonVariables.ExtensionTempTerminalState temp_msg = 'Transitioning state in extension' blob_report_msg, file_report_msg = get_status_to_report( temp_status, temp_result, temp_msg, None) if (hutil.is_status_file_exists()): status_report_to_file(file_report_msg) status_report_to_blob(blob_report_msg) backup_logger.log('doing freeze now...', True) #partial logging before freeze if (para_parser is not None and para_parser.logsBlobUri is not None and para_parser.logsBlobUri != ""): backup_logger.commit_to_blob(para_parser.logsBlobUri) else: backup_logger.log( "the logs blob uri is not there, so do not upload log." ) backup_logger.log('commandToExecute is ' + commandToExecute, True) PluginHostObj = PluginHost(logger=backup_logger) PluginHostErrorCode, dobackup, g_fsfreeze_on = PluginHostObj.pre_check( ) doFsConsistentbackup = False if not (PluginHostErrorCode == CommonVariables.FailedPrepostPluginhostConfigParsing or PluginHostErrorCode == CommonVariables.FailedPrepostPluginConfigParsing or PluginHostErrorCode == CommonVariables.FailedPrepostPluginhostConfigNotFound or PluginHostErrorCode == CommonVariables. FailedPrepostPluginhostConfigPermissionError or PluginHostErrorCode == CommonVariables.FailedPrepostPluginConfigNotFound or PluginHostErrorCode == CommonVariables. FailedPrepostPluginConfigPermissionError): backup_logger.log( 'App Consistent Consistent Backup Enabled', True) HandlerUtil.HandlerUtility.add_to_telemetery_data( "isPrePostEnabled", "true") if (PluginHostErrorCode != CommonVariables.PrePost_PluginStatus_Success): backup_logger.log( 'Triggering File System Consistent Backup because of error code' + ExtensionErrorCodeHelper.ExtensionErrorCodeHelper. StatusCodeStringBuilder(PluginHostErrorCode), True) doFsConsistentbackup = True if not doFsConsistentbackup: preResult = PluginHostObj.pre_script() dobackup = preResult.continueBackup if (g_fsfreeze_on == False and preResult.anyScriptFailed): dobackup = False if dobackup: freeze_snapshot(thread_timeout) backup_logger.log('unfreeze ends...') if not doFsConsistentbackup: postResult = PluginHostObj.post_script() if not postResult.continueBackup: dobackup = False if (g_fsfreeze_on == False and postResult.anyScriptFailed): dobackup = False if not dobackup: if run_result == CommonVariables.success and PluginHostErrorCode != CommonVariables.PrePost_PluginStatus_Success: run_status = 'error' run_result = PluginHostErrorCode hutil.SetExtErrorCode(PluginHostErrorCode) error_msg = 'Plugin Host Precheck Failed' error_msg = error_msg + ExtensionErrorCodeHelper.ExtensionErrorCodeHelper.StatusCodeStringBuilder( hutil.ExtErrorCode) backup_logger.log(error_msg, True) if run_result == CommonVariables.success: pre_plugin_errors = preResult.errors for error in pre_plugin_errors: if error.errorCode != CommonVariables.PrePost_PluginStatus_Success and error.errorCode != CommonVariables.PrePost_ScriptStatus_Warning: run_status = 'error' run_result = error.errorCode hutil.SetExtErrorCode(error.errorCode) error_msg = 'PreScript failed for the plugin ' + error.pluginName error_msg = error_msg + ExtensionErrorCodeHelper.ExtensionErrorCodeHelper.StatusCodeStringBuilder( hutil.ExtErrorCode) backup_logger.log(error_msg, True) break if run_result == CommonVariables.success: post_plugin_errors = postResult.errors for error in post_plugin_errors: if error.errorCode != CommonVariables.PrePost_PluginStatus_Success and error.errorCode != CommonVariables.PrePost_ScriptStatus_Warning: run_status = 'error' run_result = error.errorCode hutil.SetExtErrorCode(error.errorCode) error_msg = 'PostScript failed for the plugin ' + error.pluginName error_msg = error_msg + ExtensionErrorCodeHelper.ExtensionErrorCodeHelper.StatusCodeStringBuilder( hutil.ExtErrorCode) backup_logger.log(error_msg, True) break if run_result == CommonVariables.success and not doFsConsistentbackup and not ( preResult.anyScriptFailed or postResult.anyScriptFailed): run_status = 'success' run_result = CommonVariables.success_appconsistent hutil.SetExtErrorCode( ExtensionErrorCodeHelper.ExtensionErrorCodeEnum. success_appconsistent) error_msg = 'Enable Succeeded with App Consistent Snapshot' backup_logger.log(error_msg, True) else: run_status = 'error' run_result = CommonVariables.error_parameter hutil.SetExtErrorCode(ExtensionErrorCodeHelper. ExtensionErrorCodeEnum.error_parameter) error_msg = 'command is not correct' backup_logger.log(error_msg, True, 'Error') except Exception as e: errMsg = 'Failed to enable the extension with error: %s, stack trace: %s' % ( str(e), traceback.format_exc()) backup_logger.log(errMsg, True, 'Error') global_error_result = e """ we do the final report here to get rid of the complex logic to handle the logging when file system be freezed issue. """ try: if (global_error_result is not None): if (hasattr(global_error_result, 'errno') and global_error_result.errno == 2): run_result = CommonVariables.error_12 hutil.SetExtErrorCode( ExtensionErrorCodeHelper.ExtensionErrorCodeEnum.error_12) elif (para_parser is None): run_result = CommonVariables.error_parameter hutil.SetExtErrorCode(ExtensionErrorCodeHelper. ExtensionErrorCodeEnum.error_parameter) else: run_result = CommonVariables.error hutil.SetExtErrorCode( ExtensionErrorCodeHelper.ExtensionErrorCodeEnum.error) run_status = 'error' error_msg += ('Enable failed.' + str(global_error_result)) status_report_msg = None HandlerUtil.HandlerUtility.add_to_telemetery_data( "extErrorCode", str(ExtensionErrorCodeHelper.ExtensionErrorCodeHelper. ExtensionErrorCodeNameDict[hutil.ExtErrorCode])) total_used_size = -1 blob_report_msg, file_report_msg = get_status_to_report( run_status, run_result, error_msg, snapshot_info_array) if (hutil.is_status_file_exists()): status_report_to_file(file_report_msg) status_report_to_blob(blob_report_msg) except Exception as e: errMsg = 'Failed to log status in extension' backup_logger.log(errMsg, True, 'Error') if (para_parser is not None and para_parser.logsBlobUri is not None and para_parser.logsBlobUri != ""): backup_logger.commit(para_parser.logsBlobUri) else: backup_logger.log( "the logs blob uri is not there, so do not upload log.") backup_logger.commit_to_local() sys.exit(0)
class Grappy(scrapy.Spider): name = "grappy" config = ConfigParser.ConfigParser() config.read('config.ini') start_urls = [config.get('PAGES', 'LOGIN')] #send login form data to login page def parse(self, response): self.init_db() yield scrapy.FormRequest.from_response( response, formxpath='//form', formdata={ 'user[email]': self.config.get('AUTH', 'USERNAME'), 'user[password]': self.config.get('AUTH', 'PASSWORD') }, callback=self.after_login) #check if login was succesful, and proceed with first profile to crawl def after_login(self, response): if "try again" in response.body: self.logger.error( "Login error! Make sure the credentials are set in the config.ini file" ) return else: self.logger.info( 'Succesfully logged in, proceed with first profile!') baseurl = self.config.get('PAGES', 'START_PROFILE') yield response.follow(baseurl, self.action) #extract data from profile, save it and proceed with next profile def action(self, response): url = '' name = '' picture = '' details = '' dob = '' website = '' activity = '' interests = '' favourite = '' about = '' name = response.css('h1.userProfileName::text').extract()[0].strip() picture = response.xpath( '//img[@class="profilePictureIcon circularIcon circularIcon--huge circularIcon--border"]/@src' ).extract()[0] url = response.request.url titles = response.xpath("//div[@class='infoBoxRowTitle']").extract() items = response.xpath("//div[@class='infoBoxRowItem']").extract() for i in range(0, len(titles)): self.logger.info("key " + titles[i] + " value " + items[i].strip()) if ("Details" in titles[i]): details = response.xpath( "//div[@class='infoBoxRowItem']")[i].xpath( "text()").extract_first().strip().replace("'", "`") if ("Birthday" in titles[i]): dob = response.xpath( "//div[@class='infoBoxRowItem']")[i].xpath( "text()").extract_first().strip().replace("'", "`") if ("Website" in titles[i]): website = response.xpath("//div[@class='infoBoxRowItem']")[ i].css('a::attr(href)').extract_first().replace("'", "`") if ("Activity" in titles[i]): activity = response.xpath( "//div[@class='infoBoxRowItem']")[i].xpath( "text()").extract_first().strip().replace("'", "`") if ("Interests" in titles[i]): interests = response.xpath( "//div[@class='infoBoxRowItem']")[i].xpath( "text()").extract_first().strip().replace("'", "`") if ("Favorite" in titles[i]): favourite = response.xpath( "//div[@class='infoBoxRowItem']")[i].xpath( "text()").extract_first().strip().replace("'", "`") if ("About" in titles[i]): about = response.xpath( "//*[contains(@id, 'freeTextContainerdisplay_user')]/text()" )[0].extract().replace("'", "`") self.insert(url, name, picture, details, dob, website, activity, interests, favourite, about) for href in response.css('div.friendName a::attr(href)'): yield response.follow(href, self.action) #insert profile information into DB def insert(self, url, name, picture, details, dob, website, activity, interests, favourite, about): conn = sqlite3.connect(self.config.get('OUTPUT', 'DB')) query = "INSERT INTO PROFILE (URL,NAME,PICTURE,DETAILS,DOB, WEBSITE, ACTIVITY, INTERESTS, FAVOURITES, ABOUT) VALUES ('" + url + "','" + name + "','" + picture + "','" + details + "','" + dob + "','" + website + "','" + activity + "','" + interests + "','" + favourite + "','" + about + "')" self.logger.info("execute query " + query) conn.execute(query) conn.commit() conn.close() #create DB and 'profile' table if nox exist def init_db(self): conn = sqlite3.connect(self.config.get('OUTPUT', 'DB')) self.logger.info("Database created successfully") conn.execute('''CREATE TABLE IF NOT EXISTS PROFILE (URL TEXT NOT NULL, NAME TEXT, PICTURE TEXT, DETAILS TEXT, DOB TEXT, WEBSITE TEXT, ACTIVITY TEXT, INTERESTS TEXT, FAVOURITES TEXT, ABOUT TEXT);''') self.logger.info("Table created successfully") conn.close()
def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('-b', '--bucket', help="The bucket name", required=True) parser.add_argument('-o', '--object', help="The object URI", required=True) parser.add_argument('-e', '--expires-in', type=int, nargs='?', dest="expires_in", help="How long the signed URL will be valid") parser.add_argument('-a', '--access-key', nargs='?', dest="access_key", default=None) parser.add_argument('-s', '--secret-key', nargs='?', dest="secret_key", default=None) if len(sys.argv[1:]) == 0: parser.print_help() sys.exit(1) args = parser.parse_args() bucket = args.bucket key = args.object if args.expires_in: seconds = args.expires_in else: seconds = 60 * 60 if args.access_key and args.secret_key: access_key = args.access_key secret_key = args.secret_key elif len([x for x in (args.access_key, args.secret_key) if x is not None]) == 1: parser.error('--access-key and --secret-key must be given together') else: config = ConfigParser.ConfigParser() config.read(expanduser("~/.boto")) if config.has_option('Credentials', 'aws_access_key_id') and config.has_option( 'Credentials', 'aws_secret_access_key'): access_key = config.get('Credentials', 'aws_access_key_id') secret_key = config.get('Credentials', 'aws_secret_access_key') else: config.read(expanduser("~/.s3cfg")) if config.has_option('default', 'access_key') and config.has_option( 'default', 'secret_key'): access_key = config.get('default', 'access_key') secret_key = config.get('default', 'secret_key') else: print( "You have to specify access_key and secret_key on the command line or in the ~/.boto, ~/.s3cfg." ) sys.exit(1) conn = S3Connection(aws_access_key_id=access_key, aws_secret_access_key=secret_key) try: buckets = conn.get_all_buckets() except S3ResponseError as e: print(e.message) sys.exit(1) if conn.lookup(bucket) is None: print('No such bucket!') sys.exit(1) else: if conn.get_bucket(bucket).get_key(key) is None: print('The key "{0}" does not exist'.format(key)) sys.exit(1) else: print conn.generate_url(seconds, 'GET', bucket, key, response_headers={ 'response-content-type': 'application/octet-stream' })
def __init__(self, config_path): self.config_path = config_path self.cc = configParser.ConfigParser() self.cc.readfp(file(config_path))
def main(): """ Retrieve network indicators from mongodb, used by cuckoo. Pass to genStixDoc to create the stix doc. Handle de-dup and filter lists for indicators, as well as stix items created in previous runs. """ ap = argparse.ArgumentParser() apg = ap.add_mutually_exclusive_group() apg.add_argument('--job-id', dest='jobId', default='', help='Cuckoo job id to query.') apg.add_argument('--md5', dest='md5', default='', help='File md5 hash to query.') apg.add_argument('--sha1', dest='sha1', default='', help='File sha1 hash to query.') apg.add_argument('--sha256', dest='sha256', default='', help='File sha256 hash to query.') apg.add_argument('--sha512', dest='sha512', default='', help='File sha512 hash to query.') args = ap.parse_args() config = ConfigParser.ConfigParser() config.read('app.conf') conn = pymongo.MongoClient(config.get('mongo','dbUrl')) with open(config.get('filterOut','fIpv4Addresses'), 'r+') as fIpv4AddressesFH: fIpv4Addresses = [line.rstrip('\n') for line in fIpv4AddressesFH] fIpv4AddressesFH.closed with open(config.get('filterOut','fHostNames'), 'r+') as fHostNamesFH: fHostNames = [line.rstrip('\n') for line in fHostNamesFH] fHostNamesFH.closed with open(config.get('filterOut','fSeenEntries'), 'w+') as fSeenEntriesFH: fSeenEntries = [line.rstrip('\n') for line in fSeenEntriesFH] fSeenEntriesFH.closed networkItems = [] ipv4Addresses = [] hostNames = [] _l.info('Starting...') fSeenEntriesFH = open(config.get('filterOut','fSeenEntries'), 'a', 0) cfg_collections = config.get('mongo','dbCollectionNames') if ',' in cfg_collections: db_collection_names = cfg_collections.split(',') else: db_collection_names = [cfg_collections] cuckoo_names = config.get('dbsList','cuckoo') if ',' in cuckoo_names: cuckoo_servers = cuckoo_names.split(',') else: cuckoo_servers = [cuckoo_names] for dbkey, dbs in enumerate(cuckoo_servers): db = conn[dbs] mongo_collection = getattr(db, db_collection_names[dbkey]) _l.debug('Connected to data source.') # Get a list of file names and hashes from db if args.jobId: cs = mongo_collection.aggregate([{"$match": {"info.id": int(args.jobId)}}, {"$group": {"_id": {"targetFileSha1": "$target.file.sha1", "targetFileSha256": "$target.file.sha256", "targetFileSha512": "$target.file.sha512", "targetFileSsdeep": "$target.file.ssdeep", "targetFileMd5": "$target.file.md5", "targetFileSize": "$target.file.size", "targetFileName": "$target.file.name"}}}]) elif args.md5: cs = mongo_collection.aggregate([{"$match": {"target.file.md5": args.md5}}, {"$group": {"_id": {"targetFileSha1": "$target.file.sha1", "targetFileSha256": "$target.file.sha256", "targetFileSha512": "$target.file.sha512", "targetFileSsdeep": "$target.file.ssdeep", "targetFileMd5": "$target.file.md5", "targetFileSize": "$target.file.size", "targetFileName": "$target.file.name"}}}]) elif args.sha1: cs = mongo_collection.aggregate([{"$match": {"target.file.sha1": args.sha1}}, {"$group": {"_id": {"targetFileSha1": "$target.file.sha1", "targetFileSha256": "$target.file.sha256", "targetFileSha512": "$target.file.sha512", "targetFileSsdeep": "$target.file.ssdeep", "targetFileMd5": "$target.file.md5", "targetFileSize": "$target.file.size", "targetFileName": "$target.file.name"}}}]) elif args.sha256: cs = mongo_collection.aggregate([{"$match": {"target.file.sha256": args.sha256}}, {"$group": {"_id": {"targetFileSha1": "$target.file.sha1", "targetFileSha256": "$target.file.sha256", "targetFileSha512": "$target.file.sha512", "targetFileSsdeep": "$target.file.ssdeep", "targetFileMd5": "$target.file.md5", "targetFileSize": "$target.file.size", "targetFileName": "$target.file.name"}}}]) elif args.sha512: cs = mongo_collection.aggregate([{"$match": {"target.file.sha512": args.sha512}}, {"$group": {"_id": {"targetFileSha1": "$target.file.sha1", "targetFileSha256": "$target.file.sha256", "targetFileSha512": "$target.file.sha512", "targetFileSsdeep": "$target.file.ssdeep", "targetFileMd5": "$target.file.md5", "targetFileSize": "$target.file.size", "targetFileName": "$target.file.name"}}}]) else: cs = mongo_collection.aggregate([{"$group": {"_id": {"targetFileSha1": "$target.file.sha1", "targetFileSha256": "$target.file.sha256", "targetFileSha512": "$target.file.sha512", "targetFileSsdeep": "$target.file.ssdeep", "targetFileMd5": "$target.file.md5", "targetFileSize": "$target.file.size", "targetFileName": "$target.file.name"}}}]) _l.debug('Executed initial aggregation query.') for i in cs['result']: try: # Get all network indicators: addresses and names networkItems[:] = [] ipv4Addresses[:] = [] hostNames[:] = [] networkUdpSrc = mongo_collection.find( { "target.file.sha1": i['_id']['targetFileSha1'], "target.file.sha256": i['_id']['targetFileSha256'], "target.file.sha512": i['_id']['targetFileSha512'], "target.file.ssdeep": i['_id']['targetFileSsdeep'], "target.file.md5": i['_id']['targetFileMd5'], "target.file.size": i['_id']['targetFileSize'], "target.file.name": i['_id']['targetFileName']}).distinct('network.udp.src') networkUdpDst = mongo_collection.find( { "target.file.sha1": i['_id']['targetFileSha1'], "target.file.sha256": i['_id']['targetFileSha256'], "target.file.sha512": i['_id']['targetFileSha512'], "target.file.ssdeep": i['_id']['targetFileSsdeep'], "target.file.md5": i['_id']['targetFileMd5'], "target.file.size": i['_id']['targetFileSize'], "target.file.name": i['_id']['targetFileName']}).distinct('network.udp.dst') networkIcmpSrc = mongo_collection.find( { "target.file.sha1": i['_id']['targetFileSha1'], "target.file.sha256": i['_id']['targetFileSha256'], "target.file.sha512": i['_id']['targetFileSha512'], "target.file.ssdeep": i['_id']['targetFileSsdeep'], "target.file.md5": i['_id']['targetFileMd5'], "target.file.size": i['_id']['targetFileSize'], "target.file.name": i['_id']['targetFileName']}).distinct('network.icmp.src') networkIcmpDst = mongo_collection.find( { "target.file.sha1": i['_id']['targetFileSha1'], "target.file.sha256": i['_id']['targetFileSha256'], "target.file.sha512": i['_id']['targetFileSha512'], "target.file.ssdeep": i['_id']['targetFileSsdeep'], "target.file.md5": i['_id']['targetFileMd5'], "target.file.size": i['_id']['targetFileSize'], "target.file.name": i['_id']['targetFileName']}).distinct('network.icmp.dst') networkTcpSrc = mongo_collection.find( { "target.file.sha1": i['_id']['targetFileSha1'], "target.file.sha256": i['_id']['targetFileSha256'], "target.file.sha512": i['_id']['targetFileSha512'], "target.file.ssdeep": i['_id']['targetFileSsdeep'], "target.file.md5": i['_id']['targetFileMd5'], "target.file.size": i['_id']['targetFileSize'], "target.file.name": i['_id']['targetFileName']}).distinct('network.tcp.src') networkTcpDst = mongo_collection.find( { "target.file.sha1": i['_id']['targetFileSha1'], "target.file.sha256": i['_id']['targetFileSha256'], "target.file.sha512": i['_id']['targetFileSha512'], "target.file.ssdeep": i['_id']['targetFileSsdeep'], "target.file.md5": i['_id']['targetFileMd5'], "target.file.size": i['_id']['targetFileSize'], "target.file.name": i['_id']['targetFileName']}).distinct('network.tcp.dst') networkDnsAnswersData = mongo_collection.find( { "target.file.sha1": i['_id']['targetFileSha1'], "target.file.sha256": i['_id']['targetFileSha256'], "target.file.sha512": i['_id']['targetFileSha512'], "target.file.ssdeep": i['_id']['targetFileSsdeep'], "target.file.md5": i['_id']['targetFileMd5'], "target.file.size": i['_id']['targetFileSize'], "target.file.name": i['_id']['targetFileName']}).distinct('network.dns.answers.data') networkDomainsIp = mongo_collection.find( { "target.file.sha1": i['_id']['targetFileSha1'], "target.file.sha256": i['_id']['targetFileSha256'], "target.file.sha512": i['_id']['targetFileSha512'], "target.file.ssdeep": i['_id']['targetFileSsdeep'], "target.file.md5": i['_id']['targetFileMd5'], "target.file.size": i['_id']['targetFileSize'], "target.file.name": i['_id']['targetFileName']}).distinct('network.domains.ip') networkHttpHost = mongo_collection.find( { "target.file.sha1": i['_id']['targetFileSha1'], "target.file.sha256": i['_id']['targetFileSha256'], "target.file.sha512": i['_id']['targetFileSha512'], "target.file.ssdeep": i['_id']['targetFileSsdeep'], "target.file.md5": i['_id']['targetFileMd5'], "target.file.size": i['_id']['targetFileSize'], "target.file.name": i['_id']['targetFileName']}).distinct('network.http.host') networkHosts = mongo_collection.find( { "target.file.sha1": i['_id']['targetFileSha1'], "target.file.sha256": i['_id']['targetFileSha256'], "target.file.sha512": i['_id']['targetFileSha512'], "target.file.ssdeep": i['_id']['targetFileSsdeep'], "target.file.md5": i['_id']['targetFileMd5'], "target.file.size": i['_id']['targetFileSize'], "target.file.name": i['_id']['targetFileName']}).distinct('network.hosts') networkDnsRequest = mongo_collection.find( { "target.file.sha1": i['_id']['targetFileSha1'], "target.file.sha256": i['_id']['targetFileSha256'], "target.file.sha512": i['_id']['targetFileSha512'], "target.file.ssdeep": i['_id']['targetFileSsdeep'], "target.file.md5": i['_id']['targetFileMd5'], "target.file.size": i['_id']['targetFileSize'], "target.file.name": i['_id']['targetFileName']}).distinct('network.dns.request') networkDomainsDomain = mongo_collection.find( { "target.file.sha1": i['_id']['targetFileSha1'], "target.file.sha256": i['_id']['targetFileSha256'], "target.file.sha512": i['_id']['targetFileSha512'], "target.file.ssdeep": i['_id']['targetFileSsdeep'], "target.file.md5": i['_id']['targetFileMd5'], "target.file.size": i['_id']['targetFileSize'], "target.file.name": i['_id']['targetFileName']}).distinct('network.domains.domain') # Aggregate all found items and remove duplicates and empty networkItems += networkUdpSrc + networkUdpDst + networkIcmpSrc + \ networkIcmpDst + networkTcpSrc + networkTcpDst + \ networkDnsAnswersData + networkDomainsIp + networkHttpHost + \ networkHosts + networkDnsRequest + networkDomainsDomain networkItems = list(set(networkItems)) networkItems = filter(None, networkItems) # Split into one list for addresses and one for host names ipv4Addresses = keepAddresses(networkItems[:]) hostNames = keepHostNames(networkItems[:]) # Delete addresses and host names if in whitelist files ipv4Addresses = delIfMatchedAddr(ipv4Addresses, fIpv4Addresses) hostNames = delIfMatchedHostName(hostNames, fHostNames) # Get file names targetFileName = mongo_collection.find( { "target.file.sha1": i['_id']['targetFileSha1'], "target.file.sha256": i['_id']['targetFileSha256'], "target.file.sha512": i['_id']['targetFileSha512'], "target.file.ssdeep": i['_id']['targetFileSsdeep'], "target.file.md5": i['_id']['targetFileMd5'], "target.file.size": i['_id']['targetFileSize'], "target.file.name": i['_id']['targetFileName']}).distinct('target.file.name') # Call the function to create the output, check if seen before first if str(i['_id']['targetFileSha1']) + ',' + \ str(i['_id']['targetFileSha256']) + ',' + \ str(i['_id']['targetFileSha512']) + ',' + \ str(i['_id']['targetFileSsdeep']) + ',' + \ str(i['_id']['targetFileMd5']) + ',' + \ str(i['_id']['targetFileSize']) not in str(fSeenEntries): if ipv4Addresses or hostNames: genStixDoc(config.get('output','outputDir'), str(i['_id']['targetFileSha1']), str(i['_id']['targetFileSha256']), str(i['_id']['targetFileSha512']), str(i['_id']['targetFileSsdeep']), str(i['_id']['targetFileMd5']), str(i['_id']['targetFileSize']), i['_id']['targetFileName'], ipv4Addresses, hostNames) # Write to file so that we can read back in as filter later fSeenEntriesFH.write(str(i['_id']['targetFileSha1']) + ',' + \ str(i['_id']['targetFileSha256']) + ',' + \ str(i['_id']['targetFileSha512']) + ',' + \ str(i['_id']['targetFileSsdeep']) + ',' + \ str(i['_id']['targetFileMd5']) + ',' + \ str(i['_id']['targetFileSize']) + '\n') _l.debug('Updated SeenEntries file with: ' + \ str(i['_id']['targetFileSha256']) + ',' + \ str(i['_id']['targetFileSha512']) + ',' + \ str(i['_id']['targetFileSsdeep']) + ',' + \ str(i['_id']['targetFileMd5']) + ',' + \ str(i['_id']['targetFileSize']) + \ ' since content has been written to stix file.\n') except Exception as e: import traceback tb = traceback.format_exc() _l.error('Row failed due to: ' + str(e) + "\n\n" + str(tb) + "\n\n" + str(repr(i))) conn.disconnect() fSeenEntriesFH.closed _l.info('Ended.')
def run_qemu_io_blkdebug(test, params, env): """ Run qemu-io blkdebug tests: 1. Create image with given parameters 2. Write the blkdebug config file 3. Try to do operate in image with qemu-io and get the error message 4. Get the error message from perror by error number set in config file 5. Compare the error message @param test: kvm test object @param params: Dictionary with the test parameters @param env: Dictionary with test environment. """ tmp_dir = params.get("tmp_dir", "/tmp") blkdebug_cfg = utils_misc.get_path(tmp_dir, params.get("blkdebug_cfg", "blkdebug.cfg")) err_command = params.get("err_command") err_event = params.get("err_event") errn_list = re.split("\s+", params.get("errn_list").strip()) re_std_msg = params.get("re_std_msg") test_timeout = int(params.get("test_timeout", "60")) pre_err_commands = params.get("pre_err_commands") image = params.get("images") blkdebug_default = params.get("blkdebug_default") error.context("Create image", logging.info) image_io = QemuImg(params.object_params(image), test.bindir, image) image_name = image_io.create(params.object_params(image)) template_name = utils_misc.get_path(test.virtdir, blkdebug_default) template = ConfigParser.ConfigParser() template.read(template_name) for errn in errn_list: log_filename = utils_misc.get_path(test.outputdir, "qemu-io-log-%s" % errn) error.context("Write the blkdebug config file", logging.info) template.set("inject-error", "event", '"%s"' % err_event) template.set("inject-error", "errno", '"%s"' % errn) error.context("Write blkdebug config file", logging.info) blkdebug = None try: blkdebug = open(blkdebug_cfg, 'w') template.write(blkdebug) finally: if blkdebug is not None: blkdebug.close() error.context("Operate in qemu-io to trigger the error", logging.info) session = qemu_io.QemuIOShellSession(test, params, image_name, blkdebug_cfg=blkdebug_cfg, log_filename=log_filename) if pre_err_commands: for cmd in re.split(",", pre_err_commands.strip()): session.cmd_output(cmd, timeout=test_timeout) output = session.cmd_output(err_command, timeout=test_timeout) error.context("Get error message from command perror", logging.info) perror_cmd = "perror %s" % errn std_msg = utils.system_output(perror_cmd) std_msg = re.findall(re_std_msg, std_msg) if std_msg: std_msg = std_msg[0] else: std_msg = "" logging.warning("Can not find error message from perror") session.close() error.context("Compare the error message", logging.info) if std_msg: if std_msg in output: logging.info("Error message is correct in qemu-io") else: fail_log = "The error message is mismatch:" fail_log += "qemu-io reports: '%s'," % output fail_log += "perror reports: '%s'" % std_msg raise error.TestFail(fail_log) else: logging.warning("Can not find error message from perror." " The output from qemu-io is %s" % output)
def __init__(self, ID=0): Cf = ConfigParser.ConfigParser() Cf.read(pyu.getlong('agent.ini', 'ini')) self.ag_opt = dict(Cf.items(ID)) self.parse()
def plotDiagram(name): filename = name + ".dat" X = load(filename) flowWEA = X[:, 0] flowNSA = X[:, 1] qWEA = X[:, 2] qNSA = X[:, 3] avgDelayWEA = X[:, 4] avgDelayNSA = X[:, 5] avgDelayA = X[:, 6] greenWEA = X[:, 7] greenNSA = X[:, 8] filename = name + '.ini' ini = ConfigParser() ini.read(filename) N = ini.getint("general", "N") cut = ini.getboolean("general", "cut") distrWE = ini.get("demand", "distrWE") distrNS = ini.get("demand", "distrNS") tlType = ini.get("TL", "tlType") vehphWEA = eval(ini.get("demand", "vehphWEA")) vehphNSA = eval(ini.get("demand", "vehphNSA")) maxDelay = max(max(avgDelayWEA), max(avgDelayNSA)) maxD = max(avgDelayA) c = maxD / (exp(1) - 1) V = [c * (exp(i) - 1) for i in frange(0, 1, 0.01)] [X, Y] = meshgrid(vehphWEA, vehphNSA) Z = griddata(flowWEA, flowNSA, avgDelayWEA, X, Y) figure(figsize=(12, 12)) subplot(2, 2, 1) # contour(X, Y, Z, 100) contourf(X, Y, Z, range(0, 180, 5)) colorbar() title('Average Delay WE') xlabel('Input flow WE') ylabel('Input flow NS') Z = griddata(flowWEA, flowNSA, avgDelayNSA, X, Y) subplot(2, 2, 2) # contour(X, Y, Z, 100) contourf(X, Y, Z, range(0, 180, 5)) colorbar() title('Average Delay NS') xlabel('Input flow WE') ylabel('Input flow NS') Z = griddata(qWEA, qNSA, avgDelayWEA, X, Y) avgDelayWECut = [Z[i][i] for i in range(len(Z))] subplot(2, 2, 3) # contour(X, Y, Z, 100) contourf(X, Y, Z, range(0, 180, 5)) colorbar() title('Average Delay WE') xlabel('Flow q WE') ylabel('Flow q NS') Z = griddata(qWEA, qNSA, avgDelayNSA, X, Y) avgDelayNSCut = [Z[i][i] for i in range(len(Z))] subplot(2, 2, 4) # contour(X, Y, Z, 100) contourf(X, Y, Z, range(0, 180, 5)) colorbar() title('Average Delay NS') xlabel('Flow q WE') ylabel('Flow q NS') suptitle("Average Delay (WE " + ("poisson" if distrWE == 'p' else "uniform") + ", NS " + ("poisson" if distrNS == 'p' else "uniform") + ", " + tlType + ")") savefig(name + "AvgDelayNSWE.png") close() Z = griddata(flowWEA, flowNSA, avgDelayA, X, Y) figure(figsize=(12, 6)) subplot(1, 2, 1) # contour(X, Y, Z, 100) contourf(X, Y, Z, range(0, 180, 5)) colorbar() title('Average Delay') xlabel('Input flow WE') ylabel('Input flow NS') Z = griddata(qWEA, qNSA, avgDelayA, X, Y) avgDelayCut = [Z[i][i] for i in range(len(Z))] subplot(1, 2, 2) # contour(X, Y, Z, 100) contourf(X, Y, Z, range(0, 180, 5)) # clim(0, maxDelay) colorbar() title('Average Delay') xlabel('Flow q WE') ylabel('Flow q NS') suptitle("Average Delay (WE " + ("poisson" if distrWE == 'p' else "uniform") + ", NS " + ("poisson" if distrNS == 'p' else "uniform") + ", " + tlType + ")") savefig(name + "AvgDelay.png") close() Z = griddata(flowWEA, flowNSA, qWEA, X, Y) figure(figsize=(12, 6)) subplot(1, 2, 1) contourf(X, Y, Z, range(0, 1250, 50)) colorbar() title('Flow q WE') xlabel('Input flow WE') ylabel('Input flow NS') Z = griddata(flowWEA, flowNSA, qNSA, X, Y) subplot(1, 2, 2) contourf(X, Y, Z, range(0, 1250, 50)) colorbar() title('Flow q NS') xlabel('Input flow WE') ylabel('Input flow NS') suptitle("Input flow vs. flow q (WE " + ("poisson" if distrWE == 'p' else "uniform") + ", NS " + ("poisson" if distrNS == 'p' else "uniform") + ", " + tlType + ")") savefig(name + "flowvsq.png") close() maxGreen = max(max(greenWEA), max(greenNSA)) Z = griddata(flowWEA, flowNSA, greenWEA, X, Y) figure(figsize=(12, 12)) subplot(2, 2, 1) contourf(X, Y, Z, range(0, 100, 5)) # clim(0, maxGreen) colorbar() title('Green-time WE') xlabel('Input flow WE') ylabel('Input flow NS') Z = griddata(flowWEA, flowNSA, greenNSA, X, Y) subplot(2, 2, 2) contourf(X, Y, Z, range(0, 100, 5)) # clim(0, maxGreen) colorbar() title('Green-time NS') xlabel('Input flow WE') ylabel('Input flow NS') Z = griddata(qWEA, qNSA, greenWEA, X, Y) greenWECut = [Z[i][i] for i in range(len(Z))] subplot(2, 2, 3) contourf(X, Y, Z, range(0, 100, 5)) # clim(0, maxGreen) colorbar() title('Green-time WE') xlabel('Flow q WE') ylabel('Flow q NS') Z = griddata(qWEA, qNSA, greenNSA, X, Y) greenNSCut = [Z[i][i] for i in range(len(Z))] subplot(2, 2, 4) contourf(X, Y, Z, range(0, 100, 5)) colorbar() title('Green-time NS') xlabel('Flow q WE') ylabel('Flow q NS') suptitle("Green-time (WE " + ("poisson" if distrWE == 'p' else "uniform") + ", NS " + ("poisson" if distrNS == 'p' else "uniform") + ", " + tlType + ")") savefig(name + "GreenTime.png") close() q = X[0] figure() title("CUT (WE " + ("poisson" if distrWE == 'p' else "uniform") + ", NS " + ("poisson" if distrNS == 'p' else "uniform") + ", " + tlType + ")") xlabel('Flow q') plot(q[0:15], avgDelayWECut[0:15], q[0:15], avgDelayNSCut[0:15], q[0:15], avgDelayCut[0:15], q[0:15], greenWECut[0:15], q[0:15], greenNSCut[0:15]) legend(('avgDelayWE', 'avgDelayNS', 'avgDelay', 'greenWE', 'greenNS'), loc='upper left') savefig(name + "CUT.png") close()
USE_SETUPCFG = os.environ.get('USE_SETUPCFG') # override use of setup.cfg with env var. if USE_SETUPCFG is not None: USE_SETUPCFG = bool(int(USE_SETUPCFG)) else: USE_SETUPCFG = True setup_cfg = 'setup.cfg' # contents of setup.cfg will override env vars, unless # USE_SETUPCFG evaluates to True. Exception is use_ncconfig, # which does not take precedence ofver USE_NCCONFIG env var. ncconfig = None use_ncconfig = None if USE_SETUPCFG and os.path.exists(setup_cfg): sys.stdout.write('reading from setup.cfg...\n') config = configparser.ConfigParser() config.read(setup_cfg) try: HDF5_dir = config.get("directories", "HDF5_dir") except: pass try: HDF5_libdir = config.get("directories", "HDF5_libdir") except: pass try: HDF5_incdir = config.get("directories", "HDF5_incdir") except: pass try: netCDF4_dir = config.get("directories", "netCDF4_dir")
def compareData(name1, name2, str): filename = name1 + '.ini' ini = ConfigParser() ini.read(filename) distrWE = ini.get("demand", "distrWE") distr = ("poisson" if (distrWE == 'p') else "uniform") vehphWEA = eval(ini.get("demand", "vehphWEA")) vehphNSA = eval(ini.get("demand", "vehphNSA")) filename1 = name1 + ".dat" data1 = load(filename1) flowWEA1 = data1[:, 0] flowNSA1 = data1[:, 1] qWEA1 = data1[:, 2] qNSA1 = data1[:, 3] avgDelayWEA1 = data1[:, 4] avgDelayNSA1 = data1[:, 5] avgDelayA1 = data1[:, 6] greenWEA1 = data1[:, 7] greenNSA1 = data1[:, 8] filename2 = name2 + ".dat" data2 = load(filename2) flowWEA2 = data2[:, 0] flowNSA2 = data2[:, 1] qWEA2 = data2[:, 2] qNSA2 = data2[:, 3] avgDelayWEA2 = data2[:, 4] avgDelayNSA2 = data2[:, 5] avgDelayA2 = data2[:, 6] greenWEA2 = data2[:, 7] greenNSA2 = data2[:, 8] [X, Y] = meshgrid(vehphWEA, vehphNSA) figure(figsize=(6, 6)) Z1 = array(griddata(flowWEA1, flowNSA1, avgDelayA1, X, Y)) Z2 = array(griddata(flowWEA2, flowNSA2, avgDelayA2, X, Y)) Z = Z1 / Z2 subplot(1, 1, 1) contourf(X, Y, Z, concatenate((frange(0, 2, 0.1), frange(2, 3, 1)))) xlabel('Input flow WE') ylabel('Input flow NS') colorbar() title('Average Delay VA vs. FC (ratio), ' + distr) savefig(str + ".png") Z1 = array(griddata(flowWEA1, flowNSA1, avgDelayWEA1, X, Y)) Z2 = array(griddata(flowWEA2, flowNSA2, avgDelayWEA2, X, Y)) Z = Z1 / Z2 figure(figsize=(12, 6)) subplot(1, 2, 1) contourf(X, Y, Z, concatenate((frange(0, 2, 0.1), frange(2, 3, 1)))) xlabel('Input flow WE') ylabel('Input flow NS') title('Average Delay WE') colorbar() Z1 = array(griddata(flowWEA1, flowNSA1, avgDelayNSA1, X, Y)) Z2 = array(griddata(flowWEA2, flowNSA2, avgDelayNSA2, X, Y)) Z = Z1 / Z2 subplot(1, 2, 2) contourf(X, Y, Z, concatenate((frange(0, 2, 0.1), frange(2, 3, 1)))) xlabel('Input flow WE') ylabel('Input flow NS') title('Average Delay NS') colorbar() suptitle('Average Delay VA vs. FC (ratio), ' + distr) savefig(str + "WENS.png")
import time import gc from sklearn import metrics import logging from sklearn import cross_validation, metrics from sklearn.model_selection import GridSearchCV from sklearn.metrics import classification_report from pca.PCAColumn import PCAColumn from Utils.config import * from Utils.string_utils import get_bool_value from pca.PCAForXgb import PCAForXgb import ConfigParser ENV_CONFIG = ConfigParser.ConfigParser() ENV_CONFIG.read("env.conf") LEARNING_ALG = ENV_CONFIG.get('LEARNING', 'algorithm') ACTIVATE_FS = get_bool_value(ENV_CONFIG.get('LEARNING', 'feature_selector')) FS_PCT = int(ENV_CONFIG.get('LEARNING', 'feature_selector_pct')) REMOVE_PCA_FEATURES = get_bool_value( ENV_CONFIG.get('LEARNING', 'remove_pca_processed_features')) class ExprWithPCA(PCAForXgb): def preprocess(self, ori_file_data, all_pca_added_df, all_encoders, is_training=False):
def connect(self): self.config = ConfigParser.ConfigParser() self.config.read([os.path.expanduser('~/.gitlab.cfg')]) host = self.config.get("default", "host") token = self.config.get("default", "token") return gitlab.Gitlab(host=host, token=token)
def getSections(self): config = ConfigParser.ConfigParser() config.read(self.__conFile) return config.sections()
#bed leveling tool import time import sys, os import serial from subprocess import call import numpy as np import json import ConfigParser import logging import re config = ConfigParser.ConfigParser() config.read('/var/www/lib/config.ini') serialconfig = ConfigParser.ConfigParser() serialconfig.read('/var/www/lib/serial.ini') #check if LOCK FILE EXISTS if os.path.isfile(config.get('task', 'lock_file')): print "printer busy" sys.exit() #Args try: logfile=str(sys.argv[1]) #param for the log file log_trace=str(sys.argv[2]) #trace log file fix_d=float(sys.argv[3]) #hight of the plane. (smaller=higher) except:
self.worksheet_v4.write(GLOBAL_LINE_v4, 6, line_data["lego_memory_v4"], self.style) if __name__ == '__main__': # init mylogging logger = mylogging.init_log(logging.DEBUG) global GLOBAL_LINE_5117 global GLOBAL_LINE_v3 global GLOBAL_LINE_v4 # init excel wow = LoadExcel() # init config_parser try: cf = ConfigParser.ConfigParser() cf.read("../conf/load_config.conf") threadsize_list=[1,2,6,10,12] #threadsize_list=[1] if len(sys.argv) == 2: #TODO model_list = sys.argv[1].split(",") for model in model_list: conf_name = "conf_%s" % model for thread_size in threadsize_list: print "==========model: %s,thread_size: %s================" % (model, thread_size) try: db_name = cf.get(conf_name, "test_db") % thread_size except Exception as e: print ("[error]: Pls Check The Modle:%s input wrong!" % model) sys.exit(1)
def main(): parser = optparse.OptionParser(usage="usage: %prog [options]") parser.add_option("-S", "--protocol", dest="protocol", default="http", help="default is http, set to https if required") parser.add_option("-P", "--port", dest="port", default="8080", help="Set Ambari Protocol") parser.add_option("-u", "--username", dest="username", default="admin", help="Ambari Username") parser.add_option("-p", "--password", dest="password", default="admin", help="Ambari Password") parser.add_option("-H", "--host", dest="host", default="localhost", help="Ambari Host") parser.add_option( "-C", "--configfile", dest="configs", default="./configs", help="Config file containing key and truststore information") (options, args) = parser.parse_args() global username global password global port global protocol global host global clustername username = options.username password = options.password port = options.port protocol = options.protocol host = options.host clustername = ambariREST( protocol, host, port, username, password, "api/v1/clusters")["items"][0]["Clusters"]["cluster_name"] installedservices = [ line["ServiceInfo"]["service_name"] for line in ambariREST( protocol, host, port, username, password, "api/v1/clusters/" + clustername + "/services")["items"] ] definitions = loaddefinitions() ambari = ambariProps(protocol, host, port, username, password, clustername) updater = propertiesupdater(definitions) Config = ConfigParser.ConfigParser() Config.read(options.configs) os.popen('PYTHONHTTPSVERIFY=0') global changeprops changeprops = { "KEYSTORELOC": Config.get("Configs", "KeyStoreLocation") + '/server.jks', "KEYSTORERANGER": Config.get("Configs", "KeyStoreLocation") + '/ranger-plugin.jks', "KEYPASS": Config.get("Configs", "KeyStorePassword"), "TRUSTSTORELOC": Config.get("Configs", "TrustStoreLocation") + '/truststore.jks', "TRUSTSTOREPASS": Config.get("Configs", "TrustStorePassword"), "RANGERCOMMONNAME": 'ranger.' + Config.get("Configs", "Domain"), "RANGERURL": replaceurl(ambari.get("admin-properties", "policymgr_external_url"), 6182), "TIMELINEURL": ambari.get("yarn-site", "yarn.timeline-service.webapp.address").split(':')[0] + ':8190', "HISTORYURL": replaceurl( 'http://' + ambari.get("mapred-site", "mapreduce.jobhistory.webapp.address"), 19889), "KMSURL": str( ambari.get("core-site", "hadoop.security.key.provider.path").replace( ':9292', ':9393')).replace('//http@', '//https@'), "ATLASURL": replaceurl(ambari.get("application-properties", "atlas.rest.address"), 21443) } for service in installedservices: if service in definitions.keys(): updater.service(service) else: continue
def user_configuration(self, configFile=None): # get a logger logger = logging.getLogger("configuration") # load and parse the provided configFile, if provided if not configFile: logger.warn( 'no user configuration file provided; using only built-in default settings' ) return # load the config file try: configparser = ConfigParser.ConfigParser() configparser.readfp(open(configFile)) logger.debug( 'successfully read and parsed user configuration file %s' % configFile) except: logger.fatal('error reading user configuration file %s' % configFile) raise #work_dir must be provided before initialising other directories self.work_dir = None if self.work_dir == None: try: self.work_dir = configparser.get('Paths', 'work') except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): if self.work_dir == None: logger.critical('Paths:work has no value!') raise Exception # look for those items that are user-configurable, and get their values # sptk_bindir= .... # a list instead of a dict because OrderedDict is not available until 2.7 # and I don't want to import theano here just for that one class # each entry is a tuple of (variable name, default value, section in config file, option name in config file) # # the type of the default value is important and controls the type that the corresponding # variable will have # # to set a default value of 'undefined' use an empty string # or the special value 'impossible', as appropriate # impossible_int = int(-99999) impossible_float = float(-99999.0) user_options = [ ('work_dir', self.work_dir, 'Paths', 'work'), ('data_dir', '', 'Paths', 'data'), ('plot_dir', '', 'Paths', 'plot'), ('plot', False, 'Utility', 'plot'), ('profile', False, 'Utility', 'profile'), ('file_id_scp', os.path.join(self.work_dir, 'data/file_id_list.scp'), 'Paths', 'file_id_list'), ('test_id_scp', os.path.join(self.work_dir, 'data/test_id_list.scp'), 'Paths', 'test_id_list'), ('GV_dir', os.path.join(self.work_dir, 'data/GV'), 'Paths', 'GV_dir'), ('in_stepw_dir', os.path.join(self.work_dir, 'data/stepw'), 'Paths', 'in_stepw_dir'), ('in_mgc_dir', os.path.join(self.work_dir, 'data/mgc'), 'Paths', 'in_mgc_dir'), ('in_lf0_dir', os.path.join(self.work_dir, 'data/lf0'), 'Paths', 'in_lf0_dir'), ('in_bap_dir', os.path.join(self.work_dir, 'data/bap'), 'Paths', 'in_bap_dir'), ('in_sp_dir', os.path.join(self.work_dir, 'data/sp'), 'Paths', 'in_sp_dir'), ('in_seglf0_dir', os.path.join(self.work_dir, 'data/lf03'), 'Paths', 'in_seglf0_dir'), ## for glottHMM ('in_F0_dir', os.path.join(self.work_dir, 'data/F0'), 'Paths', 'in_F0_dir'), ('in_Gain_dir', os.path.join(self.work_dir, 'data/Gain'), 'Paths', 'in_Gain_dir'), ('in_HNR_dir', os.path.join(self.work_dir, 'data/HNR'), 'Paths', 'in_HNR_dir'), ('in_LSF_dir', os.path.join(self.work_dir, 'data/LSF'), 'Paths', 'in_LSF_dir'), ('in_LSFsource_dir', os.path.join(self.work_dir, 'data/LSFsource'), 'Paths', 'in_LSFsource_dir'), ## for joint duration ('in_seq_dur_dir', os.path.join(self.work_dir, 'data/S2S_dur'), 'Paths', 'in_seq_dur_dir'), ('in_dur_dir', os.path.join(self.work_dir, 'data/dur'), 'Paths', 'in_dur_dir'), ('nn_norm_temp_dir', os.path.join(self.work_dir, 'data/step_hidden9'), 'Paths', 'nn_norm_temp_dir'), ('process_labels_in_work_dir', False, 'Labels', 'process_labels_in_work_dir'), ('label_style', 'HTS', 'Labels', 'label_style'), ('label_type', 'state_align', 'Labels', 'label_type'), ('in_label_align_dir', os.path.join(self.work_dir, 'data/label_state_align'), 'Labels', 'label_align'), ('question_file_name', os.path.join(self.work_dir, 'data/questions.hed'), 'Labels', 'question_file_name'), ('silence_pattern', ['*-#+*'], 'Labels', 'silence_pattern'), ('subphone_feats', 'full', 'Labels', 'subphone_feats'), ('xpath_file_name', os.path.join(self.work_dir, 'data/xml_labels/xpaths.txt'), 'Labels', 'xpath_file_name'), ('label_config_file', 'configuration/examplelabelconfigfile.py', 'Labels', 'label_config'), ('add_frame_features', True, 'Labels', 'add_frame_features'), ('fill_missing_values', False, 'Labels', 'fill_missing_values'), ('xpath_label_align_dir', os.path.join(self.work_dir, 'data/label_state_align'), 'Labels', 'xpath_label_align'), ('enforce_silence', False, 'Labels', 'enforce_silence'), ('remove_silence_using_binary_labels', False, 'Labels', 'remove_silence_using_binary_labels'), ('precompile_xpaths', True, 'Labels', 'precompile_xpaths'), ('iterate_over_frames', True, 'Labels', 'iterate_over_frames'), ('appended_input_dim', 0, 'Labels', 'appended_input_dim'), ('buffer_size', 200000, 'Data', 'buffer_size'), ('train_file_number', impossible_int, 'Data', 'train_file_number'), ('valid_file_number', impossible_int, 'Data', 'valid_file_number'), ('test_file_number', impossible_int, 'Data', 'test_file_number'), ('log_path', os.path.join(self.work_dir, 'log'), 'Paths', 'log_path'), ('log_file', '', 'Paths', 'log_file'), ('log_config_file', 'configuration/exampleloggingconfigfile.conf', 'Paths', 'log_config_file'), ('sptk_bindir', 'tools/bin/SPTK-3.9', 'Paths', 'sptk'), ('straight_bindir', 'tools/bin/straight', 'Paths', 'straight'), ('world_bindir', 'tools/bin/WORLD', 'Paths', 'world'), ('network_type', 'RNN', 'Architecture', 'network_type'), ('model_type', 'DNN', 'Architecture', 'model_type'), ('hidden_layer_type', ['TANH', 'TANH', 'TANH', 'TANH', 'TANH', 'TANH'], 'Architecture', 'hidden_layer_type'), ('output_layer_type', 'LINEAR', 'Architecture', 'output_layer_type'), ('sequential_training', False, 'Architecture', 'sequential_training'), ('dropout_rate', 0.0, 'Architecture', 'dropout_rate'), ## some config variables for token projection DNN ('scheme', 'stagewise', 'Architecture', 'scheme'), ('index_to_project', 0, 'Architecture', 'index_to_project'), ('projection_insize', 10000, 'Architecture', 'projection_insize'), ('projection_outsize', 10, 'Architecture', 'projection_outsize'), ('initial_projection_distrib', 'gaussian', 'Architecture', 'initial_projection_distrib'), ('projection_weights_output_dir', 'some_path', 'Architecture', 'projection_weights_output_dir'), ('layers_with_projection_input', [0], 'Architecture', 'layers_with_projection_input'), ('projection_learning_rate_scaling', 1.0, 'Architecture', 'projection_learning_rate_scaling'), ('learning_rate', 0.0002, 'Architecture', 'learning_rate'), ('l2_reg', 0.00001, 'Architecture', 'L2_regularization'), ('l1_reg', 0.0, 'Architecture', 'L1_regularization'), ('batch_size', 16, 'Architecture', 'batch_size'), ('training_epochs', 25, 'Architecture', 'training_epochs'), ('hidden_activation', 'tanh', 'Architecture', 'hidden_activation'), ('output_activation', 'linear', 'Architecture', 'output_activation'), ('hidden_layer_size', [1024, 1024, 1024, 1024, 1024, 1024], 'Architecture', 'hidden_layer_size'), ('private_hidden_sizes', [1024], 'Architecture', 'private_hidden_sizes'), ('stream_weights', [1.0], 'Architecture', 'stream_weights'), ('private_l2_reg', 0.00001, 'Architecture', 'private_l2_reg'), ('warmup_epoch', 5, 'Architecture', 'warmup_epoch'), ('warmup_momentum', 0.3, 'Architecture', 'warmup_momentum'), ('momentum', 0.9, 'Architecture', 'momentum'), ('warmup_epoch', 5, 'Architecture', 'warmup_epoch'), ('mdn_component', 1, 'Architecture', 'mdn_component'), ('var_floor', 0.01, 'Architecture', 'var_floor'), ('beta_opt', False, 'Architecture', 'beta_opt'), ('eff_sample_size', 0.8, 'Architecture', 'eff_sample_size'), ('mean_log_det', -100.0, 'Architecture', 'mean_log_det'), ('start_from_trained_model', '_', 'Architecture', 'start_from_trained_model'), ('use_rprop', 0, 'Architecture', 'use_rprop'), ('mgc_dim', 60, 'Outputs', 'mgc'), ('dmgc_dim', 60 * 3, 'Outputs', 'dmgc'), ('vuv_dim', 1, 'Outputs', 'vuv'), ('lf0_dim', 1, 'Outputs', 'lf0'), ('dlf0_dim', 1 * 3, 'Outputs', 'dlf0'), ('bap_dim', 25, 'Outputs', 'bap'), ('dbap_dim', 25 * 3, 'Outputs', 'dbap'), ('cmp_dim', (60 * 3) + 1 + (1 * 3) + (25 * 3), 'Outputs', 'cmp'), ('stepw_dim', 55, 'Outputs', 'stepw_dim'), ('temp_sp_dim', 1025, 'Outputs', 'temp_sp_dim'), ('seglf0_dim', 7, 'Outputs', 'seglf0_dim'), ('delta_win', [-0.5, 0.0, 0.5], 'Outputs', 'delta_win'), ('acc_win', [1.0, -2.0, 1.0], 'Outputs', 'acc_win'), ('do_MLPG', True, 'Outputs', 'do_MLPG'), ## for GlottHMM ('F0_dim', 1, 'Outputs', 'F0'), ('dF0_dim', 1 * 3, 'Outputs', 'dF0'), ('Gain_dim', 1, 'Outputs', 'Gain'), ('dGain_dim', 1 * 3, 'Outputs', 'dGain'), ('HNR_dim', 5, 'Outputs', 'HNR'), ('dHNR_dim', 5 * 3, 'Outputs', 'dHNR'), ('LSF_dim', 30, 'Outputs', 'LSF'), ('dLSF_dim', 30 * 3, 'Outputs', 'dLSF'), ('LSFsource_dim', 10, 'Outputs', 'LSFsource'), ('dLSFsource_dim', 10 * 3, 'Outputs', 'dLSFsource'), ## for joint dur:- ('seq_dur_dim', 1, 'Outputs', 'seq_dur'), ('remove_silence_from_dur', True, 'Outputs', 'remove_silence_from_dur'), ('dur_dim', 5, 'Outputs', 'dur'), ('dur_feature_type', 'numerical', 'Outputs', 'dur_feature_type'), ('output_feature_normalisation', 'MVN', 'Outputs', 'output_feature_normalisation'), ('multistream_switch', False, 'Streams', 'multistream_switch'), # ('use_private_hidden' , False, 'Streams', 'use_private_hidden'), ('output_features', ['mgc', 'lf0', 'vuv', 'bap'], 'Streams', 'output_features'), ('gen_wav_features', ['mgc', 'bap', 'lf0'], 'Streams', 'gen_wav_features'), # ('stream_mgc_hidden_size' , 192 , 'Streams', 'stream_mgc_hidden_size'), # ('stream_lf0_hidden_size' , 32 , 'Streams', 'stream_lf0_hidden_size'), # ('stream_vuv_hidden_size' , 32 , 'Streams', 'stream_vuv_hidden_size'), # ('stream_bap_hidden_size' , 128 , 'Streams', 'stream_bap_hidden_size'), # ('stream_stepw_hidden_size' , 64 , 'Streams', 'stream_stepw_hidden_size'), # ('stream_seglf0_hidden_size', 64 , 'Streams', 'stream_seglf0_hidden_size'), # ('stream_cmp_hidden_size' , 256 , 'Streams', 'stream_cmp_hidden_size'), #when multi-stream is disabled, use this to indicate the final hidden layer size #if this is also not provided, use the top common hidden layer size ## Glott HMM -- dummy values -- haven't used private streams:-- # ('stream_F0_hidden_size' , 192 , 'Streams', 'stream_F0_hidden_size'), # ('stream_Gain_hidden_size' , 192 , 'Streams', 'stream_Gain_hidden_size'), # ('stream_HNR_hidden_size' , 192 , 'Streams', 'stream_HNR_hidden_size'), # ('stream_LSF_hidden_size' , 192 , 'Streams', 'stream_LSF_hidden_size'), # ('stream_LSFsource_hidden_size' , 192 , 'Streams', 'stream_LSFsource_hidden_size'), ## joint dur -- dummy values -- haven't used private streams:-- # ('stream_dur_hidden_size' , 192 , 'Streams', 'stream_dur_hidden_size'), # ('stream_sp_hidden_size' , 1024, 'Streams', 'stream_sp_hidden_size'), # ('stream_weight_mgc' , 1.0, 'Streams', 'stream_weight_mgc'), # ('stream_weight_lf0' , 3.0, 'Streams', 'stream_weight_lf0'), # ('stream_weight_vuv' , 1.0, 'Streams', 'stream_weight_vuv'), # ('stream_weight_bap' , 1.0, 'Streams', 'stream_weight_bap'), # ('stream_weight_stepw' , 0.0, 'Streams', 'stream_weight_stepw'), # ('stream_weight_seglf0', 1.0, 'Streams', 'stream_weight_seglf0'), # ('stream_weight_sp' , 1.0, 'Streams', 'stream_weight_sp'), ## Glott HMM - unused? # ('stream_weight_F0' , 1.0, 'Streams', 'stream_weight_F0'), # ('stream_weight_Gain' , 1.0, 'Streams', 'stream_weight_Gain'), # ('stream_weight_HNR' , 1.0, 'Streams', 'stream_weight_HNR'), # ('stream_weight_LSF' , 1.0, 'Streams', 'stream_weight_LSF'), # ('stream_weight_LSFsource' , 1.0, 'Streams', 'stream_weight_LSFsource'), ## dur - unused? # ('stream_weight_dur' , 1.0, 'Streams', 'stream_weight_dur'), # ('stream_lf0_lr' , 0.5, 'Streams', 'stream_lf0_lr'), # ('stream_vuv_lr' , 0.5, 'Streams', 'stream_vuv_lr'), ('vocoder_type', 'STRAIGHT', 'Waveform', 'vocoder_type'), ('sr', 48000, 'Waveform', 'samplerate'), ('fl', 4096, 'Waveform', 'framelength'), ('shift', 1000 * 240 / 48000, 'Waveform', 'frameshift'), ('sp_dim', (4096 / 2) + 1, 'Waveform', 'sp_dim'), # fw_alpha: 'Bark' or 'ERB' allowing deduction of alpha, or explicity float value (e.g. 0.77) ('fw_alpha', 0.77, 'Waveform', 'fw_alpha'), ('pf_coef', 1.4, 'Waveform', 'postfilter_coef'), ('co_coef', 2047, 'Waveform', 'minimum_phase_order'), ('use_cep_ap', True, 'Waveform', 'use_cep_ap'), ('do_post_filtering', True, 'Waveform', 'do_post_filtering'), ('apply_GV', False, 'Waveform', 'apply_GV'), ('test_synth_dir', 'test_synthesis/wav', 'Waveform', 'test_synth_dir'), ('DurationModel', False, 'Processes', 'DurationModel'), ('AcousticModel', False, 'Processes', 'AcousticModel'), ('GenTestList', False, 'Processes', 'GenTestList'), ('NORMLAB', False, 'Processes', 'NORMLAB'), ('MAKEDUR', False, 'Processes', 'MAKEDUR'), ('MAKECMP', False, 'Processes', 'MAKECMP'), ('NORMCMP', False, 'Processes', 'NORMCMP'), ('TRAINDNN', False, 'Processes', 'TRAINDNN'), ('DNNGEN', False, 'Processes', 'DNNGEN'), ('GENWAV', False, 'Processes', 'GENWAV'), ('CALMCD', False, 'Processes', 'CALMCD'), ('NORMSTEP', False, 'Processes', 'NORMSTEP'), ('GENBNFEA', False, 'Processes', 'GENBNFEA'), ('mgc_ext', '.mgc', 'Extensions', 'mgc_ext'), ('bap_ext', '.bap', 'Extensions', 'bap_ext'), ('lf0_ext', '.lf0', 'Extensions', 'lf0_ext'), ('cmp_ext', '.cmp', 'Extensions', 'cmp_ext'), ('lab_ext', '.lab', 'Extensions', 'lab_ext'), ('utt_ext', '.utt', 'Extensions', 'utt_ext'), ('stepw_ext', '.stepw', 'Extensions', 'stepw_ext'), ('sp_ext', '.sp', 'Extensions', 'sp_ext'), ## GlottHMM ('F0_ext', '.F0', 'Extensions', 'F0_ext'), ('Gain_ext', '.Gain', 'Extensions', 'Gain_ext'), ('HNR_ext', '.HNR', 'Extensions', 'HNR_ext'), ('LSF_ext', '.LSF', 'Extensions', 'LSF_ext'), ('LSFsource_ext', '.LSFsource', 'Extensions', 'LSFsource_ext'), ## joint dur ('dur_ext', '.dur', 'Extensions', 'dur_ext'), ] # this uses exec(...) which is potentially dangerous since arbitrary code could be executed for (variable, default, section, option) in user_options: value = None try: # first, look for a user-set value for this variable in the config file value = configparser.get(section, option) user_or_default = 'user' except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): # use default value, if there is one if (default == None) or \ (default == '') or \ ((type(default) == int) and (default == impossible_int)) or \ ((type(default) == float) and (default == impossible_float)) : logger.critical('%20s has no value!' % (section + ":" + option)) raise Exception else: value = default user_or_default = 'default' if type(default) == str: exec('self.%s = "%s"' % (variable, value)) elif type(default) == int: exec('self.%s = int(%s)' % (variable, value)) elif type(default) == float: exec('self.%s = float(%s)' % (variable, value)) elif type(default) == bool: exec('self.%s = bool(%s)' % (variable, value)) elif type(default) == list: exec('self.%s = list(%s)' % (variable, value)) else: logger.critical( 'Variable %s has default value of unsupported type %s', variable, type(default)) raise Exception( 'Internal error in configuration settings: unsupported default type' ) logger.info('%20s has %7s value %s' % (section + ":" + option, user_or_default, value)) self.combined_feature_name = '' for feature_name in self.output_features: self.combined_feature_name += '_' self.combined_feature_name += feature_name self.combined_model_name = self.model_type for hidden_type in self.hidden_layer_type: self.combined_model_name += '_' + hidden_type self.combined_model_name += '_' + self.output_layer_type
def parse(filename): global config configname = filename config = ConfigParser.ConfigParser() config.read([filename, os.path.expanduser('~/.' + filename)])
# @{ ## import time import os from oeqa.runtime.wifi import wifi import string try: import ConfigParser except: import configparser as ConfigParser from oeqa.oetest import oeRuntimeTest from oeqa.utils.helper import shell_cmd_timeout from oeqa.utils.decorators import tag ssid_config = ConfigParser.ConfigParser() config_path = os.path.join(os.path.dirname(__file__), "files/config.ini") ssid_config.readfp(open(config_path)) @tag(TestType="FVT") class CommWiFiConect(oeRuntimeTest): """ @class CommWiFiConect """ def setUp(self): ''' initialize wifi class @fn setUp @param self @return ''' self.wifi = wifi.WiFiFunction(self.target)
def conf(): config = ConfigParser.ConfigParser() config.read('manifest.ini') return config