def do_mqtt_forward(config, section, verbose): # default configuration default = SafeConfigParser({"hostname": "localhost", "port": "1883", "auth": "False", "user": "******", "password": "******", "retain": "False", "qos": "0"}) default.optionxform = str default.read(config) # configuration cfg = SafeConfigParser({"client_id": "mqtt-forward-"+section+'-'+str(os.getpid()), "hostname": default.get("mqtt-forward", "hostname"), "port": default.get("mqtt-forward", "port"), "auth": default.get("mqtt-forward", "auth"), "transform": None, "plain": "False", "retain": default.get("mqtt-forward", "retain"), "qos": default.get("mqtt-forward", "qos"), "user": default.get("mqtt-forward", "user"), "password": default.get("mqtt-forward", "password")}) cfg.optionxform = str cfg.read(config) # publication setup pubsection = cfg.get(section, "pub") if not cfg.has_section(pubsection): transform = pubsection pubtopic = None pubsection = section else: transform = cfg.get(pubsection, "transform") pubtopic = cfg.get(pubsection, "topic") if transform in transforms: transform = transforms[transform] if eval(cfg.get(pubsection, "auth")): pubauth = { "username": cfg.get(pubsection, "user"), "password": cfg.get(pubsection, "password") } else: pubauth = None pub = { "hostname": cfg.get(pubsection, "hostname"), "port": eval(cfg.get(pubsection, "port")), "auth": pubauth, "client_id": cfg.get(pubsection, "client_id"), "topic": pubtopic, "transform": transform, "plain": eval(cfg.get(pubsection, "plain")), "retain": eval(cfg.get(pubsection, "retain")), "qos": eval(cfg.get(pubsection, "qos")), "last": {} } # subscription setup subsection = cfg.get(section, "sub") if not cfg.has_section(subsection): subtopic = subsection subsection = section else: subtopic = cfg.get(subsection, "topic") subplain = eval(cfg.get(subsection, "plain")) sub = mqtt.Client(client_id=cfg.get(subsection, "client_id")+"_sub", userdata=(verbose, pub, subtopic, eval(cfg.get(subsection, "plain")))) sub.on_connect = on_connect sub.on_message = on_message if eval(cfg.get(subsection, "auth")): sub.username_pw_set(cfg.get(subsection, "user"), cfg.get(subsection, "password")) sub.connect(cfg.get(subsection, "hostname"), eval(cfg.get(subsection, "port")), 60) if verbose > 3: print(str(datetime.datetime.utcnow())+" subscribed @"+subtopic+" [" + subsection + "]: Plain = "+str(subplain)) # Loop until done... sub.loop_start() while not done: sleep(1) sub.loop_stop() return 0
def __init__(self, path, default='etc/client_defaults.cfg'): self.path = path # read defaults tmp = SafeConfigParser() tmp.optionxform = str tmp.read(default) self._config_options_dict(tmp) # read file tmp = SafeConfigParser() tmp.optionxform = str tmp.read(path) self._config_options_dict(tmp) self._populate_partitions()
def copy_inputs(config_file, InputsDir): config_dict = read_config(config_file) config = SafeConfigParser() config.optionxform = str config.read(config_file) new_config = os.path.join(InputsDir, os.path.split(config_file)[1]) # ---------------------------------------------------------------- # # copy the inputs for key, section in config_dict.iteritems(): if 'FILE_NAME' in section.keys(): new_file_name = os.path.join(InputsDir, os.path.split(section['FILE_NAME'])[1]) copyfile(section['FILE_NAME'], new_file_name) # update the config file for an easy restart config.set(key, 'FILE_NAME', os.path.join(InputsDir, os.path.split(section['FILE_NAME'])[1])) # update the config_dict with the new value config_dict[key]['FILE_NAME'] = new_file_name # ---------------------------------------------------------------- # # ---------------------------------------------------------------- # # write the new configuration file with open(new_config, 'w') as configfile: config.write(configfile) # ---------------------------------------------------------------- # return config_dict
def dumpDictToConfig(self, newhierdict, configfilename): newparser = SafeConfigParser() newparser.optionxform = str for section_name in sorted(newhierdict.keys()): newparser.add_section(section_name) [newparser.set(section_name, k, v) for k, v in sorted(newhierdict[section_name].items())] newparser.write(open(configfilename, 'w'))
def __init__ (self, filename = "output.cfg"): from numpy import dtype from ConfigParser import SafeConfigParser from ast import literal_eval parser = SafeConfigParser () parser.optionxform = str parser.read ("output.cfg") for section in parser.sections (): for item in parser.items (section): key, string = item try: self[key] = literal_eval (string) except: try: self[key] = parser.getboolean (section, key) except: self[key] = string self["dx"] = self["Lx"]/self["nx"] self["dz"] = self["Lz"]/self["nz"] self["npar"] = self["nx"]*self["nz"]*self["npc"] self["rtype"] = dtype ("f" + str ( self["precision"])) self["ctype"] = dtype ("c" + str (2*self["precision"])) # Allow e.g. cfg["dx"] to be accessed through cfg.dx self.__dict__.update (self)
def prepare_answers(path, topo, config): with NamedTemporaryFile() as temp: warn_if_fail(get(path, temp.name)) temp.flush() parser = SafeConfigParser() parser.optionxform = str parser.read(temp.name) parser.set("general", "CONFIG_PROVISION_DEMO", "y") parser.set("general", "CONFIG_PROVISION_TEMPEST", "y") parser.set("general", "CONFIG_PROVISION_TEMPEST_REPO_REVISION", "master") parser.set("general", "CONFIG_KEYSTONE_ADMIN_PW", "Cisco123") parser.set("general", "CONFIG_KEYSTONE_DEMO_PW", "secret") parser.set("general", "CONFIG_DEBUG_MODE", "y") parser.set("general", "CONFIG_NTP_SERVERS", "10.81.254.202,ntp.esl.cisco.com") parser.set("general", "CONFIG_NOVA_NETWORK_DEFAULTFLOATINGPOOL", "public") if topo == "2role": parser.set("general", "CONFIG_CONTROLLER_HOST", config['servers']['control-server'][0]['ip']) parser.set("general", "CONFIG_COMPUTE_HOSTS", config['servers']['compute-server'][0]['ip']) if topo == "3role": parser.set("general", "CONFIG_CONTROLLER_HOST", config['servers']['build-server'][0]['ip']) parser.set("general", "CONFIG_NETWORK_HOSTS", config['servers']['control-server'][0]['ip']) parser.set("general", "CONFIG_COMPUTE_HOSTS", config['servers']['compute-server'][0]['ip']) with open("installed_answers", "w") as f: parser.write(f) warn_if_fail(put("installed_answers", "~/installed_answers"))
def readMetadata(baseDir): metadata = SafeConfigParser() metadata.optionxform = str file = codecs.open(getMetadataPath(baseDir), 'rb', encoding='utf-8') metadata.readfp(file) file.close() return metadata
def parseConfig(self, cfg, prefix=DEFAULT_OPT_PREFIX, section='DEFAULT'): '''Read config settings from a file, config parser object or dict @type cfg: basestring / ConfigParser derived type / dict @param cfg: configuration file path or ConfigParser type object @type prefix: basestring @param prefix: prefix for option names e.g. "attributeQuery." @type section: basetring @param section: configuration file section from which to extract parameters. ''' if isinstance(cfg, basestring): cfgFilePath = path.expandvars(cfg) # Add a 'here' helper option for setting dir paths in the config # file hereDir = path.abspath(path.dirname(cfgFilePath)) _cfg = SafeConfigParser(defaults={'here': hereDir}) # Make option name reading case sensitive _cfg.optionxform = str _cfg.read(cfgFilePath) items = _cfg.items(section) elif isinstance(cfg, ConfigParser): items = cfg.items(section) elif isinstance(cfg, dict): items = cfg.items() else: raise AttributeError('Expecting basestring, ConfigParser or dict ' 'type for "cfg" attribute; got %r type' % type(cfg)) self.__parseFromItems(items, prefix=prefix)
def get_config(): """ Returns parsed configuration file (SafeConfigParser instance). File paths that will be checked: ~/.sitescripts, ~/sitescripts.ini, /etc/sitescripts, /etc/sitescripts.ini """ paths = [] # Allow SITESCRIPTS_CONFIG variable to override config path if 'SITESCRIPTS_CONFIG' in os.environ: paths.append(os.environ['SITESCRIPTS_CONFIG']) # For debugging - accept configuration in user's profile paths.append(os.path.expanduser('~/.sitescripts')) paths.append(os.path.expanduser('~/sitescripts.ini')) # Server-wide configuration if no custom found paths.append('/etc/sitescripts') paths.append('/etc/sitescripts.ini') for path in paths: path = os.path.abspath(path) if os.path.exists(path): config = SafeConfigParser() config.optionxform = lambda x: x config.read(path) return config raise Exception('No config file found. Please put sitescripts.ini into your home directory or /etc')
def __init__(self, path, default=os.path.join(os.path.dirname(os.path.abspath(__file__)), 'etc/client_defaults.cfg')): self.path = path # read defaults tmp = SafeConfigParser() tmp.optionxform = str tmp.read(default) self._config_options_dict(tmp) # read file tmp = SafeConfigParser() tmp.optionxform = str tmp.read(path) self._config_options_dict(tmp) self._populate_partitions()
def __init__(self, configFile, setEnv=False): """ Purpose: Constructor :param self: class object itself :param configFile: configuration file to read :param setEnv: Whether to export the key/value pairs in the environment :raises ValueError """ parser = SafeConfigParser() # make option names case sensitive parser.optionxform = str # read file contents if configFile: # file was provided fileContents = parser.read(configFile) else: # file was not provided raise Exception("Configuration file is not provided") if not fileContents: # unable to read file contents raise ValueError(self.__class__.__name__ + ": File " + configFile + " could not be read.") # use parser._sections to populate dictionary for item in parser._sections: self.m_dictionary.update(parser.items(item)) # If user has requested to set the environment, # export all the key/value pairs into the environment if setEnv == True: for key in self.m_dictionary: os.environ[key] = self.m_dictionary[key]
def _load_cfg_parser(self, filename='apb.ini'): ''' Load config parser from specific INI filename. ''' defaults = {} try: cfg_parser = SafeConfigParser(defaults, allow_no_value=True) except TypeError: cfg_parser = SafeConfigParser(defaults) cfg_parser.optionxform = str found_filename = None if filename == os.path.basename(filename): base_filename = filename cwd = os.getcwd() while cwd: filename = os.path.join(cwd, base_filename) if os.path.exists(filename): found_filename = filename break if os.path.dirname(cwd) == cwd: break cwd = os.path.dirname(cwd) else: found_filename = filename if found_filename: cfg_parser.readfp(open(found_filename, 'r'), found_filename) return cfg_parser
def parse_cfg_with_sections(stream): """Return as dict of dict of ... """ #Experimental: """ ConfigParser sections become sub-sub sections when separated by dots. [foo.bar] baz = 42 is equivalent to JSON {"foo": {"bar": {"baz": 42}}} """ content = stream.read() result = dict() try: jdict = json.loads(NativeIO(content).read()) return jdict except ValueError: pass #logger.exception('Could not parse stream as JSON.') try: config = ConfigParser() #strict=False? config.optionxform = str config.readfp(NativeIO(content)) sections = config.sections() for sec in sections: result[sec] = dict(config.items(sec)) return result except: raise
def checkVersion(self): #print "in Checkver " + self.versions_cfg missing_products = [] self.allowed_ext.append(self.py_ver_ext[self.python_version]) config = SafeConfigParser() config.optionxform = lambda orig: orig config.read(self.versions_cfg) """ returns a tuple list of key-value pairs """ iversions = config.items('versions') #print iversions iversions_keys = [value[0] for value in iversions] for prod in iversions_keys: print 'checking ' + prod + ' .....' prod_exists = 0 for ext in self.allowed_ext: if prod_exists == 0: prod_url = self.package_index + "/" + prod+"/" + prod + "-" + config.get('versions',prod)+ext print '\tchecking ' + prod_url link_exists = self.checkLink(prod_url) if (link_exists == 1) : #print '\t\tfound ' + prod_url prod_exists = 1 else: pass #print '\t\tnot found ' + prod_url if prod_exists == 0: print 'adding missing ' + prod + ' == ' + config.get('versions',prod) missing_products.append(prod + ' == ' + config.get('versions',prod)) #print 'Missing Products = ' + str(len(missing_products)) return missing_products
def ensure_definition(definition, definition_type): """Load all alerts/reports into DATABASE """ parser = SafeConfigParser() parser.optionxform = str parser.read(definition) if definition_type == "alert": collection = ALERTS_COLLECTION elif definition_type == "report": collection = REPORTS_COLLECTION for section in parser.sections(): params = defaultdict(dict) params["name"] = section for name, value in parser.items(section): value = map(lambda y: y.strip(), value.split(", ")) if name[0] == "~": params[name[1:]] = value[0] else: params[name] = value DATABASE[collection].update({"name": section}, params, upsert=True) LOGR.info("Ensured {0} definition for {1}". format(definition_type, section))
def from_config(cls, cfg, prefix=PREFIX, section='DEFAULT'): '''Create query from config file settings @type cfg: basestring /ConfigParser derived type @param cfg: configuration file path or ConfigParser type object @type prefix: basestring @param prefix: prefix for option names e.g. "attributeQuery." @type section: baestring @param section: configuration file section from which to extract parameters. ''' if isinstance(cfg, basestring): cfg_filepath = os.path.expandvars(cfg) here_dir = os.path.dirname(cfg_filepath) _cfg = SafeConfigParser(defaults=dict(here=here_dir)) _cfg.optionxform = str _cfg.read(cfg_filepath) elif isinstance(cfg, ConfigParser): _cfg = cfg else: raise AttributeError('Expecting basestring or ConfigParser type ' 'for "cfg" attribute; got %r type' % type(cfg)) # Get items for this section as a dictionary so that from_kw can # used to update the object kw = dict(_cfg.items(section)) if 'prefix' not in kw and prefix: kw['prefix'] = prefix return cls.from_kw(**kw)
def load_local_config(config_file, required=[], optional=[]): """Loads config from ~/.cloudconnect and from os.environ, and merges values into current globals()""" from ConfigParser import SafeConfigParser config = SafeConfigParser() config.optionxform = str new_config = {} if os.path.exists(config_file): if "DEBUG_SETTINGS" in os.environ: print "Reading '%s'" % config_file config.read(config_file) env = determine_env() if env in config.sections(): for key in config.options(env): val = config.get(env, key) if key in os.environ: val = os.environ[key] val = cast_env_vars(val) if "DEBUG_SETTINGS" in os.environ: print "Setting %s to %s" % (key, str(val)) new_config[key] = val for key in required: if key in os.environ: new_config[key] = cast_env_vars(os.environ[key]) for key in optional: if key in os.environ: new_config[key] = cast_env_vars(os.environ[key]) return new_config
def create_config(self): """ Save current settings to config.ini """ config = SafeConfigParser() config.optionxform = str config.add_section('Desktop Entry') config.set('Desktop Entry', 'Encoding', 'UTF-8') config.set('Desktop Entry', 'Version', '1.0') config.set('Desktop Entry', 'Type', 'Application') config.set('Desktop Entry', 'Name', 'Bonjob') config.set('Desktop Entry', 'Icon', 'bonjob.png') config.set('Desktop Entry', 'Path', self.get_app_path()) config.set('Desktop Entry', 'Exec', 'python bonjob.py') config.set('Desktop Entry', 'StartupNotify', 'false') config.set('Desktop Entry', 'X-UnityGenerated', 'true') #config.set('Desktop Entry', 'OnlyShowIn', 'Unity') try: conf_path = self.get_abspath('.local/share/applications/bonjob.desktop') config.write(open(conf_path, 'w')) except IOError as e: logger.warning('[-] %s' % e) raise SetupError else: logger.info('[+] Create bonjob.desktop in %s' % conf_path)
def parseConfig(self, cfg, prefix='', section='DEFAULT'): '''Read config file settings :type cfg: basestring /ConfigParser derived type :param cfg: configuration file path or ConfigParser type object :type prefix: basestring :param prefix: prefix for option names e.g. "attributeQuery." :type section: baestring :param section: configuration file section from which to extract parameters. ''' if isinstance(cfg, basestring): cfgFilePath = path.expandvars(cfg) hereDir = path.dirname(cfgFilePath) _cfg = SafeConfigParser(defaults=dict(here=hereDir)) _cfg.optionxform = str _cfg.read(cfgFilePath) elif isinstance(cfg, ConfigParser): _cfg = cfg else: raise AttributeError('Expecting basestring or ConfigParser type ' 'for "cfg" attribute; got %r type' % type(cfg)) # Get items for this section as a dictionary so that parseKeywords can # used to update the object kw = dict(_cfg.items(section)) if 'prefix' not in kw and prefix: kw['prefix'] = prefix self.parseKeywords(**kw)
def getLoginSession(): url = 'https://twitter.com' r = requests.get(url,verify=False) cookies = dict(r.cookies) page = r.text soup = BeautifulSoup(page); input = soup.findAll('input',{'name':'authenticity_token'}); authenticity_token = input[0]['value']; config = SafeConfigParser() config.optionxform = str config.read("tags.cfg") username = config.get('USER','username') password = config.get('USER','password') login_url = 'https://twitter.com/sessions' params = {} params['session[username_or_email]'] = username params['session[password]'] = password params['authenticity_token'] = authenticity_token params['redirect_after_login'] = '******' params['return_to_ssl'] = 'true'; login_req = requests.post(login_url,data=params,cookies=cookies,verify=False,proxies=proxies,headers=headers,allow_redirects=False) login_cookies = login_req.cookies; check_url = 'https://twitter.com/' check_req = requests.get(check_url,headers=headers,proxies=proxies,verify=False,cookies=login_cookies); return login_cookies;
def retweet(tag): login_cookies = getLoginSession() config = SafeConfigParser() config.optionxform = str config.read("tags.cfg") tags = dict(config.items('TAGS')) for val in tags.keys(): tag = tags[val] url='https://twitter.com/search?src=typd&q='+tag r = requests.get(url,verify=False,proxies=proxies,headers=headers,cookies=login_cookies) page = r.text soup = BeautifulSoup(page) input = soup.findAll('input',{'name':'authenticity_token'}); div = soup.findAll('div',{'class':'account-group js-mini-current-user'}) data_user_id = div[0]['data-screen-name'] referer = 'https://twitter.com/'+data_user_id headers['Referer'] = referer; print headers; authenticity_token = input[0]['value'] print authenticity_token params = {} params['authenticity_token'] = authenticity_token tweets = soup.findAll('li',{'data-item-type':'tweet'}) for tweet in tweets: params['id'] = tweet['data-item-id'] retweet_url = 'https://twitter.com/i/tweet/retweet' print params r_retweet = requests.post(retweet_url,data=params,verify=False,cookies=login_cookies,proxies=proxies,headers=headers)
def testDatabaseContent(self): script_output_file = os.path.abspath('script.out') self.gnathub.run(script=Script.db2cfg(), output=script_output_file) parser = SafeConfigParser() parser.optionxform = str parser.read(script_output_file) sections = parser.sections() for s in SECTIONS: self.assertTrue(s in sections, "Missing section: " + s) self.assertTrue(parser.has_option(TEXT_IO[0], EXTERNAL_CALL[0]), "Should be an external call") self.assertEqual(parser.get(TEXT_IO[0], EXTERNAL_CALL[0]), EXTERNAL_CALL[1], 'Wrong message') self.assertTrue(parser.has_option(B_SIMPLE_ADB[0], ENTRY_POINT[0]), "Entry point not found") self.assertTrue(parser.get(B_SIMPLE_ADB[0], ENTRY_POINT[0]).endswith(ENTRY_POINT[1]), 'unexpected value for the entry point') for metric in METRICS_SIMPLE_4_1: self.assertTrue( parser.has_option(SIMPLE_4_1[0], metric), 'missing entry for "%s"' % metric)
def updateMqtt(NeoStat, updateInterval): cfg = SafeConfigParser() cfg.optionxform = str cfg.read(os.path.normpath(os.path.dirname(os.path.realpath(__file__)) + "/neohub.conf")) host = cfg.get("mqtt", "hostname") port = eval(cfg.get("mqtt", "port")) topic = cfg.get("mqtt", "topic") qos = eval(cfg.get("mqtt", "qos")) retain = eval(cfg.get("mqtt", "retain")) client_id = "neohub" + NeoStat.name + str(os.getpid()) if eval(cfg.get("mqtt", "auth")): auth = {"username": cfg.get("mqtt", "user"), "password": cfg.get("mqtt", "password")} else: auth = None while True: try: NeoStat.update() time.sleep(updateInterval) msgs = [{"topic": topic + NeoStat.name, "payload": """{ "temperature" : """ + str(NeoStat.current_temperature) + """, "status" : """ + "\"" + NeoStat.operation + "\"" " " + "}", "qos": qos, "retain": retain}] log.debug("msgs = " + str(msgs)) publish.multiple(msgs, hostname=host, port=port, client_id=client_id, auth=auth) except: log.error("Unable to publish message")
def __init__(self, params, status, calib_data, service_name): """ :param params: pywws configuration. :type params: :class:`pywws.DataStore.params` :param status: pywws status store. :type status: :class:`pywws.DataStore.status` :param calib_data: 'calibrated' data. :type calib_data: :class:`pywws.DataStore.calib_store` :param service_name: name of service to upload to. :type service_name: string """ self.logger = logging.getLogger('pywws.ToService(%s)' % service_name) self.params = params self.status = status self.data = calib_data self.service_name = service_name # 'derived' services such as 'underground_rf' share their # parent's config and templates config_section = self.service_name.split('_')[0] if config_section == self.service_name: self.parent = None else: self.parent = config_section self.old_response = None self.old_ex = None # set default socket timeout, so urlopen calls don't hang forever socket.setdefaulttimeout(30) # open params file service_params = SafeConfigParser() service_params.optionxform = str service_params.readfp(open(os.path.join( os.path.dirname(__file__), 'services', '%s.ini' % (self.service_name)))) # get URL self.server = service_params.get('config', 'url') # get fixed part of upload data self.fixed_data = dict() for name, value in service_params.items('fixed'): if value[0] == '*': value = self.params.get(config_section, value[1:], 'unknown') self.fixed_data[name] = value # create templater self.templater = Template.Template( self.params, self.status, self.data, self.data, None, None, use_locale=False) self.template_file = os.path.join( os.path.dirname(__file__), 'services', '%s_template_%s.txt' % ( config_section, self.params.get('config', 'ws type'))) # get other parameters self.catchup = eval(service_params.get('config', 'catchup')) self.use_get = eval(service_params.get('config', 'use get')) self.expected_result = eval(service_params.get('config', 'result'))
def save(self, parfile=None): """Store the current object for later use.""" parser = SafeConfigParser() parser.optionxform = str parser.add_section('material') parser.set('material', 'title', str(self.title)) parser.set('material', 'RE', str(self.RE)) parser.add_section('parameters') parser.set('parameters', 'B20', str(self.B20)) parser.set('parameters', 'B22', str(self.B22)) parser.set('parameters', 'B40', str(self.B40)) parser.set('parameters', 'B42', str(self.B42)) parser.set('parameters', 'B43', str(self.B43)) parser.set('parameters', 'B44', str(self.B44)) parser.set('parameters', 'B60', str(self.B60)) parser.set('parameters', 'B62', str(self.B62)) parser.set('parameters', 'B63', str(self.B63)) parser.set('parameters', 'B64', str(self.B64)) parser.set('parameters', 'B66', str(self.B66)) parser.set('parameters', 'Hz', str(self.Hz)) parser.set('parameters', 'Hx', str(self.Hx)) if parfile is None: parfile = '%s.cfg' % self.title with open(parfile, 'wb') as cfg: parser.write(cfg)
def get_configuration(): """Loads the xeno configuration object. This method loads a ConfigParser.SafeConfigParser from the configuration path. If the configuration path does not exist, this method returns an empty SafeConfigParser. Returns: An initialized (but possibly empty) ConfigParser.SafeConfigParser. """ # Grab the configuration file path config_file_path = configuration_file_path() # Create a configuration parser configuration = SafeConfigParser() # Make it case-sensitive configuration.optionxform = str # Try to read in any existing configuration try: configuration.read(config_file_path) except Exception as e: print_error( 'Unable to read configuration file ({0}): {1}'.format( config_file_path, str(e) ) ) exit(1) return configuration
def main(): import argparse from operator import itemgetter parser = argparse.ArgumentParser() parser.add_argument("input", nargs='+', metavar="XML") parser.add_argument("-c", "--config", required=True) parser.add_argument("-o", "--output", default="output.csv") args = parser.parse_args() output = tablib.Dataset() config = SafeConfigParser() config.optionxform = lambda value: value config.read([args.config]) for fname in args.input: print "Parsing '%s'..." % fname info = extract_from_pdf(fname, config) values = [] if not output.headers: output.headers = sorted(info.keys()) for k, v in sorted(info.iteritems(), key=itemgetter(0)): values.append(v) output.append(values) print "Writing to '%s'..." % args.output with open(args.output, 'wb') as fp: fp.write(output.csv)
def __init__(self, *file): parser = SafeConfigParser() parser.optionxform = str found = parser.read(file) if not found: raise ValueError('No config file found!') for name in sections: self.__dict__.update(parser.items(name))
def __init__(self, *file): parser = SafeConfigParser() parser.optionxform = str found = parser.read(file) if not found: sys.exit(Fore.RED + "No config file found!" + Style.RESET_ALL) for name in sections: self.__dict__.update(parser.items(name))
def active(): conf_file = "conf/pingc.conf" cp = SafeConfigParser() cp.optionxform = str # Preserves case sensitivity cp.readfp(open(conf_file, "r")) section = "Main" printLine("[D] checking pingc.conf for checkedin", flag) return cp.get(section, "checkedin")
def initialiseModels(argv, update, initMode='training'): # argv[1] = dataPath # argv[2] = modelPath # argv[3] = driverName # update = 'update' or 'new' from SAM.SAM_Core import SAMDriver as Driver dataPath = argv[0] modelPath = argv[1] driverName = argv[2] print argv stringCommand = 'from SAM.SAM_Drivers import ' + driverName + ' as Driver' print stringCommand exec stringCommand mySAMpy = Driver() mode = update trainName = dataPath.split('/')[-1] # participantList is extracted from number of subdirectories of dataPath participantList = [ f for f in listdir(dataPath) if isdir(join(dataPath, f)) ] off = 17 print '-------------------' print 'Training Settings:' print print 'Data Path: '.ljust(off), dataPath print 'Model Path: '.ljust(off), modelPath print 'Participants: '.ljust(off), participantList print 'Model Root Name: '.ljust(off), trainName print 'Training Mode:'.ljust(off), mode print 'Driver:'.ljust(off), driverName print '-------------------' print 'Loading Parameters...' print temporalFlag = False modeConfig = '' found = '' try: parser = SafeConfigParser() found = parser.read(dataPath + "/config.ini") if parser.has_option(trainName, 'update_mode'): modeConfig = parser.get(trainName, 'update_mode') else: modeConfig = 'update' print modeConfig except IOError: pass defaultParamsList = [ 'experiment_number', 'model_type', 'model_num_inducing', 'model_num_iterations', 'model_init_iterations', 'verbose', 'Quser', 'kernelString', 'ratioData', 'update_mode', 'model_mode', 'windowSize' ] mySAMpy.experiment_number = None mySAMpy.model_type = None mySAMpy.kernelString = None mySAMpy.fname = None mySAMpy.ratioData = None if initMode == 'training' and (mode == 'new' or modeConfig == 'new' or 'exp' not in modelPath): print 'Loading training parameters from: \n ', '\t' + dataPath + "/config.ini" try: default = False parser = SafeConfigParser() parser.optionxform = str found = parser.read(dataPath + "/config.ini") # load parameters from config file if parser.has_option(trainName, 'experiment_number'): mySAMpy.experiment_number = int( parser.get(trainName, 'experiment_number')) elif '.pickle' in modelPath: mySAMpy.experiment_number = int( modelPath.split('__')[-2].replace('exp', '')) + 1 else: fail = True print 'No experiment_number found' if parser.has_option(trainName, 'model_type'): mySAMpy.model_type = parser.get(trainName, 'model_type') else: default = True mySAMpy.model_type = 'mrd' if parser.has_option(trainName, 'model_num_inducing'): mySAMpy.model_num_inducing = int( parser.get(trainName, 'model_num_inducing')) else: default = True mySAMpy.model_num_inducing = 30 if parser.has_option(trainName, 'model_num_iterations'): mySAMpy.model_num_iterations = int( parser.get(trainName, 'model_num_iterations')) else: default = True mySAMpy.model_num_iterations = 700 if parser.has_option(trainName, 'model_init_iterations'): mySAMpy.model_init_iterations = int( parser.get(trainName, 'model_init_iterations')) else: default = True mySAMpy.model_init_iterations = 2000 if parser.has_option(trainName, 'verbose'): mySAMpy.verbose = parser.get(trainName, 'verbose') == 'True' else: default = True mySAMpy.verbose = False if parser.has_option(trainName, 'model_mode'): mySAMpy.model_mode = parser.get(trainName, 'model_mode') if mySAMpy.model_mode == 'temporal' and parser.has_option( trainName, 'windowSize'): mySAMpy.windowSize = int( parser.get(trainName, 'windowSize')) else: temporalFlag = True else: default = True mySAMpy.model_mode = 'single' if parser.has_option(trainName, 'Quser'): mySAMpy.Quser = int(parser.get(trainName, 'Quser')) else: default = True mySAMpy.Quser = 2 if parser.has_option(trainName, 'kernelString'): mySAMpy.kernelString = parser.get(trainName, 'kernelString') else: default = True mySAMpy.kernelString = "GPy.kern.RBF(Q, ARD=False) + GPy.kern.Bias(Q) + GPy.kern.White(Q)" if parser.has_option(trainName, 'ratioData'): mySAMpy.ratioData = int(parser.get(trainName, 'ratioData')) else: default = True mySAMpy.ratioData = 50 if default: print 'Default settings applied' mySAMpy.paramsDict = dict() mySAMpy.loadParameters(parser, trainName) except IOError: print 'IO Exception reading ', found pass else: print 'Loading parameters from: \n ', '\t' + modelPath try: parser = SafeConfigParser() parser.optionxform = str found = parser.read(dataPath + "/config.ini") # load parameters from config file if parser.has_option(trainName, 'experiment_number'): mySAMpy.experiment_number = int( parser.get(trainName, 'experiment_number')) else: mySAMpy.experiment_number = int( modelPath.split('__')[-2].replace('exp', '')) modelPickle = pickle.load(open(modelPath + '.pickle', 'rb')) mySAMpy.paramsDict = dict() for j in parser.options(trainName): if j not in defaultParamsList: print j mySAMpy.paramsDict[j] = modelPickle[j] mySAMpy.ratioData = modelPickle['ratioData'] mySAMpy.model_type = modelPickle['model_type'] mySAMpy.model_mode = modelPickle['model_mode'] if mySAMpy.model_mode == 'temporal': mySAMpy.windowSize = modelPickle['windowSize'] mySAMpy.model_type = 'mrd' mySAMpy.model_num_inducing = modelPickle['model_num_inducing'] mySAMpy.model_num_iterations = modelPickle['model_num_iterations'] mySAMpy.model_init_iterations = modelPickle[ 'model_init_iterations'] mySAMpy.verbose = modelPickle['verbose'] mySAMpy.Quser = modelPickle['Quser'] mySAMpy.kernelString = modelPickle['kernelString'] try: mySAMpy.listOfModels = modelPickle['listOfModels'] mySAMpy.classifiers = modelPickle['classifiers'] mySAMpy.classif_thresh = modelPickle['classif_thresh'] except: pass except IOError: print 'IO Exception reading ', found pass if 'exp' in modelPath: fnameProto = '/'.join(modelPath.split('/')[:-1]) + '/' + dataPath.split('/')[-1] + '__' + driverName + \ '__' + mySAMpy.model_type + '__exp' + str(mySAMpy.experiment_number) else: fnameProto = modelPath + dataPath.split('/')[-1] + '__' + driverName + '__' + mySAMpy.model_type + \ '__exp' + str(mySAMpy.experiment_number) print 'Full model name: \n', '\t' + fnameProto print '-------------------' print mySAMpy.save_model = False mySAMpy.economy_save = True mySAMpy.visualise_output = False # test_mode = True mySAMpy.readData(dataPath, participantList) # at this point, all the data that will be eventually used for training is contained in mySAMpy.Y # and mySAMpy.L contains all labels if any (depending on mrd model or bgplvm model) # mySAMpy.L is a list of labels while mySAMpy.Y is a numpy array of data # mySAMpy.Y should have 2 dimensions, length of dimension 0 = number of instances # length of dimension 1 = length of feature vector if mySAMpy.model_mode != 'temporal': # get list of labels mySAMpy.textLabels = list(set(mySAMpy.L)) # convert L from list of strings to array of indices mySAMpy.L = np.asarray( [mySAMpy.textLabels.index(i) for i in mySAMpy.L])[:, None] mySAMpy.textLabels = mySAMpy.textLabels else: mySAMpy.X, mySAMpy.Y = transformTimeSeriesToSeq( mySAMpy.Y1, mySAMpy.windowSize) mySAMpy.L, mySAMpy.tmp = transformTimeSeriesToSeq( mySAMpy.U1, mySAMpy.windowSize) mm = [mySAMpy] # mm.append(mySAMpy) # mm[0] contains root model # this is the only model in the case of a single model # or contains all info for the rest of the models in case of multiple models # if mySAMpy.model_mode == 'single' or mySAMpy.model_mode == 'temporal': mm[0].participantList = ['all'] else: mm[0].participantList = ['root'] + mySAMpy.textLabels for k in range(len(mm[0].participantList)): if mm[0].participantList[k] == 'all': minData = len(mm[k].L) mm[0].fname = fnameProto mm[0].model_type = mySAMpy.model_type Ntr = int(mySAMpy.ratioData * minData / 100) else: if k > 0: mm.append(Driver()) # extract subset of data corresponding to this model inds = [ i for i in range(len(mm[0].Y['L'])) if mm[0].Y['L'][i] == k - 1 ] mm[k].Y = mm[0].Y['Y'][inds] mm[k].L = mm[0].Y['L'][inds] mm[k].Quser = mm[0].Quser mm[k].verbose = mm[0].verbose print 'Object class: ', mm[0].participantList[k] minData = len(inds) mm[k].fname = fnameProto + '__L' + str(k - 1) mm[0].listOfModels.append(mm[k].fname) mm[k].model_type = 'bgplvm' Ntr = int(mySAMpy.ratioData * minData / 100) else: mm[0].listOfModels = [] mm[0].fname = fnameProto mm[0].SAMObject.kernelString = '' minData = len(mm[0].L) Ntr = int(mySAMpy.ratioData * minData / 100) mm[k].modelLabel = mm[0].participantList[k] if mm[0].model_mode != 'temporal': [Yall, Lall, YtestAll, LtestAll] = mm[k].prepareData(mm[k].model_type, Ntr, randSeed=mm[0].experiment_number) mm[k].Yall = Yall mm[k].Lall = Lall mm[k].YtestAll = YtestAll mm[k].LtestAll = LtestAll elif mm[0].model_mode == 'temporal': [Xall, Yall, Lall, XtestAll, YtestAll, LtestAll] = mm[k].prepareData(mm[k].model_type, Ntr, randSeed=mm[0].experiment_number) mm[k].Xall = Xall mm[k].Yall = Yall mm[k].Lall = Lall mm[k].XtestAll = XtestAll mm[k].YtestAll = YtestAll mm[k].LtestAll = LtestAll print 'minData = ' + str(minData) print 'ratioData = ' + str(mySAMpy.ratioData) if initMode == 'training': for k in range(len(mm[0].participantList)): # for k = 0 check if multiple model or not if mm[0].participantList[k] != 'root': print "Training with ", mm[0].model_num_inducing, 'inducing points for ', \ mm[0].model_init_iterations, '|', mm[0].model_num_iterations mm[k].training(mm[0].model_num_inducing, mm[0].model_num_iterations, mm[0].model_init_iterations, mm[k].fname, mm[0].save_model, mm[0].economy_save, keepIfPresent=False, kernelStr=mm[0].kernelString) if mm[0].visualise_output: ax = mm[k].SAMObject.visualise() visualiseInfo = dict() visualiseInfo['ax'] = ax else: visualiseInfo = None else: for k in range(len(mm[0].participantList)): # for k = 0 check if multiple model or not if mm[0].participantList[k] != 'root': print "Training with ", mm[0].model_num_inducing, 'inducing points for ', \ mm[0].model_init_iterations, '|', mm[0].model_num_iterations mm[k].training(mm[0].model_num_inducing, mm[0].model_num_iterations, mm[0].model_init_iterations, mm[k].fname, mm[0].save_model, mm[0].economy_save, keepIfPresent=True, kernelStr=mm[0].kernelString) return mm
def createConfigParser(sourceConfigFile, ignoreCase=True, additional_config_dir=""): """ Creates a config parser instance using the Oncotator conventions for config files. Note: ALL config files must end with .config In order of precedence: basicConfig file (.config) localBasicConfig file (.local.config) The filename convention for a local basic config is as follows: myConfig.config (the basicConfig) myConfig.local.config All sections, keys, and values in the local config supersede the basic config. No local configs should be committed to git. IMPORTANT: If the config file is not found. Attempts to find it again by prepending "configs/" and then checking the additional_config_dir. If still not found, assumes that the file is part of the install package. Precedence: 1) File name relative to current dir (CWD) 2) Filename configs/CWD 3) Additional directory (additional_config_dir) 4) Package resources .local.config can be found in any of these locations. In other words, test.config can be package resources and test.local.config can be in ~/.oncotator, and it will be as if these two files are local to each other. :param sourceConfigFile: :param ignoreCase: :param additional_config_dir: Lowest precedence directory to search. Does NOT look for a config subdirectory here """ # TODO: Must be more graceful way of handling this. # TODO: Add ~/.oncotator to areas that are searched for config files. sourceConfigFP = ConfigUtils._locateConfigFile(sourceConfigFile, additional_config_dir) config = SafeConfigParser() if not ignoreCase: config.optionxform = str config.readfp(sourceConfigFP) # Create the local filename string by grabbing the ending of .config and configBaseFilename = os.path.basename(sourceConfigFP.name) pattern = "\.config$" configLocalFilename = re.sub(pattern, ".local.config", configBaseFilename) dirName = os.path.dirname(sourceConfigFile) if dirName <> "": dirName = dirName + '/' configLocalSourceFilename = dirName + configLocalFilename configLocalSourceFP = ConfigUtils._locateConfigFile( configLocalSourceFilename, isRelaxedLogging=True, additional_config_dir=additional_config_dir) if configLocalSourceFP <> None: config.readfp(configLocalSourceFP) return config
for val in result: print >>f, '%g\t' % val, print >>f if __name__ == '__main__': if(not args.dir is None): print 'size\titers\tH\t\terrRes\t\tincomp' for val in process_dir(args.dir): print '%g\t' % val, print exit(0) cfgfile = args.config cfg = SafeConfigParser(allow_no_value=True) cfg.optionxform = str cfg.read(cfgfile) # --- data dir --- # data dir pattern: # $(data_path)/$(var_with_val}-$(data_path_suffix)/ # ${batchvar_with_val}/ data_path = cfg.get('xscft','dataPath') data_path_suffix = cfg.get('xscft','dataPathSuffix') var_with_val = get_var_with_val(cfg) dir1 = os.path.join(data_path,var_with_val+data_path_suffix) bname = cfg.get('xscft','activeBatchName') if(args.result is None): result_file = os.path.join(dir1, 'result-'+bname+'.txt') else:
def __init__(self, params, status, calib_data, service_name): """ :param params: pywws configuration. :type params: :class:`pywws.DataStore.params` :param status: pywws status store. :type status: :class:`pywws.DataStore.status` :param calib_data: 'calibrated' data. :type calib_data: :class:`pywws.DataStore.calib_store` :param service_name: name of service to upload to. :type service_name: string """ self.logger = logging.getLogger('pywws.ToService(%s)' % service_name) self.params = params self.status = status self.data = calib_data self.service_name = service_name # 'derived' services such as 'underground_rf' share their # parent's config and templates config_section = self.service_name.split('_')[0] if config_section == self.service_name: self.parent = None else: self.parent = config_section self.old_response = None self.old_ex = None # set default socket timeout, so urlopen calls don't hang forever if eval(self.params.get('config', 'asynchronous', 'False')): socket.setdefaulttimeout(60) else: socket.setdefaulttimeout(20) # open params file service_params = SafeConfigParser() service_params.optionxform = str param_string = pkg_resources.resource_string( 'pywws', 'services/%s.ini' % (self.service_name)) if sys.version_info[0] >= 3: param_string = param_string.decode('utf-8') service_params.readfp(StringIO.StringIO(param_string)) # get URL self.server = service_params.get('config', 'url') parsed_url = urlparse.urlsplit(self.server) if parsed_url.scheme == 'aprs': self.send_data = self.aprs_send_data server, port = parsed_url.netloc.split(':') self.server = (server, int(port)) elif parsed_url.scheme == 'mqtt': self.send_data = self.mqtt_send_data else: self.send_data = self.http_send_data self.use_get = eval(service_params.get('config', 'use get')) # get fixed part of upload data self.fixed_data = dict() for name, value in service_params.items('fixed'): if value[0] == '*': value = self.params.get(config_section, value[1:], 'unknown') self.fixed_data[name] = value # create templater self.templater = Template.Template(self.params, self.status, self.data, self.data, None, None, use_locale=False) template_name = self.params.get(config_section, 'template', 'default') if template_name != 'default': template_dir = self.params.get( 'paths', 'templates', os.path.expanduser('~/weather/templates/')) self.template_file = open( os.path.join(template_dir, template_name), 'rb') else: template_name = 'services/%s_template_%s.txt' % ( config_section, self.params.get('config', 'ws type')) if not pkg_resources.resource_exists('pywws', template_name): template_name = 'services/%s_template_1080.txt' % ( config_section) self.template_file = pkg_resources.resource_stream( 'pywws', template_name) # get other parameters self.auth_type = service_params.get('config', 'auth_type') if self.auth_type == 'basic': user = self.params.get(config_section, 'user', 'unknown') password = self.params.get(config_section, 'password', 'unknown') details = user + ':' + password self.auth = 'Basic ' + base64.b64encode( details.encode('utf-8')).decode('utf-8') elif self.auth_type == 'mqtt': self.user = self.params.get(config_section, 'user', 'unknown') self.password = self.params.get(config_section, 'password', 'unknown') self.catchup = eval(service_params.get('config', 'catchup')) self.expected_result = eval(service_params.get('config', 'result')) self.interval = eval(service_params.get('config', 'interval')) self.interval = max(self.interval, 40) self.interval = timedelta(seconds=self.interval) # move 'last update' from params to status last_update = self.params.get_datetime(self.service_name, 'last update') if last_update: self.params.unset(self.service_name, 'last update') self.status.set('last update', self.service_name, last_update.isoformat(' ')) # set timestamp of first data to upload self.next_update = datetime.utcnow() - max( timedelta(days=self.catchup), self.interval)
def target_repos_from_config(*config_filenames): config = SafeConfigParser() config.optionxform = str # package names are case sensitive for filename in config_filenames: print 'reading %s' % filename config.readfp(open(filename, 'r')) repos = {} testing_repos = {} skipped = set() for section in sorted(config.sections()): descr, _, rest = section.partition('.') if not rest: koji_profile = config.get(section, 'source') downgradeable = True if config.has_option(section, 'downgradeable'): downgradeable = config.getboolean(section, 'downgradeable') all_packages = False if config.has_option(section, 'all-packages'): all_packages = config.getboolean(section, 'all-packages') signed = False if config.has_option(section, 'signed'): signed = config.getboolean(section, 'signed') repos[descr] = TargetRepo(name=config.get(section, 'name'), distro=config.get(section, 'distro'), arches=config.get(section, 'arches').split(), tag=config.get(section, 'tag'), downgradeable=downgradeable, all_packages=all_packages, signed=signed, koji_profile=koji_profile) testing_repos[descr] = TargetRepo( name=config.get(section, 'testing-name'), distro=config.get(section, 'distro'), arches=config.get(section, 'arches').split(), tag=config.get(section, 'testing-tag'), downgradeable=downgradeable, all_packages=all_packages, signed=False, # testing repos are always unsigned for now koji_profile=koji_profile) if config.has_option(section, 'skip') and config.getboolean( section, 'skip'): skipped.add(descr) if config.has_option(section, 'excluded-rpms'): for name in config.get(section, 'excluded-rpms').split(): repos[descr].add_excluded_rpm(name) testing_repos[descr].add_excluded_rpm(name) elif rest == 'packages': for package_name, rpm_names in config.items(section): repos[descr].add_package(package_name, rpm_names.split()) testing_repos[descr].add_package(package_name, rpm_names.split()) elif rest.startswith('packages.'): tag, _, source = rest[len('packages.'):].partition('.') for package_name, rpm_names in config.items(section): repos[descr].add_package(package_name, rpm_names.split(), tag=tag, koji_profile=source or None) # appending -testing here is perhaps an unwise hack... # we just need to work towards eliminating these exceptional sections testing_tag = tag + '-testing' if not tag.endswith( '-testing') else tag testing_repos[descr].add_package(package_name, rpm_names.split(), tag=testing_tag, koji_profile=source or None) else: raise ValueError('Unrecognised section: %s' % rest) for descr in skipped: del repos[descr] del testing_repos[descr] return sorted(repos.values() + testing_repos.values())
def WriteConfig(): cfg = SafeConfigParser() cfg.optionxform = lambda option: option section = 'config' cfg.add_section(section) cfg.set(section, "path", autosub.PATH) cfg.set(section, "seriespath", autosub.SERIESPATH) cfg.set(section, "bckpath", autosub.BCKPATH) cfg.set(section, 'downloaddutch', str(autosub.DOWNLOADDUTCH)) cfg.set(section, 'downloadeng', str(autosub.DOWNLOADENG)) cfg.set(section, "subeng", autosub.SUBENG) cfg.set(section, "subnl", autosub.SUBNL) cfg.set(section, "fallbacktoeng", str(autosub.FALLBACKTOENG)) cfg.set(section, "notifyen", str(autosub.NOTIFYEN)) cfg.set(section, "notifynl", str(autosub.NOTIFYNL)) cfg.set(section, "wantedfirst", str(autosub.WANTEDFIRST)) cfg.set(section, "launchbrowser", str(autosub.LAUNCHBROWSER)) cfg.set(section, "skiphiddendirs", str(autosub.SKIPHIDDENDIRS)) cfg.set(section, "englishsubdelete", str(autosub.ENGLISHSUBDELETE)) cfg.set(section, "addic7ed", str(autosub.ADDIC7ED)) cfg.set(section, "opensubtitles", str(autosub.OPENSUBTITLES)) cfg.set(section, "podnapisi", str(autosub.PODNAPISI)) cfg.set(section, "subscene", str(autosub.SUBSCENE)) cfg.set(section, "hearingimpaired", str(autosub.HI)) cfg.set(section, 'minmatchscore', str(autosub.MINMATCHSCORE)) cfg.set(section, "configversion", str(autosub.CONFIGVERSION)) cfg.set(section, 'searchinterval', str(autosub.SEARCHINTERVAL)) cfg.set(section, "browserrefresh", str(autosub.BROWSERREFRESH)) cfg.set(section, "postprocesscmd", autosub.POSTPROCESSCMD) cfg.set(section, "opensubtitlesuser", autosub.OPENSUBTITLESUSER) cfg.set(section, "opensubtitlespasswd", autosub.OPENSUBTITLESPASSWD) cfg.set(section, "addic7eduser", autosub.ADDIC7EDUSER) cfg.set(section, "addic7edpasswd", autosub.ADDIC7EDPASSWD) cfg.set(section, "subcodec", autosub.SUBCODEC) cfg.set(section, "skipstringnl", autosub.SKIPSTRINGNL) cfg.set(section, "skipstringen", autosub.SKIPSTRINGEN) cfg.set(section, "skipfoldersnl", autosub.SKIPFOLDERSNL) cfg.set(section, "skipfoldersen", autosub.SKIPFOLDERSEN) section = 'webserver' cfg.add_section(section) cfg.set(section, "webserverip", str(autosub.WEBSERVERIP)) cfg.set(section, 'webserverport', str(autosub.WEBSERVERPORT)) cfg.set(section, "username", autosub.USERNAME) cfg.set(section, "password", autosub.PASSWORD) cfg.set(section, "webroot", autosub.WEBROOT) section = 'logfile' cfg.add_section(section) cfg.set(section, "logfile", autosub.LOGFILE) cfg.set(section, "loglevel", logging.getLevelName(autosub.LOGLEVEL)) cfg.set(section, "loglevelconsole", logging.getLevelName(autosub.LOGLEVELCONSOLE)) cfg.set(section, "logsize", str(autosub.LOGSIZE)) cfg.set(section, "lognum", str(autosub.LOGNUM)) section = 'notify' cfg.add_section(section) if autosub.NOTIFYMAIL: cfg.set(section, "notifymail", str(autosub.NOTIFYMAIL)) cfg.set(section, "mailsrv", autosub.MAILSRV) cfg.set(section, 'mailfromaddr', autosub.MAILFROMADDR) cfg.set(section, "mailtoaddr", autosub.MAILTOADDR) cfg.set(section, "mailusername", autosub.MAILUSERNAME) cfg.set(section, "mailpassword", autosub.MAILPASSWORD) cfg.set(section, "mailsubject", autosub.MAILSUBJECT) cfg.set(section, "mailencryption", autosub.MAILENCRYPTION) cfg.set(section, "mailauth", autosub.MAILAUTH) if autosub.NOTIFYGROWL: cfg.set(section, "notifygrowl", str(autosub.NOTIFYGROWL)) cfg.set(section, "growlhost", autosub.GROWLHOST) cfg.set(section, "growlport", autosub.GROWLPORT) cfg.set(section, "growlpass", autosub.GROWLPASS) if autosub.NOTIFYNMA: cfg.set(section, "notifynma", str(autosub.NOTIFYNMA)) cfg.set(section, "nmaapi", autosub.NMAAPI) cfg.set(section, "nmapriority", str(autosub.NMAPRIORITY)) if autosub.NOTIFYTWITTER: cfg.set(section, "notifytwitter", str(autosub.NOTIFYTWITTER)) cfg.set(section, "twitterkey", autosub.TWITTERKEY) cfg.set(section, "twittersecret", autosub.TWITTERSECRET) if autosub.NOTIFYPROWL: cfg.set(section, "notifyprowl", str(autosub.NOTIFYPROWL)) cfg.set(section, "prowlapi", autosub.PROWLAPI) cfg.set(section, "prowlpriority", str(autosub.PROWLPRIORITY)) if autosub.NOTIFYTELEGRAM: cfg.set(section, "notifytelegram", str(autosub.NOTIFYTELEGRAM)) cfg.set(section, "telegramapi", autosub.TELEGRAMAPI) cfg.set(section, "telegramid", autosub.TELEGRAMID) if autosub.NOTIFYPUSHALOT: cfg.set(section, "notifypushalot", str(autosub.NOTIFYPUSHALOT)) cfg.set(section, "pushalotapi", autosub.PUSHALOTAPI) if autosub.NOTIFYPUSHBULLET: cfg.set(section, "notifypushbullet", str(autosub.NOTIFYPUSHBULLET)) cfg.set(section, "pushbulletapi", autosub.PUSHBULLETAPI) if autosub.NOTIFYPUSHOVER: cfg.set(section, "notifypushover", str(autosub.NOTIFYPUSHOVER)) cfg.set(section, "pushoverappkey", autosub.PUSHOVERAPPKEY) cfg.set(section, "pushoveruserkey", autosub.PUSHOVERUSERKEY) cfg.set(section, "pushoverpriority", str(autosub.PUSHOVERPRIORITY)) if autosub.NOTIFYBOXCAR2: cfg.set(section, "notifyboxcar2", str(autosub.NOTIFYBOXCAR2)) cfg.set(section, "boxcar2token", autosub.BOXCAR2TOKEN) if autosub.NOTIFYPLEX: cfg.set(section, "notifyplex", str(autosub.NOTIFYPLEX)) cfg.set(section, "plexserverhost", autosub.PLEXSERVERHOST) cfg.set(section, "plexserverport", autosub.PLEXSERVERPORT) cfg.set(section, "plexserverusername", autosub.PLEXSERVERUSERNAME) cfg.set(section, "plexserverpassword", autosub.PLEXSERVERPASSWORD) section = 'skipshow' cfg.add_section(section) for Show in autosub.SKIPSHOW: if re.match("^[0-9 ,.-]+$", autosub.SKIPSHOW[Show]): cfg.set(section, Show.replace(':', '~'), autosub.SKIPSHOW[Show]) section = 'namemapping' cfg.add_section(section) for Name in autosub.USERNAMEMAPPING: cfg.set(section, Name.replace(':', '~'), autosub.USERNAMEMAPPING[Name]) section = 'addic7edmapping' cfg.add_section(section) for Name in autosub.USERADDIC7EDMAPPING: cfg.set(section, Name, autosub.USERADDIC7EDMAPPING[Name]) try: with open(autosub.CONFIGFILE, 'wb') as cfile: cfg.write(cfile) except Exception as error: return error # here we read the config back because the the UPPERCASE variants of the config (for searching) has to be filled ReadConfig() return 'Config has been saved.'
def main(): try: opts, servers = getopt.getopt(sys.argv[1:], "ahd", ["all", "help", "debug"]) except getopt.GetoptError as e: logger.critical(str(e)) usage() sys.exit(2) runall = False debug = False for o, a in opts: if o in ("-a", "--all"): runall = True elif o in ("-d", "--debug"): debug = True elif o in ("-h", "--help"): usage() sys.exit() else: assert False, "unhandled option: %s" % o if debug: logger.setLevel(logging.DEBUG) if servers == [] and runall == False: logger.error("Please specify an APPP server to run, or use '-a' to run all servers! Exiting...") sys.exit(1) cfg = SafeConfigParser() cfg.optionxform = str runservers = {} try: cfg.readfp(open(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'APPP.ini'))) if runall == True: logger.info("Attempting to run all APPP servers defined in APPP.ini") servers = list(cfg.sections()) if servers == []: logger.error("No APPP servers defined in APPP.ini! Exiting...") sys.exit(1) for srv in servers: if not cfg.has_section(srv): logger.error(srv + " is not defined in APPP.ini! Exiting...") continue server = config(cfg, srv) if not server: continue runservers[srv] = server except KeyboardInterrupt: logger.info("Shutting down...") sys.exit(0) except Exception as e: logger.critical(str(e)) sys.exit(1) cfg = SafeConfigParser() cfg.optionxform = str try: logger.info("Read DNS.ini") cfg.readfp(open(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'DNS.ini'))) for item in cfg.items("DNS"): items = item[1].split(',') i = 0 while i < len(items): items[i] = (items[i].strip(), 5) i = i + 1 dns[item[0]] = items except Exception as e: logger.info("Can not read DNS.ini" + str(e)) if len(runservers) == 0: logger.critical("No runnable APPP server found! Exiting...") sys.exit(1) logger.info('Checking APPP certificates...') if not dump_ca(): sys.exit(1) logger.info('APPP certificates OK.') server_threads = {} for server in runservers: server_threads[server] = spawn(name=server, tgt=start_server, args=(runservers[server],)) try: while not flag_exit: if activeCount() == 1: logger.info('No active APPP server running.') break sleep(5) except KeyboardInterrupt: pass logger.info("Shutting down...") sys.exit(0)
def get_run_options(): from ConfigParser import SafeConfigParser configFileName = scriptName + ".ini" if not os.path.isfile(configFileName): configPath = os.path.join(scriptDir, configFileName) else: configPath = os.path.join('.', configFileName) configSectionName = scriptName config = SafeConfigParser() config.optionxform = str config.read(configPath) configOptions = {} if config.has_section(configSectionName): for (k, v) in config.items(configSectionName): if v == "": continue configOptions[k] = v defaults = { 'outputDir': './results', 'bclToFastqMaxCores': 12, 'samtoolsSortMemPerCore': 1000000000, # samtools sort uses about 2x what you tell it to... 'alnMaxCores': 8, # presumably bwa aln will become increasingly inefficient per core, so we don't want to let this go forever... 'isKeepFastq': True, # important to keep these during testing, but not for production } defaults.update(configOptions) parser = get_option_parser(defaults, configFileName) (options, args) = parser.parse_args() if options.isAllHelp: parser = get_option_parser(defaults, configFileName, True) parser.print_help() sys.exit(2) if len(args): # or (len(sys.argv) == 1): parser.print_help() sys.exit(2) # sanitize arguments before writing defaults, check for missing arguments after: # def checkListRepeats(list, itemLabel): if list == None: return if len(set(list)) != len(list): parser.error("Repeated %s entries" % (itemLabel)) if options.bclBasecallsDirList != None: for i, bclDir in enumerate(options.bclBasecallsDirList): options.bclBasecallsDirList[i] = checkDirArg( bclDir, "bcl basecalls directory") # tmp for testing: # checkListRepeats(options.bclBasecallsDirList,"bcl basecalls directory") if (options.bclTilePatternList != None) and \ (len(options.bclBasecallsDirList) != len(options.bclTilePatternList)) : parser.error("Unexpected number of bclTilPattern entries") checkListRepeats(options.sampleNameList, "sample name") options.casavaDir = checkDirArg(options.casavaDir, "casava directory") options.genomeFasta = checkFileArg(options.genomeFasta, "genome fasta file") options.bwaBin = checkFileArg(options.bwaBin, "bwa binary") options.samtoolsBin = checkFileArg(options.samtoolsBin, "samtools binary") if options.isWriteConfig == True: if not config.has_section(configSectionName): config.add_section(configSectionName) for k, v in vars(options).iteritems(): if k == "isWriteConfig": continue if v == None: v = "" config.set(configSectionName, k, str(v)) configfp = open(configFileName, "w") config.write(configfp) configfp.close() sys.exit(0) def noArgOrError(msg): if len(sys.argv) <= 1: parser.print_help() sys.exit(2) else: parser.error(msg) def assertOption(arg, label): if arg == None: noArgOrError("No %s specified" % (label)) def assertList(list, itemLabel): if (list == None) or (len(list) == 0): noArgOrError("List containing %s (s) is empty or missing" % (itemLabel)) else: for item in list: assertOption(item, itemLabel) assertList(options.bclBasecallsDirList, "bcl basecalls directory") assertList(options.sampleNameList, "sample name") assertOption(options.genomeFasta, "genome fasta file") assertOption(options.outputDir, "output directory") assertOption(options.casavaDir, "casava directory") assertOption(options.picardDir, "picard directory") assertOption(options.bwaBin, "bwa binary") assertOption(options.samtoolsBin, "samtools binary") return options
def ReadConfig(configfile): """ Read the config file and set all the variables. """ # Read config file cfg = SafeConfigParser() cfg.optionxform = str try: with codecs.open(autosub.CONFIGFILE, 'r', autosub.SYSENCODING) as f: cfg.readfp(f) except: Message = WriteConfig() return # First we check whether the config has been upgraded if autosub.CONFIGVERSION < version.configversion: upgradeConfig(cfg, autosub.CONFIGVERSION, version.configversion) elif autosub.CONFIGVERSION > version.configversion: print "Config: ERROR! Config version higher then this version of AutoSub supports. Update AutoSub!" os._exit(1) section = 'config' if not cfg.has_section(section): cfg.add_section(section) if cfg.has_option(section, "configversion"): autosub.CONFIGVERSION = cfg.getint("config", "configversion") if cfg.has_option(section, "wantedfirst"): autosub.WANTEDFIRST = cfg.getboolean(section, "wantedfirst") if cfg.has_option(section, 'downloaddutch'): autosub.DOWNLOADDUTCH = cfg.getboolean(section, 'downloaddutch') if cfg.has_option(section, 'downloadeng'): autosub.DOWNLOADENG = cfg.getboolean(section, 'downloadeng') if cfg.has_option(section, "fallbacktoeng"): autosub.FALLBACKTOENG = cfg.getboolean(section, "fallbacktoeng") if cfg.has_option(section, "notifyen"): autosub.NOTIFYEN = cfg.getboolean(section, "notifyen") if cfg.has_option(section, "notifynl"): autosub.NOTIFYNL = cfg.getboolean(section, "notifynl") if cfg.has_option(section, "launchbrowser"): autosub.LAUNCHBROWSER = cfg.getboolean(section, "launchbrowser") if cfg.has_option(section, "skiphiddendirs"): autosub.SKIPHIDDENDIRS = cfg.getboolean(section, "skiphiddendirs") if cfg.has_option(section, "englishsubdelete"): autosub.ENGLISHSUBDELETE = cfg.getboolean(section, "englishsubdelete") if cfg.has_option(section, "podnapisi"): autosub.PODNAPISI = cfg.getboolean(section, "podnapisi") if cfg.has_option(section, "subscene"): autosub.SUBSCENE = cfg.getboolean(section, "subscene") if cfg.has_option(section, "addic7ed"): autosub.ADDIC7ED = cfg.getboolean(section, "addic7ed") if cfg.has_option(section, "hearingimpaired"): autosub.HI = cfg.getboolean(section, "hearingimpaired") if cfg.has_option(section, 'minmatchscore'): autosub.MINMATCHSCORE = cfg.getint(section, 'minmatchscore') if cfg.has_option(section, 'searchinterval'): autosub.SEARCHINTERVAL = cfg.getint(section, 'searchinterval') if cfg.has_option(section, "browserrefresh"): autosub.BROWSERREFRESH = cfg.getint(section, "browserrefresh") if cfg.has_option(section, "subeng"): autosub.SUBENG = cfg.get(section, "subeng") if cfg.has_option(section, "subnl"): autosub.SUBNL = cfg.get(section, "subnl") if cfg.has_option(section, "postprocesscmd"): autosub.POSTPROCESSCMD = cfg.get(section, "postprocesscmd") if cfg.has_option(section, "opensubtitlesuser"): autosub.OPENSUBTITLESUSER = cfg.get(section, "opensubtitlesuser") if cfg.has_option(section, "opensubtitlespasswd"): autosub.OPENSUBTITLESPASSWD = cfg.get(section, "opensubtitlespasswd") if cfg.has_option(section, "addic7eduser"): autosub.ADDIC7EDUSER = cfg.get(section, "addic7eduser") if cfg.has_option(section, "addic7edpasswd"): autosub.ADDIC7EDPASSWD = cfg.get(section, "addic7edpasswd") if cfg.has_option(section, "logfile"): autosub.LOGFILE = cfg.get(section, "logfile") if cfg.has_option(section, "subcoautosub"): autosub.SUBCODEC = cfg.get(section, "subcodec") if cfg.has_option(section, "rootpath"): autosub.SERIESPATH = cfg.get(section, "rootpath") if cfg.has_option(section, "seriespath"): autosub.SERIESPATH = cfg.get(section, "seriespath") if cfg.has_option(section, "skipstringnl"): autosub.SKIPSTRINGNL = cfg.get(section, "skipstringnl") if cfg.has_option(section, "skipstringen"): autosub.SKIPSTRINGEN = cfg.get(section, "skipstringen") if cfg.has_option(section, "skipfoldersnl"): autosub.SKIPFOLDERSNL = cfg.get(section, "skipfoldersnl") if cfg.has_option(section, "skipfolderSen"): autosub.SKIPFOLDERSEN = cfg.get(section, "skipfoldersen") # ******************* # * Logfile Section * # ******************* section = 'logfile' if not cfg.has_section(section): cfg.add_section(section) if cfg.has_option(section, "loglevel"): LogLevel = cfg.get(section, "loglevel").upper() if LogLevel == u'ERROR': autosub.LOGLEVEL = logging.ERROR elif LogLevel == u"WARNING": autosub.LOGLEVEL = logging.WARNING elif LogLevel == u"DEBUG": autosub.LOGLEVEL = logging.DEBUG elif LogLevel == u"INFO": autosub.LOGLEVEL = logging.INFO elif LogLevel == u"CRITICAL": autosub.LOGLEVEL = logging.CRITICAL if cfg.has_option(section, "loglevelconsole"): LogLevel = cfg.get(section, "loglevelconsole").upper() if LogLevel == u'ERROR': autosub.LOGLEVELCONSOLE = logging.ERROR elif LogLevel == u"WARNING": autosub.LOGLEVELCONSOLE = logging.WARNING elif LogLevel == u"DEBUG": autosub.LOGLEVELCONSOLE = logging.DEBUG elif LogLevel == u"INFO": autosub.LOGLEVELCONSOLE = logging.INFO elif LogLevel == u"CRITICAL": autosub.LOGLEVELCONSOLE = logging.CRITICAL if cfg.has_option(section, "logsize"): autosub.LOGSIZE = cfg.getint(section, "logsize") if cfg.has_option(section, "lognum"): autosub.LOGNUM = cfg.getint(section, "lognum") # ****************************** # * Cherrypy Webserver Section * # ****************************** section = 'webserver' if not cfg.has_section(section): cfg.add_section(section) if cfg.has_option(section, 'webserverip'): autosub.WEBSERVERIP = cfg.get(section, 'webserverip') if cfg.has_option(section, 'webserverport'): autosub.WEBSERVERPORT = int(cfg.get(section, 'webserverport')) if cfg.has_option(section, 'webroot'): autosub.WEBROOT = cfg.get(section, 'webroot') if cfg.has_option(section, 'username'): autosub.USERNAME = cfg.get(section, 'username') if cfg.has_option(section, 'password'): autosub.PASSWORD = cfg.get(section, 'password') # ******************** # * SkipShow Section * # ******************** section = 'skipshow' if not cfg.has_section(section): cfg.add_section(section) autosub.SKIPSHOWUPPER = {} autosub.SKIPSHOW = {} SkipShows = dict(cfg.items(section)) #autosub.SKIPSHOW = dict(cfg.items('skipshow')) # The following 5 lines convert the skipshow to uppercase. And also convert the variables to a list # also replace the "~" with ":" neccesary because the config parser sees ":" as a delimiter for show in SkipShows: if re.match("^[0-9 ,.-]+$", SkipShows[show]): autosub.SKIPSHOW[show.replace('~', ':')] = SkipShows[show] autosub.SKIPSHOWUPPER[show.upper().replace('~', ':')] = [ Item.strip() for Item in SkipShows[show].split(',') ] # *********************** # * Namemapping Section * # *********************** section = 'namemapping' if not cfg.has_section(section): cfg.add_section(section) NameMapping = dict(cfg.items(section)) autosub.USERNAMEMAPPING = {} autosub.USERNAMEMAPPINGUPPER = {} for Name in NameMapping: if NameMapping[Name].isdigit(): autosub.USERNAMEMAPPING[Name.replace('~', ':')] = NameMapping[Name] autosub.USERNAMEMAPPINGUPPER[Name.upper().replace('~', ':')] = [ Item.strip() for Item in NameMapping[Name].split(',') ] # ******************************** # * Addic7ed Namemapping Section * # ******************************** section = 'addic7edmapping' if not cfg.has_section(section): cfg.add_section(section) NameMapping = dict(cfg.items(section)) autosub.USERADDIC7EDMAPPING = {} for Name in NameMapping: if Name.isdigit and NameMapping[Name].isdigit(): autosub.USERADDIC7EDMAPPING[Name] = NameMapping[Name] # ****************** # * Notify Section * # ****************** section = 'notify' if not cfg.has_section(section): cfg.add_section(section) if cfg.has_option(section, 'notifymail'): autosub.NOTIFYMAIL = cfg.getboolean(section, 'notifymail') if cfg.has_option(section, 'mailsrv'): autosub.MAILSRV = cfg.get(section, 'mailsrv') if cfg.has_option(section, 'mailfromaddr'): autosub.MAILFROMADDR = cfg.get(section, 'mailfromaddr') if cfg.has_option(section, 'mailtoaddr'): autosub.MAILTOADDR = cfg.get(section, 'mailtoaddr') if cfg.has_option(section, 'mailusername'): autosub.MAILUSERNAME = cfg.get(section, 'mailusername') if cfg.has_option(section, 'mailpassword'): autosub.MAILPASSWORD = cfg.get(section, 'mailpassword') if cfg.has_option(section, 'mailsubject'): autosub.MAILSUBJECT = cfg.get(section, 'mailsubject') if cfg.has_option(section, 'mailencryption'): autosub.MAILENCRYPTION = cfg.get(section, 'mailencryption') if cfg.has_option(section, 'mailauth'): autosub.MAILAUTH = cfg.get(section, 'mailauth') if cfg.has_option(section, 'notifygrowl'): autosub.NOTIFYGROWL = cfg.getboolean(section, 'notifygrowl') if cfg.has_option(section, 'growlhost'): autosub.GROWLHOST = cfg.get(section, 'growlhost') if cfg.has_option(section, 'growlport'): autosub.GROWLPORT = cfg.get(section, 'growlport') if cfg.has_option(section, 'growlpass'): autosub.GROWLPASS = cfg.get(section, 'growlpass') if cfg.has_option(section, 'notifytwitter'): autosub.NOTIFYTWITTER = cfg.getboolean(section, 'notifytwitter') if cfg.has_option(section, 'twitterkey'): autosub.TWITTERKEY = cfg.get(section, 'twitterkey') if cfg.has_option(section, 'twittersecret'): autosub.TWITTERSECRET = cfg.get(section, 'twittersecret') if cfg.has_option(section, 'notifynma'): autosub.NOTIFYNMA = cfg.getboolean(section, 'notifynma') if cfg.has_option(section, 'nmaapi'): autosub.NMAAPI = cfg.get(section, 'nmaapi') if cfg.has_option(section, 'nmapriority'): autosub.NMAPRIORITY = cfg.getint(section, 'nmapriority') if cfg.has_option(section, 'notifyprowl'): autosub.NOTIFYPROWL = cfg.getboolean(section, 'notifyprowl') if cfg.has_option(section, 'prowlapi'): autosub.PROWLAPI = cfg.get(section, 'prowlapi') if cfg.has_option(section, 'prowlpriority'): autosub.PROWLPRIORITY = cfg.getint(section, 'prowlpriority') if cfg.has_option(section, 'notifypushalot'): autosub.NOTIFYPUSHALOT = cfg.getboolean(section, 'notifypushalot') if cfg.has_option(section, 'pushalotapi'): autosub.PUSHALOTAPI = cfg.get(section, 'pushalotapi') if cfg.has_option(section, 'notifypushbullet'): autosub.NOTIFYPUSHBULLET = cfg.getboolean(section, 'notifypushbullet') if cfg.has_option(section, 'pushbulletapi'): autosub.PUSHBULLETAPI = cfg.get(section, 'pushbulletapi') if cfg.has_option(section, 'notifypushover'): autosub.NOTIFYPUSHOVER = cfg.getboolean(section, 'notifypushover') if cfg.has_option(section, 'pushoverappkey'): autosub.PUSHOVERAPPKEY = cfg.get(section, 'pushoverappkey') if cfg.has_option(section, 'pushoveruserkey'): autosub.PUSHOVERUSERKEY = cfg.get(section, 'pushoveruserkey') if cfg.has_option(section, 'notifyboxcar2'): autosub.NOTIFYBOXCAR2 = cfg.getboolean(section, 'notifyboxcar2') if cfg.has_option(section, 'boxcar2token'): autosub.BOXCAR2TOKEN = cfg.get(section, 'boxcar2token') if cfg.has_option(section, 'notifyplex'): autosub.NOTIFYPLEX = cfg.getboolean(section, 'notifyplex') if cfg.has_option(section, 'plexserverhost'): autosub.PLEXSERVERHOST = cfg.get(section, 'plexserverhost') if cfg.has_option(section, 'plexserverport'): autosub.PLEXSERVERPORT = cfg.get(section, 'plexserverport') # Settings autosub.NAMEMAPPING = { "Against the Wall": "1836237", "alcatraz": "1728102", "almost human": "2654580", "alphas": "1183865", "american dad": "0397306", "american horror story": "1844624", "appropriate adult": "1831575", "Are You There Chelsea": "1826989", "atlantis": "2705602", "atlantis 2013": "2705602", "awkward": "1663676", "back in the game": "2655470", "Bates Motel": "2188671", "beauty and the beast": "2193041", "beauty and the beast 2012": "2193041", "betrayal": "2751074", "blue bloods": "1595859", "boardwalk empire": "0979432", "bob's burgers": "1561755", "bobs burgers": "1561755", "Body of Proof": "1587669", "borgen": "1526318", "breakout kings": "1590961", "breaking bad": "903747", "Castle (2009)": "1219024", "castle 2009": "1219024", "charlie's angels 2011": "1760943", "Charlies Angels 2011": "1760943", "chicago fire": "2261391", "chicago fire (2012)": "2261391", "chicago pd": "2805096", "chicago p.d": "2805096", "chicago p.d.": "2805096", "Common Law 2012": "1771072", "continuum": "1954347", "covert affairs": "1495708", "cracked (2013)": "2078576", "criminal minds": "0452046", "csi": "0247082", "csi crime scene investigation": "0247082", "Csi Miami": "0313043", "csi new york": "0395843", "csi ny": "0395843", "Da Vinci's Demons": "2094262", "Dallas 2012": "1723760", "desperate housewives": "0410975", "devious maids": "2226342", "Doctor Who": "0436992", "Doctor Who (2005)": "0436992", "don't trust the b---- in apartment 23": "1819509", "dont trust the bitch in apartment 23": "1819509", "dracula": "2296682", "dracula (2013)": "2296682", "DreamWorks Dragons: Riders of Berk": "2325846", "eastbound & down": "0866442", "eastbound and down": "0866442", "emily owens m d": "2290339", "Falling skies": "1462059", "Fast N Loud": "2346169", "Femme Fatales": "1841108", "Franklin and Bash": "1600199", "Free Agents": "1839481", "Free Agents Us": "1839481", "fringe": "1119644", "game of thrones": "0944947", "Glee": "1327801", "Grey's Anatomy": "0413573", "Greys Anatomy": "0413573", "grimm": "1830617", "harry's law": "1582453", "Harrys Law": "1582453", "haven": "1519931", "Hawaii Five 0": "1600194", "Hawaii Five 0 2010": "1600194", "Hawaii Five-0": "1600194", "hawaii five-0 2010": "1600194", "hello ladies": "2378794", "homeland": "1796960", "hostages": "2647258", "house of cards 2013": "1856010", "how i met your mother": "0460649", "How To Survive The End Of The World": "3377330", "Intelligence us": "2693776", "king": "1804880", "kings of crash": "2623754", "Last Man Standing": "1828327", "Last Man Standing Us": "1828327", "law and order svu": "0203259", "law and order uk": "1166893", "longmire": "1836037", "luck": "1578887", "luther": "1474684", "Man Up": "1828238", "marvel's agents of s h i e l d": "2364582", "marvels agents of s h i e l d": "2364582", "marvel agents of shield": "2364582", "agents of s h i e l d": "2364582", "masters of sex": "2137109", "Melissa And Joey": "1597420", "Merlin": "1199099", "Merlin 2008": "1199099", "Mike and Molly": "1608180", "missing 2012": "1828246", "mockingbird lane": "2130271", "modern family": "1442437", "moonshiners": "1877005", "Mr Sunshine": "1583638", "nashville": "2281375", "nashville 2012": "2281375", "ncis": "0364845", "Ncis Los Angeles": "1378167", "Necessary Roughness": "1657505", "new girl": "1826940", "new tricks": "0362357", "nip tuck": "0361217", "nip-tuck": "0361217", "once upon a time": "1843230", "once upon time": "1843230", "once upon a time 2011": "1843230", "once upon a time in wonderland": "2802008", "oppenheimer (1980)": "0078037", "Parks and Recreation": "1266020", "person of interest": "1839578", "played": "2886812", "pretty little liars": "1578873", "Prime Suspect Us": "1582456", "primeval new world": "2295953", "ray donovan": "2249007", "reign 2013": "2710394", "Revolution": "2070791", "Revolution 2012": "2070791", "Rizzoli And Isles": "1551632", "rookie blue": "1442065", "Scandal": "1837576", "scandal (2012)": "1837576", "Scandal 2012": "1837576", "Scandal US": "1837576", "scott and bailey": "1843678", "sean saves the world": "2715776", "Shameless Us": "1586680", "silent witness": "0115355", "Sinbad": "1979918", "sleepy hollow": "2647544", "snooki and jwoww": "2083701", "sons of anarchy": "1124373", "South Park": "0121955", "Spartacus": "1442449", "Spartacus Blood And Sand": "1442449", "Spartacus Gods Of The Arena": "1758429", "spartacus vengeance": "1442449", "star wars the clone wars": "0458290", "suburgatory": "1741256", "suits": "1632701", "sun, sex and suspicious parents": "1832153", "super fun night": "2298477", "The After": "3145422", "the americans 2013": "2149175", "the americans (2013)": "2149175", "the americans": "2149175", "the big bang theory": "898266", "the biggest loser": "0429318", "the blacklist": "2741602", "the client list": "2022170", "the closer": "0458253", "the dukes of hazzard": "78607", "the gadget show": "0830851", "The Kennedys": "1567215", "the killing (2011)": "1637727", "The La Complex": "1794147", "The Legend Of Korra": "1695360", "the lying game": "1798274", "the mentalist": "1196946", "the newsroom (2012)": "1870479", "the newsroom 2012": "1870479", "the o c": "0362359", "the office us": "0386676", "the originals": "2632424", "the piglet files": "0098895", "the protector": "1836417", "The River": "1836195", "the tomorrow people us": "2660734", "the walking dead": "1520211", "the wire": "306414", "the wrong mans": "2603596", "thundercats 2011": "1666278", "Touch": "1821681", "trophy wife": "2400736", "two and a half men": "0369179", "under the dome": "1553656", "unforgettable": "1842530", "untouchables-the venture bros": "0417373", "Up All Night 2011": "1843323", "utopia": "2384811", "Vegas": "2262383", "white collar": "1358522", "xiii the series 2011": "1713938" } autosub.NAMEMAPPINGUPPER = {} for x in autosub.NAMEMAPPING.keys(): autosub.NAMEMAPPINGUPPER[x.upper()] = autosub.NAMEMAPPING[x] autosub.LASTESTDOWNLOAD = []
def __init__(self, params, calib_data, service_name=None): """ :param params: pywws configuration. :type params: :class:`pywws.DataStore.params` :param calib_data: 'calibrated' data. :type calib_data: :class:`pywws.DataStore.calib_store` :keyword service_name: name of service to upload to. :type service_name: string """ if service_name: self.config_section = service_name self.logger = logging.getLogger('pywws.ToService(%s)' % service_name) else: self.logger = logging.getLogger('pywws.%s' % self.__class__.__name__) self.params = params self.data = calib_data self.old_response = None self.old_ex = None # set default socket timeout, so urlopen calls don't hang forever socket.setdefaulttimeout(30) # open params file service_params = SafeConfigParser() service_params.optionxform = str service_params.readfp( open( os.path.join(os.path.dirname(__file__), 'services', '%s.ini' % (self.config_section)))) # get URL self.server = service_params.get('config', 'url') # get fixed part of upload data self.fixed_data = dict() for name, value in service_params.items('fixed'): if value[0] == '*': value = self.params.get(self.config_section, value[1:], 'unknown') self.fixed_data[name] = value # create templater self.templater = Template.Template(self.params, self.data, self.data, None, None, use_locale=False) self.template_file = os.path.join( os.path.dirname(__file__), 'services', '%s_template_%s.txt' % (service_name, self.params.get('fixed', 'ws type'))) # get other parameters self.catchup = eval(service_params.get('config', 'catchup')) self.use_get = eval(service_params.get('config', 'use get')) rapid_fire = eval(service_params.get('config', 'rapidfire')) if rapid_fire: self.server_rf = service_params.get('config', 'url-rf') self.fixed_data_rf = dict(self.fixed_data) for name, value in service_params.items('fixed-rf'): self.fixed_data_rf[name] = value else: self.server_rf = self.server self.fixed_data_rf = self.fixed_data self.expected_result = eval(service_params.get('config', 'result'))
from ConfigParser import SafeConfigParser from orphics.tools.io import Plotter, dictFromSection, listFromConfig from szlib.szcounts import ClusterCosmology from orphics.tools.stats import bin2D from szlib.sims import BattagliaSims, getKappaSZ from enlib.fft import fft, ifft import os print "Done importing modules..." outDir = os.environ['WWW'] + "plots/kappatest/" # === COSMOLOGY === cosmologyName = 'LACosmology' # from ini file iniFile = "../SZ_filter/input/params.ini" Config = SafeConfigParser() Config.optionxform = str Config.read(iniFile) lmax = 8000 cosmoDict = dictFromSection(Config, cosmologyName) constDict = dictFromSection(Config, 'constants') cc = ClusterCosmology(cosmoDict, constDict, lmax, pickling=True) TCMB = 2.7255e6 # === NFW CLUSTER === massOverh = 2.e14 concentration = 3.2 zL = 0.7 sourceZ = 1100. overdensity = 500. critical = True
def initialiseModels(argv, update, initMode='training'): """Initialise SAM Model data structure, training parameters and user parameters. This method starts by initialising the required Driver from the driver name in argv[3] if it exists in SAM_Drivers folder. The standard model parameters and the specific user parameters are then initialised and the data is read in by the SAMDriver.readData method to complete the model data structure. This method then replicates the model data structure for training with multiple models if it is required in the config file loaded by the Driver. Args: argv_0: dataPath containing the data that is to be trained on. argv_1: modelPath containing the path of where the model is to be stored. argv_2: driverName containing the name of the driver class that is to be loaded from SAM_Drivers folder. update: String having either a value of 'update' or 'new'. 'new' will load the parameters as set in the config file of the driver being loaded present in the dataPath directory. This is used to train a new model from scratch. 'update' will check for an existing model in the modelPath directory and load the parameters from this model if it exists. This is used for retraining a model when new data becomes available. initMode: String having either a value of 'training' or 'interaction'. 'training' takes into consideration the value of update in loading the parameters. (Used by trainSAMModel.py) 'interaction' loads the parameters directly from the model if the model exists. (Used by interactionSAMModel.py) Returns: The output is a list of SAMDriver models. The list is of length 1 when the config file requests a single model or a list of length n+1 for a config file requesting multiple models where n is the number of requested models. The number of models either depends on the number of directories present in the dataPath or from the length of textLabels returned from the SAMDriver.readData method. """ from SAM.SAM_Core import SAMDriver as Driver dataPath = argv[0] modelPath = argv[1] driverName = argv[2] logging.info(argv) stringCommand = 'from SAM.SAM_Drivers import ' + driverName + ' as Driver' logging.info(stringCommand) exec stringCommand mySAMpy = Driver() mode = update trainName = dataPath.split('/')[-1] # participantList is extracted from number of subdirectories of dataPath participantList = [ f for f in listdir(dataPath) if isdir(join(dataPath, f)) ] off = 17 logging.info('-------------------') logging.info('Training Settings:') logging.info('') logging.info('Init mode: '.ljust(off) + str(initMode)) logging.info('Data Path: '.ljust(off) + str(dataPath)) logging.info('Model Path: '.ljust(off) + str(modelPath)) logging.info('Participants: '.ljust(off) + str(participantList)) logging.info('Model Root Name: '.ljust(off) + str(trainName)) logging.info('Training Mode:'.ljust(off) + str(mode)) logging.info('Driver:'.ljust(off) + str(driverName)) logging.info('-------------------') logging.info('Loading Parameters...') logging.info('') modeConfig = '' found = '' try: parser = SafeConfigParser() found = parser.read(dataPath + "/config.ini") if parser.has_option(trainName, 'update_mode'): modeConfig = parser.get(trainName, 'update_mode') else: modeConfig = 'update' logging.info(modeConfig) except IOError: pass defaultParamsList = [ 'experiment_number', 'model_type', 'model_num_inducing', 'model_num_iterations', 'model_init_iterations', 'verbose', 'Quser', 'kernelString', 'ratioData', 'update_mode', 'model_mode', 'temporalModelWindowSize', 'optimiseRecall', 'classificationDict', 'useMaxDistance', 'calibrateUnknown' ] mySAMpy.experiment_number = None mySAMpy.model_type = None mySAMpy.kernelString = None mySAMpy.fname = None mySAMpy.ratioData = None if initMode == 'training' and (mode == 'new' or modeConfig == 'new' or 'exp' not in modelPath): logging.info('Loading training parameters from:' + str(dataPath) + "/config.ini") try: default = False parser = SafeConfigParser() parser.optionxform = str found = parser.read(dataPath + "/config.ini") mySAMpy.experiment_number = 'exp' if parser.has_option(trainName, 'model_type'): mySAMpy.model_type = parser.get(trainName, 'model_type') else: default = True mySAMpy.model_type = 'mrd' if parser.has_option(trainName, 'model_num_inducing'): mySAMpy.model_num_inducing = int( parser.get(trainName, 'model_num_inducing')) else: default = True mySAMpy.model_num_inducing = 30 if parser.has_option(trainName, 'model_num_iterations'): mySAMpy.model_num_iterations = int( parser.get(trainName, 'model_num_iterations')) else: default = True mySAMpy.model_num_iterations = 700 if parser.has_option(trainName, 'model_init_iterations'): mySAMpy.model_init_iterations = int( parser.get(trainName, 'model_init_iterations')) else: default = True mySAMpy.model_init_iterations = 2000 if parser.has_option(trainName, 'verbose'): mySAMpy.verbose = parser.get(trainName, 'verbose') == 'True' else: default = True mySAMpy.verbose = False if parser.has_option(trainName, 'optimiseRecall'): mySAMpy.optimiseRecall = int( parser.get(trainName, 'optimiseRecall')) else: default = True mySAMpy.optimiseRecall = 200 if parser.has_option(trainName, 'useMaxDistance'): mySAMpy.useMaxDistance = parser.get(trainName, 'useMaxDistance') == 'True' else: mySAMpy.useMaxDistance = False if parser.has_option(trainName, 'calibrateUnknown'): mySAMpy.calibrateUnknown = parser.get( trainName, 'calibrateUnknown') == 'True' else: mySAMpy.calibrateUnknown = False if parser.has_option(trainName, 'model_mode'): mySAMpy.model_mode = parser.get(trainName, 'model_mode') if mySAMpy.model_mode == 'temporal' and parser.has_option( trainName, 'temporalModelWindowSize'): mySAMpy.temporalWindowSize = int( parser.get(trainName, 'temporalModelWindowSize')) else: temporalFlag = True else: default = True mySAMpy.model_mode = 'single' if parser.has_option(trainName, 'Quser'): mySAMpy.Quser = int(parser.get(trainName, 'Quser')) else: default = True mySAMpy.Quser = 2 if parser.has_option(trainName, 'kernelString'): mySAMpy.kernelString = parser.get(trainName, 'kernelString') else: default = True mySAMpy.kernelString = "GPy.kern.RBF(Q, ARD=False) + GPy.kern.Bias(Q) + GPy.kern.White(Q)" if parser.has_option(trainName, 'ratioData'): mySAMpy.ratioData = int(parser.get(trainName, 'ratioData')) else: default = True mySAMpy.ratioData = 50 if default: logging.info('Default settings applied') mySAMpy.paramsDict = dict() mySAMpy.loadParameters(parser, trainName) except IOError: logging.warning('IO Exception reading ', found) pass else: logging.info('Loading parameters from: \n \t' + str(modelPath)) try: parser = SafeConfigParser() parser.optionxform = str found = parser.read(dataPath + "/config.ini") # load parameters from config file mySAMpy.experiment_number = modelPath.split('__')[-1] modelPickle = pickle.load(open(modelPath + '.pickle', 'rb')) mySAMpy.paramsDict = dict() for j in parser.options(trainName): if j not in defaultParamsList: logging.info(str(j)) mySAMpy.paramsDict[j] = modelPickle[j] mySAMpy.ratioData = modelPickle['ratioData'] mySAMpy.model_type = modelPickle['model_type'] mySAMpy.model_mode = modelPickle['model_mode'] if mySAMpy.model_mode == 'temporal': mySAMpy.temporalModelWindowSize = modelPickle[ 'temporalModelWindowSize'] mySAMpy.model_type = 'mrd' mySAMpy.model_num_inducing = modelPickle['model_num_inducing'] mySAMpy.model_num_iterations = modelPickle['model_num_iterations'] mySAMpy.model_init_iterations = modelPickle[ 'model_init_iterations'] mySAMpy.verbose = modelPickle['verbose'] mySAMpy.Quser = modelPickle['Quser'] mySAMpy.optimiseRecall = modelPickle['optimiseRecall'] mySAMpy.kernelString = modelPickle['kernelString'] mySAMpy.calibrated = modelPickle['calibrated'] # try loading classification parameters for multiple model implementation try: mySAMpy.useMaxDistance = modelPickle['useMaxDistance'] except: logging.warning( 'Failed to load useMaxDistace. Possible reasons: ' 'Not saved or multiple model implementation') mySAMpy.calibrateUnknown = modelPickle['calibrateUnknown'] if mySAMpy.calibrateUnknown: mySAMpy.classificationDict = modelPickle['classificationDict'] except IOError: logging.warning('IO Exception reading ', found) pass if 'exp' in modelPath or 'best' in modelPath or 'backup' in modelPath: fnameProto = '/'.join(modelPath.split('/')[:-1]) + '/' + dataPath.split('/')[-1] + '__' + driverName + \ '__' + mySAMpy.model_type + '__' + str(mySAMpy.experiment_number) else: fnameProto = modelPath + dataPath.split('/')[-1] + '__' + driverName + '__' + mySAMpy.model_type + \ '__' + str(mySAMpy.experiment_number) logging.info('Full model name: ' + str(fnameProto)) logging.info('-------------------') logging.info('') mySAMpy.save_model = False mySAMpy.economy_save = True mySAMpy.visualise_output = False # test_mode = True mySAMpy.readData(dataPath, participantList) if mySAMpy.model_mode != 'temporal': # get list of labels mySAMpy.textLabels = list(set(mySAMpy.L)) # convert L from list of strings to array of indices mySAMpy.L = np.asarray( [mySAMpy.textLabels.index(i) for i in mySAMpy.L])[:, None] mySAMpy.textLabels = mySAMpy.textLabels else: mySAMpy.X, mySAMpy.Y = transformTimeSeriesToSeq( mySAMpy.Y1, mySAMpy.temporalModelWindowSize) mySAMpy.L, mySAMpy.tmp = transformTimeSeriesToSeq( mySAMpy.U1, mySAMpy.temporalModelWindowSize) mm = [mySAMpy] # mm.append(mySAMpy) # mm[0] contains root model # this is the only model in the case of a single model # or contains all info for the rest of the models in case of multiple models # if mySAMpy.model_mode == 'single' or mySAMpy.model_mode == 'temporal': mm[0].participantList = ['all'] else: mm[0].participantList = ['root'] + mySAMpy.textLabels for k in range(len(mm[0].participantList)): if mm[0].participantList[k] == 'all': normaliseData = True minData = len(mm[k].L) mm[0].fname = fnameProto mm[0].model_type = mySAMpy.model_type Ntr = int(mySAMpy.ratioData * minData / 100) else: if k > 0: mm.append(Driver()) # extract subset of data corresponding to this model inds = [ i for i in range(len(mm[0].Y['L'])) if mm[0].Y['L'][i] == k - 1 ] mm[k].Y = mm[0].Y['Y'][inds] mm[k].L = mm[0].Y['L'][inds] mm[k].Quser = mm[0].Quser mm[k].verbose = mm[0].verbose logging.info('Object class: ' + str(mm[0].participantList[k])) minData = len(inds) mm[k].fname = fnameProto + '__L' + str(k - 1) mm[0].listOfModels.append(mm[k].fname) mm[k].model_type = 'bgplvm' Ntr = int(mySAMpy.ratioData * minData / 100) normaliseData = True else: normaliseData = False mm[0].listOfModels = [] mm[0].fname = fnameProto mm[0].SAMObject.kernelString = '' minData = len(mm[0].L) Ntr = int(mySAMpy.ratioData * minData / 100) mm[k].modelLabel = mm[0].participantList[k] if mm[0].model_mode != 'temporal': [Yall, Lall, YtestAll, LtestAll] = mm[k].prepareData(mm[k].model_type, Ntr, randSeed=0, normalise=normaliseData) mm[k].Yall = Yall mm[k].Lall = Lall mm[k].YtestAll = YtestAll mm[k].LtestAll = LtestAll elif mm[0].model_mode == 'temporal': [Xall, Yall, Lall, XtestAll, YtestAll, LtestAll] = mm[k].prepareData(mm[k].model_type, Ntr, randSeed=0, normalise=normaliseData) mm[k].Xall = Xall mm[k].Yall = Yall mm[k].Lall = Lall mm[k].XtestAll = XtestAll mm[k].YtestAll = YtestAll mm[k].LtestAll = LtestAll logging.info('minData = ' + str(minData)) logging.info('ratioData = ' + str(mySAMpy.ratioData)) logging.info( '-------------------------------------------------------------------------------------------------' ) if initMode == 'training': samOptimiser.deleteModel(modelPath, 'exp') for k in range(len(mm[0].participantList)): # for k = 0 check if multiple model or not if mm[0].participantList[k] != 'root': logging.info("Training with " + str(mm[0].model_num_inducing) + ' inducing points for ' + str(mm[0].model_init_iterations) + '|' + str(mm[0].model_num_iterations)) logging.info("Fname:" + str(mm[k].fname)) mm[k].training(mm[0].model_num_inducing, mm[0].model_num_iterations, mm[0].model_init_iterations, mm[k].fname, mm[0].save_model, mm[0].economy_save, keepIfPresent=False, kernelStr=mm[0].kernelString) if mm[0].visualise_output: ax = mm[k].SAMObject.visualise() visualiseInfo = dict() visualiseInfo['ax'] = ax else: visualiseInfo = None else: for k in range(len(mm[0].participantList)): # for k = 0 check if multiple model or not if mm[0].participantList[k] != 'root': logging.info("Training with " + str(mm[0].model_num_inducing) + ' inducing points for ' + str(mm[0].model_init_iterations) + '|' + str(mm[0].model_num_iterations)) mm[k].training(mm[0].model_num_inducing, mm[0].model_num_iterations, mm[0].model_init_iterations, mm[k].fname, mm[0].save_model, mm[0].economy_save, keepIfPresent=True, kernelStr=mm[0].kernelString) return mm
import getpass as gp import re import sys import os from glob import glob from pprint import pprint from ConfigParser import SafeConfigParser # MPC Imports from mpc.tessa import contexts # Relative Path Config Setup __location__ = os.path.dirname(os.path.realpath(__file__)) __config__ = os.path.join(__location__, "config.ini") try: parser = SafeConfigParser() parser.optionxform = str parser.read(__config__) config = {} for section in parser.sections(): config[section] = {} for sub_item, value in parser.items(section): config[section][ sub_item] = value if ',' not in value else value.split(',') root = config['map']['server'] except KeyError as err: print err raise IOError("File not found %s" % __config__) class SaveData(object): """ Class putting together all the data and interfacing with the UI
def ReadConfig(): """ Read the config file and set all the variables. """ # Read config file cfg = SafeConfigParser() cfg.optionxform = lambda option: option try: with codecs.open(autosub.CONFIGFILE, 'r', autosub.SYSENCODING) as f: cfg.readfp(f) except: #No config found so we create a default config Message = WriteConfig() return # First we check whether the config has been upgraded if autosub.CONFIGVERSION < version.configversion: upgradeConfig(cfg, autosub.CONFIGVERSION, version.configversion) elif autosub.CONFIGVERSION > version.configversion: print "Config: ERROR! Config version higher then this version of AutoSub supports. Update AutoSub!" os._exit(1) section = 'config' if not cfg.has_section(section): cfg.add_section(section) if cfg.has_option(section, "configversion"): autosub.CONFIGVERSION = cfg.getint("config", "configversion") if cfg.has_option(section, "wantedfirst"): autosub.WANTEDFIRST = cfg.getboolean(section, "wantedfirst") if cfg.has_option(section, 'downloaddutch'): autosub.DOWNLOADDUTCH = cfg.getboolean(section, 'downloaddutch') if cfg.has_option(section, 'downloadeng'): autosub.DOWNLOADENG = cfg.getboolean(section, 'downloadeng') if cfg.has_option(section, "fallbacktoeng"): autosub.FALLBACKTOENG = cfg.getboolean(section, "fallbacktoeng") if cfg.has_option(section, "notifyen"): autosub.NOTIFYEN = cfg.getboolean(section, "notifyen") if cfg.has_option(section, "notifynl"): autosub.NOTIFYNL = cfg.getboolean(section, "notifynl") if cfg.has_option(section, "launchbrowser"): autosub.LAUNCHBROWSER = cfg.getboolean(section, "launchbrowser") if cfg.has_option(section, "skiphiddendirs"): autosub.SKIPHIDDENDIRS = cfg.getboolean(section, "skiphiddendirs") if cfg.has_option(section, "englishsubdelete"): autosub.ENGLISHSUBDELETE = cfg.getboolean(section, "englishsubdelete") if cfg.has_option(section, "podnapisi"): autosub.PODNAPISI = cfg.getboolean(section, "podnapisi") if cfg.has_option(section, "subscene"): autosub.SUBSCENE = cfg.getboolean(section, "subscene") if cfg.has_option(section, "addic7ed"): autosub.ADDIC7ED = cfg.getboolean(section, "addic7ed") if cfg.has_option(section, "opensubtitles"): autosub.OPENSUBTITLES = cfg.getboolean(section, "opensubtitles") if cfg.has_option(section, "hearingimpaired"): autosub.HI = cfg.getboolean(section, "hearingimpaired") if cfg.has_option(section, 'minmatchscore'): autosub.MINMATCHSCORE = cfg.getint(section, 'minmatchscore') if cfg.has_option(section, 'searchinterval'): autosub.SEARCHINTERVAL = cfg.getint(section, 'searchinterval') if cfg.has_option(section, "browserrefresh"): autosub.BROWSERREFRESH = cfg.getint(section, "browserrefresh") if cfg.has_option(section, "path"): autosub.PATH = os.path.normpath(cfg.get(section, "path")) if cfg.has_option(section, "rootpath"): autosub.SERIESPATH = os.path.normpath(cfg.get(section, "rootpath")) if cfg.has_option(section, "seriespath"): autosub.SERIESPATH = os.path.normpath(cfg.get(section, "seriespath")) if autosub.SERIESPATH == '.': autosub.SERIESPATH = u'' if cfg.has_option(section, "bckpath"): autosub.BCKPATH = os.path.normpath(cfg.get(section, "bckpath")) if autosub.BCKPATH == '.': autosub.BCKPATH = u'' if cfg.has_option(section, "subeng"): autosub.SUBENG = cfg.get(section, "subeng") if cfg.has_option(section, "subnl"): autosub.SUBNL = cfg.get(section, "subnl") if cfg.has_option(section, "postprocesscmd"): autosub.POSTPROCESSCMD = cfg.get(section, "postprocesscmd") if cfg.has_option(section, "opensubtitlesuser"): autosub.OPENSUBTITLESUSER = cfg.get(section, "opensubtitlesuser") if cfg.has_option(section, "opensubtitlespasswd"): autosub.OPENSUBTITLESPASSWD = cfg.get(section, "opensubtitlespasswd") if cfg.has_option(section, "addic7eduser"): autosub.ADDIC7EDUSER = cfg.get(section, "addic7eduser") if cfg.has_option(section, "addic7edpasswd"): autosub.ADDIC7EDPASSWD = cfg.get(section, "addic7edpasswd") if cfg.has_option(section, "logfile"): autosub.LOGFILE = cfg.get(section, "logfile") if cfg.has_option(section, "subcodec"): autosub.SUBCODEC = cfg.get(section, "subcodec") if cfg.has_option(section, "skipstringnl"): autosub.SKIPSTRINGNL = cfg.get(section, "skipstringnl") if cfg.has_option(section, "skipstringen"): autosub.SKIPSTRINGEN = cfg.get(section, "skipstringen") if cfg.has_option(section, "skipfoldersnl"): autosub.SKIPFOLDERSNL = cfg.get(section, "skipfoldersnl") if cfg.has_option(section, "skipfoldersen"): autosub.SKIPFOLDERSEN = cfg.get(section, "skipfoldersen") # ******************* # * Logfile Section * # ******************* section = 'logfile' if not cfg.has_section(section): cfg.add_section(section) if cfg.has_option(section, "logfile"): autosub.LOGFILE = cfg.get(section, "logfile") if cfg.has_option(section, "loglevel"): LogLevel = cfg.get(section, "loglevel").upper() if LogLevel == u'ERROR': autosub.LOGLEVEL = logging.ERROR elif LogLevel == u"WARNING": autosub.LOGLEVEL = logging.WARNING elif LogLevel == u"DEBUG": autosub.LOGLEVEL = logging.DEBUG elif LogLevel == u"INFO": autosub.LOGLEVEL = logging.INFO elif LogLevel == u"CRITICAL": autosub.LOGLEVEL = logging.CRITICAL if cfg.has_option(section, "loglevelconsole"): LogLevel = cfg.get(section, "loglevelconsole").upper() if LogLevel == u'ERROR': autosub.LOGLEVELCONSOLE = logging.ERROR elif LogLevel == u"WARNING": autosub.LOGLEVELCONSOLE = logging.WARNING elif LogLevel == u"DEBUG": autosub.LOGLEVELCONSOLE = logging.DEBUG elif LogLevel == u"INFO": autosub.LOGLEVELCONSOLE = logging.INFO elif LogLevel == u"CRITICAL": autosub.LOGLEVELCONSOLE = logging.CRITICAL if cfg.has_option(section, "logsize"): autosub.LOGSIZE = cfg.getint(section, "logsize") if cfg.has_option(section, "lognum"): autosub.LOGNUM = cfg.getint(section, "lognum") # ****************************** # * Cherrypy Webserver Section * # ****************************** section = 'webserver' if not cfg.has_section(section): cfg.add_section(section) if cfg.has_option(section, 'webserverip'): autosub.WEBSERVERIP = str(cfg.get(section, 'webserverip')) if cfg.has_option(section, 'webserverport'): autosub.WEBSERVERPORT = int(cfg.get(section, 'webserverport')) if cfg.has_option(section, 'webroot'): autosub.WEBROOT = str(cfg.get(section, 'webroot')) if cfg.has_option(section, 'username'): autosub.USERNAME = str(cfg.get(section, 'username')) if cfg.has_option(section, 'password'): autosub.PASSWORD = str(cfg.get(section, 'password')) # ******************** # * SkipShow Section * # ******************** section = 'skipshow' if not cfg.has_section(section): cfg.add_section(section) autosub.SKIPSHOWUPPER = {} autosub.SKIPSHOW = {} SkipShows = dict(cfg.items(section)) #autosub.SKIPSHOW = dict(cfg.items('skipshow')) # The following 5 lines convert the skipshow to uppercase. And also convert the variables to a list # also replace the "~" with ":" neccesary because the config parser sees ":" as a delimiter # The UPPER version is for searching, the normal for dispaly in the user interface for show in SkipShows: if re.match("^[0-9 ,.-]+$", SkipShows[show]): autosub.SKIPSHOW[show.replace('~', ':')] = SkipShows[show] autosub.SKIPSHOWUPPER[show.upper().replace('~', ':')] = [ Item.strip() for Item in SkipShows[show].split(',') ] # ******************************** # * Addic7ed Namemapping Section * # ******************************** section = 'addic7edmapping' if not cfg.has_section(section): cfg.add_section(section) autosub.USERADDIC7EDMAPPING = {} try: autosub.USERADDIC7EDMAPPING = dict(cfg.items(section)) except: pass for ImdbId in autosub.USERADDIC7EDMAPPING.iterkeys(): if not (ImdbId.isdigit and autosub.USERADDIC7EDMAPPING[ImdbId].isdigit()): del autosub.USERADDIC7EDMAPPING[ImdbId] print 'ReadConfig: Addic7ed mapping has an unkown format.', ImdbId, ' = ', autosub.USERADDIC7EDMAPPING[ ImdbId] # Settings # **************************** # * User Namemapping Section * # **************************** section = 'namemapping' if not cfg.has_section(section): cfg.add_section(section) NameMapping = dict(cfg.items(section)) autosub.USERNAMEMAPPING = {} for ConfigName in NameMapping.iterkeys(): if NameMapping[ConfigName].isdigit(): Name = ConfigName.replace('~', ':') if not Name.upper() in autosub.NAMEMAPPING.keys(): autosub.NAMEMAPPING[Name.upper()] = [ NameMapping[ConfigName].strip(), u'' ] autosub.USERNAMEMAPPING[Name] = NameMapping[ConfigName].strip() else: print 'ReadConfig: Username mapping has an unknown format.', ConfigName, ' = ', NameMapping[ ConfigName] # ****************** # * Notify Section * # ****************** section = 'notify' if not cfg.has_section(section): cfg.add_section(section) if cfg.has_option(section, 'notifymail'): autosub.NOTIFYMAIL = cfg.getboolean(section, 'notifymail') if cfg.has_option(section, 'mailsrv'): autosub.MAILSRV = cfg.get(section, 'mailsrv') if cfg.has_option(section, 'mailfromaddr'): autosub.MAILFROMADDR = cfg.get(section, 'mailfromaddr') if cfg.has_option(section, 'mailtoaddr'): autosub.MAILTOADDR = cfg.get(section, 'mailtoaddr') if cfg.has_option(section, 'mailusername'): autosub.MAILUSERNAME = cfg.get(section, 'mailusername') if cfg.has_option(section, 'mailpassword'): autosub.MAILPASSWORD = cfg.get(section, 'mailpassword') if cfg.has_option(section, 'mailsubject'): autosub.MAILSUBJECT = cfg.get(section, 'mailsubject') if cfg.has_option(section, 'mailencryption'): autosub.MAILENCRYPTION = cfg.get(section, 'mailencryption') if cfg.has_option(section, 'mailauth'): autosub.MAILAUTH = cfg.get(section, 'mailauth') if cfg.has_option(section, 'notifygrowl'): autosub.NOTIFYGROWL = cfg.getboolean(section, 'notifygrowl') if cfg.has_option(section, 'growlhost'): autosub.GROWLHOST = cfg.get(section, 'growlhost') if cfg.has_option(section, 'growlport'): autosub.GROWLPORT = cfg.get(section, 'growlport') if cfg.has_option(section, 'growlpass'): autosub.GROWLPASS = cfg.get(section, 'growlpass') if cfg.has_option(section, 'notifytwitter'): autosub.NOTIFYTWITTER = cfg.getboolean(section, 'notifytwitter') if cfg.has_option(section, 'twitterkey'): autosub.TWITTERKEY = cfg.get(section, 'twitterkey') if cfg.has_option(section, 'twittersecret'): autosub.TWITTERSECRET = cfg.get(section, 'twittersecret') if cfg.has_option(section, 'notifynma'): autosub.NOTIFYNMA = cfg.getboolean(section, 'notifynma') if cfg.has_option(section, 'nmaapi'): autosub.NMAAPI = cfg.get(section, 'nmaapi') if cfg.has_option(section, 'nmapriority'): autosub.NMAPRIORITY = cfg.getint(section, 'nmapriority') if cfg.has_option(section, 'notifyprowl'): autosub.NOTIFYPROWL = cfg.getboolean(section, 'notifyprowl') if cfg.has_option(section, 'prowlapi'): autosub.PROWLAPI = cfg.get(section, 'prowlapi') if cfg.has_option(section, 'prowlpriority'): autosub.PROWLPRIORITY = cfg.getint(section, 'prowlpriority') if cfg.has_option(section, 'notifytelegram'): autosub.NOTIFYTELEGRAM = cfg.getboolean(section, 'notifytelegram') if cfg.has_option(section, 'telegramapi'): autosub.TELEGRAMAPI = cfg.get(section, 'telegramapi') if cfg.has_option(section, 'telegramid'): autosub.TELEGRAMID = cfg.get(section, 'telegramid') if cfg.has_option(section, 'notifypushalot'): autosub.NOTIFYPUSHALOT = cfg.getboolean(section, 'notifypushalot') if cfg.has_option(section, 'pushalotapi'): autosub.PUSHALOTAPI = cfg.get(section, 'pushalotapi') if cfg.has_option(section, 'notifypushbullet'): autosub.NOTIFYPUSHBULLET = cfg.getboolean(section, 'notifypushbullet') if cfg.has_option(section, 'pushbulletapi'): autosub.PUSHBULLETAPI = cfg.get(section, 'pushbulletapi') if cfg.has_option(section, 'notifypushover'): autosub.NOTIFYPUSHOVER = cfg.getboolean(section, 'notifypushover') if cfg.has_option(section, 'pushoverappkey'): autosub.PUSHOVERAPPKEY = cfg.get(section, 'pushoverappkey') if cfg.has_option(section, 'pushoveruserkey'): autosub.PUSHOVERUSERKEY = cfg.get(section, 'pushoveruserkey') if cfg.has_option(section, 'pushoverpriority'): autosub.PUSHOVERPRIORITY = cfg.getint(section, 'pushoverpriority') if cfg.has_option(section, 'notifyboxcar2'): autosub.NOTIFYBOXCAR2 = cfg.getboolean(section, 'notifyboxcar2') if cfg.has_option(section, 'boxcar2token'): autosub.BOXCAR2TOKEN = cfg.get(section, 'boxcar2token') if cfg.has_option(section, 'notifyplex'): autosub.NOTIFYPLEX = cfg.getboolean(section, 'notifyplex') if cfg.has_option(section, 'plexserverhost'): autosub.PLEXSERVERHOST = cfg.get(section, 'plexserverhost') if cfg.has_option(section, 'plexserverport'): autosub.PLEXSERVERPORT = cfg.get(section, 'plexserverport') if cfg.has_option(section, 'plexserverusername'): autosub.PLEXSERVERUSERNAME = cfg.get(section, 'plexserverusername') if cfg.has_option(section, 'plexserverpassword'): autosub.PLEXSERVERPASSWORD = cfg.get(section, 'plexserverpassword')
def testDatabaseContent(self): script_output_file = os.path.abspath('script.out') self.gnathub.run(script=Script.db2cfg(), output=script_output_file) parser = SafeConfigParser() parser.optionxform = str parser.read(script_output_file) self.assertListEqual(sorted(parser.sections()), SECTIONS) self.assertTrue(parser.has_option(F_ADB, 'warnings'), 'missing "warnings" entry') self.assertTrue(parser.has_option(F_ADB, 'identifier_casing'), 'missing "identifier_casing" entry') self.assertTrue( parser.has_option(P_ADB, 'unconstrained_array_returns'), 'missing "unconstrained_array_returns" entry') self.assertTrue( parser.has_option(P_ADS_1, 'unconstrained_array_returns'), 'missing "unconstrained_array_returns" entry') self.assertTrue(parser.has_option(PG_ADB_1, 'improper_returns'), 'missing "improper_returns" entry') self.assertTrue(parser.has_option(PG_ADB_2, 'improper_returns'), 'missing "improper_returns" entry') self.assertTrue( parser.has_option(PG_ADB_3, 'unconstrained_array_returns'), 'missing "unconstrained_array_returns" entry') self.assertTrue(parser.has_option(PG_ADB_4, 'warnings'), 'missing "warnings" entry') self.assertTrue( parser.has_option(PG_ADB_5, 'unconstrained_array_returns'), 'missing "unconstrained_array_returns" entry') self.assertTrue(parser.has_option(PG_ADB_6, 'warnings'), 'missing "warnings" entry') self.assertTrue( parser.has_option(PG_ADS_1, 'unconstrained_array_returns'), 'missing "unconstrained_array_returns" entry') self.assertTrue( parser.has_option(PG_ADS_2, 'unconstrained_array_returns'), 'missing "unconstrained_array_returns" entry') self.assertTrue( parser.has_option(PG_ADS_3, 'unconstrained_array_returns'), 'missing "unconstrained_array_returns" entry') self.assertTrue( parser.has_option(PG_ADS_4, 'unconstrained_array_returns'), 'missing "unconstrained_array_returns" entry') self.assertTrue(parser.has_option(PG_ADS_5, 'recursive_subprograms'), 'missing "recursive_subprograms" entry') self.assertTrue(parser.has_option(PG_ADS_6, 'recursive_subprograms'), 'missing "recursive_subprograms" entry') self.assertTrue( parser.has_option(PGG_ADS_1, 'unconstrained_array_returns'), 'missing "unconstrained_array_returns" entry') self.assertTrue(parser.has_option(PGG_ADS_2, 'warnings'), 'missing "warnings" entry') self.assertTrue(parser.has_option(SIMPLE_ADB_1, 'improper_returns'), 'missing "improper_returns" entry') self.assertTrue(parser.has_option(SIMPLE_ADB_3, 'identifier_suffixes'), 'missing "identifier_suffixes" entry') self.assertTrue(parser.has_option(SIMPLE_ADB_3, 'identifier_prefixes'), 'missing "identifier_prefixes" entry')
def test_modelWithParams(modelName, driverName, datacollectionOnly, randomRecall, probRecall, bufferLen, recency, transmissionDelay): # constant variables automaticOpen = True connectDirect = True frameLen = 15 yrf = yarp.ResourceFinder() yrf.setVerbose(True) yrf.setDefaultContext("samSupervisor") yrf.setDefaultConfigFile("default.ini") yrf.configure([]) rootPath = yrf.find("root_path").asString() name = rootPath + '/Models/' + modelName + '__' + driverName + '*.pickle' fname = glob.glob(name) assert len(fname) > 0, 'model file not found' modelFileName = fname[0] interactionConfFile = yrf.find("config_path").asString() interactionConfFile = yrf.findFile(interactionConfFile) # copyfile(interactionConfFile, interactionConfFile+'.bkp') parser = SafeConfigParser() parser.optionxform = str parser.read(interactionConfFile) assert modelName in parser.sections(), 'model name not in parser sections' items = parser.items(modelName) for j in items: if j[0] == 'dataIn': dataInPort = j[1].split(' ')[0] elif j[0] == 'rpcBase': rpcInPort = j[1] elif j[0] == 'callSign': askCommand = [k for k in j[1].split(',') if 'label' in k][0] elif j[0] == 'collectionMethod': collectionMethod = j[1].split(' ')[0] if collectionMethod == 'continuous': parser.set(modelName, 'collectionMethod', 'continuous ' + str(bufferLen)) elif j[0] == 'recency' and collectionMethod == 'continuous': parser.set(modelName, 'recency', str(recency)) parser.write(open(interactionConfFile, 'wb')) def checkRecall(rpcPort, askCommand): # check correct response cmBottle = yarp.Bottle() rpBottle = yarp.Bottle() cmBottle.addString(askCommand) rpcPort.write(cmBottle, rpBottle) results.append(rpBottle.toString()) return rpBottle networkFound = yarp.Network.checkNetwork() assert networkFound, 'Yarpserver not found' processesList = [] # open data port dataPortName = '/modelTest/data:o' dataPort = yarp.BufferedPortBottle() dataPort.open(dataPortName) # open query port rpcPortName = '/modelTest/rpc' rpcPort = yarp.RpcClient() rpcPort.open(rpcPortName) # open interactionSAMModel if connectDirect: if automaticOpen: args = ' '.join([ rootPath + '/Data/' + modelName, modelFileName.replace('.pickle', ''), interactionConfFile, driverName, 'False' ]) pyFile = 'interactionSAMModel.py' interactionCMD = ' '.join([pyFile, args]) print interactionCMD windowedCMD = "bash -c \"" + interactionCMD + "\"" interactionProcess = subprocess.Popen(['xterm', '-e', windowedCMD], shell=False) # wait until model loaded time.sleep(7) else: # check interactionSAMModel present pass yarp.Network.connect(rpcPortName, rpcInPort + ':i') else: yarp.Network.connect(rpcPortName, '/sam/rpc:i') # send load Actions3 # connect data to /sam/actions/actionData:i yarp.Network.connect(dataPortName, dataInPort) # yarp.Network.connect(dataPortName, '/reader') # check length of data log file dataFile = open(join('./noisyActionData/recordedData', 'data.log'), 'r') for i, l in enumerate(dataFile): pass lenDataFile = i + 1 print lenDataFile dataFile.close() # send each line with a pause of 0.1s and query model every frameLen data points. # check correct response and continued operation os samSupervisor processBreak = False dataFile = open(join('./noisyActionData/recordedData', 'data.log'), 'r') results = [] for curr in range(lenDataFile): line = dataFile.readline() lineParts = line.split(' ') line = ' '.join(lineParts[2:]) dataBottle = dataPort.prepare() dataBottle.fromString(line) dataPort.write() time.sleep(transmissionDelay) if randomRecall and random.random( ) < probRecall and not datacollectionOnly: rpBottle = checkRecall(rpcPort, askCommand) pollResult = interactionProcess.poll() print "{0:.2f}".format( curr * 100.0 / lenDataFile ) + '%\t', 'Request Response:', rpBottle.toString(), pollResult if curr % frameLen == 0: if not datacollectionOnly and not randomRecall: rpBottle = checkRecall(rpcPort, askCommand) # check reply and status of process pollResult = interactionProcess.poll() print "{0:.2f}".format( curr * 100.0 / lenDataFile ) + '%\t', 'Request Response:', rpBottle.toString(), pollResult else: pollResult = interactionProcess.poll() print "{0:.2f}".format( curr * 100.0 / lenDataFile) + '%\t', pollResult if pollResult is not None: processBreak = True break dataFile.close() try: print 'Terminating process' interactionProcess.send_signal(signal.SIGINT) except: pass retCode = interactionProcess.wait() if processBreak: testResult = False else: testResult = True return testResult
force=False) else: # one db per GFF file base_name = os.path.splitext(gff_file)[0] db_file = base_name + normalize_filext(args['-e']) try: db = gff.create_db(gff_file, db_file, merge_strategy=unique_keys, verbose=debug, pragmas=pragmas, force=False) except sql.OperationalError: raise IOError( "Database file '{0}' already exists.".format(db_file)) except sql.IntegrityError as err: remove_file(db_file) raise IOError( "{0} in database '{1}'.".format(err, db_file)) else: # rdf mode rdf_format = args['-o'] cfg_file = args['-c'] if os.path.exists(cfg_file) is False: raise IOError("Config file '{0}' not found.".format(cfg_file)) SafeConfigParser.validate = validate # add new method cfg = SafeConfigParser() cfg.optionxform = str # option names case sensitive cfg.read(cfg_file) cfg.validate() gff.FeatureDB.triplify = triplify # add new method for db_file in args['DB_FILE']: db = gff.FeatureDB(db_file) db.triplify(rdf_format, cfg)
def get_config(self): """ Gets the current configuration as a SafeConfigParser from BrandServiceProxy. """ config = SafeConfigParser() config.optionxform = str config.readfp(StringIO(self.get_config_str())) return config
def ReadConfig(): """ Read the config file and set all the variables. """ # Read config file cfg = SafeConfigParser() cfg.optionxform = lambda option: option try: with CodecsOpen(autosub.CONFIGFILE, 'r', autosub.SYSENCODING) as f: cfg.readfp(f) except Exception as error: if error.errno == 2: print "No config found so a default config.properties is created." Message = WriteConfig() return else: print error.message sys.exit(1) section = 'config' if not cfg.has_section(section): print "No config found so a default config.properties is created." Message = WriteConfig() return if cfg.has_option(section, "configversion"): autosub.CONFIGVERSION = cfg.getint("config", "configversion") if cfg.has_option(section, "wantedfirst"): autosub.WANTEDFIRST = cfg.getboolean(section, "wantedfirst") if cfg.has_option(section, 'downloaddutch'): autosub.DOWNLOADDUTCH = cfg.getboolean(section, 'downloaddutch') if cfg.has_option(section, 'downloadeng'): autosub.DOWNLOADENG = cfg.getboolean(section, 'downloadeng') if cfg.has_option(section, "fallbacktoeng"): autosub.FALLBACKTOENG = cfg.getboolean(section, "fallbacktoeng") if cfg.has_option(section, "notifyen"): autosub.NOTIFYEN = cfg.getboolean(section, "notifyen") if cfg.has_option(section, "notifynl"): autosub.NOTIFYNL = cfg.getboolean(section, "notifynl") if cfg.has_option(section, "launchbrowser"): autosub.LAUNCHBROWSER = cfg.getboolean(section, "launchbrowser") if cfg.has_option(section, "skiphiddendirs"): autosub.SKIPHIDDENDIRS = cfg.getboolean(section, "skiphiddendirs") if cfg.has_option(section, "englishsubdelete"): autosub.ENGLISHSUBDELETE = cfg.getboolean(section, "englishsubdelete") if cfg.has_option(section, "podnapisi"): autosub.PODNAPISI = cfg.getboolean(section, "podnapisi") if cfg.has_option(section, "subscene"): autosub.SUBSCENE = cfg.getboolean(section, "subscene") if cfg.has_option(section, "addic7ed"): autosub.ADDIC7ED = cfg.getboolean(section, "addic7ed") if cfg.has_option(section, "opensubtitles"): autosub.OPENSUBTITLES = cfg.getboolean(section, "opensubtitles") if cfg.has_option(section, "hearingimpaired"): autosub.HI = cfg.getboolean(section, "hearingimpaired") if cfg.has_option(section, "equalmatch"): autosub.EQUALMATCH = cfg.getboolean(section, "equalmatch") if cfg.has_option(section, 'minmatchscore'): autosub.MINMATCHSCORE = cfg.getint(section, 'minmatchscore') if cfg.has_option(section, 'searchinterval'): autosub.SEARCHINTERVAL = cfg.getint(section, 'searchinterval') if cfg.has_option(section, "browserrefresh"): autosub.BROWSERREFRESH = cfg.getint(section, "browserrefresh") if cfg.has_option(section, "path"): autosub.PATH = os.path.normpath(cfg.get(section, "path")) if cfg.has_option(section, "rootpath"): autosub.SERIESPATH = os.path.normpath(cfg.get(section, "rootpath")) if cfg.has_option(section, "seriespath"): autosub.SERIESPATH = os.path.normpath(cfg.get(section, "seriespath")) if autosub.SERIESPATH == '.': autosub.SERIESPATH = u'' if cfg.has_option(section, "bckpath"): autosub.BCKPATH = os.path.normpath(cfg.get(section, "bckpath")) if autosub.BCKPATH =='.': autosub.BCKPATH = u'' if cfg.has_option(section, "subeng"): autosub.SUBENG = cfg.get(section, "subeng") if cfg.has_option(section, "subnl"): autosub.SUBNL = cfg.get(section, "subnl") if cfg.has_option(section, "postprocesscmd"): autosub.POSTPROCESSCMD = cfg.get(section, "postprocesscmd") if cfg.has_option(section, "opensubtitlesuser"): autosub.OPENSUBTITLESUSER = cfg.get(section, "opensubtitlesuser") if cfg.has_option(section, "opensubtitlespasswd"): autosub.OPENSUBTITLESPASSWD = cfg.get(section, "opensubtitlespasswd") if cfg.has_option(section, "addic7eduser"): autosub.ADDIC7EDUSER = cfg.get(section, "addic7eduser") if cfg.has_option(section, "addic7edpasswd"): autosub.ADDIC7EDPASSWD = cfg.get(section, "addic7edpasswd") if cfg.has_option(section, "tvdbuser"): autosub.TVDBUSER = cfg.get(section, "tvdbuser") if cfg.has_option(section, "tvdbaccountid"): autosub.TVDBACCOUNTID = cfg.get(section, "tvdbaccountid") if cfg.has_option(section, "subcodec"): autosub.SUBCODEC = cfg.get(section, "subcodec") if cfg.has_option(section,'subpermissions'): Temp = cfg.get(section,"subpermissions") autosub.SUBRIGHTS['owner'] = int(Temp[1:2]) autosub.SUBRIGHTS['group'] = int(Temp[2:3]) autosub.SUBRIGHTS['world'] = int(Temp[3:4]) if cfg.has_option(section, "skipstringnl"): autosub.SKIPSTRINGNL = cfg.get(section, "skipstringnl") if cfg.has_option(section, "skipstringen"): autosub.SKIPSTRINGEN = cfg.get(section, "skipstringen") if cfg.has_option(section, "skipfoldersnl"): autosub.SKIPFOLDERSNL = cfg.get(section, "skipfoldersnl") if cfg.has_option(section, "skipfoldersen"): autosub.SKIPFOLDERSEN = cfg.get(section, "skipfoldersen") # ******************* # * Logfile Section * # ******************* section = 'logfile' if not cfg.has_section(section): cfg.add_section(section) autosub.LOGFILE = cfg.get(section, "logfile") if cfg.has_option(section, "logfile") else os.path.join(autosub.CONFIGPATH,autosub.LOGNAME) if cfg.has_option(section, "logfile"): autosub.LOGFILE = cfg.get(section, "logfile") if cfg.has_option(section, "loglevel"): autosub.LOGLEVEL = cfg.getint(section, "loglevel") if cfg.has_option(section, "loglevelconsole"): autosub.LOGLEVELCONSOLE = cfg.getint(section, "loglevelconsole") if cfg.has_option(section, "logsize"): autosub.LOGSIZE = cfg.getint(section, "logsize") if cfg.has_option(section, "lognum"): autosub.LOGNUM = cfg.getint(section, "lognum") if cfg.has_option(section, "logsize"): autosub.LOGSIZE = cfg.getint(section, "logsize") if cfg.has_option(section, "lognum"): autosub.LOGNUM = cfg.getint(section, "lognum") # ****************************** # * Cherrypy Webserver Section * # ****************************** section = 'webserver' if not cfg.has_section(section): cfg.add_section(section) if cfg.has_option(section, 'webserverip'): autosub.WEBSERVERIP = str(cfg.get(section, 'webserverip')) if cfg.has_option(section, 'webserverport'): autosub.WEBSERVERPORT = str(cfg.get(section, 'webserverport')) if cfg.has_option(section, 'webroot'): autosub.WEBROOT = str(cfg.get(section, 'webroot')) if cfg.has_option(section, 'username'): autosub.USERNAME = str(cfg.get(section, 'username')) if cfg.has_option(section, 'password'): autosub.PASSWORD = str(cfg.get(section, 'password')) # ******************** # * rlsgrp Section * # ******************** section = 'releasegroup' if not cfg.has_section(section): cfg.add_section(section) if cfg.has_option(section, 'mustmatch'): autosub.MUSTMATCH = cfg.get(section, 'mustmatch').lower().split(',') # ******************** # * SkipShow Section * # ******************** section = 'skipshow' if not cfg.has_section(section): cfg.add_section(section) autosub.SKIPSHOWUPPER = {} autosub.SKIPSHOW = {} SkipShows = dict(cfg.items(section)) #autosub.SKIPSHOW = dict(cfg.items('skipshow')) # The following 5 lines convert the skipshow to uppercase. And also convert the variables to a list # also replace the "~" with ":" neccesary because the config parser sees ":" as a delimiter # The UPPER version is for searching, the normal for dispaly in the user interface for show in SkipShows: if re.match("^[0-9 ,.-]+$", SkipShows[show]): autosub.SKIPSHOW[show.replace('~',':')] = SkipShows[show] autosub.SKIPSHOWUPPER[show.upper().replace('~',':')] = [Item.strip() for Item in SkipShows[show].split(',')] # ******************************** # * Addic7ed Namemapping Section * # ******************************** section = 'addic7edmapping' if not cfg.has_section(section): cfg.add_section(section) autosub.USERADDIC7EDMAPPING.clear() try: A7Mapping = dict(cfg.items(section)) except: pass for ImdbId in A7Mapping.iterkeys(): if (ImdbId.isdigit and A7Mapping[ImdbId].isdigit()): autosub.USERADDIC7EDMAPPING[ImdbId] = int(A7Mapping[ImdbId]) # **************************** # * User Namemapping Section * # **************************** section = 'namemapping' if not cfg.has_section(section): cfg.add_section(section) NameMapping = dict(cfg.items(section)) autosub.USERNAMEMAPPING={} for ConfigName in NameMapping.iterkeys(): if NameMapping[ConfigName].isdigit(): Name = ConfigName.replace('~',':') if not Name.upper() in autosub.NAMEMAPPING.keys(): autosub.NAMEMAPPING[Name.upper()] = [NameMapping[ConfigName].strip(),u''] autosub.USERNAMEMAPPING[Name] = NameMapping[ConfigName].strip() else: print 'ReadConfig: Username mapping has an unknown format.',ConfigName,' = ',NameMapping[ConfigName] # ****************** # * Notify Section * # ****************** section = 'notify' if not cfg.has_section(section): cfg.add_section(section) if cfg.has_option(section, 'notifymail'): autosub.NOTIFYMAIL = cfg.getboolean(section, 'notifymail') if cfg.has_option(section, 'mailsrv'): autosub.MAILSRV = cfg.get(section, 'mailsrv') if cfg.has_option(section, 'mailfromaddr'): autosub.MAILFROMADDR = cfg.get(section, 'mailfromaddr') if cfg.has_option(section, 'mailtoaddr'): autosub.MAILTOADDR = cfg.get(section, 'mailtoaddr') if cfg.has_option(section, 'mailusername'): autosub.MAILUSERNAME = cfg.get(section, 'mailusername') if cfg.has_option(section, 'mailpassword'): autosub.MAILPASSWORD = cfg.get(section, 'mailpassword') if cfg.has_option(section, 'mailsubject'): autosub.MAILSUBJECT = cfg.get(section, 'mailsubject') if cfg.has_option(section, 'mailencryption'): autosub.MAILENCRYPTION = cfg.get(section, 'mailencryption') if cfg.has_option(section, 'mailauth'): autosub.MAILAUTH = cfg.get(section, 'mailauth') if cfg.has_option(section, 'notifygrowl'): autosub.NOTIFYGROWL = cfg.getboolean(section, 'notifygrowl') if cfg.has_option(section, 'growlhost'): autosub.GROWLHOST = cfg.get(section, 'growlhost') if cfg.has_option(section, 'growlport'): autosub.GROWLPORT = cfg.get(section, 'growlport') if cfg.has_option(section, 'growlpass'): autosub.GROWLPASS = cfg.get(section, 'growlpass') if cfg.has_option(section, 'notifytwitter'): autosub.NOTIFYTWITTER = cfg.getboolean(section, 'notifytwitter') if cfg.has_option(section, 'twitterkey'): autosub.TWITTERKEY = cfg.get(section, 'twitterkey') if cfg.has_option(section, 'twittersecret'): autosub.TWITTERSECRET = cfg.get(section, 'twittersecret') if cfg.has_option(section, 'notifynma'): autosub.NOTIFYNMA = cfg.getboolean(section, 'notifynma') if cfg.has_option(section, 'nmaapi'): autosub.NMAAPI = cfg.get(section, 'nmaapi') if cfg.has_option(section, 'nmapriority'): autosub.NMAPRIORITY = cfg.getint(section, 'nmapriority') if cfg.has_option(section, 'notifyprowl'): autosub.NOTIFYPROWL = cfg.getboolean(section, 'notifyprowl') if cfg.has_option(section, 'prowlapi'): autosub.PROWLAPI = cfg.get(section, 'prowlapi') if cfg.has_option(section, 'prowlpriority'): autosub.PROWLPRIORITY = cfg.getint(section, 'prowlpriority') if cfg.has_option(section, 'notifytelegram'): autosub.NOTIFYTELEGRAM = cfg.getboolean(section, 'notifytelegram') if cfg.has_option(section, 'telegramapi'): autosub.TELEGRAMAPI = cfg.get(section, 'telegramapi') if cfg.has_option(section, 'telegramid'): autosub.TELEGRAMID = cfg.get(section, 'telegramid') if cfg.has_option(section, 'notifypushalot'): autosub.NOTIFYPUSHALOT = cfg.getboolean(section, 'notifypushalot') if cfg.has_option(section, 'pushalotapi'): autosub.PUSHALOTAPI = cfg.get(section, 'pushalotapi') if cfg.has_option(section, 'notifypushbullet'): autosub.NOTIFYPUSHBULLET = cfg.getboolean(section, 'notifypushbullet') if cfg.has_option(section, 'pushbulletapi'): autosub.PUSHBULLETAPI = cfg.get(section, 'pushbulletapi') if cfg.has_option(section, 'notifypushover'): autosub.NOTIFYPUSHOVER = cfg.getboolean(section, 'notifypushover') if cfg.has_option(section, 'pushoverappkey'): autosub.PUSHOVERAPPKEY = cfg.get(section, 'pushoverappkey') if cfg.has_option(section, 'pushoveruserkey'): autosub.PUSHOVERUSERKEY = cfg.get(section, 'pushoveruserkey') if cfg.has_option(section, 'pushoverpriority'): autosub.PUSHOVERPRIORITY = cfg.getint(section, 'pushoverpriority') if cfg.has_option(section, 'notifyboxcar2'): autosub.NOTIFYBOXCAR2 = cfg.getboolean(section, 'notifyboxcar2') if cfg.has_option(section, 'boxcar2token'): autosub.BOXCAR2TOKEN = cfg.get(section, 'boxcar2token') if cfg.has_option(section, 'notifyplex'): autosub.NOTIFYPLEX = cfg.getboolean(section, 'notifyplex') if cfg.has_option(section, 'plexserverhost'): autosub.PLEXSERVERHOST = cfg.get(section, 'plexserverhost') if cfg.has_option(section, 'plexserverport'): autosub.PLEXSERVERPORT = cfg.get(section, 'plexserverport') if cfg.has_option(section, 'plexserverusername'): autosub.PLEXSERVERUSERNAME = cfg.get(section, 'plexserverusername') if cfg.has_option(section, 'plexserverpassword'): autosub.PLEXSERVERPASSWORD = cfg.get(section, 'plexserverpassword') if cfg.has_option(section, 'notifykodi'): autosub.NOTIFYKODI = cfg.getboolean(section, 'notifykodi') if cfg.has_option(section, 'kodiserverhost'): autosub.KODISERVERHOST = cfg.get(section, 'kodiserverhost') if cfg.has_option(section, 'kodiserverport'): autosub.KODISERVERPORT = cfg.get(section, 'kodiserverport') if cfg.has_option(section, 'kodiserverusername'): autosub.KODISERVERUSERNAME = cfg.get(section, 'kodiserverusername') if cfg.has_option(section, 'kodiserverpassword'): autosub.KODISERVERPASSWORD = cfg.get(section, 'kodiserverpassword') if cfg.has_option(section, 'kodiupdateonce'): autosub.KODIUPDATEONCE = cfg.getboolean(section, 'kodiupdateonce') autosub.MINMATCHDSP = '{0:05b}'.format(autosub.MINMATCHSCORE).replace('1','X').replace('0','-') # Here we check whether the config has been upgraded # if so we make the changes and write the config back to the config file autosub.CONFIGVERSION = cfg.getint("config", "configversion") if cfg.has_option("config", "configversion") else 0 if autosub.CONFIGVERSION < autosub.version.configversion: _upgradeConf(cfg, autosub.CONFIGVERSION, autosub.version.configversion) elif autosub.CONFIGVERSION > autosub.version.configversion: print "Config: ERROR! Config version higher then this version of AutoSub supports. Update AutoSub!" os._exit(1)
def parseConfigFile(filename, preserve_case=True): parser = SafeConfigParser() if preserve_case: parser.optionxform = str parser.read(filename) return parser
def _call_parser(conf_file): parser = SafeConfigParser() parser.optionxform = str # to preserve case parser.read(conf_file) return parser
def from_file(cls, cfgfile): cfg = SafeConfigParser(allow_no_value=True) cfg.optionxform = str cfg.read(cfgfile) return cls(cfg)
from subprocess import check_call from ConfigParser import SafeConfigParser from time import sleep ap = ArgumentParser() ap.add_argument("--configFile", action="store", required=True, help='Configuration file containing arguments') ap.add_argument("--numJobs", action="store", type=int, required=True, help='Number of jobs into which run is split') ap.add_argument("--jobNum", action="store", type=int, required=True, help='Number of this job') args = ap.parse_args() cp = SafeConfigParser() cp.optionxform = str cp.read(args.configFile) fMin = cp.getfloat('param-space','f_min') fMax = cp.getfloat('param-space','f_max') fFullBand = fMax-fMin fBand = fFullBand / args.numJobs fStart = fMin + args.jobNum * fBand toplistPattern = cp.get('filename-patterns','toplist_name') toplistName = toplistPattern % (args.jobNum,args.numJobs) logfilePattern = cp.get('filename-patterns','logfile_name') logfileName = logfilePattern % (args.jobNum,args.numJobs) # Pass along the arguments from the ini file
def get_gconfig(): parser = SafeConfigParser() parser.optionxform = str parser.read(os.path.join(CONFIG.system_path, 'game.cfg')) return parser