def do_mqtt_forward(config, section, verbose):
# default configuration
default = SafeConfigParser({"hostname": "localhost", "port": "1883", "auth": "False", "user": "******", "password": "******", "retain": "False", "qos": "0"})
default.optionxform = str
default.read(config)
# configuration
cfg = SafeConfigParser({"client_id": "mqtt-forward-"+section+'-'+str(os.getpid()), "hostname": default.get("mqtt-forward", "hostname"), "port": default.get("mqtt-forward", "port"), "auth": default.get("mqtt-forward", "auth"), "transform": None, "plain": "False", "retain": default.get("mqtt-forward", "retain"), "qos": default.get("mqtt-forward", "qos"), "user": default.get("mqtt-forward", "user"), "password": default.get("mqtt-forward", "password")})
cfg.optionxform = str
cfg.read(config)
# publication setup
pubsection = cfg.get(section, "pub")
if not cfg.has_section(pubsection):
transform = pubsection
pubtopic = None
pubsection = section
else:
transform = cfg.get(pubsection, "transform")
pubtopic = cfg.get(pubsection, "topic")
if transform in transforms:
transform = transforms[transform]
if eval(cfg.get(pubsection, "auth")):
pubauth = { "username": cfg.get(pubsection, "user"), "password": cfg.get(pubsection, "password") }
else:
pubauth = None
pub = { "hostname": cfg.get(pubsection, "hostname"), "port": eval(cfg.get(pubsection, "port")), "auth": pubauth, "client_id": cfg.get(pubsection, "client_id"), "topic": pubtopic, "transform": transform, "plain": eval(cfg.get(pubsection, "plain")), "retain": eval(cfg.get(pubsection, "retain")), "qos": eval(cfg.get(pubsection, "qos")), "last": {} }
# subscription setup
subsection = cfg.get(section, "sub")
if not cfg.has_section(subsection):
subtopic = subsection
subsection = section
else:
subtopic = cfg.get(subsection, "topic")
subplain = eval(cfg.get(subsection, "plain"))
sub = mqtt.Client(client_id=cfg.get(subsection, "client_id")+"_sub", userdata=(verbose, pub, subtopic, eval(cfg.get(subsection, "plain"))))
sub.on_connect = on_connect
sub.on_message = on_message
if eval(cfg.get(subsection, "auth")):
sub.username_pw_set(cfg.get(subsection, "user"), cfg.get(subsection, "password"))
sub.connect(cfg.get(subsection, "hostname"), eval(cfg.get(subsection, "port")), 60)
if verbose > 3:
print(str(datetime.datetime.utcnow())+" subscribed @"+subtopic+" [" + subsection + "]: Plain = "+str(subplain))
# Loop until done...
sub.loop_start()
while not done:
sleep(1)
sub.loop_stop()
return 0
def __init__(self, path, default='etc/client_defaults.cfg'):
self.path = path
# read defaults
tmp = SafeConfigParser()
tmp.optionxform = str
tmp.read(default)
self._config_options_dict(tmp)
# read file
tmp = SafeConfigParser()
tmp.optionxform = str
tmp.read(path)
self._config_options_dict(tmp)
self._populate_partitions()
def copy_inputs(config_file, InputsDir):
config_dict = read_config(config_file)
config = SafeConfigParser()
config.optionxform = str
config.read(config_file)
new_config = os.path.join(InputsDir, os.path.split(config_file)[1])
# ---------------------------------------------------------------- #
# copy the inputs
for key, section in config_dict.iteritems():
if 'FILE_NAME' in section.keys():
new_file_name = os.path.join(InputsDir,
os.path.split(section['FILE_NAME'])[1])
copyfile(section['FILE_NAME'], new_file_name)
# update the config file for an easy restart
config.set(key, 'FILE_NAME',
os.path.join(InputsDir,
os.path.split(section['FILE_NAME'])[1]))
# update the config_dict with the new value
config_dict[key]['FILE_NAME'] = new_file_name
# ---------------------------------------------------------------- #
# ---------------------------------------------------------------- #
# write the new configuration file
with open(new_config, 'w') as configfile:
config.write(configfile)
# ---------------------------------------------------------------- #
return config_dict
def dumpDictToConfig(self, newhierdict, configfilename):
newparser = SafeConfigParser()
newparser.optionxform = str
for section_name in sorted(newhierdict.keys()):
newparser.add_section(section_name)
[newparser.set(section_name, k, v) for k, v in sorted(newhierdict[section_name].items())]
newparser.write(open(configfilename, 'w'))
def __init__ (self, filename = "output.cfg"):
from numpy import dtype
from ConfigParser import SafeConfigParser
from ast import literal_eval
parser = SafeConfigParser ()
parser.optionxform = str
parser.read ("output.cfg")
for section in parser.sections ():
for item in parser.items (section):
key, string = item
try:
self[key] = literal_eval (string)
except:
try:
self[key] = parser.getboolean (section, key)
except:
self[key] = string
self["dx"] = self["Lx"]/self["nx"]
self["dz"] = self["Lz"]/self["nz"]
self["npar"] = self["nx"]*self["nz"]*self["npc"]
self["rtype"] = dtype ("f" + str ( self["precision"]))
self["ctype"] = dtype ("c" + str (2*self["precision"]))
# Allow e.g. cfg["dx"] to be accessed through cfg.dx
self.__dict__.update (self)
def prepare_answers(path, topo, config):
with NamedTemporaryFile() as temp:
warn_if_fail(get(path, temp.name))
temp.flush()
parser = SafeConfigParser()
parser.optionxform = str
parser.read(temp.name)
parser.set("general", "CONFIG_PROVISION_DEMO", "y")
parser.set("general", "CONFIG_PROVISION_TEMPEST", "y")
parser.set("general", "CONFIG_PROVISION_TEMPEST_REPO_REVISION", "master")
parser.set("general", "CONFIG_KEYSTONE_ADMIN_PW", "Cisco123")
parser.set("general", "CONFIG_KEYSTONE_DEMO_PW", "secret")
parser.set("general", "CONFIG_DEBUG_MODE", "y")
parser.set("general", "CONFIG_NTP_SERVERS", "10.81.254.202,ntp.esl.cisco.com")
parser.set("general", "CONFIG_NOVA_NETWORK_DEFAULTFLOATINGPOOL", "public")
if topo == "2role":
parser.set("general", "CONFIG_CONTROLLER_HOST", config['servers']['control-server'][0]['ip'])
parser.set("general", "CONFIG_COMPUTE_HOSTS", config['servers']['compute-server'][0]['ip'])
if topo == "3role":
parser.set("general", "CONFIG_CONTROLLER_HOST", config['servers']['build-server'][0]['ip'])
parser.set("general", "CONFIG_NETWORK_HOSTS", config['servers']['control-server'][0]['ip'])
parser.set("general", "CONFIG_COMPUTE_HOSTS", config['servers']['compute-server'][0]['ip'])
with open("installed_answers", "w") as f:
parser.write(f)
warn_if_fail(put("installed_answers", "~/installed_answers"))
def readMetadata(baseDir): metadata = SafeConfigParser() metadata.optionxform = str file = codecs.open(getMetadataPath(baseDir), 'rb', encoding='utf-8') metadata.readfp(file) file.close() return metadata
def parseConfig(self, cfg, prefix=DEFAULT_OPT_PREFIX, section='DEFAULT'):
'''Read config settings from a file, config parser object or dict
@type cfg: basestring / ConfigParser derived type / dict
@param cfg: configuration file path or ConfigParser type object
@type prefix: basestring
@param prefix: prefix for option names e.g. "attributeQuery."
@type section: basetring
@param section: configuration file section from which to extract
parameters.
'''
if isinstance(cfg, basestring):
cfgFilePath = path.expandvars(cfg)
# Add a 'here' helper option for setting dir paths in the config
# file
hereDir = path.abspath(path.dirname(cfgFilePath))
_cfg = SafeConfigParser(defaults={'here': hereDir})
# Make option name reading case sensitive
_cfg.optionxform = str
_cfg.read(cfgFilePath)
items = _cfg.items(section)
elif isinstance(cfg, ConfigParser):
items = cfg.items(section)
elif isinstance(cfg, dict):
items = cfg.items()
else:
raise AttributeError('Expecting basestring, ConfigParser or dict '
'type for "cfg" attribute; got %r type' %
type(cfg))
self.__parseFromItems(items, prefix=prefix)
def get_config():
"""
Returns parsed configuration file (SafeConfigParser instance). File paths
that will be checked: ~/.sitescripts, ~/sitescripts.ini, /etc/sitescripts,
/etc/sitescripts.ini
"""
paths = []
# Allow SITESCRIPTS_CONFIG variable to override config path
if 'SITESCRIPTS_CONFIG' in os.environ:
paths.append(os.environ['SITESCRIPTS_CONFIG'])
# For debugging - accept configuration in user's profile
paths.append(os.path.expanduser('~/.sitescripts'))
paths.append(os.path.expanduser('~/sitescripts.ini'))
# Server-wide configuration if no custom found
paths.append('/etc/sitescripts')
paths.append('/etc/sitescripts.ini')
for path in paths:
path = os.path.abspath(path)
if os.path.exists(path):
config = SafeConfigParser()
config.optionxform = lambda x: x
config.read(path)
return config
raise Exception('No config file found. Please put sitescripts.ini into your home directory or /etc')
def __init__(self, path, default=os.path.join(os.path.dirname(os.path.abspath(__file__)),
'etc/client_defaults.cfg')):
self.path = path
# read defaults
tmp = SafeConfigParser()
tmp.optionxform = str
tmp.read(default)
self._config_options_dict(tmp)
# read file
tmp = SafeConfigParser()
tmp.optionxform = str
tmp.read(path)
self._config_options_dict(tmp)
self._populate_partitions()
def __init__(self, configFile, setEnv=False):
""" Purpose: Constructor
:param self: class object itself
:param configFile: configuration file to read
:param setEnv: Whether to export the key/value pairs in the environment
:raises ValueError
"""
parser = SafeConfigParser()
# make option names case sensitive
parser.optionxform = str
# read file contents
if configFile:
# file was provided
fileContents = parser.read(configFile)
else:
# file was not provided
raise Exception("Configuration file is not provided")
if not fileContents:
# unable to read file contents
raise ValueError(self.__class__.__name__ + ": File " + configFile + " could not be read.")
# use parser._sections to populate dictionary
for item in parser._sections:
self.m_dictionary.update(parser.items(item))
# If user has requested to set the environment,
# export all the key/value pairs into the environment
if setEnv == True:
for key in self.m_dictionary:
os.environ[key] = self.m_dictionary[key]
def _load_cfg_parser(self, filename='apb.ini'):
'''
Load config parser from specific INI filename.
'''
defaults = {}
try:
cfg_parser = SafeConfigParser(defaults, allow_no_value=True)
except TypeError:
cfg_parser = SafeConfigParser(defaults)
cfg_parser.optionxform = str
found_filename = None
if filename == os.path.basename(filename):
base_filename = filename
cwd = os.getcwd()
while cwd:
filename = os.path.join(cwd, base_filename)
if os.path.exists(filename):
found_filename = filename
break
if os.path.dirname(cwd) == cwd:
break
cwd = os.path.dirname(cwd)
else:
found_filename = filename
if found_filename:
cfg_parser.readfp(open(found_filename, 'r'), found_filename)
return cfg_parser
def parse_cfg_with_sections(stream):
"""Return as dict of dict of ...
"""
#Experimental:
"""
ConfigParser sections become sub-sub sections when separated by dots.
[foo.bar]
baz = 42
is equivalent to JSON
{"foo": {"bar": {"baz": 42}}}
"""
content = stream.read()
result = dict()
try:
jdict = json.loads(NativeIO(content).read())
return jdict
except ValueError:
pass #logger.exception('Could not parse stream as JSON.')
try:
config = ConfigParser() #strict=False?
config.optionxform = str
config.readfp(NativeIO(content))
sections = config.sections()
for sec in sections:
result[sec] = dict(config.items(sec))
return result
except:
raise
def checkVersion(self):
#print "in Checkver " + self.versions_cfg
missing_products = []
self.allowed_ext.append(self.py_ver_ext[self.python_version])
config = SafeConfigParser()
config.optionxform = lambda orig: orig
config.read(self.versions_cfg)
"""
returns a tuple list of key-value pairs
"""
iversions = config.items('versions')
#print iversions
iversions_keys = [value[0] for value in iversions]
for prod in iversions_keys:
print 'checking ' + prod + ' .....'
prod_exists = 0
for ext in self.allowed_ext:
if prod_exists == 0:
prod_url = self.package_index + "/" + prod+"/" + prod + "-" + config.get('versions',prod)+ext
print '\tchecking ' + prod_url
link_exists = self.checkLink(prod_url)
if (link_exists == 1) :
#print '\t\tfound ' + prod_url
prod_exists = 1
else:
pass
#print '\t\tnot found ' + prod_url
if prod_exists == 0:
print 'adding missing ' + prod + ' == ' + config.get('versions',prod)
missing_products.append(prod + ' == ' + config.get('versions',prod))
#print 'Missing Products = ' + str(len(missing_products))
return missing_products
def ensure_definition(definition, definition_type):
"""Load all alerts/reports into DATABASE
"""
parser = SafeConfigParser()
parser.optionxform = str
parser.read(definition)
if definition_type == "alert":
collection = ALERTS_COLLECTION
elif definition_type == "report":
collection = REPORTS_COLLECTION
for section in parser.sections():
params = defaultdict(dict)
params["name"] = section
for name, value in parser.items(section):
value = map(lambda y: y.strip(), value.split(", "))
if name[0] == "~":
params[name[1:]] = value[0]
else:
params[name] = value
DATABASE[collection].update({"name": section},
params, upsert=True)
LOGR.info("Ensured {0} definition for {1}".
format(definition_type, section))
def from_config(cls, cfg, prefix=PREFIX, section='DEFAULT'):
'''Create query from config file settings
@type cfg: basestring /ConfigParser derived type
@param cfg: configuration file path or ConfigParser type object
@type prefix: basestring
@param prefix: prefix for option names e.g. "attributeQuery."
@type section: baestring
@param section: configuration file section from which to extract
parameters.
'''
if isinstance(cfg, basestring):
cfg_filepath = os.path.expandvars(cfg)
here_dir = os.path.dirname(cfg_filepath)
_cfg = SafeConfigParser(defaults=dict(here=here_dir))
_cfg.optionxform = str
_cfg.read(cfg_filepath)
elif isinstance(cfg, ConfigParser):
_cfg = cfg
else:
raise AttributeError('Expecting basestring or ConfigParser type '
'for "cfg" attribute; got %r type' % type(cfg))
# Get items for this section as a dictionary so that from_kw can
# used to update the object
kw = dict(_cfg.items(section))
if 'prefix' not in kw and prefix:
kw['prefix'] = prefix
return cls.from_kw(**kw)
def load_local_config(config_file, required=[], optional=[]):
"""Loads config from ~/.cloudconnect and from os.environ, and merges values into current globals()"""
from ConfigParser import SafeConfigParser
config = SafeConfigParser()
config.optionxform = str
new_config = {}
if os.path.exists(config_file):
if "DEBUG_SETTINGS" in os.environ:
print "Reading '%s'" % config_file
config.read(config_file)
env = determine_env()
if env in config.sections():
for key in config.options(env):
val = config.get(env, key)
if key in os.environ:
val = os.environ[key]
val = cast_env_vars(val)
if "DEBUG_SETTINGS" in os.environ:
print "Setting %s to %s" % (key, str(val))
new_config[key] = val
for key in required:
if key in os.environ:
new_config[key] = cast_env_vars(os.environ[key])
for key in optional:
if key in os.environ:
new_config[key] = cast_env_vars(os.environ[key])
return new_config
def create_config(self):
""" Save current settings to config.ini """
config = SafeConfigParser()
config.optionxform = str
config.add_section('Desktop Entry')
config.set('Desktop Entry', 'Encoding', 'UTF-8')
config.set('Desktop Entry', 'Version', '1.0')
config.set('Desktop Entry', 'Type', 'Application')
config.set('Desktop Entry', 'Name', 'Bonjob')
config.set('Desktop Entry', 'Icon', 'bonjob.png')
config.set('Desktop Entry', 'Path', self.get_app_path())
config.set('Desktop Entry', 'Exec', 'python bonjob.py')
config.set('Desktop Entry', 'StartupNotify', 'false')
config.set('Desktop Entry', 'X-UnityGenerated', 'true')
#config.set('Desktop Entry', 'OnlyShowIn', 'Unity')
try:
conf_path = self.get_abspath('.local/share/applications/bonjob.desktop')
config.write(open(conf_path, 'w'))
except IOError as e:
logger.warning('[-] %s' % e)
raise SetupError
else:
logger.info('[+] Create bonjob.desktop in %s' % conf_path)
def parseConfig(self, cfg, prefix='', section='DEFAULT'):
'''Read config file settings
:type cfg: basestring /ConfigParser derived type
:param cfg: configuration file path or ConfigParser type object
:type prefix: basestring
:param prefix: prefix for option names e.g. "attributeQuery."
:type section: baestring
:param section: configuration file section from which to extract
parameters.
'''
if isinstance(cfg, basestring):
cfgFilePath = path.expandvars(cfg)
hereDir = path.dirname(cfgFilePath)
_cfg = SafeConfigParser(defaults=dict(here=hereDir))
_cfg.optionxform = str
_cfg.read(cfgFilePath)
elif isinstance(cfg, ConfigParser):
_cfg = cfg
else:
raise AttributeError('Expecting basestring or ConfigParser type '
'for "cfg" attribute; got %r type' % type(cfg))
# Get items for this section as a dictionary so that parseKeywords can
# used to update the object
kw = dict(_cfg.items(section))
if 'prefix' not in kw and prefix:
kw['prefix'] = prefix
self.parseKeywords(**kw)
def getLoginSession():
url = 'https://twitter.com'
r = requests.get(url,verify=False)
cookies = dict(r.cookies)
page = r.text
soup = BeautifulSoup(page);
input = soup.findAll('input',{'name':'authenticity_token'});
authenticity_token = input[0]['value'];
config = SafeConfigParser()
config.optionxform = str
config.read("tags.cfg")
username = config.get('USER','username')
password = config.get('USER','password')
login_url = 'https://twitter.com/sessions'
params = {}
params['session[username_or_email]'] = username
params['session[password]'] = password
params['authenticity_token'] = authenticity_token
params['redirect_after_login'] = '******'
params['return_to_ssl'] = 'true';
login_req = requests.post(login_url,data=params,cookies=cookies,verify=False,proxies=proxies,headers=headers,allow_redirects=False)
login_cookies = login_req.cookies;
check_url = 'https://twitter.com/'
check_req = requests.get(check_url,headers=headers,proxies=proxies,verify=False,cookies=login_cookies);
return login_cookies;
def retweet(tag):
login_cookies = getLoginSession()
config = SafeConfigParser()
config.optionxform = str
config.read("tags.cfg")
tags = dict(config.items('TAGS'))
for val in tags.keys():
tag = tags[val]
url='https://twitter.com/search?src=typd&q='+tag
r = requests.get(url,verify=False,proxies=proxies,headers=headers,cookies=login_cookies)
page = r.text
soup = BeautifulSoup(page)
input = soup.findAll('input',{'name':'authenticity_token'});
div = soup.findAll('div',{'class':'account-group js-mini-current-user'})
data_user_id = div[0]['data-screen-name']
referer = 'https://twitter.com/'+data_user_id
headers['Referer'] = referer;
print headers;
authenticity_token = input[0]['value']
print authenticity_token
params = {}
params['authenticity_token'] = authenticity_token
tweets = soup.findAll('li',{'data-item-type':'tweet'})
for tweet in tweets:
params['id'] = tweet['data-item-id']
retweet_url = 'https://twitter.com/i/tweet/retweet'
print params
r_retweet = requests.post(retweet_url,data=params,verify=False,cookies=login_cookies,proxies=proxies,headers=headers)
def testDatabaseContent(self):
script_output_file = os.path.abspath('script.out')
self.gnathub.run(script=Script.db2cfg(), output=script_output_file)
parser = SafeConfigParser()
parser.optionxform = str
parser.read(script_output_file)
sections = parser.sections()
for s in SECTIONS:
self.assertTrue(s in sections, "Missing section: " + s)
self.assertTrue(parser.has_option(TEXT_IO[0], EXTERNAL_CALL[0]),
"Should be an external call")
self.assertEqual(parser.get(TEXT_IO[0], EXTERNAL_CALL[0]),
EXTERNAL_CALL[1],
'Wrong message')
self.assertTrue(parser.has_option(B_SIMPLE_ADB[0], ENTRY_POINT[0]),
"Entry point not found")
self.assertTrue(parser.get(B_SIMPLE_ADB[0],
ENTRY_POINT[0]).endswith(ENTRY_POINT[1]),
'unexpected value for the entry point')
for metric in METRICS_SIMPLE_4_1:
self.assertTrue(
parser.has_option(SIMPLE_4_1[0], metric),
'missing entry for "%s"' % metric)
def updateMqtt(NeoStat, updateInterval):
cfg = SafeConfigParser()
cfg.optionxform = str
cfg.read(os.path.normpath(os.path.dirname(os.path.realpath(__file__)) + "/neohub.conf"))
host = cfg.get("mqtt", "hostname")
port = eval(cfg.get("mqtt", "port"))
topic = cfg.get("mqtt", "topic")
qos = eval(cfg.get("mqtt", "qos"))
retain = eval(cfg.get("mqtt", "retain"))
client_id = "neohub" + NeoStat.name + str(os.getpid())
if eval(cfg.get("mqtt", "auth")):
auth = {"username": cfg.get("mqtt", "user"), "password": cfg.get("mqtt", "password")}
else:
auth = None
while True:
try:
NeoStat.update()
time.sleep(updateInterval)
msgs = [{"topic": topic + NeoStat.name, "payload": """{ "temperature" : """ + str(NeoStat.current_temperature)
+ """, "status" : """ + "\"" + NeoStat.operation + "\"" " "
+ "}", "qos": qos, "retain": retain}]
log.debug("msgs = " + str(msgs))
publish.multiple(msgs, hostname=host, port=port, client_id=client_id, auth=auth)
except:
log.error("Unable to publish message")
def __init__(self, params, status, calib_data, service_name):
"""
:param params: pywws configuration.
:type params: :class:`pywws.DataStore.params`
:param status: pywws status store.
:type status: :class:`pywws.DataStore.status`
:param calib_data: 'calibrated' data.
:type calib_data: :class:`pywws.DataStore.calib_store`
:param service_name: name of service to upload to.
:type service_name: string
"""
self.logger = logging.getLogger('pywws.ToService(%s)' % service_name)
self.params = params
self.status = status
self.data = calib_data
self.service_name = service_name
# 'derived' services such as 'underground_rf' share their
# parent's config and templates
config_section = self.service_name.split('_')[0]
if config_section == self.service_name:
self.parent = None
else:
self.parent = config_section
self.old_response = None
self.old_ex = None
# set default socket timeout, so urlopen calls don't hang forever
socket.setdefaulttimeout(30)
# open params file
service_params = SafeConfigParser()
service_params.optionxform = str
service_params.readfp(open(os.path.join(
os.path.dirname(__file__), 'services', '%s.ini' % (self.service_name))))
# get URL
self.server = service_params.get('config', 'url')
# get fixed part of upload data
self.fixed_data = dict()
for name, value in service_params.items('fixed'):
if value[0] == '*':
value = self.params.get(config_section, value[1:], 'unknown')
self.fixed_data[name] = value
# create templater
self.templater = Template.Template(
self.params, self.status, self.data, self.data, None, None,
use_locale=False)
self.template_file = os.path.join(
os.path.dirname(__file__), 'services', '%s_template_%s.txt' % (
config_section, self.params.get('config', 'ws type')))
# get other parameters
self.catchup = eval(service_params.get('config', 'catchup'))
self.use_get = eval(service_params.get('config', 'use get'))
self.expected_result = eval(service_params.get('config', 'result'))
def save(self, parfile=None):
"""Store the current object for later use."""
parser = SafeConfigParser()
parser.optionxform = str
parser.add_section('material')
parser.set('material', 'title', str(self.title))
parser.set('material', 'RE', str(self.RE))
parser.add_section('parameters')
parser.set('parameters', 'B20', str(self.B20))
parser.set('parameters', 'B22', str(self.B22))
parser.set('parameters', 'B40', str(self.B40))
parser.set('parameters', 'B42', str(self.B42))
parser.set('parameters', 'B43', str(self.B43))
parser.set('parameters', 'B44', str(self.B44))
parser.set('parameters', 'B60', str(self.B60))
parser.set('parameters', 'B62', str(self.B62))
parser.set('parameters', 'B63', str(self.B63))
parser.set('parameters', 'B64', str(self.B64))
parser.set('parameters', 'B66', str(self.B66))
parser.set('parameters', 'Hz', str(self.Hz))
parser.set('parameters', 'Hx', str(self.Hx))
if parfile is None:
parfile = '%s.cfg' % self.title
with open(parfile, 'wb') as cfg:
parser.write(cfg)
def get_configuration():
"""Loads the xeno configuration object.
This method loads a ConfigParser.SafeConfigParser from the configuration
path. If the configuration path does not exist, this method returns an
empty SafeConfigParser.
Returns:
An initialized (but possibly empty) ConfigParser.SafeConfigParser.
"""
# Grab the configuration file path
config_file_path = configuration_file_path()
# Create a configuration parser
configuration = SafeConfigParser()
# Make it case-sensitive
configuration.optionxform = str
# Try to read in any existing configuration
try:
configuration.read(config_file_path)
except Exception as e:
print_error(
'Unable to read configuration file ({0}): {1}'.format(
config_file_path,
str(e)
)
)
exit(1)
return configuration
def main():
import argparse
from operator import itemgetter
parser = argparse.ArgumentParser()
parser.add_argument("input", nargs='+', metavar="XML")
parser.add_argument("-c", "--config", required=True)
parser.add_argument("-o", "--output", default="output.csv")
args = parser.parse_args()
output = tablib.Dataset()
config = SafeConfigParser()
config.optionxform = lambda value: value
config.read([args.config])
for fname in args.input:
print "Parsing '%s'..." % fname
info = extract_from_pdf(fname, config)
values = []
if not output.headers:
output.headers = sorted(info.keys())
for k, v in sorted(info.iteritems(), key=itemgetter(0)):
values.append(v)
output.append(values)
print "Writing to '%s'..." % args.output
with open(args.output, 'wb') as fp:
fp.write(output.csv)
def __init__(self, *file):
parser = SafeConfigParser()
parser.optionxform = str
found = parser.read(file)
if not found:
raise ValueError('No config file found!')
for name in sections:
self.__dict__.update(parser.items(name))
def __init__(self, *file): parser = SafeConfigParser() parser.optionxform = str found = parser.read(file) if not found: sys.exit(Fore.RED + "No config file found!" + Style.RESET_ALL) for name in sections: self.__dict__.update(parser.items(name))
def active():
conf_file = "conf/pingc.conf"
cp = SafeConfigParser()
cp.optionxform = str # Preserves case sensitivity
cp.readfp(open(conf_file, "r"))
section = "Main"
printLine("[D] checking pingc.conf for checkedin", flag)
return cp.get(section, "checkedin")
def initialiseModels(argv, update, initMode='training'):
# argv[1] = dataPath
# argv[2] = modelPath
# argv[3] = driverName
# update = 'update' or 'new'
from SAM.SAM_Core import SAMDriver as Driver
dataPath = argv[0]
modelPath = argv[1]
driverName = argv[2]
print argv
stringCommand = 'from SAM.SAM_Drivers import ' + driverName + ' as Driver'
print stringCommand
exec stringCommand
mySAMpy = Driver()
mode = update
trainName = dataPath.split('/')[-1]
# participantList is extracted from number of subdirectories of dataPath
participantList = [
f for f in listdir(dataPath) if isdir(join(dataPath, f))
]
off = 17
print '-------------------'
print 'Training Settings:'
print
print 'Data Path: '.ljust(off), dataPath
print 'Model Path: '.ljust(off), modelPath
print 'Participants: '.ljust(off), participantList
print 'Model Root Name: '.ljust(off), trainName
print 'Training Mode:'.ljust(off), mode
print 'Driver:'.ljust(off), driverName
print '-------------------'
print 'Loading Parameters...'
print
temporalFlag = False
modeConfig = ''
found = ''
try:
parser = SafeConfigParser()
found = parser.read(dataPath + "/config.ini")
if parser.has_option(trainName, 'update_mode'):
modeConfig = parser.get(trainName, 'update_mode')
else:
modeConfig = 'update'
print modeConfig
except IOError:
pass
defaultParamsList = [
'experiment_number', 'model_type', 'model_num_inducing',
'model_num_iterations', 'model_init_iterations', 'verbose', 'Quser',
'kernelString', 'ratioData', 'update_mode', 'model_mode', 'windowSize'
]
mySAMpy.experiment_number = None
mySAMpy.model_type = None
mySAMpy.kernelString = None
mySAMpy.fname = None
mySAMpy.ratioData = None
if initMode == 'training' and (mode == 'new' or modeConfig == 'new'
or 'exp' not in modelPath):
print 'Loading training parameters from: \n ', '\t' + dataPath + "/config.ini"
try:
default = False
parser = SafeConfigParser()
parser.optionxform = str
found = parser.read(dataPath + "/config.ini")
# load parameters from config file
if parser.has_option(trainName, 'experiment_number'):
mySAMpy.experiment_number = int(
parser.get(trainName, 'experiment_number'))
elif '.pickle' in modelPath:
mySAMpy.experiment_number = int(
modelPath.split('__')[-2].replace('exp', '')) + 1
else:
fail = True
print 'No experiment_number found'
if parser.has_option(trainName, 'model_type'):
mySAMpy.model_type = parser.get(trainName, 'model_type')
else:
default = True
mySAMpy.model_type = 'mrd'
if parser.has_option(trainName, 'model_num_inducing'):
mySAMpy.model_num_inducing = int(
parser.get(trainName, 'model_num_inducing'))
else:
default = True
mySAMpy.model_num_inducing = 30
if parser.has_option(trainName, 'model_num_iterations'):
mySAMpy.model_num_iterations = int(
parser.get(trainName, 'model_num_iterations'))
else:
default = True
mySAMpy.model_num_iterations = 700
if parser.has_option(trainName, 'model_init_iterations'):
mySAMpy.model_init_iterations = int(
parser.get(trainName, 'model_init_iterations'))
else:
default = True
mySAMpy.model_init_iterations = 2000
if parser.has_option(trainName, 'verbose'):
mySAMpy.verbose = parser.get(trainName, 'verbose') == 'True'
else:
default = True
mySAMpy.verbose = False
if parser.has_option(trainName, 'model_mode'):
mySAMpy.model_mode = parser.get(trainName, 'model_mode')
if mySAMpy.model_mode == 'temporal' and parser.has_option(
trainName, 'windowSize'):
mySAMpy.windowSize = int(
parser.get(trainName, 'windowSize'))
else:
temporalFlag = True
else:
default = True
mySAMpy.model_mode = 'single'
if parser.has_option(trainName, 'Quser'):
mySAMpy.Quser = int(parser.get(trainName, 'Quser'))
else:
default = True
mySAMpy.Quser = 2
if parser.has_option(trainName, 'kernelString'):
mySAMpy.kernelString = parser.get(trainName, 'kernelString')
else:
default = True
mySAMpy.kernelString = "GPy.kern.RBF(Q, ARD=False) + GPy.kern.Bias(Q) + GPy.kern.White(Q)"
if parser.has_option(trainName, 'ratioData'):
mySAMpy.ratioData = int(parser.get(trainName, 'ratioData'))
else:
default = True
mySAMpy.ratioData = 50
if default:
print 'Default settings applied'
mySAMpy.paramsDict = dict()
mySAMpy.loadParameters(parser, trainName)
except IOError:
print 'IO Exception reading ', found
pass
else:
print 'Loading parameters from: \n ', '\t' + modelPath
try:
parser = SafeConfigParser()
parser.optionxform = str
found = parser.read(dataPath + "/config.ini")
# load parameters from config file
if parser.has_option(trainName, 'experiment_number'):
mySAMpy.experiment_number = int(
parser.get(trainName, 'experiment_number'))
else:
mySAMpy.experiment_number = int(
modelPath.split('__')[-2].replace('exp', ''))
modelPickle = pickle.load(open(modelPath + '.pickle', 'rb'))
mySAMpy.paramsDict = dict()
for j in parser.options(trainName):
if j not in defaultParamsList:
print j
mySAMpy.paramsDict[j] = modelPickle[j]
mySAMpy.ratioData = modelPickle['ratioData']
mySAMpy.model_type = modelPickle['model_type']
mySAMpy.model_mode = modelPickle['model_mode']
if mySAMpy.model_mode == 'temporal':
mySAMpy.windowSize = modelPickle['windowSize']
mySAMpy.model_type = 'mrd'
mySAMpy.model_num_inducing = modelPickle['model_num_inducing']
mySAMpy.model_num_iterations = modelPickle['model_num_iterations']
mySAMpy.model_init_iterations = modelPickle[
'model_init_iterations']
mySAMpy.verbose = modelPickle['verbose']
mySAMpy.Quser = modelPickle['Quser']
mySAMpy.kernelString = modelPickle['kernelString']
try:
mySAMpy.listOfModels = modelPickle['listOfModels']
mySAMpy.classifiers = modelPickle['classifiers']
mySAMpy.classif_thresh = modelPickle['classif_thresh']
except:
pass
except IOError:
print 'IO Exception reading ', found
pass
if 'exp' in modelPath:
fnameProto = '/'.join(modelPath.split('/')[:-1]) + '/' + dataPath.split('/')[-1] + '__' + driverName + \
'__' + mySAMpy.model_type + '__exp' + str(mySAMpy.experiment_number)
else:
fnameProto = modelPath + dataPath.split('/')[-1] + '__' + driverName + '__' + mySAMpy.model_type + \
'__exp' + str(mySAMpy.experiment_number)
print 'Full model name: \n', '\t' + fnameProto
print '-------------------'
print
mySAMpy.save_model = False
mySAMpy.economy_save = True
mySAMpy.visualise_output = False
# test_mode = True
mySAMpy.readData(dataPath, participantList)
# at this point, all the data that will be eventually used for training is contained in mySAMpy.Y
# and mySAMpy.L contains all labels if any (depending on mrd model or bgplvm model)
# mySAMpy.L is a list of labels while mySAMpy.Y is a numpy array of data
# mySAMpy.Y should have 2 dimensions, length of dimension 0 = number of instances
# length of dimension 1 = length of feature vector
if mySAMpy.model_mode != 'temporal':
# get list of labels
mySAMpy.textLabels = list(set(mySAMpy.L))
# convert L from list of strings to array of indices
mySAMpy.L = np.asarray(
[mySAMpy.textLabels.index(i) for i in mySAMpy.L])[:, None]
mySAMpy.textLabels = mySAMpy.textLabels
else:
mySAMpy.X, mySAMpy.Y = transformTimeSeriesToSeq(
mySAMpy.Y1, mySAMpy.windowSize)
mySAMpy.L, mySAMpy.tmp = transformTimeSeriesToSeq(
mySAMpy.U1, mySAMpy.windowSize)
mm = [mySAMpy]
# mm.append(mySAMpy)
# mm[0] contains root model
# this is the only model in the case of a single model
# or contains all info for the rest of the models in case of multiple models
#
if mySAMpy.model_mode == 'single' or mySAMpy.model_mode == 'temporal':
mm[0].participantList = ['all']
else:
mm[0].participantList = ['root'] + mySAMpy.textLabels
for k in range(len(mm[0].participantList)):
if mm[0].participantList[k] == 'all':
minData = len(mm[k].L)
mm[0].fname = fnameProto
mm[0].model_type = mySAMpy.model_type
Ntr = int(mySAMpy.ratioData * minData / 100)
else:
if k > 0:
mm.append(Driver())
# extract subset of data corresponding to this model
inds = [
i for i in range(len(mm[0].Y['L']))
if mm[0].Y['L'][i] == k - 1
]
mm[k].Y = mm[0].Y['Y'][inds]
mm[k].L = mm[0].Y['L'][inds]
mm[k].Quser = mm[0].Quser
mm[k].verbose = mm[0].verbose
print 'Object class: ', mm[0].participantList[k]
minData = len(inds)
mm[k].fname = fnameProto + '__L' + str(k - 1)
mm[0].listOfModels.append(mm[k].fname)
mm[k].model_type = 'bgplvm'
Ntr = int(mySAMpy.ratioData * minData / 100)
else:
mm[0].listOfModels = []
mm[0].fname = fnameProto
mm[0].SAMObject.kernelString = ''
minData = len(mm[0].L)
Ntr = int(mySAMpy.ratioData * minData / 100)
mm[k].modelLabel = mm[0].participantList[k]
if mm[0].model_mode != 'temporal':
[Yall, Lall, YtestAll,
LtestAll] = mm[k].prepareData(mm[k].model_type,
Ntr,
randSeed=mm[0].experiment_number)
mm[k].Yall = Yall
mm[k].Lall = Lall
mm[k].YtestAll = YtestAll
mm[k].LtestAll = LtestAll
elif mm[0].model_mode == 'temporal':
[Xall, Yall, Lall, XtestAll, YtestAll,
LtestAll] = mm[k].prepareData(mm[k].model_type,
Ntr,
randSeed=mm[0].experiment_number)
mm[k].Xall = Xall
mm[k].Yall = Yall
mm[k].Lall = Lall
mm[k].XtestAll = XtestAll
mm[k].YtestAll = YtestAll
mm[k].LtestAll = LtestAll
print 'minData = ' + str(minData)
print 'ratioData = ' + str(mySAMpy.ratioData)
if initMode == 'training':
for k in range(len(mm[0].participantList)):
# for k = 0 check if multiple model or not
if mm[0].participantList[k] != 'root':
print "Training with ", mm[0].model_num_inducing, 'inducing points for ', \
mm[0].model_init_iterations, '|', mm[0].model_num_iterations
mm[k].training(mm[0].model_num_inducing,
mm[0].model_num_iterations,
mm[0].model_init_iterations,
mm[k].fname,
mm[0].save_model,
mm[0].economy_save,
keepIfPresent=False,
kernelStr=mm[0].kernelString)
if mm[0].visualise_output:
ax = mm[k].SAMObject.visualise()
visualiseInfo = dict()
visualiseInfo['ax'] = ax
else:
visualiseInfo = None
else:
for k in range(len(mm[0].participantList)):
# for k = 0 check if multiple model or not
if mm[0].participantList[k] != 'root':
print "Training with ", mm[0].model_num_inducing, 'inducing points for ', \
mm[0].model_init_iterations, '|', mm[0].model_num_iterations
mm[k].training(mm[0].model_num_inducing,
mm[0].model_num_iterations,
mm[0].model_init_iterations,
mm[k].fname,
mm[0].save_model,
mm[0].economy_save,
keepIfPresent=True,
kernelStr=mm[0].kernelString)
return mm
def createConfigParser(sourceConfigFile,
ignoreCase=True,
additional_config_dir=""):
""" Creates a config parser instance using the Oncotator conventions for config files.
Note: ALL config files must end with .config
In order of precedence:
basicConfig file (.config)
localBasicConfig file (.local.config)
The filename convention for a local basic config is as follows:
myConfig.config (the basicConfig)
myConfig.local.config
All sections, keys, and values in the local config supersede the basic config. No local configs should be
committed to git.
IMPORTANT: If the config file is not found. Attempts to find it again by prepending "configs/" and then checking
the additional_config_dir.
If still not found, assumes that the file is part of the install package.
Precedence:
1) File name relative to current dir (CWD)
2) Filename configs/CWD
3) Additional directory (additional_config_dir)
4) Package resources
.local.config can be found in any of these locations.
In other words, test.config can be package resources and test.local.config can be in ~/.oncotator,
and it will be as if these two files are local to each other.
:param sourceConfigFile:
:param ignoreCase:
:param additional_config_dir: Lowest precedence directory to search. Does NOT look for a config subdirectory here
"""
# TODO: Must be more graceful way of handling this.
# TODO: Add ~/.oncotator to areas that are searched for config files.
sourceConfigFP = ConfigUtils._locateConfigFile(sourceConfigFile,
additional_config_dir)
config = SafeConfigParser()
if not ignoreCase:
config.optionxform = str
config.readfp(sourceConfigFP)
# Create the local filename string by grabbing the ending of .config and
configBaseFilename = os.path.basename(sourceConfigFP.name)
pattern = "\.config$"
configLocalFilename = re.sub(pattern, ".local.config",
configBaseFilename)
dirName = os.path.dirname(sourceConfigFile)
if dirName <> "":
dirName = dirName + '/'
configLocalSourceFilename = dirName + configLocalFilename
configLocalSourceFP = ConfigUtils._locateConfigFile(
configLocalSourceFilename,
isRelaxedLogging=True,
additional_config_dir=additional_config_dir)
if configLocalSourceFP <> None:
config.readfp(configLocalSourceFP)
return config
for val in result:
print >>f, '%g\t' % val,
print >>f
if __name__ == '__main__':
if(not args.dir is None):
print 'size\titers\tH\t\terrRes\t\tincomp'
for val in process_dir(args.dir):
print '%g\t' % val,
print
exit(0)
cfgfile = args.config
cfg = SafeConfigParser(allow_no_value=True)
cfg.optionxform = str
cfg.read(cfgfile)
# --- data dir ---
# data dir pattern:
# $(data_path)/$(var_with_val}-$(data_path_suffix)/
# ${batchvar_with_val}/
data_path = cfg.get('xscft','dataPath')
data_path_suffix = cfg.get('xscft','dataPathSuffix')
var_with_val = get_var_with_val(cfg)
dir1 = os.path.join(data_path,var_with_val+data_path_suffix)
bname = cfg.get('xscft','activeBatchName')
if(args.result is None):
result_file = os.path.join(dir1, 'result-'+bname+'.txt')
else:
def __init__(self, params, status, calib_data, service_name):
"""
:param params: pywws configuration.
:type params: :class:`pywws.DataStore.params`
:param status: pywws status store.
:type status: :class:`pywws.DataStore.status`
:param calib_data: 'calibrated' data.
:type calib_data: :class:`pywws.DataStore.calib_store`
:param service_name: name of service to upload to.
:type service_name: string
"""
self.logger = logging.getLogger('pywws.ToService(%s)' % service_name)
self.params = params
self.status = status
self.data = calib_data
self.service_name = service_name
# 'derived' services such as 'underground_rf' share their
# parent's config and templates
config_section = self.service_name.split('_')[0]
if config_section == self.service_name:
self.parent = None
else:
self.parent = config_section
self.old_response = None
self.old_ex = None
# set default socket timeout, so urlopen calls don't hang forever
if eval(self.params.get('config', 'asynchronous', 'False')):
socket.setdefaulttimeout(60)
else:
socket.setdefaulttimeout(20)
# open params file
service_params = SafeConfigParser()
service_params.optionxform = str
param_string = pkg_resources.resource_string(
'pywws', 'services/%s.ini' % (self.service_name))
if sys.version_info[0] >= 3:
param_string = param_string.decode('utf-8')
service_params.readfp(StringIO.StringIO(param_string))
# get URL
self.server = service_params.get('config', 'url')
parsed_url = urlparse.urlsplit(self.server)
if parsed_url.scheme == 'aprs':
self.send_data = self.aprs_send_data
server, port = parsed_url.netloc.split(':')
self.server = (server, int(port))
elif parsed_url.scheme == 'mqtt':
self.send_data = self.mqtt_send_data
else:
self.send_data = self.http_send_data
self.use_get = eval(service_params.get('config', 'use get'))
# get fixed part of upload data
self.fixed_data = dict()
for name, value in service_params.items('fixed'):
if value[0] == '*':
value = self.params.get(config_section, value[1:], 'unknown')
self.fixed_data[name] = value
# create templater
self.templater = Template.Template(self.params,
self.status,
self.data,
self.data,
None,
None,
use_locale=False)
template_name = self.params.get(config_section, 'template', 'default')
if template_name != 'default':
template_dir = self.params.get(
'paths', 'templates',
os.path.expanduser('~/weather/templates/'))
self.template_file = open(
os.path.join(template_dir, template_name), 'rb')
else:
template_name = 'services/%s_template_%s.txt' % (
config_section, self.params.get('config', 'ws type'))
if not pkg_resources.resource_exists('pywws', template_name):
template_name = 'services/%s_template_1080.txt' % (
config_section)
self.template_file = pkg_resources.resource_stream(
'pywws', template_name)
# get other parameters
self.auth_type = service_params.get('config', 'auth_type')
if self.auth_type == 'basic':
user = self.params.get(config_section, 'user', 'unknown')
password = self.params.get(config_section, 'password', 'unknown')
details = user + ':' + password
self.auth = 'Basic ' + base64.b64encode(
details.encode('utf-8')).decode('utf-8')
elif self.auth_type == 'mqtt':
self.user = self.params.get(config_section, 'user', 'unknown')
self.password = self.params.get(config_section, 'password',
'unknown')
self.catchup = eval(service_params.get('config', 'catchup'))
self.expected_result = eval(service_params.get('config', 'result'))
self.interval = eval(service_params.get('config', 'interval'))
self.interval = max(self.interval, 40)
self.interval = timedelta(seconds=self.interval)
# move 'last update' from params to status
last_update = self.params.get_datetime(self.service_name,
'last update')
if last_update:
self.params.unset(self.service_name, 'last update')
self.status.set('last update', self.service_name,
last_update.isoformat(' '))
# set timestamp of first data to upload
self.next_update = datetime.utcnow() - max(
timedelta(days=self.catchup), self.interval)
def target_repos_from_config(*config_filenames):
config = SafeConfigParser()
config.optionxform = str # package names are case sensitive
for filename in config_filenames:
print 'reading %s' % filename
config.readfp(open(filename, 'r'))
repos = {}
testing_repos = {}
skipped = set()
for section in sorted(config.sections()):
descr, _, rest = section.partition('.')
if not rest:
koji_profile = config.get(section, 'source')
downgradeable = True
if config.has_option(section, 'downgradeable'):
downgradeable = config.getboolean(section, 'downgradeable')
all_packages = False
if config.has_option(section, 'all-packages'):
all_packages = config.getboolean(section, 'all-packages')
signed = False
if config.has_option(section, 'signed'):
signed = config.getboolean(section, 'signed')
repos[descr] = TargetRepo(name=config.get(section, 'name'),
distro=config.get(section, 'distro'),
arches=config.get(section,
'arches').split(),
tag=config.get(section, 'tag'),
downgradeable=downgradeable,
all_packages=all_packages,
signed=signed,
koji_profile=koji_profile)
testing_repos[descr] = TargetRepo(
name=config.get(section, 'testing-name'),
distro=config.get(section, 'distro'),
arches=config.get(section, 'arches').split(),
tag=config.get(section, 'testing-tag'),
downgradeable=downgradeable,
all_packages=all_packages,
signed=False, # testing repos are always unsigned for now
koji_profile=koji_profile)
if config.has_option(section, 'skip') and config.getboolean(
section, 'skip'):
skipped.add(descr)
if config.has_option(section, 'excluded-rpms'):
for name in config.get(section, 'excluded-rpms').split():
repos[descr].add_excluded_rpm(name)
testing_repos[descr].add_excluded_rpm(name)
elif rest == 'packages':
for package_name, rpm_names in config.items(section):
repos[descr].add_package(package_name, rpm_names.split())
testing_repos[descr].add_package(package_name,
rpm_names.split())
elif rest.startswith('packages.'):
tag, _, source = rest[len('packages.'):].partition('.')
for package_name, rpm_names in config.items(section):
repos[descr].add_package(package_name,
rpm_names.split(),
tag=tag,
koji_profile=source or None)
# appending -testing here is perhaps an unwise hack...
# we just need to work towards eliminating these exceptional sections
testing_tag = tag + '-testing' if not tag.endswith(
'-testing') else tag
testing_repos[descr].add_package(package_name,
rpm_names.split(),
tag=testing_tag,
koji_profile=source or None)
else:
raise ValueError('Unrecognised section: %s' % rest)
for descr in skipped:
del repos[descr]
del testing_repos[descr]
return sorted(repos.values() + testing_repos.values())
def WriteConfig():
cfg = SafeConfigParser()
cfg.optionxform = lambda option: option
section = 'config'
cfg.add_section(section)
cfg.set(section, "path", autosub.PATH)
cfg.set(section, "seriespath", autosub.SERIESPATH)
cfg.set(section, "bckpath", autosub.BCKPATH)
cfg.set(section, 'downloaddutch', str(autosub.DOWNLOADDUTCH))
cfg.set(section, 'downloadeng', str(autosub.DOWNLOADENG))
cfg.set(section, "subeng", autosub.SUBENG)
cfg.set(section, "subnl", autosub.SUBNL)
cfg.set(section, "fallbacktoeng", str(autosub.FALLBACKTOENG))
cfg.set(section, "notifyen", str(autosub.NOTIFYEN))
cfg.set(section, "notifynl", str(autosub.NOTIFYNL))
cfg.set(section, "wantedfirst", str(autosub.WANTEDFIRST))
cfg.set(section, "launchbrowser", str(autosub.LAUNCHBROWSER))
cfg.set(section, "skiphiddendirs", str(autosub.SKIPHIDDENDIRS))
cfg.set(section, "englishsubdelete", str(autosub.ENGLISHSUBDELETE))
cfg.set(section, "addic7ed", str(autosub.ADDIC7ED))
cfg.set(section, "opensubtitles", str(autosub.OPENSUBTITLES))
cfg.set(section, "podnapisi", str(autosub.PODNAPISI))
cfg.set(section, "subscene", str(autosub.SUBSCENE))
cfg.set(section, "hearingimpaired", str(autosub.HI))
cfg.set(section, 'minmatchscore', str(autosub.MINMATCHSCORE))
cfg.set(section, "configversion", str(autosub.CONFIGVERSION))
cfg.set(section, 'searchinterval', str(autosub.SEARCHINTERVAL))
cfg.set(section, "browserrefresh", str(autosub.BROWSERREFRESH))
cfg.set(section, "postprocesscmd", autosub.POSTPROCESSCMD)
cfg.set(section, "opensubtitlesuser", autosub.OPENSUBTITLESUSER)
cfg.set(section, "opensubtitlespasswd", autosub.OPENSUBTITLESPASSWD)
cfg.set(section, "addic7eduser", autosub.ADDIC7EDUSER)
cfg.set(section, "addic7edpasswd", autosub.ADDIC7EDPASSWD)
cfg.set(section, "subcodec", autosub.SUBCODEC)
cfg.set(section, "skipstringnl", autosub.SKIPSTRINGNL)
cfg.set(section, "skipstringen", autosub.SKIPSTRINGEN)
cfg.set(section, "skipfoldersnl", autosub.SKIPFOLDERSNL)
cfg.set(section, "skipfoldersen", autosub.SKIPFOLDERSEN)
section = 'webserver'
cfg.add_section(section)
cfg.set(section, "webserverip", str(autosub.WEBSERVERIP))
cfg.set(section, 'webserverport', str(autosub.WEBSERVERPORT))
cfg.set(section, "username", autosub.USERNAME)
cfg.set(section, "password", autosub.PASSWORD)
cfg.set(section, "webroot", autosub.WEBROOT)
section = 'logfile'
cfg.add_section(section)
cfg.set(section, "logfile", autosub.LOGFILE)
cfg.set(section, "loglevel", logging.getLevelName(autosub.LOGLEVEL))
cfg.set(section, "loglevelconsole",
logging.getLevelName(autosub.LOGLEVELCONSOLE))
cfg.set(section, "logsize", str(autosub.LOGSIZE))
cfg.set(section, "lognum", str(autosub.LOGNUM))
section = 'notify'
cfg.add_section(section)
if autosub.NOTIFYMAIL:
cfg.set(section, "notifymail", str(autosub.NOTIFYMAIL))
cfg.set(section, "mailsrv", autosub.MAILSRV)
cfg.set(section, 'mailfromaddr', autosub.MAILFROMADDR)
cfg.set(section, "mailtoaddr", autosub.MAILTOADDR)
cfg.set(section, "mailusername", autosub.MAILUSERNAME)
cfg.set(section, "mailpassword", autosub.MAILPASSWORD)
cfg.set(section, "mailsubject", autosub.MAILSUBJECT)
cfg.set(section, "mailencryption", autosub.MAILENCRYPTION)
cfg.set(section, "mailauth", autosub.MAILAUTH)
if autosub.NOTIFYGROWL:
cfg.set(section, "notifygrowl", str(autosub.NOTIFYGROWL))
cfg.set(section, "growlhost", autosub.GROWLHOST)
cfg.set(section, "growlport", autosub.GROWLPORT)
cfg.set(section, "growlpass", autosub.GROWLPASS)
if autosub.NOTIFYNMA:
cfg.set(section, "notifynma", str(autosub.NOTIFYNMA))
cfg.set(section, "nmaapi", autosub.NMAAPI)
cfg.set(section, "nmapriority", str(autosub.NMAPRIORITY))
if autosub.NOTIFYTWITTER:
cfg.set(section, "notifytwitter", str(autosub.NOTIFYTWITTER))
cfg.set(section, "twitterkey", autosub.TWITTERKEY)
cfg.set(section, "twittersecret", autosub.TWITTERSECRET)
if autosub.NOTIFYPROWL:
cfg.set(section, "notifyprowl", str(autosub.NOTIFYPROWL))
cfg.set(section, "prowlapi", autosub.PROWLAPI)
cfg.set(section, "prowlpriority", str(autosub.PROWLPRIORITY))
if autosub.NOTIFYTELEGRAM:
cfg.set(section, "notifytelegram", str(autosub.NOTIFYTELEGRAM))
cfg.set(section, "telegramapi", autosub.TELEGRAMAPI)
cfg.set(section, "telegramid", autosub.TELEGRAMID)
if autosub.NOTIFYPUSHALOT:
cfg.set(section, "notifypushalot", str(autosub.NOTIFYPUSHALOT))
cfg.set(section, "pushalotapi", autosub.PUSHALOTAPI)
if autosub.NOTIFYPUSHBULLET:
cfg.set(section, "notifypushbullet", str(autosub.NOTIFYPUSHBULLET))
cfg.set(section, "pushbulletapi", autosub.PUSHBULLETAPI)
if autosub.NOTIFYPUSHOVER:
cfg.set(section, "notifypushover", str(autosub.NOTIFYPUSHOVER))
cfg.set(section, "pushoverappkey", autosub.PUSHOVERAPPKEY)
cfg.set(section, "pushoveruserkey", autosub.PUSHOVERUSERKEY)
cfg.set(section, "pushoverpriority", str(autosub.PUSHOVERPRIORITY))
if autosub.NOTIFYBOXCAR2:
cfg.set(section, "notifyboxcar2", str(autosub.NOTIFYBOXCAR2))
cfg.set(section, "boxcar2token", autosub.BOXCAR2TOKEN)
if autosub.NOTIFYPLEX:
cfg.set(section, "notifyplex", str(autosub.NOTIFYPLEX))
cfg.set(section, "plexserverhost", autosub.PLEXSERVERHOST)
cfg.set(section, "plexserverport", autosub.PLEXSERVERPORT)
cfg.set(section, "plexserverusername", autosub.PLEXSERVERUSERNAME)
cfg.set(section, "plexserverpassword", autosub.PLEXSERVERPASSWORD)
section = 'skipshow'
cfg.add_section(section)
for Show in autosub.SKIPSHOW:
if re.match("^[0-9 ,.-]+$", autosub.SKIPSHOW[Show]):
cfg.set(section, Show.replace(':', '~'), autosub.SKIPSHOW[Show])
section = 'namemapping'
cfg.add_section(section)
for Name in autosub.USERNAMEMAPPING:
cfg.set(section, Name.replace(':', '~'), autosub.USERNAMEMAPPING[Name])
section = 'addic7edmapping'
cfg.add_section(section)
for Name in autosub.USERADDIC7EDMAPPING:
cfg.set(section, Name, autosub.USERADDIC7EDMAPPING[Name])
try:
with open(autosub.CONFIGFILE, 'wb') as cfile:
cfg.write(cfile)
except Exception as error:
return error
# here we read the config back because the the UPPERCASE variants of the config (for searching) has to be filled
ReadConfig()
return 'Config has been saved.'
def main():
try:
opts, servers = getopt.getopt(sys.argv[1:], "ahd", ["all", "help", "debug"])
except getopt.GetoptError as e:
logger.critical(str(e))
usage()
sys.exit(2)
runall = False
debug = False
for o, a in opts:
if o in ("-a", "--all"):
runall = True
elif o in ("-d", "--debug"):
debug = True
elif o in ("-h", "--help"):
usage()
sys.exit()
else:
assert False, "unhandled option: %s" % o
if debug: logger.setLevel(logging.DEBUG)
if servers == [] and runall == False:
logger.error("Please specify an APPP server to run, or use '-a' to run all servers! Exiting...")
sys.exit(1)
cfg = SafeConfigParser()
cfg.optionxform = str
runservers = {}
try:
cfg.readfp(open(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'APPP.ini')))
if runall == True:
logger.info("Attempting to run all APPP servers defined in APPP.ini")
servers = list(cfg.sections())
if servers == []:
logger.error("No APPP servers defined in APPP.ini! Exiting...")
sys.exit(1)
for srv in servers:
if not cfg.has_section(srv):
logger.error(srv + " is not defined in APPP.ini! Exiting...")
continue
server = config(cfg, srv)
if not server:
continue
runservers[srv] = server
except KeyboardInterrupt:
logger.info("Shutting down...")
sys.exit(0)
except Exception as e:
logger.critical(str(e))
sys.exit(1)
cfg = SafeConfigParser()
cfg.optionxform = str
try:
logger.info("Read DNS.ini")
cfg.readfp(open(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'DNS.ini')))
for item in cfg.items("DNS"):
items = item[1].split(',')
i = 0
while i < len(items):
items[i] = (items[i].strip(), 5)
i = i + 1
dns[item[0]] = items
except Exception as e:
logger.info("Can not read DNS.ini" + str(e))
if len(runservers) == 0:
logger.critical("No runnable APPP server found! Exiting...")
sys.exit(1)
logger.info('Checking APPP certificates...')
if not dump_ca(): sys.exit(1)
logger.info('APPP certificates OK.')
server_threads = {}
for server in runservers:
server_threads[server] = spawn(name=server, tgt=start_server, args=(runservers[server],))
try:
while not flag_exit:
if activeCount() == 1:
logger.info('No active APPP server running.')
break
sleep(5)
except KeyboardInterrupt:
pass
logger.info("Shutting down...")
sys.exit(0)
def get_run_options():
from ConfigParser import SafeConfigParser
configFileName = scriptName + ".ini"
if not os.path.isfile(configFileName):
configPath = os.path.join(scriptDir, configFileName)
else:
configPath = os.path.join('.', configFileName)
configSectionName = scriptName
config = SafeConfigParser()
config.optionxform = str
config.read(configPath)
configOptions = {}
if config.has_section(configSectionName):
for (k, v) in config.items(configSectionName):
if v == "": continue
configOptions[k] = v
defaults = {
'outputDir': './results',
'bclToFastqMaxCores': 12,
'samtoolsSortMemPerCore':
1000000000, # samtools sort uses about 2x what you tell it to...
'alnMaxCores':
8, # presumably bwa aln will become increasingly inefficient per core, so we don't want to let this go forever...
'isKeepFastq':
True, # important to keep these during testing, but not for production
}
defaults.update(configOptions)
parser = get_option_parser(defaults, configFileName)
(options, args) = parser.parse_args()
if options.isAllHelp:
parser = get_option_parser(defaults, configFileName, True)
parser.print_help()
sys.exit(2)
if len(args): # or (len(sys.argv) == 1):
parser.print_help()
sys.exit(2)
# sanitize arguments before writing defaults, check for missing arguments after:
#
def checkListRepeats(list, itemLabel):
if list == None: return
if len(set(list)) != len(list):
parser.error("Repeated %s entries" % (itemLabel))
if options.bclBasecallsDirList != None:
for i, bclDir in enumerate(options.bclBasecallsDirList):
options.bclBasecallsDirList[i] = checkDirArg(
bclDir, "bcl basecalls directory")
# tmp for testing:
# checkListRepeats(options.bclBasecallsDirList,"bcl basecalls directory")
if (options.bclTilePatternList != None) and \
(len(options.bclBasecallsDirList) != len(options.bclTilePatternList)) :
parser.error("Unexpected number of bclTilPattern entries")
checkListRepeats(options.sampleNameList, "sample name")
options.casavaDir = checkDirArg(options.casavaDir, "casava directory")
options.genomeFasta = checkFileArg(options.genomeFasta,
"genome fasta file")
options.bwaBin = checkFileArg(options.bwaBin, "bwa binary")
options.samtoolsBin = checkFileArg(options.samtoolsBin, "samtools binary")
if options.isWriteConfig == True:
if not config.has_section(configSectionName):
config.add_section(configSectionName)
for k, v in vars(options).iteritems():
if k == "isWriteConfig": continue
if v == None: v = ""
config.set(configSectionName, k, str(v))
configfp = open(configFileName, "w")
config.write(configfp)
configfp.close()
sys.exit(0)
def noArgOrError(msg):
if len(sys.argv) <= 1:
parser.print_help()
sys.exit(2)
else:
parser.error(msg)
def assertOption(arg, label):
if arg == None:
noArgOrError("No %s specified" % (label))
def assertList(list, itemLabel):
if (list == None) or (len(list) == 0):
noArgOrError("List containing %s (s) is empty or missing" %
(itemLabel))
else:
for item in list:
assertOption(item, itemLabel)
assertList(options.bclBasecallsDirList, "bcl basecalls directory")
assertList(options.sampleNameList, "sample name")
assertOption(options.genomeFasta, "genome fasta file")
assertOption(options.outputDir, "output directory")
assertOption(options.casavaDir, "casava directory")
assertOption(options.picardDir, "picard directory")
assertOption(options.bwaBin, "bwa binary")
assertOption(options.samtoolsBin, "samtools binary")
return options
def ReadConfig(configfile):
"""
Read the config file and set all the variables.
"""
# Read config file
cfg = SafeConfigParser()
cfg.optionxform = str
try:
with codecs.open(autosub.CONFIGFILE, 'r', autosub.SYSENCODING) as f:
cfg.readfp(f)
except:
Message = WriteConfig()
return
# First we check whether the config has been upgraded
if autosub.CONFIGVERSION < version.configversion:
upgradeConfig(cfg, autosub.CONFIGVERSION, version.configversion)
elif autosub.CONFIGVERSION > version.configversion:
print "Config: ERROR! Config version higher then this version of AutoSub supports. Update AutoSub!"
os._exit(1)
section = 'config'
if not cfg.has_section(section): cfg.add_section(section)
if cfg.has_option(section, "configversion"):
autosub.CONFIGVERSION = cfg.getint("config", "configversion")
if cfg.has_option(section, "wantedfirst"):
autosub.WANTEDFIRST = cfg.getboolean(section, "wantedfirst")
if cfg.has_option(section, 'downloaddutch'):
autosub.DOWNLOADDUTCH = cfg.getboolean(section, 'downloaddutch')
if cfg.has_option(section, 'downloadeng'):
autosub.DOWNLOADENG = cfg.getboolean(section, 'downloadeng')
if cfg.has_option(section, "fallbacktoeng"):
autosub.FALLBACKTOENG = cfg.getboolean(section, "fallbacktoeng")
if cfg.has_option(section, "notifyen"):
autosub.NOTIFYEN = cfg.getboolean(section, "notifyen")
if cfg.has_option(section, "notifynl"):
autosub.NOTIFYNL = cfg.getboolean(section, "notifynl")
if cfg.has_option(section, "launchbrowser"):
autosub.LAUNCHBROWSER = cfg.getboolean(section, "launchbrowser")
if cfg.has_option(section, "skiphiddendirs"):
autosub.SKIPHIDDENDIRS = cfg.getboolean(section, "skiphiddendirs")
if cfg.has_option(section, "englishsubdelete"):
autosub.ENGLISHSUBDELETE = cfg.getboolean(section, "englishsubdelete")
if cfg.has_option(section, "podnapisi"):
autosub.PODNAPISI = cfg.getboolean(section, "podnapisi")
if cfg.has_option(section, "subscene"):
autosub.SUBSCENE = cfg.getboolean(section, "subscene")
if cfg.has_option(section, "addic7ed"):
autosub.ADDIC7ED = cfg.getboolean(section, "addic7ed")
if cfg.has_option(section, "hearingimpaired"):
autosub.HI = cfg.getboolean(section, "hearingimpaired")
if cfg.has_option(section, 'minmatchscore'):
autosub.MINMATCHSCORE = cfg.getint(section, 'minmatchscore')
if cfg.has_option(section, 'searchinterval'):
autosub.SEARCHINTERVAL = cfg.getint(section, 'searchinterval')
if cfg.has_option(section, "browserrefresh"):
autosub.BROWSERREFRESH = cfg.getint(section, "browserrefresh")
if cfg.has_option(section, "subeng"):
autosub.SUBENG = cfg.get(section, "subeng")
if cfg.has_option(section, "subnl"):
autosub.SUBNL = cfg.get(section, "subnl")
if cfg.has_option(section, "postprocesscmd"):
autosub.POSTPROCESSCMD = cfg.get(section, "postprocesscmd")
if cfg.has_option(section, "opensubtitlesuser"):
autosub.OPENSUBTITLESUSER = cfg.get(section, "opensubtitlesuser")
if cfg.has_option(section, "opensubtitlespasswd"):
autosub.OPENSUBTITLESPASSWD = cfg.get(section, "opensubtitlespasswd")
if cfg.has_option(section, "addic7eduser"):
autosub.ADDIC7EDUSER = cfg.get(section, "addic7eduser")
if cfg.has_option(section, "addic7edpasswd"):
autosub.ADDIC7EDPASSWD = cfg.get(section, "addic7edpasswd")
if cfg.has_option(section, "logfile"):
autosub.LOGFILE = cfg.get(section, "logfile")
if cfg.has_option(section, "subcoautosub"):
autosub.SUBCODEC = cfg.get(section, "subcodec")
if cfg.has_option(section, "rootpath"):
autosub.SERIESPATH = cfg.get(section, "rootpath")
if cfg.has_option(section, "seriespath"):
autosub.SERIESPATH = cfg.get(section, "seriespath")
if cfg.has_option(section, "skipstringnl"):
autosub.SKIPSTRINGNL = cfg.get(section, "skipstringnl")
if cfg.has_option(section, "skipstringen"):
autosub.SKIPSTRINGEN = cfg.get(section, "skipstringen")
if cfg.has_option(section, "skipfoldersnl"):
autosub.SKIPFOLDERSNL = cfg.get(section, "skipfoldersnl")
if cfg.has_option(section, "skipfolderSen"):
autosub.SKIPFOLDERSEN = cfg.get(section, "skipfoldersen")
# *******************
# * Logfile Section *
# *******************
section = 'logfile'
if not cfg.has_section(section): cfg.add_section(section)
if cfg.has_option(section, "loglevel"):
LogLevel = cfg.get(section, "loglevel").upper()
if LogLevel == u'ERROR':
autosub.LOGLEVEL = logging.ERROR
elif LogLevel == u"WARNING":
autosub.LOGLEVEL = logging.WARNING
elif LogLevel == u"DEBUG":
autosub.LOGLEVEL = logging.DEBUG
elif LogLevel == u"INFO":
autosub.LOGLEVEL = logging.INFO
elif LogLevel == u"CRITICAL":
autosub.LOGLEVEL = logging.CRITICAL
if cfg.has_option(section, "loglevelconsole"):
LogLevel = cfg.get(section, "loglevelconsole").upper()
if LogLevel == u'ERROR':
autosub.LOGLEVELCONSOLE = logging.ERROR
elif LogLevel == u"WARNING":
autosub.LOGLEVELCONSOLE = logging.WARNING
elif LogLevel == u"DEBUG":
autosub.LOGLEVELCONSOLE = logging.DEBUG
elif LogLevel == u"INFO":
autosub.LOGLEVELCONSOLE = logging.INFO
elif LogLevel == u"CRITICAL":
autosub.LOGLEVELCONSOLE = logging.CRITICAL
if cfg.has_option(section, "logsize"):
autosub.LOGSIZE = cfg.getint(section, "logsize")
if cfg.has_option(section, "lognum"):
autosub.LOGNUM = cfg.getint(section, "lognum")
# ******************************
# * Cherrypy Webserver Section *
# ******************************
section = 'webserver'
if not cfg.has_section(section): cfg.add_section(section)
if cfg.has_option(section, 'webserverip'):
autosub.WEBSERVERIP = cfg.get(section, 'webserverip')
if cfg.has_option(section, 'webserverport'):
autosub.WEBSERVERPORT = int(cfg.get(section, 'webserverport'))
if cfg.has_option(section, 'webroot'):
autosub.WEBROOT = cfg.get(section, 'webroot')
if cfg.has_option(section, 'username'):
autosub.USERNAME = cfg.get(section, 'username')
if cfg.has_option(section, 'password'):
autosub.PASSWORD = cfg.get(section, 'password')
# ********************
# * SkipShow Section *
# ********************
section = 'skipshow'
if not cfg.has_section(section): cfg.add_section(section)
autosub.SKIPSHOWUPPER = {}
autosub.SKIPSHOW = {}
SkipShows = dict(cfg.items(section))
#autosub.SKIPSHOW = dict(cfg.items('skipshow'))
# The following 5 lines convert the skipshow to uppercase. And also convert the variables to a list
# also replace the "~" with ":" neccesary because the config parser sees ":" as a delimiter
for show in SkipShows:
if re.match("^[0-9 ,.-]+$", SkipShows[show]):
autosub.SKIPSHOW[show.replace('~', ':')] = SkipShows[show]
autosub.SKIPSHOWUPPER[show.upper().replace('~', ':')] = [
Item.strip() for Item in SkipShows[show].split(',')
]
# ***********************
# * Namemapping Section *
# ***********************
section = 'namemapping'
if not cfg.has_section(section): cfg.add_section(section)
NameMapping = dict(cfg.items(section))
autosub.USERNAMEMAPPING = {}
autosub.USERNAMEMAPPINGUPPER = {}
for Name in NameMapping:
if NameMapping[Name].isdigit():
autosub.USERNAMEMAPPING[Name.replace('~', ':')] = NameMapping[Name]
autosub.USERNAMEMAPPINGUPPER[Name.upper().replace('~', ':')] = [
Item.strip() for Item in NameMapping[Name].split(',')
]
# ********************************
# * Addic7ed Namemapping Section *
# ********************************
section = 'addic7edmapping'
if not cfg.has_section(section): cfg.add_section(section)
NameMapping = dict(cfg.items(section))
autosub.USERADDIC7EDMAPPING = {}
for Name in NameMapping:
if Name.isdigit and NameMapping[Name].isdigit():
autosub.USERADDIC7EDMAPPING[Name] = NameMapping[Name]
# ******************
# * Notify Section *
# ******************
section = 'notify'
if not cfg.has_section(section): cfg.add_section(section)
if cfg.has_option(section, 'notifymail'):
autosub.NOTIFYMAIL = cfg.getboolean(section, 'notifymail')
if cfg.has_option(section, 'mailsrv'):
autosub.MAILSRV = cfg.get(section, 'mailsrv')
if cfg.has_option(section, 'mailfromaddr'):
autosub.MAILFROMADDR = cfg.get(section, 'mailfromaddr')
if cfg.has_option(section, 'mailtoaddr'):
autosub.MAILTOADDR = cfg.get(section, 'mailtoaddr')
if cfg.has_option(section, 'mailusername'):
autosub.MAILUSERNAME = cfg.get(section, 'mailusername')
if cfg.has_option(section, 'mailpassword'):
autosub.MAILPASSWORD = cfg.get(section, 'mailpassword')
if cfg.has_option(section, 'mailsubject'):
autosub.MAILSUBJECT = cfg.get(section, 'mailsubject')
if cfg.has_option(section, 'mailencryption'):
autosub.MAILENCRYPTION = cfg.get(section, 'mailencryption')
if cfg.has_option(section, 'mailauth'):
autosub.MAILAUTH = cfg.get(section, 'mailauth')
if cfg.has_option(section, 'notifygrowl'):
autosub.NOTIFYGROWL = cfg.getboolean(section, 'notifygrowl')
if cfg.has_option(section, 'growlhost'):
autosub.GROWLHOST = cfg.get(section, 'growlhost')
if cfg.has_option(section, 'growlport'):
autosub.GROWLPORT = cfg.get(section, 'growlport')
if cfg.has_option(section, 'growlpass'):
autosub.GROWLPASS = cfg.get(section, 'growlpass')
if cfg.has_option(section, 'notifytwitter'):
autosub.NOTIFYTWITTER = cfg.getboolean(section, 'notifytwitter')
if cfg.has_option(section, 'twitterkey'):
autosub.TWITTERKEY = cfg.get(section, 'twitterkey')
if cfg.has_option(section, 'twittersecret'):
autosub.TWITTERSECRET = cfg.get(section, 'twittersecret')
if cfg.has_option(section, 'notifynma'):
autosub.NOTIFYNMA = cfg.getboolean(section, 'notifynma')
if cfg.has_option(section, 'nmaapi'):
autosub.NMAAPI = cfg.get(section, 'nmaapi')
if cfg.has_option(section, 'nmapriority'):
autosub.NMAPRIORITY = cfg.getint(section, 'nmapriority')
if cfg.has_option(section, 'notifyprowl'):
autosub.NOTIFYPROWL = cfg.getboolean(section, 'notifyprowl')
if cfg.has_option(section, 'prowlapi'):
autosub.PROWLAPI = cfg.get(section, 'prowlapi')
if cfg.has_option(section, 'prowlpriority'):
autosub.PROWLPRIORITY = cfg.getint(section, 'prowlpriority')
if cfg.has_option(section, 'notifypushalot'):
autosub.NOTIFYPUSHALOT = cfg.getboolean(section, 'notifypushalot')
if cfg.has_option(section, 'pushalotapi'):
autosub.PUSHALOTAPI = cfg.get(section, 'pushalotapi')
if cfg.has_option(section, 'notifypushbullet'):
autosub.NOTIFYPUSHBULLET = cfg.getboolean(section, 'notifypushbullet')
if cfg.has_option(section, 'pushbulletapi'):
autosub.PUSHBULLETAPI = cfg.get(section, 'pushbulletapi')
if cfg.has_option(section, 'notifypushover'):
autosub.NOTIFYPUSHOVER = cfg.getboolean(section, 'notifypushover')
if cfg.has_option(section, 'pushoverappkey'):
autosub.PUSHOVERAPPKEY = cfg.get(section, 'pushoverappkey')
if cfg.has_option(section, 'pushoveruserkey'):
autosub.PUSHOVERUSERKEY = cfg.get(section, 'pushoveruserkey')
if cfg.has_option(section, 'notifyboxcar2'):
autosub.NOTIFYBOXCAR2 = cfg.getboolean(section, 'notifyboxcar2')
if cfg.has_option(section, 'boxcar2token'):
autosub.BOXCAR2TOKEN = cfg.get(section, 'boxcar2token')
if cfg.has_option(section, 'notifyplex'):
autosub.NOTIFYPLEX = cfg.getboolean(section, 'notifyplex')
if cfg.has_option(section, 'plexserverhost'):
autosub.PLEXSERVERHOST = cfg.get(section, 'plexserverhost')
if cfg.has_option(section, 'plexserverport'):
autosub.PLEXSERVERPORT = cfg.get(section, 'plexserverport')
# Settings
autosub.NAMEMAPPING = {
"Against the Wall": "1836237",
"alcatraz": "1728102",
"almost human": "2654580",
"alphas": "1183865",
"american dad": "0397306",
"american horror story": "1844624",
"appropriate adult": "1831575",
"Are You There Chelsea": "1826989",
"atlantis": "2705602",
"atlantis 2013": "2705602",
"awkward": "1663676",
"back in the game": "2655470",
"Bates Motel": "2188671",
"beauty and the beast": "2193041",
"beauty and the beast 2012": "2193041",
"betrayal": "2751074",
"blue bloods": "1595859",
"boardwalk empire": "0979432",
"bob's burgers": "1561755",
"bobs burgers": "1561755",
"Body of Proof": "1587669",
"borgen": "1526318",
"breakout kings": "1590961",
"breaking bad": "903747",
"Castle (2009)": "1219024",
"castle 2009": "1219024",
"charlie's angels 2011": "1760943",
"Charlies Angels 2011": "1760943",
"chicago fire": "2261391",
"chicago fire (2012)": "2261391",
"chicago pd": "2805096",
"chicago p.d": "2805096",
"chicago p.d.": "2805096",
"Common Law 2012": "1771072",
"continuum": "1954347",
"covert affairs": "1495708",
"cracked (2013)": "2078576",
"criminal minds": "0452046",
"csi": "0247082",
"csi crime scene investigation": "0247082",
"Csi Miami": "0313043",
"csi new york": "0395843",
"csi ny": "0395843",
"Da Vinci's Demons": "2094262",
"Dallas 2012": "1723760",
"desperate housewives": "0410975",
"devious maids": "2226342",
"Doctor Who": "0436992",
"Doctor Who (2005)": "0436992",
"don't trust the b---- in apartment 23": "1819509",
"dont trust the bitch in apartment 23": "1819509",
"dracula": "2296682",
"dracula (2013)": "2296682",
"DreamWorks Dragons: Riders of Berk": "2325846",
"eastbound & down": "0866442",
"eastbound and down": "0866442",
"emily owens m d": "2290339",
"Falling skies": "1462059",
"Fast N Loud": "2346169",
"Femme Fatales": "1841108",
"Franklin and Bash": "1600199",
"Free Agents": "1839481",
"Free Agents Us": "1839481",
"fringe": "1119644",
"game of thrones": "0944947",
"Glee": "1327801",
"Grey's Anatomy": "0413573",
"Greys Anatomy": "0413573",
"grimm": "1830617",
"harry's law": "1582453",
"Harrys Law": "1582453",
"haven": "1519931",
"Hawaii Five 0": "1600194",
"Hawaii Five 0 2010": "1600194",
"Hawaii Five-0": "1600194",
"hawaii five-0 2010": "1600194",
"hello ladies": "2378794",
"homeland": "1796960",
"hostages": "2647258",
"house of cards 2013": "1856010",
"how i met your mother": "0460649",
"How To Survive The End Of The World": "3377330",
"Intelligence us": "2693776",
"king": "1804880",
"kings of crash": "2623754",
"Last Man Standing": "1828327",
"Last Man Standing Us": "1828327",
"law and order svu": "0203259",
"law and order uk": "1166893",
"longmire": "1836037",
"luck": "1578887",
"luther": "1474684",
"Man Up": "1828238",
"marvel's agents of s h i e l d": "2364582",
"marvels agents of s h i e l d": "2364582",
"marvel agents of shield": "2364582",
"agents of s h i e l d": "2364582",
"masters of sex": "2137109",
"Melissa And Joey": "1597420",
"Merlin": "1199099",
"Merlin 2008": "1199099",
"Mike and Molly": "1608180",
"missing 2012": "1828246",
"mockingbird lane": "2130271",
"modern family": "1442437",
"moonshiners": "1877005",
"Mr Sunshine": "1583638",
"nashville": "2281375",
"nashville 2012": "2281375",
"ncis": "0364845",
"Ncis Los Angeles": "1378167",
"Necessary Roughness": "1657505",
"new girl": "1826940",
"new tricks": "0362357",
"nip tuck": "0361217",
"nip-tuck": "0361217",
"once upon a time": "1843230",
"once upon time": "1843230",
"once upon a time 2011": "1843230",
"once upon a time in wonderland": "2802008",
"oppenheimer (1980)": "0078037",
"Parks and Recreation": "1266020",
"person of interest": "1839578",
"played": "2886812",
"pretty little liars": "1578873",
"Prime Suspect Us": "1582456",
"primeval new world": "2295953",
"ray donovan": "2249007",
"reign 2013": "2710394",
"Revolution": "2070791",
"Revolution 2012": "2070791",
"Rizzoli And Isles": "1551632",
"rookie blue": "1442065",
"Scandal": "1837576",
"scandal (2012)": "1837576",
"Scandal 2012": "1837576",
"Scandal US": "1837576",
"scott and bailey": "1843678",
"sean saves the world": "2715776",
"Shameless Us": "1586680",
"silent witness": "0115355",
"Sinbad": "1979918",
"sleepy hollow": "2647544",
"snooki and jwoww": "2083701",
"sons of anarchy": "1124373",
"South Park": "0121955",
"Spartacus": "1442449",
"Spartacus Blood And Sand": "1442449",
"Spartacus Gods Of The Arena": "1758429",
"spartacus vengeance": "1442449",
"star wars the clone wars": "0458290",
"suburgatory": "1741256",
"suits": "1632701",
"sun, sex and suspicious parents": "1832153",
"super fun night": "2298477",
"The After": "3145422",
"the americans 2013": "2149175",
"the americans (2013)": "2149175",
"the americans": "2149175",
"the big bang theory": "898266",
"the biggest loser": "0429318",
"the blacklist": "2741602",
"the client list": "2022170",
"the closer": "0458253",
"the dukes of hazzard": "78607",
"the gadget show": "0830851",
"The Kennedys": "1567215",
"the killing (2011)": "1637727",
"The La Complex": "1794147",
"The Legend Of Korra": "1695360",
"the lying game": "1798274",
"the mentalist": "1196946",
"the newsroom (2012)": "1870479",
"the newsroom 2012": "1870479",
"the o c": "0362359",
"the office us": "0386676",
"the originals": "2632424",
"the piglet files": "0098895",
"the protector": "1836417",
"The River": "1836195",
"the tomorrow people us": "2660734",
"the walking dead": "1520211",
"the wire": "306414",
"the wrong mans": "2603596",
"thundercats 2011": "1666278",
"Touch": "1821681",
"trophy wife": "2400736",
"two and a half men": "0369179",
"under the dome": "1553656",
"unforgettable": "1842530",
"untouchables-the venture bros": "0417373",
"Up All Night 2011": "1843323",
"utopia": "2384811",
"Vegas": "2262383",
"white collar": "1358522",
"xiii the series 2011": "1713938"
}
autosub.NAMEMAPPINGUPPER = {}
for x in autosub.NAMEMAPPING.keys():
autosub.NAMEMAPPINGUPPER[x.upper()] = autosub.NAMEMAPPING[x]
autosub.LASTESTDOWNLOAD = []
def __init__(self, params, calib_data, service_name=None):
"""
:param params: pywws configuration.
:type params: :class:`pywws.DataStore.params`
:param calib_data: 'calibrated' data.
:type calib_data: :class:`pywws.DataStore.calib_store`
:keyword service_name: name of service to upload to.
:type service_name: string
"""
if service_name:
self.config_section = service_name
self.logger = logging.getLogger('pywws.ToService(%s)' %
service_name)
else:
self.logger = logging.getLogger('pywws.%s' %
self.__class__.__name__)
self.params = params
self.data = calib_data
self.old_response = None
self.old_ex = None
# set default socket timeout, so urlopen calls don't hang forever
socket.setdefaulttimeout(30)
# open params file
service_params = SafeConfigParser()
service_params.optionxform = str
service_params.readfp(
open(
os.path.join(os.path.dirname(__file__), 'services',
'%s.ini' % (self.config_section))))
# get URL
self.server = service_params.get('config', 'url')
# get fixed part of upload data
self.fixed_data = dict()
for name, value in service_params.items('fixed'):
if value[0] == '*':
value = self.params.get(self.config_section, value[1:],
'unknown')
self.fixed_data[name] = value
# create templater
self.templater = Template.Template(self.params,
self.data,
self.data,
None,
None,
use_locale=False)
self.template_file = os.path.join(
os.path.dirname(__file__), 'services', '%s_template_%s.txt' %
(service_name, self.params.get('fixed', 'ws type')))
# get other parameters
self.catchup = eval(service_params.get('config', 'catchup'))
self.use_get = eval(service_params.get('config', 'use get'))
rapid_fire = eval(service_params.get('config', 'rapidfire'))
if rapid_fire:
self.server_rf = service_params.get('config', 'url-rf')
self.fixed_data_rf = dict(self.fixed_data)
for name, value in service_params.items('fixed-rf'):
self.fixed_data_rf[name] = value
else:
self.server_rf = self.server
self.fixed_data_rf = self.fixed_data
self.expected_result = eval(service_params.get('config', 'result'))
from ConfigParser import SafeConfigParser from orphics.tools.io import Plotter, dictFromSection, listFromConfig from szlib.szcounts import ClusterCosmology from orphics.tools.stats import bin2D from szlib.sims import BattagliaSims, getKappaSZ from enlib.fft import fft, ifft import os print "Done importing modules..." outDir = os.environ['WWW'] + "plots/kappatest/" # === COSMOLOGY === cosmologyName = 'LACosmology' # from ini file iniFile = "../SZ_filter/input/params.ini" Config = SafeConfigParser() Config.optionxform = str Config.read(iniFile) lmax = 8000 cosmoDict = dictFromSection(Config, cosmologyName) constDict = dictFromSection(Config, 'constants') cc = ClusterCosmology(cosmoDict, constDict, lmax, pickling=True) TCMB = 2.7255e6 # === NFW CLUSTER === massOverh = 2.e14 concentration = 3.2 zL = 0.7 sourceZ = 1100. overdensity = 500. critical = True
def initialiseModels(argv, update, initMode='training'):
"""Initialise SAM Model data structure, training parameters and user parameters.
This method starts by initialising the required Driver from the driver name in argv[3] if it exists
in SAM_Drivers folder. The standard model parameters and the specific user parameters are then initialised
and the data is read in by the SAMDriver.readData method to complete the model data structure. This method
then replicates the model data structure for training with multiple models if it is required in the config
file loaded by the Driver.
Args:
argv_0: dataPath containing the data that is to be trained on.
argv_1: modelPath containing the path of where the model is to be stored.
argv_2: driverName containing the name of the driver class that is to be loaded from SAM_Drivers folder.
update: String having either a value of 'update' or 'new'. 'new' will load the parameters as set in the
config file of the driver being loaded present in the dataPath directory. This is used to train a
new model from scratch. 'update' will check for an existing model in the modelPath directory and
load the parameters from this model if it exists. This is used for retraining a model when new
data becomes available.
initMode: String having either a value of 'training' or 'interaction'. 'training' takes into consideration
the value of update in loading the parameters. (Used by trainSAMModel.py) 'interaction' loads
the parameters directly from the model if the model exists. (Used by interactionSAMModel.py)
Returns:
The output is a list of SAMDriver models. The list is of length 1 when the config file requests a single
model or a list of length n+1 for a config file requesting multiple models where n is the number of
requested models. The number of models either depends on the number of directories present in the dataPath
or from the length of textLabels returned from the SAMDriver.readData method.
"""
from SAM.SAM_Core import SAMDriver as Driver
dataPath = argv[0]
modelPath = argv[1]
driverName = argv[2]
logging.info(argv)
stringCommand = 'from SAM.SAM_Drivers import ' + driverName + ' as Driver'
logging.info(stringCommand)
exec stringCommand
mySAMpy = Driver()
mode = update
trainName = dataPath.split('/')[-1]
# participantList is extracted from number of subdirectories of dataPath
participantList = [
f for f in listdir(dataPath) if isdir(join(dataPath, f))
]
off = 17
logging.info('-------------------')
logging.info('Training Settings:')
logging.info('')
logging.info('Init mode: '.ljust(off) + str(initMode))
logging.info('Data Path: '.ljust(off) + str(dataPath))
logging.info('Model Path: '.ljust(off) + str(modelPath))
logging.info('Participants: '.ljust(off) + str(participantList))
logging.info('Model Root Name: '.ljust(off) + str(trainName))
logging.info('Training Mode:'.ljust(off) + str(mode))
logging.info('Driver:'.ljust(off) + str(driverName))
logging.info('-------------------')
logging.info('Loading Parameters...')
logging.info('')
modeConfig = ''
found = ''
try:
parser = SafeConfigParser()
found = parser.read(dataPath + "/config.ini")
if parser.has_option(trainName, 'update_mode'):
modeConfig = parser.get(trainName, 'update_mode')
else:
modeConfig = 'update'
logging.info(modeConfig)
except IOError:
pass
defaultParamsList = [
'experiment_number', 'model_type', 'model_num_inducing',
'model_num_iterations', 'model_init_iterations', 'verbose', 'Quser',
'kernelString', 'ratioData', 'update_mode', 'model_mode',
'temporalModelWindowSize', 'optimiseRecall', 'classificationDict',
'useMaxDistance', 'calibrateUnknown'
]
mySAMpy.experiment_number = None
mySAMpy.model_type = None
mySAMpy.kernelString = None
mySAMpy.fname = None
mySAMpy.ratioData = None
if initMode == 'training' and (mode == 'new' or modeConfig == 'new'
or 'exp' not in modelPath):
logging.info('Loading training parameters from:' + str(dataPath) +
"/config.ini")
try:
default = False
parser = SafeConfigParser()
parser.optionxform = str
found = parser.read(dataPath + "/config.ini")
mySAMpy.experiment_number = 'exp'
if parser.has_option(trainName, 'model_type'):
mySAMpy.model_type = parser.get(trainName, 'model_type')
else:
default = True
mySAMpy.model_type = 'mrd'
if parser.has_option(trainName, 'model_num_inducing'):
mySAMpy.model_num_inducing = int(
parser.get(trainName, 'model_num_inducing'))
else:
default = True
mySAMpy.model_num_inducing = 30
if parser.has_option(trainName, 'model_num_iterations'):
mySAMpy.model_num_iterations = int(
parser.get(trainName, 'model_num_iterations'))
else:
default = True
mySAMpy.model_num_iterations = 700
if parser.has_option(trainName, 'model_init_iterations'):
mySAMpy.model_init_iterations = int(
parser.get(trainName, 'model_init_iterations'))
else:
default = True
mySAMpy.model_init_iterations = 2000
if parser.has_option(trainName, 'verbose'):
mySAMpy.verbose = parser.get(trainName, 'verbose') == 'True'
else:
default = True
mySAMpy.verbose = False
if parser.has_option(trainName, 'optimiseRecall'):
mySAMpy.optimiseRecall = int(
parser.get(trainName, 'optimiseRecall'))
else:
default = True
mySAMpy.optimiseRecall = 200
if parser.has_option(trainName, 'useMaxDistance'):
mySAMpy.useMaxDistance = parser.get(trainName,
'useMaxDistance') == 'True'
else:
mySAMpy.useMaxDistance = False
if parser.has_option(trainName, 'calibrateUnknown'):
mySAMpy.calibrateUnknown = parser.get(
trainName, 'calibrateUnknown') == 'True'
else:
mySAMpy.calibrateUnknown = False
if parser.has_option(trainName, 'model_mode'):
mySAMpy.model_mode = parser.get(trainName, 'model_mode')
if mySAMpy.model_mode == 'temporal' and parser.has_option(
trainName, 'temporalModelWindowSize'):
mySAMpy.temporalWindowSize = int(
parser.get(trainName, 'temporalModelWindowSize'))
else:
temporalFlag = True
else:
default = True
mySAMpy.model_mode = 'single'
if parser.has_option(trainName, 'Quser'):
mySAMpy.Quser = int(parser.get(trainName, 'Quser'))
else:
default = True
mySAMpy.Quser = 2
if parser.has_option(trainName, 'kernelString'):
mySAMpy.kernelString = parser.get(trainName, 'kernelString')
else:
default = True
mySAMpy.kernelString = "GPy.kern.RBF(Q, ARD=False) + GPy.kern.Bias(Q) + GPy.kern.White(Q)"
if parser.has_option(trainName, 'ratioData'):
mySAMpy.ratioData = int(parser.get(trainName, 'ratioData'))
else:
default = True
mySAMpy.ratioData = 50
if default:
logging.info('Default settings applied')
mySAMpy.paramsDict = dict()
mySAMpy.loadParameters(parser, trainName)
except IOError:
logging.warning('IO Exception reading ', found)
pass
else:
logging.info('Loading parameters from: \n \t' + str(modelPath))
try:
parser = SafeConfigParser()
parser.optionxform = str
found = parser.read(dataPath + "/config.ini")
# load parameters from config file
mySAMpy.experiment_number = modelPath.split('__')[-1]
modelPickle = pickle.load(open(modelPath + '.pickle', 'rb'))
mySAMpy.paramsDict = dict()
for j in parser.options(trainName):
if j not in defaultParamsList:
logging.info(str(j))
mySAMpy.paramsDict[j] = modelPickle[j]
mySAMpy.ratioData = modelPickle['ratioData']
mySAMpy.model_type = modelPickle['model_type']
mySAMpy.model_mode = modelPickle['model_mode']
if mySAMpy.model_mode == 'temporal':
mySAMpy.temporalModelWindowSize = modelPickle[
'temporalModelWindowSize']
mySAMpy.model_type = 'mrd'
mySAMpy.model_num_inducing = modelPickle['model_num_inducing']
mySAMpy.model_num_iterations = modelPickle['model_num_iterations']
mySAMpy.model_init_iterations = modelPickle[
'model_init_iterations']
mySAMpy.verbose = modelPickle['verbose']
mySAMpy.Quser = modelPickle['Quser']
mySAMpy.optimiseRecall = modelPickle['optimiseRecall']
mySAMpy.kernelString = modelPickle['kernelString']
mySAMpy.calibrated = modelPickle['calibrated']
# try loading classification parameters for multiple model implementation
try:
mySAMpy.useMaxDistance = modelPickle['useMaxDistance']
except:
logging.warning(
'Failed to load useMaxDistace. Possible reasons: '
'Not saved or multiple model implementation')
mySAMpy.calibrateUnknown = modelPickle['calibrateUnknown']
if mySAMpy.calibrateUnknown:
mySAMpy.classificationDict = modelPickle['classificationDict']
except IOError:
logging.warning('IO Exception reading ', found)
pass
if 'exp' in modelPath or 'best' in modelPath or 'backup' in modelPath:
fnameProto = '/'.join(modelPath.split('/')[:-1]) + '/' + dataPath.split('/')[-1] + '__' + driverName + \
'__' + mySAMpy.model_type + '__' + str(mySAMpy.experiment_number)
else:
fnameProto = modelPath + dataPath.split('/')[-1] + '__' + driverName + '__' + mySAMpy.model_type + \
'__' + str(mySAMpy.experiment_number)
logging.info('Full model name: ' + str(fnameProto))
logging.info('-------------------')
logging.info('')
mySAMpy.save_model = False
mySAMpy.economy_save = True
mySAMpy.visualise_output = False
# test_mode = True
mySAMpy.readData(dataPath, participantList)
if mySAMpy.model_mode != 'temporal':
# get list of labels
mySAMpy.textLabels = list(set(mySAMpy.L))
# convert L from list of strings to array of indices
mySAMpy.L = np.asarray(
[mySAMpy.textLabels.index(i) for i in mySAMpy.L])[:, None]
mySAMpy.textLabels = mySAMpy.textLabels
else:
mySAMpy.X, mySAMpy.Y = transformTimeSeriesToSeq(
mySAMpy.Y1, mySAMpy.temporalModelWindowSize)
mySAMpy.L, mySAMpy.tmp = transformTimeSeriesToSeq(
mySAMpy.U1, mySAMpy.temporalModelWindowSize)
mm = [mySAMpy]
# mm.append(mySAMpy)
# mm[0] contains root model
# this is the only model in the case of a single model
# or contains all info for the rest of the models in case of multiple models
#
if mySAMpy.model_mode == 'single' or mySAMpy.model_mode == 'temporal':
mm[0].participantList = ['all']
else:
mm[0].participantList = ['root'] + mySAMpy.textLabels
for k in range(len(mm[0].participantList)):
if mm[0].participantList[k] == 'all':
normaliseData = True
minData = len(mm[k].L)
mm[0].fname = fnameProto
mm[0].model_type = mySAMpy.model_type
Ntr = int(mySAMpy.ratioData * minData / 100)
else:
if k > 0:
mm.append(Driver())
# extract subset of data corresponding to this model
inds = [
i for i in range(len(mm[0].Y['L']))
if mm[0].Y['L'][i] == k - 1
]
mm[k].Y = mm[0].Y['Y'][inds]
mm[k].L = mm[0].Y['L'][inds]
mm[k].Quser = mm[0].Quser
mm[k].verbose = mm[0].verbose
logging.info('Object class: ' + str(mm[0].participantList[k]))
minData = len(inds)
mm[k].fname = fnameProto + '__L' + str(k - 1)
mm[0].listOfModels.append(mm[k].fname)
mm[k].model_type = 'bgplvm'
Ntr = int(mySAMpy.ratioData * minData / 100)
normaliseData = True
else:
normaliseData = False
mm[0].listOfModels = []
mm[0].fname = fnameProto
mm[0].SAMObject.kernelString = ''
minData = len(mm[0].L)
Ntr = int(mySAMpy.ratioData * minData / 100)
mm[k].modelLabel = mm[0].participantList[k]
if mm[0].model_mode != 'temporal':
[Yall, Lall, YtestAll,
LtestAll] = mm[k].prepareData(mm[k].model_type,
Ntr,
randSeed=0,
normalise=normaliseData)
mm[k].Yall = Yall
mm[k].Lall = Lall
mm[k].YtestAll = YtestAll
mm[k].LtestAll = LtestAll
elif mm[0].model_mode == 'temporal':
[Xall, Yall, Lall, XtestAll, YtestAll,
LtestAll] = mm[k].prepareData(mm[k].model_type,
Ntr,
randSeed=0,
normalise=normaliseData)
mm[k].Xall = Xall
mm[k].Yall = Yall
mm[k].Lall = Lall
mm[k].XtestAll = XtestAll
mm[k].YtestAll = YtestAll
mm[k].LtestAll = LtestAll
logging.info('minData = ' + str(minData))
logging.info('ratioData = ' + str(mySAMpy.ratioData))
logging.info(
'-------------------------------------------------------------------------------------------------'
)
if initMode == 'training':
samOptimiser.deleteModel(modelPath, 'exp')
for k in range(len(mm[0].participantList)):
# for k = 0 check if multiple model or not
if mm[0].participantList[k] != 'root':
logging.info("Training with " + str(mm[0].model_num_inducing) +
' inducing points for ' +
str(mm[0].model_init_iterations) + '|' +
str(mm[0].model_num_iterations))
logging.info("Fname:" + str(mm[k].fname))
mm[k].training(mm[0].model_num_inducing,
mm[0].model_num_iterations,
mm[0].model_init_iterations,
mm[k].fname,
mm[0].save_model,
mm[0].economy_save,
keepIfPresent=False,
kernelStr=mm[0].kernelString)
if mm[0].visualise_output:
ax = mm[k].SAMObject.visualise()
visualiseInfo = dict()
visualiseInfo['ax'] = ax
else:
visualiseInfo = None
else:
for k in range(len(mm[0].participantList)):
# for k = 0 check if multiple model or not
if mm[0].participantList[k] != 'root':
logging.info("Training with " + str(mm[0].model_num_inducing) +
' inducing points for ' +
str(mm[0].model_init_iterations) + '|' +
str(mm[0].model_num_iterations))
mm[k].training(mm[0].model_num_inducing,
mm[0].model_num_iterations,
mm[0].model_init_iterations,
mm[k].fname,
mm[0].save_model,
mm[0].economy_save,
keepIfPresent=True,
kernelStr=mm[0].kernelString)
return mm
import getpass as gp
import re
import sys
import os
from glob import glob
from pprint import pprint
from ConfigParser import SafeConfigParser
# MPC Imports
from mpc.tessa import contexts
# Relative Path Config Setup
__location__ = os.path.dirname(os.path.realpath(__file__))
__config__ = os.path.join(__location__, "config.ini")
try:
parser = SafeConfigParser()
parser.optionxform = str
parser.read(__config__)
config = {}
for section in parser.sections():
config[section] = {}
for sub_item, value in parser.items(section):
config[section][
sub_item] = value if ',' not in value else value.split(',')
root = config['map']['server']
except KeyError as err:
print err
raise IOError("File not found %s" % __config__)
class SaveData(object):
""" Class putting together all the data and interfacing with the UI
def ReadConfig():
"""
Read the config file and set all the variables.
"""
# Read config file
cfg = SafeConfigParser()
cfg.optionxform = lambda option: option
try:
with codecs.open(autosub.CONFIGFILE, 'r', autosub.SYSENCODING) as f:
cfg.readfp(f)
except:
#No config found so we create a default config
Message = WriteConfig()
return
# First we check whether the config has been upgraded
if autosub.CONFIGVERSION < version.configversion:
upgradeConfig(cfg, autosub.CONFIGVERSION, version.configversion)
elif autosub.CONFIGVERSION > version.configversion:
print "Config: ERROR! Config version higher then this version of AutoSub supports. Update AutoSub!"
os._exit(1)
section = 'config'
if not cfg.has_section(section): cfg.add_section(section)
if cfg.has_option(section, "configversion"):
autosub.CONFIGVERSION = cfg.getint("config", "configversion")
if cfg.has_option(section, "wantedfirst"):
autosub.WANTEDFIRST = cfg.getboolean(section, "wantedfirst")
if cfg.has_option(section, 'downloaddutch'):
autosub.DOWNLOADDUTCH = cfg.getboolean(section, 'downloaddutch')
if cfg.has_option(section, 'downloadeng'):
autosub.DOWNLOADENG = cfg.getboolean(section, 'downloadeng')
if cfg.has_option(section, "fallbacktoeng"):
autosub.FALLBACKTOENG = cfg.getboolean(section, "fallbacktoeng")
if cfg.has_option(section, "notifyen"):
autosub.NOTIFYEN = cfg.getboolean(section, "notifyen")
if cfg.has_option(section, "notifynl"):
autosub.NOTIFYNL = cfg.getboolean(section, "notifynl")
if cfg.has_option(section, "launchbrowser"):
autosub.LAUNCHBROWSER = cfg.getboolean(section, "launchbrowser")
if cfg.has_option(section, "skiphiddendirs"):
autosub.SKIPHIDDENDIRS = cfg.getboolean(section, "skiphiddendirs")
if cfg.has_option(section, "englishsubdelete"):
autosub.ENGLISHSUBDELETE = cfg.getboolean(section, "englishsubdelete")
if cfg.has_option(section, "podnapisi"):
autosub.PODNAPISI = cfg.getboolean(section, "podnapisi")
if cfg.has_option(section, "subscene"):
autosub.SUBSCENE = cfg.getboolean(section, "subscene")
if cfg.has_option(section, "addic7ed"):
autosub.ADDIC7ED = cfg.getboolean(section, "addic7ed")
if cfg.has_option(section, "opensubtitles"):
autosub.OPENSUBTITLES = cfg.getboolean(section, "opensubtitles")
if cfg.has_option(section, "hearingimpaired"):
autosub.HI = cfg.getboolean(section, "hearingimpaired")
if cfg.has_option(section, 'minmatchscore'):
autosub.MINMATCHSCORE = cfg.getint(section, 'minmatchscore')
if cfg.has_option(section, 'searchinterval'):
autosub.SEARCHINTERVAL = cfg.getint(section, 'searchinterval')
if cfg.has_option(section, "browserrefresh"):
autosub.BROWSERREFRESH = cfg.getint(section, "browserrefresh")
if cfg.has_option(section, "path"):
autosub.PATH = os.path.normpath(cfg.get(section, "path"))
if cfg.has_option(section, "rootpath"):
autosub.SERIESPATH = os.path.normpath(cfg.get(section, "rootpath"))
if cfg.has_option(section, "seriespath"):
autosub.SERIESPATH = os.path.normpath(cfg.get(section, "seriespath"))
if autosub.SERIESPATH == '.':
autosub.SERIESPATH = u''
if cfg.has_option(section, "bckpath"):
autosub.BCKPATH = os.path.normpath(cfg.get(section, "bckpath"))
if autosub.BCKPATH == '.':
autosub.BCKPATH = u''
if cfg.has_option(section, "subeng"):
autosub.SUBENG = cfg.get(section, "subeng")
if cfg.has_option(section, "subnl"):
autosub.SUBNL = cfg.get(section, "subnl")
if cfg.has_option(section, "postprocesscmd"):
autosub.POSTPROCESSCMD = cfg.get(section, "postprocesscmd")
if cfg.has_option(section, "opensubtitlesuser"):
autosub.OPENSUBTITLESUSER = cfg.get(section, "opensubtitlesuser")
if cfg.has_option(section, "opensubtitlespasswd"):
autosub.OPENSUBTITLESPASSWD = cfg.get(section, "opensubtitlespasswd")
if cfg.has_option(section, "addic7eduser"):
autosub.ADDIC7EDUSER = cfg.get(section, "addic7eduser")
if cfg.has_option(section, "addic7edpasswd"):
autosub.ADDIC7EDPASSWD = cfg.get(section, "addic7edpasswd")
if cfg.has_option(section, "logfile"):
autosub.LOGFILE = cfg.get(section, "logfile")
if cfg.has_option(section, "subcodec"):
autosub.SUBCODEC = cfg.get(section, "subcodec")
if cfg.has_option(section, "skipstringnl"):
autosub.SKIPSTRINGNL = cfg.get(section, "skipstringnl")
if cfg.has_option(section, "skipstringen"):
autosub.SKIPSTRINGEN = cfg.get(section, "skipstringen")
if cfg.has_option(section, "skipfoldersnl"):
autosub.SKIPFOLDERSNL = cfg.get(section, "skipfoldersnl")
if cfg.has_option(section, "skipfoldersen"):
autosub.SKIPFOLDERSEN = cfg.get(section, "skipfoldersen")
# *******************
# * Logfile Section *
# *******************
section = 'logfile'
if not cfg.has_section(section): cfg.add_section(section)
if cfg.has_option(section, "logfile"):
autosub.LOGFILE = cfg.get(section, "logfile")
if cfg.has_option(section, "loglevel"):
LogLevel = cfg.get(section, "loglevel").upper()
if LogLevel == u'ERROR':
autosub.LOGLEVEL = logging.ERROR
elif LogLevel == u"WARNING":
autosub.LOGLEVEL = logging.WARNING
elif LogLevel == u"DEBUG":
autosub.LOGLEVEL = logging.DEBUG
elif LogLevel == u"INFO":
autosub.LOGLEVEL = logging.INFO
elif LogLevel == u"CRITICAL":
autosub.LOGLEVEL = logging.CRITICAL
if cfg.has_option(section, "loglevelconsole"):
LogLevel = cfg.get(section, "loglevelconsole").upper()
if LogLevel == u'ERROR':
autosub.LOGLEVELCONSOLE = logging.ERROR
elif LogLevel == u"WARNING":
autosub.LOGLEVELCONSOLE = logging.WARNING
elif LogLevel == u"DEBUG":
autosub.LOGLEVELCONSOLE = logging.DEBUG
elif LogLevel == u"INFO":
autosub.LOGLEVELCONSOLE = logging.INFO
elif LogLevel == u"CRITICAL":
autosub.LOGLEVELCONSOLE = logging.CRITICAL
if cfg.has_option(section, "logsize"):
autosub.LOGSIZE = cfg.getint(section, "logsize")
if cfg.has_option(section, "lognum"):
autosub.LOGNUM = cfg.getint(section, "lognum")
# ******************************
# * Cherrypy Webserver Section *
# ******************************
section = 'webserver'
if not cfg.has_section(section): cfg.add_section(section)
if cfg.has_option(section, 'webserverip'):
autosub.WEBSERVERIP = str(cfg.get(section, 'webserverip'))
if cfg.has_option(section, 'webserverport'):
autosub.WEBSERVERPORT = int(cfg.get(section, 'webserverport'))
if cfg.has_option(section, 'webroot'):
autosub.WEBROOT = str(cfg.get(section, 'webroot'))
if cfg.has_option(section, 'username'):
autosub.USERNAME = str(cfg.get(section, 'username'))
if cfg.has_option(section, 'password'):
autosub.PASSWORD = str(cfg.get(section, 'password'))
# ********************
# * SkipShow Section *
# ********************
section = 'skipshow'
if not cfg.has_section(section): cfg.add_section(section)
autosub.SKIPSHOWUPPER = {}
autosub.SKIPSHOW = {}
SkipShows = dict(cfg.items(section))
#autosub.SKIPSHOW = dict(cfg.items('skipshow'))
# The following 5 lines convert the skipshow to uppercase. And also convert the variables to a list
# also replace the "~" with ":" neccesary because the config parser sees ":" as a delimiter
# The UPPER version is for searching, the normal for dispaly in the user interface
for show in SkipShows:
if re.match("^[0-9 ,.-]+$", SkipShows[show]):
autosub.SKIPSHOW[show.replace('~', ':')] = SkipShows[show]
autosub.SKIPSHOWUPPER[show.upper().replace('~', ':')] = [
Item.strip() for Item in SkipShows[show].split(',')
]
# ********************************
# * Addic7ed Namemapping Section *
# ********************************
section = 'addic7edmapping'
if not cfg.has_section(section): cfg.add_section(section)
autosub.USERADDIC7EDMAPPING = {}
try:
autosub.USERADDIC7EDMAPPING = dict(cfg.items(section))
except:
pass
for ImdbId in autosub.USERADDIC7EDMAPPING.iterkeys():
if not (ImdbId.isdigit
and autosub.USERADDIC7EDMAPPING[ImdbId].isdigit()):
del autosub.USERADDIC7EDMAPPING[ImdbId]
print 'ReadConfig: Addic7ed mapping has an unkown format.', ImdbId, ' = ', autosub.USERADDIC7EDMAPPING[
ImdbId]
# Settings
# ****************************
# * User Namemapping Section *
# ****************************
section = 'namemapping'
if not cfg.has_section(section): cfg.add_section(section)
NameMapping = dict(cfg.items(section))
autosub.USERNAMEMAPPING = {}
for ConfigName in NameMapping.iterkeys():
if NameMapping[ConfigName].isdigit():
Name = ConfigName.replace('~', ':')
if not Name.upper() in autosub.NAMEMAPPING.keys():
autosub.NAMEMAPPING[Name.upper()] = [
NameMapping[ConfigName].strip(), u''
]
autosub.USERNAMEMAPPING[Name] = NameMapping[ConfigName].strip()
else:
print 'ReadConfig: Username mapping has an unknown format.', ConfigName, ' = ', NameMapping[
ConfigName]
# ******************
# * Notify Section *
# ******************
section = 'notify'
if not cfg.has_section(section): cfg.add_section(section)
if cfg.has_option(section, 'notifymail'):
autosub.NOTIFYMAIL = cfg.getboolean(section, 'notifymail')
if cfg.has_option(section, 'mailsrv'):
autosub.MAILSRV = cfg.get(section, 'mailsrv')
if cfg.has_option(section, 'mailfromaddr'):
autosub.MAILFROMADDR = cfg.get(section, 'mailfromaddr')
if cfg.has_option(section, 'mailtoaddr'):
autosub.MAILTOADDR = cfg.get(section, 'mailtoaddr')
if cfg.has_option(section, 'mailusername'):
autosub.MAILUSERNAME = cfg.get(section, 'mailusername')
if cfg.has_option(section, 'mailpassword'):
autosub.MAILPASSWORD = cfg.get(section, 'mailpassword')
if cfg.has_option(section, 'mailsubject'):
autosub.MAILSUBJECT = cfg.get(section, 'mailsubject')
if cfg.has_option(section, 'mailencryption'):
autosub.MAILENCRYPTION = cfg.get(section, 'mailencryption')
if cfg.has_option(section, 'mailauth'):
autosub.MAILAUTH = cfg.get(section, 'mailauth')
if cfg.has_option(section, 'notifygrowl'):
autosub.NOTIFYGROWL = cfg.getboolean(section, 'notifygrowl')
if cfg.has_option(section, 'growlhost'):
autosub.GROWLHOST = cfg.get(section, 'growlhost')
if cfg.has_option(section, 'growlport'):
autosub.GROWLPORT = cfg.get(section, 'growlport')
if cfg.has_option(section, 'growlpass'):
autosub.GROWLPASS = cfg.get(section, 'growlpass')
if cfg.has_option(section, 'notifytwitter'):
autosub.NOTIFYTWITTER = cfg.getboolean(section, 'notifytwitter')
if cfg.has_option(section, 'twitterkey'):
autosub.TWITTERKEY = cfg.get(section, 'twitterkey')
if cfg.has_option(section, 'twittersecret'):
autosub.TWITTERSECRET = cfg.get(section, 'twittersecret')
if cfg.has_option(section, 'notifynma'):
autosub.NOTIFYNMA = cfg.getboolean(section, 'notifynma')
if cfg.has_option(section, 'nmaapi'):
autosub.NMAAPI = cfg.get(section, 'nmaapi')
if cfg.has_option(section, 'nmapriority'):
autosub.NMAPRIORITY = cfg.getint(section, 'nmapriority')
if cfg.has_option(section, 'notifyprowl'):
autosub.NOTIFYPROWL = cfg.getboolean(section, 'notifyprowl')
if cfg.has_option(section, 'prowlapi'):
autosub.PROWLAPI = cfg.get(section, 'prowlapi')
if cfg.has_option(section, 'prowlpriority'):
autosub.PROWLPRIORITY = cfg.getint(section, 'prowlpriority')
if cfg.has_option(section, 'notifytelegram'):
autosub.NOTIFYTELEGRAM = cfg.getboolean(section, 'notifytelegram')
if cfg.has_option(section, 'telegramapi'):
autosub.TELEGRAMAPI = cfg.get(section, 'telegramapi')
if cfg.has_option(section, 'telegramid'):
autosub.TELEGRAMID = cfg.get(section, 'telegramid')
if cfg.has_option(section, 'notifypushalot'):
autosub.NOTIFYPUSHALOT = cfg.getboolean(section, 'notifypushalot')
if cfg.has_option(section, 'pushalotapi'):
autosub.PUSHALOTAPI = cfg.get(section, 'pushalotapi')
if cfg.has_option(section, 'notifypushbullet'):
autosub.NOTIFYPUSHBULLET = cfg.getboolean(section, 'notifypushbullet')
if cfg.has_option(section, 'pushbulletapi'):
autosub.PUSHBULLETAPI = cfg.get(section, 'pushbulletapi')
if cfg.has_option(section, 'notifypushover'):
autosub.NOTIFYPUSHOVER = cfg.getboolean(section, 'notifypushover')
if cfg.has_option(section, 'pushoverappkey'):
autosub.PUSHOVERAPPKEY = cfg.get(section, 'pushoverappkey')
if cfg.has_option(section, 'pushoveruserkey'):
autosub.PUSHOVERUSERKEY = cfg.get(section, 'pushoveruserkey')
if cfg.has_option(section, 'pushoverpriority'):
autosub.PUSHOVERPRIORITY = cfg.getint(section, 'pushoverpriority')
if cfg.has_option(section, 'notifyboxcar2'):
autosub.NOTIFYBOXCAR2 = cfg.getboolean(section, 'notifyboxcar2')
if cfg.has_option(section, 'boxcar2token'):
autosub.BOXCAR2TOKEN = cfg.get(section, 'boxcar2token')
if cfg.has_option(section, 'notifyplex'):
autosub.NOTIFYPLEX = cfg.getboolean(section, 'notifyplex')
if cfg.has_option(section, 'plexserverhost'):
autosub.PLEXSERVERHOST = cfg.get(section, 'plexserverhost')
if cfg.has_option(section, 'plexserverport'):
autosub.PLEXSERVERPORT = cfg.get(section, 'plexserverport')
if cfg.has_option(section, 'plexserverusername'):
autosub.PLEXSERVERUSERNAME = cfg.get(section, 'plexserverusername')
if cfg.has_option(section, 'plexserverpassword'):
autosub.PLEXSERVERPASSWORD = cfg.get(section, 'plexserverpassword')
def testDatabaseContent(self):
script_output_file = os.path.abspath('script.out')
self.gnathub.run(script=Script.db2cfg(), output=script_output_file)
parser = SafeConfigParser()
parser.optionxform = str
parser.read(script_output_file)
self.assertListEqual(sorted(parser.sections()), SECTIONS)
self.assertTrue(parser.has_option(F_ADB, 'warnings'),
'missing "warnings" entry')
self.assertTrue(parser.has_option(F_ADB, 'identifier_casing'),
'missing "identifier_casing" entry')
self.assertTrue(
parser.has_option(P_ADB, 'unconstrained_array_returns'),
'missing "unconstrained_array_returns" entry')
self.assertTrue(
parser.has_option(P_ADS_1, 'unconstrained_array_returns'),
'missing "unconstrained_array_returns" entry')
self.assertTrue(parser.has_option(PG_ADB_1, 'improper_returns'),
'missing "improper_returns" entry')
self.assertTrue(parser.has_option(PG_ADB_2, 'improper_returns'),
'missing "improper_returns" entry')
self.assertTrue(
parser.has_option(PG_ADB_3, 'unconstrained_array_returns'),
'missing "unconstrained_array_returns" entry')
self.assertTrue(parser.has_option(PG_ADB_4, 'warnings'),
'missing "warnings" entry')
self.assertTrue(
parser.has_option(PG_ADB_5, 'unconstrained_array_returns'),
'missing "unconstrained_array_returns" entry')
self.assertTrue(parser.has_option(PG_ADB_6, 'warnings'),
'missing "warnings" entry')
self.assertTrue(
parser.has_option(PG_ADS_1, 'unconstrained_array_returns'),
'missing "unconstrained_array_returns" entry')
self.assertTrue(
parser.has_option(PG_ADS_2, 'unconstrained_array_returns'),
'missing "unconstrained_array_returns" entry')
self.assertTrue(
parser.has_option(PG_ADS_3, 'unconstrained_array_returns'),
'missing "unconstrained_array_returns" entry')
self.assertTrue(
parser.has_option(PG_ADS_4, 'unconstrained_array_returns'),
'missing "unconstrained_array_returns" entry')
self.assertTrue(parser.has_option(PG_ADS_5, 'recursive_subprograms'),
'missing "recursive_subprograms" entry')
self.assertTrue(parser.has_option(PG_ADS_6, 'recursive_subprograms'),
'missing "recursive_subprograms" entry')
self.assertTrue(
parser.has_option(PGG_ADS_1, 'unconstrained_array_returns'),
'missing "unconstrained_array_returns" entry')
self.assertTrue(parser.has_option(PGG_ADS_2, 'warnings'),
'missing "warnings" entry')
self.assertTrue(parser.has_option(SIMPLE_ADB_1, 'improper_returns'),
'missing "improper_returns" entry')
self.assertTrue(parser.has_option(SIMPLE_ADB_3, 'identifier_suffixes'),
'missing "identifier_suffixes" entry')
self.assertTrue(parser.has_option(SIMPLE_ADB_3, 'identifier_prefixes'),
'missing "identifier_prefixes" entry')
def test_modelWithParams(modelName, driverName, datacollectionOnly,
randomRecall, probRecall, bufferLen, recency,
transmissionDelay):
# constant variables
automaticOpen = True
connectDirect = True
frameLen = 15
yrf = yarp.ResourceFinder()
yrf.setVerbose(True)
yrf.setDefaultContext("samSupervisor")
yrf.setDefaultConfigFile("default.ini")
yrf.configure([])
rootPath = yrf.find("root_path").asString()
name = rootPath + '/Models/' + modelName + '__' + driverName + '*.pickle'
fname = glob.glob(name)
assert len(fname) > 0, 'model file not found'
modelFileName = fname[0]
interactionConfFile = yrf.find("config_path").asString()
interactionConfFile = yrf.findFile(interactionConfFile)
# copyfile(interactionConfFile, interactionConfFile+'.bkp')
parser = SafeConfigParser()
parser.optionxform = str
parser.read(interactionConfFile)
assert modelName in parser.sections(), 'model name not in parser sections'
items = parser.items(modelName)
for j in items:
if j[0] == 'dataIn':
dataInPort = j[1].split(' ')[0]
elif j[0] == 'rpcBase':
rpcInPort = j[1]
elif j[0] == 'callSign':
askCommand = [k for k in j[1].split(',') if 'label' in k][0]
elif j[0] == 'collectionMethod':
collectionMethod = j[1].split(' ')[0]
if collectionMethod == 'continuous':
parser.set(modelName, 'collectionMethod',
'continuous ' + str(bufferLen))
elif j[0] == 'recency' and collectionMethod == 'continuous':
parser.set(modelName, 'recency', str(recency))
parser.write(open(interactionConfFile, 'wb'))
def checkRecall(rpcPort, askCommand):
# check correct response
cmBottle = yarp.Bottle()
rpBottle = yarp.Bottle()
cmBottle.addString(askCommand)
rpcPort.write(cmBottle, rpBottle)
results.append(rpBottle.toString())
return rpBottle
networkFound = yarp.Network.checkNetwork()
assert networkFound, 'Yarpserver not found'
processesList = []
# open data port
dataPortName = '/modelTest/data:o'
dataPort = yarp.BufferedPortBottle()
dataPort.open(dataPortName)
# open query port
rpcPortName = '/modelTest/rpc'
rpcPort = yarp.RpcClient()
rpcPort.open(rpcPortName)
# open interactionSAMModel
if connectDirect:
if automaticOpen:
args = ' '.join([
rootPath + '/Data/' + modelName,
modelFileName.replace('.pickle', ''), interactionConfFile,
driverName, 'False'
])
pyFile = 'interactionSAMModel.py'
interactionCMD = ' '.join([pyFile, args])
print interactionCMD
windowedCMD = "bash -c \"" + interactionCMD + "\""
interactionProcess = subprocess.Popen(['xterm', '-e', windowedCMD],
shell=False)
# wait until model loaded
time.sleep(7)
else:
# check interactionSAMModel present
pass
yarp.Network.connect(rpcPortName, rpcInPort + ':i')
else:
yarp.Network.connect(rpcPortName, '/sam/rpc:i')
# send load Actions3
# connect data to /sam/actions/actionData:i
yarp.Network.connect(dataPortName, dataInPort)
# yarp.Network.connect(dataPortName, '/reader')
# check length of data log file
dataFile = open(join('./noisyActionData/recordedData', 'data.log'), 'r')
for i, l in enumerate(dataFile):
pass
lenDataFile = i + 1
print lenDataFile
dataFile.close()
# send each line with a pause of 0.1s and query model every frameLen data points.
# check correct response and continued operation os samSupervisor
processBreak = False
dataFile = open(join('./noisyActionData/recordedData', 'data.log'), 'r')
results = []
for curr in range(lenDataFile):
line = dataFile.readline()
lineParts = line.split(' ')
line = ' '.join(lineParts[2:])
dataBottle = dataPort.prepare()
dataBottle.fromString(line)
dataPort.write()
time.sleep(transmissionDelay)
if randomRecall and random.random(
) < probRecall and not datacollectionOnly:
rpBottle = checkRecall(rpcPort, askCommand)
pollResult = interactionProcess.poll()
print "{0:.2f}".format(
curr * 100.0 / lenDataFile
) + '%\t', 'Request Response:', rpBottle.toString(), pollResult
if curr % frameLen == 0:
if not datacollectionOnly and not randomRecall:
rpBottle = checkRecall(rpcPort, askCommand)
# check reply and status of process
pollResult = interactionProcess.poll()
print "{0:.2f}".format(
curr * 100.0 / lenDataFile
) + '%\t', 'Request Response:', rpBottle.toString(), pollResult
else:
pollResult = interactionProcess.poll()
print "{0:.2f}".format(
curr * 100.0 / lenDataFile) + '%\t', pollResult
if pollResult is not None:
processBreak = True
break
dataFile.close()
try:
print 'Terminating process'
interactionProcess.send_signal(signal.SIGINT)
except:
pass
retCode = interactionProcess.wait()
if processBreak:
testResult = False
else:
testResult = True
return testResult
force=False)
else: # one db per GFF file
base_name = os.path.splitext(gff_file)[0]
db_file = base_name + normalize_filext(args['-e'])
try:
db = gff.create_db(gff_file, db_file,
merge_strategy=unique_keys,
verbose=debug, pragmas=pragmas,
force=False)
except sql.OperationalError:
raise IOError(
"Database file '{0}' already exists.".format(db_file))
except sql.IntegrityError as err:
remove_file(db_file)
raise IOError(
"{0} in database '{1}'.".format(err, db_file))
else: # rdf mode
rdf_format = args['-o']
cfg_file = args['-c']
if os.path.exists(cfg_file) is False:
raise IOError("Config file '{0}' not found.".format(cfg_file))
SafeConfigParser.validate = validate # add new method
cfg = SafeConfigParser()
cfg.optionxform = str # option names case sensitive
cfg.read(cfg_file)
cfg.validate()
gff.FeatureDB.triplify = triplify # add new method
for db_file in args['DB_FILE']:
db = gff.FeatureDB(db_file)
db.triplify(rdf_format, cfg)
def get_config(self):
""" Gets the current configuration as a SafeConfigParser from BrandServiceProxy. """
config = SafeConfigParser()
config.optionxform = str
config.readfp(StringIO(self.get_config_str()))
return config
def ReadConfig():
"""
Read the config file and set all the variables.
"""
# Read config file
cfg = SafeConfigParser()
cfg.optionxform = lambda option: option
try:
with CodecsOpen(autosub.CONFIGFILE, 'r', autosub.SYSENCODING) as f:
cfg.readfp(f)
except Exception as error:
if error.errno == 2:
print "No config found so a default config.properties is created."
Message = WriteConfig()
return
else:
print error.message
sys.exit(1)
section = 'config'
if not cfg.has_section(section):
print "No config found so a default config.properties is created."
Message = WriteConfig()
return
if cfg.has_option(section, "configversion"): autosub.CONFIGVERSION = cfg.getint("config", "configversion")
if cfg.has_option(section, "wantedfirst"): autosub.WANTEDFIRST = cfg.getboolean(section, "wantedfirst")
if cfg.has_option(section, 'downloaddutch'): autosub.DOWNLOADDUTCH = cfg.getboolean(section, 'downloaddutch')
if cfg.has_option(section, 'downloadeng'): autosub.DOWNLOADENG = cfg.getboolean(section, 'downloadeng')
if cfg.has_option(section, "fallbacktoeng"): autosub.FALLBACKTOENG = cfg.getboolean(section, "fallbacktoeng")
if cfg.has_option(section, "notifyen"): autosub.NOTIFYEN = cfg.getboolean(section, "notifyen")
if cfg.has_option(section, "notifynl"): autosub.NOTIFYNL = cfg.getboolean(section, "notifynl")
if cfg.has_option(section, "launchbrowser"): autosub.LAUNCHBROWSER = cfg.getboolean(section, "launchbrowser")
if cfg.has_option(section, "skiphiddendirs"): autosub.SKIPHIDDENDIRS = cfg.getboolean(section, "skiphiddendirs")
if cfg.has_option(section, "englishsubdelete"): autosub.ENGLISHSUBDELETE = cfg.getboolean(section, "englishsubdelete")
if cfg.has_option(section, "podnapisi"): autosub.PODNAPISI = cfg.getboolean(section, "podnapisi")
if cfg.has_option(section, "subscene"): autosub.SUBSCENE = cfg.getboolean(section, "subscene")
if cfg.has_option(section, "addic7ed"): autosub.ADDIC7ED = cfg.getboolean(section, "addic7ed")
if cfg.has_option(section, "opensubtitles"): autosub.OPENSUBTITLES = cfg.getboolean(section, "opensubtitles")
if cfg.has_option(section, "hearingimpaired"): autosub.HI = cfg.getboolean(section, "hearingimpaired")
if cfg.has_option(section, "equalmatch"): autosub.EQUALMATCH = cfg.getboolean(section, "equalmatch")
if cfg.has_option(section, 'minmatchscore'): autosub.MINMATCHSCORE = cfg.getint(section, 'minmatchscore')
if cfg.has_option(section, 'searchinterval'): autosub.SEARCHINTERVAL = cfg.getint(section, 'searchinterval')
if cfg.has_option(section, "browserrefresh"): autosub.BROWSERREFRESH = cfg.getint(section, "browserrefresh")
if cfg.has_option(section, "path"): autosub.PATH = os.path.normpath(cfg.get(section, "path"))
if cfg.has_option(section, "rootpath"): autosub.SERIESPATH = os.path.normpath(cfg.get(section, "rootpath"))
if cfg.has_option(section, "seriespath"): autosub.SERIESPATH = os.path.normpath(cfg.get(section, "seriespath"))
if autosub.SERIESPATH == '.': autosub.SERIESPATH = u''
if cfg.has_option(section, "bckpath"): autosub.BCKPATH = os.path.normpath(cfg.get(section, "bckpath"))
if autosub.BCKPATH =='.': autosub.BCKPATH = u''
if cfg.has_option(section, "subeng"): autosub.SUBENG = cfg.get(section, "subeng")
if cfg.has_option(section, "subnl"): autosub.SUBNL = cfg.get(section, "subnl")
if cfg.has_option(section, "postprocesscmd"): autosub.POSTPROCESSCMD = cfg.get(section, "postprocesscmd")
if cfg.has_option(section, "opensubtitlesuser"): autosub.OPENSUBTITLESUSER = cfg.get(section, "opensubtitlesuser")
if cfg.has_option(section, "opensubtitlespasswd"): autosub.OPENSUBTITLESPASSWD = cfg.get(section, "opensubtitlespasswd")
if cfg.has_option(section, "addic7eduser"): autosub.ADDIC7EDUSER = cfg.get(section, "addic7eduser")
if cfg.has_option(section, "addic7edpasswd"): autosub.ADDIC7EDPASSWD = cfg.get(section, "addic7edpasswd")
if cfg.has_option(section, "tvdbuser"): autosub.TVDBUSER = cfg.get(section, "tvdbuser")
if cfg.has_option(section, "tvdbaccountid"): autosub.TVDBACCOUNTID = cfg.get(section, "tvdbaccountid")
if cfg.has_option(section, "subcodec"): autosub.SUBCODEC = cfg.get(section, "subcodec")
if cfg.has_option(section,'subpermissions'):
Temp = cfg.get(section,"subpermissions")
autosub.SUBRIGHTS['owner'] = int(Temp[1:2])
autosub.SUBRIGHTS['group'] = int(Temp[2:3])
autosub.SUBRIGHTS['world'] = int(Temp[3:4])
if cfg.has_option(section, "skipstringnl"): autosub.SKIPSTRINGNL = cfg.get(section, "skipstringnl")
if cfg.has_option(section, "skipstringen"): autosub.SKIPSTRINGEN = cfg.get(section, "skipstringen")
if cfg.has_option(section, "skipfoldersnl"): autosub.SKIPFOLDERSNL = cfg.get(section, "skipfoldersnl")
if cfg.has_option(section, "skipfoldersen"): autosub.SKIPFOLDERSEN = cfg.get(section, "skipfoldersen")
# *******************
# * Logfile Section *
# *******************
section = 'logfile'
if not cfg.has_section(section): cfg.add_section(section)
autosub.LOGFILE = cfg.get(section, "logfile") if cfg.has_option(section, "logfile") else os.path.join(autosub.CONFIGPATH,autosub.LOGNAME)
if cfg.has_option(section, "logfile"): autosub.LOGFILE = cfg.get(section, "logfile")
if cfg.has_option(section, "loglevel"): autosub.LOGLEVEL = cfg.getint(section, "loglevel")
if cfg.has_option(section, "loglevelconsole"): autosub.LOGLEVELCONSOLE = cfg.getint(section, "loglevelconsole")
if cfg.has_option(section, "logsize"): autosub.LOGSIZE = cfg.getint(section, "logsize")
if cfg.has_option(section, "lognum"): autosub.LOGNUM = cfg.getint(section, "lognum")
if cfg.has_option(section, "logsize"): autosub.LOGSIZE = cfg.getint(section, "logsize")
if cfg.has_option(section, "lognum"): autosub.LOGNUM = cfg.getint(section, "lognum")
# ******************************
# * Cherrypy Webserver Section *
# ******************************
section = 'webserver'
if not cfg.has_section(section): cfg.add_section(section)
if cfg.has_option(section, 'webserverip'): autosub.WEBSERVERIP = str(cfg.get(section, 'webserverip'))
if cfg.has_option(section, 'webserverport'): autosub.WEBSERVERPORT = str(cfg.get(section, 'webserverport'))
if cfg.has_option(section, 'webroot'): autosub.WEBROOT = str(cfg.get(section, 'webroot'))
if cfg.has_option(section, 'username'): autosub.USERNAME = str(cfg.get(section, 'username'))
if cfg.has_option(section, 'password'): autosub.PASSWORD = str(cfg.get(section, 'password'))
# ********************
# * rlsgrp Section *
# ********************
section = 'releasegroup'
if not cfg.has_section(section): cfg.add_section(section)
if cfg.has_option(section, 'mustmatch'): autosub.MUSTMATCH = cfg.get(section, 'mustmatch').lower().split(',')
# ********************
# * SkipShow Section *
# ********************
section = 'skipshow'
if not cfg.has_section(section): cfg.add_section(section)
autosub.SKIPSHOWUPPER = {}
autosub.SKIPSHOW = {}
SkipShows = dict(cfg.items(section))
#autosub.SKIPSHOW = dict(cfg.items('skipshow'))
# The following 5 lines convert the skipshow to uppercase. And also convert the variables to a list
# also replace the "~" with ":" neccesary because the config parser sees ":" as a delimiter
# The UPPER version is for searching, the normal for dispaly in the user interface
for show in SkipShows:
if re.match("^[0-9 ,.-]+$", SkipShows[show]):
autosub.SKIPSHOW[show.replace('~',':')] = SkipShows[show]
autosub.SKIPSHOWUPPER[show.upper().replace('~',':')] = [Item.strip() for Item in SkipShows[show].split(',')]
# ********************************
# * Addic7ed Namemapping Section *
# ********************************
section = 'addic7edmapping'
if not cfg.has_section(section): cfg.add_section(section)
autosub.USERADDIC7EDMAPPING.clear()
try:
A7Mapping = dict(cfg.items(section))
except:
pass
for ImdbId in A7Mapping.iterkeys():
if (ImdbId.isdigit and A7Mapping[ImdbId].isdigit()):
autosub.USERADDIC7EDMAPPING[ImdbId] = int(A7Mapping[ImdbId])
# ****************************
# * User Namemapping Section *
# ****************************
section = 'namemapping'
if not cfg.has_section(section): cfg.add_section(section)
NameMapping = dict(cfg.items(section))
autosub.USERNAMEMAPPING={}
for ConfigName in NameMapping.iterkeys():
if NameMapping[ConfigName].isdigit():
Name = ConfigName.replace('~',':')
if not Name.upper() in autosub.NAMEMAPPING.keys():
autosub.NAMEMAPPING[Name.upper()] = [NameMapping[ConfigName].strip(),u'']
autosub.USERNAMEMAPPING[Name] = NameMapping[ConfigName].strip()
else:
print 'ReadConfig: Username mapping has an unknown format.',ConfigName,' = ',NameMapping[ConfigName]
# ******************
# * Notify Section *
# ******************
section = 'notify'
if not cfg.has_section(section): cfg.add_section(section)
if cfg.has_option(section, 'notifymail'): autosub.NOTIFYMAIL = cfg.getboolean(section, 'notifymail')
if cfg.has_option(section, 'mailsrv'): autosub.MAILSRV = cfg.get(section, 'mailsrv')
if cfg.has_option(section, 'mailfromaddr'): autosub.MAILFROMADDR = cfg.get(section, 'mailfromaddr')
if cfg.has_option(section, 'mailtoaddr'): autosub.MAILTOADDR = cfg.get(section, 'mailtoaddr')
if cfg.has_option(section, 'mailusername'): autosub.MAILUSERNAME = cfg.get(section, 'mailusername')
if cfg.has_option(section, 'mailpassword'): autosub.MAILPASSWORD = cfg.get(section, 'mailpassword')
if cfg.has_option(section, 'mailsubject'): autosub.MAILSUBJECT = cfg.get(section, 'mailsubject')
if cfg.has_option(section, 'mailencryption'): autosub.MAILENCRYPTION = cfg.get(section, 'mailencryption')
if cfg.has_option(section, 'mailauth'): autosub.MAILAUTH = cfg.get(section, 'mailauth')
if cfg.has_option(section, 'notifygrowl'): autosub.NOTIFYGROWL = cfg.getboolean(section, 'notifygrowl')
if cfg.has_option(section, 'growlhost'): autosub.GROWLHOST = cfg.get(section, 'growlhost')
if cfg.has_option(section, 'growlport'): autosub.GROWLPORT = cfg.get(section, 'growlport')
if cfg.has_option(section, 'growlpass'): autosub.GROWLPASS = cfg.get(section, 'growlpass')
if cfg.has_option(section, 'notifytwitter'): autosub.NOTIFYTWITTER = cfg.getboolean(section, 'notifytwitter')
if cfg.has_option(section, 'twitterkey'): autosub.TWITTERKEY = cfg.get(section, 'twitterkey')
if cfg.has_option(section, 'twittersecret'): autosub.TWITTERSECRET = cfg.get(section, 'twittersecret')
if cfg.has_option(section, 'notifynma'): autosub.NOTIFYNMA = cfg.getboolean(section, 'notifynma')
if cfg.has_option(section, 'nmaapi'): autosub.NMAAPI = cfg.get(section, 'nmaapi')
if cfg.has_option(section, 'nmapriority'): autosub.NMAPRIORITY = cfg.getint(section, 'nmapriority')
if cfg.has_option(section, 'notifyprowl'): autosub.NOTIFYPROWL = cfg.getboolean(section, 'notifyprowl')
if cfg.has_option(section, 'prowlapi'): autosub.PROWLAPI = cfg.get(section, 'prowlapi')
if cfg.has_option(section, 'prowlpriority'): autosub.PROWLPRIORITY = cfg.getint(section, 'prowlpriority')
if cfg.has_option(section, 'notifytelegram'): autosub.NOTIFYTELEGRAM = cfg.getboolean(section, 'notifytelegram')
if cfg.has_option(section, 'telegramapi'): autosub.TELEGRAMAPI = cfg.get(section, 'telegramapi')
if cfg.has_option(section, 'telegramid'): autosub.TELEGRAMID = cfg.get(section, 'telegramid')
if cfg.has_option(section, 'notifypushalot'): autosub.NOTIFYPUSHALOT = cfg.getboolean(section, 'notifypushalot')
if cfg.has_option(section, 'pushalotapi'): autosub.PUSHALOTAPI = cfg.get(section, 'pushalotapi')
if cfg.has_option(section, 'notifypushbullet'): autosub.NOTIFYPUSHBULLET = cfg.getboolean(section, 'notifypushbullet')
if cfg.has_option(section, 'pushbulletapi'): autosub.PUSHBULLETAPI = cfg.get(section, 'pushbulletapi')
if cfg.has_option(section, 'notifypushover'): autosub.NOTIFYPUSHOVER = cfg.getboolean(section, 'notifypushover')
if cfg.has_option(section, 'pushoverappkey'): autosub.PUSHOVERAPPKEY = cfg.get(section, 'pushoverappkey')
if cfg.has_option(section, 'pushoveruserkey'): autosub.PUSHOVERUSERKEY = cfg.get(section, 'pushoveruserkey')
if cfg.has_option(section, 'pushoverpriority'): autosub.PUSHOVERPRIORITY = cfg.getint(section, 'pushoverpriority')
if cfg.has_option(section, 'notifyboxcar2'): autosub.NOTIFYBOXCAR2 = cfg.getboolean(section, 'notifyboxcar2')
if cfg.has_option(section, 'boxcar2token'): autosub.BOXCAR2TOKEN = cfg.get(section, 'boxcar2token')
if cfg.has_option(section, 'notifyplex'): autosub.NOTIFYPLEX = cfg.getboolean(section, 'notifyplex')
if cfg.has_option(section, 'plexserverhost'): autosub.PLEXSERVERHOST = cfg.get(section, 'plexserverhost')
if cfg.has_option(section, 'plexserverport'): autosub.PLEXSERVERPORT = cfg.get(section, 'plexserverport')
if cfg.has_option(section, 'plexserverusername'): autosub.PLEXSERVERUSERNAME = cfg.get(section, 'plexserverusername')
if cfg.has_option(section, 'plexserverpassword'): autosub.PLEXSERVERPASSWORD = cfg.get(section, 'plexserverpassword')
if cfg.has_option(section, 'notifykodi'): autosub.NOTIFYKODI = cfg.getboolean(section, 'notifykodi')
if cfg.has_option(section, 'kodiserverhost'): autosub.KODISERVERHOST = cfg.get(section, 'kodiserverhost')
if cfg.has_option(section, 'kodiserverport'): autosub.KODISERVERPORT = cfg.get(section, 'kodiserverport')
if cfg.has_option(section, 'kodiserverusername'): autosub.KODISERVERUSERNAME = cfg.get(section, 'kodiserverusername')
if cfg.has_option(section, 'kodiserverpassword'): autosub.KODISERVERPASSWORD = cfg.get(section, 'kodiserverpassword')
if cfg.has_option(section, 'kodiupdateonce'): autosub.KODIUPDATEONCE = cfg.getboolean(section, 'kodiupdateonce')
autosub.MINMATCHDSP = '{0:05b}'.format(autosub.MINMATCHSCORE).replace('1','X').replace('0','-')
# Here we check whether the config has been upgraded
# if so we make the changes and write the config back to the config file
autosub.CONFIGVERSION = cfg.getint("config", "configversion") if cfg.has_option("config", "configversion") else 0
if autosub.CONFIGVERSION < autosub.version.configversion:
_upgradeConf(cfg, autosub.CONFIGVERSION, autosub.version.configversion)
elif autosub.CONFIGVERSION > autosub.version.configversion:
print "Config: ERROR! Config version higher then this version of AutoSub supports. Update AutoSub!"
os._exit(1)
def parseConfigFile(filename, preserve_case=True):
parser = SafeConfigParser()
if preserve_case:
parser.optionxform = str
parser.read(filename)
return parser
def _call_parser(conf_file):
parser = SafeConfigParser()
parser.optionxform = str # to preserve case
parser.read(conf_file)
return parser
def from_file(cls, cfgfile):
cfg = SafeConfigParser(allow_no_value=True)
cfg.optionxform = str
cfg.read(cfgfile)
return cls(cfg)
from subprocess import check_call
from ConfigParser import SafeConfigParser
from time import sleep
ap = ArgumentParser()
ap.add_argument("--configFile", action="store", required=True,
help='Configuration file containing arguments')
ap.add_argument("--numJobs", action="store", type=int, required=True,
help='Number of jobs into which run is split')
ap.add_argument("--jobNum", action="store", type=int, required=True,
help='Number of this job')
args = ap.parse_args()
cp = SafeConfigParser()
cp.optionxform = str
cp.read(args.configFile)
fMin = cp.getfloat('param-space','f_min')
fMax = cp.getfloat('param-space','f_max')
fFullBand = fMax-fMin
fBand = fFullBand / args.numJobs
fStart = fMin + args.jobNum * fBand
toplistPattern = cp.get('filename-patterns','toplist_name')
toplistName = toplistPattern % (args.jobNum,args.numJobs)
logfilePattern = cp.get('filename-patterns','logfile_name')
logfileName = logfilePattern % (args.jobNum,args.numJobs)
# Pass along the arguments from the ini file
def get_gconfig():
parser = SafeConfigParser()
parser.optionxform = str
parser.read(os.path.join(CONFIG.system_path, 'game.cfg'))
return parser
