def loadWebConf():
""" Load configuration parameters to be used in web.
By default read from: scipion.conf
"""
# Read menus from users' config file.
cp = ConfigParser()
cp.optionxform = str # keep case (stackoverflow.com/questions/1611799)
webConf = OrderedDict()
confFile = os.environ['SCIPION_CONFIG']
assert cp.read(confFile) != [], 'Missing file %s' % confFile
# Set options from WEB main section
def setw(option, default):
if cp.has_option('WEB', option):
webConf[option] = cp.get('WEB', option)
else:
webConf[option] = default
setw('SITE_URL', 'scipion.cnb.csic.es')
setw('ABSOLUTE_URL', '')
# Load some parameters per protocol
protocols = OrderedDict()
webConf['PROTOCOLS'] = protocols
if cp.has_section('WEB_PROTOCOLS'):
for protName in cp.options('WEB_PROTOCOLS'):
protocols[protName] = json.loads(cp.get('WEB_PROTOCOLS', protName))
return webConf
def getDataSources(fName = None):
''' return data sources directories for this machine.
directories are defined in datasources.ini or provided filepath'''
import socket
from ConfigParser import ConfigParser
pcName = socket.gethostname()
p = ConfigParser()
p.optionxform = str
if fName is None:
fName = 'datasources.ini'
p.read(fName)
if pcName not in p.sections():
raise NameError('Host name section %s not found in file %s' %(pcName,fName))
dataSources = {}
for option in p.options(pcName):
dataSources[option] = p.get(pcName,option)
return dataSources
def parse_config_file(config_type, config_filename):
from ConfigParser import ConfigParser
config_dic = {}
if not os.path.exists(config_filename):
print "%s: Error: %s config file does not exist [%s]" % (time.strftime("%H:%M:%S"), config_type, config_filename)
config_dic['CONFIG_EXISTS'] = False
else:
config_dic['CONFIG_EXISTS'] = True
# get global config values
parser_dic = ConfigParser()
parser_dic.optionxform = str
parser_dic.read(config_filename)
if DEBUG:
print "\t>> %s Config file: %s" % (config_type, config_filename)
# print "\t >> %s Sections %s" % (config_type, parser_dic.sections())
for section_name in parser_dic.sections():
if DEBUG:
print '\t >> %s Section: %s' % (config_type, section_name)
# print '\t >> %s Options: %s' % (config_type, parser_dic.options(section_name))
for name, value in parser_dic.items(section_name):
if DEBUG:
print '\t >> %s Value: %s = %s' % (config_type, name, value)
config_dic[name] = value
return config_dic
def fromIniFile(iniFilename, sectionName):
iniFilename = op.abspath(op.expanduser(iniFilename))
def resolvePath(path):
"""
Path may be given
- as relative to the ini file path
- qualified with "~"
Resolve it whatever it is.
None is passed through.
"""
if path is None: return None
iniDirectory = op.dirname(iniFilename)
path = op.expanduser(path)
if op.isabs(path):
return path
else:
return op.abspath(op.join(iniDirectory, path))
cp = ConfigParser()
cp.optionxform=str
cp.read(iniFilename)
opts = dict(cp.items(sectionName))
return Fixture(trcFname=resolvePath(opts.get("Traces")),
plsFname=resolvePath(opts.get("Pulses")),
basFname=resolvePath(opts.get("Bases")),
refFname=resolvePath(opts.get("Reference")),
alnFname=resolvePath(opts.get("Alignments")))
def loadEmailConf():
""" Load configuration parameters to be used for emailing.
By default read from: scipion.conf
"""
# Read menus from users' config file.
cp = ConfigParser()
cp.optionxform = str # keep case (stackoverflow.com/questions/1611799)
emailConf = OrderedDict()
confFile = os.environ['SCIPION_CONFIG']
assert cp.read(confFile) != [], 'Missing file %s' % confFile
# Set options from EMAIL main section
def setm(option, default):
if cp.has_option('EMAIL', option):
emailConf[option] = cp.get('EMAIL', option)
else:
emailConf[option] = default
setm('EMAIL_HOST', 'localhost')
setm('EMAIL_PORT', '25')
setm('EMAIL_HOST_USER', 'smtpuser')
setm('EMAIL_HOST_PASSWORD', 'smtppassword')
setm('EMAIL_USE_TLS', False)
setm('EMAIL_FOR_SUBSCRIPTION', '*****@*****.**')
return emailConf
def config(self, key=None, default=u""):
"""
Return the configuration file location or values of keys.
"""
# First try a direct environment variable.
try: fn = os.environ["IMPERIAL_CONFIG"]
except KeyError:
# Next, are we on Windows?
if SHGetFolderPath: fn = SHGetFolderPath(0, CSIDL_APPDATA, None, 0) + "\\imperial.ini"
# Or Linux? (God I hope this works on Mac, too!)
elif xdg_config_home: fn = os.path.join(xdg_config_home, ".imperial")
# Something else..? (Bonus points: it even fits 8.3)
else: fn = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), "imperial" + os.extsep + "ini")
#endtry
if key is None: return fn
else:
config = ConfigParser()
config.optionxform = str
config.read(fn)
if type(key) in [str, unicode]:
try: ret = config.get("Imperial", key)
except (NoSectionError, NoOptionError): ret = default
else:
ret = {}
for x in key:
try: ret[x] = (config.get("Imperial", x))
except (NoSectionError, NoOptionError):
# Passed defaults by dict.
try: ret[x] = key[x]
# No explicit defaulting.
except TypeError: ret[x] = u""
#endtry
#endfor
return ret
def __init__(self, config_file='config.ini'):
parser = ConfigParser()
parser.optionxform=str
parser.read(config_file)
self.config = {}
for section in parser.sections():
self.config[section] = {}
for key, val in parser.items(section):
if section == "Aggregations":
try:
val = json.loads(val)
except:
raise Exception(
"Could not parse aggregation {}:\n{}".format(
key,
val
)
)
self.config[section][key] = val
if not 'Authentication' in self.config:
raise Exception("Configuration missing Authentication section")
elif not 'auth_token' in self.config['Authentication']:
raise Exception("Authentication section missing auth_token")
elif len(str(self.config['Authentication']['auth_token'])) < 30:
raise Exception("Invalid auth_token")
self.hipchat = hypchat.HypChat(
self.config['Authentication']['auth_token']
)
self.get_self()
self.make_rooms()
def main(opts):
config = ConfigParser()
config.optionxform=str
if opts.ini == '':
print 'ERROR: You must specify a valid ini file path. Use -h for help'
return
if not os.path.isfile(opts.ini):
print 'ERROR: Specified ini file is not found'
return
config.read(opts.ini)
utils.setOptions(opts.options, config)
print utils.showOptions(config, ['General', 'DB', 'Load', 'Query'])
# Set logging
logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s', datefmt=utils.DEFAULT_TIMEFORMAT, level=getattr(logging, config.get('General','LogLevel')))
if not opts.show:
loadermodule = config.get('General','Loader')
loadername = loadermodule.split('.')[-1]
exec 'from ' + loadermodule + ' import ' + loadername
loader = getattr(sys.modules[loadermodule], loadername)(config)
# Loads data into DB
loader.run()
def loadINI(f):
parser = ConfigParser()
parser.optionxform = str
parser.read(f)
for section in parser.sections():
yield section, ((k, parse_value(v))
for k, v in parser.items(section, raw=True) if v != '')
def read(self, filename):
# clean the sections
for section in self.keys():
del self[section]
from ConfigParser import ConfigParser
config = ConfigParser()
config.optionxform = str
config.read(filename)
for this in config.sections():
sec = eval("Params" + this + "()")
options = config.options(this)
for option in options:
value = config.get(this, option)
# HERE, we need to interpret the content of the config file.
# values could be int or float, in which case we want to cast the string
# to a float. If the string is True/False, we want also a cast
try:
try:
value = int(value)
except:
value = float(value)
except:
if isinstance(value, str):
if value == 'True':
value = True
elif value == 'False':
value = False
elif value.strip() == '':
value = None
setattr(getattr(sec, option), 'value', value)
self[this] = sec
return config
def loadEmailConf():
""" Load configuration parameters to be used for emailing.
By default read from: scipion.conf
"""
# Read menus from users' config file.
cp = ConfigParser()
cp.optionxform = str # keep case (stackoverflow.com/questions/1611799)
emailConf = OrderedDict()
confFile = os.environ['SCIPION_CONFIG']
assert cp.read(confFile) != [], 'Missing file %s' % confFile
# Set options from EMAIL main section
def setm(option, default):
if cp.has_option('EMAIL', option):
emailConf[option] = cp.get('EMAIL', option)
else:
emailConf[option] = default
setm('EMAIL_HOST', 'localhost')
setm('EMAIL_PORT', '25')
setm('EMAIL_HOST_USER', 'smtpuser')
setm('EMAIL_HOST_PASSWORD', 'smtppassword')
setm('EMAIL_USE_TLS', False)
setm('EMAIL_FOR_SUBSCRIPTION', '*****@*****.**')
return emailConf
def read(self, policy):
ret = False
inftable = self.apply_map()
current_section = None
# So here we would declare a boolean,
# that would get changed to TRUE.
#
# If at any point in time a GPO was applied,
# then we return that boolean at the end.
inf_conf = ConfigParser()
inf_conf.optionxform = str
try:
inf_conf.readfp(StringIO(policy))
except:
inf_conf.readfp(StringIO(policy.decode('utf-16')))
for section in inf_conf.sections():
current_section = inftable.get(section)
if not current_section:
continue
for key, value in inf_conf.items(section):
if current_section.get(key):
(att, setter) = current_section.get(key)
value = value.encode('ascii', 'ignore')
ret = True
setter(self.logger, self.ldb, self.gp_db, self.lp, att,
value).update_samba()
self.gp_db.commit()
return ret
def from_file(cls, file_name):
""" Make a Config from a file. """
file_name = os.path.expanduser(file_name)
parser = ConfigParser()
# IMPORTANT: Turn off downcasing of option names.
parser.optionxform = str
parser.read(file_name)
cfg = Config()
if parser.has_section('index_values'):
for repo_id in parser.options('index_values'):
cfg.version_table[repo_id] = (parser.getint(
'index_values', repo_id))
if parser.has_section('request_usks'):
for repo_dir in parser.options('request_usks'):
cfg.request_usks[repo_dir] = parser.get(
'request_usks', repo_dir)
if parser.has_section('insert_usks'):
for repo_id in parser.options('insert_usks'):
cfg.insert_usks[repo_id] = parser.get('insert_usks', repo_id)
# ignored = fms_id|usk_hash|usk_hash|...
if parser.has_section('fmsread_trust_map'):
cfg.fmsread_trust_map.clear() # Wipe defaults.
for ordinal in parser.options('fmsread_trust_map'):
fields = parser.get('fmsread_trust_map',
ordinal).strip().split('|')
Config.validate_trust_map_entry(cfg, fields)
cfg.fmsread_trust_map[fields[0]] = tuple(fields[1:])
Config.update_defaults(parser, cfg)
cfg.file_name = file_name
return cfg
def createDictFromConfig():
""" Read the configuration from scipion/config/scipionbox.conf.
A dictionary will be created where each key will be a section starting
by MICROSCOPE:, all variables that are in the GLOBAL section will be
inherited by default.
"""
# Read from users' config file.
confGlobalDict = OrderedDict()
confDict = OrderedDict()
cp = ConfigParser()
cp.optionxform = str # keep case (stackoverflow.com/questions/1611799)
confFile = pw.getConfigPath("scipionbox.conf")
print "Reading conf file: ", confFile
cp.read(confFile)
GLOBAL = "GLOBAL"
MICROSCOPE = "MICROSCOPE:"
if not GLOBAL in cp.sections():
raise Exception("Missing section %s in %s" % (GLOBAL, confFile))
else:
for opt in cp.options(GLOBAL):
confGlobalDict[opt] = cp.get(GLOBAL, opt)
for section in cp.sections():
if section != GLOBAL and section.startswith(MICROSCOPE):
sectionDict = OrderedDict(confGlobalDict)
for opt in cp.options(section):
sectionDict[opt] = cp.get(section, opt)
confDict[section.replace(MICROSCOPE, '')] = sectionDict
return confDict
def fromIniFile(iniFilename, sectionName):
iniFilename = op.abspath(op.expanduser(iniFilename))
def resolvePath(path):
"""
Path may be given
- as relative to the ini file path
- qualified with "~"
Resolve it whatever it is.
None is passed through.
"""
if path is None: return None
iniDirectory = op.dirname(iniFilename)
path = op.expanduser(path)
if op.isabs(path):
return path
else:
return op.abspath(op.join(iniDirectory, path))
cp = ConfigParser()
cp.optionxform = str
cp.read(iniFilename)
opts = dict(cp.items(sectionName))
return Fixture(trcFname=resolvePath(opts.get("Traces")),
plsFname=resolvePath(opts.get("Pulses")),
basFname=resolvePath(opts.get("Bases")),
refFname=resolvePath(opts.get("Reference")),
alnFname=resolvePath(opts.get("Alignments")))
def get_log_aggregation_Dir_Locations(cls):
'''
Gets base dir to for log aggregation.
Returns a tuple of (str, str, str, str).
Returns a tuple of (cluster name, temporary cluster dir for logs,
temporary dir for applications, test component)
'''
config = ConfigParser()
reportconf = os.path.join(Config.getEnv('ARTIFACTS_DIR'),
'test_report.conf')
SECTION = "HW-QE-PUBLISH-REPORT"
config.optionxform = str
config.read(reportconf)
CLUSTER_NAME = config.get(SECTION, "CLUSTER_NAME")
logUtilCloudbreak.LOCAL_TMP_CLUSTER_DIR = os.path.join(
Config.getEnv('ARTIFACTS_DIR'), CLUSTER_NAME)
logUtilCloudbreak.LOCAL_TMP_APP_STORAGE = os.path.join(
cls.LOCAL_TMP_CLUSTER_DIR, "Application-logs")
logUtilCloudbreak.COMPONENT = ''
if config.has_option(SECTION, 'TESTSUITE_COMPONENT'):
logUtilCloudbreak.COMPONENT = config.get(SECTION,
'TESTSUITE_COMPONENT')
logger.info("Set logUtilCloudbreak.COMPONENT to %s",
logUtilCloudbreak.COMPONENT)
return (CLUSTER_NAME, logUtilCloudbreak.LOCAL_TMP_CLUSTER_DIR,
logUtilCloudbreak.LOCAL_TMP_APP_STORAGE,
logUtilCloudbreak.COMPONENT)
def dict_conf(filename):
"""
Return dict object for *.conf file
"""
f, ext = os.path.splitext(filename)
ext = ext.lower()
if ext == "conf" or ext == "ini":
# python config via config parser
config = ConfigParser()
config.optionxform=str
config.read(filename)
rv = {}
for section in config.sections():
rv[section] = {}
for key,value in config.items(section):
rv[section][key] = value.strip('"').strip("'").decode("string_escape")
return rv
else:
# other type of config, use munge
if munge_config:
src = munge_config.parse_url(filename)
return src.cls().load(open(filename)).get("vodka")
else:
raise Exception("'%s' type of config encountered, install munge" % ext)
def from_file(cls, file_name):
""" Make a Config from a file. """
file_name = os.path.expanduser(file_name)
parser = ConfigParser()
# IMPORTANT: Turn off downcasing of option names.
parser.optionxform = str
parser.read(file_name)
cfg = Config()
if parser.has_section('index_values'):
for repo_id in parser.options('index_values'):
cfg.version_table[repo_id] = (
parser.getint('index_values', repo_id))
if parser.has_section('request_usks'):
for repo_dir in parser.options('request_usks'):
cfg.request_usks[repo_dir] = parser.get('request_usks',
repo_dir)
if parser.has_section('insert_usks'):
for repo_id in parser.options('insert_usks'):
cfg.insert_usks[repo_id] = parser.get('insert_usks', repo_id)
# ignored = fms_id|usk_hash|usk_hash|...
if parser.has_section('fmsread_trust_map'):
cfg.fmsread_trust_map.clear() # Wipe defaults.
for ordinal in parser.options('fmsread_trust_map'):
fields = parser.get('fmsread_trust_map',
ordinal).strip().split('|')
Config.validate_trust_map_entry(cfg, fields)
cfg.fmsread_trust_map[fields[0]] = tuple(fields[1:])
Config.update_defaults(parser, cfg)
cfg.file_name = file_name
return cfg
def read_config(fn):
'''Return a case sensitive ConfigParser by reading the file "fn"'''
cp = ConfigParser()
cp.optionxform = str
cp.read(fn)
return cp
def __init__(self,options):
import sys, os
from ConfigParser import ConfigParser
self.options = options
CNF = ConfigParser()
CNF.optionxform=str # prevent script to ignore cases
CNF.read( options.filename ) # reading config file
# - If section [general settings] is missing or
# section [parameters] missing: stop
if not CNF.has_section("general settings"):
sys.exit("ERROR: file does not contain required [general settings] section!")
elif not CNF.has_section("parameters"):
sys.exit("ERROR: file does not contain required [parameters] section!")
# - Else areading [general settings] section, we need
# three arguments which is 'city','user','password'. If
# at least one is missing: stop.
required = ['url','user','password','city']
for req in required:
if not CNF.has_option('general settings',req):
sys.exit("ERROR: option \"%s\" in section [general settings] not defined." % req)
# - Read these 3 strings
self.url = CNF.get('general settings','url')
self.user = CNF.get('general settings','user')
self.password = CNF.get('general settings','password')
self.city = CNF.get('general settings','city')
# - If logfile is defined: take argument. Else
# logfile is just options.filename+"log"
if not CNF.has_option('general settings','logfile'):
self.logfile = "%s.log" % options.filename
else:
self.logfile = CNF.get('general settings','logfile')
self.logfile = self.logfile.replace("%city%",self.city)
self.logfile = self.logfile.replace("%user%",self.user)
# - Now reading parameters
self.data = {}
params = CNF.items('parameters')
for rec in params:
self.data[rec[0]] = CNF.getfloat('parameters',rec[0])
# - Create options string
self.args = "city=%s&user=%s&password=%s" % (self.city,self.user,self.password)
for key, val in self.data.iteritems():
self.args += "&%s=%.1f" % (key,val)
# - Create list object wihch can be used in subprocess.
self.cmd = ['wget',self.url,'--post-data',self.args,'-O',self.logfile]
# - Create command string (for copy and paste users)
# For wget with POST request
self.cmdstring = "wget %s --post-data \"%s\" -O %s" % (self.url,self.args,self.logfile)
# - GET request
if not self.url[-1] == "/":
self.geturl = "%s/?%s" % (self.url,self.args)
else:
self.geturl = "%s?%s" % (self.url,self.args)
def parse_config(self, config_file):
"""Parse config file."""
# Transform initial "=" sign to "#" (comment),
# then all "|" signs to "="
conf_str =\
open(config_file, 'r').read().replace('=', '#').replace('|', '=')
# Prepend a [root] namespace for ConfigParser compatibility
conf_str = '[root]\n' + conf_str
conf_fp = StringIO(conf_str)
cfg = ConfigParser()
# Keep variable names in capital letters
cfg.optionxform = str
cfg.readfp(conf_fp)
self.conf = dict(cfg.items('root'))
self.conf['soil_conditions'] = False
self.lon0 = float(self.conf['MAP_XYLIM'].split(',')[0])
self.lon1 = float(self.conf['MAP_XYLIM'].split(',')[1])
self.lat0 = float(self.conf['MAP_XYLIM'].split(',')[2])
self.lat1 = float(self.conf['MAP_XYLIM'].split(',')[3])
legend_locations = {
'LL': 'lower left',
'LR': 'lower right',
'UL': 'upper left',
'UR': 'upper right',
}
self.legend_loc = legend_locations[self.conf['MAP_LEGEND_LOC']]
try:
self.colorbar_bcsf = bool(self.conf['COLORBAR_BCSF'])
except Exception:
self.colorbar_bcsf = False
def test_read_init(self):
"""Test that the plugin __init__ will validate on plugins.qgis.org."""
# You should update this list according to the latest in
# https://github.com/qgis/qgis-django/blob/master/qgis-app/
# plugins/validator.py
required_metadata = [
'name', 'description', 'version', 'qgisMinimumVersion', 'email',
'author'
]
file_path = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, 'metadata.txt'))
LOGGER.info(file_path)
metadata = []
parser = ConfigParser()
parser.optionxform = str
parser.read(file_path)
message = 'Cannot find a section named "general" in %s' % file_path
assert parser.has_section('general'), message
metadata.extend(parser.items('general'))
for expectation in required_metadata:
message = ('Cannot find metadata "%s" in metadata source (%s).' %
(expectation, file_path))
self.assertIn(expectation, dict(metadata), message)
def downloadTweets(username, downloadNewerThanId=-1, downloadOlderThanId=999999999999999999):
highestIdDownloaded = 0
storedInfo = ConfigParser()
storedInfo.optionxform = str #Makes sure options preserve their case. Prolly breaks something down the line, but CASE!
twitterInfoFilename = os.path.join(GlobalStore.scriptfolder, 'data', 'TwitterInfo.dat')
if os.path.exists(twitterInfoFilename):
storedInfo.read(twitterInfoFilename)
if not storedInfo.has_section(username):
storedInfo.add_section(username)
if storedInfo.has_option(username, "highestIdDownloaded"):
highestIdDownloaded = int(storedInfo.get(username, "highestIdDownloaded"))
headers = {"Authorization": "{} {}".format(GlobalStore.commandhandler.apikeys.get('twitter', 'tokentype'), GlobalStore.commandhandler.apikeys.get('twitter', 'token'))}
params = {"screen_name": username, "count": "200", "trim_user": "******", "exclude_replies": "true", "include_rts": "false"}
if downloadNewerThanId > -1:
params["since_id"] = downloadNewerThanId
tweets = {}
lowestIdFound = downloadOlderThanId
newTweetsFound = True
while newTweetsFound:
params["max_id"] = lowestIdFound
req = requests.get("https://api.twitter.com/1.1/statuses/user_timeline.json", headers=headers, params=params)
apireply = json.loads(req.text)
newTweetsFound = False
for tweet in apireply:
tweettext = tweet["text"].replace("\n", " ").encode(encoding="utf-8", errors="replace")
#print "Tweet {}: {}".format(tweet["id"], tweettext)
if tweet["id"] not in tweets:
#print " storing tweet"
newTweetsFound = True
tweets[tweet["id"]] = tweettext
tweetId = int(tweet["id"])
lowestIdFound = min(lowestIdFound, tweetId-1)
highestIdDownloaded = max(highestIdDownloaded, tweetId)
#else:
# print " skipping duplicate tweet"
#All tweets downloaded. Time to process them
tweetfile = open(os.path.join(GlobalStore.scriptfolder, 'data', "tweets-{}.txt".format(username)), "a")
#Sort the keys before saving, so we're writing from oldest to newest, so in the same order as the Twitter timeline (Not absolutely necessary, but it IS neat and tidy)
for id in sorted(tweets.keys()):
tweetfile.write(tweets[id] + "\n")
tweetfile.close()
storedInfo.set(username, "highestIdDownloaded", highestIdDownloaded)
linecount = 0
if storedInfo.has_option(username, "linecount"):
linecount = storedInfo.getint(username, "linecount")
linecount += len(tweets)
storedInfo.set(username, "linecount", linecount)
storedInfoFile = open(twitterInfoFilename, "w")
storedInfo.write(storedInfoFile)
storedInfoFile.close()
return True
def saveObject(self, _filename):
configparser = ConfigParser()
configparser.optionxform = str
configparser.add_section('general')
configparser.set('general', 'type', self.type)
configparser.set('general', 'identification', self.identification)
configparser.add_section(self.identification)
for k, l in self.getThermoInputDict().iteritems():
configparser.set(self.identification, k, l[0])
for k, l in self.getAeroInputDict().iteritems():
configparser.set(self.identification, k, l[0])
for k, l in self.getThermoOutputDict().iteritems():
configparser.set(self.identification, k, l[0])
for k, l in self.getAeroOutputDict().iteritems():
configparser.set(self.identification, k, l[0])
for i in self.subcomponentList:
dummy = getattr(self, i[0])
configparser.add_section(dummy.identification)
for k, l in dummy.getThermoInputDict().iteritems():
configparser.set(dummy.identification, k, l[0])
for k, l in dummy.getAeroInputDict().iteritems():
configparser.set(dummy.identification, k, l[0])
for k, l in dummy.getThermoOutputDict().iteritems():
configparser.set(dummy.identification, k, l[0])
for k, l in dummy.getAeroOutputDict().iteritems():
configparser.set(dummy.identification, k, l[0])
with open(_filename, 'wb') as configfile:
configparser.write(configfile)
def prepareExport(self,fName='migration.ini'):
"""
create an ini file that is used for exporting files.
file is saved in the root folder
"""
# create export mapping (source->dest). this is a multiline string
mapping = '\n'
for oldPath in self.folders():
mapping+='->'.join((oldPath, newPath(oldPath,self.dateRange(oldPath))))+'\n'
p = ConfigParser()
p.optionxform = str
p.add_section('Export')
p.set('Export','root',self.root)
p.set('Export','dest',None)
p.set('Export','mapping',mapping)
self.log.info( 'Writing '+fName)
with open(fName,'w') as fid:
p.write(fid)
def getAliasModuleName(cls, moduleName, configFile, logoutput=False): # pylint: disable=unused-argument
'''
Gets alias module name if a user specifies an entry in alias file.
Returns a String.
'''
if cls.cachedConfigAliasModuleName is None:
cls.cachedConfigAliasModuleName = {}
try:
config = ConfigParser()
config.optionxform = str #make it case sensitive
config.read(configFile)
for section in config.sections():
nameValueList = config.items(section)
for (name, value) in nameValueList:
cls.cachedConfigAliasModuleName[name] = value
except Exception as ex:
logger.info(
"Exception is thrown when reading file in getAliasModuleName. %s",
ex)
finally:
#logger.info("config alias module name = ", cls.cachedConfigAliasModuleName)
pass
if moduleName in cls.cachedConfigAliasModuleName:
moduleName = cls.cachedConfigAliasModuleName[moduleName]
return moduleName
def confFileFromDB(gt, gtConfFileName, gtConnString, authpath):
# check if the original conf file exists
if not os.path.isfile(gtConfFileName):
# get the conf file from DB
print "Getting the conf file for GT: " + gt + " from DB....be patinet!"
dbtoconf_cfg = ConfigParser()
dbtoconf_cfg.optionxform = str
dbtoconf_cfg.add_section("Common")
dbtoconf_cfg.set("Common","Account","CMS_COND_31X_GLOBALTAG")
#dbtoconf_cfg.set("Common","Conn_string_gtag","frontier://cmsfrontier:8000/FrontierProd/CMS_COND_31X_GLOBALTAG")
dbtoconf_cfg.set("Common","Conn_string_gtag",gtConnString)
dbtoconf_cfg.set("Common","AuthPath",authpath)
dbtoconf_cfg.set("Common","Globtag",gt)
dbtoconf_cfg.set("Common","Confoutput",gtConfFileName)
dbtoconf_file = open("dbtoconf.cfg", 'wb')
dbtoconf_cfg.write(dbtoconf_file)
dbtoconf_file.close()
#dbtoconf_cmd = 'eval `scram runtime -csh`; dbtoconf.py'
dbtoconf_cmd = 'dbtoconf.py'
statusAndOutput = commands.getstatusoutput(dbtoconf_cmd)
if statusAndOutput[0] != 0:
print statusAndOutput[1]
# check again
if not os.path.isfile(gtConfFileName):
return False
return True
def get_conf_stanzas(conf_name):
'''Get stanzas of `conf_name`
:param conf_name: Config file.
:type conf_name: ``string``
:returns: Config stanzas.
:rtype: ``dict``
Usage::
>>> stanzas = get_conf_stanzas('server')
>>> return: {'serverName': 'testServer', 'sessionTimeout': '1h', ...}
'''
if conf_name.endswith('.conf'):
conf_name = conf_name[:-5]
# TODO: dynamically caculate SPLUNK_HOME
btool_cli = [op.join(os.environ['SPLUNK_HOME'], 'bin', 'btool'),
conf_name, 'list']
p = subprocess.Popen(btool_cli,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, _ = p.communicate()
out = StringIO(out)
parser = ConfigParser()
parser.optionxform = str
parser.readfp(out)
out = {}
for section in parser.sections():
out[section] = {item[0]: item[1] for item in parser.items(section)}
return out
def read_rss_feeds_ini(rss_ini, verbose=False):
sources = []
if rss_ini is None:
rss_ini = os.getenv("NB_RSS", None)
if rss_ini is None:
return sources
else:
from ConfigParser import ConfigParser
parser = ConfigParser(allow_no_value=True)
parser.optionxform = str
parser.read(rss_ini)
for source_url in parser.sections():
http_source_url = "http://" + source_url
if verbose:
print u"SOURCE:", http_source_url, u"\nFEEDS:"
source = Source(http_source_url)
sources.append(source)
for rss_url, _ in parser.items(source_url):
http_rss_url = "http://" + rss_url
if verbose:
print u"\t", http_rss_url
source.feeds.append(
Feed(http_rss_url))
return sources
def createDictFromConfig():
""" Read the configuration from scipion/config/scipionbox.conf.
A dictionary will be created where each key will be a section starting
by MICROSCOPE:, all variables that are in the GLOBAL section will be
inherited by default.
"""
# Read from users' config file.
confGlobalDict = OrderedDict()
confDict = OrderedDict()
cp = ConfigParser()
cp.optionxform = str # keep case (stackoverflow.com/questions/1611799)
confFile = pw.getConfigPath("scipionbox.conf")
print "Reading conf file: ", confFile
cp.read(confFile)
GLOBAL = "GLOBAL"
MICROSCOPE = "MICROSCOPE:"
if not GLOBAL in cp.sections():
raise Exception("Missing section %s in %s" % (GLOBAL, confFile))
else:
for opt in cp.options(GLOBAL):
confGlobalDict[opt] = cp.get(GLOBAL, opt)
for section in cp.sections():
if section != GLOBAL and section.startswith(MICROSCOPE):
sectionDict = OrderedDict(confGlobalDict)
for opt in cp.options(section):
sectionDict[opt] = cp.get(section, opt)
confDict[section.replace(MICROSCOPE, '')] = sectionDict
return confDict
def getDataSources(fName=None):
''' return data sources directories for this machine.
directories are defined in datasources.ini or provided filepath'''
import socket
from ConfigParser import ConfigParser
pcName = socket.gethostname()
p = ConfigParser()
p.optionxform = str
if fName is None:
fName = 'datasources.ini'
p.read(fName)
if pcName not in p.sections():
raise NameError('Host name section %s not found in file %s' %
(pcName, fName))
dataSources = {}
for option in p.options(pcName):
dataSources[option] = p.get(pcName, option)
return dataSources
def getConfig(testFolder, iniFileName):
""" Gets configuraiton handle from an ini file """
config = ConfigParser()
config.optionxform=str
config.read(os.path.abspath(os.path.join(testFolder, iniFileName )))
return config
def test_read_init(self):
"""Test that the plugin __init__ will validate on plugins.qgis.org."""
# You should update this list according to the latest in
# https://github.com/qgis/qgis-django/blob/master/qgis-app/
# plugins/validator.py
required_metadata = [
'name',
'description',
'version',
'qgisMinimumVersion',
'email',
'author']
file_path = os.path.abspath(os.path.join(
os.path.dirname(__file__), os.pardir,
'metadata.txt'))
LOGGER.info(file_path)
metadata = []
parser = ConfigParser()
parser.optionxform = str
parser.read(file_path)
message = 'Cannot find a section named "general" in %s' % file_path
assert parser.has_section('general'), message
metadata.extend(parser.items('general'))
for expectation in required_metadata:
message = ('Cannot find metadata "%s" in metadata source (%s).' % (
expectation, file_path))
self.assertIn(expectation, dict(metadata), message)
def _reset(self):
config = ConfigParser()
config.optionxform = str
config.read(self._config_path)
config.set('CARLA/LevelSettings', 'NumberOfVehicles',
self._driver_conf.cars)
config.set('CARLA/LevelSettings', 'NumberOfPedestrians',
self._driver_conf.pedestrians)
config.set('CARLA/LevelSettings', 'WeatherId',
self._driver_conf.weather)
# Write down a temporary init_file to be used on the experiments
temp_f_name = 'p' + str(self._driver_conf.pedestrians)+'_c' + str(self._driver_conf.cars)+"_w"\
+ str(self._driver_conf.weather) +'.ini'
with open(temp_f_name, 'w') as configfile:
config.write(configfile)
self._start_time = time.time()
self.positions = self.carla.requestNewEpisode(self._config_path)
self._current_goal = random.randint(0, len(self.positions))
self.carla.newEpisode(random.randint(0, len(self.positions)))
self._dist_to_activate = random.randint(100, 500)
def read_inifile_parameters(self, configfile):
self._configfile = configfile
parser = ConfigParser()
parser.optionxform = self._optionxform
parser.read(configfile)
for section in parser.sections():
group = section
for option in parser.options(section):
key = (option, group)
if key in self._inifile_parameters:
ptype = self._inifile_parameters[key]["ptype"]
dtype = self._inifile_parameters[key]["dtype"]
value = self.interpret_value(parser.get(group, option),
dtype=dtype)
if is_quantity(self._inifile_parameters[key]["default"]):
value = new_quantity(
val,
to_quantity(
self._inifile_parameters[key]["default"]).unit)
self._inifile_parameters[key]["value"] = value
else:
value = self.interpret_value(parser.get(group, option))
self._inifile_parameters[key] = dict(
group_name=group,
name=option,
set_name=group,
default=value,
value=value,
short=option,
ptype="ini",
dtype="unknown",
description="unknown parameter read from %s" %
configfile)
def read_config(fn):
'''Return a case sensitive ConfigParser by reading the file "fn"'''
cp = ConfigParser()
cp.optionxform = str
cp.read(fn)
return cp
def checkConf(fpath, ftemplate, remove=[], keep=[]):
"Check that all the variables in the template are in the config file too"
# Remove from the checks the sections in "remove", and if "keep"
# is used only check those sections.
# Read the config file fpath and the template ftemplate
cf = ConfigParser()
cf.optionxform = str # keep case (stackoverflow.com/questions/1611799)
assert cf.read(fpath) != [], 'Missing file %s' % fpath
ct = ConfigParser()
ct.optionxform = str
assert ct.read(ftemplate) != [], 'Missing file %s' % ftemplate
# Keep only the sections we want to compare from the files.
for section in set(remove) - set(keep):
ct.remove_section(section)
cf.remove_section(section)
if keep:
for section in set(ct.sections()) - set(keep):
ct.remove_section(section)
cf.remove_section(section)
df = dict([(s, set(cf.options(s))) for s in cf.sections()])
dt = dict([(s, set(ct.options(s))) for s in ct.sections()])
# That funny syntax to create the dictionaries works with old pythons.
if df == dt:
print(green("All the expected sections and options found in " + fpath))
else:
print("Found differences between the configuration file\n %s\n"
"and the template file\n %s" % (fpath, ftemplate))
sf = set(df.keys())
st = set(dt.keys())
for s in sf - st:
print("Section %s exists in the configuration file but "
"not in the template." % red(s))
for s in st - sf:
print("Section %s exists in the template but "
"not in the configuration file." % red(s))
for s in st & sf:
for o in df[s] - dt[s]:
print("In section %s, option %s exists in the configuration "
"file but not in the template." % (red(s), red(o)))
for o in dt[s] - df[s]:
print("In section %s, option %s exists in the template "
"but not in the configuration file." % (red(s), red(o)))
def readBuildConfiguration(self, cfg_file):
cfg = ConfigParser()
cfg.optionxform = str
if cfg.read(cfg_file):
logging.debug("Using config file found at %s" % cfg_file)
self.config = cfg
else:
raise IOError("Cannot find file %s" % cfg_file)
def load_configs(path_to_configs):
"""Load configs from default location."""
configs = ConfigParser()
# Load configs preserving case
configs.optionxform = str
# Individual configs accessed via configs object.
configs.readfp(open(path_to_configs))
return configs
def read(self, policy):
inf_conf = ConfigParser()
inf_conf.optionxform = str
try:
inf_conf.readfp(StringIO(policy))
except:
inf_conf.readfp(StringIO(policy.decode('utf-16')))
return inf_conf
def parse_config(self, config_file):
cfg = ConfigParser()
cfg.optionxform = str
cfg.read(config_file)
self.results_fn = cfg.get('global', 'results')
self.read_featurizers(cfg)
self.read_pipelines(cfg)
self.read_experiments(cfg)
def configtojson(flname):
c = ConfigParser()
c.optionxform = str
c.read(flname)
data = {}
for section in c.sections():
data[section] = dict(c.items(section))
return data
def write_options_dict_to_config(config_file, section, data):
parser = ConfigParser()
parser.optionxform = str
parser.add_section(section)
for key in data.keys():
parser.set(section, key, data[key])
with open(config_file, 'w') as f:
parser.write(f)
def loadINI(f):
parser = ConfigParser()
parser.optionxform = str
parser.read(f)
for section in parser.sections():
for k, v in parser.items(section, raw=True):
if v != '':
yield section, k, parse_value(v)
def readBuildConfiguration(self, cfg_file):
cfg = ConfigParser()
cfg.optionxform = str
if cfg.read(cfg_file):
logging.debug("Using config file found at %s"%cfg_file)
self.config = cfg
else:
raise IOError("Cannot find file %s"%cfg_file)
def login(conf=None, klass=None, proxy=None):
"""Login to smugmug using the contents of the configuration file.
If no configuration file is specified then a file named C{.pysmugrc} in
the user's home directory is used if it exists.
The format is a standard configuration parseable by C{ConfigParser}. The main
section C{pysmug} is required.The key C{login} references which section to use
for authentication with SmugMug. The key C{smugmug} is optional and can specify
an alternate C{SmugMug} class to instantiate. This is an example file::
[pysmug]
login=login_withHash
smugmug=pysmug.SmugTool
[login_withHash]
APIKey = <my api key>
userId = <my user id>
passwordHash = <my password hash>
[login_anonymously]
APIKey = <my api key>
@type conf: string
@param conf: path to a configuration file
@type klass: C{SmugMug} class
@param klass: class to instantiate
@param proxy: address of proxy server if one is required (http[s]://localhost[:8080])
@raise ValueError: if no configuration file is found
"""
import os
from ConfigParser import ConfigParser
if not conf:
home = os.environ.get("HOME", None)
if not home:
raise ValueError("unknown home directory")
conf = os.path.join(home, ".pysmugrc")
if not os.path.exists(conf):
raise ValueError("'%s' not found" % (conf))
config = ConfigParser()
config.optionxform = str
config.read(conf)
if not klass:
klass = SmugMug
if config.has_option("pysmug", "smugmug"):
path = config.get("pysmug", "smugmug")
i = path.rfind(".")
module, attr = path[:i], path[i + 1:]
mod = __import__(module, globals(), locals(), [attr])
klass = getattr(mod, attr)
m = klass(proxy=proxy)
auth = config.get("pysmug", "login")
keys = dict(config.items(auth))
return getattr(m, auth)(**keys)
def login(conf=None, klass=None, proxy=None):
"""Login to smugmug using the contents of the configuration file.
If no configuration file is specified then a file named C{.pysmugrc} in
the user's home directory is used if it exists.
The format is a standard configuration parseable by C{ConfigParser}. The main
section C{pysmug} is required.The key C{login} references which section to use
for authentication with SmugMug. The key C{smugmug} is optional and can specify
an alternate C{SmugMug} class to instantiate. This is an example file::
[pysmug]
login=login_withHash
smugmug=pysmug.SmugTool
[login_withHash]
APIKey = <my api key>
userId = <my user id>
passwordHash = <my password hash>
[login_anonymously]
APIKey = <my api key>
@type conf: string
@param conf: path to a configuration file
@type klass: C{SmugMug} class
@param klass: class to instantiate
@param proxy: address of proxy server if one is required (http[s]://localhost[:8080])
@raise ValueError: if no configuration file is found
"""
import os
from ConfigParser import ConfigParser
if not conf:
home = os.environ.get("HOME", None)
if not home:
raise ValueError("unknown home directory")
conf = os.path.join(home, ".pysmugrc")
if not os.path.exists(conf):
raise ValueError("'%s' not found" % (conf))
config = ConfigParser()
config.optionxform = str
config.read(conf)
if not klass:
klass = SmugMug
if config.has_option("pysmug", "smugmug"):
path = config.get("pysmug", "smugmug")
i = path.rfind(".")
module, attr = path[:i], path[i+1:]
mod = __import__(module, globals(), locals(), [attr])
klass = getattr(mod, attr)
m = klass(proxy=proxy)
auth = config.get("pysmug", "login")
keys = dict(config.items(auth))
return getattr(m, auth)(**keys)
def _buildConfDict(self):
"""Build configuration dictionary that we will use """
if self.splunkEmbedded and not STANDALONE:
self.logger.info('Retrieving eventgen configurations from /configs/eventgen')
self._confDict = entity.getEntities('configs/eventgen', count=-1, sessionKey=self.sessionKey)
else:
self.logger.info('Retrieving eventgen configurations with ConfigParser()')
# We assume we're in a bin directory and that there are default and local directories
conf = ConfigParser()
# Make case sensitive
conf.optionxform = str
currentdir = os.getcwd()
# If we're running standalone (and thusly using configParser)
# only pick up eventgen-standalone.conf.
conffiles = [ ]
if len(sys.argv) > 1:
if len(sys.argv[1]) > 0:
if os.path.exists(sys.argv[1]):
conffiles = [os.path.join(self.grandparentdir, 'default', 'eventgen.conf'),
sys.argv[1]]
if len(conffiles) == 0:
conffiles = [os.path.join(self.grandparentdir, 'default', 'eventgen.conf'),
os.path.join(self.grandparentdir, 'local', 'eventgen.conf')]
self.logger.debug('Reading configuration files for non-splunkembedded: %s' % conffiles)
conf.read(conffiles)
sections = conf.sections()
ret = { }
orig = { }
for section in sections:
ret[section] = dict(conf.items(section))
# For compatibility with Splunk's configs, need to add the app name to an eai:acl key
ret[section]['eai:acl'] = { 'app': self.grandparentdir.split(os.sep)[-1] }
# orig[section] = dict(conf.items(section))
# ret[section] = { }
# for item in orig[section]:
# results = re.match('(token\.\d+)\.(\w+)', item)
# if results != None:
# ret[section][item] = orig[section][item]
# else:
# if item.lower() in [x.lower() for x in self._validSettings]:
# newitem = self._validSettings[[x.lower() for x in self._validSettings].index(item.lower())]
# ret[section][newitem] = orig[section][item]
self._confDict = ret
# Have to look in the data structure before normalization between what Splunk returns
# versus what ConfigParser returns.
logobj = logging.getLogger('eventgen')
if self._confDict['global']['debug'].lower() == 'true' \
or self._confDict['global']['debug'].lower() == '1':
logobj.setLevel(logging.DEBUG)
if self._confDict['global']['verbose'].lower() == 'true' \
or self._confDict['global']['verbose'].lower() == '1':
logobj.setLevel(logging.DEBUGV)
self.logger.debug("ConfDict returned %s" % pprint.pformat(dict(self._confDict)))
def _load_config():
config = ConfigParser()
config.optionxform = str # make it preserve case
# defaults
config.readfp(StringIO(_DEFAULT_CONFIG))
# update from config file
config.read(os.path.join(_STASH_ROOT, f) for f in _STASH_CONFIG_FILES)
return config
def parse_config(fd):
"""Parse configuration file content to TQL list per component
config file is TQL list per platform
"""
config = ConfigParser()
config.optionxform = str
with fd:
config.readfp(fd)
return config.items(SECTION_NAME)
def resetQueue(cfgName, cfgFile, newVersion):
# print confirmation message
print warning("*** Warning, reset GT conf file: " + cfgName)
confirm = raw_input('Proceed? (y/N)')
confirm = confirm.lower() #convert to lowercase
if confirm != 'y':
return
print "Resetting: " + cfgName
newconfig = ConfigParser(dict_type=OrderedDict)
newconfig.optionxform = str
# starting tag
newconfig.add_section("Common")
newconfig.set("Common",'OldGT', cfgFile.get('Common','NewGT'))
# new GT name
newgtname = cfgFile.get('Common','NewGT').split('_V')[0] + '_V' + newVersion
if newgtname == cfgFile.get('Common','NewGT'):
print "New tag : " + newgtname + " equal to old tag: " + cfgFile.get('Common','NewGT') + " skipping the reset"
return
newconfig.set("Common",'NewGT', newgtname)
newconfig.set("Common",'Passwd', cfgFile.get('Common','Passwd'))
newconfig.set("Common",'Environment', cfgFile.get('Common','Environment'))
newconfig.set("Common",'GTType', cfgFile.get('Common','GTType'))
newconfig.add_section("Tags")
newconfig.add_section("Connect")
newconfig.add_section("Account")
newconfig.add_section("AddRecord")
newconfig.add_section("RmRecord")
newconfig.add_section("TagManipulation")
newconfig.set("TagManipulation",'check', 'new')
newconfig.add_section("Comments")
newconfig.set("Comments","Release",cfgFile.get("Comments","Release"))
newconfig.set("Comments","Scope",cfgFile.get("Comments","Scope"))
newconfig.set("Comments","Changes",'')
if cfgFile.has_option("Comments","Class"):
newconfig.set("Comments","Class",cfgFile.get("Comments","Class"))
newconfig.add_section("Pending")
if cfgFile.has_section("Pending"):
for item in cfgFile.items("Pending"):
newconfig.set("Pending",item[0],item[1])
# write the file
configfile = open(cfgName, 'wb')
newconfig.write(configfile)
configfile.close()
cvsCommit(cfgName,'reset for new GT')
return
def _get_financial_item_algos(self, cfgfilepath):
""""""
fin_item_cfg = ConfigParser()
fin_item_cfg.optionxform = str
fin_item_cfg.readfp(codecs.open(cfgfilepath, 'r', 'utf-8-sig'))
self.fin_item_names = fin_item_cfg.options('financial_item')
self.fin_item_algos = {}
for name in self.fin_item_names:
exec("import procfindata.financial_item_algos_by_sql.{} as finalgo".format(name))
self.fin_item_algos[name] = finalgo
def parse(data): """Parses an xpd file Returns: A dict containing the properties """ config = ConfigParser() config.optionxform = str config.readfp(StringIO(data)) return dict(config.items(constants.sectionName))
def dump(items): """Builds an xpd file""" config = ConfigParser() config.optionxform = str config.add_section(constants.sectionName) for k, v in items.iteritems(): config.set(constants.sectionName, k, v) f = StringIO() config.write(f) return f.getvalue()
