def main():
global DEBUG
if "--register" in sys.argv or "--unregister" in sys.argv:
import win32com.server.register
win32com.server.register.UseCommandLine(PyEmail)
elif "py2exe" in sys.argv:
from distutils.core import setup
from .nsis import build_installer, Target
import py2exe
setup(
name="PyEmail",
version=__version__,
description="Interfaz PyAfipWs Email %s",
long_description=__doc__,
author="Mariano Reingart",
author_email="*****@*****.**",
url="http://www.sistemasagiles.com.ar",
license="GNU GPL v3",
com_server=["pyemail"],
console=[],
options={
"py2exe": {
"includes": [
"email.generator",
"email.iterators",
"email.message",
"email.utils",
],
"optimize":
2,
"excludes": [
"pywin",
"pywin.dialogs",
"pywin.dialogs.list",
"win32ui",
"distutils.core",
"py2exe",
"nsis",
],
#'skip_archive': True,
}
},
data_files=[
(".", ["licencia.txt"]),
],
cmdclass={"py2exe": build_installer},
)
elif "/Automate" in sys.argv:
# MS seems to like /automate to run the class factories.
import win32com.server.localserver
# win32com.server.localserver.main()
# start the server.
win32com.server.localserver.serve([PyEmail._reg_clsid_])
elif "/prueba" in sys.argv:
pyemail = PyEmail()
import getpass
i = sys.argv.index("/prueba")
if i + 2 < len(sys.argv):
usuario = sys.argv[sys.argv.index("/prueba") + 1]
clave = sys.argv[sys.argv.index("/prueba") + 2]
else:
usuario = input("usuario:")
clave = getpass.getpass("clave:")
ok = pyemail.Conectar("smtp.gmail.com",
usuario=usuario,
clave=clave,
puerto=587)
print("login ok?", ok, pyemail.Excepcion)
print(pyemail.Traceback)
ok = pyemail.Enviar(usuario, "prueba", usuario, "prueba!", None)
print("mail enviado?", ok, pyemail.Excepcion)
ok = pyemail.Salir()
else:
config = SafeConfigParser()
config.read("rece.ini")
if "/debug" in sys.argv:
DEBUG = True
print("VERSION", __version__)
sys.argv.remove("/debug")
if len(sys.argv) < 3:
print("Parámetros: motivo destinatario [mensaje] [archivo]")
sys.exit(1)
conf_mail = dict(config.items("MAIL"))
motivo = sys.argv[1]
destinatario = sys.argv[2]
mensaje = len(sys.argv) > 3 and sys.argv[3] or conf_mail["cuerpo"]
archivo = len(sys.argv) > 4 and sys.argv[4] or None
print("Motivo: ", motivo)
print("Destinatario: ", destinatario)
print("Mensaje: ", mensaje)
print("Archivo: ", archivo)
pyemail = PyEmail()
ok = pyemail.Conectar(
conf_mail["servidor"],
conf_mail["usuario"],
conf_mail["clave"],
conf_mail.get("puerto", 25),
)
if ok:
pyemail.Enviar(conf_mail["remitente"], motivo, destinatario,
mensaje, archivo)
else:
print(pyemail.Traceback)
def getlist(self, section, option):
return [x.strip() for x in self.get(section, option).split(',')]
SafeConfigParser.getlist = getlist
files = [
os.path.join(os.path.dirname(os.path.realpath(__file__)),
'../default-config.ini'),
os.path.join(os.path.dirname(os.path.realpath(__file__)), '../config.ini'),
'/etc/voctomix/voctocore.ini',
'/etc/voctomix.ini', # deprecated
'/etc/voctocore.ini',
os.path.expanduser('~/.voctomix.ini'), # deprecated
os.path.expanduser('~/.voctocore.ini'),
]
if Args.ini_file is not None:
files.append(Args.ini_file)
Config = SafeConfigParser()
readfiles = Config.read(files)
log = logging.getLogger('ConfigParser')
log.debug('considered config-files: \n%s',
"\n".join(["\t\t" + os.path.normpath(file) for file in files]))
log.debug('successfully parsed config-files: \n%s',
"\n".join(["\t\t" + os.path.normpath(file) for file in readfiles]))
from configparser import SafeConfigParser
import pythoncom
import wmi
from utils import conf_filename, wait_for_result, log_fw
from PyQt5.QtWidgets import QMessageBox, QDialog, QLabel, QVBoxLayout
from PyQt5.QtCore import Qt
from threading import Thread
from copyer import run
parser = SafeConfigParser()
def is_removable_drive(drive_letter):
pythoncom.CoInitialize()
c = wmi.WMI()
drive_letter = drive_letter.replace('\\', '')
return len([
localDisk
for localDisk in c.Win32_LogicalDisk(Description='Removable Disk',
DeviceID=drive_letter)
]) > 0
def get_serial_by_drive_letter(drive_letter):
pythoncom.CoInitialize()
c = wmi.WMI()
logical_drive = [
logicalDisk.Antecedent.deviceID
for logicalDisk in c.Win32_LogicalDiskToPartition()
if logicalDisk.Dependent.deviceID.lower() in drive_letter.lower()
][0]
def fromFile(cls, initFile, initList=None):
"""Loads specified features from file
Parameters
----------
initFile : str
Path to file
initList : list
List of classes to load
"""
parser = SafeConfigParser()
parser.read(initFile)
corpdb = parser.get('constants', 'corpdb') if parser.has_option(
'constants', 'corpdb') else dlac.DEF_CORPDB
corptable = parser.get('constants', 'corptable') if parser.has_option(
'constants', 'corptable') else dlac.DEF_CORPTABLE
correl_field = parser.get(
'constants', 'correl_field') if parser.has_option(
'constants', 'correl_field') else dlac.DEF_CORREL_FIELD
mysql_host = parser.get(
'constants', 'mysql_host') if parser.has_option(
'constants', 'mysql_host') else dlac.MYSQL_HOST
message_field = parser.get(
'constants', 'message_field') if parser.has_option(
'constants', 'message_field') else dlac.DEF_MESSAGE_FIELD
messageid_field = parser.get(
'constants', 'messageid_field') if parser.has_option(
'constants', 'messageid_field') else dlac.DEF_MESSAGEID_FIELD
encoding = parser.get('constants', 'encoding') if parser.has_option(
'constants', 'encoding') else dlac.DEF_ENCODING
if parser.has_option('constants', 'use_unicode'):
use_unicode = True if parser.get(
'constants', 'use_unicode') == "True" else False
else:
use_unicode = dlac.DEF_UNICODE_SWITCH
lexicondb = parser.get('constants', 'lexicondb') if parser.has_option(
'constants', 'lexicondb') else dlac.DEF_LEXICON_DB
if parser.has_option('constants', 'feattable'):
if len(parser.get('constants', 'feattable').split(",")) > 0:
featureTable = [
f.strip()
for f in parser.get('constants', 'feattable').split(",")
]
else:
featureTable = parser.get('constants', 'feattable')
else:
featureTable = dlac.DEF_FEAT_TABLE
featNames = parser.get('constants', 'featnames') if parser.has_option(
'constants', 'featnames') else dlac.DEF_FEAT_NAMES
date_field = parser.get(
'constants', 'date_field') if parser.has_option(
'constants', 'date_field') else dlac.DEF_DATE_FIELD
outcome_table = parser.get(
'constants', 'outcometable') if parser.has_option(
'constants', 'outcometable') else dlac.DEF_OUTCOME_TABLE
outcome_value_fields = [
o.strip()
for o in parser.get('constants', 'outcomefields').split(",")
] if parser.has_option('constants', 'outcomefields') else [
dlac.DEF_OUTCOME_FIELD
] # possible list
outcome_controls = [
o.strip()
for o in parser.get('constants', 'outcomecontrols').split(",")
] if parser.has_option(
'constants',
'outcomecontrols') else dlac.DEF_OUTCOME_CONTROLS # possible list
outcome_interaction = [
o.strip()
for o in parser.get('constants', 'outcomeinteraction').split(",")
] if parser.has_option(
'constants', 'outcomeinteraction'
) else dlac.DEF_OUTCOME_CONTROLS # possible list
group_freq_thresh = int(parser.get(
'constants', 'groupfreqthresh')) if parser.has_option(
'constants',
'groupfreqthresh') else dlac.getGroupFreqThresh(correl_field)
featureMappingTable = parser.get(
'constants', 'featlabelmaptable') if parser.has_option(
'constants', 'featlabelmaptable') else ''
featureMappingLex = parser.get(
'constants', 'featlabelmaplex') if parser.has_option(
'constants', 'featlabelmaplex') else ''
output_name = parser.get('constants',
'outputname') if parser.has_option(
'constants', 'outputname') else ''
wordTable = parser.get('constants', 'wordTable') if parser.has_option(
'constants', 'wordTable') else None
model = parser.get('constants', 'model') if parser.has_option(
'constants', 'model') else dlac.DEF_MODEL
feature_selection = dlac.DEF_FEATURE_SELECTION_MAPPING[parser.get(
'constants', 'featureselection')] if parser.has_option(
'constants', 'featureselection') else ''
feature_selection_string = parser.get(
'constants', 'featureselectionstring') if parser.has_option(
'constants', 'featureselectionstring') else ''
if initList:
init = initList
else:
init = [
o.strip() for o in parser.get('constants', 'init').split(",")
] if parser.has_option('constants', 'init') else [
'fw', 'fg', 'fe', 'fr', 'og', 'oa', 'rp', 'cp'
]
return cls(corpdb=corpdb,
corptable=corptable,
correl_field=correl_field,
mysql_host=mysql_host,
message_field=message_field,
messageid_field=messageid_field,
encoding=encoding,
use_unicode=use_unicode,
lexicondb=lexicondb,
featureTable=featureTable,
featNames=featNames,
date_field=date_field,
outcome_table=outcome_table,
outcome_value_fields=outcome_value_fields,
outcome_controls=outcome_controls,
outcome_interaction=outcome_interaction,
group_freq_thresh=group_freq_thresh,
featureMappingTable=featureMappingTable,
featureMappingLex=featureMappingLex,
output_name=output_name,
wordTable=wordTable,
model=model,
feature_selection=feature_selection,
feature_selection_string=feature_selection_string,
init=init)
def main():
def ensure_dir(path):
"""Ensure that path is a directory creating it if necessary.
If path already exists and is not a directory, print an error
message and quit with sys.exit().
Parameters:
path String specifying the path to ensure
Return value:
path
"""
if not os.path.exists(path):
os.makedirs(path)
elif not os.path.isdir(path):
print("error: '%s' is not a directory" % path)
sys.exit(1)
return path
def config_default(config, section, option, default):
"""Set default values for options that do not have a value."""
try:
config.get(section, option)
except NoSectionError:
config.add_section(section)
config.set(section, option, default)
except NoOptionError:
config.set(section, option, default)
config_dir = ensure_dir(os.path.expanduser("~/.diceware.py"))
cache_dir = ensure_dir(os.path.join(config_dir, "cache"))
# Parse config file
config_file = os.path.join(config_dir, "config")
config = SafeConfigParser()
config.read(config_file)
config_default(config, "defaults", "lang", "en")
config_default(config, "defaults", "words", "5")
config_default(config, "defaults", "special", "0")
config_default(config, "defaults", "file", "")
config_default(config, "defaults", "separator", " ")
# Sanity checks for config options
if config.get("defaults", "lang") not in WORD_LIST_URLS.keys():
print("error: '%s' is not a valid value for option 'lang'" %
config.get("defaults", "lang"))
sys.exit(1)
try:
config.getint("defaults", "words")
config.getint("defaults", "special")
except ValueError:
print("error: 'words' and 'special' options must have integer values")
sys.exit(1)
# Parse command line arguments
parser = OptionParser()
parser.add_option("-g",
"--grid",
dest="grid",
action="store_true",
help="Instead of a single line, generate NxN grid of " +
"words. This makes eavesdropping harder")
parser.add_option("-n",
"--words",
dest="words",
type="int",
metavar="N",
help="generate N words (default: %default)",
default=config.getint("defaults", "words"))
parser.add_option("-s",
"--special",
dest="special",
type="int",
metavar="M",
help="insert M special characters (default: %default)",
default=config.getint("defaults", "special"))
parser.add_option(
"-f",
"--file",
dest="file",
metavar="FILE",
help="override the `lang' option and read the word list " +
"from FILE",
default=config.get("defaults", "file"))
parser.add_option(
"-p",
"--separator",
dest="separator",
type="string",
metavar="P",
help="specify the separator between words (default: %default)",
default=config.get("defaults", "separator"))
linguas = sorted(WORD_LIST_URLS.keys())
parser.add_option("-l",
"--lang",
dest="lang",
metavar="LANG",
type="choice",
choices=linguas,
help="use the word list for LANG (" +
", ".join(linguas) + ") (default: %default)",
default=config.get("defaults", "lang"))
options, args = parser.parse_args()
if args or options.words < 1 or options.special < 0:
parser.print_help()
sys.exit(0)
parser.destroy()
# --file has higher precedence than --lang
if options.file:
try:
fobj = open(options.file)
except IOError:
print("error: unable to open word list file '%s'" % options.file)
sys.exit(1)
try:
word_list = read_word_list(fobj)
except ValueError as e:
print("error: %s" % e)
sys.exit(1)
else:
word_list = get_word_list(cache_dir, options.lang)
if not options.grid:
words, with_specials = generate(word_list, options.words,
options.special)
print("passphrase : %s" % options.separator.join(words))
if options.special > 0:
print("with specials: %s" % options.separator.join(with_specials))
else:
words, length = generate_grid(word_list, options.words,
options.special)
for word_row in words:
print(" ".join([word.ljust(length) for word in word_row]))
def create_from_config_file(cls,
config_file="~/.hpecp.conf",
profile=None):
"""Create a ContainerPlatformClient object from a configuration file.
Parameters
----------
config_file : str
The configuration filename and path
profile : str
If the configuration file has multiple profile sections, you
can select the profile to use.
Returns
-------
ContainerPlatformClient:
An instance of ContainerPlatformClient is returned.
Example
-------
Below is an example configuration file.
[default]
api_host = 127.0.0.1
api_port = 8080
use_ssl = True
verify_ssl = False
warn_ssl = False
[demoserver]
username = admin
password = admin123
tenant = /api/v1/tenant/2
"""
_log = Logger.get_logger()
if profile is None:
profile = "default"
if config_file.startswith("~"):
file_path = config_file[1:]
file_path = file_path.lstrip("/")
config_file = os.path.join(os.path.expanduser("~"), file_path)
if not os.path.exists(config_file):
raise ContainerPlatformClientException(
"Could not find configuration file '{}'".format(config_file))
config = SafeConfigParser()
config.readfp(codecs.open(config_file, "r", "utf8"))
assert (profile
in config.sections()), "'{}' section not found in '{}'".format(
profile, config_file)
assert ("username" in config[profile]
or "username" in config["default"]), (
"'username' not found in section '{}' or in "
"the default section".format(profile))
assert ("password" in config[profile]
or "password" in config["default"]), (
"'password' not found in section '{}' "
"or in the default section".format(profile))
assert ("api_host" in config[profile]
or "api_host" in config["default"]), (
"'api_host' not found in section '{}' or in "
"the default section".format(profile))
assert ("api_port" in config[profile]
or "api_port" in config["default"]), (
"'api_port' not found in section '{}' or in "
"the default section".format(profile))
assert ("use_ssl" in config[profile]
or "use_ssl" in config["default"]), (
"'use_ssl' not found in section '{}' or in"
"the default section".format(profile))
assert ("verify_ssl" in config[profile]
or "verify_ssl" in config["default"]), (
"'verify_ssl' not found in section '{}' or in"
"the default section".format(profile))
assert ("warn_ssl" in config[profile]
or "warn_ssl" in config["default"]), (
"'warn_ssl' not found in section '{}' or in"
"the default section".format(profile))
# tenant parameter is optional
def get_config_value(key, profile):
if key in config[profile]:
_log.debug("Found '{}' in profile '{}'".format(key, profile))
return config[profile][key]
else:
try:
val = config["default"][key]
_log.debug("Found '{}' in profile '{}'".format(
key, "default"))
return val
except Exception:
_log.debug("Could not find '{}' in profile '{}'".format(
key, profile))
return None
username = str(get_config_value("username", profile))
password = str(get_config_value("password", profile))
api_host = str(get_config_value("api_host", profile))
api_port = int(get_config_value("api_port", profile))
use_ssl = str(get_config_value("use_ssl", profile))
verify_ssl = str(get_config_value("verify_ssl", profile))
warn_ssl = str(get_config_value("warn_ssl", profile))
# optional parameter
tenant = get_config_value("tenant", profile)
if tenant:
assert isinstance(tenant, str) and re.match(
r"\/api\/v1\/tenant\/[0-9]+", tenant), (
"'tenant' must have format '/api/v1/tenant/[0-9]+' in '{}'"
).format(config_file)
if use_ssl == "False":
use_ssl = False
else:
use_ssl = True
# verify_ssl could be a path
if verify_ssl == "False":
verify_ssl = False
if warn_ssl == "False":
warn_ssl = False
else:
warn_ssl = True
return cls(
username,
password,
api_host,
api_port,
use_ssl,
verify_ssl,
warn_ssl,
tenant,
)
def read_config(self):
'''
Gets info from config file
'''
self.config = SafeConfigParser()
self.config.read(self.config_file)
def __init__(self, file):
# Script starts from here
print("Arguement is : ", file)
self.fname = os.path.basename(file)
print("VHDL filename is : ", self.fname)
self.home = os.path.expanduser("~")
self.parser = SafeConfigParser()
self.parser.read(
os.path.join(self.home, os.path.join('.nghdl', 'config.ini')))
self.ngspice_home = self.parser.get('NGSPICE', 'NGSPICE_HOME')
self.release_dir = self.parser.get('NGSPICE', 'RELEASE')
self.src_home = self.parser.get('SRC', 'SRC_HOME')
self.licensefile = self.parser.get('SRC', 'LICENSE')
# #### Creating connection_info.txt file from vhdl file #### #
read_vhdl = open(file, 'r')
vhdl_data = read_vhdl.readlines()
read_vhdl.close()
start_flag = -1 # Used for scaning part of data
scan_data = []
# p=re.search('port(.*?)end',read_vhdl,re.M|re.I|re.DOTALL).group()
for item in vhdl_data:
if re.search('port', item, re.I):
start_flag = 1
elif re.search("end", item, re.I):
start_flag = 0
if start_flag == 1:
item = re.sub("port", " ", item, flags=re.I)
item = re.sub("\(", " ", item, flags=re.I) # noqa
item = re.sub("\)", " ", item, flags=re.I) # noqa
item = re.sub(";", " ", item, flags=re.I)
scan_data.append(item.rstrip())
scan_data = [_f for _f in scan_data if _f]
elif start_flag == 0:
break
port_info = []
self.port_vector_info = []
for item in scan_data:
print("Scan Data :", item)
if re.search("in", item, flags=re.I):
if re.search("std_logic_vector", item, flags=re.I):
temp = re.compile(r"\s*std_logic_vector\s*", flags=re.I)
elif re.search("std_logic", item, flags=re.I):
temp = re.compile(r"\s*std_logic\s*", flags=re.I)
else:
raise ValueError("Please check your vhdl " +
"code for datatype of input port")
elif re.search("out", item, flags=re.I):
if re.search("std_logic_vector", item, flags=re.I):
temp = re.compile(r"\s*std_logic_vector\s*", flags=re.I)
elif re.search("std_logic", item, flags=re.I):
temp = re.compile(r"\s*std_logic\s*", flags=re.I)
else:
raise ValueError("Please check your vhdl " +
"code for datatype of output port")
else:
raise ValueError(
"Please check the in/out direction of your port")
lhs = temp.split(item)[0]
rhs = temp.split(item)[1]
bit_info = re.compile(r"\s*downto\s*", flags=re.I).split(rhs)[0]
if bit_info:
port_info.append(lhs + ":" + str(int(bit_info) + int(1)))
self.port_vector_info.append(1)
else:
port_info.append(lhs + ":" + str(int(1)))
self.port_vector_info.append(0)
print("Port Info :", port_info)
# Open connection_info.txt file
con_ifo = open('connection_info.txt', 'w')
for item in port_info:
word = item.split(':')
con_ifo.write(word[0].strip() + ' ' + word[1].strip() + ' ' +
word[2].strip())
con_ifo.write("\n")
con_ifo.close()
def parse_ini(self):
"""Read ini file for visar setup and analysis settings."""
cp = SafeConfigParser(defaults=self.DEFAULTS)
if self.filenames['ini'] is not None:
cp.read(self.filenames['ini'])
return cp
def __init__(self, path=CONFIGPATH):
self.this_parser = SafeConfigParser()
self.path = path
self.read()
self.data = {}
self.get_values()
def upload_books(self,
files,
names,
on_card=None,
end_session=True,
metadata=None):
"""Process sending the book to the Kobo device."""
if self.modifying_css():
common.log.info(
"KoboTouchExtended:upload_books:Searching for device-specific "
"CSS file")
device_css_file_name = self.KOBO_EXTRA_CSSFILE
try:
if self.isAuraH2O():
device_css_file_name = "kobo_extra_AURAH2O.css"
elif self.isAuraHD():
device_css_file_name = "kobo_extra_AURAHD.css"
elif self.isAura():
device_css_file_name = "kobo_extra_AURA.css"
elif self.isClaraHD():
device_css_file_name = "kobo_extra_CLARA.css"
elif self.isForma():
device_css_file_name = "kobo_extra_FORMA.css"
elif self.isGlo():
device_css_file_name = "kobo_extra_GLO.css"
elif self.isGloHD():
device_css_file_name = "kobo_extra_GLOHD.css"
elif self.isLibraH2O():
device_css_file_name = "kobo_extra_LIBRA.css"
elif self.isMini():
device_css_file_name = "kobo_extra_MINI.css"
elif self.isTouch():
device_css_file_name = "kobo_extra_TOUCH.css"
except AttributeError:
common.log.warning(
"KoboTouchExtended:upload_books:Calibre version too old "
"to handle some specific devices, falling back to "
"generic file {0}".format(device_css_file_name))
device_css_file_name = os.path.join(self.configdir,
device_css_file_name)
if os.path.isfile(device_css_file_name):
common.log.info(
"KoboTouchExtended:upload_books:Found device-specific "
"file {0}".format(device_css_file_name))
shutil.copy(
device_css_file_name,
os.path.join(self._main_prefix, self.KOBO_EXTRA_CSSFILE),
)
else:
common.log.info(
"KoboTouchExtended:upload_books:No device-specific CSS "
"file found (expecting {0})".format(device_css_file_name))
kobo_config_file = os.path.join(self._main_prefix, ".kobo", "Kobo",
"Kobo eReader.conf")
if os.path.isfile(kobo_config_file):
cfg = SafeConfigParser(allow_no_value=True)
cfg.optionxform = str
cfg.read(kobo_config_file)
if not cfg.has_section("FeatureSettings"):
cfg.add_section("FeatureSettings")
common.log.info(
"KoboTouchExtended:upload_books:Setting FeatureSettings."
"FullBookPageNumbers to {0}".format(
"true" if self.full_page_numbers else "false"))
cfg.set(
"FeatureSettings",
"FullBookPageNumbers",
"true" if self.full_page_numbers else "false",
)
with open(kobo_config_file, "w") as cfgfile:
cfg.write(cfgfile)
return super(KOBOTOUCHEXTENDED,
self).upload_books(files, names, on_card, end_session,
metadata)
def __init__(self):
config_parser = argparse.ArgumentParser(add_help=False)
config_parser.add_argument("-f",
"--from-config",
help="Specify config file",
metavar="FILE")
args, _ = config_parser.parse_known_args()
defaults = {}
if args.from_config:
config = SafeConfigParser()
config.read([args.from_config])
defaults = dict(config.items("Defaults"))
formatter = argparse.ArgumentDefaultsHelpFormatter
self.parser = argparse.ArgumentParser(formatter_class=formatter,
parents=[config_parser],
description=__doc__)
self.parser.add_argument("--data-provider",
"-d",
type=str,
default="static")
self.parser.add_argument("--input-data-path",
"-n",
type=str,
default="data/input/coinbase-1h-btc-usd.csv")
self.parser.add_argument("--reward-strategy",
"-r",
type=str,
default="incremental-profit",
dest="reward_strat")
self.parser.add_argument("--pair", "-p", type=str, default="BTC/USD")
self.parser.add_argument("--debug", "-D", action='store_false')
self.parser.add_argument('--mini-batches',
type=int,
default=1,
help='Mini batches',
dest='n_minibatches')
self.parser.add_argument('--train-split-percentage',
type=float,
default=0.8,
help='Train set percentage')
self.parser.add_argument('--verbose-model',
type=int,
default=1,
help='Verbose model',
dest='model_verbose')
self.parser.add_argument('--params-db-path',
type=str,
default='sqlite:///data/params.db',
help='Params path')
self.parser.add_argument('--tensorboard-path',
type=str,
default=os.path.join('data', 'tensorboard'),
help='Tensorboard path')
self.parser.add_argument('--parallel-jobs',
type=int,
default=multiprocessing.cpu_count(),
help='How many processes in parallel')
subparsers = self.parser.add_subparsers(help='Command', dest="command")
optimize_parser = subparsers.add_parser(
'optimize', description='Optimize model parameters')
optimize_parser.add_argument('--trials',
type=int,
default=1,
help='Number of trials')
optimize_parser.add_argument(
'--prune-evals',
type=int,
default=2,
help='Number of pruning evaluations per trial')
optimize_parser.add_argument(
'--eval-tests',
type=int,
default=1,
help='Number of tests per pruning evaluation')
train_parser = subparsers.add_parser('train',
description='Train model')
train_parser.add_argument('--epochs',
type=int,
default=10,
help='Number of epochs to train')
train_parser.add_argument('--save-every',
type=int,
default=1,
help='Save the trained model every n epochs')
train_parser.add_argument('--no-test',
dest="test_trained",
action="store_false",
help='Test each saved model')
train_parser.add_argument('--render-test',
dest="render_test",
action="store_true",
help='Render the test environment')
train_parser.add_argument('--no-report',
dest="render_report",
action="store_false",
help='Render the performance report')
train_parser.add_argument('--save-report',
dest="save_report",
action="store_true",
help='Save the performance report as .html')
test_parser = subparsers.add_parser('test', description='Test model')
test_parser.add_argument('--model-epoch',
type=int,
default=0,
help='Model epoch index')
test_parser.add_argument('--no-render',
dest="render_env",
action="store_false",
help='Render the test environment')
test_parser.add_argument('--no-report',
dest="render_report",
action="store_false",
help='Render the performance report')
test_parser.add_argument('--save-report',
dest="save_report",
action="store_true",
help='Save the performance report as .html')
trade_parser = subparsers.add_parser(
'trade', description='Trade with saved model')
trade_parser.add_argument("--data-provider",
"-d",
type=str,
default="static")
trade_parser.add_argument('--model-epoch',
type=int,
default=0,
help='Model epoch index')
trade_parser.add_argument('--no-render',
dest="render_env",
action="store_false",
help='Render the test environment')
trade_parser.add_argument('--no-report',
dest="render_report",
action="store_false",
help='Render the performance report')
trade_parser.add_argument('--save-report',
dest="save_report",
action="store_true",
help='Save the performance report as .html')
subparsers.add_parser('update-static-data',
description='Update static data')
self.parser.set_defaults(**defaults)
def continuity_error(ini_fname, delta_t, cell_id, X, channel_indices):
# run the model using the supplied configuration
pytopkapi.run(ini_fname)
# parse the config file
config = SafeConfigParser()
config.read(ini_fname)
precip_fname = config.get('input_files', 'file_rain')
ET_fname = config.get('input_files', 'file_ET')
group_name = config.get('groups', 'group_name')
result_fname = config.get('output_files', 'file_out')
# write model version
print('PyTOPKAPI version = %s' % pytopkapi.__version__)
# compute the terms in the continuity eqn.
initial_storage, final_storage = compute_storage(result_fname)
print('Initial storage = ', initial_storage)
print('Final storage = ', final_storage)
precip_vol = compute_precip_volume(precip_fname, group_name, X)
print('Precipitation = ', precip_vol)
evapo_vol = compute_evapot_volume(result_fname, X)
print('Evapotranspiration = ', evapo_vol)
open_water_evap_vol = compute_evap_volume(result_fname, channel_indices)
print('Channel evaporation = ', open_water_evap_vol)
channel_runoff_vol = compute_channel_runoff(result_fname, delta_t, cell_id)
print('Channel runoff (outlet) = ', channel_runoff_vol)
overland_runoff_vol = compute_overland_runoff(result_fname, delta_t,
cell_id)
print('Overland runoff (outlet) = ', overland_runoff_vol)
soil_drainage_vol = compute_soil_drainage(result_fname, delta_t, cell_id)
print('Soil drainage (outlet) = ', soil_drainage_vol)
down_drainage_vol = compute_down_drainage(result_fname, delta_t, cell_id)
print('Non-channel drainage (outlet) = ', down_drainage_vol)
input = precip_vol
output = evapo_vol \
+ open_water_evap_vol \
+ channel_runoff_vol \
+ down_drainage_vol
delta_storage = final_storage - initial_storage
error = delta_storage - (input - output)
if precip_vol > 0:
precip_error = abs((error / precip_vol) * 100.0)
else:
precip_error = None
stor_error = abs((error / initial_storage) * 100.0)
print('Continuity error = ', error)
print('Error as % precip. = ', precip_error)
print('Error as % initial storage = ', stor_error)
os.remove(result_fname)
return error, precip_error, stor_error
def plot_as_wires(cfg_file, db_name, syn_db):
# cfg_file = sys.argv[1]
# db_name = sys.argv[2]
parser = SafeConfigParser()
parser.read(cfg_file)
colors = ['r', 'g', 'b', 'c', 'm', 'k']
markers = [".", "o", "s", "^", "p", "*"]
c_mapping = {}
m_mapping = {}
conn = sqlite3.connect(db_name)
cursor = conn.cursor()
cursor.execute("select distinct name from swc_data order by name")
rets = cursor.fetchall()
names = []
c = 0
for entity in rets:
print("entity: %s with color %s" %
(str(entity[0]), colors[c % len(colors)]))
names.append(entity[0])
c_mapping[str(entity[0])] = colors[c % len(colors)]
c = c + 1
times = []
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.set_xlabel("X")
ax.set_ylabel("Y")
ax.set_zlabel("Z")
xmin, ymin, zmin = 0.0, 0.0, 0.0
dim_xyz = eval(parser.get("substrate", "dim_xyz"))
xmax = dim_xyz[0]
ymax = dim_xyz[1]
zmax = dim_xyz[2]
ax.set_xlim([xmin, xmax])
ax.set_ylim([ymin, ymax])
ax.set_zlim([zmin, zmax])
ax.view_init(elev=e_start, azim=a_start)
for name in names:
cursor.execute("select * from swc_data where name=? order by id",
(str(name), ))
rets = cursor.fetchall()
i = 0
points = []
u = np.linspace(0, 2 * np.pi, 20)
v = np.linspace(0, np.pi, 20)
soma = rets[0]
soma_x = soma[3]
soma_y = soma[4]
soma_z = soma[5]
radius = soma[9]
x = soma_x + (radius * np.outer(np.cos(u), np.sin(v)))
y = soma_y + (radius * np.outer(np.sin(u), np.sin(v)))
z = soma_z + (radius * np.outer(np.ones(np.size(u)), np.cos(v)))
ax.plot_surface(x, y, z, rstride=4, cstride=4, color=c_mapping[name])
xlist = []
ylist = []
zlist = []
for entity in rets:
from_point = np.array([entity[3], entity[4], entity[5]])
to_point = np.array([entity[6], entity[7], entity[8]])
xlist.extend([from_point[0], to_point[0]]) #xpairs[-1])
xlist.append(np.nan) #None)
ylist.extend([from_point[1], to_point[1]]) #ypairs[-1])
ylist.append(np.nan) #None)
zlist.extend([from_point[2], to_point[2]]) #zpairs[-1])
zlist.append(np.nan) #None)
t0 = timer()
plt.plot(xlist, ylist, zlist, color=c_mapping[name], alpha=0.5)
t1 = timer()
times.append(t1 - t0)
if len(syn_db) > 0:
syn_conn = sqlite3.connect(syn_db)
syn_cursor = syn_conn.cursor()
syn_cursor.execute("select * from synapses order by pre_syn_entity")
rets = syn_cursor.fetchall()
xs = []
ys = []
zs = []
count = 0
for ret in rets:
xs.append(ret[2])
ys.append(ret[3])
zs.append(ret[4])
count = count + 1
ax.scatter(xs, ys, zs, c="b", marker="o")
print("no_synapses (count)=", count)
t0 = timer()
if db_name.startswith(".."):
out_name = db_name.split("/")[-1].split(".")[0] + "_wire.pdf"
else:
out_name = db_name.split(".")[0] + "_wire.pdf"
plt.savefig(out_name)
t1 = timer()
print("writing the figure took: %fs" % (t1 - t0))
times.append(t1 - t0)
return np.sum(times)
def __init__(self):
self.config = SafeConfigParser()
self.config.read('config.ini')
def main():
config = configparser.ConfigParser()
config.read('config.ini')
arquivo = config['default']['video']
font = cv2.FONT_HERSHEY_SIMPLEX
posicao = 1
new = False
capture = cv2.VideoCapture(arquivo)
_, image = capture.read()
image_line1 = np.hstack((image, image))
image_line2 = np.hstack((image, image))
image = np.vstack((image_line1, image_line2))
blur = int(config['default']['blur'])
Bsize = int(config['default']['Bsize'])
Hmin = int(config['default']['Hmin'])
Hmax = int(config['default']['Hmax'])
Smin = int(config['default']['Smin'])
Smax = int(config['default']['Smax'])
Vmin = int(config['default']['Vmin'])
Vmax = int(config['default']['Vmax'])
cv2.namedWindow("image")
cv2.createTrackbar('Frame','image',0,int(capture.get(cv2.CAP_PROP_FRAME_COUNT)),frame_change)
cv2.createTrackbar('Blur','image',blur,30,sliders_update)
cv2.createTrackbar('Hmin','image',Hmin,100,sliders_update)
cv2.createTrackbar('Hmax','image',Hmax,179,sliders_update)
cv2.createTrackbar('Smin','image',Smin,255,sliders_update)
cv2.createTrackbar('Smax','image',Smax,255,sliders_update)
cv2.createTrackbar('Vmin','image',Vmin,255,sliders_update)
cv2.createTrackbar('Vmax','image',Vmax,255,sliders_update)
cv2.createTrackbar('Bsize','image',Bsize,50,sliders_update)
rmin = Bsize - int(Bsize/3) # Raio minimo para ser considerado um objeto circular (em pixels)
rmax = Bsize + int(Bsize/3)
while True:
if new:
new = False
capture.set(cv2.CAP_PROP_POS_FRAMES, posicao)
_, image_raw = capture.read()
blur = int(cv2.getTrackbarPos('Blur', 'image'))
if blur%2 == 0:
blur += 1
Hmin = int(cv2.getTrackbarPos('Hmin', 'image'))
Hmax = int(cv2.getTrackbarPos('Hmax', 'image'))
Smin = int(cv2.getTrackbarPos('Smin', 'image'))
Smax = int(cv2.getTrackbarPos('Smax', 'image'))
Vmin = int(cv2.getTrackbarPos('Vmin', 'image'))
Vmax = int(cv2.getTrackbarPos('Vmax', 'image'))
Bsize = int(cv2.getTrackbarPos('Bsize', 'image'))
Hmax = int(cv2.getTrackbarPos('Hmax', 'image'))
# TODO: max nao pode ser menor que min para H, S e V
image_blur = cv2.blur(image_raw, (blur, blur))
image_hsv = cv2.cvtColor(image_blur, cv2.COLOR_BGR2HSV)
image_thresh = cv2.inRange(image_hsv,np.array((Hmin, Smin, Vmin)), np.array((Hmax, Smax, Vmax)))
image_thresh = cv2.blur(image_thresh,(blur, blur))
try:
rmin = Bsize - int(Bsize/3)
rmax = Bsize + int(Bsize/3)
print(rmin, rmax)
cir = cv2.HoughCircles(image_thresh,cv2.HOUGH_GRADIENT,1,200,
param1=25,param2=25,minRadius=5,maxRadius=20)
if cir is not None:
for i in cir:
for j in i:
if j[0] > 0:
cv2.circle(image_raw,(j[0],j[1]), int(j[2]), (255,255,0),5)
print(f'x={j[0]} \t y={j[1]}')
except Exception as e:
print('Exception: ', e)
cv2.putText(image_raw,'sair: q',(10,30), font, 1, (0, 0, 255), 2, cv2.LINE_AA)
cv2.putText(image_raw,'reset: r',(10,90), font, 1, (0, 0, 255), 2, cv2.LINE_AA)
cv2.putText(image_raw,'Salvar: s',(10,150), font, 1, (0, 0, 255), 2, cv2.LINE_AA)
image_line1 = np.hstack((image_raw, image_blur))
image_thresh = cv2.cvtColor(image_thresh, cv2.COLOR_GRAY2BGR)
image_line2 = np.hstack((image_thresh, image_blur))
image = np.vstack((image_line1, image_line2))
cv2.imshow("image", image)
key = cv2.waitKey(1) & 0xFF
# r para resset
if key == ord("r"):
posicao = 1
capture.set(cv2.CAP_PROP_POS_FRAMES, posicao)
_, image = capture.read()
elif key == ord('s'):
blur = int(cv2.getTrackbarPos('Blur', 'image'))
if blur%2 == 0:
blur += 1
Hmin = int(cv2.getTrackbarPos('Hmin', 'image'))
Hmax = int(cv2.getTrackbarPos('Hmax', 'image'))
Smin = int(cv2.getTrackbarPos('Smin', 'image'))
Smax = int(cv2.getTrackbarPos('Smax', 'image'))
Vmin = int(cv2.getTrackbarPos('Vmin', 'image'))
Vmax = int(cv2.getTrackbarPos('Vmax', 'image'))
Bsize = int(cv2.getTrackbarPos('Bsize', 'image'))
parser = SafeConfigParser()
parser.read('config.ini')
parser.set('atual', 'blur', str(blur))
parser.set('atual', 'Hmin', str(Hmin))
parser.set('atual', 'Smin', str(Smin))
parser.set('atual', 'Smax', str(Smax))
parser.set('atual', 'Vmin', str(Vmin))
parser.set('atual', 'Vmax', str(Vmax))
parser.set('atual', 'Bsize', str(Bsize))
with open('config.ini', 'w+') as configfile:
parser.write(configfile)
elif key == ord("q"):
break
cv2.destroyAllWindows()
def test_start_and_kill_faraday_server():
"""
Starts the server and then send a signal to kill the
process gracefully.
The idea is to catch a broken faraday-server.py
After sending the signal we wait for 15 seconds and
if the server didn't stop we fail the test also.
"""
server_port = 5988
while daemonize.is_server_running(server_port) and server_port < 6500:
server_port += 1
if server_port > 6500:
raise Exception('No free ports could be found')
if 'POSTGRES_DB' in os.environ:
# I'm on gitlab ci runner
# I will overwrite server.ini
connection_string = 'postgresql+psycopg2://{username}:{password}@postgres/{database}'.format(
username=os.environ['POSTGRES_USER'],
password=os.environ['POSTGRES_PASSWORD'],
database=os.environ['POSTGRES_DB'],
)
faraday_config = SafeConfigParser()
config_path = os.path.expanduser('~/.faraday/config/server.ini')
faraday_config.read(config_path)
try:
faraday_config.add_section('database')
except DuplicateSectionError:
pass
faraday_config.set('database', 'connection_string', connection_string)
with open(config_path, 'w') as faraday_config_file:
faraday_config.write(faraday_config_file)
command = ['faraday-manage', 'create-tables']
subproc = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
subproc.wait()
std, err = subproc.communicate()
assert subproc.returncode == 0, ('Create tables failed!', std, err)
command = [
'faraday-server', '--port', '{0}'.format(server_port), '--debug'
]
subproc = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
start = datetime.now()
while subproc.returncode is None:
now = datetime.now()
delta = now - start
if delta.seconds > 140:
raise UserWarning('Faraday server test timeout!')
if delta.seconds > 30:
subproc.send_signal(signal.SIGTERM)
subproc.wait()
subproc.poll()
subproc.poll()
time.sleep(0.1)
out, err = subproc.communicate()
if subproc.returncode != 0:
log_path = os.path.expanduser('~/.faraday/logs/faraday-server.log')
with open(log_path, 'r') as log_file:
print(log_file.read())
assert subproc.returncode == 0, (out, err, command)
def _sendAdminEmail(self, user, seller, topic, prod_num, message,
request_file, restricted_active, request, product_obj):
config = SafeConfigParser()
config.read('/code/config.ini')
db = MySQLdb.connect(
host=config.get('main',
'mysql_host'), # your host, usually localhost
user=config.get('main', 'mysql_name'), # your username
passwd=config.get('main', 'mysql_pw'), # your password
db=config.get('main', 'mysql_db')) # your database
cur = db.cursor()
log = config.get('main', 'checkout_log_path')
NP_PATH = config.get('main', 'np_path')
username = user.username
user_email = user.email
topic = topic
# Password with 6 characters (lower case + number)
original_password = ''.join(
random.choice(string.ascii_lowercase + string.digits)
for _ in range(12))
if not cur.execute("select (1) from users where username = %s limit 1",
(username, )):
command = NP_PATH + ' ' + '-p' + ' ' + original_password
command_bytes = command.encode('utf-8')
pw_bytes = subprocess.Popen(command_bytes,
stdout=subprocess.PIPE,
shell=True).communicate()[0]
password = pw_bytes.decode().rstrip('\n')
cur.execute(
"insert into users (username,pw,user_id) values (%s,%s,%s)",
(username, password, user.id)) # stdout: ignore '\n'
# Send password to email
subject = 'Your new password'
msg = "Your password to I3 is: " + original_password
email = EmailMessage(subject, msg, to=[user_email])
email.send()
# Record email as notification
notification_box = request.user.get_notification_box()
notification_item = NotificationItem(
notification_box=notification_box, subject=subject, body=msg)
notification_item.save()
# To do: make topic as a product obj that can be linked to
flow_obj = Flow.objects.create(user=request.user,
topic=topic,
direction='in',
state='inactive')
flow_obj.save()
# send to the user which topic is able to pub/sub
# when the topic is unrestricted: insert to acls and send confirmation back to buyer
if not restricted_active:
subject = 'New product purchased'
msg = 'Now you can subscribe to topic: ' + topic + '.'
email = EmailMessage(subject, msg, to=[user_email])
email.send()
# Record email as notification
notification_box = request.user.get_notification_box()
notification_item = NotificationItem(
notification_box=notification_box, subject=subject, body=msg)
notification_item.save()
subject = 'New buyer of an unrestricted topic'
msg = 'Buyer ' + username + ' just bought product ' + topic + '.'
email = EmailMessage(subject, msg, to=[seller.email])
email.send()
# Record email as notification
notification_box = seller.get_notification_box()
notification_item = NotificationItem(
notification_box=notification_box, subject=subject, body=msg)
notification_item.save()
# insert into acls table
rw = 1 # seller: can read and write
if product_obj.sensor_type >= 2:
rw = 2
cur.execute(
"insert into acls (username,topic,rw,user_id, topic_id) values (%s,%s,%s,%s,%s)",
(username, topic, str(rw), user.id, product_obj.id))
# write new sub info to log
with open(log, 'a') as f:
f.write(
str(time.time()) + ': New Sub ' + username + ' ' + topic +
' ' + str(prod_num) + '\n')
else:
restricted_request_obj = RestrictedRequest(
seller=product_obj.seller,
requester=request.user,
product=product_obj,
price=product_obj.price,
quantity=prod_num,
intention=message,
attachment=request_file)
restricted_request_obj.save()
subject = 'New product purchased (to be confirmed)'
msg = 'Waiting seller to confirm purchase of ' + topic + '.'
email = EmailMessage(subject, msg, to=[user_email])
email.send()
# Record email as notification
notification_box = request.user.get_notification_box()
notification_item = NotificationItem(
notification_box=notification_box, subject=subject, body=msg)
notification_item.save()
subject = 'New buyer of a restricted topic'
msg = 'Buyer ' + username + ' just bought product ' + topic + '. You need to approve the purchase.'
email = EmailMessage(subject, msg, to=[seller.email])
email.send()
# Record email as notification
notification_box = seller.get_notification_box()
notification_item = NotificationItem(
notification_box=notification_box, subject=subject, body=msg)
notification_item.save()
db.commit()
def get_options(self, test_args=None):
"""Parse command-line arguments passed to the desiInstall script.
Parameters
----------
test_args : :class:`list`
Normally, this method is called without arguments, and
:data:`sys.argv` is parsed. Arguments should only be passed for
testing purposes.
Returns
-------
:class:`argparse.Namespace`
A simple object containing the parsed options. Also, the
attribute `options` is set.
"""
from argparse import ArgumentParser
log = logging.getLogger(__name__ + '.DesiInstall.get_options')
check_env = {'MODULESHOME': None,
'DESI_PRODUCT_ROOT': None,
'USER': None,
'LANG': None}
for e in check_env:
try:
check_env[e] = environ[e]
except KeyError:
if e == 'DESI_PRODUCT_ROOT' and 'NERSC_HOST' in environ:
log.debug('The environment variable DESI_PRODUCT_ROOT ' +
'is not set, but this is probably not a ' +
'problem at NERSC.')
else:
log.warning(('The environment variable {0} is not ' +
'set!').format(e))
parser = ArgumentParser(description="Install DESI software.",
prog=self.executable)
parser.add_argument('-a', '--anaconda', action='store', dest='anaconda',
default=self.anaconda_version(), metavar='VERSION',
help="Set the version of the DESI+Anaconda software stack.")
parser.add_argument('-b', '--bootstrap', action='store_true',
dest='bootstrap',
help=("Run in bootstrap mode to install the " +
"desiutil product."))
parser.add_argument('-C', '--compile-c', action='store_true',
dest='force_build_type',
help=("Force C/C++ install mode, even if a " +
"setup.py file is detected (WARNING: " +
"this is for experts only)."))
parser.add_argument('-c', '--configuration', action='store',
dest='config_file', default='',
metavar='FILE',
help=("Override built-in configuration with " +
"data from FILE."))
parser.add_argument('-d', '--default', action='store_true',
dest='default',
help='Make this version the default version.')
parser.add_argument('-F', '--force', action='store_true',
dest='force',
help=('Overwrite any existing installation of ' +
'this product/version.'))
parser.add_argument('-k', '--keep', action='store_true',
dest='keep',
help='Keep the exported build directory.')
parser.add_argument('-m', '--module-home', action='store',
dest='moduleshome',
default=check_env['MODULESHOME'],
metavar='DIR',
help='Set or override the value of $MODULESHOME')
parser.add_argument('-M', '--module-dir', action='store',
dest='moduledir',
default='',
metavar='DIR',
help="Install module files in DIR.")
parser.add_argument('-r', '--root', action='store',
dest='root',
default=check_env['DESI_PRODUCT_ROOT'],
metavar='DIR',
help=('Set or override the value of ' +
'$DESI_PRODUCT_ROOT'))
parser.add_argument('-t', '--test', action='store_true',
dest='test',
help=('Test Mode.. Do not actually install ' +
'anything.'))
parser.add_argument('-U', '--username', action='store',
dest='username',
default=check_env['USER'],
metavar='USER',
help="Set svn username to USER.")
parser.add_argument('-v', '--verbose', action='store_true',
dest='verbose',
help='Print extra information.')
parser.add_argument('-V', '--version', action='version',
version='%(prog)s ' + desiutilVersion)
parser.add_argument('-x', '--cross-install', action='store_true',
dest='cross_install',
help=('Make the install available on multiple ' +
'systems (e.g. NERSC).'))
parser.add_argument('product', nargs='?',
default='NO PACKAGE',
help='Name of product to install.')
parser.add_argument('product_version', nargs='?',
default='NO VERSION',
help='Version of product to install.')
if test_args is None: # pragma: no cover
self.options = parser.parse_args()
else:
self.options = parser.parse_args(test_args)
self.ll = logging.INFO
if self.options.verbose or self.options.test or self.test:
self.ll = logging.DEBUG
logging.getLogger(__name__).setLevel(self.ll)
if test_args is not None:
log.debug('Called parse_args() with: {0}'.format(
' '.join(test_args)))
log.debug('Set log level to {0}.'.format(
logging.getLevelName(self.ll)))
self.config = None
if self.options.config_file:
log.debug("Detected configuration file: {0}.".format(self.options.config_file))
c = SafeConfigParser()
status = c.read([self.options.config_file])
if status[0] == self.options.config_file:
self.config = c
log.debug("Successfully parsed {0}.".format(self.options.config_file))
return self.options
python nodotjs/server.py <MODE> <CONFIG_PATH>
Look at `config.ini` for defined modes. Defaults are `production`,
`staging`, and `test`.""")
exit(1)
MODE = sys.argv[1]
if len(sys.argv) > 2:
CONFIG_PATH = sys.argv[2]
else:
CONFIG_PATH = 'config.ini'
print(CONFIG_PATH)
PARSER = SafeConfigParser()
if not len(PARSER.read(CONFIG_PATH)):
print("No config.ini file found in this directory. Writing a config...")
modes = ['production', 'staging', 'test']
for i in range(0, len(modes)): # ew redis dbs made me loop like this
mode = modes[i]
PARSER.add_section(mode)
PARSER.set(mode, 'db', str(i))
PARSER.set(mode, 'cookie_secret', str(uuid.uuid4()))
PARSER.set(mode, 'timeout', '30')
PARSER.set(mode, 'port', '7000')
PARSER.set(mode, 'templates_dir', './templates')
try:
import sys, os
import time
import json
import driver
import shutil
import boto3
import botocore
from boto3.dynamodb.conditions import Key, Attr
# # Import utility helpers
# sys.path.insert(1, os.path.realpath(os.path.pardir))
# import helpers
# Get configuration
from configparser import SafeConfigParser
config = SafeConfigParser(os.environ)
config.read('ann_config.ini')
# Add utility code here
AwsRegionName = config['aws']['AwsRegionName']
TableName = config['aws']['TableName']
CnetId = config['aws']['CnetId']
OutputBucket = config['aws']['OutputBucket']
SNS_JobResults = config['aws']['SNS_JobResults']
SNS_Archive = config['aws']['SNS_Archive']
"""A rudimentary timer for coarse-grained profiling
"""
class Timer(object):
def __init__(self, verbose=True):
def get_conf(conf_path):
"""
Read the configuration file and return it
"""
conf_path = conf_path
parser = SafeConfigParser()
parser.read(conf_path)
parser.add_section('paths')
# Default
parser.set('packages', 'safe_conf', 'True')
if not parser.has_section('settings'):
raise ConfigException('Missing settings section in configuration file')
if not parser.has_option('settings', 'var_path'):
raise ConfigException(
'Missing var_path option in settings section in configuration file'
)
# Create paths
parser.set('paths', 'local_db',
os.path.join(parser.get('settings', 'var_path'), 'packages.db'))
parser.set('paths', 'lock',
os.path.join(parser.get('settings', 'var_path'), 'db.lock'))
parser.set('paths', 'infos',
os.path.join(parser.get('settings', 'var_path'), 'infos'))
parser.set(
'paths', 'web_data',
os.path.join(parser.get('settings', 'var_path'), 'ubik-web.dat'))
# Detect system info
if not parser.has_section('system'):
parser.add_section('system')
if not parser.has_option('system', 'arch'):
if isit.bit32:
parser.set('system', 'arch', 'i386')
elif isit.bit64:
parser.set('system', 'arch', 'x86_64')
else:
parser.set('system', 'arch', parser.get('system', 'arch').lower())
if not parser.has_option('system', 'dist'):
dist = 'unknown'
vers = 'unknown'
if isit.osx:
dist = 'osx'
vers = isit.osx_version
elif isit.linux:
if isit.debian:
dist = "debian"
if isit.debian_version:
vers = isit.debian_version
elif isit.ubuntu:
dist = "ubuntu"
if isit.ubuntu_version:
vers = isit.ubuntu_version.replace('.', '')
elif isit.centos:
dist = "centos"
if isit.centos_version:
vers = isit.centos_version
elif isit.redhat:
dist = "redhat"
if isit.redhat_version:
vers = isit.redhat_version
elif isit.archlinux:
dist = "archlinux"
if isit.archlinux_version:
vers = isit.archlinux_version
parser.set('system', 'dist', dist)
parser.set('system', 'vers', vers)
else:
parser.set('system', 'dist', parser.get('system', 'dist').lower())
if parser.has_option('system', 'vers'):
parser.set('system', 'vers', parser.get('system', 'vers').lower())
return parser
def run(ini_file='TOPKAPI.ini',
verbose=False,
quiet=False,
parallel_exec=True,
nworkers=int(mp.cpu_count() - 1)):
"""Run the model.
Parameters
----------
ini_file : str
The name of the PyTOPKAPI initialization file. This file describes the
locations of the parameter files and model setup options. Default is to
use a file named `TOPKAPI.ini` in the current directory.
verbose : bool
Prints runtime information [default False - don't display runtime
info]. Is independent of the `quiet` keyword argument.
quiet : bool
Toggles whether to display an informational banner at runtime [default
False - display banner]. Is independent of the `verbose` keyword
argument.
nworkers : int
Number of worker processes to spawn for solving each cell's time-series
in parallel. Default is one fewer than CPU count reported by
multiprocessing.
"""
##================================##
## Read the input file (*.ini) ##
##================================##
config = SafeConfigParser()
config.read(ini_file)
##~~~~~~ Numerical_options ~~~~~~##
solve_s = config.getint('numerical_options', 'solve_s')
solve_o = config.getint('numerical_options', 'solve_o')
solve_c = config.getint('numerical_options', 'solve_c')
##~~~~~~~~~~~ input files ~~~~~~~~~~~##
#Param
file_global_param = config.get('input_files', 'file_global_param')
file_cell_param = config.get('input_files', 'file_cell_param')
#Rain
file_rain = config.get('input_files', 'file_rain')
#ETP
file_ET = config.get('input_files', 'file_ET')
#~~~~~~~~~~~ Group (simulated event) ~~~~~~~~~~~##
group_name = config.get('groups', 'group_name')
##~~~~~~ Calibration ~~~~~~##
fac_L = config.getfloat('calib_params', 'fac_L')
fac_Ks = config.getfloat('calib_params', 'fac_Ks')
fac_n_o = config.getfloat('calib_params', 'fac_n_o')
fac_n_c = config.getfloat('calib_params', 'fac_n_c')
##~~~~~~ External flows ~~~~~~##
external_flow = config.getboolean('external_flow', 'external_flow')
if external_flow:
file_Qexternal_flow = config.get('external_flow',
'file_Qexternal_flow')
Xexternal_flow = config.getfloat('external_flow', 'Xexternal_flow')
Yexternal_flow = config.getfloat('external_flow', 'Yexternal_flow')
##~~~~~~~~~~~ output files ~~~~~~~~~~##
file_out = config.get('output_files', 'file_out')
ut.check_file_exist(file_out) #create path_out if it doesn't exist
if os.path.exists(file_out):
first_run = False
else:
first_run = True
append_output = config.getboolean('output_files', 'append_output')
if append_output is True:
fmode = 'a'
else:
fmode = 'w'
##============================##
## Read the forcing data ##
##============================##
if verbose:
print('Read the forcing data')
#~~~~Rainfall
h5_rain = h5py.File(file_rain)
dset_name = '/{}/rainfall'.format(group_name)
rainfall_forcing = h5_rain[dset_name][...]
h5_rain.close()
#~~~~ETr - Reference crop ET
h5_ET = h5py.File(file_ET)
dset_name = '/{}/ETr'.format(group_name)
ETr_forcing = h5_ET[dset_name][...]
#~~~~ETo - Open water potential evap.
dset_name = '/{}/ETo'.format(group_name)
ET0_forcing = h5_ET[dset_name][...]
h5_ET.close()
#~~~~external_flow flows
if external_flow:
external_flow_records = np.loadtxt(file_Qexternal_flow)[:, 5]
else:
external_flow_records = None
##============================##
## Pretreatment of input data ##
##============================##
if verbose:
print('Pretreatment of input data')
#~~~~Read Global parameters file
X, Dt, alpha_s, \
alpha_o, alpha_c, \
A_thres, W_min, W_max = pm.read_global_parameters(file_global_param)
#~~~~Read Cell parameters file
ar_cell_label, ar_coorx, \
ar_coory, channel_flag, \
Xc, ar_dam, \
ar_tan_beta, ar_tan_beta_channel, \
ar_L, Ks, \
ar_theta_r, ar_theta_s, \
ar_n_o, ar_n_c, \
ar_cell_down, ar_pVs_t0, \
ar_Vo_t0, ar_Qc_t0, \
Kc, psi_b, lamda = pm.read_cell_parameters(file_cell_param)
#~~~~Number of cell in the catchment
nb_cell = len(ar_cell_label)
#~~~~Computation of cell order
node_hierarchy = pm.compute_node_hierarchy(ar_cell_label, ar_cell_down)
ar_label_sort = pm.sort_cell(ar_cell_label, ar_cell_down)
#~~~~Computation of upcells
li_cell_up = pm.direct_up_cell(ar_cell_label, ar_cell_down, ar_label_sort)
#~~~~Computation of drained area
ar_A_drained = pm.drained_area(ar_label_sort, li_cell_up, X)
#~~~~Apply calibration factors to the parameter values
ar_L = ar_L * fac_L
Ks = Ks * fac_Ks
ar_n_o = ar_n_o * fac_n_o
ar_n_c = ar_n_c * fac_n_c
if verbose:
print('Max L=', max(ar_L))
print('Max Ks=', max(Ks))
print('Max n_o=', max(ar_n_o))
print('Max n_c=', max(ar_n_c))
#~~~~Computation of model parameters from physical parameters
Vsm, b_s, b_o, \
W, b_c = pm.compute_cell_param(X, Xc, Dt, alpha_s,
alpha_o, alpha_c, nb_cell,
A_thres, W_max, W_min,
channel_flag, ar_tan_beta,
ar_tan_beta_channel, ar_L,
Ks, ar_theta_r, ar_theta_s,
ar_n_o, ar_n_c, ar_A_drained)
#~~~~Look for the cell of external_flow tunnel
if external_flow:
cell_external_flow = ut.find_cell_coordinates(ar_cell_label,
Xexternal_flow,
Yexternal_flow, ar_coorx,
ar_coory, channel_flag)
if verbose:
print('external flows will be taken into account for cell no',\
cell_external_flow, ' coordinates ('\
,Xexternal_flow,',',Yexternal_flow,')')
else:
cell_external_flow = None
#~~~~Number of simulation time steps
nb_time_step = rainfall_forcing.shape[0]
##=============================##
## Variable array definition ##
##=============================##
## Initialisation of the reservoirs
#Matrix of soil,overland and channel store at the begining of the time step
if append_output and not first_run:
if verbose:
print('Initialize from simulation file')
h5file_in = h5py.File(file_out)
Vs_t0 = h5file_in['/Soil/V_s'][-1, :]
Vc_t0 = h5file_in['/Channel/V_c'][-1, :]
Vo_t0 = h5file_in['/Overland/V_o'][-1, :]
h5file_in.close()
else:
if verbose:
print('Initialize from parameters')
Vs_t0 = fl.initial_volume_soil(ar_pVs_t0, Vsm)
Vo_t0 = ar_Vo_t0
Vc_t0 = fl.initial_volume_channel(ar_Qc_t0, W, X, ar_n_c)
##=============================##
## HDF5 output file definition ##
##=============================##
h5file, dset_Vs, dset_Vo, dset_Vc, \
dset_Qs_out, dset_Qo_out, dset_Qc_out, \
dset_Q_down, dset_ET_out, dset_Ec_out \
= ut.open_simulation_file(file_out, fmode,
Vs_t0, Vo_t0, Vc_t0, no_data,
nb_cell, nb_time_step,
append_output, first_run)
eff_theta = ar_theta_s - ar_theta_r
##===========================##
## Core of the Model ##
##===========================##
if not quiet:
ut.show_banner(ini_file, nb_cell, nb_time_step)
progress_desc = 'Simulation'
else:
progress_desc = 'PyTOPKAPI v{}'.format(pytopkapi.__version__)
# prepare parameter dict
exec_params = {
'nb_cell': nb_cell,
'nb_time_step': nb_time_step,
'progress_desc': progress_desc,
'Dt': Dt,
'rainfall_forcing': rainfall_forcing,
'ETr_forcing': ETr_forcing,
'ET0_forcing': ET0_forcing,
'psi_b': psi_b,
'lamda': lamda,
'eff_theta': eff_theta,
'Ks': Ks,
'X': X,
'b_s': b_s,
'b_o': b_o,
'b_c': b_c,
'alpha_s': alpha_s,
'alpha_o': alpha_o,
'alpha_c': alpha_c,
'Vs_t0': Vs_t0,
'Vo_t0': Vo_t0,
'Vc_t0': Vc_t0,
'Vsm': Vsm,
'dset_Vs': dset_Vs,
'dset_Vo': dset_Vo,
'dset_Vc': dset_Vc,
'dset_Qs_out': dset_Qs_out,
'dset_Qo_out': dset_Qo_out,
'dset_Qc_out': dset_Qc_out,
'dset_Q_down': dset_Q_down,
'dset_ET_out': dset_ET_out,
'dset_Ec_out': dset_Ec_out,
'solve_s': solve_s,
'solve_o': solve_o,
'solve_c': solve_c,
'channel_flag': channel_flag,
'W': W,
'Xc': Xc,
'Kc': Kc,
'cell_external_flow': cell_external_flow,
'external_flow_records': external_flow_records,
'node_hierarchy': node_hierarchy,
'li_cell_up': li_cell_up,
'nworkers': nworkers
}
if not parallel_exec:
# Serial execution. Solve by timestep in a single process.
# Outer loop timesteps - inner loop cells
_serial_execute(exec_params)
else:
# Parallel execution. Solve by cell using multiple processes.
# Outer loop cells - inner loop timesteps
_parallel_execute(exec_params)
h5file.close()
def get_default_config(work_dir='.', section='recon-neonatal-cortex'):
"""Get default configuration."""
# directories
session_dir = os.path.join('%(WorkDir)s', '%(SubjectId)s-%(SessionId)s')
input_dir = os.path.join(session_dir, 'input')
temp_dir = os.path.join(session_dir, 'temp')
mesh_dir = os.path.join(session_dir, 'meshes')
logs_dir = os.path.join(session_dir, 'logs')
# configuration
config = SafeConfigParser(defaults={'work_dir': work_dir, 'temp_dir': temp_dir})
section = args.section
config.add_section(section)
config.set(section, 'logs_dir', logs_dir)
# input file paths
config.set(section, 'input_t1w_image', os.path.join(input_dir, 't1w-image.nii.gz'))
config.set(section, 'input_t2w_image', os.path.join(input_dir, 't2w-image.nii.gz'))
config.set(section, 'input_brain_mask', os.path.join(input_dir, 'brain-mask.nii.gz'))
config.set(section, 'input_labels_image', os.path.join(input_dir, 'brain-labels.nii.gz'))
config.set(section, 'input_tissues_image', os.path.join(input_dir, 'tissue-labels.nii.gz'))
# intermediate file paths
config.set(section, 't1w_image', os.path.join(temp_dir, 't1w-image.nii.gz'))
config.set(section, 't2w_image', os.path.join(temp_dir, 't2w-image.nii.gz'))
config.set(section, 'brain_mask', os.path.join(temp_dir, 'brain-mask.nii.gz'))
config.set(section, 'white_matter_mask', os.path.join(temp_dir, 'white-matter-mask.nii.gz'))
config.set(section, 'gray_matter_mask', os.path.join(temp_dir, 'gray-matter-mask.nii.gz'))
config.set(section, 'deep_gray_matter_mask', os.path.join(temp_dir, 'deep-gray-matter-mask.nii.gz'))
config.set(section, 'corpus_callosum_mask', os.path.join(temp_dir, 'corpus-callosum-mask.nii.gz'))
config.set(section, 'ventricles_mask', os.path.join(temp_dir, 'ventricles-mask.nii.gz'))
config.set(section, 'ventricles_dmap', os.path.join(temp_dir, 'ventricles-dmap.nii.gz'))
config.set(section, 'regions_mask', os.path.join(temp_dir, 'regions.nii.gz'))
config.set(section, 'cortical_hull_dmap', os.path.join(temp_dir, 'cortical-hull-dmap.nii.gz'))
# output file paths
config.set(section, 'brain_mesh', os.path.join(mesh_dir, 'brain.vtp'))
config.set(section, 'bs_cb_mesh', os.path.join(mesh_dir, 'bs+cb.vtp'))
config.set(section, 'internal_mesh', os.path.join(mesh_dir, 'internal.vtp'))
config.set(section, 'cerebrum_mesh', os.path.join(mesh_dir, 'cerebrum.vtp'))
config.set(section, 'right_cerebrum_mesh', os.path.join(mesh_dir, 'cerebrum-rh.vtp'))
config.set(section, 'left_cerebrum_mesh', os.path.join(mesh_dir, 'cerebrum-lh.vtp'))
config.set(section, 'white_mesh', os.path.join(mesh_dir, 'white.vtp'))
config.set(section, 'right_white_mesh', os.path.join(mesh_dir, 'white-rh.vtp'))
config.set(section, 'left_white_mesh', os.path.join(mesh_dir, 'white-lh.vtp'))
config.set(section, 'pial_mesh', os.path.join(mesh_dir, 'pial.vtp'))
config.set(section, 'right_pial_mesh', os.path.join(mesh_dir, 'pial-rh.vtp'))
config.set(section, 'left_pial_mesh', os.path.join(mesh_dir, 'pial-lh.vtp'))
# parameters of subdivide-brain-image step to create regions mask
config.set(section, 'subcortex_closing', '5')
config.set(section, 'brainstem_closing', '5')
config.set(section, 'cerebellum_closing', '5')
# default labels used when none specified are those of Draw-EM (all_labels)
config.set(section, 'white_matter_labels', '51..82')
config.set(section, 'gray_matter_labels', '5..16,20..39')
config.set(section, 'deep_gray_matter_labels', '1..4,40..47,85..87')
config.set(section, 'lateral_ventricles_labels', '49,50')
config.set(section, 'corpus_callosum_labels', '48')
config.set(section, 'inter_hemisphere_labels', '40..47,85..87')
config.set(section, 'brainstem_labels', '19')
config.set(section, 'cerebellum_labels', '17,18')
rh_labels = [] # right hemisphere structures
rh_labels.extend(range(2, 48, 2))
rh_labels.extend(range(50, 63, 2))
rh_labels.extend(range(63, 82, 2))
rh_labels.append(86)
config.set(section, 'right_hemisphere_labels', ','.join([str(x) for x in rh_labels]))
lh_labels = [] # left hemisphere structures
lh_labels.extend(range(1, 18, 2))
lh_labels.extend(range(21, 62, 2))
lh_labels.extend(range(64, 83, 2))
lh_labels.append(87)
config.set(section, 'left_hemisphere_labels', ','.join([str(x) for x in lh_labels]))
return config
def __init__(self, filename):
self.settings = SafeConfigParser()
self.filename = filename
self.load()
def initialise(**kwargs):
Config.__parser_config = SafeConfigParser()
p = os.path.dirname(__file__)
file = os.path.join(p, 'project.config')
Config.__parser_config.read(file)
def generate_substitutions_from_package(package,
os_name,
os_version,
ros_distro,
installation_prefix='/usr',
deb_inc=0,
peer_packages=None,
releaser_history=None,
fallback_resolver=None,
native=False):
peer_packages = peer_packages or []
data = {}
# Name, Version, Description
data['Name'] = package.name
data['Version'] = package.version
data['Description'] = format_description(package.description)
# Websites
websites = [str(url) for url in package.urls if url.type == 'website']
homepage = websites[0] if websites else ''
if homepage == '':
warning("No homepage set, defaulting to ''")
data['Homepage'] = homepage
# Debian Increment Number
data['DebianInc'] = '' if native else '-{0}'.format(deb_inc)
# Debian Package Format
data['format'] = 'native' if native else 'quilt'
# Package name
data['Package'] = sanitize_package_name(package.name)
# Installation prefix
data['InstallationPrefix'] = installation_prefix
# Resolve dependencies
depends = package.run_depends + package.buildtool_export_depends
build_depends = package.build_depends + package.buildtool_depends + package.test_depends
unresolved_keys = depends + build_depends + package.replaces + package.conflicts
# The installer key is not considered here, but it is checked when the keys are checked before this
resolved_deps = resolve_dependencies(
unresolved_keys, os_name, os_version, ros_distro,
peer_packages + [d.name for d in package.replaces + package.conflicts],
fallback_resolver)
data['Depends'] = sorted(set(format_depends(depends, resolved_deps)))
data['BuildDepends'] = sorted(
set(format_depends(build_depends, resolved_deps)))
data['Replaces'] = sorted(
set(format_depends(package.replaces, resolved_deps)))
data['Conflicts'] = sorted(
set(format_depends(package.conflicts, resolved_deps)))
# Build-type specific substitutions.
build_type = package.get_build_type()
if build_type == 'catkin':
pass
elif build_type == 'cmake':
pass
elif build_type == 'ament_cmake':
pass
elif build_type == 'ament_python':
# Don't set the install-scripts flag if it's already set in setup.cfg.
package_path = os.path.abspath(os.path.dirname(package.filename))
setup_cfg_path = os.path.join(package_path, 'setup.cfg')
data['pass_install_scripts'] = True
if os.path.isfile(setup_cfg_path):
setup_cfg = SafeConfigParser()
setup_cfg.read([setup_cfg_path])
if (setup_cfg.has_option('install', 'install-scripts')
or setup_cfg.has_option('install', 'install_scripts')):
data['pass_install_scripts'] = False
else:
error("Build type '{}' is not supported by this version of bloom.".
format(build_type),
exit=True)
# Set the distribution
data['Distribution'] = os_version
# Use the time stamp to set the date strings
stamp = datetime.datetime.now(tz.tzlocal())
data['Date'] = stamp.strftime('%a, %d %b %Y %T %z')
data['YYYY'] = stamp.strftime('%Y')
# Maintainers
maintainers = []
for m in package.maintainers:
maintainers.append(str(m))
data['Maintainer'] = maintainers[0]
data['Maintainers'] = ', '.join(maintainers)
# Changelog
changelogs = get_changelogs(package, releaser_history)
if changelogs and package.version not in [x[0] for x in changelogs]:
warning("")
warning(
"A CHANGELOG.rst was found, but no changelog for this version was found."
)
warning(
"You REALLY should have a entry (even a blank one) for each version of your package."
)
warning("")
if not changelogs:
# Ensure at least a minimal changelog
changelogs = []
if package.version not in [x[0] for x in changelogs]:
changelogs.insert(0, (
package.version, get_rfc_2822_date(datetime.datetime.now()),
' * Autogenerated, no changelog for this version found in CHANGELOG.rst.',
package.maintainers[0].name, package.maintainers[0].email))
bad_changelog = False
# Make sure that the first change log is the version being released
if package.version != changelogs[0][0]:
error("")
error("The version of the first changelog entry '{0}' is not the "
"same as the version being currently released '{1}'.".format(
package.version, changelogs[0][0]))
bad_changelog = True
# Make sure that the current version is the latest in the changelog
for changelog in changelogs:
if parse_version(package.version) < parse_version(changelog[0]):
error("")
error(
"There is at least one changelog entry, '{0}', which has a "
"newer version than the version of package '{1}' being released, '{2}'."
.format(changelog[0], package.name, package.version))
bad_changelog = True
if bad_changelog:
error("This is almost certainly by mistake, you should really take a "
"look at the changelogs for the package you are releasing.")
error("")
if not maybe_continue('n', 'Continue anyways'):
sys.exit("User quit.")
data['changelogs'] = changelogs
# Use debhelper version 7 for oneric, otherwise 9
data['debhelper_version'] = 7 if os_version in ['oneiric'] else 9
# Summarize dependencies
summarize_dependency_mapping(data, depends, build_depends, resolved_deps)
# Copyright
licenses = []
separator = '\n' + '=' * 80 + '\n\n'
for l in package.licenses:
if hasattr(l, 'file') and l.file is not None:
license_file = os.path.join(os.path.dirname(package.filename),
l.file)
if not os.path.exists(license_file):
error("License file '{}' is not found.".format(license_file),
exit=True)
license_text = open(license_file, 'r').read()
if not license_text.endswith('\n'):
license_text += '\n'
licenses.append(license_text)
data['Copyright'] = separator.join(licenses)
def convertToUnicode(obj):
if sys.version_info.major == 2:
if isinstance(obj, str):
return unicode(obj.decode('utf8'))
elif isinstance(obj, unicode):
return obj
else:
if isinstance(obj, bytes):
return str(obj.decode('utf8'))
elif isinstance(obj, str):
return obj
if isinstance(obj, list):
for i, val in enumerate(obj):
obj[i] = convertToUnicode(val)
return obj
elif isinstance(obj, type(None)):
return None
elif isinstance(obj, tuple):
obj_tmp = list(obj)
for i, val in enumerate(obj_tmp):
obj_tmp[i] = convertToUnicode(obj_tmp[i])
return tuple(obj_tmp)
elif isinstance(obj, int):
return obj
raise RuntimeError('need to deal with type %s' % (str(type(obj))))
for item in data.items():
data[item[0]] = convertToUnicode(item[1])
return data
def confParser(self):
'''Parse a config file to find potential conf file settings.
@author: rsn
'''
success = False
# This script should be run from [stonixroot]/src/MacBuild. We must
os.chdir("../..")
self.STONIX_ROOT = os.getcwd()
os.chdir("src/MacBuild")
macbuild_root = os.getcwd()
myconf = os.path.join(macbuild_root, 'macbuild.conf')
print(myconf)
if os.path.isfile(myconf):
self.parser = SafeConfigParser()
candidates = [myconf, 'not_a_real_conf.conf']
found = self.parser.read(candidates)
missing = set(candidates) - set(found)
try:
dict1 = {}
for section in self.parser.sections():
dict1[section] = self.configSectionMap(section)
print(dict1)
except:
#####
# happens if there was a problem attempting to read the config
# file, Initializing class variables.
self.STONIX = "stonix"
self.STONIXICON = "stonix_icon"
self.STONIXVERSION = self.APPVERSION
self.STONIX4MAC = "stonix4mac"
self.STONIX4MACICON = "stonix_icon"
self.STONIX4MACVERSION = self.APPVERSION
#-- Internal libraries
from .macbuildlib import macbuildlib
self.mbl = macbuildlib(self.logger)
self.PYUIC = self.mbl.getpyuicpath()
self.codesignVerbose = 'vvvv'
self.codesignDeep = True
else:
#####
# Config file read, initializing class variables.
self.STONIX = dict1['stonix']['app']
self.STONIXICON = dict1['stonix']['app_icon']
self.STONIXVERSION = dict1['stonix']['app_version']
self.STONIX4MAC = dict1['stonix']['wrapper']
self.STONIX4MACICON = dict1['stonix']['wrapper_icon']
self.STONIX4MACVERSION = dict1['stonix']['wrapper_version']
self.PYUIC = dict1['libpaths']['pyuic']
self.PYPATHS = dict1['libpaths']['pythonpath'].split(':')
self.logger.log(
lp.INFO, 'attempting to get codesigning information...')
self.codesignVerbose = dict1['codesign']['verbose']
if re.match('^True$', dict1['codesign']['deep']):
self.codesignDeep = True
else:
self.codesignDeep = False
self.logger.log(lp.INFO, "Grabbed codesign info...")
for path in self.PYPATHS:
sys.path.append(path)
#-- Internal libraries
try:
from .macbuildlib import macbuildlib
self.mbl = macbuildlib(self.logger, self.PYPATHS)
except Exception as err:
raise
self.logger.log(lp.INFO, "... macbuildlib loaded ...")
finally:
self.hiddenimports = self.mbl.getHiddenImports()
self.logger.log(lp.DEBUG,
"Hidden imports: " + str(self.hiddenimports))
success = True
return success
def _parse(paths, prepos, ignored_map, ignored_location_map, local_config,
portdir):
"""Parse files in paths to load config"""
parser = SafeConfigParser()
# use read_file/readfp in order to control decoding of unicode
try:
# Python >=3.2
read_file = parser.read_file
source_kwarg = 'source'
except AttributeError:
read_file = parser.readfp
source_kwarg = 'filename'
recursive_paths = []
for p in paths:
if isinstance(p, basestring):
recursive_paths.extend(_recursive_file_list(p))
else:
recursive_paths.append(p)
for p in recursive_paths:
if isinstance(p, basestring):
f = None
try:
f = io.open(_unicode_encode(p,
encoding=_encodings['fs'],
errors='strict'),
mode='r',
encoding=_encodings['repo.content'],
errors='replace')
except EnvironmentError:
pass
else:
# The 'source' keyword argument is needed since otherwise
# ConfigParser in Python <3.3.3 may throw a TypeError
# because it assumes that f.name is a native string rather
# than binary when constructing error messages.
kwargs = {source_kwarg: p}
read_file(f, **portage._native_kwargs(kwargs))
finally:
if f is not None:
f.close()
elif isinstance(p, io.StringIO):
kwargs = {source_kwarg: "<io.StringIO>"}
read_file(p, **portage._native_kwargs(kwargs))
else:
raise TypeError(
"Unsupported type %r of element %r of 'paths' argument" %
(type(p), p))
prepos['DEFAULT'] = RepoConfig("DEFAULT",
parser.defaults(),
local_config=local_config)
for sname in parser.sections():
optdict = {}
for oname in parser.options(sname):
optdict[oname] = parser.get(sname, oname)
repo = RepoConfig(sname, optdict, local_config=local_config)
if repo.sync_type is not None and repo.sync_uri is None:
writemsg_level("!!! %s\n" % _(
"Repository '%s' has sync-type attribute, but is missing sync-uri attribute"
) % sname,
level=logging.ERROR,
noiselevel=-1)
continue
if repo.sync_uri is not None and repo.sync_type is None:
writemsg_level("!!! %s\n" % _(
"Repository '%s' has sync-uri attribute, but is missing sync-type attribute"
) % sname,
level=logging.ERROR,
noiselevel=-1)
continue
if repo.sync_type not in (None, "cvs", "git", "rsync"):
writemsg_level("!!! %s\n" % _(
"Repository '%s' has sync-type attribute set to unsupported value: '%s'"
) % (sname, repo.sync_type),
level=logging.ERROR,
noiselevel=-1)
continue
if repo.sync_type == "cvs" and repo.sync_cvs_repo is None:
writemsg_level("!!! %s\n" % _(
"Repository '%s' has sync-type=cvs, but is missing sync-cvs-repo attribute"
) % sname,
level=logging.ERROR,
noiselevel=-1)
continue
# For backward compatibility with locations set via PORTDIR and
# PORTDIR_OVERLAY, delay validation of the location and repo.name
# until after PORTDIR and PORTDIR_OVERLAY have been processed.
prepos[sname] = repo
def build_erfs_survey_collection():
# self.hdf5_filename = os.path.join(os.path.dirname(ERF_HDF5_DATA_DIR),'erf','erf.h5')
erfs_survey_collection = SurveyCollection()
for year in range(2006, 2010):
surveys = erfs_survey_collection.surveys
yr = str(year)[2:]
yr1 = str(year + 1)[2:]
eec_variables = [
'noi', 'noicon', 'noindiv', 'noiper', 'noimer', 'ident', 'naia',
'naim', 'lien', 'acteu', 'stc', 'contra', 'titc', 'mrec', 'forter',
'rstg', 'retrai', 'lpr', 'cohab', 'sexe', 'agepr', 'rga', 'statut',
'txtppb', 'encadr', 'prosa', 'nbsala', 'chpub', 'dip11'
]
eec_rsa_variables = ["sp0" + str(i)
for i in range(0, 10)] + ["sp10", "sp11"] + [
'sitant', 'adeben', 'datant', 'raistp',
'amois', 'adfdap', 'ancentr', 'ancchom',
'dimtyp', 'rabsp', 'raistp', 'rdem',
'ancinatm'
]
eec_aah_variables = ["rc1rev", "maahe"]
eec_variables += eec_rsa_variables + eec_aah_variables
erf_tables = {
"erf_menage": {
"RData_filename": "menage" + yr,
"year": year,
"variables": None,
},
"eec_menage": {
"RData_filename": "mrf" + yr + "e" + yr + "t4",
"year": year,
"variables": None,
},
"foyer": {
"RData_filename": "foyer" + yr,
"year": year,
"variables": None,
},
"erf_indivi": {
"RData_filename":
"indivi" + yr,
"year":
year,
"variables": [
'noi',
'noindiv',
'ident',
'declar1',
'quelfic',
'persfip',
'declar2',
'persfipd',
'wprm',
"zsali",
"zchoi",
"ztsai",
"zreti",
"zperi",
"zrsti",
"zalri",
"zrtoi",
"zragi",
"zrici",
"zrnci",
"zsalo",
"zchoo",
"ztsao",
"zreto",
"zpero",
"zrsto",
"zalro",
"zrtoo",
"zrago",
"zrico",
"zrnco",
],
},
"eec_indivi": {
"RData_filename": "irf" + yr + "e" + yr + "t4",
"year": year,
"variables": eec_variables,
},
"eec_cmp_1": {
"RData_filename": "icomprf" + yr + "e" + yr1 + "t1",
"year": year,
"variables": eec_variables,
},
"eec_cmp_2": {
"RData_filename": "icomprf" + yr + "e" + yr1 + "t2",
"year": year,
"variables": eec_variables,
},
"eec_cmp_3": {
"RData_filename": "icomprf" + yr + "e" + yr1 + "t3",
"year": year,
"variables": eec_variables,
},
}
# Build absolute path for RData_filename
from configparser import SafeConfigParser
parser = SafeConfigParser()
config_local_ini = os.path.join(CONFIG_DIR, 'config_local.ini')
config_ini = os.path.join(CONFIG_DIR, 'config.ini')
found = parser.read(config_local_ini, config_ini)
data_directory = parser.get('data', 'input_directory')
for table in erf_tables:
table["RData_filename"] = os.path.join(
os.path.dirname(data_directory), 'R', 'erf')
def initialize(self):
"""
Initialize survey data
"""
self.initialize_erf(tables=tables)
# self.initialize_logement()
def initialize_erf(self, tables=None):
"""
"""
year = self.year
erf = SurveyDescription()
yr = str(year)[2:]
yr1 = str(year + 1)[2:]
erf_tables_to_process = {
"erf_menage": "menage" + yr,
"eec_menage": "mrf" + yr + "e" + yr + "t4",
"foyer": "foyer" + yr,
"erf_indivi": "indivi" + yr,
"eec_indivi": "irf" + yr + "e" + yr + "t4",
"eec_cmp_1": "icomprf" + yr + "e" + yr1 + "t1",
"eec_cmp_2": "icomprf" + yr + "e" + yr1 + "t2",
"eec_cmp_3": "icomprf" + yr + "e" + yr1 + "t3"
}
RData_dir = os.path.join(os.path.dirname(DATA_DIR), 'R', 'erf')
variables = [
'noi', 'noindiv', 'ident', 'declar1', 'quelfic', 'persfip',
'declar2', 'persfipd', 'wprm', "zsali", "zchoi", "ztsai", "zreti",
"zperi", "zrsti", "zalri", "zrtoi", "zragi", "zrici", "zrnci",
"zsalo", "zchoo", "ztsao", "zreto", "zpero", "zrsto", "zalro",
"zrtoo", "zrago", "zrico", "zrnco"
]
variables_eec = [
'noi', 'noicon', 'noindiv', 'noiper', 'noimer', 'ident', 'naia',
'naim', 'lien', 'acteu', 'stc', 'contra', 'titc', 'mrec', 'forter',
'rstg', 'retrai', 'lpr', 'cohab', 'sexe', 'agepr', 'rga', 'statut',
'txtppb', 'encadr', 'prosa', 'nbsala', 'chpub', 'dip11'
]
variables_eec_rsa = ["sp0" + str(i)
for i in range(0, 10)] + ["sp10", "sp11"] + [
'sitant', 'adeben', 'datant', 'raistp',
'amois', 'adfdap', 'ancentr', 'ancchom',
'dimtyp', 'rabsp', 'raistp', 'rdem',
'ancinatm'
]
variables_eec_aah = ["rc1rev", "maahe"]
variables_eec += variables_eec_rsa + variables_eec_aah
erf_tables = {
"erf_menage": {
"RData_filename": "menage" + yr,
"variables": None
},
"eec_menage": {
"RData_filename": "mrf" + yr + "e" + yr + "t4",
"variables": None
},
"foyer": {
"RData_filename": "foyer" + yr,
"variables": None
},
"erf_indivi": {
"RData_filename": "indivi" + yr,
"variables": variables
},
"eec_indivi": {
"RData_filename": "irf" + yr + "e" + yr + "t4",
"variables": variables_eec
},
"eec_cmp_1": {
"RData_filename": "icomprf" + yr + "e" + yr1 + "t1",
"variables": variables_eec
},
"eec_cmp_2": {
"RData_filename": "icomprf" + yr + "e" + yr1 + "t2",
"variables": variables_eec
},
"eec_cmp_3": {
"RData_filename": "icomprf" + yr + "e" + yr1 + "t3",
"variables": variables_eec
}
}
RData_dir = os.path.join(os.path.dirname(DATA_DIR), 'R', 'erf')
if tables is None:
erf_tables_to_process = erf_tables
else:
erf_tables_to_process = tables
for name in erf_tables_to_process:
erf.insert_table(name=name,
RData_filename=RData_filename,
RData_dir=RData_dir,
variables=variables)
self.surveys["erf"] = erf
def initialize_logement(self):
"""
"""
year = self.year
lgt = SurveyDescription()
yr = str(year)[2:]
yr1 = str(year + 1)[2:]
if yr == "03":
lgt_men = "menage"
lgt_logt = None
renameidlgt = dict(ident='ident')
elif yr in ["06", "07", "08", "09"]:
lgt_men = "menage1"
lgt_lgt = "logement"
renameidlgt = dict(idlog='ident')
lgt_tables_to_process = {
"adresse": "adresse",
"lgt_menage": lgt_men,
"lgt_logt": lgt_lgt
}
RData_dir = os.path.join(os.path.dirname(DATA_DIR), 'R', 'logement')
for name, RData_filename in lgt_tables_to_process.items():
lgt.insert_table(name=name,
RData_filename=RData_filename,
RData_dir=RData_dir)
self.surveys["lgt"] = lgt
def initialize_patrimoine(self, year):
"""
TODO:
"""
pat_tables_to_process = {
"pat_individu": "individu",
"pat_menage": "meange",
"pat_produit": "produit",
"pat_transmission": "transm"
}
pat_data_dir = os.path.join(os.path.dirname(DATA_DIR), 'R',
'patrimoine')
pat = {
"name": "patrimoine",
"data_dir": os.path.join(os.path.dirname(DATA_DIR), 'R',
'patrimoine'),
"tables_to_process": pat_tables_to_process
}
def set_config(self, **kwargs):
"""
Set configuration parameters
Parameters
----------
year : int, default None
year of the survey
"""
if self.year is not None:
year = self.year
else:
raise Exception("year should be defined")
store = HDFStore(self.hdf5_filename)
for survey_name, description in self.surveys.items():
for destination_table_name, tables in description.tables.items():
data_dir = tables["RData_dir"]
R_table_name = tables["RData_filename"]
try:
variables = tables["variables"]
except Exception:
variables = None
print(variables)
self.store_survey(survey_name, R_table_name,
destination_table_name, data_dir, variables)
def store_survey(self,
survey_name,
R_table_name,
destination_table_name,
data_dir,
variables=None,
force_recreation=True):
"""
Store a R data table in an HDF5 file
Parameters
----------
survey_name : string
the name of the survey
R_table_name : string
the name of the R data table
destination_table_name : string
the name of the table in the HDFStore
data_dir : path
the directory where to find the RData file
variables : list of string, default None
When not None, list of the variables to keep
"""
gc.collect()
year = self.year
def get_survey_year(survey_name, year):
if survey_name == "logement":
if year == 2003:
return 2003
elif year in range(2006, 2010):
return 2006
if survey_name == "patrimoine":
return 2004
else:
return year
print("creating %s" % (destination_table_name))
table_Rdata = R_table_name + ".Rdata"
filename = os.path.join(data_dir,
str(get_survey_year(survey_name, year)),
table_Rdata)
print(filename)
if not os.path.isfile(filename):
raise Exception("filename do not exists")
rpy.r.load(filename)
stored_table = com.load_data(R_table_name)
store = HDFStore(self.hdf5_filename)
store_path = str(self.year) + "/" + destination_table_name
if store_path in store:
if force_recreation is not True:
print(store_path + "already exists, do not re-create and exit")
store.close()
return
if variables is not None:
print(store)
print(store_path)
print(variables)
variables_stored = list(
set(variables).intersection(set(stored_table.columns)))
print(list(set(variables).difference((set(stored_table.columns)))))
store[store_path] = stored_table[variables_stored]
else:
store[store_path] = stored_table
store.close()
del stored_table
gc.collect()
def get_value(self, variable, table=None):
"""
Get value
Parameters
----------
variable : string
name of the variable
table : string, default None
name of the table where to get variable
Returns
-------
df : DataFrame, default None
A DataFrame containing the variable
"""
df = self.get_values([variable], table)
return df
def get_values(self, variables=None, table=None):
"""
Get values
Parameters
----------
variables : list of strings, default None
list of variables names, if None return the whole table
table : string, default None
name of the table where to get the variables
Returns
-------
df : DataFrame, default None
A DataFrame containing the variables
"""
store = HDFStore(self.hdf5_filename)
df = store[str(self.year) + "/" + table]
# If no variables read the whole table
if variables is None:
return df
diff = set(variables) - set(df.columns)
if diff:
raise Exception("The following variable(s) %s are missing" % diff)
variables = list(set(variables).intersection(df.columns))
df = df[variables]
return df
def get_of_value(self, variable, table=None):
"""
Get value
Parameters
----------
variable : string
name of the variable
table : string, default None
name of the table where to get variable
Returns
-------
df : DataFrame, default None
A DataFrame containing the variable
"""
df = self.get_of_values([variable], table)
return df
def get_of_values(self, variables=None, table=None):
"""
Get values
Parameters
----------
variables : list of strings, default None
list of variables names, if None return the whole table
table : string, default None
name of the table where to get the variables
Returns
-------
df : DataFrame, default None
A DataFrame containing the variables
"""
store = HDFStore(self.hdf5_filename)
df = store[str(self.year) + "/" + table]
# If no variables read the whole table
if variables is None:
return df
from openfisca_france.data.erf import get_erf2of, get_of2erf
of2erf = get_of2erf()
to_be_renamed_variables = set(of2erf.keys()).intersection(variables)
renamed_variables = []
for variable in to_be_renamed_variables:
renamed_variables.append(of2erf[variable])
if renamed_variables:
variables = list(
set(variables).difference(
to_be_renamed_variables)) + renamed_variables
# if table is None:
# for test_table in self.tables.keys:
# if set(variables) < set(self.tables[test_table].columns):
# table = test_table
# print("using guessed table :", table)
# break
#
# if table is None:
# print("varname not found in any tables")
# df = None
# else:
variables = list(set(variables).intersection(df.columns))
df = df[variables]
# rename variables according to their name in openfisca
erf2of = get_erf2of()
to_be_renamed_variables = set(list(
erf2of.values())).intersection(variables)
if to_be_renamed_variables:
for var in to_be_renamed_variables:
df.rename(columns={var: erf2of[var]}, inplace=True)
return df
