def parse(filename='default.conf'):
config = configparser.ConfigParser()
config._interpolation = configparser.ExtendedInterpolation()
config.read(filename)
return config
parser.add_argument(
"--trapping_clips",
help=
"A csv containing urls of all trapping clips. Columns 'trapping_clips'"
"and 'trapping_ans'")
parser.add_argument('--num_responses_per_clip',
help='Number of response per clip required',
default=5,
type=int)
# check input arguments
args = parser.parse_args()
assert os.path.exists(args.cfg), f"No config file in {args.cfg}"
cfg = CP.ConfigParser()
cfg._interpolation = CP.ExtendedInterpolation()
cfg.read(args.cfg)
if args.clips:
assert os.path.exists(
args.clips), f"No csv file containing clips in {args.clips}"
elif cfg.has_option('RatingClips', 'RatingClipsConfigurations'):
assert len(cfg['RatingClips']['RatingClipsConfigurations']
) > 0, f"No cloud store for clips specified in config"
else:
assert True, "Neither clips file not cloud store provided for rating clips"
if args.gold_clips:
assert os.path.exists(
args.gold_clips
), f"No csv file containing gold clips in {args.gold_clips}"
import boto3
from utils import message_generator
import os, json, uuid
import configparser
settings = configparser.ConfigParser()
settings._interpolation = configparser.ExtendedInterpolation()
settings.read('config.ini')
# Create SQS client
sqs = boto3.client('sqs')
queue_url = settings.get('AccessKeys', 'queue_url')
num_msgs = 100
for i,x in enumerate(range(0,num_msgs)):
file_name = str(uuid.uuid1())
data = message_generator.gen_req(file_name, i)
# Send message to SQS queue
response = sqs.send_message(
QueueUrl=queue_url,
DelaySeconds=10,
MessageAttributes=data['MessageAttributes'],
MessageBody=data['MessageBody']
)
print(i, response['MessageId'])
#!/usr/bin/python
import praw
import urllib.request
import os
import time
import datetime
import re
import configparser
# Parse config file
config = configparser.ConfigParser()
config._DEFAULT_INTERPOLATION = configparser.ExtendedInterpolation()
config.read('config.ini')
# Path to save images to.
PATH = str(config.get('APPLICATION', 'path'))
# Get bot data and subreddit instance.
reddit = praw.Reddit('BOT')
subreddit_str = str(input("Enter subreddit: "))
subreddit = reddit.subreddit(subreddit_str)
print("")
print("Bot running. Parsing posts...")
print("")
# Iterate through posts in hot section of subreddit. Only iterates limit number of times.
for submission in subreddit.hot(limit=60):
valid_types = (".jpg", ".png")
url_string = str(submission.url)
def configure_suite(config_file):
"""
Populates a global datastructure with the parameters from the suite's
configuration file.
This function performs the following steps:
1. read in the file ``config_file`` via the ConfigParser module using
extended interpolation
2. check that the sections ``variables`` and ``paths`` are disjoint
3. extract the environment variables given in the ``ENV`` section
4. save all entries from the ``variables`` section in the global
datastructure
5. interpret all entries in the ``paths`` section as relative paths from the
configuration file, expand them to absolute paths and save them in the
global datastructure
For further information concerning the rationale behind this, please consult
the documentation in ``doc.md``.
"""
if not os.path.exists(config_file):
raise ValueError("Test suite config file {:s} does not exist".format(
os.path.abspath(config_file)))
config = configparser.ConfigParser(
interpolation=configparser.ExtendedInterpolation(),
delimiters=(':'),
comment_prefixes=('#'))
config.read(config_file)
_parameters["suite_root"] = os.path.split(os.path.abspath(config_file))[0]
_parameters["timeout"] = config.getfloat("General",
"timeout",
fallback=1.0)
if 'variables' in config and 'paths' in config:
intersecting_keys = set(config["paths"].keys())\
.intersection(set(config["variables"].keys()))
if len(intersecting_keys) > 0:
raise ValueError(
"The sections 'paths' and 'variables' must not share keys, "
"but they have the following common key{:s}: {:s}".format(
's' if len(intersecting_keys) > 1 else '',
', '.join(k for k in intersecting_keys)))
# extract variables from the environment
for key in config['ENV']:
if key in config['ENV fallback']:
fallback = config['ENV fallback'][key]
else:
fallback = ""
config['ENV'][key] = os.getenv(config['ENV'][key]) or fallback
if 'variables' in config:
for key in config['variables']:
_parameters[key] = config['variables'][key]
if 'paths' in config:
for key in config['paths']:
rel_path = config['paths'][key]
abs_path = os.path.abspath(
os.path.join(_parameters["suite_root"], rel_path))
if not os.path.exists(abs_path):
raise ValueError(
"Path replacement for {short}: {abspath} does not exist"
" (was expanded from {rel})".format(short=key,
abspath=abs_path,
rel=rel_path))
_parameters[key] = abs_path
def __init__(self, configfile, **kwargs):
kwargs.setdefault("interpolation",
configparser.ExtendedInterpolation())
super().__init__(**kwargs)
assert configfile is not None
self.configfile = configfile
def read_config(path):
config = My_Config_Parser(inline_comment_prefixes=['#'], interpolation=configparser.ExtendedInterpolation())
assert os.path.isfile(path)
config.read(path)
return config
def __init__(self, config_file, env):
self.config = configparser.ConfigParser(
interpolation=configparser.ExtendedInterpolation())
self.config.read(config_file)
self.env = env
def parse(self, sections=None):
# Parameters are put directly into this object, based on their name in the config file
# or the command line.
# NOTE: Parameters with default values which evaluate to TRUE will ALWAYS override the file!
#
# self.configValues is the result of reading the config file
# self.args is the result from the parser
# Parse the command line (in case the configfile has been overridden)
self.args = self.parser.parse_args()
# Set values from the configfile
configParser = configparser.ConfigParser(
interpolation=configparser.ExtendedInterpolation())
configParser.read(self.args.configfile)
# Get the default section
configValues = dict(configParser[configParser.default_section])
# Add any sections we've been asked to include. If there is a collision between earlier and
# later sections, the later section wins; note that the default section is the earliest one.
# We use the consolidated dictionary for interpolation of values from the command line.
if sections:
if isinstance(sections, str):
sections = (sections, )
for s in sections:
for name in configParser[s]:
configValues[name] = configParser[s][name]
# Now, put values from the configfile into the parms object:
for name in configValues:
self.__dict__[name] = configValues[name]
# Override with non-false values from the command line (or the default).
# If no value is in the configfile, use the command line or default whether it's true or false.
# Do interpolation on strings against the consolidated dictionary of values from the configfile.
# We use section and key 'xyzzy' for interpolation; by this point, everything we might
# interpolate against has been promoted to the configValues dictionary.
configParser.remove_section('xyzzy')
resolver_name = 'xyzzy'
resolver_section = 'xyzzy'
configParser.add_section(resolver_section)
args = vars(self.args)
for name in list(args.keys()):
if (args[name] or name not in self.__dict__):
if isinstance(args[name], str):
try:
configParser.set(resolver_section, resolver_name,
args[name])
args[name] = configParser.get(resolver_section,
resolver_name,
vars=configValues)
except ValueError:
pass # Ignore interpolation problems here
self.__dict__[name] = args[name]
# And handle dbhost specially to make sure it exists:
if 'dbhost' not in self.__dict__ or not self.dbhost:
self.dbhost = 'localhost'
def parse_config(cfg_file):
"""Parse configuration file for essential variables during flux analysis.
Parameters
----------
cfg_file : str
Path to configuration file.
Returns
-------
collections.OrderedDict
Ordered dictionary with variables for each section. A list of
input files found, given the input directory and file pattern, is
also appended to the 'Inputs' section.
"""
# Ordered dictionary based on dflts to give to the parser
dflt_dict = OrderedDict((
("EC Inputs",
OrderedDict((
("input_directory", _DFLTS["ec_input_directory"]),
("file_pattern", _DFLTS["ec_file_pattern"]),
("colnames", _DFLTS["ec_colnames"]),
("sample_frequency", _DFLTS["ec_sample_frequency"]),
))
),
("EC Outputs",
OrderedDict((
("summary_file", _DFLTS["ec_summary_file"]),
))
),
("EC Despiking",
OrderedDict((
("despike_win_width", _DFLTS["ec_despike_win_width"]),
("despike_step", _DFLTS["ec_despike_step"]),
("despike_nreps", _DFLTS["ec_despike_nreps"]),
))
),
("EC Motion Correction",
OrderedDict((
("sonic_xoffset",
_DFLTS["ec_sonic_xoffset"]),
("imu_xoffset",
_DFLTS["ec_imu_xoffset"]),
("tilt_window_width",
_DFLTS["ec_tilt_window_width"]),
("imu2anemometer_pos",
_DFLTS["imu2anemometer_pos"]),
("imu_xyz_idx",
_DFLTS["imu_xyz_idx"]),
("imu2rhs_linaccel_mult",
_DFLTS["imu2rhs_linaccel_mult"]),
("imu2rhs_angaccel_mult",
_DFLTS["imu2rhs_angaccel_mult"]),
("complementary_filter_period",
_DFLTS["ec_complementary_filter_period"]),
("accel_highpass_cutoff",
_DFLTS["ec_accel_highpass_cutoff"]),
))
),
("UW Inputs",
OrderedDict((
("input_directory", _DFLTS["uw_input_directory"]),
("file_pattern", _DFLTS["uw_file_pattern"]),
("colnames", _DFLTS["uw_colnames"]),
("uw_intake_depth", _DFLTS["uw_intake_depth"]),
("uw_regress_temperature_external",
_DFLTS["uw_regress_temperature_external"]),
("uw_temperature_external_coefs",
_DFLTS["uw_temperature_external_coefs"]),
("anemometer2d_height", _DFLTS["anemometer2d_height"]),
))
),
("UW Outputs",
OrderedDict((
("pco2_directory", _DFLTS["uw_pco2_directory"]),
))
),))
# Set up the parser to interpolate across sections
config = cfg.ConfigParser(interpolation=cfg.ExtendedInterpolation())
config.read_dict(dflt_dict) # set up our specific defaults
# Replace defaults with what we're given
config.read_file(open(cfg_file))
# Copy where we'll replace strings with other types
py_dict = dflt_dict.copy()
# Loop through all items and clean them up to generate our variables as
# lists of strings
for sec in config.sections():
for opt in config.options(sec):
opt_value = config.get(sec, opt)
# Just replace and skip if we have the same as defaults
if dflt_dict[sec][opt] == opt_value:
py_dict[sec][opt] = opt_value.split(",")
else:
# Otherwise move on and remove double quotes, newlines, and
# spaces
clean_opt = re.sub('["\n ]+', "", opt_value)
config.set(sec, opt, clean_opt)
# Replace values with lists, splitting on comma character,
# on our local dictionary
py_dict[sec][opt] = clean_opt.split(",")
# Extract single elements, and convert to floats and arrays where
# appropriate
if len(py_dict[sec][opt]) == 1:
py_dict[sec][opt] = py_dict[sec][opt][0]
if sec in _SCALAR_OPTS and opt in _SCALAR_OPTS[sec]:
py_dict[sec][opt] = float(py_dict[sec][opt])
if sec in _VECTOR_OPTS and opt in _VECTOR_OPTS[sec]:
py_dict[sec][opt] = [float(x) for x in py_dict[sec][opt]]
# Check input directories exist
if not osp.exists(py_dict["EC Inputs"]["input_directory"]):
raise Exception("Input directory doesn't exist")
if not osp.exists(py_dict["UW Inputs"]["input_directory"]):
raise Exception("Input directory doesn't exist")
# Check if we have a valid sampling frequency
if py_dict["EC Inputs"]["sample_frequency"] <= 0:
raise Exception("Sampling frequency must be greater than zero")
# Check if we have a valid tilt window width
if py_dict["EC Motion Correction"]["tilt_window_width"] <= 0:
raise Exception("Tilt window width must be greater than zero")
# Check if we have valid despiking parameters
if py_dict["EC Despiking"]["despike_win_width"] <= 0:
raise Exception("Despiking window width must be greater than zero")
if py_dict["EC Despiking"]["despike_step"] <= 0:
raise Exception("Despiking step size must be greater than zero")
if py_dict["EC Despiking"]["despike_nreps"] < 0:
raise Exception(("The number of despiking iterations"
"cannot be negative"))
# Sort input file lists
ec_input_files = glob.glob(osp.join(config["EC Inputs"]["input_directory"],
config["EC Inputs"]["file_pattern"]))
ec_input_files.sort()
py_dict["EC Inputs"]["input_files"] = ec_input_files
uw_input_files = glob.glob(osp.join(config["UW Inputs"]["input_directory"],
config["UW Inputs"]["file_pattern"]))
uw_input_files.sort()
py_dict["UW Inputs"]["input_files"] = uw_input_files
# Check if we have all legal names for header of input files (don't
# care about first 2 time columns).
illegal_names = ((set(py_dict["EC Inputs"]["colnames"][2:]) |
set(py_dict["UW Inputs"]["colnames"][2:])) -
set(_INCOLS_ALL.keys()))
legal_names = set(_INCOLS_ALL.keys()) - set(illegal_names)
if len(illegal_names) > 0:
logger.info("There are unknown column names in config file: %s",
list(illegal_names))
ec_legal_names = set(py_dict["EC Inputs"]["colnames"][2:]) & legal_names
if len(ec_legal_names) < 1:
logger.error("There are no legal EC column names in config file")
else:
ec_dtypes = {key: _INCOLS_ALL[key] for key in ec_legal_names}
py_dict["EC Inputs"]["dtypes"] = ec_dtypes
uw_legal_names = set(py_dict["UW Inputs"]["colnames"][2:]) & legal_names
if len(uw_legal_names) < 1:
logger.error("There are no legal UW column names in config file")
else:
uw_dtypes = {key: _INCOLS_ALL[key] for key in uw_legal_names}
py_dict["UW Inputs"]["dtypes"] = uw_dtypes
return py_dict
def load(self, interpolation=configparser.ExtendedInterpolation()):
config = configparser.ConfigParser(interpolation=interpolation)
config.read(self.location, encoding="utf-8")
return config
def __init__(self, *args, **kwargs):
kwargs['interpolation'] = configparser.ExtendedInterpolation()
kwargs['default_section'] = self.defaultsect
super().__init__(*args, **kwargs)
def main():
# Format ISO date
dateiso = datetime.datetime.now().strftime('%Y%m%d')
# Config
config = configparser.ConfigParser()
config._interpolation = configparser.ExtendedInterpolation()
config.read('config.ini')
# Folder and file names
xmlfilename = config['paths']['xmlfilename']
xmlfolder = config['paths']['xmlfoldername']
if config['paths']['prefix'] == 'yes':
if config['paths']['xmlfilename'] == '':
xmlfilename = dateiso
else:
xmlfilename = ('%s_%s' % (dateiso, xmlfilename))
if config['paths']['prefix'] == 'no':
if config['paths']['xmlfilename'] == '':
print('xmlfilename = empty.\nEnter a name in config.ini.')
exit()
if config['paths']['prefix'] == '':
if config['paths']['xmlfilename'] == '':
print('xmlfilename = empty.\nEnter a name in config.ini.')
exit()
xmlout = ('%s/%s.xml' % (xmlfolder, xmlfilename))
print('\nfolder/xmlfile: %s\n' % xmlout)
# SBID List
with open(config['paths']['sbidlistname']) as f:
sbidlist = [line.strip() for line in f]
f.close()
print('SBID list: ')
for i in sbidlist:
print(i)
# Build XML object
xmldocs = json2xml(config=config, sbidlist=sbidlist)
# Check and create xml folder output
if xmldocs:
if os.path.exists(xmlfolder):
pass
else:
os.mkdir(xmlfolder)
# Write the XML file
with open(xmlout, encoding='utf-8-sig', mode='w') as f:
# Declaration with quotation marks
f.write(u'<?xml version="1.0" encoding="utf-8"?>\n')
f.write(xmldocs.decode('utf-8'))
f.close()
def _preprocess(self, host_chroot_spec, host_chroot_sources):
rpkg_conf_path = os.path.join(host_chroot_sources, 'rpkg.conf')
force_enable = self.opts.get('force_enable', False)
if not force_enable:
if not os.path.isfile(rpkg_conf_path):
self.log.info("rpkg.conf not found. "
"Skipping rpkg preprocessing step.")
return
parser = configparser.ConfigParser(
interpolation=configparser.ExtendedInterpolation())
try:
parser.read(rpkg_conf_path)
except configparser.ParsingError as e:
raise PkgError("Parsing of %s failed with error: %s" % (rpkg_conf_path, repr(e)))
try:
preprocess_spec = parser.getboolean('rpkg', 'preprocess_spec')
except (configparser.Error, ValueError):
self.log.warning(
"Could not get boolean value of rpkg.preprocess_spec option from rpkg.conf.")
preprocess_spec = False
if not preprocess_spec:
self.log.info("preprocess_spec not enabled in rpkg.conf. "
"Skipping rpkg preprocessing step.")
return
# try to locate spec file in SOURCES, which will be our input
host_chroot_sources_spec = os.path.join(host_chroot_sources,
os.path.basename(host_chroot_spec))
if not os.path.isfile(host_chroot_sources_spec):
raise PkgError("%s is not a file. Spec file needs to be among sources." %
host_chroot_sources_spec)
self.log.info("Installing rpkg preprocessing requires...")
self._install_requires(self.opts.get('requires', []))
# get rid of host rootdir prefixes
rootdir_prefix = self.buildroot.make_chroot_path()
chroot_spec = host_chroot_spec.replace(rootdir_prefix, '')
chroot_sources = host_chroot_sources.replace(rootdir_prefix, '')
chroot_sources_spec = host_chroot_sources_spec.replace(rootdir_prefix, '')
command_str = self.opts.get('cmd') % {'source_spec': chroot_sources_spec,
'target_spec': chroot_spec}
command = shlex.split(command_str)
# determine whether to use private network or not based on rpmbuild_networking
private_network = (not self.config.get('rpmbuild_networking', False))
self.buildroot.doChroot(
command,
shell=False,
cwd=chroot_sources,
logger=self.buildroot.build_log,
uid=self.buildroot.chrootuid,
gid=self.buildroot.chrootgid,
user=self.buildroot.chrootuser,
unshare_net=private_network,
nspawn_args=self.config.get('nspawn_args', []),
printOutput=self.config.get('print_main_output', True)
)
def read_layer_config(layer_config_filepath: Path) -> dict:
# Init
if not layer_config_filepath.exists():
raise Exception(
f"Layer config file not found: {layer_config_filepath}")
# Read config file...
layer_config = configparser.ConfigParser(
interpolation=configparser.ExtendedInterpolation(),
converters={
'list': lambda x: [i.strip() for i in x.split(',')],
'listint': lambda x: [int(i.strip()) for i in x.split(',')],
'dict': lambda x: None if x is None else json.loads(x),
'path': lambda x: Path(x)
})
layer_config.read(layer_config_filepath)
# Prepare data
image_layers = {}
for image_layer in layer_config.sections():
# First check if the image_layer code doesn't contain 'illegal' characters
if any(illegal_char in image_layer
for illegal_char in illegal_chars_in_codes):
raise Exception(
f"Section name [{image_layer}] in layer config should not contain any of the following characters: {illegal_chars_in_codes}, in {layer_config_filepath}"
)
# Init layer with all parameters in the section as dict
image_layers[image_layer] = dict(layer_config[image_layer])
# If the layer source(s) are specified in a json parameter, parse it
if 'layersources' in image_layers[image_layer]:
image_layers[image_layer]['layersources'] = layer_config[
image_layer].getdict('layersources')
# Give default values to some optional properties of a server
for layersource in image_layers[image_layer]['layersources']:
if 'random_sleep' not in layersource:
layersource['random_sleep'] = 0
else:
# If not, the layersource should be specified in seperate parameters
layersource = {}
layersource['wms_server_url'] = layer_config[image_layer].get(
'wms_server_url')
layersource['wms_version'] = layer_config[image_layer].get(
'wms_version', fallback='1.3.0')
# The layer names and layer styles are lists
layersource['layernames'] = layer_config[image_layer].getlist(
'wms_layernames')
layersource['layerstyles'] = layer_config[image_layer].getlist(
'wms_layerstyles')
# Some more properties
layersource['bands'] = layer_config[image_layer].getlist(
'bands', fallback=None)
layersource['random_sleep'] = layer_config[image_layer].getint(
'random_sleep', fallback=0)
image_layers[image_layer]['layersources'] = [layersource]
# Read nb_concurrent calls param
image_layers[image_layer]['nb_concurrent_calls'] = (
layer_config[image_layer].getint('nb_concurrent_calls',
fallback=6))
# Check if a region of interest is specified as file or bbox
roi_filepath = layer_config[image_layer].getpath('roi_filepath',
fallback=None)
image_layers[image_layer]['roi_filepath'] = roi_filepath
bbox_tuple = None
if layer_config.has_option(image_layer, 'bbox'):
bbox_list = layer_config[image_layer].getlist('bbox')
bbox_tuple = (float(bbox_list[0]), float(bbox_list[1]),
float(bbox_list[2]), float(bbox_list[3]))
image_layers[image_layer]['bbox'] = bbox_tuple
image_layers[image_layer]['bbox'] = bbox_tuple
# Check if the grid xmin and xmax are specified
grid_xmin = 0
if layer_config.has_option(image_layer, 'grid_xmin'):
grid_xmin = layer_config[image_layer].getfloat('grid_xmin')
image_layers[image_layer]['grid_xmin'] = grid_xmin
grid_ymin = 0
if layer_config.has_option(image_layer, 'grid_ymin'):
grid_ymin = layer_config[image_layer].getfloat('grid_ymin')
image_layers[image_layer]['grid_ymin'] = grid_ymin
# Check if a image_pixels_ignore_border is specified
image_pixels_ignore_border = layer_config[image_layer].getint(
'image_pixels_ignore_border', fallback=0)
image_layers[image_layer][
'image_pixels_ignore_border'] = image_pixels_ignore_border
return image_layers
def setParameters(self, configfile):
param = configparser.SafeConfigParser(
interpolation=configparser.ExtendedInterpolation())
#param = configparser.SafeConfigParser(os.environ)#interpolation = EnvInterpolation())
# param = configparser.SafeConfigParser(interpolation = )
configfile_str = open(configfile).read()
#configfile_str = os.path.expandvars(configfile_str)
configfile_str = configfile_str.replace(
"$PWD", os.path.abspath(os.path.dirname(configfile)))
param.read_string(configfile_str)
print(param)
self.dir_model_output = param.get('PATH', 'dir_model_output')
self.dir_vocab_output = param.get('PATH', 'dir_vocab_output')
self.dir_model = param.get('PATH', 'dir_model')
self.path_log = param.get('PATH', 'path_log')
self.filename_train = param.get("PATH", 'filename_train')
self.filename_test = param.get('PATH', 'filename_test')
self.filename_dev = param.get('PATH', 'filename_dev')
self.filename_words = param.get('PATH', 'filename_words')
self.filename_tags = param.get('PATH', 'filename_tags')
self.filename_chars = param.get('PATH', 'filename_chars')
#embedding types: Glove, w2v, fasttext
try:
self.embedding_type = param.get("EMBEDDINGS", "embedding_type")
except configparser.NoOptionError:
self.embedding_type = "Glove"
self.dim_word = param.getint('EMBEDDINGS', 'dim_word')
self.dim_char = param.getint('EMBEDDINGS', 'dim_char')
self.filename_embeddings = param.get('EMBEDDINGS',
'filename_embeddings')
self.filename_embeddings_trimmed = param.get(
'EMBEDDINGS', 'filename_embeddings_trimmed')
self.use_pretrained = param.getboolean('EMBEDDINGS', 'use_pretrained')
self.use_large_embeddings = param.getboolean('EMBEDDINGS',
'use_large_embeddings')
self.oov_size = 0
self.oov_current_size = 0
if param.has_option('EMBEDDINGS', 'oov_size'):
self.oov_size = param.getint('EMBEDDINGS', 'oov_size')
self.max_iter = param['PARAM']['max_iter']
if not self.embedding_type == "fasttext" and self.oov_size > 0:
sys.stderr.write(
"Embeddings for unknown words cannot be generated for " +
self.embedding_type +
", thus the parameter oov_size will be set to zero\n")
self.oov_size = 0
if self.use_pretrained == False and self.use_large_embeddings == True:
sys.stderr.write(
"If you want to train embeddings from scratch the use_large_embedding option is not valid\n"
)
if (self.max_iter == "None"):
self.max_iter = None
self.train_embeddings = param.getboolean('PARAM', 'train_embeddings')
try:
self.lowercase = param.getboolean("PARAM", "lowercase")
except configparser.NoOptionError:
self.lowercase = True
self.nepochs = param.getint('PARAM', 'nepochs')
self.dropout = param.getfloat('PARAM', 'dropout')
self.batch_size = param.getint('PARAM', 'batch_size')
self.lr_method = param.get('PARAM', 'lr_method')
self.lr = param.getfloat('PARAM', 'lr')
self.lr_decay = param.getfloat('PARAM', 'lr_decay')
self.clip = param.getint('PARAM', 'clip')
self.nepoch_no_imprv = param.getint('PARAM', 'nepoch_no_imprv')
self.hidden_size_char = param.getint('PARAM', 'hidden_size_char')
self.hidden_size_lstm = param.getint('PARAM', 'hidden_size_lstm')
self.use_crf = param.getboolean('PARAM', 'use_crf')
self.use_chars = param.getboolean('PARAM', 'use_chars')
self.oov_words = []
def parse_conf_file(self):
"""
Parse the file ``dirs.USER_CONF_DIR_FILE`` with the following sections
:[Commons]:
Try to find user directories; if they exist add them to
`dirs.USER_DIRS` and `sys.path`
For the other sections, OrderedDicts are returned with the class names
as keys and dictionaries with options as values.
:[Input Widgets]:
Store (user) input widgets in `fb.input_dict`
:[Plot Widgets]:
Store (user) plot widgets in `fb.plot_dict`
:[Filter Designs]:
Store (user) filter designs in `fb.filter_dict`
:[Fixpoint Widgets]:
Store (user) fixpoint widgets in `fb.fixpoint_dict`
Parameters
----------
None
Returns
-------
None
"""
CONF_VERSION = 2
try:
# Test whether user config file is readable, this is necessary as
# configParser quietly fails when the file doesn't exist
if not os.access(dirs.USER_CONF_DIR_FILE, os.R_OK):
raise IOError('Config file "{0}" cannot be read.'.format(
dirs.USER_CONF_DIR_FILE))
# -----------------------------------------------------------------
# setup an instance of config parser, allow keys without value
# -----------------------------------------------------------------
self.conf = configparser.ConfigParser(allow_no_value=True)
# preserve case of parsed options by overriding optionxform():
# Set it to function str()
self.conf.optionxform = str
# Allow interpolation across sections, ${Dirs:dir1}
self.conf._interpolation = configparser.ExtendedInterpolation(
) # PY3 only
self.conf.read(dirs.USER_CONF_DIR_FILE)
logger.info(
'Parsing config file\n\t"{0}"\n\t\twith sections:\n\t{1}'.
format(dirs.USER_CONF_DIR_FILE, str(self.conf.sections())))
# -----------------------------------------------------------------
# Parsing [Common]
#------------------------------------------------------------------
self.commons = self.parse_conf_section("Common")
if not 'version' in self.commons or int(
self.commons['version'][0]) != CONF_VERSION:
logger.critical("Config file\n\t'{0:s}'\n\thas the wrong version '{2}' "
"(required: '{1}').\n"
"\tYou can either edit the file or delete it.\n\tWhen deleted, "
"a new configuration file will be created at restart."\
.format(dirs.USER_CONF_DIR_FILE, CONF_VERSION, int(self.commons['version'][0])))
sys.exit()
if 'user_dirs' in self.commons:
for d in self.commons['user_dirs']:
d = os.path.abspath(os.path.normpath(d))
if os.path.isdir(d):
dirs.USER_DIRS.append(d)
if d not in sys.path:
sys.path.append(d)
else:
logger.warning(
"User directory doesn't exist:\n\t{0}\n".format(d))
if dirs.USER_DIRS:
logger.info("User directory(s):\n\t{0}\n".format(
dirs.USER_DIRS))
else:
logger.warning(
'No valid user directory found in "{0}\n.'.format(
dirs.USER_CONF_DIR_FILE))
# -----------------------------------------------------------------
# Parsing [Input Widgets]
#------------------------------------------------------------------
fb.input_classes = self.build_class_dict("Input Widgets",
"input_widgets")
# -----------------------------------------------------------------
# Parsing [Plot Widgets]
#------------------------------------------------------------------
fb.plot_classes = self.build_class_dict("Plot Widgets",
"plot_widgets")
# -----------------------------------------------------------------
# Parsing [Filter Designs]
#------------------------------------------------------------------
fb.filter_classes = self.build_class_dict("Filter Designs",
"filter_designs")
# currently, option "opt" can only be an association with a fixpoint
# widget, so replace key "opt" by key "fix":
# Convert to list in any case
for c in fb.filter_classes:
if 'opt' in fb.filter_classes[c]:
fb.filter_classes[c]['fix'] = fb.filter_classes[c].pop(
'opt')
if 'fix' in fb.filter_classes[c] and type(
fb.filter_classes[c]['fix']) == str:
fb.filter_classes[c]['fix'] = fb.filter_classes[c][
'fix'].split(',')
# -----------------------------------------------------------------
# Parsing [Fixpoint Filters]
#------------------------------------------------------------------
fb.fixpoint_classes = self.build_class_dict(
"Fixpoint Widgets", "fixpoint_widgets")
logger.info("\nFixpoint_widgets: \n{0}\n".format(
fb.fixpoint_classes))
# First check whether fixpoint options of the filter designs are
# valid fixpoint classes by comparing them to the verified items of
# fb.fixpoint_classes:
for c in fb.filter_classes:
if 'fix' in fb.filter_classes[c]:
for w in fb.filter_classes[c]['fix']:
if w not in fb.fixpoint_classes:
logger.warning('Removing invalid fixpoint module\n\t"{0}" for filter class "{1}".'\
.format(w,c))
fb.filter_classes[c]['fix'].remove(w)
# merge fb.filter_classes info "filter class":[fx_class1, fx_class2]
# and fb.fixpoint_classes info "fixpoint class":[fil_class1, fil_class2]
# into the fb.filter_classes dict
# collect all fixpoint widgets (keys in fb.fixpoint_classes) which
# have the class name c as a value
fix_wdg = {
k
for k, val in fb.fixpoint_classes.items()
if c in val['opt']
}
if len(fix_wdg) > 0:
if 'fix' in fb.filter_classes[c]:
#... and merge it with the fixpoint options of class c
fix_wdg = fix_wdg.union(fb.filter_classes[c]['fix'])
fb.filter_classes[c].update({'fix': list(fix_wdg)})
# ----- Exceptions ----------------------
except configparser.DuplicateSectionError as e:
logger.critical('{0} in config file "{1}".'.format(
e, dirs.USER_CONF_DIR_FILE))
sys.exit()
except configparser.ParsingError as e:
logger.critical('Parsing Error in config file "{0}:\n{1}".'.format(
dirs.USER_CONF_DIR_FILE, e))
sys.exit()
except configparser.Error as e:
logger.critical('{0} in config file "{1}".'.format(
e, dirs.USER_CONF_DIR_FILE))
sys.exit()
def get_config():
CONFIG_FILE = os.environ.get("CONFIG_FILE", 'config_files/config.dev.ini')
config = configparser.ConfigParser(interpolation=configparser.ExtendedInterpolation())
config.read(CONFIG_FILE)
return config
def main_body():
'''Main body of this file'''
parser = argparse.ArgumentParser()
# Configurations: read noisyspeech_synthesizer.cfg and gather inputs
parser.add_argument(
'--cfg',
default='pdns_synthesizer_icassp2022.cfg',
help='Read noisyspeech_synthesizer.cfg for all the details')
parser.add_argument('--cfg_str', type=str, default='noisy_speech')
args = parser.parse_args()
params = dict()
params['args'] = args
cfgpath = os.path.join(args.cfg)
# os.path.join(os.path.dirname(__file__), args.cfg)
assert os.path.exists(cfgpath), f'No configuration file as [{cfgpath}]'
cfg = CP.ConfigParser()
cfg._interpolation = CP.ExtendedInterpolation()
cfg.read(cfgpath)
params['cfg'] = cfg._sections[args.cfg_str]
cfg = params['cfg']
clean_dir = os.path.join('datasets/clean')
if cfg['speech_dir'] != 'None':
clean_dir = cfg['speech_dir']
if not os.path.exists(clean_dir):
assert False, ('Clean speech data is required')
if cfg['speech_dir2'] != 'None':
clean_dir2 = cfg['speech_dir2']
if cfg['spkid_csv'] != 'None':
spkid_csv = cfg['spkid_csv']
if not os.path.exists(clean_dir2):
assert False, ('Clean speech2 data is required')
if cfg['rir_dir'] != 'None':
rir_dir = cfg['rir_dir']
if cfg['noise_dir'] != 'None':
noise_dir = cfg['noise_dir']
if not os.path.exists(noise_dir):
assert False, ('Clean speech data is required')
print(clean_dir)
print(clean_dir2)
print(noise_dir)
print(spkid_csv)
print(rir_dir)
if cfg['noise_dir'] != 'None':
noise_dir = cfg['noise_dir']
if not os.path.exists:
assert False, ('Noise data is required')
params['fs'] = int(cfg['sampling_rate'])
params['audioformat'] = cfg['audioformat']
params['audio_length'] = float(cfg['audio_length'])
params['silence_length'] = float(cfg['silence_length'])
params['total_hours'] = float(cfg['total_hours'])
# clean singing speech
params['clean_singing'] = str(cfg['clean_singing'])
params['singing_choice'] = int(cfg['singing_choice'])
# rir
params['rir_choice'] = int(cfg['rir_choice'])
params['lower_t60'] = float(cfg['lower_t60'])
params['upper_t60'] = float(cfg['upper_t60'])
params['rir_table_csv'] = str(cfg['rir_table_csv'])
params['clean_speech_t60_csv'] = str(cfg['clean_speech_t60_csv'])
if cfg['fileindex_start'] != 'None' and cfg['fileindex_start'] != 'None':
params['num_files'] = int(cfg['fileindex_end']) - int(
cfg['fileindex_start'])
params['fileindex_start'] = int(cfg['fileindex_start'])
params['fileindex_end'] = int(cfg['fileindex_end'])
else:
params['num_files'] = int(
(params['total_hours'] * 60 * 60) / params['audio_length'])
params['fileindex_start'] = 0
params['fileindex_end'] = int(params['num_files'])
print('Number of files to be synthesized:', params['num_files'])
params['is_test_set'] = utils.str2bool(cfg['is_test_set'])
params['clean_activity_threshold'] = float(cfg['clean_activity_threshold'])
params['noise_activity_threshold'] = float(cfg['noise_activity_threshold'])
params['snr_lower'] = int(cfg['snr_lower'])
params['snr_upper'] = int(cfg['snr_upper'])
params['randomize_snr'] = utils.str2bool(cfg['randomize_snr'])
params['target_level_lower'] = int(cfg['target_level_lower'])
params['target_level_upper'] = int(cfg['target_level_upper'])
if 'snr' in cfg.keys():
params['snr'] = int(cfg['snr'])
else:
params['snr'] = int((params['snr_lower'] + params['snr_upper']) / 2)
params['noisyspeech_dir'] = utils.get_dir(cfg, 'noisy_destination',
'noisy')
params['clean_proc_dir'] = utils.get_dir(cfg, 'clean_destination', 'clean')
params['noise_proc_dir'] = utils.get_dir(cfg, 'noise_destination', 'noise')
if 'speech_csv' in cfg.keys() and cfg['speech_csv'] != 'None':
cleanfilenames = pd.read_csv(cfg['speech_csv'])
cleanfilenames = cleanfilenames['filename']
else:
cleanfilenames = []
for path in Path(cfg['speech_dir']).rglob('*.wav'):
cleanfilenames.append(str(path.resolve()))
selected_primary = []
selected_secondary = []
with open(spkid_csv, 'r') as file:
my_reader = csv.reader(file, delimiter=',')
for row in my_reader:
if row[1] == 'primary':
selected_primary.append(row)
elif row[1] == 'secondary':
selected_secondary.append(row)
cleanfilenames = []
for row in selected_primary:
cleanfilenames.append(row[0])
cleanfilenames2 = []
for row in selected_secondary:
cleanfilenames2.append(row[0])
params['cleanfilenames'] = cleanfilenames
shuffle(cleanfilenames2)
params['cleanfilenames2'] = cleanfilenames2
rirfilenames = []
for path in Path(cfg['rir_dir']).rglob('*.wav'):
rirfilenames.append(str(path.resolve()))
shuffle(rirfilenames)
params['myrir'] = rirfilenames
if 'noise_csv' in cfg.keys() and cfg['noise_csv'] != 'None':
noisefilenames = pd.read_csv(cfg['noise_csv'])
noisefilenames = noisefilenames['filename']
else:
noisefilenames = glob.glob(
os.path.join(noise_dir, params['audioformat']))
if len(noisefilenames) != 0:
shuffle(noisefilenames)
params['noisefilenames'] = noisefilenames
else:
noisedirs = glob.glob(os.path.join(noise_dir, '*'))
if cfg['noise_types_excluded'] != 'None':
dirstoexclude = cfg['noise_types_excluded'].split(',')
for dirs in dirstoexclude:
noisedirs.remove(dirs)
shuffle(noisedirs)
params['noisedirs'] = noisedirs
# Call main_gen() to generate audio
clean_source_files, clean_clipped_files, clean_low_activity_files, \
noise_source_files, noise_clipped_files, noise_low_activity_files = main_gen(params)
# Create log directory if needed, and write log files of clipped and low activity files
log_dir = utils.get_dir(cfg, 'log_dir', 'Logs')
utils.write_log_file(log_dir, 'source_files.csv',
clean_source_files + noise_source_files)
utils.write_log_file(log_dir, 'clipped_files.csv',
clean_clipped_files + noise_clipped_files)
utils.write_log_file(log_dir, 'low_activity_files.csv', \
clean_low_activity_files + noise_low_activity_files)
# Compute and print stats about percentange of clipped and low activity files
total_clean = len(clean_source_files) + len(clean_clipped_files) + len(
clean_low_activity_files)
total_noise = len(noise_source_files) + len(noise_clipped_files) + len(
noise_low_activity_files)
pct_clean_clipped = round(len(clean_clipped_files) / total_clean * 100, 1)
pct_noise_clipped = round(len(noise_clipped_files) / total_noise * 100, 1)
pct_clean_low_activity = round(
len(clean_low_activity_files) / total_clean * 100, 1)
pct_noise_low_activity = round(
len(noise_low_activity_files) / total_noise * 100, 1)
print("Of the " + str(total_clean) + " clean speech files analyzed, " + \
str(pct_clean_clipped) + "% had clipping, and " + str(pct_clean_low_activity) + \
"% had low activity " + "(below " + str(params['clean_activity_threshold']*100) + \
"% active percentage)")
print("Of the " + str(total_noise) + " noise files analyzed, " + str(pct_noise_clipped) + \
"% had clipping, and " + str(pct_noise_low_activity) + "% had low activity " + \
"(below " + str(params['noise_activity_threshold']*100) + "% active percentage)")
def load_plugin(self, name, config_names, plugin_params=None):
if config_names is not None:
self.logger.info("Loading plugin [%s] with configuration(s) [%s].",
name, config_names)
else:
self.logger.info("Loading plugin [%s] with default configuration.",
name)
root_path = self._plugin_root_path(name)
if not config_names:
# maybe we only have a config folder but nothing else (e.g. if there is only an install hook)
if io.exists(root_path):
return PluginDescriptor(name=name,
config=config_names,
root_path=root_path)
else:
core_plugin = self._core_plugin(name)
if core_plugin:
return core_plugin
# If we just have a plugin name then we assume that this is a community plugin and the user has specified a download URL
else:
self.logger.info(
"The plugin [%s] is neither a configured nor an official plugin. Assuming that this is a community "
"plugin not requiring any configuration and you have set a proper download URL.",
name)
return PluginDescriptor(name)
else:
variables = {}
config_paths = []
# used for deduplication
known_config_bases = set()
# used to determine whether this is a core plugin
core_plugin = self._core_plugin(name)
for config_name in config_names:
config_file = self._plugin_file(name, config_name)
# Do we have an explicit configuration for this plugin?
if not io.exists(config_file):
if core_plugin:
raise exceptions.SystemSetupError(
"Plugin [%s] does not provide configuration [%s]. List the available plugins "
"and configurations with %s list elasticsearch-plugins "
"--distribution-version=VERSION." %
(name, config_name, PROGRAM_NAME))
else:
raise exceptions.SystemSetupError(
"Unknown plugin [%s]. List the available plugins with %s list "
"elasticsearch-plugins --distribution-version=VERSION."
% (name, PROGRAM_NAME))
config = configparser.ConfigParser(
interpolation=configparser.ExtendedInterpolation())
# Do not modify the case of option keys but read them as is
config.optionxform = lambda option: option
config.read(config_file)
if "config" in config and "base" in config["config"]:
config_bases = config["config"]["base"].split(",")
for base in config_bases:
if base and base not in known_config_bases:
config_paths.append(
os.path.join(root_path, base, "templates"))
known_config_bases.add(base)
if "variables" in config.sections():
for k, v in config["variables"].items():
variables[k] = v
# add all plugin params here to override any defaults
if plugin_params:
variables.update(plugin_params)
# maybe one of the configs is really just for providing variables. However, we still require one config base overall.
if len(config_paths) == 0:
raise exceptions.SystemSetupError(
"At least one config base is required for plugin [%s]" %
name)
return PluginDescriptor(name=name,
core_plugin=core_plugin is not None,
config=config_names,
root_path=root_path,
config_paths=config_paths,
variables=variables)
def main():
try:
# ------------------------------------------
# Platform check
#-------------------------------------------
sys.tracebacklimit = 0
if platform.system() == 'Windows':
iswindows = True
slash = '\\'
else:
iswindows = False
slash = '/'
# ------------------------------------------
# Read the config file
#-------------------------------------------
settings = configparser.ConfigParser()
settings._interpolation = configparser.ExtendedInterpolation()
try:
settings.read('config.ini')
except Exception:
sys.exit(
'\n**ERROR**\nCould not open configuration file. It should be in the same folder as the script and named \'config.ini\'\n'
)
print(
"\n-------------------------\nMemespector Python script\n-------------------------"
)
try:
projectFolder = settings['Project']['ProjectFolder']
dir_path = os.path.dirname(os.path.realpath(__file__))
dataFolder = dir_path + slash + settings['Folders'][
'DataFolder'] + slash
cacheFolder = dir_path + slash + settings['Folders'][
'CacheFolder'] + slash
outputsFolder = dir_path + slash + settings['Folders'][
'OutputsFolder'] + slash + projectFolder + slash
imageCpFolder = outputsFolder + settings['Folders'][
'ImageCopyFolder'] + slash
cacheCopyFolder = outputsFolder + settings['Folders'][
'CacheCopyFolder'] + slash
makeNetwork = f.yn(settings['OutputConfiguration']['MakeNetwork'])
makeWebNetwork = f.yn(
settings['OutputConfiguration']['MakeWebEntitiesNetwork'])
imagesRemote = f.yn(
settings['SourceImagesLocation']['ImagesRemote'])
absolutePath = f.yn(
settings['SourceImagesLocation']['AbsolutePath'])
forceBase64 = f.yn(settings['SourceImagesLocation']['ForceBase64'])
saveImageCopy = f.yn(
settings['SourceImagesLocation']['SaveImageCopy'])
inputImageFolder = settings['SourceImagesLocation'][
'InputImageFolder']
labelThreshold = float(
settings['OutputConfiguration']['LabelScoreFilter'])
includeScore = f.yn(
settings['OutputConfiguration']['IncludeScoreInSpreadsheet'])
inputFileName = settings['InputConfiguration']['InputFile']
inputFilePath = dataFolder + inputFileName
delimiter = settings['InputConfiguration']['Delimiter'].encode(
'utf-8').decode('unicode_escape')
imagesColumn = settings['InputConfiguration']['ImagesColumn']
linkColumn = settings['InputConfiguration']['LinkColumn']
if linkColumn == "":
linkColumn = imagesColumn
procLimit = int(settings['InputConfiguration']['Limit'])
imagesRemote = f.yn(
settings['SourceImagesLocation']['ImagesRemote'])
forceBase64 = f.yn(settings['SourceImagesLocation']['ForceBase64'])
labelDetection = f.yn(settings['ApiRequestFeatures']['Label'])
safeSearchDetection = f.yn(
settings['ApiRequestFeatures']['SafeSearch'])
textDetection = f.yn(settings['ApiRequestFeatures']['Text'])
webDetection = f.yn(settings['ApiRequestFeatures']['Web'])
faceDetection = f.yn(settings['ApiRequestFeatures']['Face'])
maxResults = settings['ApiRequestFeatures']['MaxResults']
apiKey = settings['ApiRequestFeatures']['ApiKey']
if maxResults == "0":
setMaxResults = False
else:
setMaxResults = True
except Exception:
sys.exit(
"\n**ERROR**\nCould not parse at least one of the settings from the config file. Please verify its contents carefully."
)
# ------------------------------------------
# Create folders
#-------------------------------------------
if not os.path.exists(cacheFolder):
os.makedirs(cacheFolder)
if os.path.exists(outputsFolder):
answer = input(
"\nATTENTION: Project folder already exists. There is risk of overwriting files. Continue? Y or N > "
)
if answer.lower() == 'n':
sys.exit('Rename project in config file.')
elif answer.lower() == 'y':
print('Continuing overwriting existing files.')
else:
sys.exit('Answer not understood. Exiting.')
else:
os.makedirs(outputsFolder)
if not os.path.exists(imageCpFolder):
os.makedirs(imageCpFolder)
if not os.path.exists(cacheCopyFolder):
os.makedirs(cacheCopyFolder)
# ------------------------------------------
# Configure Processing attributes
# ------------------------------------------
apirequest = VisionApiRequest(apiKey, setMaxResults, maxResults,
labelDetection, safeSearchDetection,
textDetection, webDetection,
faceDetection)
apirequest.printModuleConfiguration()
print("HANDLING OF IMAGE SOURCE\n")
if imagesRemote:
print("\tOnline images")
if forceBase64:
print(
"\tLocal intermediation of data submission (forcing base64)"
)
else:
print("\tRemotely retrieved by Google")
if saveImageCopy:
print("\tSaving local copies of processed images")
else:
print("\tNot saving local copies")
else:
print("\tProcessing local images")
if absolutePath:
print("\tReading file paths as absolute")
else:
print("\tReading file paths as relative to: " +
inputImageFolder)
print()
# ------------------------------------------
# Get input
# ------------------------------------------
try:
inputFile = open(inputFilePath, encoding='utf8')
except Exception:
sys.exit(
'\n**ERROR**\nInput file could not be opened. Please check the configuration file.\n'
)
print("Delimiter: " + delimiter)
csvDialect = csv.Sniffer().sniff(inputFile.read(1024),
delimiters=delimiter)
if not csvDialect.escapechar:
csvDialect.escapechar = "\\"
print("Escape char: " + str(csvDialect.escapechar))
inputFile.seek(0)
inputCSV = csv.reader(inputFile, csvDialect)
inputHeader = next(inputCSV)
if imagesColumn in inputHeader:
imagesColumnIdx = inputHeader.index(imagesColumn)
else:
print(inputHeader)
sys.exit(
'\n**ERROR**\nImage column could not be found in input file. Please check the configuration file.\n'
)
if makeNetwork or makeWebNetwork:
if linkColumn in inputHeader:
linkColumnIdx = inputHeader.index(linkColumn)
else:
print(inputHeader)
sys.exit(
'\n**ERROR**\nLink column could not be found in input file. Please check the configuration file\n'
)
# Workaround for Facebook URLs messing with filename extensions
if ("created_time_unix" in inputHeader) or ("post_published_unix"
in inputHeader):
isFacebook = True
else:
isFacebook = False
numImages = sum(1 for row in inputCSV)
inputFile.seek(0)
if procLimit > numImages or procLimit == 0:
procLimit = numImages
print("DATA FILE\n")
print("\tSource file path\n\t\t" + inputFilePath)
print("\tNumber of entries\n\t\t" + str(numImages))
print("\tProcessing limit\n\t\t" + str(procLimit))
print("\tImage column header\n\t\t" + imagesColumn + "(" +
str(imagesColumnIdx) + ")")
print("\n-------------------------")
# ------------------------------------------
# Create output files
# ------------------------------------------
outputFileName = "annotated_" + inputFileName
outputFilePath = outputsFolder + outputFileName
try:
outputFile = open(outputFilePath, 'w', newline='', encoding='utf8')
except Exception:
sys.exit(
'\n**ERROR**\nOutput file could not be created. Please check the configuration file.\n'
)
outputCSV = csv.writer(outputFile, csvDialect)
# Add columns to input file
outputHeader = inputHeader + [
'original_filename', 'image_id', 'file_ext', 'copy_filename',
'gv_ss_adult', 'gv_ss_spoof', 'gv_ss_medical', 'gv_ss_violence',
'gv_labels', 'gv_text', 'gv_text_lang', 'gv_web_entities',
'gv_web_full_matching_images', 'gv_web_partial_matching_images',
'gv_web_pages_with_matching_images',
'gv_web_visually_similar_images', 'gv_num_faces', 'gv_face_joy',
'gv_face_sorrow', 'gv_face_anger', 'gv_face_surprise'
]
outputCSV.writerow(outputHeader)
if makeNetwork:
graph = nx.Graph()
graphfilename = "img-label-net_" + inputFileName.split(
".")[0] + ".gexf"
graphfilepath = outputsFolder + graphfilename
foundlabels = []
if makeWebNetwork:
webgraph = nx.Graph()
webgraphfilename = "img-webentities-net_" + inputFileName.split(
".")[0] + ".gexf"
webgraphfilepath = outputsFolder + webgraphfilename
foundwebentities = []
# ------------------------------------------
# Process images
# ------------------------------------------
next(inputCSV)
for i in range(procLimit):
print("\nImage %s of %s" % (i + 1, procLimit))
inputRow = next(inputCSV)
outputRow = inputRow
# Retrieve image path from input
imagePath = inputRow[imagesColumnIdx]
# If image is local and path is not absolute, make up absolute path
if not imagesRemote and not absolutePath:
imagePath = inputImageFolder + imagePath
# Clean file name and extension from path if Facebook data
if isFacebook:
originalFileName = os.path.basename(
re.findall(".+\/(.+?)\?", imagePath)[0])
else:
originalFileName = os.path.basename(imagePath)
extension = os.path.splitext(originalFileName)[1]
# Create hash for image url
hashObj = hashlib.sha1(imagePath.encode('utf-8'))
imageHash = hashObj.hexdigest()
print("\tImage ID: %s" % (imageHash))
# Make image copy
copyFilename = imageHash + extension
if (saveImageCopy):
copyFilePath = imageCpFolder + copyFilename
if not os.path.isfile(copyFilePath):
print("\tCopying image...", end="")
sys.stdout.flush()
try:
image = requests.get(imagePath, allow_redirects=True)
except Exception:
print(
'\n\t**ERROR**\n\tCould not retrieve remote image. Check data, internet connection, configuration.\n'
)
outputCSV.writerow(outputRow)
continue
open(copyFilePath, "wb").write(image.content)
print("done")
else:
print("\tCopy already existed")
if makeNetwork:
nodelink = inputRow[linkColumnIdx]
graph.add_node(imageHash,
type='image',
label='_image',
file=copyFilename,
link=nodelink)
if makeWebNetwork:
nodelink = inputRow[linkColumnIdx]
webgraph.add_node(imageHash,
type='image',
label='_image',
file=copyFilename,
link=nodelink)
# Process image
responseFile = cacheFolder + imageHash + '.json'
responseFileCp = cacheCopyFolder + imageHash + '.json'
if not (os.path.isfile(responseFile)):
if imagesRemote and forceBase64 and saveImageCopy:
# If images are remote but are to be processed through local base64 coding and copies have been made, use copies for network traffic efficiency.
apirequest.annotateImage(copyFilePath, isRemote=False)
responseData = apirequest.getResponse()
if not responseData:
outputCSV.writerow(outputRow)
continue
else:
apirequest.annotateImage(imagePath,
isRemote=imagesRemote,
base64Encode=forceBase64)
responseData = apirequest.getResponse()
if not responseData:
outputCSV.writerow(outputRow)
continue
with open(responseFile, 'w', encoding='utf8') as outFile:
json.dump(responseData, outFile, indent=4, sort_keys=True)
copyfile(responseFile, responseFileCp)
else:
# If there is a json annotation file save in cache, use that instead of processing the image again.
print(
"\t*ATTENTION* Using cached content (remove all files in the cache folder if you see this message and the tool is not working yet)"
)
copyfile(responseFile, responseFileCp)
responseData = json.load(open(responseFile, encoding='utf8'))
# Parse API response
try:
response = responseData['responses'][0]
except Exception:
print(
'\t*ATTENTION* Vision API returned an error for this image. Check JSON file for details.\n\tMoving on to next image.\n'
)
outputCSV.writerow(outputRow)
continue
# Safe Search
if 'safeSearchAnnotation' in response:
gv_ss_adult = response['safeSearchAnnotation']['adult']
gv_ss_spoof = response['safeSearchAnnotation']['spoof']
gv_ss_medical = response['safeSearchAnnotation']['medical']
gv_ss_violence = response['safeSearchAnnotation']['violence']
else:
gv_ss_adult = "NONE"
gv_ss_spoof = "NONE"
gv_ss_medical = "NONE"
gv_ss_violence = "NONE"
# Labels
if 'labelAnnotations' in response:
gv_labels = []
for label in response['labelAnnotations']:
if label['score'] < labelThreshold:
continue
if label['mid'] == '':
label_id = "_" + label['description']
else:
label_id = label['mid']
if makeNetwork:
if not label_id in foundlabels:
foundlabels.append(label_id)
graph.add_node(label_id,
type='gv_label',
label=label['description'],
mid=label['mid'],
description=label['description'])
graph.add_edge(imageHash,
label_id,
score=label['score'],
topicality=label['topicality'])
if includeScore:
gv_labels.append(label['description'] + "(" +
str(label['score']) + ")")
else:
gv_labels.append(label['description'])
gv_labels = ",".join(gv_labels)
else:
gv_labels = "NONE"
# Text
if 'textAnnotations' in response:
gv_text = response['textAnnotations'][0]['description']
gv_text = re.sub("[\n\t\r]", " ", gv_text)
gv_text_lang = response['textAnnotations'][0]['locale']
else:
gv_text = "NONE"
gv_text_lang = "NONE"
# Web Detection
if 'webDetection' in response:
if 'fullMatchingImages' in response['webDetection']:
gv_web_full_matching_images = []
for url in response['webDetection']['fullMatchingImages']:
gv_web_full_matching_images.append(url['url'].replace(
",", "%2C"))
gv_web_full_matching_images = ",".join(
gv_web_full_matching_images)
else:
gv_web_full_matching_images = "NONE"
if 'pagesWithMatchingImages' in response['webDetection']:
gv_web_pages_with_matching_images = []
for url in response['webDetection'][
'pagesWithMatchingImages']:
gv_web_pages_with_matching_images.append(
url['url'].replace(",", "%2C"))
gv_web_pages_with_matching_images = ",".join(
gv_web_pages_with_matching_images)
else:
gv_web_pages_with_matching_images = "NONE"
if 'partialMatchingImages' in response['webDetection']:
gv_web_partial_matching_images = []
for url in response['webDetection'][
'partialMatchingImages']:
gv_web_partial_matching_images.append(
url['url'].replace(",", "%2C"))
gv_web_partial_matching_images = ",".join(
gv_web_partial_matching_images)
else:
gv_web_partial_matching_images = "NONE"
if 'visuallySimilarImages' in response['webDetection']:
gv_web_visually_similar_images = []
for url in response['webDetection'][
'visuallySimilarImages']:
gv_web_visually_similar_images.append(
url['url'].replace(",", "%2C"))
gv_web_visually_similar_images = ",".join(
gv_web_visually_similar_images)
else:
gv_web_visually_similar_images = "NONE"
if 'webEntities' in response['webDetection']:
gv_web_entities = []
for entity in response['webDetection']['webEntities']:
if 'entityId' in entity and 'description' in entity and 'score' in entity:
entity_id = entity['entityId']
description = entity['description']
gv_web_entities.append(description + "(" +
str(entity['score']) + ")")
if makeWebNetwork:
if not entity_id in foundwebentities:
foundwebentities.append(entity_id)
# This likely adds redundant attributes but may make it more compatible with the label network graph
webgraph.add_node(entity_id,
type='gv_web_entity',
label=description,
mid=entity_id,
description=description)
webgraph.add_edge(imageHash,
entity_id,
score=entity['score'],
topicality=entity['score'])
else:
description = "NONE"
gv_web_entities = ",".join(gv_web_entities)
else:
gv_web_entities = "NONE"
else:
gv_web_full_matching_images = "NONE"
gv_web_pages_with_matching_images = "NONE"
gv_web_partial_matching_images = "NONE"
gv_web_visually_similar_images = "NONE"
gv_web_entities = "NONE"
# Face
if 'faceAnnotations' in response:
gv_num_faces = 0
gv_face_joy = 'VERY_UNLIKELY'
gv_face_sorrow = 'VERY_UNLIKELY'
gv_face_anger = 'VERY_UNLIKELY'
gv_face_surprise = 'VERY_UNLIKELY'
for face in response['faceAnnotations']:
gv_face_joy = f.likelihoodCompare(gv_face_joy,
face['joyLikelihood'])
gv_face_sorrow = f.likelihoodCompare(
gv_face_sorrow, face['sorrowLikelihood'])
gv_face_anger = f.likelihoodCompare(
gv_face_anger, face['angerLikelihood'])
gv_face_surprise = f.likelihoodCompare(
gv_face_surprise, face['surpriseLikelihood'])
gv_num_faces += 1
else:
gv_face_joy = 'NONE'
gv_face_sorrow = 'NONE'
gv_face_anger = 'NONE'
gv_face_surprise = 'NONE'
gv_num_faces = '0'
# Add values to output row
outputRow.append(originalFileName)
outputRow.append(imageHash)
outputRow.append(extension)
outputRow.append(copyFilename)
outputRow.append(gv_ss_adult)
outputRow.append(gv_ss_spoof)
outputRow.append(gv_ss_medical)
outputRow.append(gv_ss_violence)
outputRow.append(gv_labels)
outputRow.append(gv_text)
outputRow.append(gv_text_lang)
outputRow.append(gv_web_entities)
outputRow.append(gv_web_full_matching_images)
outputRow.append(gv_web_partial_matching_images)
outputRow.append(gv_web_pages_with_matching_images)
outputRow.append(gv_web_visually_similar_images)
outputRow.append(gv_num_faces)
outputRow.append(gv_face_joy)
outputRow.append(gv_face_sorrow)
outputRow.append(gv_face_anger)
outputRow.append(gv_face_surprise)
# Write results to output file
outputCSV.writerow(outputRow)
if makeNetwork:
nx.write_gexf(graph, graphfilepath)
if makeWebNetwork:
nx.write_gexf(webgraph, webgraphfilepath)
except KeyboardInterrupt:
if makeNetwork:
if graph:
nx.write_gexf(graph, graphfilepath)
if makeWebNetwork:
if webgraph:
nx.write_gexf(webgraph, webgraphfilepath)
print("\n\n**Script interrupted by user**\n\n")
except Exception:
traceback.print_exc(file=sys.stdout)
sys.exit(0)
def main():
# load configurations from settings file using configparser()
settingsfile = '../settings/settings'
if os.path.exists(settingsfile):
config = configparser.ConfigParser()
config._interpolation = configparser.ExtendedInterpolation()
config.read(settingsfile)
paths = config['Paths']
outputdir = paths['output_dir'] + "/"
emailsentfilecsv = paths['emailsentfile']
logincredfile = paths['login_cred']
else:
print(
"No settings file in ../settings/settings. Using default values.")
parent_directory = ".."
inputdir = parent_directory + 'datainput/'
outputdir = parent_directory + 'dataoutput/'
logincredfile = inputdir + 'login.cred'
if os.path.exists(emailsentfilecsv):
dfsentemails = pd.read_csv(emailsentfilecsv)
else:
dfsentemails = pd.DataFrame({'date': [], 'email': []})
print("Saving data to: ", emailsentfilecsv)
# parse command-line arguments using argparse()
parser = argparse.ArgumentParser()
parser.add_argument('-v',
'--verbose',
help='Verbose mode [Default=Off]',
action='store_true')
parser.add_argument('-l',
'--login',
help='Enter login info manually [Default=Off]',
action='store_true')
parser.add_argument('-n',
'--dryrun',
help='Dry run (not send emails)',
action='store_true')
args = parser.parse_args()
verbose = args.verbose
login = args.login
dryrun = args.dryrun
if dryrun:
print(
"In dryrun the [emailsentfilecsv] is not created so that multiples runs shows data output."
)
# BEGIN MANUAL LOGIN
# login credentials from file to avoid exposing them in code
if not login:
if os.path.exists(logincredfile):
f = open(logincredfile, "r")
uname = f.readline().strip()
pwd = f.readline().strip()
f.close()
else:
print('Login credentials file not found. Enter it manually.')
uname, pwd = getlogininfo()
else:
uname, pwd = getlogininfo()
# set up the SMTP server
s = smtplib.SMTP(host="smtp-mail.outlook.com", port=587)
s.starttls()
print("Logging in")
s.login(uname, pwd)
emailto = ""
emailmsg = ""
counter = 0
# For each contact, send the email:
for emailfile in glob.glob(outputdir + '*.txt'):
coursename = re.sub(
' ', '',
re.sub(r'letter-', '',
re.sub(r'.txt', '',
os.path.basename(emailfile)).strip(" ")))
emailsubject = "[" + coursename + "] " + "Unattempted assignments"
if verbose:
print('+' * 80, "\n")
print(emailsubject, "\n")
with open(emailfile, 'r') as f:
line = f.readline()
while line:
if line == '=' * 70 + '\n':
createmessage(dfsentemails, s, emailsentfilecsv, counter,
emailto, emailsubject, emailmsg, verbose,
dryrun)
# after sending the previous email start a new email message
counter = counter + 1
emailto = f.readline()
emailmsg = ""
elif line == "EOF":
createmessage(dfsentemails, s, emailsentfilecsv, counter,
emailto, emailsubject, emailmsg, verbose,
dryrun)
# after sending the previous email start a new email message
counter = 0
emailto = ""
emailmsg = ""
else:
emailmsg = emailmsg + line
line = f.readline()
# print(line)
# if counter > 1:
# break
s.quit()
def parser(cls):
cfg = configparser.ConfigParser(
interpolation=configparser.ExtendedInterpolation())
cfg.optionxform = str
return cfg
def __init__(self, grid_conf_file, args):
self.conf = configparser.ConfigParser(
interpolation=configparser.ExtendedInterpolation())
self.conf.read_file(open(grid_conf_file, 'r'))
def __init__(self, config_name, test=False):
self.logger = logging.getLogger('%s.%s' % (__name__, config_name))
self.logger.setLevel(logging.DEBUG)
self.error = True
self.status = 'Backup failed!'
self.pid_created = False
script_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
# Load the global configuration file
configfile_global = os.path.join(script_dir, 'rsync-backup.conf')
self.global_config = configparser.ConfigParser(
interpolation=configparser.ExtendedInterpolation())
self.global_config.read_file(open(configfile_global))
# Load the backup configuration file
configfile_backup = os.path.join(script_dir, 'conf.d',
'%s.conf' % config_name)
self.config = configparser.ConfigParser(
interpolation=configparser.ExtendedInterpolation())
self.config.read_file(open(configfile_backup))
self.test = test
current_datetime = datetime.now()
self.rules = configfile_backup.replace('.conf', '.rules')
self.timestamp = current_datetime.strftime('%Y-%m-%d-%H%M%S')
self.backup_root = os.path.join(
self.global_config.get('general', 'backup_root'),
self.config.get('general', 'label'))
self.log_dir = os.path.join(self.backup_root, 'logs')
self.log_file = os.path.join(self.log_dir, '%s.log' % self.timestamp)
self.to_addrs = set(
self.config.get('reporting',
'to_addrs',
fallback=self.global_config.get(
'reporting', 'to_addrs')).split(','))
self.pidfile = '/var/run/backup/backup-%s.pid' % (self.config.get(
'general', 'label'))
self.cache_dir = os.path.join(self.backup_root, 'cache')
self.backups_dir = os.path.join(self.backup_root, 'backups')
self.last_verification_file = os.path.join(self.cache_dir,
'last_verification')
self.umask = int(
self.global_config.get('general', 'umask', fallback='0o077'), 8)
os.umask(self.umask)
# Configure backup intervals
self.intervals = {
'snapshot': {
'retention':
self.config.getint('retention',
'snapshot',
fallback=self.global_config.getint(
'retention', 'snapshot')),
},
'daily': {
'retention':
self.config.getint('retention',
'daily',
fallback=self.global_config.getint(
'retention', 'daily')),
},
'monthly': {
'retention':
self.config.getint('retention',
'monthly',
fallback=self.global_config.getint(
'retention', 'monthly')),
},
'yearly': {
'retention':
self.config.getint('retention',
'yearly',
fallback=self.global_config.getint(
'retention', 'yearly')),
}
}
# Check if backup is already running and set up logging
self._is_running()
self._create_dirs()
self._prepare_logging()
def configTest(self):
# setup the configuration parser
self.config = configparser.SafeConfigParser(interpolation=configparser.ExtendedInterpolation())
# Set the config parser to make option names case sensitive.
self.config.optionxform = str
# fill ENV section with environemt variables
self.config.add_section('ENV')
for k,v in os.environ.items():
self.config.set('ENV', k, v.replace("$","$$"))
# log the list of files - note that the argument parser
# puts the input files in a list, with the first member
# being the list of input files
self.log['inifiles'] = self.args.ini_files[0]
# initialize the list of active sections
self.actives = []
# if they specified a list to execute, then use it
sections = []
if self.args.section:
sections = self.args.section.split(",")
skip = False
elif self.args.skipsections:
sections = self.args.skipsections.split(",")
skip = True
else:
sections = None
# cycle thru the input files
for testFile in self.log['inifiles']:
if not os.path.isfile(testFile):
print("Test description file",testFile,"not found!")
sys.exit(1)
self.config.read(self.log['inifiles'])
for section in self.config.sections():
if section.startswith("SKIP") or section.startswith("skip"):
# users often want to temporarily ignore a section
# of their test definition file, but don't want to
# remove it lest they forget what it did. So let
# them just mark the section as "skip" to be ignored
continue
# if we are to filter the sections, then do so
takeus = True
if sections is not None:
found = False
for sec in sections:
if sec == section:
found = True
sections.remove(sec)
if skip:
takeus = False
break
if not found and not skip:
takeus = False
if takeus:
self.actives.append(section)
if self.logger is not None:
self.logger.verbose_print("SECTION: " + section)
self.logger.verbose_print(self.config.items(section))
if sections is not None and 0 != len(sections) and not skip:
print("ERROR: sections were specified for execution and not found:",sections)
sys.exit(1)
return
import configparser
conf = configparser.ConfigParser(interpolation=configparser.ExtendedInterpolation())
# conf['DEFAULT'] = {}
# conf['DEFAULT']['path'] = 'c:/f/d'
# conf['lh'] = {}
# conf['lh']['path'] = '${path}/init.com'
# with open('1.txt', 'a', encoding='utf-8') as f:
# conf.write(f)
conf.read('1.txt')
print(conf.get('lh','path'))
import select
import configparser
import time
import sys
timeout = 1
log = open("log.log", "w")
try:
os.mkdir("logs-" + sys.argv[1])
except:
pass
programs = []
programs = configparser.ConfigParser(
interpolation=configparser.ExtendedInterpolation(), default_section="")
programs.read("entrylist.ini")
stats = {}
test_data = [
i.split() for i in open(sys.argv[1]).read().split("\n") if i.strip()
]
def pad(s, l):
return s + (l - len(s)) * " "
def print(*args, print=print, **kwargs):
s = " ".join(str(i) for i in args)
log.write(s + kwargs.get("end", "\n"))
tmpDir = 'tmp'
headerPart = os.path.join(tmpDir, '~header')
binPart = os.path.join(tmpDir, '~bin')
footerPart = os.path.join(tmpDir, '~footer')
# Command line args
if len(sys.argv) == 1:
print("Usage: pack.py <config file>")
print("Example: pack.py configs/letv-x355pro.ini")
quit()
configFile = sys.argv[1]
# Parse config file
config = configparser.ConfigParser(
interpolation=configparser.ExtendedInterpolation())
#config = configparser.ConfigParser()
config.read(configFile)
# Main
main = config['Main']
firmwareFileName = main['FirmwareFileName']
projectFolder = main['ProjectFolder']
useHexValuesPrefix = utils.str2bool(main['useHexValuesPrefix'])
SCRIPT_FIRMWARE_FILE_NAME = main['SCRIPT_FIRMWARE_FILE_NAME']
DRAM_BUF_ADDR = main['DRAM_BUF_ADDR']
MAGIC_FOOTER = main['MAGIC_FOOTER']
HEADER_SIZE = utils.sizeInt(main['HEADER_SIZE'])
# Header
def main():
"""Main procedure for training the GAN.
It will run through all classes and train the GANs
for each class.
The following command line arguments are supported:
--traintool: script to call
--tfrecords: path to the TFrecords
--gans: path to the GANs folder
--size: image size (part of folder name)
--gpus: number of GPUs to use
--kimg: number of training rounds (in 1000s of images)
"""
# read config file with defaults
config = configparser.ConfigParser(
interpolation=configparser.ExtendedInterpolation())
config.read('config.ini')
# first step: parse config-file related arguments
cfg_parser = argparse.ArgumentParser(description='Train GAN',
add_help=False)
cfg_parser.add_argument(
'-e',
'--env',
type=str,
default=configparser.DEFAULTSECT,
help='base environment to use (in file `config.ini`)')
# parse config file related args
cfg_args = cfg_parser.parse_known_args()[0]
# add defaults from environment
defaults = dict(config.items(cfg_args.env))
# add other argument parser arguments
parser = argparse.ArgumentParser(parents=[cfg_parser])
parser.set_defaults(**defaults)
parser.add_argument('--traintool', type=str, help='script to call')
parser.add_argument('--tfrecords', type=str, help='path to the TFrecords')
parser.add_argument('--gans', type=str, help='path to the GANs folder')
parser.add_argument('--size',
type=str,
help='image size (part of folder name)')
parser.add_argument('--gpus', type=int, help='number of GPUs to use')
parser.add_argument('--kimg',
type=int,
help='number of training rounds (in 1000s of images)')
# parse command line arguments
args = parser.parse_args()
# does the source folder exist?
train_path = f"{args.tfrecords}_{args.size}/"
if not os.path.isdir(train_path):
print(f"Error: source folder `{train_path}` does not exist!",
file=sys.stderr)
os._exit(1)
# run through classes
for cls in range(5):
# source path for this class
source_path = train_path + "/" + str(cls)
if not os.path.isdir(source_path):
print(f"Error: Class folder `{source_path}` not found",
file=sys.stderr)
os._exit(2)
# start existing script
print(f"Training GAN for class {cls}...")
subprocess.run([
sys.executable, args.traintool, f"--num-gpus={args.gpus}",
f"--data-dir={train_path}", f"--dataset={cls}",
f"--total-kimg={args.kimg}",
f"--result-dir={args.gans}_{args.size}", "--config=config-f",
"--mirror-augment=true"
])