def read():
default_config = Config({
'credentials': {
'stack-exchange': {
'email': '',
'password': '',
},
},
'resources': {
'db': 'sqlite:///:memory:',
},
'state': {
'params': {
'--server': 'se',
'--room': '70095',
'--user': '******',
},
'room-tailed-times': {
'se': {},
'so': {},
'mse': {},
},
},
})
global_config = Config(toml.loads(open('~/.stack.chat.toml')))
local_config = Config(
default_config, global_config, toml.loads(open('./.stack.chat.toml')))
return local_config
def _parse_pipfile(self, contents):
# If any outline tables are present...
if ("[packages." in contents) or ("[dev-packages." in contents):
data = toml.loads(contents)
# Convert all outline tables to inline tables.
for section in ("packages", "dev-packages"):
for package in data.get(section, {}):
# Convert things to inline tables — fancy :)
if hasattr(data[section][package], "keys"):
_data = data[section][package]
data[section][package] = toml._get_empty_inline_table(dict)
data[section][package].update(_data)
# We lose comments here, but it's for the best.)
try:
return contoml.loads(toml.dumps(data, preserve=True))
except RuntimeError:
return toml.loads(toml.dumps(data, preserve=True))
else:
# Fallback to toml parser, for large files.
try:
return contoml.loads(contents)
except Exception:
return toml.loads(contents)
def parsed_pipfile(self):
# Open the pipfile, read it into memory.
with open(self.pipfile_location) as f:
contents = f.read()
# If any outline tables are present...
if ('[packages.' in contents) or ('[dev-packages.' in contents):
data = toml.loads(contents)
# Convert all outline tables to inline tables.
for section in ('packages', 'dev-packages'):
for package in data.get(section, {}):
# Convert things to inline tables — fancy :)
if hasattr(data[section][package], 'keys'):
_data = data[section][package]
data[section][package] = toml._get_empty_inline_table(dict)
data[section][package].update(_data)
# We lose comments here, but it's for the best.)
try:
return contoml.loads(toml.dumps(data, preserve=True))
except RuntimeError:
return toml.loads(toml.dumps(data, preserve=True))
else:
# Fallback to toml parser, for large files.
try:
return contoml.loads(contents)
except Exception:
return toml.loads(contents)
def test_normal(self, capsys, table_name, header, value, expected):
writer = table_writer_class()
writer.table_name = table_name
writer.headers = header
writer.value_matrix = value
writer.write_table()
out, err = capsys.readouterr()
print_test_result(expected=expected, actual=out, error=err)
assert toml.loads(out) == toml.loads(expected)
def get_server():
config_file = os.path.join(config.get_config_dir(), 'ias_proxy.toml')
LOGGER.info('Loading IAS Proxy config from: %s', config_file)
# Lack of a config file is a fatal error, so let the exception percolate
# up to caller
with open(config_file) as fd:
proxy_config = toml.loads(fd.read())
# Verify the integrity (as best we can) of the TOML configuration file
valid_keys = set(['proxy_name', 'proxy_port', 'ias_url', 'spid_cert_file'])
found_keys = set(proxy_config.keys())
invalid_keys = found_keys.difference(valid_keys)
if invalid_keys:
raise \
ValueError(
'IAS Proxy config file contains the following invalid '
'keys: {}'.format(
', '.join(sorted(list(invalid_keys)))))
missing_keys = valid_keys.difference(found_keys)
if missing_keys:
raise \
ValueError(
'IAS Proxy config file missing the following keys: '
'{}'.format(
', '.join(sorted(list(missing_keys)))))
return IasProxyServer(proxy_config)
def __init__(self, context):
self.context = context
def get_env_bool(var, default):
# Contents of env vars are strings by default. This returns the
# boolean value of the specified environment variable, or the
# speciried default if the var doesn't contain True or False
return {'True': True, 'False': False}.get(os.environ.get(var), default)
def resolverelative(category, key):
# Allow ~
self.config[category][key] = path.expanduser(self.config[category][key])
# Resolve relative paths
self.config[category][key] = path.join(context.topdir,
self.config[category][key])
if not hasattr(self.context, "bootstrapped"):
self.context.bootstrapped = False
config_path = path.join(context.topdir, ".servobuild")
if path.exists(config_path):
with open(config_path) as f:
self.config = toml.loads(f.read())
else:
self.config = {}
# Handle missing/default items
self.config.setdefault("tools", {})
default_cache_dir = os.environ.get("SERVO_CACHE_DIR",
path.join(context.topdir, ".servo"))
self.config["tools"].setdefault("cache-dir", default_cache_dir)
resolverelative("tools", "cache-dir")
default_cargo_home = os.environ.get("CARGO_HOME",
path.join(context.topdir, ".cargo"))
self.config["tools"].setdefault("cargo-home-dir", default_cargo_home)
resolverelative("tools", "cargo-home-dir")
context.sharedir = self.config["tools"]["cache-dir"]
self.config["tools"].setdefault("use-rustup", True)
self.config["tools"].setdefault("rustc-with-gold", get_env_bool("SERVO_RUSTC_WITH_GOLD", True))
self.config.setdefault("build", {})
self.config["build"].setdefault("android", False)
self.config["build"].setdefault("mode", "")
self.config["build"].setdefault("debug-mozjs", False)
self.config["build"].setdefault("ccache", "")
self.config["build"].setdefault("rustflags", "")
self.config["build"].setdefault("incremental", None)
self.config["build"].setdefault("thinlto", False)
self.config["build"].setdefault("webgl-backtrace", False)
self.config["build"].setdefault("dom-backtrace", False)
self.config.setdefault("android", {})
self.config["android"].setdefault("sdk", "")
self.config["android"].setdefault("ndk", "")
self.config["android"].setdefault("toolchain", "")
# Set default android target
self.handle_android_target("armv7-linux-androideabi")
def load_toml_rest_api_config(filename):
"""Returns a RestApiConfig created by loading a TOML file from the
filesystem.
"""
if not os.path.exists(filename):
LOGGER.info(
"Skipping rest api loading from non-existent config file: %s",
filename)
return RestApiConfig()
LOGGER.info("Loading rest api information from config: %s", filename)
try:
with open(filename) as fd:
raw_config = fd.read()
except IOError as e:
raise RestApiConfigurationError(
"Unable to load rest api configuration file: {}".format(str(e)))
toml_config = toml.loads(raw_config)
invalid_keys = set(toml_config.keys()).difference(
['bind', 'connect', 'timeout'])
if invalid_keys:
raise RestApiConfigurationError(
"Invalid keys in rest api config: {}".format(
", ".join(sorted(list(invalid_keys)))))
config = RestApiConfig(
bind=toml_config.get("bind", None),
connect=toml_config.get('connect', None),
timeout=toml_config.get('timeout', None)
)
return config
def _get_dir(toml_config_setting,
sawtooth_home_dir,
windows_dir,
default_dir):
"""Determines the directory path based on configuration.
Arguments:
toml_config_setting (str): The name of the config setting related
to the directory which will appear in path.toml.
sawtooth_home_dir (str): The directory under the SAWTOOTH_HOME
environment variable. For example, for 'data' if the data
directory is $SAWTOOTH_HOME/data.
windows_dir (str): The windows path relative to the computed base
directory.
default_dir (str): The default path on Linux.
Returns:
directory (str): The path.
"""
conf_file = os.path.join(get_config_dir(), 'path.toml')
if os.path.exists(conf_file):
with open(conf_file) as fd:
raw_config = fd.read()
toml_config = toml.loads(raw_config)
if toml_config_setting in toml_config:
return toml_config[toml_config_setting]
return _select_dir(sawtooth_home_dir, windows_dir, default_dir)
def main():
import time
conf_fn = "config/ak_client.toml"
with open(conf_fn, "r") as conf_fh:
cfg = toml.loads(conf_fh.read())
conf = cfg["redis"]
rclient = RedisLua(conf)
key = "eqm011"
i = 0
while 1:
result = "info:%s" %(i)
rclient.save(key, i%4, result)
i = i +1
print i
time.sleep(5)
def check_lock(file_name, contents):
def find_reverse_dependencies(name, content):
for package in itertools.chain([content.get("root", {})], content["package"]):
for dependency in package.get("dependencies", []):
if dependency.startswith("{} ".format(name)):
yield package["name"], dependency
if not file_name.endswith(".lock"):
raise StopIteration
# Package names to be neglected (as named by cargo)
exceptions = config["ignore"]["packages"]
content = toml.loads(contents)
packages_by_name = {}
for package in content.get("package", []):
if "replace" in package:
continue
source = package.get("source", "")
if source == r"registry+https://github.com/rust-lang/crates.io-index":
source = "crates.io"
packages_by_name.setdefault(package["name"], []).append((package["version"], source))
for (name, packages) in packages_by_name.iteritems():
has_duplicates = len(packages) > 1
duplicates_allowed = name in exceptions
if has_duplicates == duplicates_allowed:
continue
if duplicates_allowed:
message = 'duplicates for `{}` are allowed, but only single version found'.format(name)
else:
message = "duplicate versions for package `{}`".format(name)
packages.sort()
packages_dependencies = list(find_reverse_dependencies(name, content))
for version, source in packages:
short_source = source.split("#")[0].replace("git+", "")
message += "\n\t\033[93mThe following packages depend on version {} from '{}':\033[0m" \
.format(version, short_source)
for name, dependency in packages_dependencies:
if version in dependency and short_source in dependency:
message += "\n\t\t" + name
yield (1, message)
# Check to see if we are transitively using any blocked packages
for package in content.get("package", []):
package_name = package.get("name")
package_version = package.get("version")
for dependency in package.get("dependencies", []):
dependency = dependency.split()
dependency_name = dependency[0]
whitelist = config['blocked-packages'].get(dependency_name)
if whitelist is not None:
if package_name not in whitelist:
fmt = "Package {} {} depends on blocked package {}."
message = fmt.format(package_name, package_version, dependency_name)
yield (1, message)
def _reload_mirrors(self, signum, frame, force=False):
print("reload mirror configs, force restart: {}".format(force))
with open(self._config_file) as f:
self._settings = toml.loads(f.read())
for mirror_opt in self._settings["mirrors"]:
name = mirror_opt["name"]
newMirCfg = MirrorConfig(self, mirror_opt)
if name in self._mirrors:
if newMirCfg.compare(self._mirrors[name]):
continue
self._mirrors[name] = newMirCfg
hooks = newMirCfg.hooks() + self.hooks()
newProvider = newMirCfg.to_provider(hooks)
self._providers[name] = newProvider
if name in self.processes:
q, p = self.processes[name]
if force:
p.terminate()
print("Terminated Job: {}".format(name))
self.run_provider(name)
else:
q.put("terminate")
print("New configuration queued to {}".format(name))
else:
print("New mirror: {}".format(name))
self.run_provider(name)
def read_config(self, config_file):
self._config_file = config_file
with open(self._config_file) as f:
self._settings = toml.loads(f.read())
self._inited = True
self._mirrors = {}
self._providers = {}
self.processes = {}
self.semaphore = Semaphore(self._settings["global"]["concurrent"])
self.channel = Queue()
self._hooks = []
self.mirror_root = self._settings["global"]["mirror_root"]
self.use_btrfs = self._settings["global"]["use_btrfs"]
self.btrfs_service_dir_tmpl = self._settings["btrfs"]["service_dir"]
self.btrfs_working_dir_tmpl = self._settings["btrfs"]["working_dir"]
self.btrfs_gc_dir_tmpl = self._settings["btrfs"]["gc_dir"]
self.status_file = self._settings["global"]["status_file"]
self.status_manager = StatusManager(self, self.status_file)
self.ctrl_addr = self._settings["global"]["ctrl_addr"]
self.ctrl_channel = Queue()
p = Process(
target=run_control_server,
args=(self.ctrl_addr, self.channel, self.ctrl_channel),
)
p.start()
self.processes["CTRL_SERVER"] = (self.ctrl_channel, p)
def __init__(self, context):
self.context = context
def resolverelative(category, key):
# Allow ~
self.config[category][key] = path.expanduser(self.config[category][key])
# Resolve relative paths
self.config[category][key] = path.join(context.topdir,
self.config[category][key])
if not hasattr(self.context, "bootstrapped"):
self.context.bootstrapped = False
config_path = path.join(context.topdir, ".servobuild")
if path.exists(config_path):
with open(config_path) as f:
self.config = toml.loads(f.read())
else:
self.config = {}
# Handle missing/default items
self.config.setdefault("tools", {})
default_cache_dir = os.environ.get("SERVO_CACHE_DIR",
path.join(context.topdir, ".servo"))
self.config["tools"].setdefault("cache-dir", default_cache_dir)
resolverelative("tools", "cache-dir")
self.config["tools"].setdefault("cargo-home-dir",
path.join(context.topdir, ".cargo"))
resolverelative("tools", "cargo-home-dir")
context.sharedir = self.config["tools"]["cache-dir"]
self.config["tools"].setdefault("system-rust", False)
self.config["tools"].setdefault("system-cargo", False)
self.config["tools"].setdefault("rust-root", "")
self.config["tools"].setdefault("cargo-root", "")
if not self.config["tools"]["system-rust"]:
self.config["tools"]["rust-root"] = path.join(
context.sharedir, "rust", self.rust_path())
if not self.config["tools"]["system-cargo"]:
self.config["tools"]["cargo-root"] = path.join(
context.sharedir, "cargo", self.cargo_build_id())
self.config["tools"].setdefault("rustc-with-gold", True)
self.config.setdefault("build", {})
self.config["build"].setdefault("android", False)
self.config["build"].setdefault("mode", "")
self.config["build"].setdefault("debug-mozjs", False)
self.config["build"].setdefault("ccache", "")
self.config.setdefault("android", {})
self.config["android"].setdefault("sdk", "")
self.config["android"].setdefault("ndk", "")
self.config["android"].setdefault("toolchain", "")
self.config["android"].setdefault("target", "arm-linux-androideabi")
self.config.setdefault("gonk", {})
self.config["gonk"].setdefault("b2g", "")
self.config["gonk"].setdefault("product", "flame")
def main():
conf_fn = "conf.toml"
with open(conf_fn) as conf_fh:
conf = toml.loads(conf_fh.read())
with open(conf["app"]["csv"], 'rb') as fh:
reader = csv.reader(fh)
i = 0
for row in reader:
i = i +1
if i > 1:
pass
#break
title = row[0]
img = gen_image(title.decode('utf-8'), conf)
gevent.sleep(0.5)
sendmail = Sendmail(conf)
sendmail.send(row[1], title, img)
print title, row[1]
gevent.sleep(0.5)
def parse_config(content):
config_file = toml.loads(content)
exclude = config_file.get("ignore", {})
# Add list of ignored directories to config
config["ignore"]["directories"] += exclude.get("directories", [])
# Add list of ignored files to config
config["ignore"]["files"] += exclude.get("files", [])
# Add list of ignored packages to config
config["ignore"]["packages"] = exclude.get("packages", [])
# Fix the paths (OS-dependent)
config['ignore']['files'] = map(lambda path: os.path.join(*path.split('/')),
config['ignore']['files'])
config['ignore']['directories'] = map(lambda path: os.path.join(*path.split('/')),
config['ignore']['directories'])
# Add dict of dir, list of expected ext to config
dirs_to_check = config_file.get("check_ext", {})
# Fix the paths (OS-dependent)
for path, exts in dirs_to_check.items():
fixed_path = os.path.join(*path.split('/'))
config['check_ext'][fixed_path] = exts
# Override default configs
user_configs = config_file.get("configs", [])
for pref in user_configs:
if pref in config:
config[pref] = user_configs[pref]
def __init__(self, context):
self.context = context
if not hasattr(self.context, "bootstrapped"):
self.context.bootstrapped = False
if not hasattr(self.context, "sharedir"):
self.context.sharedir = path.join(path.expanduser("~/"), ".servo")
config_path = path.join(context.topdir, ".servobuild")
if path.exists(config_path):
self.config = toml.loads(open(config_path).read())
else:
self.config = {}
# Handle missing/default items
self.config.setdefault("tools", {})
self.config["tools"].setdefault("system-rust", False)
self.config["tools"].setdefault("system-cargo", False)
self.config["tools"].setdefault("rust-root", "")
self.config["tools"].setdefault("cargo-root", "")
if not self.config["tools"]["system-rust"]:
self.config["tools"]["rust-root"] = path.join(
context.sharedir, "rust", *self.rust_snapshot_path().split("/"))
if not self.config["tools"]["system-cargo"]:
self.config["tools"]["cargo-root"] = path.join(
context.sharedir, "cargo", self.cargo_build_id())
self.config.setdefault("build", {})
self.config["build"].setdefault("android", False)
self.config.setdefault("android", {})
self.config["android"].setdefault("sdk", "")
self.config["android"].setdefault("ndk", "")
self.config["android"].setdefault("toolchain", "")
def __init__(self):
conf_fn = "conf.toml"
with open(conf_fn) as conf_fh:
toml_str = conf_fh.read()
self.conf = toml.loads(toml_str)
def main() -> None:
data = toml.load(sys.stdin)
assert list(data.keys()) == ["source"]
# this value is non deterministic
data["source"]["vendored-sources"]["directory"] = "@vendor@"
lines = []
inner = data["source"]
for source, attrs in sorted(inner.items()):
lines.append("[source.{}]".format(quote(source)))
if source == "vendored-sources":
lines.append('"directory" = "@vendor@"\n')
else:
for key, value in sorted(attrs.items()):
attr = "{} = {}".format(quote(key), quote(value))
lines.append(attr)
lines.append("")
result = "\n".join(lines)
real = toml.loads(result)
assert real == data, "output = {} while input = {}".format(real, data)
print(result)
def __parse_config(self, configfile):
"""
Extracts the defined configuration parameters for the parser.
Important pre-condition is the user must configure elements and rules against a predefined enumeration.
:param configfile: configuration file
:return None:
"""
if os.path.exists(configfile):
cfg = open(configfile)
topconfig = toml.loads(cfg.read())
cfg.close()
if 'Filters' in topconfig:
if 'sources' in topconfig['Filters'][0]:
self._sources = topconfig['Filters'][0]['sources']
if 'rules' in topconfig['Filters'][0]:
temp = topconfig['Filters'][0]['rules']
for x in temp:
if x.lower() in self._accepted_rules:
self._rules.append(x.lower())
else:
self._logger.warning("Incompatible rule for parser " + x)
if 'start_offset' in topconfig['Filters'][0]:
self._line_offset = topconfig['Filters'][0]['start_offset']
def setUp(self):
"""
Initialize the system configuration and the user configuration.
Note the forward slash replacements for the user configuration. This is due to the forward slash being a
restricted character in TOML(package used to parse configuration files in LQMT).
"""
# relative pathing variables. Replace function calls for Windows compatibility.
self.directory = os.path.dirname(__file__)
self.alerts = self.directory + "/test_data/"
self.alerts = self.alerts.replace("\\", "/")
self.logging = self.directory + "/test_data/test-logs/lqmt"
self.logging = self.logging.replace("\\", "/")
self.whitelist = self.directory + "/test_data/whitelist/whitelist.txt"
self.whitelist = self.whitelist.replace("\\", "/")
self.whitelist_db = self.directory + "/test_data/whitelist/whitelist.db"
self.whitelist_db = self.whitelist_db.replace("\\", "/")
# configurations initialized
sysconf = SystemConfig()
self.sys_config = sysconf.getConfig()
config = USERCONFIG.format(self.alerts, self.logging, self.whitelist, self.whitelist_db)
self.toml_config = toml.loads(config)
self.toml_config = self.toml_config["Tools"]["FlexText"][0] # dirty way of parsing userconfig for ToolConfig
self.user_config = LQMToolConfig(config)
self.toolConfig = ToolConfig(self.toml_config, csvToolInfo={""}, unhandledCSV={""})
def parse(self):
# Open the Pipfile.
with open(self.filename) as f:
content = f.read()
# Load the default configuration.
default_config = {
u'source': [{u'url': u'https://pypi.python.org/simple', u'verify_ssl': True, 'name': "pypi"}],
u'packages': {},
u'requires': {},
u'dev-packages': {}
}
config = {}
config.update(default_config)
# Load the Pipfile's configuration.
config.update(toml.loads(content))
# Structure the data for output.
data = {
'_meta': {
'sources': config['source'],
'requires': config['requires']
},
}
# TODO: Validate given data here.
self.groups['default'] = config['packages']
self.groups['develop'] = config['dev-packages']
# Update the data structure with group information.
data.update(self.groups)
return data
def get_pages_from_dirs(dirs):
pages = []
for dir in dirs:
page = {}
path_to_metadata_file = path.join(dir, conf["metadata_file"])
with open(path_to_metadata_file) as metadata_file:
page = toml.loads(metadata_file.read())
try:
for key in conf["required_keys_in_page"]:
page[key]
except KeyError:
message = "`{}` key is missing from metadata file!".format(key)
logger.error(message)
exit()
content_path = path.join(dir, page["content_path"])
content_path = glob.glob(content_path)
if len(content_path) != 1:
logger.error("Content path matched less or more than needed!")
exit()
content_path = content_path[0]
with open(content_path) as content_file:
page["content"] = content_file.read()
pages.append(page)
return pages
def get_config():
# Read config parameters from a TOML file.
config = None
config_file_path = sys.argv[1]
with open(config_file_path) as config_file:
config = toml.loads(config_file.read())
return config
def read_toml(content):
check_section_header(content)
for k, v in toml.loads(content, collections.OrderedDict).items():
if len(v.values()) and isinstance(list(v.values())[0], dict):
raise RuntimeError('malformed section header -- forgot quotes?', k)
pname, version = split_name(k)
yield WhitelistRule(pname=pname, version=version, **v)
def load_config(filename):
if not path.exists(filename):
print 'no such file: {0}'.format(filename)
exit()
with open(filename) as fid:
config = toml.loads(fid.read())
return config
def load_config():
filename = sys.argv[1] if len(sys.argv) > 1 else 'benchmark.toml'
with open(filename) as configfile:
config = toml.loads(configfile.read())
items = config['item']
default = items['default']
if 'options' in config:
options = config['options']
else:
options = dict(ref_result="C",
time_field="elapsed",
show_diff_below=0.9,
verbose=True)
ret_items = {}
for name, item in zip(items, items.values()):
if name == 'default':
continue
if name.startswith('"') and name.endswith('"'):
import ast
name = ast.literal_eval(name)
profile = dict(default)
profile.update(item)
profile['name'] = name
for k, v in zip(profile, profile.values()):
if type(v) is str:
profile[k] = Template(v).safe_substitute(**profile)
ret_items[name] = profile
return ret_items, options
def repack(host, targets, channel='stable', suffix=''):
print("Repacking rust for %s..." % host)
url = 'https://static.rust-lang.org/dist/channel-rust-' + channel + '.toml'
req = requests.get(url)
req.raise_for_status()
manifest = toml.loads(req.content)
if manifest['manifest-version'] != '2':
print('ERROR: unrecognized manifest version %s.' % manifest['manifest-version'])
return
print('Using manifest for rust %s as of %s.' % (channel, manifest['date']))
print('Fetching packages...')
rustc = fetch_package(manifest, 'rustc', host)
cargo = fetch_package(manifest, 'cargo', host)
stds = fetch_std(manifest, targets)
print('Installing packages...')
tar_basename = 'rustc-' + host
if suffix:
tar_basename += '-' + suffix
tar_basename += '-repack'
install_dir = 'rustc'
subprocess.check_call(['rm', '-rf', install_dir])
install(os.path.basename(rustc['url']), install_dir)
install(os.path.basename(cargo['url']), install_dir)
for std in stds:
install(os.path.basename(std['url']), install_dir)
pass
print('Tarring %s...' % tar_basename)
tar_options, tar_ext = tar_for_host(host)
subprocess.check_call(['tar', tar_options, tar_basename + tar_ext, install_dir])
subprocess.check_call(['rm', '-rf', install_dir])
def _loadfile(fname, logger):
conf_dict = {}
if os.path.exists(fname):
# file exists
if HAVE_TOML:
conf_dict = toml.loads(open(fname).read())
else:
raise ImportError("No module named toml")
include_files = []
if "include" in conf_dict.keys():
if "file" in conf_dict["include"].keys():
f = conf_dict["include"]["file"]
include_files.append(os.path.expanduser(f))
if "directory" in conf_dict["include"].keys():
d = conf_dict["include"]["directory"]
include_files = include_files + sorted([os.path.join(dp, f) for dp, dn, fn in os.walk(os.path.expanduser(d)) for f in fn])
for f in include_files:
try:
_merge(conf_dict, _loadfile(f, logger))
except Exception as e:
logger.error("Config file parse error: " + str(f) + " " + str(e).split("\n")[0])
if HAVE_JSONSCHEMA:
validate(conf_dict, json.loads(_confspec))
else:
raise ImportError("No module named jsonschema")
return conf_dict
def load_toml_path_config(filename):
"""Returns a PathConfig created by loading a TOML file from the
filesystem.
"""
if not os.path.exists(filename):
LOGGER.info(
"Skipping path loading from non-existent config file: %s",
filename)
return PathConfig()
LOGGER.info("Loading path information from config: %s", filename)
try:
with open(filename) as fd:
raw_config = fd.read()
except IOError as e:
raise LocalConfigurationError(
"Unable to load path configuration file: {}".format(str(e)))
toml_config = toml.loads(raw_config)
invalid_keys = set(toml_config.keys()).difference(
['data_dir', 'key_dir', 'log_dir'])
if invalid_keys:
raise LocalConfigurationError("Invalid keys in path config: {}".format(
", ".join(sorted(list(invalid_keys)))))
config = PathConfig(
config_dir=None,
data_dir=toml_config.get('data_dir', None),
key_dir=toml_config.get('key_dir', None),
log_dir=toml_config.get('log_dir', None)
)
return config
def test_convert_example_test(self):
example = textwrap.dedent('''\
# This is a TOML document. Boom.
title = "TOML Example"
[owner]
name = "Tom Preston-Werner"
organization = "GitHub"
bio = "GitHub Cofounder & CEO\\nLikes tater tots and beer."
dob = 1979-05-27T07:32:00Z # First class dates? Why not?
[database] # comment after a category
server = "192.168.1.1" # comment after a key
ports = [ 8001, 8001, 8002 ]
connection_max = 5000
enabled = true
[servers]
# You can indent as you please.
[servers.alpha]
ip = "10.0.0.1"
dc = "eqdc10"
# Tabs or spaces. TOML don't care.
\t[servers.beta]
\tip = "10.0.0.2"
\tdc = "eqdc10"
[clients]
# Line breaks are OK when inside arrays
data = [
["gamma", "delta"],
[1, 2]
]
''')
expected = dict(
title = "TOML Example",
owner = dict(
name = "Tom Preston-Werner",
organization = "GitHub",
bio = "GitHub Cofounder & CEO\nLikes tater tots and beer.",
dob = datetime.datetime(1979, 5, 27, 7, 32, 0)
),
database = dict(
server = "192.168.1.1",
ports = [8001, 8001, 8002],
connection_max = 5000,
enabled = True,
),
servers = dict(
alpha = dict(ip = "10.0.0.1", dc = "eqdc10"),
beta = dict(ip = "10.0.0.2", dc = "eqdc10")
),
clients = dict(data = [["gamma", "delta"], [1, 2]]),
)
self.assertEqual(toml.loads(example), expected)
def create_default_config(cls):
""" Create default config file if not exists """
if not os.path.exists(cls.get_config_file_path()):
config = toml.loads(cls.default_config)
with open(AppUtil.get_config_file_path(), "w") as f:
toml.dump(config, f)
def __init__(self):
# config_dir = Path(__file__).resolve().parent.parent.joinpath("/config")
# super().__init__(toml.load(config_dir.joinpath(default_config)))
super().__init__(toml.loads(self.CONFIG))
from setuptools import setup, find_packages
from os import path
from io import open
here = path.abspath(path.dirname(__file__))
# Get the long description from the README file
with open(path.join(here, "README.md"), encoding="utf-8") as f:
long_description = f.read()
with open(path.join(here, "Pipfile"), encoding="utf-8") as f:
import toml # requires pyproject.toml file
tobeinstalled = toml.loads(f.read())
dct = tobeinstalled['packages'] # dictionary
install_requires = [
x +
dct[x] if dct[x] != "*" and not issubclass(dct[x].__class__, dict) else x
for x in dct
]
dct = tobeinstalled['dev-packages']
extras_require = {
'dev': [
x + dct[x]
if dct[x] != "*" and not issubclass(dct[x].__class__, dict) else x
for x in dct
]
}
# Note: this requires a pyproject.toml file with the following contents:
# [build-system]
import sys
from logging.config import fileConfig
import toml
from alembic import context
from sqlalchemy import engine_from_config, pool
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
config = context.config
# Interpret the config file for Python logging.
# This line sets up loggers basically.
fileConfig(config.config_file_name)
with open("settings.toml") as conffile:
toml_data = toml.loads(conffile.read())
config.set_main_option("sqlalchemy.url", toml_data["common"]["database_engine_dsn"])
# add your model's MetaData object here
# for 'autogenerate' support
sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..')))
from db import Base
target_metadata = Base.metadata
# other values from the config, defined by the needs of env.py,
# can be acquired:
# my_important_option = config.get_main_option("my_important_option")
# ... etc.
def __init__(self, context):
self.context = context
def get_env_bool(var, default):
# Contents of env vars are strings by default. This returns the
# boolean value of the specified environment variable, or the
# speciried default if the var doesn't contain True or False
return {
'True': True,
'False': False
}.get(os.environ.get(var), default)
def resolverelative(category, key):
# Allow ~
self.config[category][key] = path.expanduser(
self.config[category][key])
# Resolve relative paths
self.config[category][key] = path.join(context.topdir,
self.config[category][key])
if not hasattr(self.context, "bootstrapped"):
self.context.bootstrapped = False
config_path = path.join(context.topdir, ".servobuild")
if path.exists(config_path):
with open(config_path) as f:
self.config = toml.loads(f.read())
else:
self.config = {}
# Handle missing/default items
self.config.setdefault("tools", {})
default_cache_dir = os.environ.get("SERVO_CACHE_DIR",
path.join(context.topdir, ".servo"))
self.config["tools"].setdefault("cache-dir", default_cache_dir)
resolverelative("tools", "cache-dir")
default_cargo_home = os.environ.get(
"CARGO_HOME", path.join(context.topdir, ".cargo"))
self.config["tools"].setdefault("cargo-home-dir", default_cargo_home)
resolverelative("tools", "cargo-home-dir")
context.sharedir = self.config["tools"]["cache-dir"]
self.config["tools"].setdefault("use-rustup", True)
self.config["tools"].setdefault(
"rustc-with-gold", get_env_bool("SERVO_RUSTC_WITH_GOLD", True))
self.config.setdefault("build", {})
self.config["build"].setdefault("android", False)
self.config["build"].setdefault("mode", "")
self.config["build"].setdefault("debug-mozjs", False)
self.config["build"].setdefault("ccache", "")
self.config["build"].setdefault("rustflags", "")
self.config["build"].setdefault("incremental", None)
self.config["build"].setdefault("thinlto", False)
self.config["build"].setdefault("webgl-backtrace", False)
self.config["build"].setdefault("dom-backtrace", False)
self.config.setdefault("android", {})
self.config["android"].setdefault("sdk", "")
self.config["android"].setdefault("ndk", "")
self.config["android"].setdefault("toolchain", "")
# Set default android target
self.handle_android_target("armv7-linux-androideabi")
def __init__(self, context):
self.context = context
def resolverelative(category, key):
# Allow ~
self.config[category][key] = path.expanduser(
self.config[category][key])
# Resolve relative paths
self.config[category][key] = path.join(context.topdir,
self.config[category][key])
if not hasattr(self.context, "bootstrapped"):
self.context.bootstrapped = False
config_path = path.join(context.topdir, ".servobuild")
if path.exists(config_path):
with open(config_path) as f:
self.config = toml.loads(f.read())
else:
self.config = {}
# Handle missing/default items
self.config.setdefault("tools", {})
default_cache_dir = os.environ.get("SERVO_CACHE_DIR",
path.join(context.topdir, ".servo"))
self.config["tools"].setdefault("cache-dir", default_cache_dir)
resolverelative("tools", "cache-dir")
self.config["tools"].setdefault("cargo-home-dir",
path.join(context.topdir, ".cargo"))
resolverelative("tools", "cargo-home-dir")
context.sharedir = self.config["tools"]["cache-dir"]
self.config["tools"].setdefault("system-rust", False)
self.config["tools"].setdefault("system-cargo", False)
self.config["tools"].setdefault("rust-root", "")
self.config["tools"].setdefault("cargo-root", "")
if not self.config["tools"]["system-rust"]:
self.config["tools"]["rust-root"] = path.join(
context.sharedir, "rust", self.rust_path())
if not self.config["tools"]["system-cargo"]:
self.config["tools"]["cargo-root"] = path.join(
context.sharedir, "cargo", self.cargo_build_id())
self.config["tools"].setdefault("rustc-with-gold", True)
self.config.setdefault("build", {})
self.config["build"].setdefault("android", False)
self.config["build"].setdefault("mode", "")
self.config["build"].setdefault("debug-mozjs", False)
self.config["build"].setdefault("ccache", "")
self.config.setdefault("android", {})
self.config["android"].setdefault("sdk", "")
self.config["android"].setdefault("ndk", "")
self.config["android"].setdefault("toolchain", "")
self.config["android"].setdefault("target", "arm-linux-androideabi")
self.config.setdefault("gonk", {})
self.config["gonk"].setdefault("b2g", "")
self.config["gonk"].setdefault("product", "flame")
def read_conf(path=""):
with open(path) as fd:
parsed_toml = toml.loads(fd.read())
return parsed_toml
from __future__ import print_function
from models.Dense3D import Dense3D
import torch
import toml
from training import Trainer
from validation import Validator
import torch.nn as nn
import os
from models.net2d import densenet121,densenet161,squeezenet1_1,vgg19_bn,resnet152,resnet152_plus
print("Loading options...")
with open('options_lip.toml', 'r') as optionsFile:
options = toml.loads(optionsFile.read())
if(options["general"]["usecudnnbenchmark"] and options["general"]["usecudnn"]):
print("Running cudnn benchmark...")
torch.backends.cudnn.benchmark = True
os.environ['CUDA_VISIBLE_DEVICES'] = options["general"]['gpuid']
torch.manual_seed(options["general"]['random_seed'])
#Create the model.
if options['general']['use_3d']:
model = Dense3D(options)##TODO:1
elif options['general']['use_slice']:
if options['general']['use_plus']:
model = resnet152_plus(options['general']['class_num'])
else:
def dump2netcdf(self, model_str='hysplit'):
"""
dump the assembled data to a netcdf file repeatatly call :meth:`save_item`
configuration (directories, names, etc) is given by the config file
"""
timestamps = np.array([
(dt - datetime.datetime(1970, 1, 1)).total_seconds()
for dt in self.dt_list
])
hours_cn = np.array([
dt.hour + dt.minute / 60. + dt.second / 3600.
for dt in self.dt_list
])
if not os.path.isdir(self.config['output_dir']):
os.makedirs(self.config['output_dir'])
ncfile = self.config['output_dir'] +\
'{}_{}_{}-output.nc'.format(
self.dt_list[0].strftime('%Y%m%d'), self.config['station']['short_name'], model_str)
# ncfile = "/home/devel/" +\
# '{}_hysplit_output.nc'.format(self.config['time']['begin_dt'].strftime('%Y%m%d'))
#dataset = netCDF4.Dataset(ncfile, 'w', format='NETCDF4')
dataset = netCDF4.Dataset(ncfile, 'w', format='NETCDF3_CLASSIC')
dim_time = dataset.createDimension('time', len(self.dt_list))
dim_height = dataset.createDimension('height', len(self.height_list))
dim_age = dataset.createDimension(
'time_age',
abs(self.config['time']['tr_duration']) + 1)
dim_cat = dataset.createDimension('categories', 7)
dim_regions = dataset.createDimension('regions',
len(list(self.geo_names.keys())))
dim_lats = dataset.createDimension('lat_thres',
len(list(self.lat_names.keys())))
# times_cn = dataset.createVariable('time', np.float32, ('time',))
# times_cn[:] = hours_cn.astype(np.float32)
# times_cn.units = "hours since " + self.begin_dt.strftime('%Y-%m-%d') + "00:00:00 +00:00"
# times_cn.long_name = "Decimal hours from midnight UTC"
# times_cn.axis = "T"
save_item(
dataset, {
'var_name': 'timestamp',
'dimension': ('time', ),
'vartype': 'i4',
'arr': timestamps.astype(np.int32),
'long_name': "Unix timestamp",
'units': "s",
'axis': 'T'
})
save_item(
dataset, {
'var_name':
'time',
'dimension': ('time', ),
'arr':
hours_cn.astype(np.float32),
'long_name':
"Decimal hours from midnight UTC",
'units':
"hours since {} 00:00:00 +00:00".format(
self.dt_list[0].strftime('%Y-%m-%d')),
'axis':
'T'
})
save_item(
dataset, {
'var_name': 'range',
'dimension': ('height', ),
'arr': np.array(self.height_list).astype(np.float32) / 1000.,
'long_name': "Height",
'units': "km",
'axis': 'Z'
})
save_item(
dataset, {
'var_name': 'age',
'dimension': ('time_age', ),
'arr': self.raw_dict['age'][0, 0],
'long_name': "Age of trajectory",
'units': "h"
})
save_item(
dataset, {
'var_name': 'no_part',
'dimension': ('time', ),
'arr': np.array(self.no_part),
'long_name': "number particles/trajectories",
'units': "no"
})
save_item(
dataset, {
'var_name': 'time_res',
'dimension': ('time', ),
'arr': np.array(self.time_res),
'long_name': "backward time resolution",
'units': "no"
})
for k in list(self.stat2d_dict.keys()):
print(k, self.stat2d_dict.get(k).shape)
dataset = save_item(
dataset, {
'var_name': k,
'dimension': ('time', 'height'),
'arr': self.stat2d_dict.get(k).copy().astype(np.float32),
'long_name': k
})
# its sufficient to save the age of the trajectory once
raw_data_keys = list(self.raw_dict.keys())
raw_data_keys.remove('age')
# chance to modify some parameter descriptions for better readability
modified_params = {
key: {
'var_name': key.lower(),
'long_name': "Hysplit " + key.lower()
}
for key in raw_data_keys
}
modified_params['height'] = {
'var_name': 'traj_height',
'long_name': "Hysplit height of air parcel"
}
if 'land_sfc_category' in list(modified_params.keys()):
modified_params['land_sfc_category'][
'long_name'] = "Modis land use category (simplified)"
for k in raw_data_keys:
print(k, self.raw_dict.get(k).shape)
dataset = save_item(
dataset, {
'var_name': modified_params[k]['var_name'],
'dimension': ('time', 'height', 'time_age'),
'arr': self.raw_dict.get(k).copy().astype(np.float32),
'long_name': modified_params[k]['long_name']
})
# save the land use
ls_data_keys = list(self.statls_dict.keys())
print('ls_data_keys ', ls_data_keys)
modified_params = {
key: {
'var_name': key,
'long_name': "land surface " + key.lower(),
'comment': str(self.ls_categories)
}
for key in ls_data_keys
}
for k in ls_data_keys:
print(k, self.statls_dict.get(k).shape)
dataset = save_item(
dataset, {
'var_name': modified_params[k]['var_name'],
'dimension': ('time', 'height', 'categories'),
'arr': self.statls_dict.get(k),
'long_name': modified_params[k]['long_name'],
'comment': modified_params[k]['comment']
})
for k in [ky for ky in ls_data_keys if 'ens' in ky]:
rel = self.statls_dict.get(k)
no_below = self.stat2d_dict.get(k + "_no_below")
no_below = np.repeat(no_below[:, :, np.newaxis],
rel.shape[-1],
axis=2)
no_below[no_below < 0] = np.nan
norm = np.array(
self.no_part) * 10 * (24. / np.array(self.time_res))
norm = np.repeat(norm[:, np.newaxis], rel.shape[1], axis=1)
norm = np.repeat(norm[:, :, np.newaxis], rel.shape[2], axis=2)
normed_time = rel * no_below / norm
normed_time[~np.isfinite(normed_time)] = -1
str_below = modified_params[k]['var_name'].replace("occ_ens_", "")
var_name = "rt_normed_landsfc_" + str_below
long_name = "normed residence time land surface " + str_below
dataset = save_item(
dataset, {
'var_name': var_name,
'dimension': ('time', 'height', 'categories'),
'arr': normed_time,
'long_name': long_name,
'comment': modified_params[k]['comment']
})
# and the geo names
gn_data_keys = list(self.statgn_dict.keys())
print('gn_data_keys ', gn_data_keys)
modified_params = {
key: {
'var_name': key,
'long_name': "geography names " + key.lower(),
'comment': str(self.geo_names)
}
for key in gn_data_keys
}
for k in gn_data_keys:
print(k, self.statgn_dict.get(k).shape)
dataset = save_item(
dataset, {
'var_name': modified_params[k]['var_name'],
'dimension': ('time', 'height', 'regions'),
'arr': self.statgn_dict.get(k),
'long_name': modified_params[k]['long_name'],
'comment': modified_params[k]['comment']
})
for k in gn_data_keys:
rel = self.statgn_dict.get(k)
no_below = self.stat2d_dict.get(k + "_no_below")
no_below = np.repeat(no_below[:, :, np.newaxis],
rel.shape[-1],
axis=2)
no_below[no_below < 0] = np.nan
norm = np.array(
self.no_part) * 10 * (24. / np.array(self.time_res))
norm = np.repeat(norm[:, np.newaxis], rel.shape[1], axis=1)
norm = np.repeat(norm[:, :, np.newaxis], rel.shape[2], axis=2)
normed_time = rel * no_below / norm
normed_time[~np.isfinite(normed_time)] = -1
str_below = modified_params[k]['var_name'].replace(
"region_ens_", "")
var_name = "rt_normed_region_" + str_below
long_name = "normed residence time named region " + str_below
dataset = save_item(
dataset, {
'var_name': var_name,
'dimension': ('time', 'height', 'regions'),
'arr': normed_time,
'long_name': long_name,
'comment': modified_params[k]['comment']
})
# TODO make statlat optional statlat_dict
lat_data_keys = list(self.statlat_dict.keys())
print('lat_data_keys ', lat_data_keys)
modified_params = {
key: {
'var_name': key,
'long_name': "lat_thres " + key.lower(),
'comment': str(self.lat_names)
}
for key in lat_data_keys
}
for k in lat_data_keys:
print("self.statlat_dict.keys()", self.statlat_dict.keys())
print(k, self.statlat_dict.get(k).shape)
dataset = save_item(
dataset, {
'var_name': modified_params[k]['var_name'],
'dimension': ('time', 'height', 'lat_thres'),
'arr': self.statlat_dict.get(k),
'long_name': modified_params[k]['long_name'],
'comment': modified_params[k]['comment']
})
for k in lat_data_keys:
rel = self.statlat_dict.get(k)
no_below = self.stat2d_dict.get(k + "_no_below")
no_below = np.repeat(no_below[:, :, np.newaxis],
rel.shape[-1],
axis=2)
no_below[no_below < 0] = np.nan
norm = np.array(
self.no_part) * 10 * (24. / np.array(self.time_res))
norm = np.repeat(norm[:, np.newaxis], rel.shape[1], axis=1)
norm = np.repeat(norm[:, :, np.newaxis], rel.shape[2], axis=2)
normed_time = rel * no_below / norm
normed_time[~np.isfinite(normed_time)] = -1
str_below = modified_params[k]['var_name'].replace("lat_ens_", "")
var_name = "rt_normed_lat_" + str_below
long_name = "normed residence time latitude " + str_below
dataset = save_item(
dataset, {
'var_name': var_name,
'dimension': ('time', 'height', 'lat_thres'),
'arr': normed_time,
'long_name': long_name,
'comment': modified_params[k]['comment']
})
# save_item(dataset, {'var_name': 'width', 'dimension': ('time', 'height'),
# 'arr': .corr_width_reg[:].filled(), 'long_name': "Spectral width",
# 'comment': "Wind profiler spectral width (standard deviation) corrected by cloud radar (only Bragg contribution)",
# 'units': "m s-1", 'units_html': "m s<sup>-1</sup>",
# 'missing_value': -99., 'plot_range': (0.01, 4.),
# 'plot_scale': "logarithmic"})
with open('output_meta.toml') as output_meta:
meta_info = toml.loads(output_meta.read())
dataset.description = meta_info['description'][model_str]
dataset.location = self.config['station']['name']
if "moving" in self.config['station'].keys(
) and self.config['station']['moving'] == True:
dataset.coordinates = "Moving Platform!"
else:
dataset.coordinates = (self.config['station']['lat'],
self.config['station']['lon'])
dataset.institution = meta_info["institution"]
dataset.authors = meta_info["authors"]
dataset.contact = meta_info["contact"]
dataset.creation_time = datetime.datetime.utcnow().strftime(
"%Y-%m-%d %H:%M UTC")
dataset.day = self.dt_list[0].day
dataset.month = self.dt_list[0].month
dataset.year = self.dt_list[0].year
dataset.git_commit, dataset.git_branch = get_git_hash()
dataset.close()
gc.collect()
def create_job(analysis,
user=None,
json_text='',
json_data={},
name=None,
state=Job.QUEUED,
uid=None,
save=True,
fill_with={}):
"""
Note: Parameter 'fill_with' needs to be a flat key:value dictionary.
"""
state = state or Job.QUEUED
owner = user or analysis.project.owner
project = analysis.project
if json_data:
json_text = hjson.dumps(json_data)
else:
json_text = json_text or analysis.json_text
# Needs the json_data to set the summary.
json_data = hjson.loads(json_text)
# Generate a meaningful job title.
name = make_job_title(recipe=analysis, data=json_data)
uid = uid or util.get_uuid(8)
# Create the job instance.
job = Job(name=name,
state=state,
json_text=json_text,
security=Job.AUTHORIZED,
project=project,
analysis=analysis,
owner=owner,
template=analysis.template,
uid=uid)
# Fill the json data.
json_data = fill_json_data(job=job,
source_data=json_data,
project=project,
fill_with=fill_with)
# Generate a meaningful job title.
name = make_job_title(recipe=analysis, data=json_data)
# Update the json_text and name
job.json_text = hjson.dumps(json_data)
job.name = name
if save:
job.save()
# Update the projects lastedit user when a job is created
#job_count = project.job_set.filter(deleted=False).count()
job_count = Job.objects.filter(deleted=False, project=project).count()
Project.objects.filter(uid=project.uid).update(lastedit_user=owner,
lastedit_date=now(),
jobs_count=job_count)
logger.info(f"Created job id={job.id} name={job.name}")
return job
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public
# License along with fedimg; if not, see http://www.gnu.org/licenses,
# or write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#
# Authors: David Gay <*****@*****.**>
# Sayan Chowdhury <*****@*****.**>
import toml
# Read in config file
with open("/etc/fedimg/fedimg-conf.toml") as conffile:
config = toml.loads(conffile.read())
# Fedimg Consumer configurations
PROCESS_COUNT = config['general']['process_count']
STATUS_FILTER = ('FINISHED_INCOMPLETE', 'FINISHED')
ACTIVE_SERVICES = config['general']['active_services']
CLEAN_UP_ON_FAILURE = config['general']['clean_up_on_failure']
DELETE_IMAGES_ON_FAILURE = config['general']['delete_images_on_failure']
AWS_UTIL_USER = config.get('aws', {}).get('util_username', {})
AWS_TEST_USER = config.get('aws', {}).get('test_username', {})
AWS_ACCESS_ID = config.get('aws', {}).get('access_id')
AWS_SECRET_KEY = config.get('aws', {}).get('secret_key')
AWS_VOLUME_SIZE = config.get('aws', {}).get('volume_size')
AWS_VOLUME_TYPES = config.get('aws', {}).get('volume_types')
from pathlib import Path
from time import time
from toml import TomlPreserveInlineDictEncoder as tpide
from toml import dumps, loads
PYPROJECT = "pyproject.toml"
p = Path(PYPROJECT)
pyproject = loads(p.read_text())
v = pyproject["tool"]["poetry"]["version"]
parts = v.split(".")[:2]
unix = str(int(time()))
parts.append(unix)
v_with_timestamp = ".".join(parts)
pyproject["tool"]["poetry"]["version"] = v_with_timestamp
p.write_text(dumps(pyproject, tpide()))
def loads(text, tool='baldrick'):
return Config(toml.loads(text)['tool'][tool])
def loads(obj: str):
return toml.loads(obj)
def load():
global engine
engine = inflect.engine()
plurals = toml.loads(pkg_resources.read_text(sizebot.data, "plurals.ini"))
for s, p in plurals["plurals"].items():
engine.defnoun(s, p)
def main():
args = sys.argv
#options supported by Armature
long_options = ["drawstuff", "sdf"]
#Get all the options, if there is an invalid option, quit
try:
opts, args = getopt.gnu_getopt(sys.argv, "", long_options)
except getopt.GetoptError as e:
print(e)
quit()
file_to_parse = None
where_to_save = None
#If no input file was given
if len(args) == 1:
print("\
Please give one argument as the ARMS file to parse, and a second, optional, argument as the save location.\n\
\ni.e. >python Armature.py my/super/cool/file.arms where/i/want/to/save")
quit()
else:
file_to_parse = Path(args[1])
#If no options are given, ask for them
if len(opts) < 1:
print("\
Please give options to select the type of file you'd like Armature to produce. You may do multiple options.\n\
\ni.e. >python Armature.py --drawstuff --sdf my/super/cool/file.arms where/i/want/to/save\n\
\nAvailible Options:")
for option in long_options:
print(f'--{option}')
quit()
#Translate the ARMS syntax into a dictionary
global arms
arms = toml.loads(file_to_parse.read_text())
convert_macros()
convert_constants()
make_shape_positions_relative_to_parents()
make_joint_positions_relative_to_parent_shape()
#Get all the creators
creators = ArmsCreatorProvider.AllArmsCreators()
#For every option
for opt in opts:
opt = opt[0]
for creator in creators:
#Write to a file if creator was requested in the command line options
if creator.can_create(opt):
#Get the save location
if len(args) > 2:
where_to_save = Path(
f"{args[2]}/{file_to_parse.stem}{creator.file_extension}"
)
else:
where_to_save = Path(
f"./{file_to_parse.stem}{creator.file_extension}")
where_to_save.write_text(creator.create_file(arms))
def deserialize(cls, s, **opts):
return toml.loads(s, **opts)
def main(argv=()) -> None:
"""Program entry point for all of the sail/sailboat terminal program."""
# Move to project root, check for sailboat.toml, and then read it.
if argv is None:
argv = []
if not argv:
argv = sys.argv
ordinal_path = os.getcwd()
while not os.path.isfile('sailboat.toml') and os.getcwd().count(
os.sep) > 2:
os.chdir('..')
if not os.path.isfile('sailboat.toml'):
needs_setup = True
data = {}
os.chdir(ordinal_path)
else:
needs_setup = False
with open('sailboat.toml', 'r', encoding="utf8") as file:
data = toml.loads(file.read())
if data == {}:
needs_setup = True
# Add required sections to data.
for required_section in ("command", "build", "release", "git"):
if required_section not in data:
data[required_section] = {}
# Search for options in argv
switches = []
for index, value in enumerate(argv[1:]):
if value.startswith('--'):
switches.append(value[2:])
elif value.startswith('-'):
switches.extend(value[1:])
else:
start_index = index
options = [argv[0], *argv[1:][start_index:]]
break
else:
options = [argv[0]]
plugins = json.loads(open(prefix + 'plugins.json').read())
if 'version' in switches or 'v' in switches:
print(
'Sailboat version {}\n\nHelp:\ncontact [email protected]\nor make an issue at cole-wilson/sailboat'
.format(sailboat.__version__))
sys.exit(0)
if 'refresh' in switches or 'r' in switches or plugins == {} or (len(
plugins.keys()) != 4):
print("Refreshing plugins list (this could take a couple seconds...)")
plugins = sailboat.refresh_plugins()
print('Done!\n')
helptext = "usage: " + sys.argv[0].split(
os.sep
)[-1] + " [options ...] [subcommand] [subcommand options ...]\n\n\tcore commands:"
for command in plugins['core'].keys():
if plugins['core'][command]['show']:
helptext += "\n\t\t- " + term.cyan + command + term.normal + ": " + plugins[
'core'][command]['description'].capitalize()
helptext += "\n\t\t- " + term.cyan + "help" + term.normal + ": Display this message."
if len(plugins['command'].keys()) != 0:
helptext += "\n\n\tother commands:"
for command in plugins['command'].keys():
if plugins['command'][command]['show'] and command in data['command']:
helptext += "\n\t\t- " + term.cyan + command + term.normal + ": " + plugins[
'command'][command]['description'].capitalize()
helptext += "\n"
if len(options) < 2:
print(helptext)
return
if needs_setup:
command = 'quickstart'
else:
command = options[1]
if command == 'help': # or 'help' in options or 'h' in options:
print(helptext)
return
try:
plugin_type, temp = sailboat.get_plugin(command)
except sailboat.PluginNotFound:
print(
'sailboat: error: {} is not a valid command. Please make sure you have installed it.'
.format(command))
return
if plugin_type != 'core' and command not in data['build'] and command not in data['release'] and command not in \
data['command']:
print(
'sailboat: error: {} *is* a valid command, but it isn\'t installed on this project. Install it with the '
'`add` command.'.format(command))
return
temp = temp(data=data, options=options[2:], name=command, prefix=prefix)
##############################
temp.run() # Run the plugin!
##############################
if plugin_type == 'core':
data = temp.data
else:
data[plugin_type][command] = temp.data[plugin_type][command]
basic_data = {}
resources = {'resources': {}}
commands = {'command': {}}
builds = {'build': {}}
release = {'release': {}}
other = {}
with open('sailboat.toml', 'w+') as f:
for key in data.keys():
if not isinstance(data[key], dict):
basic_data[key] = data[key]
elif key == 'resources' and len(data[key].values()) > 0:
resources['resources'] = data[key]
elif key == 'command' and len(data[key].values()) > 0:
commands['command'] = data[key]
elif key == 'build' and len(data[key].values()) > 0:
builds['build'] = data[key]
elif key == 'release' and len(data[key].values()) > 0:
release['release'] = data[key]
elif key not in ('build', 'release', 'command', 'resources'):
other[key] = data[key]
resources = resources if resources != {'resources': {}} else {}
commands = commands if commands != {'command': {}} else {}
builds = builds if builds != {'build': {}} else {}
release = release if release != {'release': {}} else {}
other = other if other != {'other': {}} else {}
o = [
*map(toml.dumps,
[basic_data, resources, commands, builds, release, other])
]
out = r"""# _ _ _ _
# ___ __ _ (_)| || |__ ___ __ _ | |_
# (_-</ _` || || || '_ \/ _ \/ _` || _|
# /__/\__,_||_||_||_.__/\___/\__,_| \__|
# Basic Setup:
{}
# Resource Setup:
{}
# Plugin Commands:
{}
# Build Routines:
{}
# Release Routines:
{}
# Other:
{}
# Thank you for using Sailboat!"""
if '_comments' in data and not data['_comments']:
print()
out = "{}\n{}\n{}\n{}\n{}\n{}\n"
out = out.format(o[0], o[1], o[2], o[3], o[4], o[5])
f.write(out)
### Initialization
# load libraries
import arcpy
import toml
import os
import sys
### Preliminary processing
# load parameters
with open("code/parameters/general.toml") as conffile:
general_params = toml.loads(conffile.read())
with open("code/parameters/remove-slivers.toml") as conffile:
slivers_params = toml.loads(conffile.read())
# set environmental variables
arcpy.env.parallelProcessingFactor = general_params['threads']
arcpy.env.overwriteOutput = True
arcpy.env.workspace = os.path.dirname(sys.argv[1])
# storing system cmds
input_PTH = sys.argv[1]
output_PTH = sys.argv[2]
### Main processing
# calculate area
arcpy.AddField_management(input_PTH, 'AREA', 'DOUBLE')
arcpy.CalculateField_management(input_PTH, 'AREA',
'!shape.area@SQUAREKILOMETERS!', 'PYTHON_9.3')
# select slivers
try:
from importlib_resources import read_text
except ModuleNotFoundError:
from importlib.resources import read_text
try:
import progressbar
except ImportError:
PROGRESSBAR_EXISTS = False
else:
PROGRESSBAR_EXISTS = True
logger = logging.getLogger(__name__)
indices_urls = toml.loads(
read_text("planetpy.pdstools.data", "indices_paths.toml"))
def list_available_index_files():
print(yaml.dump(indices_urls, default_flow_style=False))
print(
"Use indices.download('mission:instrument:index') to download in index file."
)
print("For example: indices.download('cassini:uvis:moon_summary'")
def replace_url_suffix(url, new_suffix=".tab"):
"""Cleanest way to replace the suffix in an URL.
Sometimes the indices have upper case filenames, this is taken care of here.
def test_array_sep():
encoder = toml.TomlArraySeparatorEncoder(separator=",\t")
d = {"a": [1, 2, 3]}
o = toml.loads(toml.dumps(d, encoder=encoder))
assert o == toml.loads(toml.dumps(o, encoder=encoder))
GITHUB = "https://github.com"
LAB_ORG = "jupyterlab"
REPO_JUPYTERLAB = f"{GITHUB}/{LAB_ORG}/jupyterlab"
REPO_LUMINO = f"{GITHUB}/{LAB_ORG}/lumino"
# don't pollute the global state
LINKS = (HERE / "repos/.yarn-links").resolve()
YARN = ["yarn", "--link-folder", LINKS]
PIP = ["python", "-m", "pip"]
LAB_APP_DIR = pathlib.Path(sys.prefix) / "share/jupyter/lab"
LAB_APP_STATIC = LAB_APP_DIR / "static"
LAB_APP_INDEX = LAB_APP_STATIC / "index.html"
REPOS_TOML = HERE / "repos.toml"
REPOS = toml.loads(REPOS_TOML.read_text())["repos"]
PATHS = {url: HERE / "repos" / pathlib.Path(url).name for url in REPOS}
HOST = "127.0.0.1"
PORT = 8080
LAB_PORT = 9999
MISSING_LUMINO_DOCS = [
"default-theme",
# TODO: https://github.com/jupyterlab/lumino/issues/154
"polling",
]
def task_lint():
"""lint the source in _this_ repo"""
all_py = [*HERE.glob("*.py"), *PA11Y.glob("*.py")]
def load(self):
with open(self.fileName) as f:
self.loaded_results = toml.loads(f.read())
self.loaded = True
def parse_with_toml(data):
"""Uses TOML syntax to parse data"""
try:
return toml.loads(f"key={data}")["key"]
except (toml.TomlDecodeError, KeyError):
return data
def __init__(self, context):
self.context = context
def get_env_bool(var, default):
# Contents of env vars are strings by default. This returns the
# boolean value of the specified environment variable, or the
# speciried default if the var doesn't contain True or False
return {
'True': True,
'False': False
}.get(os.environ.get(var), default)
def resolverelative(category, key):
# Allow ~
self.config[category][key] = path.expanduser(
self.config[category][key])
# Resolve relative paths
self.config[category][key] = path.join(context.topdir,
self.config[category][key])
if not hasattr(self.context, "bootstrapped"):
self.context.bootstrapped = False
config_path = path.join(context.topdir, ".servobuild")
if path.exists(config_path):
with open(config_path) as f:
self.config = toml.loads(f.read())
else:
self.config = {}
# Handle missing/default items
self.config.setdefault("tools", {})
default_cache_dir = os.environ.get("SERVO_CACHE_DIR",
path.join(context.topdir, ".servo"))
self.config["tools"].setdefault("cache-dir", default_cache_dir)
resolverelative("tools", "cache-dir")
default_cargo_home = os.environ.get(
"CARGO_HOME", path.join(context.topdir, ".cargo"))
self.config["tools"].setdefault("cargo-home-dir", default_cargo_home)
resolverelative("tools", "cargo-home-dir")
context.sharedir = self.config["tools"]["cache-dir"]
self.config["tools"].setdefault("system-rust", False)
self.config["tools"].setdefault("system-cargo", False)
self.config["tools"].setdefault("rust-root", "")
self.config["tools"].setdefault("cargo-root", "")
if not self.config["tools"]["system-cargo"]:
self.config["tools"]["cargo-root"] = path.join(
context.sharedir, "cargo", self.cargo_build_id())
self.config["tools"].setdefault(
"rustc-with-gold", get_env_bool("SERVO_RUSTC_WITH_GOLD", True))
# https://github.com/rust-lang/rust/pull/39754
platforms_with_rustc_alt_builds = [
"unknown-linux-gnu", "apple-darwin", "pc-windows-msvc"
]
llvm_assertions_default = ("SERVO_RUSTC_LLVM_ASSERTIONS" in os.environ
or host_platform()
not in platforms_with_rustc_alt_builds)
self.config.setdefault("build", {})
self.config["build"].setdefault("android", False)
self.config["build"].setdefault("mode", "")
self.config["build"].setdefault("llvm-assertions",
llvm_assertions_default)
self.config["build"].setdefault("debug-mozjs", False)
self.config["build"].setdefault("ccache", "")
self.config["build"].setdefault("rustflags", "")
self.config["build"].setdefault("incremental", False)
self.config.setdefault("android", {})
self.config["android"].setdefault("sdk", "")
self.config["android"].setdefault("ndk", "")
self.config["android"].setdefault("toolchain", "")
self.config["android"].setdefault("platform", "android-18")
self.config["android"].setdefault("target", "arm-linux-androideabi")
self.set_use_stable_rust(False)
def parse_toml(filename):
toml_string = toml_file(filename)
parsed_toml = toml.loads(toml_string)
return parsed_toml
def check_config_file(config_file, print_text=True):
# Check if config file exists
if not os.path.exists(config_file):
print("%s config file is required but was not found" % config_file)
sys.exit(1)
# Load configs from servo-tidy.toml
with open(config_file) as content:
conf_file = content.read()
lines = conf_file.splitlines(True)
if print_text:
print '\rChecking the config file...'
config_content = toml.loads(conf_file)
exclude = config_content.get("ignore", {})
# Check for invalid listed ignored directories
exclude_dirs = exclude.get("directories", [])
skip_dirs = ["./target", "./tests"]
invalid_dirs = [
d for d in exclude_dirs
if not os.path.isdir(d) and not any(s in d for s in skip_dirs)
]
# Check for invalid listed ignored files
invalid_files = [
f for f in exclude.get("files", []) if not os.path.exists(f)
]
current_table = ""
for idx, line in enumerate(lines):
# Ignore comment lines
if line.strip().startswith("#"):
continue
# Check for invalid tables
if re.match("\[(.*?)\]", line.strip()):
table_name = re.findall(r"\[(.*?)\]", line)[0].strip()
if table_name not in ("configs", "blocked-packages", "ignore",
"check_ext"):
yield config_file, idx + 1, "invalid config table [%s]" % table_name
current_table = table_name
continue
# Print invalid listed ignored directories
if current_table == "ignore" and invalid_dirs:
for d in invalid_dirs:
if line.strip().strip('\'",') == d:
yield config_file, idx + 1, "ignored directory '%s' doesn't exist" % d
invalid_dirs.remove(d)
break
# Print invalid listed ignored files
if current_table == "ignore" and invalid_files:
for f in invalid_files:
if line.strip().strip('\'",') == f:
yield config_file, idx + 1, "ignored file '%s' doesn't exist" % f
invalid_files.remove(f)
break
# Skip if there is no equal sign in line, assuming it's not a key
if "=" not in line:
continue
key = line.split("=")[0].strip()
# Check for invalid keys inside [configs] and [ignore] table
if (current_table == "configs" and key not in config
or current_table == "ignore" and key not in config["ignore"] or
# Any key outside of tables
current_table == ""):
yield config_file, idx + 1, "invalid config key '%s'" % key
# Parse config file
parse_config(config_content)
def check_lock(file_name, contents):
def find_reverse_dependencies(name, content):
for package in itertools.chain([content.get("root", {})],
content["package"]):
for dependency in package.get("dependencies", []):
if dependency.startswith("{} ".format(name)):
yield package["name"], dependency
if not file_name.endswith(".lock"):
raise StopIteration
# Package names to be neglected (as named by cargo)
exceptions = config["ignore"]["packages"]
content = toml.loads(contents)
packages_by_name = {}
for package in content.get("package", []):
if "replace" in package:
continue
source = package.get("source", "")
if source == r"registry+https://github.com/rust-lang/crates.io-index":
source = "crates.io"
packages_by_name.setdefault(package["name"], []).append(
(package["version"], source))
for (name, packages) in packages_by_name.iteritems():
has_duplicates = len(packages) > 1
duplicates_allowed = name in exceptions
if has_duplicates == duplicates_allowed:
continue
if duplicates_allowed:
message = 'duplicates for `{}` are allowed, but only single version found'.format(
name)
else:
message = "duplicate versions for package `{}`".format(name)
packages.sort()
packages_dependencies = list(find_reverse_dependencies(name, content))
for version, source in packages:
short_source = source.split("#")[0].replace("git+", "")
message += "\n\t\033[93mThe following packages depend on version {} from '{}':\033[0m" \
.format(version, short_source)
for name, dependency in packages_dependencies:
if version in dependency and short_source in dependency:
message += "\n\t\t" + name
yield (1, message)
# Check to see if we are transitively using any blocked packages
for package in content.get("package", []):
package_name = package.get("name")
package_version = package.get("version")
for dependency in package.get("dependencies", []):
dependency = dependency.split()
dependency_name = dependency[0]
whitelist = config['blocked-packages'].get(dependency_name)
if whitelist is not None:
if package_name not in whitelist:
fmt = "Package {} {} depends on blocked package {}."
message = fmt.format(package_name, package_version,
dependency_name)
yield (1, message)
import random
from random import randint
import os
import time
from pymongo import MongoClient
import commands
import json
import re
import pdb
import requests, json
import sys
import urllib
global reg, a, b
with open("/var/chandni-chowk/configs/app.development.toml") as conffile:
config = toml.loads(conffile.read())
mongo_client = MongoClient(config['app']['mongo_conn_str'])
db = mongo_client.dsp
with open("/var/chandni-chowk/configs/app.test.toml") as conffile1:
config1 = toml.loads(conffile1.read())
reg = config1["app"]["regions"]
#print reg
for i in range(0, 1000):
def mul_random_regs():
regions = [
'Delhi NCR', 'Hyderabad', 'Bangalore', 'TN/Pondicherry', 'Kerala',
'Pun/Har/Cha/HP/J%26K', 'Uttar Pradesh', 'West Bengal',
'North East', 'Orissa', 'Jharkhand', 'Bihar', 'Maharashtra/Goa',
def from_content(cls, content: str) -> Any:
"""Load toml from string."""
cls._check_toml()
return toml.loads(content)
