def load_json(f):
import json, jsoncomment
if type(f) == str:
f = open(f, 'r')
jc = jsoncomment.JsonComment(json)
r = jc.load(f)
# ro = jc.load(f, object_hook=lambda d: namedtuple('X', d.keys())(*d.values()))
f.close()
return r
def load(self):
parser = jsoncomment.JsonComment(json)
with open(self.config_path) as file:
parsed_object = parser.load(file)
try:
self._config = self._validator.validate(parsed_object, namespace=True, copy=False)
except dschema.ValidationError as e:
raise TGMinerConfigException("Config Error: " + str(e))
self.__dict__.update(self._config.__dict__)
def __init_config_from_file(self, path):
def add_dict(config, d):
for k, v in d.items():
kv = config.get(k, None)
if isinstance(kv, dict) and isinstance(v, dict):
add_dict(config=kv, d=v)
else:
config[k] = v
with open(path) as f:
add_dict(config=self.__config,
d=jsoncomment.JsonComment(json).loads(f.read()))
def from_json(filepath):
# type: (Union[str, Path]) -> Database
'''
Constructs a Database instance from a given JSON filepath.
Args:
filepath(Path or str): Path to JSON config file.
Returns:
Database: Database instance.
'''
with open(filepath) as f:
config = jsonc.JsonComment().load(f)
return Database(config)
def pytest_generate_tests(metafunc):
if "test_shiftr" == metafunc.definition.name:
jsc = jsoncomment.JsonComment()
os_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'shiftr_tests')
lst = []
ids = []
for json_file in os.listdir(os_path):
try:
js = jsc.loads(read_file(os.path.join(os_path, json_file)))
lst.append(
JoltTestObject(json_file, js['input'], js['spec'],
js['expected']))
ids.append(json_file)
except JSONDecodeError:
pass
metafunc.parametrize("shiftr_test", lst, ids=ids)
def add_lyrics_lumi(part_results, lyrics_file, year):
parser = jsoncomment.JsonComment(json)
s = open(lyrics_file).read()
lyrics = parser.loads(s, encoding="utf-8")
i = 0
for l in lyrics:
song = clean_song_name(l["song"])
found = False
for pr in part_results:
if clean_song_name(pr['song_name']) == song:
pr['lyrics'] = l['lyrics'].encode("utf-8")
i += 1
found = True
break
if not found:
print "ERROR: year ", year, " song ", song, " lyrics not matched"
print "INFO: for year {} {}/{} songs have lyrics available".format(
year, i, len(part_results))
return part_results
def do_parse(file_path):
""" @todo """
try:
with io.open(file_path, 'rb') as fp:
s = fp.read()
except Exception as e:
return (str(e), None)
try:
h = chardet.detect(s)
try:
with io.open(file_path, 'r', encoding=h['encoding']) as fp:
s = fp.read()
except Exception as e:
return (str(e), None)
except Exception as e:
try:
with io.open(file_path, 'r') as fp:
s = fp.read()
except Exception as e:
return (str(e), None)
parser = jsoncomment.JsonComment(json)
try:
j = parser.loads(s)
return ('', j)
except Exception as e:
# Strip out single line comments
lines = s.splitlines()
s = ''
for line in lines:
line = re.sub(r'^\s*//.*$', '', line)
s += line + "\n"
try:
j = parser.loads(s)
except Exception as e2:
j = None
rv = '%s (%s)' % (str(e), h['encoding'])
return (rv, j)
def test_on_event_save_button_error(dash_duo):
with TemporaryDirectory() as root:
test_config_path = Path(RESOURCES_DIR, 'test_config.json')
with open(test_config_path) as f:
config = jsonc.JsonComment().load(f)
config['foo'] = 'bar'
config_path = Path(root, 'config.json')
with open(config_path, 'w') as f:
json.dump(config, f)
app.run(app.APP, config_path, debug=True, test=True)
test_app = app.APP
dash_duo.start_server(test_app)
config = test_app.api.config
# click on config tab
dash_duo.find_elements('#tabs .tab')[2].click()
time.sleep(0.1)
# click init
dash_duo.find_elements('#init-button')[-1].click()
time.sleep(0.12)
# delete config
os.remove(config_path)
# save bad config
dash_duo.find_elements('#save-button')[-1].click()
time.sleep(0.1)
dash_duo.wait_for_element('#error')
assert not config_path.is_file()
result = dash_duo.wait_for_element(
'#error tr td:last-child > div').text
assert result == 'DataError'
def parse_json_file_content(raw_content):
# type: (str) -> Dict
'''
Parses JSON file content as supplied by HTML request.
Args:
raw_content (str): Raw JSON file content.
Raises:
ValueError: If header is invalid.
JSONDecodeError: If JSON is invalid.
Returns:
dict: JSON content or reponse dict with error.
'''
header, content = raw_content.split(',')
temp = header.split('/')[-1].split(';')[0] # type: str
if temp != 'json':
msg = f'File header is not JSON. Header: {header}.'
raise ValueError(msg)
output = base64.b64decode(content).decode('utf-8')
return jsonc.JsonComment().loads(output)
def test_on_event_save_button(dash_duo):
with TemporaryDirectory() as root:
test_config_path = Path(RESOURCES_DIR, 'test_config.json')
with open(test_config_path) as f:
config = jsonc.JsonComment().load(f)
config['columns'] = ['date']
config_path = Path(root, 'config.json')
with open(config_path, 'w') as f:
json.dump(config, f)
app.run(app.APP, config_path, debug=True, test=True)
test_app = app.APP
dash_duo.start_server(test_app)
config = test_app.api.config
# click on config tab
dash_duo.find_elements('#tabs .tab')[2].click()
time.sleep(0.1)
# click init
dash_duo.find_elements('#init-button')[-1].click()
time.sleep(0.1)
# delete config
os.remove(config_path)
# save bad config
dash_duo.find_elements('#save-button')[-1].click()
time.sleep(0.1)
assert config_path.is_file()
with open(config_path) as f:
result = json.load(f)['columns']
assert result == ['date']
import logging
import json
import jsoncomment
import buv_types
from validate import incrementalValidateAndAdd
json=jsoncomment.JsonComment(json)
# in memory 'DB' of all data, mapping SHA256 strings to
# objects.
all_data={}
# initial membership proposal
genesis_members_hash=None
# maps handles to addresses, any seen
addr_for_handle={}
def reset():
global all_data
all_data={}
from buv_types import Vote, ProposalMetadata, MemberDict, ProposalText, Election
def for_type(t):
for v in all_data.itervalues():
if isinstance(v, t):
yield v
def readFile(fn):
logging.info("Reading file "+fn)
def run(self):
""" @todo docstring me """
if len(sys.argv) >= 3:
filespec = sys.argv[2]
else:
filespec = '*.json'
if len(sys.argv) >= 2:
dir_name = sys.argv[1]
else:
dir_name = '.'
self.dir = dir_name
self.logger = logging.getLogger()
self.logger.setLevel(logging.INFO)
logger2 = logging.getLogger('urllib3')
logger2.setLevel(logging.CRITICAL)
parser = jsoncomment.JsonComment(json)
if not os.path.isdir(TMP_DIR):
os.makedirs(TMP_DIR)
mask = dir_name + '/' + filespec
logging.info("==> Processing dir %s", mask)
for file in glob.glob(mask):
self.file = os.path.basename(file)
self.basename = os.path.splitext(self.file)[0]
logging.info("--> Processing file %s", file)
with io.open(file, 'r', encoding='utf-8') as f:
self.data = f.read()
orig_data = self.data
j = parser.loads(self.data)
_hash = ''
if self.check_hash and 'hash' in j:
_hash = j['hash']
scheme = self.process(j, 'homepage')
scheme = self.process(j, 'license')
scheme = self.process(j, 'url', _hash)
if self.schemes_changed(scheme):
logging.info("run: url: scheme=%s", scheme)
self.fix_schemes(j, 'autoupdate', scheme)
scheme = self.process(j, 'checkver')
if 'checkver' in j:
if isinstance(j['checkver'], dict):
scheme = self.process(j['checkver'], 'github')
if 'architecture' in j:
scheme = self.process(j['architecture'], '32bit', '',
'architecture')
if self.schemes_changed(scheme):
logging.info("run: architecture.32bit: scheme=%s", scheme)
if 'autoupdate' in j:
if 'architecture' in j['autoupdate']:
self.fix_schemes(j['autoupdate']['architecture'],
'32bit', scheme,
'autoupdate.architecture')
scheme = self.process(j['architecture'], '64bit', '',
'architecture')
if self.schemes_changed(scheme):
logging.info("run: architecture.64bit: scheme=%s", scheme)
if 'autoupdate' in j:
if 'architecture' in j['autoupdate']:
self.fix_schemes(j['autoupdate']['architecture'],
'64bit', scheme,
'autoupdate.architecture')
if self.data != orig_data:
logging.info("Updating %s", file)
if os.path.isfile(file + '.bak'):
os.remove(file + '.bak')
os.rename(file, file + '.bak')
with io.open(file, 'w', encoding='utf-8') as f:
f.write(self.data)
def do_parse(file_path):
"""@todo"""
try:
with io.open(file_path, "rb") as fp: # read binary
json_data = fp.read()
except Exception as e:
return (str(e), None)
h = {}
h["encoding"] = "unknown"
try:
h = chardet.detect(json_data)
try:
with io.open(file_path, "r", encoding=h["encoding"]) as fp:
json_data = fp.read()
except Exception as e:
return (str(e), None)
except Exception:
try:
with io.open(file_path, "r") as fp: # read non-binary
json_data = fp.read()
except Exception as e:
return (str(e), None)
parser = jsoncomment.JsonComment(json)
try:
j = parser.loads(json_data)
rv = ""
try:
jsonschema.validate(j, scoop_schema_data)
return ("", j)
except Exception as e:
err = str(e)
err = parse_validation_error(err)
# print(
# "\nError: Invalid json: %s:\n%s\n%s\n%s"
# % (os.path.basename(file_path), "=" * 80, err, "=" * 80)
# )
m = re.search(r"(Failed validating.*)", err)
if m is not None:
err = m.group(1)
else:
err = "Failed schema validation against %s" % scoop_schema_name
if re.search(r"additionalProperties", err):
return (rv, j)
rv = err
return (rv, j)
except Exception as e:
# Strip out single line comments
lines = json_data.splitlines()
s = ""
for line in lines:
line = re.sub(r"^\s*//.*$", "", line)
s += line + "\n"
try:
j = parser.loads(s)
except Exception:
j = None
rv = "%s (%s)" % (str(e), h["encoding"])
return (rv, j)
def main(args=None):
if not args:
args = sys.argv[1:]
logging.basicConfig(level=logging.DEBUG)
_logger.info(
"initialization script of container argument {0}".format(args))
parser = argparse.ArgumentParser(
description="init container 0 - javaagent")
parser.add_argument("--jacoco",
action="store_true",
default=False,
dest="jacoco",
help="activate usage of jacoco agent")
parser.add_argument("--jacocoagent",
action="store",
default=None,
dest="jacocoagent",
help="set jacoco agent property")
args, boot_args = parser.parse_known_args(args)
javaagent = None
javaagent_format = "-javaagent:{pathjar}=output={output},address={address},port={port},includes={includes}"
if args != None:
if args.jacoco:
javaagent = javaagent_format.format(
pathjar="/opt/cohorte/extra/jacoco/jacocoagent.jar",
output="tcpserver",
address="*",
port="6300",
includes="com.cohorte.*:org.cohorte.*")
_logger.info(
"-jacoco true is setted, construct automatic javaagent={}".
format(javaagent))
elif args.jacocoagent != None:
# use
javaagent = "-javaagent:{pathjar}" + args.jacocoagent.format(
pathjar="/opt/cohorte/extra/jacoco/jacocoagent.jar")
_logger.info(
"-jacocoagent is setted , construct javaagent={}".format(
javaagent))
if javaagent is not None:
w_parser = jsoncomment.JsonComment(json)
_logger.info(
"read composition file in order to retreieve the isolate name and add vm_args if necessary"
)
for isolate_name in get_list_isolate():
w_isolate_json = get_json_from_file(w_parser, isolate_name)
_logger.info("update isolate configuration files {}".format(
isolate_name))
if "vm_args" in w_isolate_json:
w_isolate_json["vm_args"].append(javaagent)
else:
w_isolate_json["vm_args"] = [javaagent]
# # add comment
w_str = '/*'\
' Modify by init_container_0.py, add jacoco agent in vm_args '\
'*/'
w_str = w_str + json.dumps(w_isolate_json, indent=4)
write_file(w_str, isolate_name, True)
else:
print("do nothin, not relevant argument")
# find in conf all isolat
for isolate_name in get_list_isolate():
real_name = isolate_name.replace("isolate_", "")[:-3]
# manager initialization of current.properties , current.properties.mdl
filename_path_mdl = "{}/install.properties.mdl".format(real_name)
filename_path = "{}/install.properties".format(real_name)
try:
with open(filename_path_mdl) as a_properties_file:
lines = a_properties_file.readlines()
new_content = "\n".join(replace_vars(os.environ, lines))
new_content = "# generated by init_container_0_iotPack" + new_content
write_file(new_content, filename_path, False)
except:
_logger.info("no install.properties file ")
pass
print("do nothin, no argument received")
