def load_conversion_and_warning_rules():
json_parser = JsonComment(json)
json_files_found = 0
try:
for name in os.listdir("ConversionRules"):
if name.endswith(".json") and name != WARNINGS_FILENAME:
json_files_found += 1
with open(os.path.join("ConversionRules", name)) as f:
convert.conversion_rules.update(json_parser.load(f))
with open(WARNINGS_PATH) as f:
warning_rules.update(json_parser.load(f))
except:
check_github_button_clicked_and_exit(
cfg.sg.Popup(
"The 'ConversionRules' folder wasn't found next to this executable. You can get the missing folder from the Legacy Mod Converter GitHub repo.",
title="Missing ConversionRules folder",
custom_text="Go to GitHub"))
if json_files_found == 0:
check_github_button_clicked_and_exit(
cfg.sg.Popup(
"The 'ConversionRules' folder didn't contain any JSON files. You can get the JSON files from the Legacy Mod Converter GitHub repo.",
title="Missing JSON files",
custom_text="Go to GitHub"))
def disease():
dis = []
with open('disease.json', encoding='utf-8') as data_file:
parser = JsonComment(json)
data = parser.load(data_file)
for a in range(0, len(data['child'])):
if len(data['child'][a]['child']) != 0:
for b in range(0, len(data['child'][a]['child'])):
if len(data['child'][a]['child'][b]['child']) != 0:
for c in range(0,
len(data['child'][a]['child'][b]['child'])):
if len(data['child'][a]['child'][b]['child'][c]
['child']) != 0:
for d in range(
0,
len(data['child'][a]['child'][b]['child']
[c]['child'])):
if len(data['child'][a]['child'][b]['child'][c]
['child'][d]['child']) != 0:
pass
else:
dis.append(
data['child'][a]['child'][b]['child']
[c]['child'][d]['name'])
else:
dis.append(data['child'][a]['child'][b]['child'][c]
['name'])
else:
dis.append(data['child'][a]['child'][b]['name'])
else:
dis.append(data['child'][a]['name'])
return dis
def load_params_from_json_file(path_to_file):
parser = JsonComment(json)
with open(path_to_file, "r") as outfile:
params = parser.load(outfile)
return params
def _load_json(self, file_name, encoding='utf-8', errors='ignore'):
try:
with open(file_name, encoding=encoding,
errors=errors) as data_file:
parser = JsonComment(json)
data = parser.load(data_file)
return data
except Exception as e:
print_yellow(f"cannot load json from {file_name}, {e}")
def load_conf(path):
"""
加载配置文件类
"""
ret = None
parser = JsonComment(json)
with open(path) as infile:
ret = parser.load(infile, object_hook=DottedDict)
return ret
def load(cls, filename, optional=False, debug=False):
'''Loads an rpggen file into the program.
optional=True if the file is optional: no error message if not found.
'''
startnum = re.compile(r"^[0123456789]+")
diceRE = re.compile(r"^[0123456789]*d[0123456789]+")
if not os.path.isfile(filename) and optional:
return
with open(filename, 'r') as file:
jsonComment = JsonComment(json)
cls.raw = jsonComment.load(file)
for d in cls.raw :
if "text" in d :
#print("loaded template: "+d['id'])
d['_type'] = 'template'
if re.search(r'.tmpl$',d['text']) :
with open(d['text'],'r') as template_file :
d['text'] = template_file.read()
cls.templates[d['id']] = d
if len(d) == 1 :
for k in d : # there is only one
id = k
d['id'] = id
if isinstance(d[k], str) :
if diceRE.search(d[k]) :
d['_type'] = 'dice'
d['roll'] = d[id]
cls.dice[d['id']] = d
else :
d['_type'] = 'table'
d['rows'] = [ ]
ii = 1
for item in d[k] :
row = Row()
row.result = item
row.start = row.stop = ii
ii += 1
d['rows'].append(row)
d['roll'] = "1d"+str(ii)
cls.tables[d['id']] = d
#print("singleton"+str(d))
else :
tableId = d['id']
if debug == True:
print("loading table: "+tableId)
rows = {}
maxnum = -sys.maxsize
minnum = sys.maxsize
for k in d :
if startnum.search(k):
rows[k] = d[k]
tab = Table(tableId, rows)
if not 'unique' in d:
tab.setUnique(False)
else:
tab.setUnique(d['unique'])
def load_split_config(config):
"""
Loads a file containing a split configuration and structures it as a dictionary.
"""
parser = JsonComment(json)
with open(config) as config_file:
cfg = parser.load(config_file)
split_config = dict()
for entry in cfg:
key = '%s:%s' % (entry['match-on-field'], entry['match-on-text'])
split_config[key] = entry['splits']
return split_config
def get_params(param_pars=None, **kw):
from jsoncomment import JsonComment
json = JsonComment()
pp = param_pars
choice = pp['choice']
config_mode = pp['config_mode']
data_path = pp['data_path']
if choice == "json":
data_path = path_norm(data_path)
cf = json.load(open(data_path, 'r'))
cf = cf[config_mode]
return cf['model_pars'], cf['data_pars'], cf['compute_pars'], cf[
'out_pars']
if choice == "test01":
log("#### Path params ##########################################")
data_path = path_norm("dataset/text/imdb.csv")
out_path = path_norm("ztest/model_tch/textcnn/")
model_path = os.path.join(out_path, "model")
data_pars = {
"data_path": path_norm("dataset/recommender/IMDB_sample.txt"),
"train_path": path_norm("dataset/recommender/IMDB_train.csv"),
"valid_path": path_norm("dataset/recommender/IMDB_valid.csv"),
"split_if_exists": True,
"frac": 0.99,
"lang": "en",
"pretrained_emb": "glove.6B.300d",
"batch_size": 64,
"val_batch_size": 64,
}
model_pars = {
"dim_channel": 100,
"kernel_height": [3, 4, 5],
"dropout_rate": 0.5,
"num_class": 2
}
compute_pars = {
"learning_rate": 0.001,
"epochs": 1,
"checkpointdir": out_path + "/checkpoint/"
}
out_pars = {
"path": model_path,
"checkpointdir": out_path + "/checkpoint/"
}
return model_pars, data_pars, compute_pars, out_pars
def _load_topology():
topo = { 'redis': 'localhost',
'database': 'localhost' }
try:
parser = JsonComment(json)
with open('/opt/stack/etc/topo.json') as fin:
topo = parser.load(fin)
except ValueError:
pass
except FileNotFoundError:
pass
return topo
def parse_manifest(manifest_path: Path):
with open(str(manifest_path), 'r') as stream:
if manifest_path.suffix == '.json':
json = JsonComment()
try:
manifest = json.load(stream)
except Exception as exc:
raise InvalidManifest(exc)
else:
try:
manifest = yaml.safe_load(stream)
except yaml.YAMLError as exc:
raise InvalidManifest(exc)
return manifest
def parse(self, filecfg):
result = {}
try:
configuration = open(filecfg, 'r')
parser = JsonComment(json)
obj = parser.load(configuration, "utf-8")
result = self.openObj(obj)
configuration.close()
except IOError as e:
print e
exit
except Exception as e:
print "Error al parsear el fichero:", filecfg
print e
return result
def get_params(param_pars={}, **kw):
from jsoncomment import JsonComment
json = JsonComment()
choice = param_pars['choice']
config_mode = param_pars['config_mode']
data_path = param_pars['data_path']
if choice == "json":
data_path = path_norm(data_path)
cf = json.load(open(data_path, mode='r'))
cf = cf[config_mode]
return cf['model_pars'], cf['data_pars'], cf['compute_pars'], cf[
'out_pars']
if choice == "test01":
log("#### Path params ##########################################")
data_path = path_norm("dataset/text/imdb.csv")
out_path = path_norm("ztest/model_keras/textcnn/model.h5")
model_path = out_path
data_pars = {
"path": data_path,
"train": 1,
"maxlen": 40,
"max_features": 5,
}
model_pars = {
"maxlen": 40,
"max_features": 5,
"embedding_dims": 50,
}
compute_pars = {
"engine": "adam",
"loss": "binary_crossentropy",
"metrics": ["accuracy"],
"batch_size": 1000,
"epochs": 1
}
out_pars = {"path": out_path, "model_path": model_path}
return model_pars, data_pars, compute_pars, out_pars
else:
raise Exception(f"Not support choice {choice} yet")
def main(schema_file, cxx=None, py=None):
"""Generate serialization/deserialization code from schema"""
logging.info('Working on file: ' + schema_file)
# validate file exists
if not os.path.isfile(schema_file):
logging.info('File not found')
return -1
# loading schema from file
json_parser = JsonComment(json)
with open(schema_file) as fd:
schema = json_parser.load(fd)
# validating the schema
status, msg = validate_protocol_schema(schema)
if not status:
logging.info('Invalid Schema:' + msg)
return -2
logging.info('Schema is valid')
# Build the protocol
protocol = Protocol(schema)
# build CXX
if cxx is not None:
if not os.path.isdir(cxx):
logging.info('CXX Dest not found')
return -1
message.make_message_cxx(protocol, cxx)
for r in protocol.endpoints:
router.make_router_cxx(protocol, r, cxx)
# build py
if py is not None:
if not os.path.isdir(py):
logging.info('PY Dest not found')
return -1
message.make_message_py(protocol, py)
for r in protocol.endpoints:
router.make_router_py(protocol, r, py)
return 0
def get_params(param_pars={}, **kw):
from jsoncomment import JsonComment
json = JsonComment()
pp = param_pars
choice = pp["choice"]
config_mode = pp["config_mode"]
data_path = pp["data_path"]
if choice == "json":
data_path = path_norm(data_path)
cf = json.load(open(data_path, mode="r"))
cf = cf[config_mode]
return cf["model_pars"], cf["data_pars"], cf["compute_pars"], cf[
"out_pars"]
if choice == "test01":
log("#### Path params ##########################################")
data_path = path_norm("dataset/text/ner_dataset.csv")
out_path = path_norm("ztest/model_keras/crf_bilstm/")
model_path = os.path.join(out_path, "model")
data_pars = {
"path": data_path,
"train": 1,
"maxlen": 400,
"max_features": 10,
}
model_pars = {}
compute_pars = {
"engine": "adam",
"loss": "binary_crossentropy",
"metrics": ["accuracy"],
"batch_size": 32,
"epochs": 1,
}
out_pars = {"path": out_path, "model_path": model_path}
log(data_pars, out_pars)
return model_pars, data_pars, compute_pars, out_pars
else:
raise Exception(f"Not support choice {choice} yet")
def make_table(exper_set, title_set):
path = Statistic_Dir()
exper_file = os.path.join(path.exper, exper_set + '.txt')
title_file = os.path.join(path.title, title_set + '.json')
table_file = os.path.join(path.table, exper_set + '.csv')
expers = parse_exper_file(exper_file)
with open(title_file, 'r') as f:
parser = JsonComment(json)
titles = parser.load(f)
# titles = commentjson.load(f)
type_list = ['hparam']
title_list = ['expernameid']
for _type, _type_title_list in titles.items():
for title in _type_title_list:
if title != 'expernameid':
type_list.append(_type)
title_list.append(title)
table = []
# table.append(type_list)
table.append(title_list)
for exper in expers:
exper_data = []
exper_stati_file = os.path.join(path.root, '__result__', exper,
'statistic.json')
with open(exper_stati_file, 'r') as f:
exper_stati = json.load(f)
for _type, title in zip(type_list, title_list):
value = exper_stati[_type][title]
# if re.match(r'^[-+]*[0-9]+.[0-9]+$', value):
# value = '%' % float(value)
exper_data.append(value)
table.append(exper_data)
# print(table)
with open(table_file, "w") as f:
writer = csv.writer(f)
writer.writerows(table)
print('table made:', table_file)
def load_data():
detector_output = git_root("models", "benchmark", "crypto-detector_output")
def join_path(source):
return os.path.join(
detector_output, f"{source}_output.crypto"
)
filenames = {source: join_path(source) for source in sources}
outputs = {source: None for source in sources}
for source in filenames:
with open(filenames[source]) as data_file:
parser = JsonComment()
outputs[source] = parser.load(data_file)
return outputs
def read_json(paramfilepath, adl=None, advanced_parsing=False):
"""
Read JSON file by reading in binary (bytes) then decoding in UTF8 (not possible with jsoncomment)
:param paramfilepath:
:param adl:
:param advanced_parsing: If this set to True read in text mode then parse:
trailing commas, comments and multi line data strings
:return:
"""
if advanced_parsing:
from jsoncomment import JsonComment
parser = JsonComment(json)
with CustomOpen(paramfilepath, mode="r", adl=adl) as data_file:
data = parser.load(data_file)
return data
else:
with CustomOpen(filename=paramfilepath, adl=adl) as data_file:
data = json.loads(data_file.read().decode(encoding))
return data
def load_json(filename):
'''A simple function to load json files.'''
with open(filename, 'r') as f:
parser = JsonComment(json) #Remove trailing commas
return parser.load(f)
def read_scenario_conf(scenario):
""" Read scenario configuration from scenarios/<scenario>.json """
parser = JsonComment(json)
with open(os.path.join('scenarios', scenario + '.json'), 'r') as f:
return parser.load(f)
# builds the path to the colors.json file we'll be reading from
# TODO add user config option
home = expanduser("~")
colorfile = "/.cache/wal/colors.json"
colorfile = home + colorfile
templateFile = "default.mustache"
# uses JsonComment as a our parser
parser = JsonComment(json)
renderer = pystache.Renderer()
# try to load the colors into a dictionary
# if fails, exit with error
with open(colorfile, 'r') as fc:
color_dict = parser.load(fc)
print(color_dict)
# create backup of original settings.json
# try to load settings.json
# if fails, exit with error
# TODO change to non dummy dir
# Should that be a user option?
with open('settings.json', 'r') as fs:
settings_dict = parser.load(fs)
renderer.render_path(templateFile, )
print(renderer.render_path(templateFile, ))
print(settings_dict)
def get_params(param_pars={}, **kw):
from jsoncomment import JsonComment ; json = JsonComment()
pp = param_pars
choice = pp['choice']
config_mode = pp['config_mode']
data_path = pp['data_path']
if choice == "json":
data_path = path_norm(data_path)
cf = json.load(open(data_path, mode='r'))
cf = cf[config_mode]
return cf['model_pars'], cf['data_pars'], cf['compute_pars'], cf['out_pars']
if choice == "test01":
log("#### Path params ##########################################")
root = path_norm()
data_path = path_norm( "dataset/text/imdb.npz" )
out_path = path_norm( "ztest/model_keras/charcnn/" )
model_path = os.path.join(out_path , "model")
model_pars = {
"embedding_size": 128,
"conv_layers": [[256, 10 ], [256, 7 ], [256, 5 ], [256, 3 ] ],
"fully_connected_layers": [
1024,
1024
],
"threshold": 1e-6,
"dropout_p": 0.1,
"optimizer": "adam",
"loss": "categorical_crossentropy"
}
data_pars = {
"train": True,
"alphabet": "abcdefghijklmnopqrstuvwxyz0123456789-,;.!?:'\"/\\|_@#$%^&*~`+-=<>()[]{}",
"alphabet_size": 69,
"input_size": 1014,
"num_of_classes": 4,
"train_data_source": path_norm("dataset/text/ag_news_csv/train.csv") ,
"val_data_source": path_norm("dataset/text/ag_news_csv/test.csv")
}
compute_pars = {
"epochs": 1,
"batch_size": 128
}
out_pars = {
"path": path_norm( "ztest/ml_keras/charcnn/charcnn.h5"),
"data_type": "pandas",
"size": [0, 0, 6],
"output_size": [0, 6]
}
return model_pars, data_pars, compute_pars, out_pars
else:
raise Exception(f"Not support choice {choice} yet")
def main():
question_list = []
# mongodb_uri = 'mongodb://*****:*****@127.0.0.1/yahoo_answers'
# client = MongoClient(mongodb_uri, 10017)
# db = client['yahoo_answers']
# collect = db['questions']
with open('source/categories.json', 'r', encoding='utf-8') as f:
parser = JsonComment(json)
categories = parser.load(f)
searchbox_id = "UHSearchBox"
searchbutton_id = "UHSearchProperty"
chrome_driver = '/Users/jo/Documents/workspaces/python/yahoo_answers_crawler/driver/chromedriver'
browser = webdriver.Chrome(chrome_driver)
base_url = "https://tw.answers.yahoo.com/"
for item in categories:
arr = item.split("|")
if len(arr) == 2:
words = arr[0] + ' ' + arr[1]
elif len(arr) == 3:
# words = arr[0] + ' ' + arr[1] + ' ' + arr[2] # less query result
words = arr[0] + ' ' + arr[2] # more query result
else:
continue
browser.get(base_url)
searchbox = browser.find_element_by_id(searchbox_id)
searchbox.clear()
searchbox.send_keys(words)
browser.find_element_by_id(searchbutton_id).click()
pageCount = 0
while True:
time.sleep(3)
pageCount += 1
if pageCount > 100:
break
html_source = browser.page_source
soup = BeautifulSoup(html_source, "html.parser")
# get all articles
try:
results = soup.find('div', {'id': 'web'}) \
.find('ol', {'class': ' reg searchCenterMiddle'}) \
.findAll('div', {'class': 'compTitle'})
for r in results:
question_dict = {
"category": item,
"question": r.text,
"isValid": ""
}
# rec_id = collect.insert_one(question_dict).inserted_id
# question_dict.update({'_id':str(rec_id)})
question_list.append(question_dict)
# find the next link, break the loop if not found
try:
next = browser.find_element_by_css_selector("a.next")
next.click()
except NoSuchElementException:
break
except:
break
# if len(question_list) > 0:
# collect.insert_many(question_list)
# question_list.clear()
# results = {
# "datas":question_list
# }
with open('output/questions.json', 'w', encoding='utf-8') as outfile:
json.dump(results, outfile, ensure_ascii=False, indent=4)
browser.quit()
helpMessage = """
Usage:
# restore config file
cfconfig reset [-p /path/to/configfile]
# remove all logging by displayId settings
cfconfig reset loggingbydisplayid [-p /path/to/configfile]
# set logging level and publisher
cfconfig set debug [-t gateway,local] [-p /path/to/configfile] [-d 15]
"""
print(helpMessage)
try:
with open(path, "r", encoding="UTF-8-SIG") as f:
config = jsoncomment.load(f)
except Exception as e:
pass
if(len(args) > 0):
cn = args[0].lower()
if cn == "reset":
if(len(args) == 1):
reset()
else:
if args[1].lower() == "loggingbydisplayid":
removeAllSpecificLogging(config)
with open(path, "w+", encoding="UTF-8-SIG") as wf:
json.dump(config, wf, indent=4, sort_keys=True)
elif cn == "set":
if not displayId:
if int(SERVER_RAM[:-1]) > max_server_ram:
print(
f"\nWARNING: Reducing server RAM from {SERVER_RAM}G to {max_server_ram}G."
)
SERVER_RAM = str(max_server_ram) + "G"
# Load secrets
SECRETS_FILE = os.path.join(SERVER_DIR, "secrets.json")
if JsonComment is not None:
COMMENT_JSON = JsonComment()
try:
with open(SECRETS_FILE, "r") as fp:
SECRETS = COMMENT_JSON.load(fp)
except FileNotFoundError:
SECRETS = {}
else:
SECRETS = {}
# Mod sync constants
MODS_NAME = "mods"
BASE_MODS_NAME = MODS_NAME + "-base"
EXTRA_MODS_NAME = MODS_NAME + "-main"
REMOVED_MODS_NAME = MODS_NAME + "-removed"
CLIENT_MODS_NAME = "client_mods"
BASE_CLIENT_MODS_NAME = CLIENT_MODS_NAME + "-base"
EXTRA_CLIENT_MODS_NAME = CLIENT_MODS_NAME + "-main"
