def write_config_to_file(data, path):
path = Path(path)
extension = path.suffix.lower()
with path.open('w') as f:
if extension in ('.yaml', '.yml'):
yaml.dump(data, f)
elif extension in ('.json', ):
json.dump(data, f)
else:
raise RuntimeError(
'Unknown file extension for the file "{}"'.format(path))
def download_engine_config(model_name):
def process_config():
engine_conf = Dict()
mode = 'evaluations' if ac_conf.evaluations else 'models'
for model in ac_conf[mode]:
model_ = model
engine_conf_ = engine_conf
if mode == 'evaluations':
engine_conf.module = model.module
model_ = model.module_config
engine_conf_ = engine_conf.module_config
engine_conf_.network_info = model_.network_info
for launcher in model_.launchers:
if launcher.framework == 'dlsdk':
engine_conf_.launchers = list()
engine_launcher = {'framework': launcher.framework}
if launcher.adapter:
engine_launcher.update({'adapter': launcher.adapter})
engine_conf_.launchers.append(engine_launcher)
engine_conf_.datasets = model_.datasets
convert_path_to_str(engine_conf)
return engine_conf
return None
def convert_path_to_str(config):
iterator = config.items() if isinstance(config,
dict) else enumerate(config)
for key, value in iterator:
if isinstance(value, (dict, list)):
convert_path_to_str(value)
elif isinstance(value, Path):
config[key] = value.as_posix()
config_path = LIBS_ROOT / Path(
'open_model_zoo/models/public/{}/accuracy-check.yml'.format(
model_name))
try:
ac_conf = Dict(
ConfigReader.merge(
Dict({
'config': config_path.as_posix(),
'definitions': OMZ_DEFINITIONS_PATH
}))[0])
except FileNotFoundError:
return None
engine_config = process_config()
with open((ENGINE_CONFIG_PATH / Path(model_name + '.json')).as_posix(),
'w') as f:
json.dump(engine_config, f, indent=4)
return engine_config
def init_config():
path = read_from_console("Input the config name", "config")
jstyleson.dump(
{
"executor": "python main.py",
"cuda": [-1],
"concurrency": 1,
"configs": {
"==base==": [],
"==more==": []
}
},
open("{}.json".format(path), "w"),
indent=4)
def test_plot_dag():
class TestPostprocess(AbstractNode):
@staticmethod
def necessary_config(node_config):
return set(["key1", "key2"])
def run(self, data_object):
return data_object
NodeFactory().register("TestPostprocess", TestPostprocess)
class Testpipeline(AbstractNode):
# def __init__(self, configuration, instance_name):
# pass
@staticmethod
def necessary_config(node_config):
return set([])
def run(self, data_object):
return data_object
NodeFactory().register("Testpipeline", Testpipeline)
class TestCleanup(AbstractNode):
@staticmethod
def necessary_config(node_config):
return set([])
def run(self, data_object):
return data_object
NodeFactory().register("TestCleanup", TestCleanup)
config = {
"implementation_config": {
"reader_config": {
"csv_reader": {
"class": "CsvReader",
"filename": "some/path/to/file",
"destinations": ["pipeline1"],
}
},
"pipeline_config": {
"pipeline1": {
"class": "Testpipeline",
"destinations": ["decision_tree_model"],
}
},
"model_config": {
"decision_tree_model": {
"class": "SklearnClassifierModel",
"model_parameters": {},
"sklearn_classifier_name": "tree.DecisionTreeClassifier",
"grid_search_scoring": "roc_auc",
"cv_folds": 3,
"mode": "predict",
"destinations": ["nodename"],
}
},
"postprocess_config": {
"nodename": {
"class": "TestPostprocess",
"key1": "val1",
"key2": "val2",
"destinations": ["write_output"],
}
},
"writer_config": {
"write_output": {
"class": "CsvWriter",
"key": "read_data",
"dir": "cache",
"filename": "some/path/to/file.csv",
"destinations": ["donothingsuccess"],
}
},
"cleanup_config": {
"donothingsuccess": {
"class": "TestCleanup",
}
},
}
}
cfilename = "test/test_dag_plotting.json"
with open(cfilename, "w") as f:
jstyleson.dump(config, f)
config = Configuration(config_location=cfilename)
dag = config.dag
dag.create_dag()
filename = "test/test_dag_plotting.png"
if os.path.exists(filename):
os.remove(filename)
dag.plot_dag(filename, traverser=ConfigLayerTraverser(config))
assert os.path.exists(filename)
if os.path.exists(cfilename):
os.remove(cfilename)
if os.path.exists(filename):
os.remove(filename)
import os, jstyleson, platform
snippetsPath = ''
if platform.system() == 'Windows':
snippetsPath = os.getenv('APPDATA') + '\\Code\\User\\snippets'
else:
snippetsPath = os.path.expanduser(
"~/Library/Application Support/Code/User/snippets")
snippetsFile = snippetsPath + '/python.json'
result = {}
with open(snippetsFile) as targetFile:
with open('python.json') as sourceFile:
result = jstyleson.load(targetFile)
source = jstyleson.load(sourceFile)
for key in source:
result[key] = source[key]
with open(snippetsFile, 'w') as targetFile:
jstyleson.dump(result, targetFile, indent=4)
if OS == 'win32':
ARCSimPath = '../build/Release/Arcsim.exe'
elif OS == 'linux':
ARCSimPath = '../build/Arcsim'
jsonConfigPath = '../conf/uniform_grid_drop.json'
jsonData = None
with open(jsonConfigPath,'r') as f:
jsonDataWithoutComment = ''.join(line for line in f if not line.startswith('//'))
jsonData = js.loads(jsonDataWithoutComment)
dirBatchingConfPathPrefix = os.getcwd() + '/batching_conf/'
dirPathName = jsonConfigPath.split('/')[-1]
dirPathName = dirPathName[:dirPathName.rfind('.')]
dirBatchingConfPathPrefix += dirPathName
# dirBatchingConfPathPrefix += '/'
if(not os.path.exists(dirBatchingConfPathPrefix)):
print('Folder {} is not existed! Create it!'.format(dirBatchingConfPathPrefix))
os.makedirs(dirBatchingConfPathPrefix)
for i in range(0,totalPointNumber):
jsonData['handles'][0]['nodes'] = i
configFileName = dirBatchingConfPathPrefix + '/' + dirPathName + '_{:0>8d}.json'.format(i)
with open(configFileName,'a') as f:
js.dump(jsonData,f)
