def build_metafiles(model): name = model['metadata']['name'] return { 'python/' + snake_case(name) + '/' + snake_case(name) + '.html' : env.get_template('main.html').render(standalone=False, **model), 'python/' + snake_case(name) + '/index.html' : env.get_template('main.html').render(standalone=True, **model), 'python/' + snake_case(name) + '/' + snake_case(name) + '_preview.html' : env.get_template('preview.html').render(**model) }
def build_locals(model): locals = model["locals"] hardware = model['metadata']['hardware'] module_name = snake_case(model['metadata']['name']) moves = {} for local in locals: name = snake_case(local["name"]) file = local["file"] row_type = local["type"] json_path = snake_case(model['metadata']['name']) + "_dataset.js" new_file = name+'.csv' new_small_file = name+'_small.csv' row_type = local["type"] keys = set() with open(file, "r") as local_file: if row_type == "json": data_list = json.load(local_file) data = [JsonLeafNodes(name+'.[0]', item).result for item in data_list] key_names = set([key for row in data for key in row.keys()]) short_key_names = shortest_unique_strings(key_names) key_name_map = dict(zip(short_key_names, key_names)) data = [OrderedDict(sorted([(short, kill_unicode(row.get(long, ''))) for short, long in key_name_map.items()])) for row in data] #json_list = [OrderedDict(sorted(flatten_json(_byteify(element), '_').items())) # for element in data_list] write_list_of_dictionaries(new_file, data) moves[new_file] = 'csv/'+module_name+'/' write_list_of_dictionaries(new_small_file, data[:hardware]) moves[new_small_file] = 'csv/'+module_name+'/' elif row_type == "csv": pass return moves
def build_csv(model): files = {} moves = {} descriptions = [] for local_file_name, fkd in build_locals(model): moves.update(local_file_name) descriptions.append(fkd) for appendix in model['metadata']['appendix']: moves[appendix['file']] = 'csv/' + snake_case( model['metadata']['name']) + '/' files.update(build_metafiles(model, descriptions)) icon_file = model['metadata']['icon'] name = snake_case(model['metadata']['name']) new_folder = 'csv/' + name + '/' if os.path.exists(icon_file): with open(icon_file, 'rb') as icon_data: files[new_folder + name + '.png'] = icon_data.read() else: model["metadata"]["icon"] = False return files, moves
def build_metafiles(model): name = model['metadata']['name'] return { 'python/' + snake_case(name) + '/' + snake_case(name) + '.html': env.get_template('main.html').render(standalone=False, **model), 'python/' + snake_case(name) + '/index.html': env.get_template('main.html').render(standalone=True, **model), 'python/' + snake_case(name) + '/' + snake_case(name) + '_preview.html': env.get_template('preview.html').render(**model) }
def build_locals(model): locals = model["locals"] hardware = model['metadata']['hardware'] module_name = snake_case(model['metadata']['name']) moves = {} for local in locals: name = snake_case(local["name"]) file = local["file"] row_type = local["type"] json_path = module_name + "_dataset.js" new_file = module_name + '.csv' new_small_file = module_name + '_small.csv' row_type = local["type"] keys = set() with open(file, "r") as local_file: if row_type == "json": data_list = json.load(local_file) data = [ JsonLeafNodes(name + '.[0]', item).result for item in data_list ] data_long_names = data key_names = set([key for row in data for key in row.keys()]) short_key_names = shortest_unique_strings(key_names) key_name_map = dict(zip(short_key_names, key_names)) comment_map = { short: model['structures_comments'].get(long, '') for short, long in key_name_map.items() } data = [ OrderedDict( sorted([(short, kill_unicode(row.get(long, ''))) for short, long in key_name_map.items()])) for row in data ] full_key_descriptions = [{ 'name': short, 'short': short, 'type': value, 'comment': comment_map.get(short), 'example': value } for short, value in data[0].items()] #json_list = [OrderedDict(sorted(flatten_json(_byteify(element), '_').items())) # for element in data_list] write_list_of_dictionaries(new_file, data) moves[new_file] = 'csv/' + module_name + '/' write_list_of_dictionaries(new_small_file, data[:hardware]) moves[new_small_file] = 'csv/' + module_name + '/' elif row_type == "csv": pass yield moves, full_key_descriptions
def build_database(model): name = snake_case(model['metadata']['name']) new_file = name+'.db' if os.path.exists(new_file): os.remove(new_file) database_file = sqlite3.connect(new_file) return new_file, database_file
def build_racket(model, fast): name = snake_case(model['metadata']['name']) new_folder = 'racket/' + name + '/' files = {} icon_file = model['metadata']['icon'] if os.path.exists(icon_file): with open(icon_file, 'rb') as icon_data: files[new_folder+name+'.png'] = icon_data.read() else: model["metadata"]["icon"] = False files.update(build_metafiles(model)) files.update(build_main(model)) if not fast: new_file, database_file = build_database(model) build_locals(model, database_file) database_file.close() moves = {new_file: new_folder} for appendix in model['metadata']['appendix']: moves[appendix['file']] = new_folder else: moves = {} return files, moves
def build_main(model): name = model['metadata']['name'] root = 'java/{name}/src/corgis/{name}/'.format(name=snake_case(name)) return { root + camel_case_caps(name) + 'Library.java': env.get_template('main.java').render(**model) }
def build_blockpy(model, fast): name = snake_case(model['metadata']['name']) new_folder = 'blockpy/' + name + '/' files = {} icon_file = model['metadata']['icon'] if os.path.exists(icon_file): with open(icon_file, 'rb') as icon_data: files[new_folder + name + '.png'] = icon_data.read() else: model["metadata"]["icon"] = False results = list(build_locals(model, new_folder)) moves = {f: new_folder for s in results for f in s[:2]} key_names = [k for s in results for k in s[2]] indexes = {k: v for s in results for k, v in s[3].items()} full_key_descriptions = [i[4] for i in results] tifa_definitions = [] for interface, s in zip(model['interfaces'], results): tifa_definitions.append( (interface['name'], TifaDefinition(s[5]).result)) files.update( build_metafiles(model, key_names, indexes, full_key_descriptions, tifa_definitions)) return files, moves
def build_python(model, fast): name = snake_case(model['metadata']['name']) new_folder = 'python/' + name + '/' files = {} icon_file = model['metadata']['icon'] if os.path.exists(icon_file): with open(icon_file, 'rb') as icon_data: files[new_folder + name + '.png'] = icon_data.read() else: model["metadata"]["icon"] = False splash_file = model['metadata']['splash'] if os.path.exists(splash_file): with open(splash_file, 'rb') as splash_data: files[new_folder + name + "_splash.png"] = splash_data.read() else: model["metadata"]["splash"] = False files.update(build_metafiles(model)) files.update(build_main(model)) if not fast: new_file, database_file = build_database(model) build_locals(model, database_file) database_file.close() moves = {new_file: new_folder} for appendix in model['metadata']['appendix']: moves[appendix['file']] = new_folder else: moves = {} return files, moves
def build_visualizer(model, fast): name = snake_case(model['metadata']['name']) new_folder = 'visualizer/' + name + '/' files = {} icon_file = model['metadata']['icon'] if os.path.exists(icon_file): with open(icon_file, 'rb') as icon_data: files[new_folder+name+'.png'] = icon_data.read() else: model["metadata"]["icon"] = False splash_file = model['metadata']['splash'] if os.path.exists(splash_file): with open(splash_file, 'rb') as splash_data: files[new_folder+name+"_splash.png"] = splash_data.read() else: model["metadata"]["splash"] = False if not fast: moves = {f: new_folder for f in build_locals(model, new_folder)} else: moves = {} files.update(build_metafiles(model)) return files, moves
def build_locals(model, database_file): locals = model["locals"] moves = {} for local in locals: name = snake_case(local["name"]) file = local["file"] type = local["type"] with open(file, "r") as local_file: if type == "json": data_list = json.load(local_file) json_list = [json.dumps(element) for element in data_list] indexes = [] index_titles = "" for index in local["indexes"]: index_name = index["name"] index_path = index["jsonpath"] indexed_value = [json_path(index_path, element) for element in data_list] indexes.append(indexed_value) index_titles += ", {} text".format(index_name) header = "{}(data{})".format(name, index_titles) blanks = "?" + (", ?" * len(indexes)) database_file.execute('CREATE TABLE '+header) if local["indexes"]: the_list = zip(json_list, zip(*indexes)) else: the_list = zip(json_list, [tuple() for x in json_list]) for row, indices in the_list: database_file.execute("INSERT INTO {} VALUES ({})".format(name, blanks), (row,)+indices) database_file.commit() elif type == "csv": pass
def build_sql(model, fast): name = snake_case(model['metadata']['name']) new_folder = 'sql/' + name + '/' files = {} icon_file = model['metadata']['icon'] if os.path.exists(icon_file): with open(icon_file, 'rb') as icon_data: files[new_folder+name+'.png'] = icon_data.read() else: model["metadata"]["icon"] = False files.update(build_metafiles(model)) if not fast: new_file, database_file = build_database(model) sql_file = name+".sql" build_locals(model, database_file, sql_file) database_file.close() moves = {new_file: new_folder, sql_file: new_folder} for appendix in model['metadata']['appendix']: moves[appendix['file']] = new_folder else: moves = {} return files, moves
def build_metafiles(model): name = snake_case(model['metadata']['name']) root = 'csv/{name}/'.format(name=name) return { root+'index.html' : env.get_template('csv_main.html').render(standalone=True, **model), root+name+'.html' : env.get_template('csv_main.html').render(standalone=False, **model) }
def build_metafiles(model, report): name = snake_case(model['metadata']['name']) root = 'metrics/' + name + '/' return { root+'index.html' : env.get_template('metrics_main.html').render(report=report, standalone=True, **model), root+name+'.html' : env.get_template('metrics_main.html').render(report=report, standalone=False, **model) }
def build_metafiles(model, key_names, indexes, full_key_descriptions, tifa_definitions): name = snake_case(model['metadata']['name']) root = 'blockpy/' + name + '/' return { root + 'index.html': env.get_template('blockpy_main.html').render( key_names=key_names, indexes=indexes, full_key_descriptions=full_key_descriptions, standalone=True, **model), root + name + '.html': env.get_template('blockpy_main.html').render( key_names=key_names, indexes=indexes, full_key_descriptions=full_key_descriptions, standalone=False, **model), root + name + '_skulpt.js': env.get_template('skulpt.js').render(key_names=key_names, indexes=indexes, tifa_definitions=tifa_definitions, **model), root + name + '_blockly.js': env.get_template('blockly.js').render(key_names=key_names, indexes=indexes, **model) }
def build_database(model): name = snake_case(model['metadata']['name']) new_file = name+'.csv' if os.path.exists(new_file): os.remove(new_file) with open(new_file, 'w') as output: pass return new_file
def build_database(model): name = snake_case(model['metadata']['name']) new_file = name + '.csv' if os.path.exists(new_file): os.remove(new_file) with open(new_file, 'w') as output: pass return new_file
def build_metafiles(model): name = snake_case(model['metadata']['name']) root = 'racket/'+name+'/' return { root + name + '.scrbl' : env.get_template('main.scrbl').render(**model), root+'index.html' : env.get_template('racket_main.html').render(standalone=True, **model), root+name+'.html' : env.get_template('racket_main.html').render(standalone=False, **model) }
def build_metafiles(model, key_names, indexes, full_key_descriptions): name = snake_case(model['metadata']['name']) root = 'blockpy/' + name + '/' return { root+'index.html' : env.get_template('blockpy_main.html').render(key_names=key_names, indexes=indexes, full_key_descriptions=full_key_descriptions, standalone=True, **model), root+name+'.html' : env.get_template('blockpy_main.html').render(key_names=key_names, indexes=indexes, full_key_descriptions=full_key_descriptions, standalone=False, **model), root + name + '_skulpt.js' : env.get_template('skulpt.js').render(key_names=key_names, indexes=indexes, **model), root + name + '_blockly.js' : env.get_template('blockly.js').render(key_names=key_names, indexes=indexes, **model) }
def build_metafiles(model): name = snake_case(model['metadata']['name']) root = 'json/{name}/'.format(name=name) return { root + 'index.html': env.get_template('json_main.html').render(standalone=True, **model), root + name + '.html': env.get_template('json_main.html').render(standalone=False, **model) }
def build_csv(model): files = {} files.update(build_metafiles(model)) moves = {} moves.update(build_locals(model)) for appendix in model['metadata']['appendix']: moves[appendix['file']] = 'csv/' + snake_case(model['metadata']['name']) + '/' icon_file = model['metadata']['icon'] name = snake_case(model['metadata']['name']) new_folder = 'csv/' + name + '/' if os.path.exists(icon_file): with open(icon_file, 'rb') as icon_data: files[new_folder+name+'.png'] = icon_data.read() else: model["metadata"]["icon"] = False return files, moves
def build_locals(model, js_path): locals = model["locals"] metadata_name = snake_case(model['metadata']['name']) model['visualized_datasets'] = {} for local in locals: name = local["name"] file = local["file"] row_type = local["type"] row = local["row"] json_path = snake_case(model['metadata']['name']) + "_dataset.js" with open(file, "r") as local_file, open(json_path, 'w') as output_file: output_file.write("_IMPORTED_DATASETS['{}'] = ".format(metadata_name)) if row_type == "json": data_list = json.load(local_file) data = [JsonLeafNodes(name+'.[0]', item).result for item in data_list] data = lod_to_dol(data) remove_outliers(data) for row in data: if row['name'] in model['structures_comments']: row['comment'] = model['structures_comments'][row['name']] is_index = row['name'] in [i['jsonpath'] for i in local['indexes']] row['index'] = is_index if is_index: print("Index:", row["name"]) key_names = [row['name'] for row in data] short_key_names = shortest_unique_strings(key_names) key_name_map = dict(zip(key_names, short_key_names)) full_key_descriptions = [ {'name': row['name'], 'short': key_name_map[row['name']], 'type': row['type'], 'comment': row.get('comment', ''), 'example': row['data'][0]} for row in data] indexes = {key_name_map[row['name']]: row for row in data if row['index']} for index_data in indexes.values(): index_data['data'] = [str(val) for val in index_data['data']] data = {key_name_map[row['name']]: row for row in data} #sample_down(data) json.dump(data, output_file, indent=2) #model['visualized_datasets'][name] = data.keys() print("File sizes:", "{}mb".format(os.stat(json_path).st_size / 1024 / 1024)) yield json_path, short_key_names, indexes, full_key_descriptions
def to_python_variable(source): was_list = is_list(source) if was_list: source = strip_list(source) #chomp out the "list(" and ")" converted_type= python_types.get(source, None) if converted_type is None: # need to convert to custom class converted_type = snake_case(source) if was_list: # if it's a list, apply it to each element return "list_of_{}".format(converted_type) else: # otherwise just return it normally return "a_{}".format(converted_type)
def build_main(model): name = model['metadata']['name'] tifa_definitions = [] locals = model["locals"] from languages.tifa import TifaDefinition for interface, local in zip(model['interfaces'], locals): file = local["file"] type = local["type"] with open(file, "r") as local_file: if type == "json": data_list = json.load(local_file) tifa_definitions.append( (interface['name'], TifaDefinition(data_list).result)) return { 'python/' + snake_case(name) + '/' + snake_case(name) + '.py': env.get_template('main.py').render(tifa_definitions=tifa_definitions, **model) }
def to_sql_variable(source): was_list = is_list(source) if was_list: source = strip_list(source) #chomp out the "list(" and ")" converted_type= sql_type_names.get(source, None) if converted_type is None: # need to convert to custom class converted_type = snake_case(source) if was_list: # if it's a list, apply it to each element return "list_of_{}".format(converted_type) else: # otherwise just return it normally return "a_{}".format(converted_type)
def build_json(model): files = {} files.update(build_metafiles(model)) moves = {} moves.update(build_locals(model)) for appendix in model['metadata']['appendix']: moves[appendix['file']] = 'json/' + snake_case( model['metadata']['name']) + '/' icon_file = model['metadata']['icon'] name = snake_case(model['metadata']['name']) new_folder = 'json/' + name + '/' if os.path.exists(icon_file): with open(icon_file, 'rb') as icon_data: files[new_folder + name + '.png'] = icon_data.read() else: model["metadata"]["icon"] = False return files, moves
def build_report(model): locals = model["locals"] hardware = model['metadata']['hardware'] module_name = snake_case(model['metadata']['name']) json_reports = {} for local in locals: name = snake_case(local["name"]) file = local["file"] type = local["type"] row = local["row"] with open(file, "r") as local_file: if type == "json": data_list = json.load(local_file) metrics = JsonMetrics({}, data_list, row) json_reports[name] = metrics.report json_reports[name]['length'] = len(data_list) json_reports[name]['size'] = os.path.getsize(file) json_reports[name]['tags'] = model['metadata']['tags'] elif type == "csv": pass return json.dumps(json_reports, indent=2)
def build_classes(model): name = model['metadata']['name'] root = 'java/{name}/src/corgis/{name}/domain/'.format( name=snake_case(name)) files = {} template = env.get_template('domain.java') for structure_name, structure in model['structures'].items(): for path, data in structure['dictionaries'].items(): filename = root + camel_case_caps( clean_invalid_characters(data['name'])) + '.java' files[filename] = template.render(dictionary=data, **model) return files
def build_metafiles(model): name = snake_case(model['metadata']['name']) root = 'java/{name}/'.format(name=name) return { root + '.classpath': env.get_template('.classpath').render(**model), root + '.project': env.get_template('.project').render(**model), root + 'build.xml': env.get_template('build.xml').render(**model), root + 'index.html': env.get_template('java_main.html').render(standalone=True, **model), root + name + '.html': env.get_template('java_main.html').render(standalone=False, **model) }
def build_locals(model): locals = model["locals"] hardware = model['metadata']['hardware'] module_name = snake_case(model['metadata']['name']) moves = {} for local in locals: name = module_name #snake_case(local["name"]) new_file = name + '.json' file = local["file"] with open(file, 'rb') as inp, open(new_file, 'wb') as out: json.dump(json.load(inp), out, indent=2) moves[new_file] = 'json/' + module_name + '/' return moves
def build_report(model): locals = model["locals"] hardware = model['metadata']['hardware'] module_name = snake_case(model['metadata']['name']) json_reports = {} for local in locals: name = snake_case(local["name"]) file = local["file"] type = local["type"] row = local["row"] with open(file, "r") as local_file: if type == "json": data_list = json.load(local_file) metrics = JsonMetrics({}, data_list, row) json_reports[name] = metrics.report json_reports[name]['length'] = len(data_list) json_reports[name]['size'] = os.path.getsize(file) json_reports[name]['tags'] = model['metadata']['tags'] json_reports[name]['description'] = model['metadata']['description']['overview'] json_reports[name]['indexes'] = [len(local['indexes']) for local in locals] elif type == "csv": pass return json.dumps(json_reports, indent=2), json_reports
def build_java(model, fast): name = snake_case(model['metadata']['name']) root = 'java/{name}/src/corgis/{name}/'.format(name=snake_case(name)) new_folder = 'java/' + name + '/' files = { new_folder + '/libs/sqlite-jdbc-3.8.11.2.jar': copy_file(java_templates + 'libs/sqlite-jdbc-3.8.11.2.jar'), new_folder + '/libs/json-simple-1.1.1.jar': copy_file(java_templates + 'libs/json-simple-1.1.1.jar') } icon_file = model['metadata']['icon'] if os.path.exists(icon_file): with open(icon_file, 'rb') as icon_data: files[new_folder + name + '.png'] = icon_data.read() else: model["metadata"]["icon"] = False files.update(build_metafiles(model)) files.update(build_main(model)) files.update(build_classes(model)) if not fast: new_file, database_file = build_database(model) build_locals(model, database_file) database_file.close() moves = {new_file: new_folder} for appendix in model['metadata']['appendix']: moves[appendix['file']] = new_folder else: moves = {} return files, moves
def post_build(model, files, moves, target): print("Building jar") name = snake_case(model['metadata']['name']) path = os.path.join(target, 'java', name) backup_location = os.getcwd() os.chdir(path) subprocess.call(["ant"], shell=True) #jar_file = os.path.join('./dist', name+'-doc-1.jar') #subprocess.call(["java", "-jar", "../../../tools/plantuml.jar", jar_file, "-o", "./docs/uml/"], shell=True) os.chdir(backup_location) return None
def build_metrics(model, fast): module_name = snake_case(model['metadata']['name']) new_folder = 'metrics/' + module_name + '/' json_data, report = build_report(model) files = {'metrics/' + module_name + '/' + module_name + '.json': json_data} icon_file = model['metadata']['icon'] if os.path.exists(icon_file): with open(icon_file, 'rb') as icon_data: files[new_folder+module_name+'.png'] = icon_data.read() else: model["metadata"]["icon"] = False files.update(build_metafiles(model, report)) return files, {}
def build_blockpy(model, fast): name = snake_case(model['metadata']['name']) new_folder = 'blockpy/' + name + '/' files = {} icon_file = model['metadata']['icon'] if os.path.exists(icon_file): with open(icon_file, 'rb') as icon_data: files[new_folder+name+'.png'] = icon_data.read() else: model["metadata"]["icon"] = False results = list(build_locals(model, new_folder)) moves = {f: new_folder for f in first_items(results)} key_names = [k for s in results for k in s[1]] indexes = {k:v for s in results for k, v in s[2].items()} full_key_descriptions = [i[3] for i in results] files.update(build_metafiles(model, key_names, indexes, full_key_descriptions)) return files, moves
def build_locals(model, database_file): locals = model["locals"] moves = {} for local in locals: name = snake_case(local["name"]) file = local["file"] type = local["type"] with open(file, "r") as local_file: if type == "json": data_list = json.load(local_file) json_list = [json.dumps(element) for element in data_list] indexes = [] index_titles = "" for index in local["indexes"]: index_name = index["name"] index_path = index["jsonpath"] indexed_value = [ json_path(index_path, element) for element in data_list ] indexes.append(indexed_value) index_titles += ", {} text".format(index_name) header = "{}(data{})".format(name, index_titles) blanks = "?" + (", ?" * len(indexes)) database_file.execute('CREATE TABLE ' + header) if local["indexes"]: the_list = zip(json_list, zip(*indexes)) else: the_list = zip(json_list, [tuple() for x in json_list]) for row, indices in the_list: database_file.execute( "INSERT INTO {} VALUES ({})".format(name, blanks), (row, ) + indices) database_file.commit() elif type == "csv": pass
def build_metrics(model, fast): module_name = snake_case(model['metadata']['name']) files = {'metrics/' + module_name + '/' + module_name + '.json': build_report(model)} return files, {}
def build_locals(model, js_path): locals = model["locals"] metadata_name = snake_case(model['metadata']['name']) model['visualized_datasets'] = {} for local in locals: name = local["name"] file = local["file"] row_type = local["type"] row = local["row"] # Complete Dataset complete_path = snake_case(model['metadata']['name']) + "_complete.js" with open(file, "r") as local_file, open(complete_path, 'w') as output_file: output_file.write( "_IMPORTED_COMPLETE_DATASETS['{}'] = Sk.ffi.remapToPy(".format( metadata_name)) if row_type == "json": data_list = json.load(local_file) json.dump(data_list, output_file, indent=2) output_file.write(");") # Linearized Dataset json_path = snake_case(model['metadata']['name']) + "_dataset.js" with open(file, "r") as local_file, open(json_path, 'w') as output_file: output_file.write( "_IMPORTED_DATASETS['{}'] = ".format(metadata_name)) if row_type == "json": data_list = json.load(local_file) data = [ JsonLeafNodes(name + '.[0]', item).result for item in data_list ] data = lod_to_dol(data) remove_outliers(data, actually_keep=model['metadata']['outliers']) for row in data: if row['name'] in model['structures_comments']: row['comment'] = model['structures_comments'][ row['name']] is_index = row['name'] in [ i['jsonpath'] for i in local['indexes'] ] row['index'] = is_index if is_index: print("Index:", row["name"]) key_names = [row['name'] for row in data] short_key_names = shortest_unique_strings(key_names) key_name_map = dict(zip(key_names, short_key_names)) full_key_descriptions = [{ 'name': row['name'], 'short': key_name_map[row['name']], 'type': row['type'], 'comment': row.get('comment', ''), 'example': row['data'][0] } for row in data] indexes = { key_name_map[row['name']]: row for row in data if row['index'] } for index_data in indexes.values(): index_data['data'] = [ str(val) for val in index_data['data'] ] data = {key_name_map[row['name']]: row for row in data} json.dump(data, output_file, indent=2) #model['visualized_datasets'][name] = data.keys() print("File sizes:", "{}mb".format(os.stat(json_path).st_size / 1024 / 1024)) yield json_path, complete_path, short_key_names, indexes, full_key_descriptions, data_list
def build_main(model): name = model['metadata']['name'] return { 'python/' + snake_case(name) + '/' + snake_case(name) + '.py': env.get_template('main.py').render(**model) }
def build_locals(model, js_path): locals = model["locals"] model['visualized_datasets'] = {} print("") for local in locals: name = snake_case(local["name"]) file = local["file"] row_type = local["type"] row = local["row"] json_path = name + ".js" json_bar_path = name + "_bar.js" with open(file, "r") as local_file, open(json_path, 'w') as output_file, open(json_bar_path, 'w') as output_bar_file: output_file.write(name+" = ") output_bar_file.write(name+"_bar = ") if row_type == "json": data_list = json.load(local_file) data = [JsonLeafNodes(name+'.[0]', item).result for item in data_list] data = lod_to_dol(data) bar_data = [] key_name_map = remove_outliers(data, actually_keep=model['metadata']['outliers']) for row in data: if row['name'] in model['structures_comments']: row['comment'] = model['structures_comments'][row['name']] is_index = row['name'] in [i['jsonpath'] for i in local['indexes']] row['index'] = is_index dol = {d['name']: d['data'] for d in data} lod = [dict(zip(dol, t)) for t in zip(*dol.values())] for index in local['indexes']: index_path = index['jsonpath'] indexed_values = {} aggregated_values = {} for chunk in lod: if index_path in chunk: category = str(chunk[index_path]) else: category = "" if category not in indexed_values: indexed_values[category] = {} for key, value in chunk.items(): if not isinstance(value, (int, float)): continue if key not in indexed_values[category]: indexed_values[category][key] = [] indexed_values[category][key].append(value) for category, items in indexed_values.items(): category = str(category) for key, values in items.items(): if key not in aggregated_values: aggregated_values[key] = {} aggregated_values[key][category.replace(',', '')] = { 'count': len(values), 'sum': sum(values), 'average': statistics.mean(values) if sum(values) else 0, 'average w/o zero': statistics.mean([v for v in values if v>0] if sum([v for v in values if v>0]) else [0]) } #inter = index_path.split('.', 2)[2] #if '.' in inter: # category, name = inter.rsplit('.', 1) #else: # category, name = inter, inter #category = category.replace('.', ' ') bar_data.append({ 'data': aggregated_values, 'name': index_path, 'indexes': [k.replace(',', '') for k in indexed_values.keys()], 'best_indexes': [k.replace(',', '') for k, v in sorted([(k, v['count']) for k, v in list(aggregated_values.values())[0].items()], key=lambda i: -i[1])[:10]], #'pretty': category.title() + ": "+name.title() 'pretty': key_name_map[index_path] }) #sample_down(data) json.dump(data, output_file, indent=2) json.dump(bar_data, output_bar_file, indent=2) #model['visualized_datasets'][name] = data.keys() print("File sizes:", "{}mb".format(os.stat(json_path).st_size/1024/1024), "{}mb".format(os.stat(json_bar_path).st_size/1024/1024)) yield json_path yield json_bar_path
def build_main(model): name = model['metadata']['name'] return {'python/' + snake_case(name) + '/' + snake_case(name) + '.py' : env.get_template('main.py').render(**model)}
def build_main(model): name = model['metadata']['name'] return {'racket/' + snake_case(name) + '/' + snake_case(name) + '.rkt' : env.get_template('main.rkt').render(**model)}