def extract_har_request_postdata_json_attributes(self): if self.no_entires(): return [] http_posts_data = [{ **er["postData"], "host": er["host"] } for er in self.har_entry_requests if "postData" in er] post_data_json_texts = [ (post_data["text"], post_data["host"]) for post_data in http_posts_data if post_data["mimeType"].startswith("application/json") ] post_data_json_array = [] for (json_text, host) in post_data_json_texts: try: post_data_json_array.append((json.loads(json_text), host)) except json.decoder.JSONDecodeError: pass post_data_df = pd.DataFrame(columns=["name", "value", "host"]) for (post_data_json, host) in post_data_json_array: flattened = json_flatten.flatten(post_data_json) for flattened_key in flattened.keys(): post_data_df = post_data_df.append( { "name": flattened_key, "value": flattened[flattened_key], "host": host, "tracker": self.detect_tracker_by_host(host) }, ignore_index=True) post_data_df = post_data_df.drop_duplicates().sort_values(by="name") # post_data_df["tracker"] = post_data_df["host"].map(self.detect_tracker) return post_data_df.to_dict("records")
def json2csv(jsonPath, filename, TestOrTrain): """ df=pandas.read_json(jsonPath+'.json') df.to_csv() df.to_csv(r'output/'+str(jsonIndex)+'.csv', index=False) """ currJson = json.load(open(jsonPath + '.json')) path = 'output/' + filename + '.csv' if not os.path.exists(path): with open(path, "w", newline='', encoding='utf-8') as csvfile: # newline='' 去除空白行 writer = csv.DictWriter(csvfile, fieldnames=dict) # 写字典的方法 writer.writeheader() for j in currJson: curr = flatten(j) #get flattened dict curr['width'] = 1280 curr['height'] = 720 curr = rename_keys(curr) curr[ 'filename'] = 'C:/Users/Administrator.WIN-2EPKD7D6018/Desktop/bdd100k-non/bdd100k/images/track/' + TestOrTrain + '/' + curr[ 'filename'].rsplit('-', 2)[0] + '-' + curr['filename'].rsplit( '-', 2)[1] + '/' + curr['filename'] with open(path, "a", newline='', encoding='utf-8') as csvfile: # newline='' 一定要写,否则写入数据有空白行 writer = csv.DictWriter(csvfile, fieldnames=dict) writer.writerow(curr) # 按行写入数据 print(filename + " write success")
def get_har_request_postdata_json_attributes(self): http_posts_data = [{ **er["postData"], "host": "" if "host" not in er else er["host"] } for er in self.har_entry_requests if "postData" in er] post_data_json_texts = [ (post_data["text"], post_data["host"]) for post_data in http_posts_data if post_data["mimeType"].startswith("application/json") ] json_texts_hosts_bag = db.from_sequence(post_data_json_texts) post_data_json_array_bag = json_texts_hosts_bag.starmap( lambda json_text, host: (self.try_json_loads(json_text), host)) post_data_json_array_bag = post_data_json_array_bag.filter( lambda element: element[0]) flattened_json_bag = post_data_json_array_bag.starmap( lambda json_obj, host: map( lambda element: { "name": element[0], "value": element[1], "host": host, "tracker": self.detect_tracker_by_host(host) }, list(json_flatten.flatten(json_obj).items()))) request_postdata_json_attributes_dict = flattened_json_bag.flatten( ).distinct(lambda o: json.dumps(o, sort_keys=True).lower()) return request_postdata_json_attributes_dict
def saveAllTrainToOneFile(): filePath = 'C:/Users/Administrator.WIN-2EPKD7D6018/Desktop/bdd100k-non/bdd100k/labels-20/box-track/train/' path = 'output/train/train_all.csv' csvfile = open(path, "w", newline='', encoding='utf-8') writer = csv.DictWriter(csvfile, fieldnames=dict) # 写字典的方法 writer.writeheader() ct = 0 for i, j, k in os.walk(filePath): # k is ####.json for path in k: jsonPath = filePath + path.rsplit('.', 1)[0] currJson = json.load(open(jsonPath + '.json')) for j in currJson: curr = flatten(j) # get flattened dict curr['width'] = 1280 curr['height'] = 720 curr = rename_keys(curr) curr['filename'] = 'C:/Users/Administrator.WIN-2EPKD7D6018/Desktop/bdd100k-non/bdd100k/images/track/train/' + \ curr['filename'].rsplit('-', 2)[0] + '-' + curr['filename'].rsplit('-', 2)[1] + '/' + \ curr['filename'] writer = csv.DictWriter(csvfile, fieldnames=dict) writer.writerow(curr) # 按行写入数据 print(str(ct) + " write success") ct += 1 print("All write success")
def json_to_csv(self, input_json: dict) -> str: """ method to handle the data conversion from JSON to CSV format :param input_json: :return: """ csv_content = json_flatten.flatten(input_json) return csv_content
def get_data(ticker): modules = "%2C".join([ 'assetProfile', 'summaryProfile', 'summaryDetail', 'esgScores', 'price', 'incomeStatementHistory', 'incomeStatementHistoryQuarterly', 'balanceSheetHistory', 'balanceSheetHistoryQuarterly', 'cashflowStatementHistory', 'cashflowStatementHistoryQuarterly', 'defaultKeyStatistics', 'financialData', 'calendarEvents', 'secFilings', 'recommendationTrend', 'upgradeDowngradeHistory', 'institutionOwnership', 'fundOwnership', 'majorDirectHolders', 'majorHoldersBreakdown', 'insiderTransactions', 'insiderHolders', 'netSharePurchaseActivity', 'earnings', 'earningsHistory', 'earningsTrend', 'industryTrend', 'indexTrend', 'sectorTrend' ]) query_url = "https://query1.finance.yahoo.com/v10/finance/quoteSummary/{}?modules={}".format( ticker, modules) while not check_internet(): print("Could not connect, trying again in 5 seconds...") time.sleep(5) try: with urllib.request.urlopen(query_url) as url: parsed = json.loads(url.read().decode()) downlaod_file_name = os.path.join("stock_fundamental_data", ticker, ticker + ".json") downlaod_flattened_file_name = os.path.join( "stock_fundamental_data", ticker, ticker + "_flattened.json") os.makedirs(os.path.dirname(downlaod_file_name), exist_ok=True) parsed = json_flatten.flatten( list(parsed["quoteSummary"]["result"])[0]) for key, value in parsed.copy().items(): if ".raw" in key or ".longFmt" in key or value == "{}" or value.lower( ) == "none" or value == "": del parsed[key] elif ".fmt" in key: parsed[key.replace(".fmt", "")] = parsed.pop(key) with open(downlaod_flattened_file_name, "w") as f: json.dump(parsed, f, indent=2, sort_keys=True) parsed = json_flatten.unflatten(parsed) with open(downlaod_file_name, "w") as f: json.dump(parsed, f, indent=2, sort_keys=True) print("Downloaded {}".format(ticker)) except: print("Data of {} doesn't exist".format(ticker)) return
def test_nested_json(self): self.assertEqual( json_flatten.flatten({ "a": 1, "b": { "c": "hello", "d": 4 }, "e": true }), '{"a": 1, "b.c": "hello", "b.d": 4, "e": true}')
def test_double_layered_nested_json(self): self.assertEqual( json_flatten.flatten({ "a": 1, "b": { "c": "hello", "d": { "e": true, "f": 6 }, "g": "seven" } }), '{"a": 1, "b.c": "hello", "b.d.e": true, "b.d.f": 6, "b.g": "seven"}' )
def test_2_nested_json(self): self.assertEqual( json_flatten.flatten({ "a": 1, "b": { "c": "hello", "d": 4 }, "e": { "f": false, "g": "example" } }), '{"a": 1, "b.c": "hello", "b.d": 4, "e.f": false, "e.g": "example"}' )
def xml_to_csv(self, input_xml: str) -> dict: """ Conversion for XML format to CSV format. For this conversion we will convert the XML into JSON first and then we will be converting that JSON to CSV format. Approach: XML -> JSON -> CSV :param input_xml: :return: """ converted_json = xmltodict.parse(input_xml) converted_csv = json_flatten.flatten(converted_json) return { 'columns': converted_csv.keys(), 'values': converted_csv.values() }
def review_numeric_lines(mid_lines): numeric_script = generate_script(mid_lines) schema = mock_schema() data = generate_mock_data(model=schema, numeric_fields=True) flattened_data = json_flatten.flatten(data) handle_numeric = lua.eval(numeric_script) handle_numeric(flattened_data) schema = mock_schema() for field in schema: _type = schema[field]['type'] if _type in numeric_types: nodes = field.split('.') if len(nodes) == 1: assert flattened_data[field] else: assert not flattened_data[field] if field[0] == 'p': assert not flattened_data[field], field
def PersonFieldsAndValues(self, person_id): # get the person's NB record response = requests.get(self.PERSON_URL % (person_id, NATIONBUILDER_API_TOKEN), timeout=self.default_timeout) self._process_response(response) # return field_path = value if the person was found if response.status_code == 200: record = response.json() flat = flatten(record) flat_list = [] for key, value in flat.items(): key_without_type = key if key.find( '$') == -1 else key[:key.find('$')] value_without_none = value if value != 'None' else '' flat_list.append( (key_without_type, value_without_none, "%s = %s" % (key_without_type, value_without_none))) flat_list.sort(key=lambda tup: tup[0]) return (flat_list) else: return None
def get_incident_list(): if 'tags_path' not in session: session['tags_path'] = {} subdomain = session["subdomain"] create_folder('resources/' + subdomain) save_path = getcwd() complete_name_tags = path.join(save_path, "resources/%s/tags.json" % subdomain) try: with open(complete_name_tags) as json_file: tags = load(json_file) except IOError: tags = {} with open(complete_name_tags, 'w+') as outfile: dump(tags, outfile) complete_name_events = path.join(save_path, "resources/%s/incidents.json" % subdomain) try: with open(complete_name_events) as json_file: incidents = load(json_file) print("Found incidents. Loading the file") except IOError: incidents = [] with open(complete_name_events, 'w+') as outfile: dump(incidents, outfile) print("Couldn't find incidents. Dumping empty list.") if not incidents: for i in range(len(session["selected_services_id"])): incidents = incidents + incident_iter_selected( session['global_token'], session['selected_windows'][i], [session['selected_services_id'][i]], session["integrations"], tags) subdomain = session["subdomain"] create_folder('resources/' + subdomain) save_path = getcwd() complete_name = path.join(save_path, "resources/%s/combos.json" % subdomain) try: with open(complete_name) as json_file: combos = load(json_file) except IOError: combos = [] with open(complete_name, 'w+') as outfile: dump(combos, outfile) keys = [] flattened_keys = [] flattened_incidents = [] ftle = [] ftle_keys = [] channels = [] channels_keys = [] alerts = [] alerts_keys = [] for incident in incidents: incident_temp = deepcopy(incident) ftle_temp = incident_temp.pop("first_trigger_log_entry") # ftle_channel = ftle_temp.pop("channel") alerts_temp = incident_temp.pop("all_alerts") flattened_incidents.append(flatten(incident_temp)) flattened_keys = list( set(flattened_keys + list(flatten(incident_temp).keys()))) ftle.append(flatten(ftle_temp)) ftle_keys = list(set(ftle_keys + list(flatten(ftle_temp).keys()))) # channels.append(flatten(ftle_channel)) # channels_keys = list(set(channels_keys + list(flatten(ftle_channel).keys()))) alerts.append(flatten(alerts_temp)) alerts_keys = list(set(alerts_keys + list(flatten(alerts_temp).keys()))) flattened_keys.sort() alerts_keys.sort() ftle_keys.sort() session["flattened_keys"] = flattened_keys session["ftle_keys"] = ftle_keys session['alerts_keys'] = alerts_keys disco_choice = session["disco_choice"] subdomain = session["subdomain"] create_folder('resources/' + subdomain) save_path = getcwd() complete_name = path.join(save_path, "resources/%s/tags.json" % subdomain) with open(complete_name, 'w+') as outfile: dump(tags, outfile) if not incidents: flash( "No incidents found in the selected window. Please select a larger window!" ) return redirect(url_for('get_service_config')) complete_name_events = path.join(save_path, "resources/%s/incidents.json" % subdomain) with open(complete_name_events, 'w+') as outfile: dump(incidents, outfile) return render_template('incident_list.html', incidents=flattened_incidents, keys=flattened_keys, alerts=alerts, alerts_keys=alerts_keys, ftle=ftle, ftle_keys=ftle_keys, channels=channels, channels_keys=channels_keys, disco_choice=disco_choice, tags=tags, provisioned=session['provisioned'], tags_exist=session['tags_exist'], subdomain=session['subdomain'], incidents_exist=session['incidents_exist'])
def test_flatten_unflatten(test_name, unflattened, flattened): assert flatten(unflattened) == flattened assert unflatten(flattened) == unflattened
def post_incident_list(): tags_path = session['tags_path'] tag_action = request.form["tag_action"] subdomain = session["subdomain"] create_folder('resources/' + subdomain) save_path = getcwd() complete_name_tags = path.join(save_path, "resources/%s/tags.json" % subdomain) try: with open(complete_name_tags) as json_file: tags = load(json_file) except IOError: tags = {} with open(complete_name_tags, 'w+') as outfile: dump(tags, outfile) if tag_action == "delete": # tags = session["tags"] tag_keys = list(tags.keys()) for tag in tag_keys: if request.form.get("tag-%s" % tag): del tags[tag] elif tag_action == "update": flattened_keys = session["flattened_keys"] ftle_keys = session["ftle_keys"] alerts_keys = session['alerts_keys'] incidents = [] # tags = session["tags"] disco_choice = session["disco_choice"] has_tags = disco_choice != "3" for i in range(len(session["selected_services_id"])): incidents = incidents + incident_iter_selected( session['global_token'], session['selected_windows'][i], [session['selected_services_id'][i]], session["integrations"][i], tags) tagged_keys = [[], []] for i in range(len(flattened_keys)): if request.form.get("incident_field_%s" % str(i)): incident_field = request.form.get("incident_field_%s" % str(i)) tagged_keys[0].append(incident_field) if incident_field not in tags: tags[incident_field] = [] for i in range(len(ftle_keys)): if request.form.get("ftle_field_%s" % str(i)): ftle_field = request.form.get("ftle_field_%s" % str(i)) tagged_keys[1].append(ftle_field) if ftle_field not in tags: tags[ftle_field] = [] for incident in incidents: incident_temp = incident ftle_temp = incident_temp.pop("first_trigger_log_entry") flattened_incident = flatten(incident_temp) ftle = flatten(ftle_temp) for key in tagged_keys[0]: if key in flattened_incident and flattened_incident[ key] not in tags[key]: tags[key].append(flattened_incident[key]) for key in tagged_keys[1]: if key in ftle and ftle[key] not in tags[key]: tags[key].append(ftle[key]) subdomain = session["subdomain"] create_folder('resources/' + subdomain) save_path = getcwd() complete_name = path.join(save_path, "resources/%s/tags.json" % subdomain) with open(complete_name, 'w+') as outfile: dump(tags, outfile) return redirect(url_for('get_incident_list'))
def test_flat_json(self): self.assertEqual(json_flatten.flatten({ "a": 1, "b": true }), '{"a": 1, "b": true}')
] # --------------------------------- Create Window --------------------------------- window = sg.Window('JSON Translator', layout) # ----- Run the Event Loop ----- # --------------------------------- Event Loop --------------------------------- while True: event, values = window.read() if event in (None, 'Exit'): break # new source file selected if event == '-SOURCE-': # clear the keys list keyList.update(values=[]) sourceDict = jf.flatten(getDictFromJSON(sourceFile.get())) sourceDictFlat = (sourceDict) keyList.update(values=list(sourceDictFlat.keys())) sourceLocale.update(value=getLocaleFromFileName(sourceFile.get()), visible=True) if event == '-TARGET-': targetDict = getDictFromJSON(targetFile.get()) targetDictFlat = jf.flatten(targetDict) targetLocale.update(value=getLocaleFromFileName(targetFile.get()), visible=True) if event == '-TARGETTEXT-': # auto save every time user types if autoSaveCheck.get() == 1: saveTarget() unsavedChanges = False else:
from json_flatten import flatten import json # Print json object as flattened one unflat_json = { 'user': { 'Rachel': { 'UserID': 1717171717, 'Email': '*****@*****.**', 'friends': ['John', 'Jeremy', 'Emily'] } } } flat_json = flatten(unflat_json) print(flat_json) #Read a json file and print as flattened one with open('mydata.json') as f: data = json.load(f) myflatdata = flatten(data) print(myflatdata) # write to a file with open("flatten_file.json", "w") as write_file: json.dump(myflatdata, write_file) #print each property separate with = intead of : and quotoes and as a property for x in myflatdata: print("%s=%s" % (x, myflatdata[x])) # Write json as properties to a file
def json_diff(original_JSON_file: str) -> dict: ''' Compute difference between left and right JSON objects :param dict left: left JSON object (original JSON) :param dict right: right JSON object (JSON generated by MSP) :param dict custom_mappings: JSON object detailing left to right custom mappings :return difference between left and right JSON objects :rtype: dict ''' print("Computing diff for: " + original_JSON_file) left: dict = {} right: dict = {} os.chdir(original_basepath) with open('{}/{}'.format(original_basepath, original_JSON_file), 'r') as f: left = json.loads(f.read()) os.chdir(new_basebath) with open('{}/{}'.format(new_basebath, original_JSON_file), 'r') as f: right = json.loads(f.read()) # pickup "PayLoad" section of left JSON for comparison left: dict = left["PayLoad"] # pickup "payLoad" section of right JSON for comparison right: dict = right["eventMessage"]["eventAttributes"]["dataElements"][ "payLoad"] # print("Flattening 'left' JSON...") left_flattened = json_flatten.flatten(left) # print("Convert 'left' JSON keys to lowercase...") left_flattened = {k.lower(): v for k, v in left_flattened.items()} # print("Flattening 'right' JSON...") right_flattened = json_flatten.flatten(right) # print("Convert 'right' JSON keys to lowercase...") # right_flattened = {k.lower(): v for k, v in right_flattened.items() if v != "" and v != 'None'} right_flattened = {k.lower(): v for k, v in right_flattened.items()} diff = {} # print("original JSON size = " + str(len(left_flattened))) # print("new JSON size = " + str(len(right_flattened))) diff["DiffStatistics"] = { "OriginalJSONSize": str(len(left_flattened)), "NewJSONSize": len(right_flattened), "Delta": abs(len(left_flattened) - len(right_flattened)) } found_diff: bool = False ''' Validate that all keys from left JSON exist in generated JSON and that their values correspond. ''' for k, v in left_flattened.items(): if "fund.funddata" in k: i = 5 k = handle_custom_mapping(k, custom_mappings) path = "" val = clean_empty(right) val = eval( repr(val).lower() ) # set val to original right dictionary, where everything is lowercase replace_key = False for key in k.split("."): if key.isnumeric() == False and key not in val: path += key break if key in val: if isinstance(val[key], list) and len(val[key]) == 1: replace_key = True path += key + ".0." val = val[key][0] else: path += key + "." val = val[key] if key.isnumeric(): path += key + "." val = val[int(key)] if path[len(path) - 1] == ".": path = path[:len(path) - 1] # remove last "." if replace_key: k = path if k not in right_flattened: diff[k] = { "Explanation": "Key '{}' with value '{}' is not present in right JSON".format( k, v) } found_diff = True elif right_flattened[k] != v: diff[k] = { "Explanation": "Value at key '{}' does not match".format(k), "Expected": v, "Actual": right_flattened[k] } found_diff = True if found_diff: os.chdir(diff_basepath) with open('{}/{}'.format(diff_basepath, original_JSON_file), 'w') as f: f.write(json.dumps(diff, indent=2))