def extract_har_request_postdata_json_attributes(self):
if self.no_entires():
return []
http_posts_data = [{
**er["postData"], "host": er["host"]
} for er in self.har_entry_requests if "postData" in er]
post_data_json_texts = [
(post_data["text"], post_data["host"])
for post_data in http_posts_data
if post_data["mimeType"].startswith("application/json")
]
post_data_json_array = []
for (json_text, host) in post_data_json_texts:
try:
post_data_json_array.append((json.loads(json_text), host))
except json.decoder.JSONDecodeError:
pass
post_data_df = pd.DataFrame(columns=["name", "value", "host"])
for (post_data_json, host) in post_data_json_array:
flattened = json_flatten.flatten(post_data_json)
for flattened_key in flattened.keys():
post_data_df = post_data_df.append(
{
"name": flattened_key,
"value": flattened[flattened_key],
"host": host,
"tracker": self.detect_tracker_by_host(host)
},
ignore_index=True)
post_data_df = post_data_df.drop_duplicates().sort_values(by="name")
# post_data_df["tracker"] = post_data_df["host"].map(self.detect_tracker)
return post_data_df.to_dict("records")
def json2csv(jsonPath, filename, TestOrTrain):
"""
df=pandas.read_json(jsonPath+'.json')
df.to_csv()
df.to_csv(r'output/'+str(jsonIndex)+'.csv', index=False)
"""
currJson = json.load(open(jsonPath + '.json'))
path = 'output/' + filename + '.csv'
if not os.path.exists(path):
with open(path, "w", newline='',
encoding='utf-8') as csvfile: # newline='' 去除空白行
writer = csv.DictWriter(csvfile, fieldnames=dict) # 写字典的方法
writer.writeheader()
for j in currJson:
curr = flatten(j) #get flattened dict
curr['width'] = 1280
curr['height'] = 720
curr = rename_keys(curr)
curr[
'filename'] = 'C:/Users/Administrator.WIN-2EPKD7D6018/Desktop/bdd100k-non/bdd100k/images/track/' + TestOrTrain + '/' + curr[
'filename'].rsplit('-', 2)[0] + '-' + curr['filename'].rsplit(
'-', 2)[1] + '/' + curr['filename']
with open(path, "a", newline='',
encoding='utf-8') as csvfile: # newline='' 一定要写，否则写入数据有空白行
writer = csv.DictWriter(csvfile, fieldnames=dict)
writer.writerow(curr) # 按行写入数据
print(filename + " write success")
def get_har_request_postdata_json_attributes(self):
http_posts_data = [{
**er["postData"], "host":
"" if "host" not in er else er["host"]
} for er in self.har_entry_requests if "postData" in er]
post_data_json_texts = [
(post_data["text"], post_data["host"])
for post_data in http_posts_data
if post_data["mimeType"].startswith("application/json")
]
json_texts_hosts_bag = db.from_sequence(post_data_json_texts)
post_data_json_array_bag = json_texts_hosts_bag.starmap(
lambda json_text, host: (self.try_json_loads(json_text), host))
post_data_json_array_bag = post_data_json_array_bag.filter(
lambda element: element[0])
flattened_json_bag = post_data_json_array_bag.starmap(
lambda json_obj, host: map(
lambda element: {
"name": element[0],
"value": element[1],
"host": host,
"tracker": self.detect_tracker_by_host(host)
}, list(json_flatten.flatten(json_obj).items())))
request_postdata_json_attributes_dict = flattened_json_bag.flatten(
).distinct(lambda o: json.dumps(o, sort_keys=True).lower())
return request_postdata_json_attributes_dict
def saveAllTrainToOneFile():
filePath = 'C:/Users/Administrator.WIN-2EPKD7D6018/Desktop/bdd100k-non/bdd100k/labels-20/box-track/train/'
path = 'output/train/train_all.csv'
csvfile = open(path, "w", newline='', encoding='utf-8')
writer = csv.DictWriter(csvfile, fieldnames=dict) # 写字典的方法
writer.writeheader()
ct = 0
for i, j, k in os.walk(filePath):
# k is ####.json
for path in k:
jsonPath = filePath + path.rsplit('.', 1)[0]
currJson = json.load(open(jsonPath + '.json'))
for j in currJson:
curr = flatten(j) # get flattened dict
curr['width'] = 1280
curr['height'] = 720
curr = rename_keys(curr)
curr['filename'] = 'C:/Users/Administrator.WIN-2EPKD7D6018/Desktop/bdd100k-non/bdd100k/images/track/train/' + \
curr['filename'].rsplit('-', 2)[0] + '-' + curr['filename'].rsplit('-', 2)[1] + '/' + \
curr['filename']
writer = csv.DictWriter(csvfile, fieldnames=dict)
writer.writerow(curr) # 按行写入数据
print(str(ct) + " write success")
ct += 1
print("All write success")
def json_to_csv(self, input_json: dict) -> str:
"""
method to handle the data conversion from JSON to CSV format
:param input_json:
:return:
"""
csv_content = json_flatten.flatten(input_json)
return csv_content
def get_data(ticker):
modules = "%2C".join([
'assetProfile', 'summaryProfile', 'summaryDetail', 'esgScores',
'price', 'incomeStatementHistory', 'incomeStatementHistoryQuarterly',
'balanceSheetHistory', 'balanceSheetHistoryQuarterly',
'cashflowStatementHistory', 'cashflowStatementHistoryQuarterly',
'defaultKeyStatistics', 'financialData', 'calendarEvents',
'secFilings', 'recommendationTrend', 'upgradeDowngradeHistory',
'institutionOwnership', 'fundOwnership', 'majorDirectHolders',
'majorHoldersBreakdown', 'insiderTransactions', 'insiderHolders',
'netSharePurchaseActivity', 'earnings', 'earningsHistory',
'earningsTrend', 'industryTrend', 'indexTrend', 'sectorTrend'
])
query_url = "https://query1.finance.yahoo.com/v10/finance/quoteSummary/{}?modules={}".format(
ticker, modules)
while not check_internet():
print("Could not connect, trying again in 5 seconds...")
time.sleep(5)
try:
with urllib.request.urlopen(query_url) as url:
parsed = json.loads(url.read().decode())
downlaod_file_name = os.path.join("stock_fundamental_data", ticker,
ticker + ".json")
downlaod_flattened_file_name = os.path.join(
"stock_fundamental_data", ticker, ticker + "_flattened.json")
os.makedirs(os.path.dirname(downlaod_file_name), exist_ok=True)
parsed = json_flatten.flatten(
list(parsed["quoteSummary"]["result"])[0])
for key, value in parsed.copy().items():
if ".raw" in key or ".longFmt" in key or value == "{}" or value.lower(
) == "none" or value == "":
del parsed[key]
elif ".fmt" in key:
parsed[key.replace(".fmt", "")] = parsed.pop(key)
with open(downlaod_flattened_file_name, "w") as f:
json.dump(parsed, f, indent=2, sort_keys=True)
parsed = json_flatten.unflatten(parsed)
with open(downlaod_file_name, "w") as f:
json.dump(parsed, f, indent=2, sort_keys=True)
print("Downloaded {}".format(ticker))
except:
print("Data of {} doesn't exist".format(ticker))
return
def test_nested_json(self):
self.assertEqual(
json_flatten.flatten({
"a": 1,
"b": {
"c": "hello",
"d": 4
},
"e": true
}), '{"a": 1, "b.c": "hello", "b.d": 4, "e": true}')
def test_double_layered_nested_json(self):
self.assertEqual(
json_flatten.flatten({
"a": 1,
"b": {
"c": "hello",
"d": {
"e": true,
"f": 6
},
"g": "seven"
}
}),
'{"a": 1, "b.c": "hello", "b.d.e": true, "b.d.f": 6, "b.g": "seven"}'
)
def test_2_nested_json(self):
self.assertEqual(
json_flatten.flatten({
"a": 1,
"b": {
"c": "hello",
"d": 4
},
"e": {
"f": false,
"g": "example"
}
}),
'{"a": 1, "b.c": "hello", "b.d": 4, "e.f": false, "e.g": "example"}'
)
def xml_to_csv(self, input_xml: str) -> dict:
"""
Conversion for XML format to CSV format.
For this conversion we will convert the XML into JSON first and then we
will be converting that JSON to CSV format.
Approach: XML -> JSON -> CSV
:param input_xml:
:return:
"""
converted_json = xmltodict.parse(input_xml)
converted_csv = json_flatten.flatten(converted_json)
return {
'columns': converted_csv.keys(),
'values': converted_csv.values()
}
def review_numeric_lines(mid_lines):
numeric_script = generate_script(mid_lines)
schema = mock_schema()
data = generate_mock_data(model=schema, numeric_fields=True)
flattened_data = json_flatten.flatten(data)
handle_numeric = lua.eval(numeric_script)
handle_numeric(flattened_data)
schema = mock_schema()
for field in schema:
_type = schema[field]['type']
if _type in numeric_types:
nodes = field.split('.')
if len(nodes) == 1:
assert flattened_data[field]
else:
assert not flattened_data[field]
if field[0] == 'p':
assert not flattened_data[field], field
def PersonFieldsAndValues(self, person_id):
# get the person's NB record
response = requests.get(self.PERSON_URL %
(person_id, NATIONBUILDER_API_TOKEN),
timeout=self.default_timeout)
self._process_response(response)
# return field_path = value if the person was found
if response.status_code == 200:
record = response.json()
flat = flatten(record)
flat_list = []
for key, value in flat.items():
key_without_type = key if key.find(
'$') == -1 else key[:key.find('$')]
value_without_none = value if value != 'None' else ''
flat_list.append(
(key_without_type, value_without_none,
"%s = %s" % (key_without_type, value_without_none)))
flat_list.sort(key=lambda tup: tup[0])
return (flat_list)
else:
return None
def get_incident_list():
if 'tags_path' not in session:
session['tags_path'] = {}
subdomain = session["subdomain"]
create_folder('resources/' + subdomain)
save_path = getcwd()
complete_name_tags = path.join(save_path,
"resources/%s/tags.json" % subdomain)
try:
with open(complete_name_tags) as json_file:
tags = load(json_file)
except IOError:
tags = {}
with open(complete_name_tags, 'w+') as outfile:
dump(tags, outfile)
complete_name_events = path.join(save_path,
"resources/%s/incidents.json" % subdomain)
try:
with open(complete_name_events) as json_file:
incidents = load(json_file)
print("Found incidents. Loading the file")
except IOError:
incidents = []
with open(complete_name_events, 'w+') as outfile:
dump(incidents, outfile)
print("Couldn't find incidents. Dumping empty list.")
if not incidents:
for i in range(len(session["selected_services_id"])):
incidents = incidents + incident_iter_selected(
session['global_token'], session['selected_windows'][i],
[session['selected_services_id'][i]], session["integrations"],
tags)
subdomain = session["subdomain"]
create_folder('resources/' + subdomain)
save_path = getcwd()
complete_name = path.join(save_path,
"resources/%s/combos.json" % subdomain)
try:
with open(complete_name) as json_file:
combos = load(json_file)
except IOError:
combos = []
with open(complete_name, 'w+') as outfile:
dump(combos, outfile)
keys = []
flattened_keys = []
flattened_incidents = []
ftle = []
ftle_keys = []
channels = []
channels_keys = []
alerts = []
alerts_keys = []
for incident in incidents:
incident_temp = deepcopy(incident)
ftle_temp = incident_temp.pop("first_trigger_log_entry")
# ftle_channel = ftle_temp.pop("channel")
alerts_temp = incident_temp.pop("all_alerts")
flattened_incidents.append(flatten(incident_temp))
flattened_keys = list(
set(flattened_keys + list(flatten(incident_temp).keys())))
ftle.append(flatten(ftle_temp))
ftle_keys = list(set(ftle_keys + list(flatten(ftle_temp).keys())))
# channels.append(flatten(ftle_channel))
# channels_keys = list(set(channels_keys + list(flatten(ftle_channel).keys())))
alerts.append(flatten(alerts_temp))
alerts_keys = list(set(alerts_keys +
list(flatten(alerts_temp).keys())))
flattened_keys.sort()
alerts_keys.sort()
ftle_keys.sort()
session["flattened_keys"] = flattened_keys
session["ftle_keys"] = ftle_keys
session['alerts_keys'] = alerts_keys
disco_choice = session["disco_choice"]
subdomain = session["subdomain"]
create_folder('resources/' + subdomain)
save_path = getcwd()
complete_name = path.join(save_path, "resources/%s/tags.json" % subdomain)
with open(complete_name, 'w+') as outfile:
dump(tags, outfile)
if not incidents:
flash(
"No incidents found in the selected window. Please select a larger window!"
)
return redirect(url_for('get_service_config'))
complete_name_events = path.join(save_path,
"resources/%s/incidents.json" % subdomain)
with open(complete_name_events, 'w+') as outfile:
dump(incidents, outfile)
return render_template('incident_list.html',
incidents=flattened_incidents,
keys=flattened_keys,
alerts=alerts,
alerts_keys=alerts_keys,
ftle=ftle,
ftle_keys=ftle_keys,
channels=channels,
channels_keys=channels_keys,
disco_choice=disco_choice,
tags=tags,
provisioned=session['provisioned'],
tags_exist=session['tags_exist'],
subdomain=session['subdomain'],
incidents_exist=session['incidents_exist'])
def test_flatten_unflatten(test_name, unflattened, flattened):
assert flatten(unflattened) == flattened
assert unflatten(flattened) == unflattened
def post_incident_list():
tags_path = session['tags_path']
tag_action = request.form["tag_action"]
subdomain = session["subdomain"]
create_folder('resources/' + subdomain)
save_path = getcwd()
complete_name_tags = path.join(save_path,
"resources/%s/tags.json" % subdomain)
try:
with open(complete_name_tags) as json_file:
tags = load(json_file)
except IOError:
tags = {}
with open(complete_name_tags, 'w+') as outfile:
dump(tags, outfile)
if tag_action == "delete":
# tags = session["tags"]
tag_keys = list(tags.keys())
for tag in tag_keys:
if request.form.get("tag-%s" % tag):
del tags[tag]
elif tag_action == "update":
flattened_keys = session["flattened_keys"]
ftle_keys = session["ftle_keys"]
alerts_keys = session['alerts_keys']
incidents = []
# tags = session["tags"]
disco_choice = session["disco_choice"]
has_tags = disco_choice != "3"
for i in range(len(session["selected_services_id"])):
incidents = incidents + incident_iter_selected(
session['global_token'], session['selected_windows'][i],
[session['selected_services_id'][i]],
session["integrations"][i], tags)
tagged_keys = [[], []]
for i in range(len(flattened_keys)):
if request.form.get("incident_field_%s" % str(i)):
incident_field = request.form.get("incident_field_%s" % str(i))
tagged_keys[0].append(incident_field)
if incident_field not in tags:
tags[incident_field] = []
for i in range(len(ftle_keys)):
if request.form.get("ftle_field_%s" % str(i)):
ftle_field = request.form.get("ftle_field_%s" % str(i))
tagged_keys[1].append(ftle_field)
if ftle_field not in tags:
tags[ftle_field] = []
for incident in incidents:
incident_temp = incident
ftle_temp = incident_temp.pop("first_trigger_log_entry")
flattened_incident = flatten(incident_temp)
ftle = flatten(ftle_temp)
for key in tagged_keys[0]:
if key in flattened_incident and flattened_incident[
key] not in tags[key]:
tags[key].append(flattened_incident[key])
for key in tagged_keys[1]:
if key in ftle and ftle[key] not in tags[key]:
tags[key].append(ftle[key])
subdomain = session["subdomain"]
create_folder('resources/' + subdomain)
save_path = getcwd()
complete_name = path.join(save_path, "resources/%s/tags.json" % subdomain)
with open(complete_name, 'w+') as outfile:
dump(tags, outfile)
return redirect(url_for('get_incident_list'))
def test_flat_json(self):
self.assertEqual(json_flatten.flatten({
"a": 1,
"b": true
}), '{"a": 1, "b": true}')
]
# --------------------------------- Create Window ---------------------------------
window = sg.Window('JSON Translator', layout)
# ----- Run the Event Loop -----
# --------------------------------- Event Loop ---------------------------------
while True:
event, values = window.read()
if event in (None, 'Exit'):
break
# new source file selected
if event == '-SOURCE-':
# clear the keys list
keyList.update(values=[])
sourceDict = jf.flatten(getDictFromJSON(sourceFile.get()))
sourceDictFlat = (sourceDict)
keyList.update(values=list(sourceDictFlat.keys()))
sourceLocale.update(value=getLocaleFromFileName(sourceFile.get()), visible=True)
if event == '-TARGET-':
targetDict = getDictFromJSON(targetFile.get())
targetDictFlat = jf.flatten(targetDict)
targetLocale.update(value=getLocaleFromFileName(targetFile.get()), visible=True)
if event == '-TARGETTEXT-':
# auto save every time user types
if autoSaveCheck.get() == 1:
saveTarget()
unsavedChanges = False
else:
from json_flatten import flatten
import json
# Print json object as flattened one
unflat_json = {
'user': {
'Rachel': {
'UserID': 1717171717,
'Email': '*****@*****.**',
'friends': ['John', 'Jeremy', 'Emily']
}
}
}
flat_json = flatten(unflat_json)
print(flat_json)
#Read a json file and print as flattened one
with open('mydata.json') as f:
data = json.load(f)
myflatdata = flatten(data)
print(myflatdata)
# write to a file
with open("flatten_file.json", "w") as write_file:
json.dump(myflatdata, write_file)
#print each property separate with = intead of : and quotoes and as a property
for x in myflatdata:
print("%s=%s" % (x, myflatdata[x]))
# Write json as properties to a file
def json_diff(original_JSON_file: str) -> dict:
'''
Compute difference between left and right JSON objects
:param dict left: left JSON object (original JSON)
:param dict right: right JSON object (JSON generated by MSP)
:param dict custom_mappings: JSON object detailing left to right custom mappings
:return difference between left and right JSON objects
:rtype: dict
'''
print("Computing diff for: " + original_JSON_file)
left: dict = {}
right: dict = {}
os.chdir(original_basepath)
with open('{}/{}'.format(original_basepath, original_JSON_file), 'r') as f:
left = json.loads(f.read())
os.chdir(new_basebath)
with open('{}/{}'.format(new_basebath, original_JSON_file), 'r') as f:
right = json.loads(f.read())
# pickup "PayLoad" section of left JSON for comparison
left: dict = left["PayLoad"]
# pickup "payLoad" section of right JSON for comparison
right: dict = right["eventMessage"]["eventAttributes"]["dataElements"][
"payLoad"]
# print("Flattening 'left' JSON...")
left_flattened = json_flatten.flatten(left)
# print("Convert 'left' JSON keys to lowercase...")
left_flattened = {k.lower(): v for k, v in left_flattened.items()}
# print("Flattening 'right' JSON...")
right_flattened = json_flatten.flatten(right)
# print("Convert 'right' JSON keys to lowercase...")
# right_flattened = {k.lower(): v for k, v in right_flattened.items() if v != "" and v != 'None'}
right_flattened = {k.lower(): v for k, v in right_flattened.items()}
diff = {}
# print("original JSON size = " + str(len(left_flattened)))
# print("new JSON size = " + str(len(right_flattened)))
diff["DiffStatistics"] = {
"OriginalJSONSize": str(len(left_flattened)),
"NewJSONSize": len(right_flattened),
"Delta": abs(len(left_flattened) - len(right_flattened))
}
found_diff: bool = False
'''
Validate that all keys from left JSON exist in generated JSON
and that their values correspond.
'''
for k, v in left_flattened.items():
if "fund.funddata" in k:
i = 5
k = handle_custom_mapping(k, custom_mappings)
path = ""
val = clean_empty(right)
val = eval(
repr(val).lower()
) # set val to original right dictionary, where everything is lowercase
replace_key = False
for key in k.split("."):
if key.isnumeric() == False and key not in val:
path += key
break
if key in val:
if isinstance(val[key], list) and len(val[key]) == 1:
replace_key = True
path += key + ".0."
val = val[key][0]
else:
path += key + "."
val = val[key]
if key.isnumeric():
path += key + "."
val = val[int(key)]
if path[len(path) - 1] == ".":
path = path[:len(path) - 1] # remove last "."
if replace_key:
k = path
if k not in right_flattened:
diff[k] = {
"Explanation":
"Key '{}' with value '{}' is not present in right JSON".format(
k, v)
}
found_diff = True
elif right_flattened[k] != v:
diff[k] = {
"Explanation": "Value at key '{}' does not match".format(k),
"Expected": v,
"Actual": right_flattened[k]
}
found_diff = True
if found_diff:
os.chdir(diff_basepath)
with open('{}/{}'.format(diff_basepath, original_JSON_file), 'w') as f:
f.write(json.dumps(diff, indent=2))
