def _write_flat(self,
profile: DatasetProfile,
indent: int = 4,
rotation_suffix: Optional[str] = None):
"""
Write output data for flat format
Parameters
----------
profile : DatasetProfile
the dataset profile to output
indent : int
The JSON indentation to use. Default is 4
"""
summary = profile.to_summary()
flat_table_path = os.path.join(self.output_path,
self.path_suffix(profile), "flat_table")
summary_df = get_dataset_frame(summary)
with open(
os.path.join(flat_table_path,
self.file_name(profile, ".csv", rotation_suffix)),
"wt") as f:
summary_df.to_csv(f, index=False)
json_flat_file = self.file_name(profile, ".json")
_suffix = rotation_suffix or ""
frequent_numbers_path = os.path.join(self.output_path,
self.path_suffix(profile),
f"freq_numbers{_suffix}")
with open(os.path.join(frequent_numbers_path, json_flat_file),
"wt") as f:
hist = flatten_dataset_histograms(summary)
json.dump(hist, f, indent=indent)
frequent_strings_path = os.path.join(self.output_path,
self.path_suffix(profile),
f"frequent_strings{_suffix}")
with open(os.path.join(frequent_strings_path, json_flat_file),
"wt") as f:
frequent_strings = flatten_dataset_frequent_strings(summary)
json.dump(frequent_strings, f, indent=indent)
histogram_path = os.path.join(self.output_path,
self.path_suffix(profile),
f"histogram{_suffix}")
with open(os.path.join(histogram_path, json_flat_file), "wt") as f:
histogram = flatten_dataset_histograms(summary)
json.dump(histogram, f, indent=indent)
def _write_json(self, profile: DatasetProfile):
"""
Write a dataset profile JSON summary to disk
"""
output_file = os.path.join(
self.output_path,
self.path_suffix(profile),
"json",
self.file_name(profile, ".json"),
)
summary = profile.to_summary()
with self.fs.open(output_file, "wt") as f:
f.write(message_to_json(summary))
def _write_json(self, profile: DatasetProfile):
"""
Write a JSON summary of the dataset profile to disk
"""
path = self.ensure_path(os.path.join(self.path_suffix(profile), "json"))
output_file = os.path.join(path, self.file_name(profile, ".json"))
path = os.path.join(self.output_path, self.path_suffix(profile))
os.makedirs(path, exist_ok=True)
# TODO: only calculate this summary once. No need to calculate it for
# _write_flat() as well
summary = profile.to_summary()
with open(output_file, "wt") as f:
f.write(message_to_json(summary))
def _write_json(self,
profile: DatasetProfile,
rotation_suffix: Optional[str] = None,
transport_params: Optional[dict] = None):
"""
Write a dataset profile JSON summary to disk
"""
output_file = os.path.join(
self.output_path,
self.path_suffix(profile),
"json",
self.file_name(profile, ".json", rotation_suffix),
)
summary = profile.to_summary()
with open(output_file, "wt", transport_params=transport_params) as f:
f.write(message_to_json(summary))
def _write_flat(self, profile: DatasetProfile, indent: int = 4):
"""
Write output data for flat format
Parameters
----------
profile : DatasetProfile
the dataset profile to output
indent : int
The JSON indentation to use. Default is 4
"""
# TODO: only calculate this summary once. No need to calculate it for
# _write_json() as well
summary = profile.to_summary()
flat_table_path = self.ensure_path(
os.path.join(self.path_suffix(profile), "flat_table")
)
summary_df = get_dataset_frame(summary)
summary_df.to_csv(
os.path.join(flat_table_path, self.file_name(profile, ".csv")), index=False
)
frequent_numbers_path = self.ensure_path(
os.path.join(self.path_suffix(profile), "freq_numbers")
)
json_flat_file = self.file_name(profile, ".json")
with open(os.path.join(frequent_numbers_path, json_flat_file), "wt") as f:
hist = flatten_dataset_frequent_numbers(summary)
json.dump(hist, f, indent=indent)
frequent_strings_path = self.ensure_path(
os.path.join(self.path_suffix(profile), "frequent_strings")
)
with open(os.path.join(frequent_strings_path, json_flat_file), "wt") as f:
frequent_strings = flatten_dataset_frequent_strings(summary)
json.dump(frequent_strings, f, indent=indent)
histogram_path = self.ensure_path(
os.path.join(self.path_suffix(profile), "histogram")
)
with open(os.path.join(histogram_path, json_flat_file), "wt") as f:
histogram = flatten_dataset_histograms(summary)
json.dump(histogram, f, indent=indent)