def upload_local_files_to_search(self):
print("uploading local files to search")
files = self.read_files_from_directory(self.vi_output_directory)
i = 0
for file in files:
path = os.path.join(self.vi_output_directory, file)
i += 1
with open(path) as f:
try:
json_object = json.load(f)
if json_object["state"] == "Processed":
parser = Parser()
intervals = parser.parse_vi_json(json_object)
intervals = list(intervals.values())
for item in intervals:
item["@search.action"] = "upload"
documents = {"value": intervals}
print(
str(i) +
f": uploading {str(file)} to search index")
self.upload_to_search(documents)
self.write_status_file(file, self.ingest_log_filename)
except ValueError:
print("could not process " + str(file))
self.write_status_file(file,
self.ingest_failure_log_filename)
def upload_files_from_storage_to_search(self):
print("uploading files from storage account to search")
vi_output_files = self.storage_client.list_files_in_container(
self.insights_container)
i = 0
for file in vi_output_files:
i += 1
try:
json_object = json.loads(
self.storage_client.get_blob_string(
self.insights_container, file.name).encode(
) # Encode as UTF-8 in case UTF-8 BOM
)
if json_object["state"] == "Processed":
parser = Parser()
intervals = parser.parse_vi_json(json_object)
intervals = list(intervals.values())
for item in intervals:
item["@search.action"] = "upload"
documents = {"value": intervals}
print(
str(i) +
f": uploading {str(file.name)} to search index")
self.upload_to_search(documents)
self.write_status_file(str(file.name),
self.ingest_log_filename)
except ValueError:
print("could not process " + str(file))
self.write_status_file(file, self.ingest_failure_log_filename)