def upload_local_files_to_search(self): print("uploading local files to search") files = self.read_files_from_directory(self.vi_output_directory) i = 0 for file in files: path = os.path.join(self.vi_output_directory, file) i += 1 with open(path) as f: try: json_object = json.load(f) if json_object["state"] == "Processed": parser = Parser() intervals = parser.parse_vi_json(json_object) intervals = list(intervals.values()) for item in intervals: item["@search.action"] = "upload" documents = {"value": intervals} print( str(i) + f": uploading {str(file)} to search index") self.upload_to_search(documents) self.write_status_file(file, self.ingest_log_filename) except ValueError: print("could not process " + str(file)) self.write_status_file(file, self.ingest_failure_log_filename)
def upload_files_from_storage_to_search(self): print("uploading files from storage account to search") vi_output_files = self.storage_client.list_files_in_container( self.insights_container) i = 0 for file in vi_output_files: i += 1 try: json_object = json.loads( self.storage_client.get_blob_string( self.insights_container, file.name).encode( ) # Encode as UTF-8 in case UTF-8 BOM ) if json_object["state"] == "Processed": parser = Parser() intervals = parser.parse_vi_json(json_object) intervals = list(intervals.values()) for item in intervals: item["@search.action"] = "upload" documents = {"value": intervals} print( str(i) + f": uploading {str(file.name)} to search index") self.upload_to_search(documents) self.write_status_file(str(file.name), self.ingest_log_filename) except ValueError: print("could not process " + str(file)) self.write_status_file(file, self.ingest_failure_log_filename)