def bulk_create_courses(self, documents):
try:
url = (
self.url
+ "/indexes/"
+ self.index_name
+ "/docs/index"
+ self.query_string
)
logging.info(f"url: {url}")
response = requests.post(url, headers=self.headers, json=documents)
except requests.exceptions.RequestException as e:
logging.exception("unexpected error loading bulk index", exc_info=True)
raise exceptions.StopEtlPipelineErrorException(e)
if response.status_code != 200:
logging.error(
f"failed to bulk load course search documents\n\
index-name: {self.index_name}\n\
status: {response.status_code}\n\
error: {requests.exceptions.HTTPError(response.text)}"
)
raise exceptions.StopEtlPipelineErrorException()
def delete_if_already_exists(self):
try:
delete_url = self.url + "/indexes/" + self.index_name + self.query_string
response = requests.delete(delete_url, headers=self.headers)
except requests.exceptions.RequestException as e:
logging.exception("unexpected error deleting index", exc_info=True)
raise exceptions.StopEtlPipelineErrorException(e)
if response.status_code == 204:
logging.warn(
f"course search index already exists, successful deletion\
prior to recreation\n index: {self.index_name}"
)
elif response.status_code != 404:
# 404 is the expected response, because normally the
# course search index will not exist
logging.error(
f"unexpected response when deleting existing search index,\
search_response: {response.json()}\nindex-name:\
{self.index_name}\nstatus: {response.status_code}"
)
raise exceptions.StopEtlPipelineErrorException
def get_version(dataset_path):
version = None
try:
split_path_name = dataset_path.split("/")
split_name = split_path_name[1].split("-")
version = split_name[2]
except IndexError:
logging.error(
f"unable to extract version from storage container path, \
dataset_path: {dataset_path}\n"
)
raise exceptions.StopEtlPipelineErrorException(IndexError)
return version
def create(self):
try:
create_url = self.url + "/synonymmaps" + self.query_string
response = requests.put(
create_url, headers=self.headers, json=self.course_synonym_schema
)
except requests.exceptions.RequestException as e:
logging.exception("unexpected error creating course synonym", exc_info=True)
raise exceptions.StopEtlPipelineErrorException(e)
if response.status_code != 201:
logging.error(
f"failed to create course search synonyms, after failing to update synonyms\n\
synonym-name: {self.synonym_name}\n\
status: {response.status_code}\n\
error: {requests.exceptions.HTTPError(response.text)}"
)
raise exceptions.StopEtlPipelineErrorException
def update(self):
self.get_synonym()
try:
update_url = (self.url + "/synonymmaps/" + self.synonym_name +
self.query_string)
response = requests.put(
update_url,
headers=self.headers,
json=self.course_synonym_schema,
)
except requests.exceptions.RequestException as e:
logging.exception("unexpected error creating course synonym",
exc_info=True)
raise exceptions.StopEtlPipelineErrorException(e)
if response.status_code == 201:
# PUT can return 201 if a new resource is created
return
if response.status_code == 204:
return
if response.status_code == 404:
logging.warning(
f"failed to update course search synonyms, unable to find synonym; try creating synonym\n\
synonym-name: {self.synonym_name}\n\
status: {response.status_code}\n\
error: {requests.exceptions.HTTPError(response.text)}"
)
self.create()
else:
logging.error(f"failed to update course search synonyms\n\
synonym-name: {self.synonym_name}\n\
status: {response.status_code}\n\
error: {requests.exceptions.HTTPError(response.text)}"
)
raise exceptions.StopEtlPipelineErrorException
def create(self):
self.get_index()
try:
create_url = self.url + "/indexes" + self.query_string
response = requests.post(create_url,
headers=self.headers,
json=self.course_schema)
except requests.exceptions.RequestException as e:
logging.exception("unexpected error creating index", exc_info=True)
raise exceptions.StopEtlPipelineErrorException(e)
if response.status_code != 201:
logging.error(f"failed to create search index\n\
index-name: {self.index_name}\n\
status: {response.status_code}\n\
error: {requests.exceptions.HTTPError(response.text)}"
)
raise exceptions.StopEtlPipelineErrorException