def _initialize_tagger(self, params):
self.info(
"Initializing Maui Server tagger '{}'".format(
self.tagger(params)))
# try to delete the tagger in case it already exists
resp = requests.delete(self.tagger_url(params))
self.debug("Trying to delete tagger {} returned status code {}"
.format(self.tagger(params), resp.status_code))
# create a new tagger
data = {'id': self.tagger(params), 'lang': params['language']}
json = {}
try:
resp = requests.post(self.endpoint(params), data=data)
self.debug("Trying to create tagger {} returned status code {}"
.format(self.tagger(params), resp.status_code))
try:
json = resp.json()
except ValueError:
pass
resp.raise_for_status()
except requests.exceptions.RequestException as err:
msg = "Creating tagger failed: {}".format(json)
raise OperationFailedException(msg) from err
def _upload_vocabulary(self):
self.info("Uploading vocabulary")
try:
resp = requests.put(self.tagger_url + '/vocab',
data=self.project.vocab.as_skos())
resp.raise_for_status()
except requests.exceptions.RequestException as err:
raise OperationFailedException(err)
def _get_project_info(self, key):
params = self._get_backend_params(None)
try:
resp = requests.get(self.tagger_url(params) + '/train')
resp.raise_for_status()
except requests.exceptions.RequestException as err:
msg = f"HTTP request failed: {err}"
raise OperationFailedException(msg) from err
try:
response = resp.json()
except ValueError as err:
msg = f"JSON decode failed: {err}"
raise OperationFailedException(msg) from err
if key in response:
return response[key]
else:
return None
def _get_project_info(self, key):
params = self._get_backend_params(None)
try:
req = requests.get(params['endpoint'].replace('/suggest', ''))
req.raise_for_status()
except requests.exceptions.RequestException as err:
msg = f"HTTP request failed: {err}"
raise OperationFailedException(msg) from err
try:
response = req.json()
except ValueError as err:
msg = f"JSON decode failed: {err}"
raise OperationFailedException(msg) from err
if key in response:
return response[key]
else:
return None
def _upload_train_file(self):
self.info("Uploading training documents")
train_path = os.path.join(self.datadir, self.TRAIN_FILE)
with open(train_path, 'rb') as train_file:
try:
resp = requests.post(self.tagger_url + '/train',
data=train_file)
resp.raise_for_status()
except requests.exceptions.RequestException as err:
raise OperationFailedException(err)
def train(self, train_x, train_y, params):
# fit the model on the training corpus
self._classifier = self._create_classifier(params)
self._classifier.fit(train_x, train_y)
# sanity check: verify that the classifier has seen both classes
if self._classifier.n_classes_ != 2:
raise OperationFailedException(
"Unable to create classifier: " +
"Not enough positive and negative examples " +
"in the training data. Please check that your training " +
"data matches your vocabulary.")
def _wait_for_train(self):
self.info("Waiting for training to be completed...")
while True:
try:
resp = requests.get(self.tagger_url + "/train")
resp.raise_for_status()
except requests.exceptions.RequestException as err:
raise OperationFailedException(err)
response = resp.json()
if response['completed']:
self.info("Training completed.")
return
time.sleep(1)
def _upload_vocabulary(self, params):
self.info("Uploading vocabulary")
json = {}
try:
resp = requests.put(self.tagger_url(params) + '/vocab',
data=self.project.vocab.as_skos())
try:
json = resp.json()
except ValueError:
pass
resp.raise_for_status()
except requests.exceptions.RequestException as err:
msg = "Uploading vocabulary failed: {}".format(json)
raise OperationFailedException(msg) from err
def _initialize_model(self):
if self._model is None:
path = os.path.join(self.datadir, self.MODEL_FILE)
self.debug('loading model from {}'.format(path))
if os.path.exists(path):
try:
self._model = omikuji.Model.load(path)
except RuntimeError:
raise OperationFailedException(
"Omikuji models trained on Annif versions older than "
"0.56 cannot be loaded. Please retrain your project.")
else:
raise NotInitializedException(
'model {} not found'.format(path),
backend_id=self.backend_id)
def _upload_train_file(self, params):
self.info("Uploading training documents")
json = {}
train_path = os.path.join(self.datadir, self.TRAIN_FILE)
with open(train_path, 'rb') as train_file:
try:
resp = requests.post(self.tagger_url(params) + '/train',
data=train_file)
try:
json = resp.json()
except ValueError:
pass
resp.raise_for_status()
except requests.exceptions.RequestException as err:
msg = "Uploading training documents failed: {}".format(json)
raise OperationFailedException(msg) from err
def _wait_for_train(self, params):
self.info("Waiting for training to be completed...")
json = {}
while True:
try:
resp = requests.get(self.tagger_url(params) + "/train")
resp.raise_for_status()
except requests.exceptions.RequestException as err:
msg = "Training failed: {}".format(json)
raise OperationFailedException(msg) from err
response = resp.json()
if response['completed']:
self.info("Training completed.")
return
time.sleep(1)
def _initialize_tagger(self):
self.info("Initializing Maui Service tagger '{}'".format(self.tagger))
# try to delete the tagger in case it already exists
resp = requests.delete(self.tagger_url)
self.debug("Trying to delete tagger {} returned status code {}"
.format(self.tagger, resp.status_code))
# create a new tagger
data = {'id': self.tagger, 'lang': self.params['language']}
try:
resp = requests.post(self.endpoint, data=data)
self.debug("Trying to create tagger {} returned status code {}"
.format(self.tagger, resp.status_code))
resp.raise_for_status()
except requests.exceptions.RequestException as err:
raise OperationFailedException(err)