def fit_process(uid):
# create a new temporary model file
fd, path = tempfile.mkstemp()
# close the temporary model file descriptor as we don't need it
os.close(fd)
# give this process a dedicated session
session = Session()
try:
ftclassifier.fit(session, uid, path)
# sgdclassifier.fit(session, uid, path)
# persist the model to the database
with open(path, 'rb') as f:
classifier = f.read()
dal.update_classifier(session, uid, classifier)
session.commit()
except:
session.rollback()
raise
finally:
session.close()
Session.remove()
# delete the temporary model file
os.unlink(path)
def populate():
"""
Populates the database for the given user with sample data.
"""
try:
id_token = request.form['id_token']
uid = verify_id_token(id_token)
except KeyError:
return "id_token required", status.HTTP_400_BAD_REQUEST
except ValueError:
return "id_token unrecognized", status.HTTP_400_BAD_REQUEST
except auth.AuthError as exc:
if exc.code == 'ID_TOKEN_REVOKED':
return "id_token revoked", status.HTTP_400_BAD_REQUEST
else:
return "id_token invalid", status.HTTP_400_BAD_REQUEST
session = Session()
try:
dal.populate(session, uid)
session.commit()
except:
session.rollback()
raise
finally:
session.close()
Session.remove()
classifier.fit(uid)
return "Sample data added for user", status.HTTP_202_ACCEPTED
def stats():
"""
Get a list of all the given user's stats.
"""
try:
id_token = request.form['id_token']
uid = verify_id_token(id_token)
except KeyError:
return "id_token required", status.HTTP_400_BAD_REQUEST
except ValueError:
return "id_token unrecognized", status.HTTP_400_BAD_REQUEST
except auth.AuthError as exc:
if exc.code == 'ID_TOKEN_REVOKED':
return "id_token revoked", status.HTTP_400_BAD_REQUEST
else:
return "id_token invalid", status.HTTP_400_BAD_REQUEST
session = Session()
try:
stats = dal.get_stats(session, uid)
session.commit()
except:
session.rollback()
raise
finally:
session.close()
Session.remove()
return json.dumps(stats), status.HTTP_200_OK
def reset():
"""
Deletes all of the user's data from the database.
"""
try:
id_token = request.form['id_token']
uid = verify_id_token(id_token)
except KeyError:
return "id_token required", status.HTTP_400_BAD_REQUEST
except ValueError:
return "id_token unrecognized", status.HTTP_400_BAD_REQUEST
except auth.AuthError as exc:
if exc.code == 'ID_TOKEN_REVOKED':
return "id_token revoked", status.HTTP_400_BAD_REQUEST
else:
return "id_token invalid", status.HTTP_400_BAD_REQUEST
session = Session()
try:
dal.delete(session, uid)
session.commit()
except:
session.rollback()
raise
finally:
session.close()
Session.remove()
return "User data deleted", status.HTTP_202_ACCEPTED
def predict():
"""
Predicts the text label of every value in the given list of unlabeled text.
"""
try:
id_token = request.form['id_token']
uid = verify_id_token(id_token)
except KeyError:
return "id_token required", status.HTTP_400_BAD_REQUEST
except ValueError:
return "id_token unrecognized", status.HTTP_400_BAD_REQUEST
except auth.AuthError as exc:
if exc.code == 'ID_TOKEN_REVOKED':
return "id_token revoked", status.HTTP_400_BAD_REQUEST
else:
return "id_token invalid", status.HTTP_400_BAD_REQUEST
try:
unlabeled_text = json.loads(request.form['unlabeled_text'])
except KeyError:
return "unlabeled_text required", status.HTTP_400_BAD_REQUEST
except ValueError:
return "unlabeled_text unrecognized", status.HTTP_400_BAD_REQUEST
session = Session()
try:
predicted_labels = classifier.predict(session, uid,
list(unlabeled_text.values()))
session.commit()
except:
session.rollback()
raise
finally:
session.close()
Session.remove()
predictions = dict(zip(unlabeled_text.keys(), predicted_labels))
return json.dumps(predictions), status.HTTP_200_OK
def add():
"""
Adds the given text to the database for a user, labeled with the given
label text, and re-fits their classifier.
"""
try:
id_token = request.form['id_token']
uid = verify_id_token(id_token)
except KeyError:
return "id_token required", status.HTTP_400_BAD_REQUEST
except ValueError:
return "id_token unrecognized", status.HTTP_400_BAD_REQUEST
except auth.AuthError as exc:
if exc.code == 'ID_TOKEN_REVOKED':
return "id_token revoked", status.HTTP_400_BAD_REQUEST
else:
return "id_token invalid", status.HTTP_400_BAD_REQUEST
try:
label = request.form['label']
except KeyError:
return "label required", status.HTTP_400_BAD_REQUEST
try:
text = request.form['text']
except KeyError:
return "text required", status.HTTP_400_BAD_REQUEST
session = Session()
try:
dal.add_labeled_text(session, uid, label, text)
session.commit()
except:
session.rollback()
raise
finally:
session.close()
Session.remove()
classifier.fit(uid)
return "Labeled text added for user", status.HTTP_202_ACCEPTED
def execute_batches(batches, quiet = False):
def run_print(string):
if quiet: return
click.echo(string)
loop = asyncio.get_event_loop()
def handler_interrupt():
run_print('Interrupt Received! ')
run_print('Cancelling the jobs...')
for task in asyncio.Task.all_tasks(loop = loop):
task.cancel()
loop.add_signal_handler(signal.SIGINT, handler_interrupt)
max_ssh = int(os.environ.setdefault('ADMINWARE_MAX_SSH', '100'))
start_delay = float(os.environ.setdefault('ADMINWARE_START_DELAY', '0.2'))
pool = concurrent.futures.ThreadPoolExecutor(max_workers = max_ssh)
async def start_tasks(tasks):
active_tasks = []
def remove_done_tasks():
for active_task in active_tasks:
if active_task.finished():
active_tasks.remove(active_task)
break
async def add_tasks():
for task in tasks:
while len(active_tasks) > max_ssh:
remove_done_tasks()
await asyncio.sleep(0.01)
asyncio.ensure_future(task, loop = loop)
active_tasks.append(task)
run_print('Starting Job: {}'.format(task.node))
await(asyncio.sleep(start_delay))
async def await_finished():
while len(active_tasks) > 0:
remove_done_tasks()
await asyncio.sleep(0.01)
try:
await add_tasks()
run_print('Waiting for jobs to finish...')
finally: await await_finished()
session = Session()
try:
for batch in batches:
session.add(batch)
session.commit()
# Ensure the models are loaded from the db
batch.jobs
batch.shell_variables
run_print('Executing: {}'.format(batch.name()))
tasks = map(lambda j: j.task(thread_pool = pool), batch.jobs)
loop.run_until_complete(start_tasks(tasks))
except concurrent.futures.CancelledError: pass
finally:
run_print('Cleaning up...')
pool.shutdown(wait = True)
run_print('Saving...')
session.commit()
Session.remove()
run_print('Done')
def remove_session(sender, response, **extra):
Session.remove()
