def create_project():
config.reload()
form = ProjectForm()
ds_forms = dict([
(cls.populates, cls()) for cls in DataSourceForm.__subclasses__()])
form.datasource_type.choices = [
(cls.populates, cls.name) for cls in DataSourceForm.__subclasses__()]
form.datasource_type.choices.insert(0, (None, "-- Select Data Source --"))
new_project = Project()
if form.validate_on_submit():
form.populate_obj(new_project)
ds_form = ds_forms[form.datasource_type.data]
if ds_form.validate():
new_project.datasource = \
getattr(models, form.datasource_type.data)()
ds_form.populate_obj(new_project.datasource)
db.session.add(new_project)
db.session.commit()
flash("New project successfully created.")
return redirect(url_for('view_project', id=new_project.id))
return render_template("project/create.html",
form=form, ds_forms=ds_forms, project=new_project)
def create_project():
config.reload()
form = ProjectForm()
ds_forms = dict([(cls.populates, cls())
for cls in DataSourceForm.__subclasses__()])
form.datasource_type.choices = [(cls.populates, cls.name)
for cls in DataSourceForm.__subclasses__()]
form.datasource_type.choices.insert(0, (None, "-- Select Data Source --"))
new_project = Project()
if form.validate_on_submit():
form.populate_obj(new_project)
ds_form = ds_forms[form.datasource_type.data]
if ds_form.validate():
new_project.datasource = \
getattr(models, form.datasource_type.data)()
ds_form.populate_obj(new_project.datasource)
db.session.add(new_project)
db.session.commit()
flash("New project successfully created.")
return redirect(url_for('view_project', id=new_project.id))
return render_template("project/create.html",
form=form,
ds_forms=ds_forms,
project=new_project)
def load_user(user_id):
username = user_id
config.reload()
user = None
if config.config.has_section("auth") \
and config.config.has_option("auth", username):
user = User(username)
return user
def validate_password(form, password_field):
username = form.username.data
password = password_field.data
if not username:
return None
config.reload()
if not (config.config.has_section("auth")
and config.config.has_option("auth", username)
and config.config.get("auth", username) == hash_pwd(password)):
raise ValidationError("Wrong username or password")
def train_project(id):
try:
logging.info("Started training project %s" % id)
project = Project.query.get(id)
project.train_status = TS.TRAINING
db.session.add(project)
db.session.commit()
# Config
config.reload()
ticket_limit = int(config.general__ticket_limit)
min_class_occur = int(config.general__min_class_occur)
C = float(config.svm__coefficient)
cache_size = int(config.svm__cache_size)
# retrieve data
# TODO: add to configuration
data = project.datasource.get_data()[-ticket_limit:]
data = utils.filter_docs(data, min_class_occur=min_class_occur)
# create training model
logging.debug("Training model for project %s" % id)
selector = selectors.TFIDFDecorator(selectors.StopWordsDecorator(
selectors.BasicSelector()))
kernel = kernels.GaussianKernel()
model = models.SVMModel(
feature_selector=selector, kernel=kernel,
C=C, cache_size=cache_size)
# train model
model.train(data)
logging.debug("Model for project %s successfully trained" % id)
# create testing model
logging.debug("Training model for project %s for testing" % id)
selector_test = selectors.TFIDFDecorator(selectors.StopWordsDecorator(
selectors.BasicSelector()))
kernel_test = kernels.GaussianKernel()
model_test = models.SVMModel(
feature_selector=selector_test, kernel=kernel_test,
C=C, cache_size=cache_size)
# split data for testing
random.shuffle(data)
n = len(data)
split_pct = 8/10.0
split_x = int(np.ceil(n*split_pct))
data_train = data[:split_x]
data_test = data[split_x:]
# train testing model
model_test.train(data_train)
logging.debug("Testing model for project %s successfully trained" % id)
# testing model
project.accuracy = tests.accuracy(model_test, data_test)
project.precision, project.recall = tests.precision_and_recall(
model_test, data_test)
# save
dump_dir = os.path.join(app.config['MODEL_FOLDER'], str(id))
if not os.path.exists(dump_dir):
os.mkdir(dump_dir)
joblib.dump(model, os.path.join(dump_dir, 'svm.pkl'))
project.train_status = TS.TRAINED
project.last_training = time.time()
db.session.add(project)
db.session.commit()
logging.info("Project %s successfully trained and updated." % id)
except Exception as ex:
logging.error("Failed to train project %s" % id)
logging.exception(ex)
project.train_status = TS.FAILED
project.training_message = "Reason: %s: %s" % (
ex.__class__.__name__, ex)
db.session.add(project)
db.session.commit()
