def main():
#load data
df = load_data('../../assignment10_data/restaurants.csv', ['CAMIS','BORO','GRADE','GRADE DATE'])
df = clean_data(df) #clean data
#question 4
sum_nyc, sum_boro = grade_sum(df) #calculate sum of test_grade in nyc and in each borough
print 'The sum of test_grade in NYC is: {} \n'.format(sum_nyc)
print 'The sum of test_grade in each boroughs is: \n {}'.format(sum_boro)
#question 5
grade_overtime_plot(df, 'nyc') #grade overtime plot for nyc
#grade overtime plot for each borough
for borough in ['BRONX', 'BROOKLYN', 'MANHATTAN', 'QUEENS', 'STATEN ISLAND']:
df_boro = df[df['BORO'] == borough]
grade_overtime_plot(df_boro, borough.lower())
#question 6
df1 = load_data('../../assignment10_data/restaurants.csv', ['CAMIS','CUISINE DESCRIPTION'])
type_name = get_top_10_nyc(df1)
df2 = load_data('../../assignment10_data/restaurants.csv', ['CAMIS','CUISINE DESCRIPTION', 'GRADE', 'GRADE DATE'])
df2 = clean_data(df2)
df2 = df2[df2['CUISINE DESCRIPTION'].isin(type_name)]
df_sum = top_10_grade_overtime(df2, type_name) #calculate score overtime for each restaurant type
top_10_plot(df_sum) #score overtime plot
top_10_colormap(df_sum) #plot correlation between any two restaurant types in NYC in color map
def new_user():
"""create a new user from register page(not social authentication)"""
if request.method == 'POST':
user_name = clean_data(request.form['user_name'])
email = clean_data(request.form['email'])
print(email)
if not request.form['password'] == request.form['cnf_password']:
flash('passwords does not match')
return redirect(url_for('new_user'))
password = clean_data(request.form['password'])
if not validate_user_email(email):
flash('Email already exist, please choose other one or login')
return redirect(url_for('new_user'))
if not validate_user_password(password):
flash('password length must be 6 digit/character or more')
return redirect(url_for('new_user'))
else:
hashed_psd = hash_password(password)
user = User(name=user_name,
email=email,
password=hashed_psd,
password_setup=True)
db_session.add(user)
try:
db_session.commit()
except:
flash('Something went wrong, please try again.')
return redirect(url_for('new_user'))
flash('new user had been created')
return redirect(url_for('login'))
else:
return render_template('register.html')
def login():
"""this function implement a conventional login
"""
if request.method == 'POST':
email = clean_data(request.form['email'])
hashed_psd = hash_password(clean_data(request.form['password']))
user = User.query.filter(User.email == email).first()
if not user:
flash('User email does not exist')
return redirect(url_for('login'))
# prevent social auth client login with empty password
if not user.password_setup:
flash('Your password have not setup yet')
return redirect(url_for('register'))
if not user.password == hashed_psd:
flash('Password incorrect')
return redirect(url_for('login'))
else:
session['user_name'] = user.name
session['user_email'] = user.email
session['user_id'] = user.id
print('user id:{} had just logged in'.format(str(user.id)))
return redirect(url_for('index'))
else:
# pass current_app to the page so it can access google client id from config
app_ = current_app
return render_template('login.html', app=app_)
def new_item():
"""Create a new Item"""
if request.method == 'POST':
slug = clean_data(request.form['slug']).replace(' ', '-')
if not validate_item_slug(slug):
flash('this slug already exist, please choose other one')
return redirect(url_for('new_item'))
name = clean_data(request.form['name'])
description = clean_data(request.form['description'])
catalog_id = clean_data(request.form['catalog_id'])
user_id = session.get('user_id')
new_item = Item(name=name,
description=description,
catalog_id=catalog_id,
slug=slug,
user_id=user_id)
db_session.add(new_item)
try:
db_session.commit()
except:
flash('Something went wrong, please try again')
return redirect(url_for('new_item'))
flash("New item '{}' had been successfully created".format(name))
print("New item '{}' had been successfully created".format(name))
return redirect(url_for('console'))
else:
catalogs = Catalog.query.all()
return render_template('new_item.html', catalogs=catalogs)
def edit_item(slug):
catalogs = Catalog.query.all()
item = Item.query.filter(Item.slug == slug).first()
# validate the item belongs to current user
if not item.user_id == session.get('user_id'):
flash("you can't edit other user's item")
print("user id:{} trying to edit other user's item".format(
str(session.get('user_id'))))
return redirect(url_for('console'))
if request.method == 'POST':
new_slug = clean_data(request.form['slug'])
new_name = clean_data(request.form['name'])
new_catalog_id = int(clean_data(request.form['catalog_id']))
new_description = clean_data(request.form['description'])
user_id = session.get('user_id')
if new_slug == slug:
item.name = new_name
item.description = new_description
item.catalog_id = new_catalog_id
item.user_id = user_id
db_session.add(item)
try:
db_session.commit()
except:
flash('something went wrong, please try again')
return render_template('edit_item.html',
item=item,
catalogs=catalogs)
return redirect(url_for('console'))
# validate new slug
elif not validate_item_slug(new_slug):
flash('URL slug already exist, please choose other one')
return render_template('edit_item.html',
item=item,
catalogs=catalogs)
else:
item.name = new_name
item.slug = new_slug
item.description = new_description
item.catalog_id = new_catalog_id
item.user_id = user_id
db_session.add(item)
try:
db_session.commit()
except:
flash('something went wrong, please try again')
return render_template('edit_item.html',
item=item,
catalogs=catalogs)
flash("Item '{}' had been modified".format(new_name))
print("Item '{}' had been modified".format(new_name))
return redirect(url_for('console'))
else:
return render_template('edit_item.html', item=item, catalogs=catalogs)
def edit_catalog(catalog_name):
"""Edit Catalog"""
catalog = Catalog.query.filter(Catalog.name == catalog_name).first()
# Validate the catalog belongs to current user
if not catalog.user_id == session.get('user_id'):
flash("you can't edit other user's catalog")
print("user id:{} trying to edit other user's catalog".format(
str(session.get('user_id'))))
return redirect(url_for('console'))
if request.method == 'POST':
new_catalog_name = clean_data(request.form['name'])
# validate catalog name is unit
if not validate_catalog_name(new_catalog_name):
flash('{} already exist, please select another name'.format(
new_catalog_name))
return render_template('edit_catalog.html',
catalog_name=catalog_name)
else:
old_catalog_name = catalog.name
catalog.name = new_catalog_name
db_session.add(catalog)
try:
db_session.commit()
except:
flash('something went wrong, please try again')
return render_template('edit_catalog.html',
catalog_name=catalog_name)
flash("Catalog name '{}' had been successfully changed to '{}'".
format(old_catalog_name, new_catalog_name))
print("Catalog name '{}' had been successfully changed to '{}'".
format(old_catalog_name, new_catalog_name))
return redirect(url_for('console'))
else:
return render_template('edit_catalog.html', catalog_name=catalog_name)
def main():
#load data
df = load_data('../../assignment10_data/restaurants.csv',
['CAMIS', 'BORO', 'GRADE', 'GRADE DATE'])
df = clean_data(df) #clean data
#question 4
sum_nyc, sum_boro = grade_sum(
df) #calculate sum of test_grade in nyc and in each borough
print 'The sum of test_grade in NYC is: {} \n'.format(sum_nyc)
print 'The sum of test_grade in each boroughs is: \n {}'.format(sum_boro)
#question 5
grade_overtime_plot(df, 'nyc') #grade overtime plot for nyc
#grade overtime plot for each borough
for borough in [
'BRONX', 'BROOKLYN', 'MANHATTAN', 'QUEENS', 'STATEN ISLAND'
]:
df_boro = df[df['BORO'] == borough]
grade_overtime_plot(df_boro, borough.lower())
#question 6
df1 = load_data('../../assignment10_data/restaurants.csv',
['CAMIS', 'CUISINE DESCRIPTION'])
type_name = get_top_10_nyc(df1)
df2 = load_data('../../assignment10_data/restaurants.csv',
['CAMIS', 'CUISINE DESCRIPTION', 'GRADE', 'GRADE DATE'])
df2 = clean_data(df2)
df2 = df2[df2['CUISINE DESCRIPTION'].isin(type_name)]
df_sum = top_10_grade_overtime(
df2, type_name) #calculate score overtime for each restaurant type
top_10_plot(df_sum) #score overtime plot
top_10_colormap(
df_sum
) #plot correlation between any two restaurant types in NYC in color map
def new_catalog():
"""Create a new catalog"""
if request.method == 'POST':
name = clean_data(request.form['name'])
if Catalog.query.filter(Catalog.name == name).first():
flash('Catalog name already exist, please choose another one')
return redirect(url_for('new_catalog'))
user_id = session.get('user_id')
new_catalog = Catalog(name=name, user_id=user_id)
db_session.add(new_catalog)
try:
db_session.commit()
except:
flash('Something went wrong, please try again')
return redirect(url_for('new_catalog'))
flash("new catalog {} had successfully added".format(name))
print("new catalog {} had successfully added".format(name))
return redirect(url_for('console'))
else:
catalogs = Catalog.query.all()
return render_template('new_catalog.html', catalogs=catalogs)
data = pd.read_csv('./data/train(3).csv')
# data = pd.read_csv('./data/hw3.csv')
labels = np.array(data['label'])
features = np.array(data['feature'])
images = []
for i in range(features.shape[0]):
images.append(features[i].split(' '))
images = np.array(images, dtype=float).reshape(features.shape[0], 48, 48,
1) / 255.0
labels = ks.utils.to_categorical(labels, 7)
return images, labels
# x,y = readData()
train_data = clean_data('./data/train(3).csv')
train = train_data.feature.reshape(-1, 48, 48, 1) / 255.0
TRAIN_SIZE = 24000
x_train = train[:TRAIN_SIZE]
y_train = train_data.onehot[:TRAIN_SIZE]
x_test = train[TRAIN_SIZE:]
y_test = train_data.onehot[TRAIN_SIZE:]
model = ks.Sequential()
model.add(
ks.layers.Convolution2D(filters=64,
kernel_size=(3, 3),
input_shape=(48, 48, 1),
padding='same'))
model.add(ks.layers.Activation('relu'))
model.add(ks.layers.BatchNormalization())
from constants import DEV_MODE, IMAGE_SIZE, SIMPLE_3LAYERS_FILENAME, LENET5_CNN_FILENAME
from lasagne_neuralnet import NeuralNet1, NeuralNet2, load_model_if_exists, predict, plot_neural_net, reshape_data
from utility import clean_data, data_preview, get_label_args, load_data, save_result, scale_data
from visualization import plot_images, plot_label_distribution, plot_lasagne_learning_curves
# Count running time
starttime = datetime.datetime.now()
# Load and preview data
train, test, label = load_data(dev_mode=DEV_MODE)
data_preview(train, test, label)
plot_images(train, label)
# Fill NA with average.
# TODO: remove abnormal samples
label = clean_data(label)
label_min, label_max, label_median, label_var = get_label_args(label)
plot_label_distribution(label)
# Convert dataframe to array and normalize the data
train_array, label_array, test_array = scale_data(train, test, label)
# Train and predict
# Simple 3-layers neural nets
print "\n[Simple 3-layers neural nets]\n"
NeuralNet1 = load_model_if_exists(NeuralNet1, SIMPLE_3LAYERS_FILENAME)
NeuralNet1.fit(train_array, label_array)
NeuralNet1.save_params_to(SIMPLE_3LAYERS_FILENAME)
submission, prediction, score = predict(NeuralNet1, test_array, IMAGE_SIZE,
label.columns.values)
from matplotlib import pyplot as plt
import tensorflow as tf
from tensorflow.python.keras.models import Sequential
from tensorflow.python.keras.layers import InputLayer, Input
from tensorflow.python.keras.layers import Reshape, MaxPooling2D
from tensorflow.python.keras.layers import Conv2D, Dense, Flatten, Dropout, Activation
from tensorflow.python.keras.layers.normalization import BatchNormalization
from tensorflow.python.keras.callbacks import TensorBoard
from tensorflow.python.keras.models import Model
from sklearn.utils.class_weight import compute_class_weight
name = ['angry','disgust', 'fear', 'happy', 'sad', 'surprise', 'neutral']
train_data = clean_data('data/train.csv')
test_data = clean_data('data/test.csv', False)
train = train_data.feature.reshape((-1, 48, 48, 1))/255
train_x = train[:-2000]
train_label = train_data.label[:-2000]
train_onehot = train_data.onehot[:-2000]
test_x = train[-2000:]
test_label = train_data.label[-2000:]
test_onehot = train_data.onehot[-2000:]
class_weight = compute_class_weight(class_weight='balanced',
classes=np.unique(train_data.label),
y=train_data.label)