def FoodDetection(self):
dir_path = os.getcwd()
print(dir_path)
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
MEAN = np.array([51.072815, 51.072815, 51.072815])
STD = np.array([108.75629, 92.98068, 85.61884])
categories = [
'healthy', 'junk', 'dessert', 'appetizer', 'mains', 'soups',
'carbs', 'protein', 'fats', 'meat'
]
"""model = build_model('inference', model_path=os.path.join(dir_path, "mobilenet - Copy.h5"))"""
model = build_model('train',
model_path=os.path.join(dir_path,
"mobilenet - Copy.h5"))
img = np.expand_dims(cv2.resize(cv2.imread(
os.path.join(dir_path, "cake4.jpg"), 1),
dsize=(224, 224),
interpolation=cv2.INTER_CUBIC),
axis=0)
for i in range(3):
img[:, :, :, i] = (img[:, :, :, i] - MEAN[i]) / STD[i]
predict1 = np.round(model.predict(img))
prediction = np.round(model.predict(img)[0])
labels = [
categories[idx]
for idx, current_prediction in enumerate(prediction)
if current_prediction == 1
]
print(predict1)
return labels
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
MEAN = np.array([1.1368206, 1.1368206, 1.1368206])
STD = np.array([17.059486, 17.059486, 17.059486])
categories = []
for i in range(1000):
categories.append(i)
parser = argparse.ArgumentParser()
parser.add_argument('--saved_model',
help='Path of the saved model',
required=True)
args = parser.parse_args()
model = build_model('inference', model_path=args.saved_model)
testX, testname = load_testdata()
print("test loaded")
testX = testX.astype(np.float32)
for i in range(3):
testX[:, :, :, i] = (testX[:, :, :, i] - MEAN[i]) / STD[i]
f = open('test.csv', 'w')
csv_writer = csv.writer(f)
csv_writer.writerow(["id", "predicted"])
res = model.predict(testX).round()
for i in range(len(res)):
def model_fn(features, labels, mode, params):
""" """
LR = params['learning_rate']
batch_size = params['batch_size']
nbr_classes = params['nbr_classes']
class_weights = params['class_weights']
class_weights = np.array(class_weights)
class_weights = np.tile(class_weights, (batch_size, 1))
# from Cerebras MNIST hybrid_model.py
# tf.set_random_seed(0) # --seed arg not yet implemented
loss = None
train_op = None
logging_hook = None
training_hook = None
eval_metric_ops = None
logging_op = None
# living in the past?
get_or_create_global_step_fn = tf.train.get_or_create_global_step
get_global_step_fn = tf.train.get_global_step
get_collection_fn = tf.get_collection
set_verbosity_fn = tf.logging.set_verbosity
optimizer_fn = tf.train.MomentumOptimizer
accuracy_fn = tf.metrics.accuracy
loss_fn = tf.losses.sparse_softmax_cross_entropy # see loss_fn below
logging_INFO = tf.logging.INFO
GraphKeys = tf.GraphKeys
summary_scalar = tf.summary.scalar
is_training = (mode == tf.estimator.ModeKeys.TRAIN)
is_evaluate = (mode == tf.estimator.ModeKeys.EVAL)
is_predict = (mode == tf.estimator.ModeKeys.PREDICT)
inputs = features #### changed to singleton return RRT 3/11 ['data']
keras_model = build_model(params, tensor=inputs)
logits = keras_model.output
predictions = tf.argmax(logits, 1)
global_step = get_or_create_global_step_fn()
loss = loss_fn(labels=labels, logits=logits)
#pdb.set_trace()
#loss = loss_fn(labels=labels, logits=logits, weights=class_weights)
hook_list = []
accuracy = accuracy_fn(labels=labels,
predictions=predictions,
name='accuracy_op')
eval_metric_ops = dict(accuracy=accuracy)
summary_scalar('accuracy', accuracy[1])
set_verbosity_fn(logging_INFO)
#GC
"""
logging_hook = tf.estimator.LoggingTensorHook(
{"loss": loss, "accuracy": accuracy[1]},
every_n_iter = 1000) #### every_n_secs = 60)
hook_list.append(logging_hook)
"""
#end GC
if is_training:
optimizer = optimizer_fn(learning_rate=LR, momentum=0.9)
update_ops = get_collection_fn(GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
train_op = optimizer.minimize(loss)
#GC
training_hook = hooks.TrainingHook(params)
logging_op = training_hook.enqueue({
"loss": loss,
"accuracy": accuracy[1]
}) #Add loss to outfeed
training_op = tf.group([train_op, logging_op
]) #Ensure logging_op is attached to graph?
#end GC
hook_list.append(training_hook)
estimator = tf.estimator.EstimatorSpec(mode=mode,
predictions=predictions,
loss=loss,
train_op=training_op,
eval_metric_ops=eval_metric_ops,
training_hooks=hook_list)
return estimator
def train_and_evaluate(args):
start_time = time()
url = 'http://35.222.54.209:5000/' + args.ticker
#url = 'https://backend-dot-deep-stock-268818.appspot.com/' + args.ticker
try:
hist_data = requests.get(url + '/historicaldata')
twit_data = requests.get(url + '/tweet_sentiments')
except requests.exceptions.RequestException as e:
raise SystemExit(e)
twit_data = twit_data.json()
clean_twit = []
for twit in twit_data:
clean_twit.append({
'date': (datetime.timestamp(
datetime.strptime(twit['tweets']['date'], "%Y-%m-%d"))) / 100,
'sent':
twit['tweets']['day_sentiment']
})
hist_data = hist_data.json()
df_hist = pd.DataFrame(hist_data)
df_hist['date'] = df_hist['date'].map(lambda x: x['$date'] / 100000)
df_hist = df_hist.sort_values('date')
df_twit = pd.DataFrame(clean_twit)
df = pd.merge(df_hist, df_twit, on=['date'], how='left')
df['sent'] = df['sent'].fillna(0)
df['PrevWeekClose'] = df['Close'].shift(7)
df['WeekReturn'] = (df['Close'] -
df['PrevWeekClose']) / df['PrevWeekClose']
input_data = df[['Open', 'High', 'Low', 'Close', 'sent']].values
targets = df['WeekReturn'].values
print(input_data.shape)
T = 7
D = input_data.shape[1]
N = len(input_data) - T
Ntrain = len(input_data) * 7 // 8
scaler = StandardScaler()
scaler.fit(input_data[:Ntrain + T - 1])
input_data = scaler.transform(input_data)
x_train = np.zeros((Ntrain, T, D))
y_train = np.zeros(Ntrain)
for t in range(Ntrain):
x_train[t, :, :] = input_data[t:t + T]
y_train[t] = targets[t + T]
#y_train[t] = (targets[t+T] > 0)
x_test = np.zeros((N - Ntrain, T, D))
y_test = np.zeros(N - Ntrain)
for u in range(N - Ntrain):
# u counts from 0...(N - Ntrain)
# t counts from Ntrain...N
t = u + Ntrain
x_test[u, :, :] = input_data[t:t + T]
y_test[u] = targets[t + T]
#y_test[u] = (targets[t+T] > 0)
model = keras_model.build_model(T, D)
print(model.summary())
with mlflow.start_run(run_name=args.ticker) as active_run:
run_id = active_run.info.run_id
mlflow.set_tag('runName', args.ticker + '-' + run_id)
r = model.fit(x_train,
y_train,
batch_size=args.batch_size,
epochs=args.num_epochs,
validation_data=(x_test, y_test))
metrics = r.history
mlflow.log_param('num_layers', len(model.layers))
mlflow.log_param('optimizer_name', type(model.optimizer).__name__)
mlflow.log_param('num_epochs', args.num_epochs)
mlflow.log_param('batch_size', args.batch_size)
_mlflow_log_metrics(metrics, 'loss')
_mlflow_log_metrics(metrics, 'accuracy')
_mlflow_log_metrics(metrics, 'val_loss')
_mlflow_log_metrics(metrics, 'val_accuracy')
try:
os.mkdir('mlflow/' + run_id)
except:
os.mkdir('mlflow')
os.mkdir('mlflow/' + run_id)
model_local_path = os.path.join('mlflow', run_id, 'model')
tf.saved_model.save(model, model_local_path)
mlflow.tensorflow.log_model(tf_saved_model_dir=model_local_path,
tf_meta_graph_tags=[tag_constants.SERVING],
tf_signature_def_key='serving_default',
artifact_path='model')
duration = time() - start_time
mlflow.log_metric('duration', duration)
mlflow.end_run()
if args.create or args.deploy:
service = discovery.build('ml', 'v1')
project_id = 'deep-stock-268818'
bucket_name = 'deep-stock-ml-bucket'
model_name = args.ticker
model_version = 'mlflow_{}'.format(run_id)
model_gcs_path = os.path.join('gs://', bucket_name, run_id, 'model')
model_helper = model_deployment.AIPlatformModel(project_id)
model_helper.upload_to_bucket(model_local_path, model_gcs_path)
model_helper.create_model(model_name)
if args.deploy:
model_helper.deploy_model(model_gcs_path, model_name, run_id)
logging.info('Model Deployment complete')
valX = valX.astype(np.float32)
trainY = trainY.astype(np.float32)
testY = testY.astype(np.float32)
valY = valY.astype(np.float32)
MEAN = np.mean(trainX, axis=(0, 1, 2))
STD = np.std(trainX, axis=(0, 1, 2))
for i in range(3):
trainX[:, :, :, i] = (trainX[:, :, :, i] - MEAN[i]) / STD[i]
testX[:, :, :, i] = (testX[:, :, :, i] - MEAN[i]) / STD[i]
valX[:, :, :, i] = (valX[:, :, :, i] - MEAN[i]) / STD[i]
f1_score_callback = ComputeF1((valX, valY))
model = build_model('train', model_name=args.model)
## Training model.
model.fit(trainX,
trainY,
batch_size=32,
epochs=25,
validation_data=(valX, valY),
callbacks=[f1_score_callback])
## Compute test F1 Score
model = load_model('model.h5')
score = F1_score(testY, model.predict(testX).round())
print('F1 Score =', score)
def process_arguments():
parser = argparse.ArgumentParser()
parser.add_argument('--steps_per_epoch',
help="Number of steps of gradient descent per epoch",
dest='steps_per_epoch',
type=int,
default=10)
parser.add_argument('--epochs',
help="Number of epochs",
dest='epochs',
type=int,
default=10)
return HParams(**parser.parse_args().__dict__)
if __name__ == "__main__":
# get arguments from command line
params = process_arguments()
model = build_model()
dataset = get_dataset("fma_small.csv")
model.fit(dataset,
steps_per_epoch=params.steps_per_epoch,
epochs=params.epochs)
for i in range(size):
a[i, :, :, :] = read_for_validation(self.data[start + i])
if self.verbose > 0:
self.progress.update()
if self.progress.n >= len(self): self.progress.close()
return a
def __len__(self):
return (len(self.data) + self.batch_size - 1) // self.batch_size
from keras_tqdm import TQDMCallback
from keras_model import build_model
model, branch_model, head_model = build_model(64e-5, 0, img_shape)
head_model.summary()
branch_model.summary()
model.summary()
plot_model(head_model, to_file='head-model.png')
pil_image.open('head-model.png')
plot_model(branch_model, to_file='branch_model.png')
pil_image.open('branch_model.png')
plot_model(model, to_file='model.png')
pil_image.open('model.png')
# A Keras generator to evaluate on the HEAD MODEL on features already pre-computed.
f = open("categories.txt", "r")
categories = f.readline().split(",")
print(categories)
#categories = [
#'healthy', 'junk', 'dessert', 'appetizer', 'mains', 'soups', 'carbs', 'protein', 'fats', 'meat'
#]
#parser = argparse.ArgumentParser()
#parser.add_argument('--image', help = 'Path to the image to be predicted', required = True)
#parser.add_argument('--saved_model', help = 'Path of the saved model', required = True)
#args = parser.parse_args()
modelPath = os.path.join("model.h5")
model = build_model('inference', model_path=modelPath)
#cap = cv2.VideoCapture(0)
#ret, img1 = cap.read()
cap = cv2.VideoCapture('new.mp4')
frame_width = int(cap.get(3))
frame_height = int(cap.get(4))
counting = 0
outputsX = ["banana", "apple", r"['salt', 'flour', 'sugar']"]
# Define the codec and create VideoWriter object.The output is stored in 'outpy.avi' file.
out = cv2.VideoWriter('output.avi', cv2.VideoWriter_fourcc('M', 'J', 'P', 'G'),
10, (frame_width, frame_height))
ret, img1 = cap.read()
def model_fn(features, labels, mode, params):
""" """
cerebras = params['cerebras']
LR = params['learning_rate']
# from Cerebras MNIST hybrid_model.py
# tf.compat.v1.set_random_seed(0) # --seed arg not yet implemented
loss = None
train_op = None
logging_hook = None
training_hook = None
eval_metric_ops = None
# living in the past?
get_or_create_global_step_fn = tf.compat.v1.train.get_or_create_global_step
get_global_step_fn = tf.compat.v1.train.get_global_step
get_collection_fn = tf.compat.v1.get_collection
set_verbosity_fn = tf.compat.v1.logging.set_verbosity
optimizer_fn = tf.compat.v1.train.MomentumOptimizer
accuracy_fn = tf.compat.v1.metrics.accuracy
loss_fn = tf.compat.v1.losses.sparse_softmax_cross_entropy # see loss_fn below
logging_INFO = tf.compat.v1.logging.INFO
GraphKeys = tf.compat.v1.GraphKeys
summary_scalar = tf.compat.v1.summary.scalar
is_training = (mode == tf.estimator.ModeKeys.TRAIN)
is_evaluate = (mode == tf.estimator.ModeKeys.EVAL)
is_predict = (mode == tf.estimator.ModeKeys.PREDICT)
#class_weights = features['weights']
inputs = features['data']
keras_model = build_model(params, tensor=inputs)
logits = keras_model.output
predictions = tf.argmax(logits, 1)
if is_training or is_evaluate or is_predict:
global_step = get_or_create_global_step_fn()
#loss = loss_fn(labels=labels, logits=logits, weights=class_weights)
loss = loss_fn(labels=labels, logits=logits)
hook_list = []
# hooks, metrics and scalars not available on Cerebras CS-1
if not cerebras:
accuracy = accuracy_fn(labels=labels,
predictions=predictions,
name='accuracy_op')
eval_metric_ops = dict(accuracy=accuracy)
summary_scalar('accuracy', accuracy[1])
set_verbosity_fn(logging_INFO)
logging_hook = tf.estimator.LoggingTensorHook(
{
"loss": loss,
"accuracy": accuracy[1]
}, every_n_iter=1000) #### every_n_secs = 60)
hook_list.append(logging_hook)
if is_training:
optimizer = optimizer_fn(learning_rate=LR, momentum=0.9)
update_ops = get_collection_fn(GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
train_op = optimizer.minimize(loss,
global_step=get_global_step_fn())
if not cerebras:
training_hook = hooks.TrainingHook(params, loss)
hook_list.append(training_hook)
estimator = tf.estimator.EstimatorSpec(mode=mode,
predictions=predictions,
loss=loss,
train_op=train_op,
eval_metric_ops=eval_metric_ops,
training_hooks=hook_list)
return estimator
finalAct="sigmoid")
# initialize the optimizer (SGD is sufficient)
opt = Adam(lr=INIT_LR, decay=INIT_LR / EPOCHS)
# compile the model using binary cross-entropy rather than
# categorical cross-entropy -- this may seem counterintuitive for
# multi-label classification, but keep in mind that the goal here
# is to treat each output label as an independent Bernoulli
# distribution
model.compile(loss="binary_crossentropy",
optimizer=opt,
metrics=["accuracy"])
else:
model = build_model('train', model_name=args["model"])
f1_score_callback = ComputeF1()
# train the network
print("[INFO] training network...")
H = model.fit(trainX,
trainY,
batch_size=BS,
validation_data=(testX, testY),
callbacks=[f1_score_callback],
epochs=EPOCHS,
verbose=2)
# save the model to disk
print("[INFO] serializing network...")
model.save(args["outModel"])
def main():
model = keras_model.build_model()
keras_model.fit(model)
pred = keras_model.predict(model)
print(pred.shape)
def model_fn(features, labels, mode, params):
""" """
cerebras = params['cerebras']
LR = params['learning_rate']
# from Cerebras MNIST hybrid_model.py
# tf.compat.v1.set_random_seed(0) # --seed arg not yet implemented
loss = None
train_op = None
logging_hook = None
training_hook = None
eval_metric_ops = None
# living in the past?
get_or_create_global_step_fn = tf.compat.v1.train.get_or_create_global_step
get_global_step_fn = tf.compat.v1.train.get_global_step
get_collection_fn = tf.compat.v1.get_collection
set_verbosity_fn = tf.compat.v1.logging.set_verbosity
optimizer_fn = tf.compat.v1.train.MomentumOptimizer
accuracy_fn = tf.compat.v1.metrics.accuracy
loss_fn = tf.compat.v1.losses.sparse_softmax_cross_entropy # see loss_fn below
# loss_fn = tf.nn.softmax_cross_entropy_with_logits_v2 # see MNIST
logging_INFO = tf.compat.v1.logging.INFO
GraphKeys = tf.compat.v1.GraphKeys
summary_scalar = tf.compat.v1.summary.scalar
is_training = (mode == tf.estimator.ModeKeys.TRAIN)
is_evaluate = (mode == tf.estimator.ModeKeys.EVAL)
is_predict = (mode == tf.estimator.ModeKeys.PREDICT)
inputs = features # no transformations needed
keras_model = build_model(params, tensor=inputs)
logits = keras_model.output
predictions = tf.argmax(logits, 1) # why axis=1?
# label and class weights are concatenated, decouple
weights = labels[:,1]
float_labels = labels[:,0]
labels = tf.cast(float_labels, dtype=tf.int64) # CS-1 flags "Cast"
if is_training or is_evaluate:
global_step = get_or_create_global_step_fn()
# loss = loss_fn(labels=labels, logits=logits) # see loss_fn, weights= not supported in "v2"
# loss = loss_fn(labels=labels, logits=logits, weights=weights) # see loss_fn
# squeezed_logits = tf.squeeze(logits)
# loss = loss_fn(labels=labels, logits=squeezed_logits, weights=weights) # see loss_fn
loss = loss_fn(labels=labels, logits=logits, weights=weights) # see loss_fn
hook_list = []
# hooks, metrics and scalars not available on Cerebras CS-1
if not cerebras:
accuracy = accuracy_fn(
labels=labels,
predictions=predictions,
name='accuracy_op')
eval_metric_ops = dict(accuracy=accuracy)
summary_scalar('accuracy', accuracy[1])
set_verbosity_fn(logging_INFO)
logging_hook = tf.estimator.LoggingTensorHook(
{"loss": loss, "accuracy": accuracy[1]},
every_n_iter = 1000) #### every_n_secs = 60)
hook_list.append(logging_hook)
if is_training:
optimizer = optimizer_fn(learning_rate=LR, momentum=0.9)
update_ops = get_collection_fn(GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
train_op = optimizer.minimize(
loss,
global_step=get_global_step_fn())
training_hook = hooks.TrainingHook(params, loss)
hook_list.append(training_hook)
estimator = tf.estimator.EstimatorSpec(
mode=mode,
predictions=predictions,
loss=loss,
train_op=train_op,
eval_metric_ops=eval_metric_ops,
training_hooks=hook_list)
return estimator