def main():
parser = argparse.ArgumentParser(description="classify the instrument in the music sample")
parser.add_argument("-i", dest="filename", required=True, help="input file in WAV format")
args = parser.parse_args()
filename = args.filename
check_format(filename)
model_instrument = load_model("29_svm_instrument")
model_family = load_model("29_svm_family")
scaler = load_model("scaler_instrument")
print("Family: %s" % predict(model_family, filename, scaler))
print("Instrument: %s" % predict(model_instrument, filename, scaler))
async def get_answer(data: review_sentiment):
review = dict(data)['review']
name_model = dict(data)['name_model']
n_class = dict(data)['n_class']
if n_class != 0:
Model = sentiment_analysis(n_class, name_model)
else:
Model = load_model(name_model)
d = {'content': [str(review)]}
df = pd.DataFrame(d)
tokenizer = Model.tokenizer
dataloader = create_data_loader(df, tokenizer, max_len=32, batch_size=8)
for data in dataloader:
input_ids = data['input_ids']
attention_mask = data['attention_mask']
prediction = Model.predict(input_ids, attention_mask)
sentiment = get_sentiment(str(prediction.numpy()[0]))
return JSONResponse(sentiment)
#if __name__ == '__main__':
#uvicorn.run(app, host='127.0.0.1', port=8000)
def get_embedding_activations(baseline_path, adv_path, data_path, desc_path, batch_size,
embedding_type):
train_data, val_data, events, vocab = loadExperimentData(desc_path=desc_path, embedding_type=embedding_type,
data_path=data_path)
embedding_dim = train_data[0][0].shape[1]
val_batches = batchify(val_data, batch_size, classifier_mode='adversarial', embedding_dim=embedding_dim, randomize=False)
crit_labels = {'low': 0, 'high': 1}
crit_output_size = len(crit_labels)
baseline_model= load_model(baseline_path)
adv_model = load_model(adv_path)
index=0
for x, y_event, y_crit, seq_lengths, sentences in val_batches:
adv_x= x.clone().detach().requires_grad_(True)
adv_model.zero_grad()
y_pred_event, y_pred_crit, _ = adv_model(adv_x, seq_lengths)
loss = adv_model.loss(y_pred_crit, y_crit, seq_lengths, crit_output_size)
#loss = loss_event + loss_crit
#loss= loss_crit
loss.backward()
adv_gradients = adv_x.grad
del loss
#del loss_crit
baseline_x = x.clone().detach().requires_grad_(True)
baseline_model.zero_grad()
y_pred_baseline, _ = baseline_model(baseline_x, seq_lengths)
loss_baseline = baseline_model.loss(y_pred_baseline, y_crit, seq_lengths)
loss_baseline.backward()
baseline_gradients = baseline_x.grad
for i in range(batch_size):
print('Current Sentence: '+str(index) +'\t'+str(y_crit[i].item())+ '\t'+ sentences[i])
index+=1
tokens= sentences[i].split()
fig, (ax1, ax2) = plt.subplots(1, 2)
#fig.set_figwidth(50)
fig.set_figwidth(9)
ax1.title.set_text('Baseline Model')
ax2.title.set_text('Adversarial Model')
visualize_saliency(baseline_gradients[i, :seq_lengths[i]], tokens, ax1)
visualize_saliency(adv_gradients[i,:seq_lengths[i]], len(tokens)*[''], ax2)
fig.savefig('../figures/figure_' +str(index)+ '.pdf')
del loss_baseline
def new_model(compile=True):
G = new_G()
D = classifier.load_model(for_test=True)
D.trainable = False
i = keras.layers.Input(input_shape)
GAN = keras.models.Model(i, D(G(i)))
if compile:
GAN.compile(optimizer='RMSProp', loss='mse', metrics=['accuracy'])
return GAN, G, D
def main():
df = pd.read_csv(args.path)
sentiment_analyser = load_model('best_model_state')
tokenizer = sentiment_analyser.tokenizer
dataloader = create_data_loader(df,
tokenizer,
max_len=args.max_len,
batch_size=args.batch_size)
for data in dataloader:
input_ids = data['input_ids']
attention_mask = data['attention_mask']
prediction = sentiment_analyser.predict(input_ids, attention_mask)
print(prediction.numpy())
import cv2
import os
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
from utils import *
import classifier as clf
import detector
def pipeline(image):
image = detector.detect(image)
return image
if __name__ == '__main__':
model = clf.load_model('model.pkl')
detector = detector.Detector(model, clf.settings)
#utils.image_preview('./test_images/test1.jpg')
#video_preview('test_video.mp4', pipeline)
video_output('project_video.mp4', pipeline)
if args.train.endswith('.npz'):
import numpy
npzarrays = numpy.load(args.train)
X, Y = npzarrays['xs'], npzarrays['ys']
else:
X, Y = feature.extract_all(reader.read_audios(args.train),
train=True,
binary_class=True,
persist=True)
model_path = classifier.train_pipeline(X, Y)
print("============")
print("model saved at " + model_path)
print("============")
if args.segment:
model = classifier.load_model(args.segment[0])
for wav in reader.read_audios(args.segment[1],
file_per_dir=args.numfiles):
predicted = classifier.predict_pipeline(wav, model)
smoothed = smoothing.smooth(predicted)
speech_portions, total_frames = writer.index_frames(smoothed)
audio_fname = os.path.join(*wav)
writer.print_durations(speech_portions, audio_fname, total_frames)
if args.out:
print('writing files')
writer.slice_speech(speech_portions, audio_fname)
if args.evaluate:
model = classifier.load_model(args.evaluate[0])
evaluation.evaluate_files(args.evaluate[1], model, args.numfiles)
if args.cpu:
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
if args.train:
start = time.perf_counter()
X, Y = feature.extract_all(reader.read_wavs(args.train),
train=True,
binary_class=args.binary)
model_path = classifier.train_pipeline(X, Y)
print("============")
print(f"model saved at {model_path}")
print(f"time elapsed: {time.perf_counter()-start:0.4f} seconds")
print("============")
if args.segment:
# include 'dat' file extension for Galaxy data files
for wav in reader.read_wavs(args.segment[1],
file_ext=['mp3', 'wav', 'mp4', 'dat']):
start = time.perf_counter()
model = classifier.load_model(args.segment[0])
predicted = classifier.predict_pipeline(wav, model)
smoothed = smoothing.smooth(predicted, int(args.threshold),
args.binary)
amp_segments = AmpSegment(wav[1], smoothed)
if args.out:
writer.save_json(amp_segments, wav, args.out)
print(
f"Finished {wav} in {time.perf_counter()-start:0.4f} seconds")
import numpy as np
import os, sys
import classifier as cf
IGM_DIR = "./images/"
NN = False
# モデルをロード
if (len(sys.argv) > 1):
if (sys.argv[1] == "CNN"):
NN = True
print("NN") if NN == False else print("CNN")
cf.load_model(NN)
for dir_name in os.listdir(IGM_DIR):
dir_size = 0
ans_list = np.zeros(10)
print("DIR:", dir_name, end="")
for filename in os.listdir(IGM_DIR + dir_name):
x = cf.load_image(IGM_DIR + dir_name + "/" + filename, NN)
ans = cf.classifier(x)
ans_list[ans] += 1
dir_size += 1
maxIndex = [i for i, x in enumerate(ans_list) if x == max(ans_list)]
print("-> Most:", maxIndex)
print("{0}:".format(dir_size), ans_list)
from sklearn.feature_extraction.text import CountVectorizer
from classifier import load_vocabulary, load_model, tf_idf, fetch_train_dataset, DEFAULT_CATEGORIES
from sys_helpers import exit_if_error, wait_for_crl_c
from kafka_helpers import KAFKA_TWEET_TOPIC, get_kafka_hosts, create_kafka_consumer
print("Loading pre-trained model...")
vocabulary, err = load_vocabulary()
exit_if_error(err)
count_vect = CountVectorizer(vocabulary=vocabulary)
count_vect._validate_vocabulary()
model, err = load_model()
exit_if_error(err)
tfidf_transformer = tf_idf(DEFAULT_CATEGORIES)[0]
print("Connecting to consume Kafka stream...")
consumer, err = create_kafka_consumer(KAFKA_TWEET_TOPIC, get_kafka_hosts())
exit_if_error(err)
print("Ready to make predictions...")
wait_for_crl_c()
for message in consumer:
msg = message.value.decode("utf-8")
X_new_counts = count_vect.transform([msg])
def detect_fake_review(url):
html=requests.get(url).text
reviews=parse_yelp_page(html)
classifier.load_model()
classify_result=classifier.check(reviews)
return update_html(html, classify_result)
heatmap = utils.get_heatmap(image, out_windows)
heatmap = self.smoother(heatmap)
heatmap = utils.apply_threshold(heatmap,3)
boxes = utils.get_labeled_boxes(heatmap)
else:
windows = utils.slide_window(image, x_start_stop=[None, None], y_start_stop=self.y_start_stop,
xy_window=(64, 64), xy_overlap=(0.85, 0.85))
boxes = self.search_windows(image, windows)
final_image = utils.draw_boxes(image, boxes, color=(0, 0, 255), thick=6)
return final_image
if __name__ == '__main__':
from classifier import load_model, settings
model = load_model('model.pkl')
detector = Detector(model, settings)
images_path = './test_images/'
fig = plt.figure(figsize=(10,10))
fig.subplots_adjust(left=0.05, bottom=0.05, right=0.98, top=0.98)
row = len(os.listdir(images_path))
col = 3
index = 0
for filename in os.listdir(images_path):
image = mpimg.imread(images_path + filename)
index += 1
plt.subplot(row, col, index)
out_windows = detector.find_multiscale(image)
window_img = utils.draw_boxes(image, out_windows, color=(0, 0, 255), thick=6)
if top_k == 1:
top_classes = [idx_to_class[int(top_labels)]]
top_probs = [float(top_probs)]
else:
top_classes = [idx_to_class[each] for each in top_labels]
return top_probs, top_classes
# [END]
# [START The main function which drives the script]
if __name__ == '__main__':
image_path = args.input
model = load_model(args.checkpoint)
if args.gpu and torch.cuda.is_available():
model.cuda()
model.eval()
top_probs, top_classes = predict(image_path,
model,
top_k=args.top_k,
cuda=args.gpu)
if args.category_names is not None:
class_to_name = load_label_map(args.category_names)
top_classes = [class_to_name[each] for each in top_classes]
#Add classifier directory so we can import it
sys.path.insert(0, os.getcwd() + '/Classifier')
import cv2
import classifier
import classifier_const
from flask import Flask, request
import unicodedata
import numpy as np
#The main server
app = Flask(__name__)
#A model trained off our data
current_model = classifier.load_model("model_final")
#GET request handler for an image | The request encodes the image
@app.route("/api/classify/", methods=['GET'])
def classify():
#Parse the raw string into raw data
image_raw = request.args.get('raw')
input_string = image_raw.rsplit('/', 3)
input_string[0] = input_string[0].replace(' ',
'+') #'+' gets encoded as ' '
for i in range(len(input_string)):
input_string[i] = unicodedata.normalize('NFKD',
input_string[i]).encode(
review = st.text_input(
"Please provide your review:",
value="This movie is super bad and the worst thing I've ever watched")
run_query = st.button("Run")
if local_model and not hugging_face_model:
n_class = 0
path = st.sidebar.text_input(
'Please enter the full path to the folder of the classification model.'
)
get_local_model = st.sidebar.button("Get_model from disk")
if get_local_model:
with st.spinner('Getting model from disk'):
local_model_ = load_model(path)
if local_model_ != None:
st.write('Got model')
else:
st.write('Model doesn not exists in the directory provided')
print(run_query)
if run_query:
print('hello')
run_local_model(review=review, path=path)
elif hugging_face_model and not local_model:
name_model = st.sidebar.text_input(
'Please enter the name of the model from huggingface hub',
value='bert-base-cased')
get_model = st.sidebar.button("Get_model from hub")
#This is to check if the model_name we typed exists in huggingface or not