def single_image_transform_model(cimage, simage): sdim, cdim = simage.shape, cimage.shape fcmodel = extract_features(cdim) fcmodel.trainable = False if cdim == sdim: fsmodel = fcmodel else: fsmodel = extract_features(sdim) fsmodel.trainable = False cimage = cimage.reshape( (1, cimage.shape[0], cimage.shape[1], cimage.shape[2])) cvals = fcmodel.predict(cimage)[0][0] simage = simage.reshape( (1, simage.shape[0], simage.shape[1], simage.shape[2])) svals = fsmodel.predict(simage)[1][0] dummy = Input(shape=cdim, dtype="float32", name="dummy_img") from keras import initializers timg = RawWeights(name='result', activation=None, initializer=initializers.uniform(0, 255))(dummy) tfeats = fcmodel(timg) cerr = Lambda(lambda x: x - cvals, name="ContentError")(tfeats[0]) serr = Lambda(lambda x: x - svals, name="StyleError")(tfeats[1]) model = Model(inputs=[dummy], outputs=[timg, cerr, serr]) return model
def on_status(self, status): """ Storing streaming tweets in the SQL database. The returned status object has all the information about a tweet. """ # Exclude all retweets. if status.retweeted or 'RT @' in status.text: return True id_str = status.id_str created_at = status.created_at text = status.text user_location = status.user.location tweet = {"text": text, "label": None} tweet_processor = PreProcessTweets() processed_tweet = tweet_processor.process_tweets([tweet]) polarity = classifier.classify(extract_features(processed_tweet[0])) print(polarity) print(processed_tweet) print(extract_features(processed_tweet[0])) insert_sql = """ INSERT INTO {} (id_str, created_at, text, polarity, user_location) VALUES (%s, %s, %s, %s, %s) """.format(settings.table_name) val = (id_str, created_at, text, polarity, user_location) with Connection() as db: db.create_table_if_not_existed(settings.table_name) db.execute(insert_sql, val)
def task(): year, months = 2019, [10, 11, 12] select = None radius, aperture_size, incident_interval, time_step = '100', '6', '25', '1' config = { 'year': year, 'months': months, 'radius': radius, 'aperture_size': aperture_size, 'incident_interval': incident_interval, 'time_step': time_step, 'select': select } month = months[0] df = pd.read_pickle( f'output/waze/{year}_{month}_{radius}_{aperture_size}_features.pkl') for month in months[1:]: temp_df = pd.read_pickle( f'output/waze/{year}_{month}_{radius}_{aperture_size}_features.pkl' ) df = df.append(temp_df, sort=False) df.fillna(0, inplace=True) # if select is not None: # df = df[-1 * int(select):] if os.path.exists( f'output/{year}_{month}_{radius}_{aperture_size}_predict_proba.pkl' ): x = pd.read_pickle( f'output/{year}_{month}_{radius}_{aperture_size}_predict_proba.pkl' ) else: x = model.predict_proba(df, int(incident_interval), time_step) x = model.extract_features(x, df) x.to_pickle( f'output/{year}_{month}_{radius}_{aperture_size}_predict_proba.pkl' ) incident_df = load_incidents(aperture_size) if os.path.exists(f'output/{year}_{month}_{radius}_{aperture_size}_y.pkl'): y = pd.read_pickle( f'output/{year}_{month}_{radius}_{aperture_size}_y.pkl') else: y = label_mapper(x, incident_df) pd.Series(y).to_pickle( f'output/{year}_{month}_{radius}_{aperture_size}_y.pkl') models = [ 'LogisticRegression', 'DecisionTreeClassifier', 'RandomForestClassifier' ] for m in models: print('model', m) cv_results, y_pred = model.cross_validate(x, y, m) for k, v in config.items(): print(k, ',', v) for k, v in cv_results.items(): print(k, ',', np.average(v)) print()
def make_training_sample(): """make_training_sample() -> (np.array of shape (8, 8, 13), (win bool, draw bool, loss bool)) Returns features and 3-tuple of bools of what the outcome is from a randomly selected board position. You must first set the global variable `tablebase` to use this! For example: train.tablebase = chess.syzygy.open_tablebases("./syzygy/") """ board = make_random_position() result = tablebase.probe_wdl(board) features = model.extract_features(board) desired_output = (result > 0, result == 0, result < 0) return features, desired_output
def predict(fen): """predict(fen: str) -> (win prob: float, draw prob: float, loss prob: float)""" board = chess.Board(fen) features = model.extract_features(board) logits, = sess.run(net.flow, feed_dict={ net.input_ph: [features], net.is_training_ph: False, }) assert logits.shape == (3, ) win, draw, loss = softmax(logits) return win, draw, loss
def generic_data_pipeline(source_data, target_data, padded_length=default_padded_length): source_data, target_data = model.extract_features( source_data), model.extract_features(target_data) source_data, target_data = model.pad_features( source_data, padded_length), model.pad_features(target_data, padded_length) source_data, target_data = model.align(source_data, target_data) source_data, target_data = source_data[:, 1:], target_data[:, 1:] source_data, target_data = model.pad_features( source_data, padded_length), model.pad_features(target_data, padded_length) source_data, target_data = model.apply_delta( source_data), model.apply_delta(target_data) return source_data, target_data
def optimize_style_model(tmodel, cimage,simage): sdim, cdim = simage.shape, cimage.shape fcmodel = extract_features(cdim) fcmodel.trainable = False if cdim==sdim: fsmodel = fcmodel else: fsmodel = extract_features(sdim) fsmodel.trainable = False cimage = cimage.reshape((1, cimage.shape[0], cimage.shape[1], cimage.shape[2])) cvals = fcmodel.predict(cimage)[0][0] simage = simage.reshape((1, simage.shape[0], simage.shape[1], simage.shape[2])) svals = fsmodel.predict(simage)[1][0] cinp = Input(shape=cimage.shape[1:],dtype="float32",name="content_img") style_shape = tmodel.input_shape[1][1:] dummy = Input(shape=style_shape,dtype="float32",name="dummy_style") from keras import initializers style = RawWeights(name='style',activation=None, initializer=initializers.uniform(-0.01,0.01))(dummy) timg = tmodel([cinp,style]) res = Lambda(lambda x:x,name="result")(timg) tfeats = fcmodel(timg) cerr = Lambda(lambda x: x-cvals, name="ContentError")(tfeats[0]) serr = Lambda(lambda x: x-svals, name="StyleError")(tfeats[1]) model = Model(inputs=[cinp,dummy],outputs=[res,cerr,serr]) return model
def cnn_model_fn(features, labels, mode): # Make predictions logits = extract_features(features, CLASSES, IMAGE_SIZE, mode) predicted_classes = tf.argmax(input=logits, axis=1) # Predict the model if mode == tf.estimator.ModeKeys.PREDICT: outputs = { 'class_ids': predicted_classes[:, tf.newaxis], "predicted_label": predicted_classes, "probabilities": tf.nn.softmax(logits, name="softmax_tensor") } return tf.estimator.EstimatorSpec(mode=mode, predictions=outputs) # If not in mode.PREDICT, compute the loss loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits) # Train the model if mode == tf.estimator.ModeKeys.TRAIN: update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): optimizer = tf.train.AdamOptimizer(learning_rate=0.0001) train_op = optimizer.minimize( loss=loss, global_step=tf.train.get_global_step()) return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op) # Eval the model # Compute evaluation metrics. accuracy = tf.metrics.accuracy(labels=labels, predictions=predicted_classes, name='acc_op') metrics = {'accuracy': accuracy} tf.summary.scalar('accuracy', accuracy[1]) return tf.estimator.EstimatorSpec(mode=mode, loss=loss, eval_metric_ops=metrics)
elif base_model == 'inception_resnet': _, x, _, is_train, features = model.inception_resnet_model(input_shape, label_shape) feature_size = 1536 else: AssertionError('Unknown base model present') sess = tf.Session() # Loading trained model checkpoint saver = utils.load_checkpoint(os.path.join(model_dir,'model'), sess, 'model') saver_unloaded = utils.load_checkpoint(os.path.join(model_dir,'model_unsaved'), sess, 'model_unsaved') # Extracting feature dictionary for the provided dataset data_pl = [x, is_train] feature_dict = model.extract_features(sess, test_iter, test_data, data_pl, features) print(f'size of feature dict : {len(feature_dict)}') print('Feature_extraction Done') # Saving feature dictionary into pickle file with open(os.path.join(DATA_DIR, 'train_features.pickle'), 'wb') as handle: pickle.dump(feature_dict, handle, protocol=pickle.HIGHEST_PROTOCOL) # Loading metadata which will be later used to create # temporal sequences if base_data == 'train': df = pd.read_csv(os.path.join(DATA_DIR, 'train_metadata.csv'), sep=',') df_labels = pd.read_csv(os.path.join(DATA_DIR, 'reshaped_stage_2_train.csv'), sep=',') df = pd.merge(df, df_labels, left_on='SOPInstanceUID', right_on='ImgID')
def analyze(source_path, target_path): click.secho('Loading Audio Files: {} {} 🔍 '.format(source_path, target_path), fg='blue') # Iterative Strategy source_dataset, target_dataset = [], [] source_data, target_data = model.extract_features( model.load_audio(source_path)), model.extract_features( model.load_audio(target_path)) shorter = min([source_data.shape[0], target_data.shape[0]]) pad_length = max([source_data.shape[0], target_data.shape[0]]) + 200 for index in tqdm(range(int(shorter / default_padded_length))): current_source_data = source_data[index * default_padded_length:index * default_padded_length + default_padded_length, :] current_target_data = target_data[index * default_padded_length:index * default_padded_length + default_padded_length, :] current_source_data, current_target_data = model.align( current_source_data, current_target_data) current_source_data, current_target_data = model.pad_features( current_source_data, pad_length=pad_length), model.pad_features(current_target_data, pad_length=pad_length) current_source_data, current_target_data = current_source_data[:, 1:], current_target_data[:, 1:] current_source_data, current_target_data = model.apply_delta( current_source_data), model.apply_delta(current_target_data) source_dataset.append(current_source_data), target_dataset.append( current_target_data) source_dataset, target_dataset = numpy.asarray( source_dataset), numpy.asarray(target_dataset) joint_distribution = utilities.math.remove_zeros_frames( model.get_joint_matrix(source_dataset, target_dataset)) # Convolution Strategy # source_dataset, target_dataset = [], [] # source_data, target_data = model.extract_features( model.load_audio(source_path) ), model.extract_features(model.load_audio(target_path)) # source_data, target_data = model.align(source_data, target_data) # shorter = min([ source_data.shape[0], target_data.shape[0] ]) # pad_length = max([ source_data.shape[0], target_data.shape[0] ]) + 200 # for index in tqdm( range( int( (shorter - default_padded_length) / default_skip_frames) - 1 ) ): # current_source_data = source_data[index + (index * default_skip_frames):index + default_padded_length + (index * default_skip_frames), :] # current_target_data = target_data[index + (index * default_skip_frames):index + default_padded_length + (index * default_skip_frames), :] # current_source_data, current_target_data = model.pad_features(current_source_data, pad_length=pad_length), model.pad_features(current_target_data, pad_length=pad_length) # current_source_data, current_target_data = current_source_data[:, 1:], current_target_data[:, 1:] # current_source_data, current_target_data = model.apply_delta(current_source_data), model.apply_delta(current_target_data) # source_dataset.append(current_source_data), target_dataset.append(current_target_data) # source_dataset, target_dataset = numpy.asarray(source_dataset), numpy.asarray(target_dataset) # joint_distribution = utilities.math.remove_zeros_frames( model.get_joint_matrix(source_dataset, target_dataset) ) # Single Process Strategy # source_data = model.load_audio(source_path) # target_data = model.load_audio(target_path) # padded_length = max([ source_data.shape[0], target_data.shape[0] ]) + default_auto_pad_length # source_data, target_data = generic_data_pipeline(source_data, target_data, padded_length) # joint_distribution = utilities.math.remove_zeros_frames( model.get_joint_matrix(source_data, target_data) ) gaussian_mixture_model = model.create_model() click.secho('Training Gaussian Mixture Model 🎓 ', fg='blue') gaussian_mixture_model.fit(joint_distribution) start_name = utilities.filesystem.extension(os.path.basename(source_path), '') end_name = utilities.filesystem.extension(os.path.basename(target_path), '') save_model_as(gaussian_mixture_model, '{}-{}.pkl'.format(start_name, end_name)) click.secho('Training Finished on Gaussian Mixture Model ({}-{}) ✓'.format( start_name, end_name), fg='green')