def mean_distances(embeddings, labels, metric, normalized): if normalized == True: embeddings = l2_normalized(embeddings) labels = array_ops.reshape(labels, [array_ops.shape(labels)[0], 1]) dist_matrix = metric(embeddings) adjacency = compose( tf_equal(labels), array_ops.transpose, )(labels) adjacency_not = compose( tf_cast(dtype=dtypes.float32), math_ops.logical_not, )(adjacency) adjacency = math_ops.cast(adjacency, dtype=dtypes.float32) pdist_matrix = tf.multiply(dist_matrix, adjacency) _ndist_matrix = tf.multiply(dist_matrix, adjacency_not) ndist_matrix = compose( tf_add(_ndist_matrix), tf_multiply(adjacency), tf_add(tf.reduce_max(dist_matrix, axis=1, keepdims=True)), )(pdist_matrix) positive_mean_distance = math_ops.reduce_mean(pdist_matrix) negative_mean_distance = math_ops.reduce_mean(_ndist_matrix) hardest_positive_dist = tf.reduce_max(pdist_matrix, axis=1, keepdims=True) hardest_mean_positive_distance = tf.reduce_mean(hardest_positive_dist) hardest_negative_dist = tf.reduce_min(ndist_matrix, axis=1, keepdims=True) hardest_mean_negative_distance = tf.reduce_mean(hardest_negative_dist) return positive_mean_distance, negative_mean_distance, hardest_mean_positive_distance, hardest_mean_negative_distance
def is_valid_user_email_and_password(user_email, password): maybe_user_auth_data = with_session(compose( maybe_dump_model, maybe_user_auth(user_email) )) return maybe_user_auth_data and verify( password, maybe_user_auth_data["password"] )
def cosine_distance(embeddings, evaluate=None): """ Compute cosine distance matrix Parameters: ----------- - embeddings: Tensor(N, E) Image embeddings, outputs of the convolutional network. N - number of samples (None) E - embedding size """ embeddings = tf.divide(embeddings, tf.norm(embeddings, axis=1, keep_dims=True)) diag_mask = compose( tf.abs, partial(tf.add, -1.), )(tf.eye(num_rows=tf.shape(embeddings)[0])) return compose( partial(tf.multiply, diag_mask), partial(tf.subtract, 1.), partial(tf.matmul, embeddings), )(tf.transpose(embeddings))
def create_model(json): name = json["name"] type_name = json["type"] datasource_id = json["datasource"] input_columns = json["input-columns"] type_id = with_session(compose( getattr("id"), query_type_by_name(type_name) )) new_model_id = with_session(create_row(ModelModel, { "name": name, "type_id": type_id, "datasource_id": datasource_id }))["id"] with_session(create_input_columns(input_columns, new_model_id)) queue_model_creation_task(json) return get_model(new_model_id)
def proxy_route(app): return compose( app.route('/', defaults={'path': ''}, methods=allowed_methods), app.route('/<path:path>', methods=allowed_methods) )
maybe_none_resolver) data = cata(tagfn(me.find_all('tr')), at3) data2 = cata(tagfn(me.find_all('td')), data) data_texts = cata(tagfn(mn.get_text()), data2) data3 = cata(tagfn(mn.find('a')), data2) data_titles = cata(tagfn(mn.get('title')), data3) data_links = cata(tagfn(mn.get('href')), data3) #data5 = cata(tagfn(maybe_none_resolver),data4) uw_titles = cata(lambda x: x.value if isinstance(x, md.Maybe) else x, data_titles) uw_texts = cata(lambda x: x.value if isinstance(x, md.Maybe) else x, data_texts) w_titles = cata( r.if_else(r.isinstance(list), r.identity, r.compose(md.First, maybe_none_resolver)), uw_titles) w_texts = cata(r.if_else(r.isinstance(list), r.identity, maybe_none_resolver), uw_texts) out = cata( r.if_else( lambda x: True if isinstance(x, tuple) and isinstance(x[0], md.Container) else False, r.apply(r.add), r.identity), (w_titles, w_texts)) out2 = cata( r.if_else(r.isinstance(md.First), lambda x: x.value.value, r.identity), out) out3 = cata(r.if_else(r.isinstance(md.Just), lambda x: x.value, r.identity), out2)
def get_recommender(id): return with_session(compose( recommender_model_to_dict, query_id(RecommenderModel, id) ))
def get_model(id): return with_session(compose( model_model_to_dict, query_id(ModelModel, id) ))
def siamese_job(source_path, model_loader, **kwargs): loss_fn = kwargs.pop('loss_fn', None) batch_size = kwargs.pop('batch_size', None) num_per_class = kwargs.pop('num_per_class', 5) lr = kwargs.pop('lr', 1e-3) normalized = kwargs.pop('normalized', True) num_classes = kwargs.pop('num_classes') normalized_input = kwargs.pop('normalized_input') tf.reset_default_graph() dirs = compose( filter_list(['.DS_Store'], False), os.listdir, )(source_path) labels = classes_to_labels(dirs) train_dirs, val_dirs, train_labels, val_labels = train_test_split( dirs, labels, test_size=0.1) if (num_classes != None): train_dirs = train_dirs[0:num_classes] train_labels = train_labels[0:num_classes] val_dirs = val_dirs[0:num_classes] val_labels = val_labels[0:num_classes] session = tf.Session() inputs, outputs, is_pretrained = model_loader(session) labels = tf.placeholder(name='labels', dtype=tf.int32, shape=(None, )) eval_samples, eval_labels = load_batch_of_images( path=source_path, num_per_class=4, image_shape=(128, 64, 3), loader=cv2_loader, dirs=train_dirs[0:3], labels=train_labels[0:3], batch_size=None, normalize=normalized_input, ) ctx['evaluator'].initialize(session=session, feed_dict={ inputs: eval_samples, labels: eval_labels, }) model = create_siamese_graph(session=session, base_model=[inputs, outputs, labels], optimizer=tf.train.AdamOptimizer, loss_fn=loss_fn, is_pretrained=is_pretrained, normalized=normalized) train_siamese_model( session=session, model=model, dirs=[train_dirs, val_dirs], labels=[train_labels, val_labels], initial_lr=lr, batch_generator=batch_of_images_generator( path=source_path, dirs=train_dirs, labels=train_labels, num_per_class=num_per_class, batch_size=batch_size, image_shape=(128, 64, 3), loader=cv2_loader, shuffle=True, normalize=normalized_input, ), batch_loader=partial( load_batch_of_images, path=source_path, num_per_class=num_per_class, image_shape=(128, 64, 3), loader=cv2_loader, normalize=normalized_input, ), is_pretrained=is_pretrained, batch_size=batch_size, **kwargs, )
def get_datasource(id): return with_session(compose( datasource_model_to_dict, query_id(DatasourceModel, id) ))
) bool_string_environ = pred_environ( is_bool_string, bool_string_to_bool, "either 'true' or 'false'" ) hash_alg_environ = pred_environ( is_hash_alg, identity, "one of {0}" % allowed_hash_algorithms ) default_hash_alg_environ = compose( hash_alg_environ, default_environ(default_hash_algoirthm) ) default_hash_rounds_environ = compose( int_string_environ, default_environ("40000") ) min_password_length_environ = compose( int_string_environ, default_environ("12") ) password_char_set_environ = default_environ(None) TARGET_SERVICE_HOST = required_environ("TARGET_SERVICE_HOST").value
from pyramda import compose def filter_list(refs, condition, l): if condition == True: return list(filter(lambda x: x in refs, l)) else: return list(filter(lambda x: x not in refs, l)) classes_to_labels = compose( list, range, len, )