import sys import mosek import utils_logging as logging logger = logging.get_logger('svm_solver', logging.INFO) class SVMSolver(object): def __init__(self, C, **kwargs): ''' solve quadratic program min ztPz + 2*ztq st. Az <= b Here: min 1/2wtw + Cxi st. for all (y1_,...yn_) in W 1/n wt sum(k){psi(xk,yk) - psi(xk,yk_)} >= 1/n sum(k) {loss(yk,yk_) - xi Our data structure: W = [{'psis':[],'losses':[]}, {'psis':[],'losses':[]}] ''' self.C = C self.use_mosek = kwargs.pop('use_mosek', True) self.Cprime = kwargs.pop('Cprime', 0) def objective(self, w, xi, wref=None): obj = 0.5 * sum([val**2 for val in w]) + self.C * xi if wref is not None: obj += 0.5 * self.Cprime * sum([(w1 - w2)**2 for w1, w2 in zip(w, wref)])
import sys import os import numpy as np import utils_logging logger = utils_logging.get_logger("weigth_functions", utils_logging.INFO) def weight_std(image, i, j, beta=1.0): """ standard weight function for touching pixel pair (i,j), wij = exp (- beta (image.flat[i] - image.flat[j])^2) """ im = np.asarray(image) wij = np.exp(-beta * (image.flat[i] - image.flat[j]) ** 2) return [wij] ##------------------------------------------------------------------------------ def weight_inv(image, i, j, beta=1.0, offset=1.0): """ inverse weight function """ im = np.asarray(image) data = np.abs(image.flat[i] - image.flat[j]) wij = 1.0 / (offset + beta * data) return [wij] ##------------------------------------------------------------------------------
import numpy as np import hashlib import mpi import logging import svm_solver reload(svm_solver) import svm_worker reload(svm_worker) import utils_logging logger = utils_logging.get_logger("struct_svm", utils_logging.DEBUG) """ class DataContainer(object): def __init__(self, data): self.data = data def __hash__(self): return id(self.data) """ class StructSVM(object): """ "Cutting-plane training of structural SVMs" Machine Learning 2009 Joachims, Thorsten Finley, Thomas Yu, Chun-Nam John
# Description: Server to conect Streak - Webhoook # Notes: Send and receive message from Gupshup # PostmanCollections: https://www.postman.com/collections/820a513fd242f49cc01d # Documentation: https://www.gupshup.io/developer/docs/bot-platform/guide/whatsapp-api-documentation-sp ######################################################################################### import sys import requests import json import time from datetime import datetime, timedelta from credentials import GUPSHUP from utils import * import urllib from utils_logging import get_logger ###################################################################################### logger = get_logger('sms_gupshup.py', level='DEBUG') ###################################################################################### def get_header(content_type): headers = headers = { 'Content-Type': content_type, 'Accept-Encoding': 'gzip, deflate, br', 'Connection': 'keep-alive', 'apikey': GUPSHUP['TOKEN'] } return headers def get_url_api(funcionality): URL = 'https://api.gupshup.io/sm/api'
''' Work in progress code. Do not copy, do not distribute without permission ! Pierre-Yves Baudin 2012 ''' import sys import os import numpy as np from scipy import sparse import utils_logging logger = utils_logging.get_logger('rwsegment',utils_logging.INFO) class BaseAnchorAPI(object): def __init__(self,anchor, anchor_weight=1.0, **kwargs): self.anchor = anchor self.labelset = anchor['labelset'] self.imask = anchor['imask'] self.anchor_weight = anchor_weight def get_labelset(self): return self.labelset def get_anchor_and_weights(self, list_D, indices): nlabel = len(self.labelset) N = np.maximum(np.max(self.imask), np.max(indices))+ 1 omega = 1./nlabel * np.ones((nlabel,N)) omega[:,self.imask] = self.anchor['data'] omega = omega[:,indices]
import logging import numpy as np import svm_worker import utils_logging logger = utils_logging.get_logger('latent_svm',utils_logging.DEBUG) from struct_svm import StructSVM class LatentSVM(object): def __init__( self, loss_function, psi, most_violated_constraint, annotation_consistent_inference, **kwargs): ## store user function self.user_loss = loss_function self.user_psi = psi self.user_mvc = most_violated_constraint self.user_aci = annotation_consistent_inference self.struct_params = kwargs ## pass parameters to struct SVM ## set latent SVM specific parameters self.niter_max = kwargs.pop('latent_niter_max',100) self.epsilon = kwargs.pop('latent_epsilon',1e-3) ## parallel stuff self.use_parallel = kwargs.pop('latent_use_parallel',False)
from scipy import sparse from scipy.sparse import linalg as splinalg def solve_qp(P,q,**kwargs): ''' solve: min(X): Xt*P*X + 2Xt*q + cst ''' if P.nnz==0: logger.error('P has no non-zeros entries') return np.zeros(q.size) maxiter = int(kwargs.pop('maxiter',1000)) tol = kwargs.pop('tol',1e-3) ## use standard conjugate gradient x,info = splinalg.cg( P,-q, maxiter=maxiter, tol=tol, ) if info!=0: logger.error('QP did not converge. info={}'.format(info)) return x import utils_logging logger = utils_logging.get_logger('solver_qp',utils_logging.DEBUG)
import numpy as np from scipy import ndimage import rwsegment reload(rwsegment) from rwsegment import BaseAnchorAPI import utils_logging logger = utils_logging.get_logger('rwsegment_prior_models',utils_logging.DEBUG) ''' class PriorModel(BaseAnchorAPI): def __init__(self,*args, **kwargs): super(PriorModel,self).__init__(*args, **kwargs) self.init_model() def init_model(self): pass def get_anchor_and_weights(self, D, indices): nlabel = len(self.labelset) N = np.maximum(np.max(self.imask), np.max(indices))+ 1 data = 1./nlabel * np.ones((nlabel,N)) data[:,self.imask] = self.anchor['data'] data = data[:,indices] weights = self.anchor_weight * np.ones((nlabel,len(indices))) return data, weights ''' class Constant(BaseAnchorAPI): def get_anchor_and_weights(self, D, indices): data, weights = super(Constant, self).get_anchor_and_weights(1, indices) return data, weights # *1
def solve_qp(P, q, **kwargs): ''' solve: min(X): Xt*P*X + 2Xt*q + cst ''' if P.nnz == 0: logger.error('P has no non-zeros entries') return np.zeros(q.size) maxiter = int(kwargs.pop('maxiter', 1000)) tol = kwargs.pop('tol', 1e-3) ## use standard conjugate gradient x, info = splinalg.cg( P, -q, maxiter=maxiter, tol=tol, ) if info != 0: logger.error('QP did not converge. info={}'.format(info)) return x import utils_logging logger = utils_logging.get_logger('solver_qp', utils_logging.DEBUG)
--data-urlencode 'Body=Hello! I am Deiner testing my code.!!!' -u MYUSER:MYPASSWORD """ ###################################################################################### import time import datetime from credentials import * from twilio.rest import Client from utils_logging import get_logger # Your Account Sid and Auth Token from twilio.com/console # DANGER! This is insecure. See http://twil.io/secure ###################################################################################### account_sid = TWILIO['ID'] auth_token = TWILIO['TOKEN'] ###################################################################################### logger = get_logger('sms_twilio.py', level='DEBUG') ###################################################################################### twilio_fiels_whatsapp = [ "originalMessage", "SmsMessageSid", "NumMedia", "SmsSid", "SmsStatus", "Body", "To", "NumSegments", "MessageSid", "AccountSid", "From", "ApiVersion", "chatId",
Work in progress code. Do not copy, do not distribute without permission ! Pierre-Yves Baudin 2012 ''' import sys import os import numpy as np from scipy import sparse import utils_logging logger = utils_logging.get_logger('rwsegment',utils_logging.DEBUG) ##------------------------------------------------------------------------------ def segment( image, anchor_api, seeds=[], weight_function=None, return_arguments=['image'], **kwargs ): ''' segment an image with method from MICCAI'12 arguments: image = image to segment anchor_api:
import numpy as np import gc import mpi import utils_logging logger = utils_logging.get_logger('svm_worker',utils_logging.INFO) #from struct_svm import DataContainer def broadcast(task, *args, **kwargs): ''' send data to SVM worker ''' comm = mpi.COMM nproc = mpi.SIZE ndata = len(args[0]) all = np.arange(ndata) ## send for iproc in range(1,nproc): some = all[np.mod(all, nproc-1) == (iproc-1)] comm.send((task, len(some), kwargs), dest=iproc) # send initial message for i in some: comm.send(tuple([i]+[arg[i] for arg in args]), dest=iproc) ## receive output = [] for n in range(ndata): source_proc = np.mod(n, comm.Get_size()-1) + 1 output.append(comm.recv(source=source_proc, tag=n)) return output
import sys import mosek import utils_logging as logging logger = logging.get_logger('svm_solver', logging.INFO) class SVMSolver(object): def __init__(self, C, **kwargs): ''' solve quadratic program min ztPz + 2*ztq st. Az <= b Here: min 1/2wtw + Cxi st. for all (y1_,...yn_) in W 1/n wt sum(k){psi(xk,yk) - psi(xk,yk_)} >= 1/n sum(k) {loss(yk,yk_) - xi Our data structure: W = [{'psis':[],'losses':[]}, {'psis':[],'losses':[]}] ''' self.C = C self.use_mosek = kwargs.pop('use_mosek', True) self.Cprime = kwargs.pop('Cprime', 0) def objective(self, w, xi, wref=None): obj = 0.5 * sum([val**2 for val in w]) + self.C * xi if wref is not None: obj += 0.5 * self.Cprime * sum([(w1-w2)**2 for w1,w2 in zip(w,wref)]) return obj
import logging import numpy as np import svm_worker import utils_logging logger = utils_logging.get_logger('latent_svm', utils_logging.DEBUG) from struct_svm import StructSVM class LatentSVM(object): def __init__(self, loss_function, psi, most_violated_constraint, annotation_consistent_inference, **kwargs): ## store user function self.user_loss = loss_function self.user_psi = psi self.user_mvc = most_violated_constraint self.user_aci = annotation_consistent_inference self.struct_params = kwargs ## pass parameters to struct SVM ## set latent SVM specific parameters self.niter_max = kwargs.pop('latent_niter_max', 100) self.epsilon = kwargs.pop('latent_epsilon', 1e-3) ## parallel stuff self.use_parallel = kwargs.pop('latent_use_parallel', False) def sequential_all_aci(self, w, xs, zs, metas, y0s): ys = [] for x, z, y0, meta in zip(xs, zs, y0s, metas):
import numpy as np from scipy import ndimage import rwsegment reload(rwsegment) from rwsegment import BaseAnchorAPI import utils_logging logger = utils_logging.get_logger('rwsegment_prior_models', utils_logging.DEBUG) ''' class PriorModel(BaseAnchorAPI): def __init__(self,*args, **kwargs): super(PriorModel,self).__init__(*args, **kwargs) self.init_model() def init_model(self): pass def get_anchor_and_weights(self, D, indices): nlabel = len(self.labelset) N = np.maximum(np.max(self.imask), np.max(indices))+ 1 data = 1./nlabel * np.ones((nlabel,N)) data[:,self.imask] = self.anchor['data'] data = data[:,indices] weights = self.anchor_weight * np.ones((nlabel,len(indices))) return data, weights ''' class Constant(BaseAnchorAPI): def get_anchor_and_weights(self, D, indices): data, weights = super(Constant, self).get_anchor_and_weights(1, indices)
u = u + step * u_nt else: logger.warning('Newton"s method did not converge in {} iterations'.format(self.maxiter)) # raise Exception('Did not converge in {} iterations'\ # .format(self.maxiter)) ## return output return u def extract_diag(self, H): ## extract diagonal from H ijH = H.nonzero() ondiagH = np.equal(*ijH) diagH = np.zeros(H.shape[0]) diagH[ijH[0][ondiagH]] = H.data[ondiagH] return diagH def line_search(self, u, du, gradu): t = 1.0 objective = self.objective b = float(self.b) while objective(u + t * du) > objective(u) + self.a * t * gradu.T * du: t = b * t return t import utils_logging logger = utils_logging.get_logger("solver_qp_constr", utils_logging.DEBUG)
else: logger.warning( 'Newton"s method did not converge in {} iterations'.format( self.maxiter)) #raise Exception('Did not converge in {} iterations'\ # .format(self.maxiter)) ## return output return u def extract_diag(self, H): ## extract diagonal from H ijH = H.nonzero() ondiagH = np.equal(*ijH) diagH = np.zeros(H.shape[0]) diagH[ijH[0][ondiagH]] = H.data[ondiagH] return diagH def line_search(self, u, du, gradu): t = 1.0 objective = self.objective b = float(self.b) while objective(u + t * du) > objective(u) + self.a * t * gradu.T * du: t = b * t return t import utils_logging logger = utils_logging.get_logger('solver_qp_constr', utils_logging.DEBUG)
import numpy as np import hashlib import mpi import logging import svm_solver reload(svm_solver) import svm_worker reload(svm_worker) import utils_logging logger = utils_logging.get_logger('struct_svm', utils_logging.DEBUG) ''' class DataContainer(object): def __init__(self, data): self.data = data def __hash__(self): return id(self.data) ''' class StructSVM(object): ''' "Cutting-plane training of structural SVMs" Machine Learning 2009 Joachims, Thorsten Finley, Thomas Yu, Chun-Nam John ''' def __init__( self,
''' Work in progress code. Do not copy, do not distribute without permission ! Pierre-Yves Baudin 2012 ''' import sys import os import numpy as np from scipy import sparse import utils_logging logger = utils_logging.get_logger('rwsegment', utils_logging.INFO) class BaseAnchorAPI(object): def __init__(self, anchor, anchor_weight=1.0, **kwargs): self.anchor = anchor self.labelset = anchor['labelset'] self.imask = anchor['imask'] self.anchor_weight = anchor_weight def get_labelset(self): return self.labelset def get_anchor_and_weights(self, list_D, indices): nlabel = len(self.labelset) N = np.maximum(np.max(self.imask), np.max(indices)) + 1 omega = 1. / nlabel * np.ones((nlabel, N)) omega[:, self.imask] = self.anchor['data']
import numpy as np import gc import mpi import utils_logging logger = utils_logging.get_logger('svm_worker', utils_logging.INFO) #from struct_svm import DataContainer def broadcast(task, *args, **kwargs): ''' send data to SVM worker ''' comm = mpi.COMM nproc = mpi.SIZE ndata = len(args[0]) all = np.arange(ndata) ## send for iproc in range(1, nproc): some = all[np.mod(all, nproc - 1) == (iproc - 1)] comm.send((task, len(some), kwargs), dest=iproc) # send initial message for i in some: comm.send(tuple([i] + [arg[i] for arg in args]), dest=iproc) ## receive output = [] for n in range(ndata): source_proc = np.mod(n, comm.Get_size() - 1) + 1 output.append(comm.recv(source=source_proc, tag=n)) return output