import sys
import mosek
import utils_logging as logging
logger = logging.get_logger('svm_solver', logging.INFO)
class SVMSolver(object):
def __init__(self, C, **kwargs):
''' solve quadratic program
min ztPz + 2*ztq
st. Az <= b
Here:
min 1/2wtw + Cxi
st. for all (y1_,...yn_) in W
1/n wt sum(k){psi(xk,yk) - psi(xk,yk_)}
>= 1/n sum(k) {loss(yk,yk_) - xi
Our data structure:
W = [{'psis':[],'losses':[]}, {'psis':[],'losses':[]}]
'''
self.C = C
self.use_mosek = kwargs.pop('use_mosek', True)
self.Cprime = kwargs.pop('Cprime', 0)
def objective(self, w, xi, wref=None):
obj = 0.5 * sum([val**2 for val in w]) + self.C * xi
if wref is not None:
obj += 0.5 * self.Cprime * sum([(w1 - w2)**2
for w1, w2 in zip(w, wref)])
import sys
import os
import numpy as np
import utils_logging
logger = utils_logging.get_logger("weigth_functions", utils_logging.INFO)
def weight_std(image, i, j, beta=1.0):
""" standard weight function
for touching pixel pair (i,j),
wij = exp (- beta (image.flat[i] - image.flat[j])^2)
"""
im = np.asarray(image)
wij = np.exp(-beta * (image.flat[i] - image.flat[j]) ** 2)
return [wij]
##------------------------------------------------------------------------------
def weight_inv(image, i, j, beta=1.0, offset=1.0):
""" inverse weight function """
im = np.asarray(image)
data = np.abs(image.flat[i] - image.flat[j])
wij = 1.0 / (offset + beta * data)
return [wij]
##------------------------------------------------------------------------------
import numpy as np
import hashlib
import mpi
import logging
import svm_solver
reload(svm_solver)
import svm_worker
reload(svm_worker)
import utils_logging
logger = utils_logging.get_logger("struct_svm", utils_logging.DEBUG)
"""
class DataContainer(object):
def __init__(self, data):
self.data = data
def __hash__(self):
return id(self.data)
"""
class StructSVM(object):
"""
"Cutting-plane training of structural SVMs"
Machine Learning 2009
Joachims, Thorsten
Finley, Thomas
Yu, Chun-Nam John
# Description: Server to conect Streak - Webhoook
# Notes: Send and receive message from Gupshup
# PostmanCollections: https://www.postman.com/collections/820a513fd242f49cc01d
# Documentation: https://www.gupshup.io/developer/docs/bot-platform/guide/whatsapp-api-documentation-sp
#########################################################################################
import sys
import requests
import json
import time
from datetime import datetime, timedelta
from credentials import GUPSHUP
from utils import *
import urllib
from utils_logging import get_logger
######################################################################################
logger = get_logger('sms_gupshup.py', level='DEBUG')
######################################################################################
def get_header(content_type):
headers = headers = {
'Content-Type': content_type,
'Accept-Encoding': 'gzip, deflate, br',
'Connection': 'keep-alive',
'apikey': GUPSHUP['TOKEN']
}
return headers
def get_url_api(funcionality):
URL = 'https://api.gupshup.io/sm/api'
'''
Work in progress code.
Do not copy, do not distribute without permission !
Pierre-Yves Baudin 2012
'''
import sys
import os
import numpy as np
from scipy import sparse
import utils_logging
logger = utils_logging.get_logger('rwsegment',utils_logging.INFO)
class BaseAnchorAPI(object):
def __init__(self,anchor, anchor_weight=1.0, **kwargs):
self.anchor = anchor
self.labelset = anchor['labelset']
self.imask = anchor['imask']
self.anchor_weight = anchor_weight
def get_labelset(self):
return self.labelset
def get_anchor_and_weights(self, list_D, indices):
nlabel = len(self.labelset)
N = np.maximum(np.max(self.imask), np.max(indices))+ 1
omega = 1./nlabel * np.ones((nlabel,N))
omega[:,self.imask] = self.anchor['data']
omega = omega[:,indices]
import logging
import numpy as np
import svm_worker
import utils_logging
logger = utils_logging.get_logger('latent_svm',utils_logging.DEBUG)
from struct_svm import StructSVM
class LatentSVM(object):
def __init__(
self,
loss_function,
psi,
most_violated_constraint,
annotation_consistent_inference,
**kwargs):
## store user function
self.user_loss = loss_function
self.user_psi = psi
self.user_mvc = most_violated_constraint
self.user_aci = annotation_consistent_inference
self.struct_params = kwargs ## pass parameters to struct SVM
## set latent SVM specific parameters
self.niter_max = kwargs.pop('latent_niter_max',100)
self.epsilon = kwargs.pop('latent_epsilon',1e-3)
## parallel stuff
self.use_parallel = kwargs.pop('latent_use_parallel',False)
from scipy import sparse
from scipy.sparse import linalg as splinalg
def solve_qp(P,q,**kwargs):
'''
solve: min(X): Xt*P*X + 2Xt*q + cst
'''
if P.nnz==0:
logger.error('P has no non-zeros entries')
return np.zeros(q.size)
maxiter = int(kwargs.pop('maxiter',1000))
tol = kwargs.pop('tol',1e-3)
## use standard conjugate gradient
x,info = splinalg.cg(
P,-q,
maxiter=maxiter,
tol=tol,
)
if info!=0:
logger.error('QP did not converge. info={}'.format(info))
return x
import utils_logging
logger = utils_logging.get_logger('solver_qp',utils_logging.DEBUG)
import numpy as np
from scipy import ndimage
import rwsegment
reload(rwsegment)
from rwsegment import BaseAnchorAPI
import utils_logging
logger = utils_logging.get_logger('rwsegment_prior_models',utils_logging.DEBUG)
'''
class PriorModel(BaseAnchorAPI):
def __init__(self,*args, **kwargs):
super(PriorModel,self).__init__(*args, **kwargs)
self.init_model()
def init_model(self):
pass
def get_anchor_and_weights(self, D, indices):
nlabel = len(self.labelset)
N = np.maximum(np.max(self.imask), np.max(indices))+ 1
data = 1./nlabel * np.ones((nlabel,N))
data[:,self.imask] = self.anchor['data']
data = data[:,indices]
weights = self.anchor_weight * np.ones((nlabel,len(indices)))
return data, weights
'''
class Constant(BaseAnchorAPI):
def get_anchor_and_weights(self, D, indices):
data, weights = super(Constant, self).get_anchor_and_weights(1, indices)
return data, weights # *1
def solve_qp(P, q, **kwargs):
'''
solve: min(X): Xt*P*X + 2Xt*q + cst
'''
if P.nnz == 0:
logger.error('P has no non-zeros entries')
return np.zeros(q.size)
maxiter = int(kwargs.pop('maxiter', 1000))
tol = kwargs.pop('tol', 1e-3)
## use standard conjugate gradient
x, info = splinalg.cg(
P,
-q,
maxiter=maxiter,
tol=tol,
)
if info != 0:
logger.error('QP did not converge. info={}'.format(info))
return x
import utils_logging
logger = utils_logging.get_logger('solver_qp', utils_logging.DEBUG)
--data-urlencode 'Body=Hello! I am Deiner testing my code.!!!'
-u MYUSER:MYPASSWORD
"""
######################################################################################
import time
import datetime
from credentials import *
from twilio.rest import Client
from utils_logging import get_logger
# Your Account Sid and Auth Token from twilio.com/console
# DANGER! This is insecure. See http://twil.io/secure
######################################################################################
account_sid = TWILIO['ID']
auth_token = TWILIO['TOKEN']
######################################################################################
logger = get_logger('sms_twilio.py', level='DEBUG')
######################################################################################
twilio_fiels_whatsapp = [
"originalMessage",
"SmsMessageSid",
"NumMedia",
"SmsSid",
"SmsStatus",
"Body",
"To",
"NumSegments",
"MessageSid",
"AccountSid",
"From",
"ApiVersion",
"chatId",
Work in progress code.
Do not copy, do not distribute without permission !
Pierre-Yves Baudin 2012
'''
import sys
import os
import numpy as np
from scipy import sparse
import utils_logging
logger = utils_logging.get_logger('rwsegment',utils_logging.DEBUG)
##------------------------------------------------------------------------------
def segment(
image,
anchor_api,
seeds=[],
weight_function=None,
return_arguments=['image'],
**kwargs
):
''' segment an image with method from MICCAI'12
arguments:
image = image to segment
anchor_api:
import numpy as np
import gc
import mpi
import utils_logging
logger = utils_logging.get_logger('svm_worker',utils_logging.INFO)
#from struct_svm import DataContainer
def broadcast(task, *args, **kwargs):
''' send data to SVM worker '''
comm = mpi.COMM
nproc = mpi.SIZE
ndata = len(args[0])
all = np.arange(ndata)
## send
for iproc in range(1,nproc):
some = all[np.mod(all, nproc-1) == (iproc-1)]
comm.send((task, len(some), kwargs), dest=iproc) # send initial message
for i in some:
comm.send(tuple([i]+[arg[i] for arg in args]), dest=iproc)
## receive
output = []
for n in range(ndata):
source_proc = np.mod(n, comm.Get_size()-1) + 1
output.append(comm.recv(source=source_proc, tag=n))
return output
import sys
import mosek
import utils_logging as logging
logger = logging.get_logger('svm_solver', logging.INFO)
class SVMSolver(object):
def __init__(self, C, **kwargs):
''' solve quadratic program
min ztPz + 2*ztq
st. Az <= b
Here:
min 1/2wtw + Cxi
st. for all (y1_,...yn_) in W
1/n wt sum(k){psi(xk,yk) - psi(xk,yk_)}
>= 1/n sum(k) {loss(yk,yk_) - xi
Our data structure:
W = [{'psis':[],'losses':[]}, {'psis':[],'losses':[]}]
'''
self.C = C
self.use_mosek = kwargs.pop('use_mosek', True)
self.Cprime = kwargs.pop('Cprime', 0)
def objective(self, w, xi, wref=None):
obj = 0.5 * sum([val**2 for val in w]) + self.C * xi
if wref is not None:
obj += 0.5 * self.Cprime * sum([(w1-w2)**2 for w1,w2 in zip(w,wref)])
return obj
import logging
import numpy as np
import svm_worker
import utils_logging
logger = utils_logging.get_logger('latent_svm', utils_logging.DEBUG)
from struct_svm import StructSVM
class LatentSVM(object):
def __init__(self, loss_function, psi, most_violated_constraint,
annotation_consistent_inference, **kwargs):
## store user function
self.user_loss = loss_function
self.user_psi = psi
self.user_mvc = most_violated_constraint
self.user_aci = annotation_consistent_inference
self.struct_params = kwargs ## pass parameters to struct SVM
## set latent SVM specific parameters
self.niter_max = kwargs.pop('latent_niter_max', 100)
self.epsilon = kwargs.pop('latent_epsilon', 1e-3)
## parallel stuff
self.use_parallel = kwargs.pop('latent_use_parallel', False)
def sequential_all_aci(self, w, xs, zs, metas, y0s):
ys = []
for x, z, y0, meta in zip(xs, zs, y0s, metas):
import numpy as np
from scipy import ndimage
import rwsegment
reload(rwsegment)
from rwsegment import BaseAnchorAPI
import utils_logging
logger = utils_logging.get_logger('rwsegment_prior_models',
utils_logging.DEBUG)
'''
class PriorModel(BaseAnchorAPI):
def __init__(self,*args, **kwargs):
super(PriorModel,self).__init__(*args, **kwargs)
self.init_model()
def init_model(self):
pass
def get_anchor_and_weights(self, D, indices):
nlabel = len(self.labelset)
N = np.maximum(np.max(self.imask), np.max(indices))+ 1
data = 1./nlabel * np.ones((nlabel,N))
data[:,self.imask] = self.anchor['data']
data = data[:,indices]
weights = self.anchor_weight * np.ones((nlabel,len(indices)))
return data, weights
'''
class Constant(BaseAnchorAPI):
def get_anchor_and_weights(self, D, indices):
data, weights = super(Constant,
self).get_anchor_and_weights(1, indices)
u = u + step * u_nt
else:
logger.warning('Newton"s method did not converge in {} iterations'.format(self.maxiter))
# raise Exception('Did not converge in {} iterations'\
# .format(self.maxiter))
## return output
return u
def extract_diag(self, H):
## extract diagonal from H
ijH = H.nonzero()
ondiagH = np.equal(*ijH)
diagH = np.zeros(H.shape[0])
diagH[ijH[0][ondiagH]] = H.data[ondiagH]
return diagH
def line_search(self, u, du, gradu):
t = 1.0
objective = self.objective
b = float(self.b)
while objective(u + t * du) > objective(u) + self.a * t * gradu.T * du:
t = b * t
return t
import utils_logging
logger = utils_logging.get_logger("solver_qp_constr", utils_logging.DEBUG)
else:
logger.warning(
'Newton"s method did not converge in {} iterations'.format(
self.maxiter))
#raise Exception('Did not converge in {} iterations'\
# .format(self.maxiter))
## return output
return u
def extract_diag(self, H):
## extract diagonal from H
ijH = H.nonzero()
ondiagH = np.equal(*ijH)
diagH = np.zeros(H.shape[0])
diagH[ijH[0][ondiagH]] = H.data[ondiagH]
return diagH
def line_search(self, u, du, gradu):
t = 1.0
objective = self.objective
b = float(self.b)
while objective(u + t * du) > objective(u) + self.a * t * gradu.T * du:
t = b * t
return t
import utils_logging
logger = utils_logging.get_logger('solver_qp_constr', utils_logging.DEBUG)
import numpy as np
import hashlib
import mpi
import logging
import svm_solver
reload(svm_solver)
import svm_worker
reload(svm_worker)
import utils_logging
logger = utils_logging.get_logger('struct_svm', utils_logging.DEBUG)
'''
class DataContainer(object):
def __init__(self, data):
self.data = data
def __hash__(self):
return id(self.data)
'''
class StructSVM(object):
'''
"Cutting-plane training of structural SVMs"
Machine Learning 2009
Joachims, Thorsten
Finley, Thomas
Yu, Chun-Nam John
'''
def __init__(
self,
'''
Work in progress code.
Do not copy, do not distribute without permission !
Pierre-Yves Baudin 2012
'''
import sys
import os
import numpy as np
from scipy import sparse
import utils_logging
logger = utils_logging.get_logger('rwsegment', utils_logging.INFO)
class BaseAnchorAPI(object):
def __init__(self, anchor, anchor_weight=1.0, **kwargs):
self.anchor = anchor
self.labelset = anchor['labelset']
self.imask = anchor['imask']
self.anchor_weight = anchor_weight
def get_labelset(self):
return self.labelset
def get_anchor_and_weights(self, list_D, indices):
nlabel = len(self.labelset)
N = np.maximum(np.max(self.imask), np.max(indices)) + 1
omega = 1. / nlabel * np.ones((nlabel, N))
omega[:, self.imask] = self.anchor['data']
import numpy as np
import gc
import mpi
import utils_logging
logger = utils_logging.get_logger('svm_worker', utils_logging.INFO)
#from struct_svm import DataContainer
def broadcast(task, *args, **kwargs):
''' send data to SVM worker '''
comm = mpi.COMM
nproc = mpi.SIZE
ndata = len(args[0])
all = np.arange(ndata)
## send
for iproc in range(1, nproc):
some = all[np.mod(all, nproc - 1) == (iproc - 1)]
comm.send((task, len(some), kwargs),
dest=iproc) # send initial message
for i in some:
comm.send(tuple([i] + [arg[i] for arg in args]), dest=iproc)
## receive
output = []
for n in range(ndata):
source_proc = np.mod(n, comm.Get_size() - 1) + 1
output.append(comm.recv(source=source_proc, tag=n))
return output
