def put(self, destfile, localfile):
"""
:param destfile:
ftp path for the localfile
:param localfile:
localfile
"""
ret = {'returncode': 0, 'msg': 'success'}
log.info('to put localfile {0} to ftp {1}'.format(localfile, destfile))
self._check_timeout()
cwd = self._ftp_con.pwd()
destdir = None
destfile = os.path.normpath(destfile)
destfile = self._get_relative_path(destfile, cwd)
rindex = destfile.rfind('/')
if rindex < 0:
destdir = cwd
file_name = destfile
elif rindex >= (len(destfile) - 1):
raise ValueError('value error, destfile {0}'.format(destfile))
else:
destdir = destfile[:rindex]
file_name = destfile.split('/')[-1]
log.info('put localfile {0} into ftp {1}'.format(localfile, destfile))
with open(localfile, 'rb') as fhandle:
try:
self._ftp_con.cwd(destdir)
ftp_cmd = 'STOR {0}'.format(file_name)
self._ftp_con.storbinary(ftp_cmd, fhandle)
except Exception as error:
ret['returncode'] = -1
ret['msg'] = 'failed to put, err:{0}'.format(error)
self._ftp_con.cwd(cwd)
return ret
def __init__(self, config):
"""
:param config:
{
"uri":"ftp://host:port",
"user":"******",
"password":"******",
"extra":None //timeout:30s
}
:raise: tlib.err.ConfigError if there's any config item missing
"""
ObjectInterface.__init__(self, config)
required_keys = ['uri', 'user', 'passwords']
if not self._validate_config(self._config, required_keys):
raise err.ConfigError(str(required_keys))
self._uri = self._config['uri']
self._user = self._config['user']
self._passwd = self._config['passwords']
self._extra = self._config['extra']
self._dufault_timeout = 30
if self._extra is not None and isinstance(self._config['extra'], int):
self._dufault_timeout = self._extra
log.info('to connect to ftp server')
self._ftp_con = ftplib.FTP()
self._host = self._uri.split(':')[1][2:]
self._port = ftplib.FTP_PORT
if len(self._uri.split(':')[2]) > 0:
self.port = int(self._uri.split(':')[2])
self._ftp_con.connect(self._host, self._port, self._dufault_timeout)
self._ftp_con.login(self._user, self._passwd)
self._last_optime = time.time()
self._timeout = 15 # idle time for ftp
def __init__(self, config):
"""
:param config: Shoule be dict like object
:raise: tlib.err.ConfigError if there's any config item missing
"""
ObjectInterface.__init__(self, config)
required_keys = ['ak', 'sk', 'endpoint', 'bucket']
if not self._validate_config(self._config, required_keys):
raise err.ConfigError(str(required_keys))
self._ak = self._config['ak']
self._sk = self._config['sk']
self._endpoint = self._config['endpoint']
self._bucket = self._config['bucket']
import boto3
from botocore import exceptions
from botocore import client as coreclient
self._s3_config = coreclient.Config(signature_version='s3v4',
s3={'addressing_style': 'path'})
logging.getLogger('boto3').setLevel(logging.INFO)
logging.getLogger('botocore').setLevel(logging.INFO)
logging.getLogger('s3transfer').setLevel(logging.INFO)
log.info('to connect to boto3')
self.__s3conn = boto3.client(
's3',
aws_access_key_id=self._ak,
aws_secret_access_key=self._sk,
endpoint_url=self._endpoint,
# region_name=conf_dict['region_name'],
config=self._s3_config)
self._exception = exceptions.ClientError
def delete(self, path):
"""delete file"""
ret = {'returncode': 0, 'msg': 'success'}
log.info('to delete ftp file: {0}'.format(path))
self._check_timeout()
cwd = os.path.normpath(self._ftp_con.pwd())
path = self._get_relative_path(path, cwd)
try:
self._ftp_con.delete(path)
except Exception as error:
ret['returncode'] = -1
ret['msg'] = str(error)
return ret
def _gis(self, maxiter):
"""
迭代尺度算法
"""
self._init_params()
for i in range(maxiter):
log.info("Iter:%s............." % i)
self.last_W = self.W[:]
self._model_ep()
###权重更新
for j, w in enumerate(self.W):
self.W[j] += 1.0 / self.C * math.log(self.ep_[j] / self.ep[j])
log.info("W:%s" % self.W)
if self._check_convergence():
break
def get(self, path, localpath):
"""
get a file into localpath
"""
ret = {'returncode': 0, 'msg': 'success'}
log.info('to get ftp file {0} to {1}'.format(path, localpath))
self._check_timeout()
cwd = self._ftp_con.pwd()
path = self._get_relative_path(path, cwd)
if localpath.endswith('/'):
localpath += path.split('/')[-1]
log.info('to get ftp {0} to local {1}'.format(path, localpath))
try:
with open(localpath, 'w+') as fhandle:
ftp_cmd = 'RETR {0}'.format(path)
resp = self._ftp_con.retrbinary(ftp_cmd, fhandle.write)
except Exception as error:
ret['returncode'] = -1
ret['msg'] = 'failed to get {0} to {1}, err:{2}'.format(
path, localpath, error)
log.error(ret['msg'])
return ret
def _pipe_asshell(cmd):
"""
run shell with subprocess.Popen
"""
tempscript = tempfile.NamedTemporaryFile(
dir=self._tmpdir, prefix=self._tmpprefix,
delete=True
)
with open(tempscript.name, 'w+b') as fhandle:
fhandle.write('cd {0};\n'.format(os.getcwd()))
fhandle.write(cmd)
shexe = self.which('sh')
cmds = [shexe, tempscript.name]
log.info(
'tlib shell execute {0} with script {1}'.format(
cmd, cmds)
)
self._subpro = subprocess.Popen(
cmds, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, preexec_fn=_signal_handle
)
self._subpro_data = self._subpro.communicate()
def _target(argcontent):
argcontent.tempscript = tempfile.NamedTemporaryFile(
dir=self._tmpdir, prefix=self._tmpprefix,
delete=True
)
with open(argcontent.tempscript.name, 'w+b') as fhandle:
fhandle.write('cd {0};\n'.format(os.getcwd()))
fhandle.write(argcontent.cmd)
shexe = self.which('sh')
cmds = [shexe, argcontent.tempscript.name]
log.info(
'to async execute {0} with script {1}'.format(
argcontent.cmd, cmds)
)
try:
argcontent.subproc = subprocess.Popen(
cmds, stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
preexec_fn=_signal_handle)
except OSError:
argcontent.ret['returncode'] = -1
argcontent.ret['stderr'] = (
'failed to execute the cmd, plz check it out\'s'
)
log.info("Iter:%s............." % i)
self.last_W = self.W[:]
self._model_ep()
###权重更新
for j, w in enumerate(self.W):
self.W[j] += 1.0 / self.C * math.log(self.ep_[j] / self.ep[j])
log.info("W:%s" % self.W)
if self._check_convergence():
break
def train(self, maxiter=1000, alg_type="gis"):
"""
@maxiter 最大迭代次数
@alg_type 算法选择 [gis|] gis迭代尺度算法
"""
if alg_type == "gis":
self._gis(maxiter)
def predict(self, sample):
X = re.split(r"\s+", sample)
p = self._pyx(X)
return p
if __name__ == "__main__":
maxent = MaxEntropy()
maxent.load_data()
maxent.train(maxiter=1000)
p = maxent.predict("sunny hot high FALSE")
log.info("predict:%s" % p)
for k in range(iter_times):
h = sigmoid(dataMatrix*weights)
error = (h - labelMat)
weights -= alpha*dataMatrix.transpose() * error
return weights
def gradAscentBatch(self, alpha=0.001, iter_times=100, batch_size=5):
"""
随机梯度下降算法
动态更新学习速率
@batch_size 随机抽batch_size个样本进行权重更新
"""
dataMatrix = mat(self.dataMat)
labelMat = mat(self.labelMat).transpose()
m, n = shape(dataMatrix)
weights = ones((n, 1))
for k in range(iter_times):
alpha = 4 / (k + 5) + 0.001
tmp_data_mat, tmp_label_mat = batchMatrix(self.dataMat, self.labelMat, batch_size)
h = sigmoid(tmp_data_mat * weights)
error = h - tmp_label_mat
weights -= alpha*tmp_data_mat.transpose()*error
return weights
if __name__ == "__main__":
lr = LogicRegression()
lr.loadDataSet()
weights = lr.gradAscentBatch()
log.info("f(x)=%s+%sx1+%sx2" % (weights[0][0], weights[1][0], weights[2][0]))