def uploadFileByFtp(iTup):
# ip, port, ftpDir, user, passwd, passiveVal, fileList
count = 0
firstTime = datetime.datetime.now()
ftp = ftpModule.createFtpLink(iTup[0], iTup[1], iTup[2], iTup[3], iTup[4],
iTup[5])
# logService.initLogging()
pidCount = 0
for i in iTup[6]:
nowtime = datetime.datetime.now()
if 100 <= (nowtime - firstTime).seconds:
ftp.quit()
ftp = ftpModule.createFtpLink(iTup[0], iTup[1], iTup[2], iTup[3],
iTup[4], iTup[5])
firstTime = datetime.datetime.now()
count += 1
ftpModule.uploadFile(ftp, i)
timeStr = str(time.strftime('%Y-%m-%d-%H-%M-%S', time.localtime()))
print "%s pid:%s--file: %s has been uploaded." % (timeStr, os.getpid(),
i)
infoStr = "%s pid:%s--file: %s has been uploaded." % (timeStr,
os.getpid(), i)
logging.info(infoStr)
pidCount = pidCount + 1
ftp.quit()
countDic = {}
countDic["reconnectNum"] = count
countDic["uploadNum"] = pidCount
return countDic
def newUploadFileByFtp(iTup):
# ip, port, ftpDir, user, passwd, passiveVal, fileList
ftp = ftpModule.createFtpLink(iTup[0], iTup[1], iTup[2], iTup[3], iTup[4],
iTup[5])
# logService.initLogging()
uploadCount = 0
discardCount = 0
# 限流，如果令牌桶中有令牌，则上传图片，如果没有，则丢弃图片
# 每秒往桶中存放3个令牌
token = tokenBucket.TokenBucket(3, 100000)
token._current_amount = 1000
for i in iTup[6]:
if token.consume(1):
ftpModule.uploadFile(ftp, i)
timeStr = str(time.strftime('%Y-%m-%d-%H-%M-%S', time.localtime()))
print "%s pid:%s--file: %s has been uploaded." % (timeStr,
os.getpid(), i)
infoStr = "%s pid:%s--file: %s has been uploaded." % (
timeStr, os.getpid(), i)
logging.info(infoStr)
uploadCount = uploadCount + 1
else:
discardCount = discardCount + 1
print time.time()
ftp.quit()
countDic = {}
countDic["uploadNum"] = uploadCount
countDic["discardNum"] = discardCount
return countDic
def bucketUploadFileByFtp(iTup):
# ip, port, ftpDir, user, passwd, passiveVal, fileList
#ftp = ftpModule.createFtpLink(iTup[0], iTup[1], iTup[2], iTup[3], iTup[4], iTup[5])
# logService.initLogging()
uploadCount = 0
# 限流，如果令牌桶中有令牌，则上传图片，如果没有，则空循环
# 每秒往桶中存放3个令牌
token = tokenBucket.TokenBucket(3, 100000)
token._current_amount = 1
index = 0
try:
while index < len(iTup[6]):
if token.consume(1):
ftp = ftpModule.createFtpLink(iTup[0], iTup[1], iTup[2],
iTup[3], iTup[4], iTup[5])
ftpModule.uploadFile(ftp, iTup[6][index])
timeStr = str(
time.strftime('%Y-%m-%d-%H-%M-%S', time.localtime()))
print "%s pid:%s--file: %s has been uploaded." % (
timeStr, os.getpid(), iTup[6][index])
uploadCount = uploadCount + 1
index = index + 1
ftp.quit()
#ftp.quit()
except Exception, e:
raise MYEXCEPTION.MyException(e)
# -*- coding: utf-8 -*-
"""
Created on 2018/4/28
@author: Simba
"""
import os
import pickle
import multiprocessing
import time
import ftpModule
def check(num):
print num/0
def some():
pool = multiprocessing.Pool(processes=3)
for i in range(10,20):
pool.apply_async(check, (i,))
# "47.105.36.177", "21", "ftpuser", "aivivi"
ftp = ftpModule.createFtpLink("47.105.36.177", "21", "aaa", "ftpuser", "aivivi", True,)
ftp.quit()
n = int(math.ceil(len(arr) / float(m)))
return [arr[i:i + n] for i in range(0, len(arr), n)]
# # ftp上传多文件
# def uploadManyFiles(ftp, fileList):
def uploadFileTup(Tup):
print "========"
#print Tup
print "%s-------%s" % (Tup, os.getpid())
#ftpModule.uploadFile(Tup[0], Tup[1])
ftp = ftpModule.createFtpLink("47.104.253.105", "2121", "test", "test", "test",
True)
def run():
parameterList = sys.argv
# 获得进程数
numOfProcess = parameterList[1]
filePathList = getImagesPathList(r"F:\newImages\newImages")
pool = multiprocessing.Pool(processes=int(numOfProcess))
# for i in range(100):
# pool.apply_async(ftpModule.uploadFile(), args=(ftp, ))
print len(filePathList)