def run(port=2121, passive_ports=range(60000, 65535), masquerade_address=None):
authorizer = CosAuthorizer()
for login_user, login_password, home_dir, permission in CosFtpConfig(
).login_users:
perm = ""
if "R" in permission:
perm = perm + authorizer.read_perms
if "W" in permission:
perm = perm + authorizer.write_perms
authorizer.add_user(login_user, login_password, home_dir, perm=perm)
handler = CosFtpHandler
handler.authorizer = authorizer
handler.abstracted_fs = CosFileSystem
handler.banner = "Welcome to COS FTP Service"
handler.permit_foreign_addresses = True
if masquerade_address is not None:
handler.masquerade_address = masquerade_address
handler.passive_ports = passive_ports
server = FTPServer(("0.0.0.0", port), handler)
server.max_cons = CosFtpConfig().max_connection_num
print "starting ftp server..."
try:
server.serve_forever()
finally:
server.close_all()
def main():
port = CosFtpConfig().listen_port
external_ip = CosFtpConfig().masquerade_address
passive_ports = CosFtpConfig().passive_ports
run(port=port, masquerade_address=external_ip, passive_ports=passive_ports)
def __init__(self, cos_client, bucket_name, object_name=None):
self._cos_client = cos_client
self._bucket_name = bucket_name
self._key_name = object_name
if CosFtpConfig().single_file_max_size > 40 * 1000 * ftp_v5.conf.common_config.GIGABYTE:
logger.error("Max file size: %d is too big. Thread: %s" % (
CosFtpConfig().single_file_max_size, threading.currentThread().getName()))
raise ValueError("Max file size: %d is too big" % CosFtpConfig().single_file_max_size)
self._min_part_size = int(math.ceil(float(CosFtpConfig().single_file_max_size) / MultipartUpload.MaxiumPartNum))
if self._min_part_size < StreamUploader.MIN_PART_SIZE:
self._min_part_size = StreamUploader.MIN_PART_SIZE # part size 最小限制为1MB
logger.info("Min part size: %d" % self._min_part_size)
self._has_init = False
self._has_commit = False
self._buffer = FifoBuffer()
self._buffer_len = 0
self._multipart_uploader = None
self._part_num = 1 # 当前上传的分片号
self._uploaded_part = dict() # 已经成功上传的分片
self._uploaded_len = 0 # 已经上传字节数
self._upload_pool = None # 用于并发上传的线程池子
def rename(self, src, dest):
logger.info("User invoke rename for {0} to {1}".format(
str(src).encode("utf-8"),
str(dest).encode("utf-8")))
logger.info("Current work directory: {0}".format(
str(self.cwd).encode("utf-8")))
assert isinstance(src, unicode), src
assert isinstance(dest, unicode), dest
if src == dest:
return
if self.isfile(src):
src_key_name = self.fs2ftp(src)[1:] # 去除头部的/
dest_key_name = self.fs2ftp(dest)[1:] # 去除头部的/
copy_source = dict()
copy_source["Bucket"] = self._bucket_name
copy_source["Key"] = src_key_name # XXX 该不该带斜线
copy_source['Region'] = CosFtpConfig().get_user_info(
self.root)['cos_region']
copy_source['Endpoint'] = CosFtpConfig().get_user_info(
self.root)['cos_endpoint']
try:
response = self._cos_client.copy_object(
Bucket=self._bucket_name,
Key=dest_key_name,
CopySource=copy_source,
CopyStatus='Copy')
self._cos_client.delete_object(Bucket=self._bucket_name,
Key=src_key_name)
except CosClientError as e:
logger.exception("Rename " + str(src).encode("utf-8") +
" to " + str(dest).encode("utf-8") +
"occurs an CosClientError.")
raise FilesystemError(
"Rename {0} to {1} failed.".format(
str(src).encode("utf-8")),
str(dest).encode("utf-8"))
except CosServiceError as e:
logger.exception("Rename " + str(src).encode("utf-8") + "to" +
str(dest).encode("utf-8") +
"occurs an CosServiceError.")
raise FilesystemError("Rename {0} to {1} failed.".format(
str(src).encode("utf-8"),
str(dest).encode("utf-8")))
except Exception as e:
logger.exception("Rename " + str(src).encode("utf-8") + "to" +
str(dest).encode("utf-8") +
"occurs an exception.")
raise FilesystemError("Rename {0} to {1} failed.".format(
str(src).encode("utf-8"),
str(dest).encode("utf-8")))
elif self.isdir(src):
raise FilesystemError("Directory renaming is not supported")
else:
raise FilesystemError("Invalid parameter!")
def __init__(self, *args, **kwargs):
super(CosFileSystem, self).__init__(*args, **kwargs)
self._cos_client = CosS3Client(CosConfig(
Appid=CosFtpConfig().appid,
Region=CosFtpConfig().region,
Access_id=CosFtpConfig().secretid,
Access_key=CosFtpConfig().secretkey),
retry=3)
self._bucket_name = CosFtpConfig().bucket
def __init__(self):
if UploadPool._isInit: # 如果已经初始化就不再初始化了
return
logger.info("init pool")
self._thread_pool = ThreadPool(CosFtpConfig().upload_thread_num) # 固定线程池的大小
self._thread_num = CosFtpConfig().upload_thread_num # 线程数目
self._semaphore = threading.Semaphore(CosFtpConfig().upload_thread_num) # 控制线程数目
self._reference_threads = set() # 引用计数
UploadPool._isInit = True
def write(self, data):
logger.debug("Receive string with length : {0} Thread: {1}".format(
len(data),
threading.currentThread().getName()))
self._buffer.write(data)
self._buffer_len += len(data)
if self._uploaded_len > CosFtpConfig().single_file_max_size:
logger.error(
"Uploading file: {0} exceeds the maximum file limit: {1}".
format(
self._key_name,
str(CosFtpConfig().single_file_max_size).encode("utf-8")))
raise IOError(
"Uploading file: {0} exceeds the maximum file limit: {1}".
format(
self._key_name,
str(CosFtpConfig().single_file_max_size).encode("utf-8")))
while self._buffer_len >= self._min_part_size:
if not self._has_init:
self._upload_pool = UploadPool()
response = self._cos_client.create_multipart_upload(
Bucket=self._bucket_name, Key=self._key_name)
self._multipart_uploader = MultipartUpload(
self._cos_client, response)
self._part_num = 1
self._isSuccess = None
self._has_init = True
def callback(part_num, result):
with StreamUploader._lock:
self._uploaded_part[str(part_num)] = result
self._isSuccess = result
def check_finish():
if self._isSuccess is not None and not self._isSuccess:
err_msg = "Uploading file:{0} failed.".format(
self._key_name)
logger.error(err_msg)
raise IOError(err_msg)
check_finish()
self._uploaded_part[str(self._part_num)] = None
self._upload_pool.apply_task(
self._multipart_uploader.upload_part, (self._buffer.read(
self._min_part_size), self._part_num, callback))
self._part_num += 1
self._buffer_len -= self._min_part_size # 只要提交到并发上传的线程池中，就可以减掉了
self._uploaded_len += self._min_part_size
logger.info("upload new part with length: {0} File: {1}".format(
self._min_part_size, self._key_name))
def __init__(self, *args, **kwargs):
super(CosFileSystem, self).__init__(*args, **kwargs)
self._cos_client = CosS3Client(
CosConfig(
Region=CosFtpConfig().get_user_info(self.root)['cos_region'],
# Endpoint=CosFtpConfig().get_user_info(self.root)['cos_endpoint'],
Scheme=CosFtpConfig().get_user_info(self.root)['cos_protocol'],
Access_id=CosFtpConfig().get_user_info(
self.root)["cos_secretid"],
Access_key=CosFtpConfig().get_user_info(
self.root)['cos_secretkey'],
# UA=version.user_agent
),
retry=3)
self._bucket_name = CosFtpConfig().get_user_info(self.root)["bucket"]
def validate_authentication(self, user_name, password, handler):
for login_user_name, login_password, login_permission in CosFtpConfig(
).login_users:
if user_name == login_user_name and password == login_password:
return True
raise AuthenticationFailed
def remove(self, path):
logger.info("user invoke remove for {0}".format(
str(path).encode("utf-8")))
logger.info("Current work directory {0}".format(
str(self.cwd).encode("utf-8")))
assert isinstance(path, unicode), path
if not CosFtpConfig().get_user_info(self.root)['delete_enable']:
raise FilesystemError("Current user is not allowed to delete.")
if self.isfile(path):
key_name = self.fs2ftp(path).strip("/")
logger.debug("key_name: {0}".format(str(key_name).encode("utf-8")))
try:
response = self._cos_client.delete_object(
Bucket=self._bucket_name, Key=key_name)
except CosClientError as e:
logger.exception(
"Remove file:{0} occurs a CosClientError.".format(
str(key_name).encode("utf-8")))
raise FilesystemError("Remove file:{0} occurs error.".format(
str(path).encode("utf-8")))
except CosServiceError as e:
logger.exception(
"Remove file:{0} occurs a CosServiceError.".format(
str(key_name).encode("utf-8")))
raise FilesystemError("Remove file:{0} occurs error.".format(
str(path).encode("utf-8")))
except Exception as e:
logger.exception("Remove file:{0} occurs an exception.".format(
str(key_name).encode("utf-8")))
raise FilesystemError("Remove file:{0} occurs error.".format(
str(path).encode("utf-8")))
logger.debug("response: {0}".format(str(response).encode("utf-8")))
def __init__(self, *args, **kwargs):
super(CosFileSystem, self).__init__(*args, **kwargs)
if CosFtpConfig().host is not None:
self._cos_client = CosS3Client(CosConfig(
Appid=CosFtpConfig().appid,
Region=None,
Access_id=CosFtpConfig().secretid,
Access_key=CosFtpConfig().secretkey,
Host=CosFtpConfig().host),
retry=3)
else:
self._cos_client = CosS3Client(CosConfig(
Appid=CosFtpConfig().appid,
Region=CosFtpConfig().region,
Access_id=CosFtpConfig().secretid,
Access_key=CosFtpConfig().secretkey,
Host=None),
retry=3)
self._bucket_name = CosFtpConfig().bucket
def rmdir(self, path):
logger.info("user invoke rmdir for {0}".format(
str(path).encode("utf-8")))
logger.info("Current work directory {0}".format(
str(self.cwd).encode("utf-8")))
assert isinstance(path, six.text_type), path
if not CosFtpConfig().get_user_info(self.root)['delete_enable']:
raise FilesystemError("Current user is not allowed to delete.")
if self.isdir(path):
dir_name = self.fs2ftp(path).strip("/") + "/"
logger.debug("dir_name:{0}".format(str(dir_name).encode("utf-8")))
response = self._cos_client.delete_object(Bucket=self._bucket_name,
Key=dir_name)
logger.debug("response:{0}".format(str(response).encode("utf-8")))
def main():
# 首先校验配置的合理性
CosFtpConfig.check_config(CosFtpConfig())
port = CosFtpConfig().listen_port
external_ip = CosFtpConfig().masquerade_address
passive_ports = CosFtpConfig().passive_ports
run(port=port, masquerade_address=external_ip, passive_ports=passive_ports)
def __init__(self, cos_client, bucket_name, object_name=None):
self._cos_client = cos_client
self._bucket_name = bucket_name
self._key_name = object_name
self._min_part_size = int(
math.ceil(
float(CosFtpConfig().single_file_max_size) /
MultipartUpload.MaxiumPartNum))
if self._min_part_size < StreamUploader.MIN_PART_SIZE:
self._min_part_size = StreamUploader.MIN_PART_SIZE # part size 最小限制为1MB
logger.info("Min part size: %d" % self._min_part_size)
self._has_init = False
self._has_commit = False
self._buffer = FifoBuffer()
self._buffer_len = 0
self._multipart_uploader = None
self._part_num = 1 # 当前上传的分片号
self._uploaded_part = dict() # 已经成功上传的分片
self._uploaded_len = 0 # 已经上传字节数
self._upload_pool = None # 用于并发上传的线程池子
class StreamUploader(object):
MIN_PART_SIZE = CosFtpConfig().min_part_size
MAX_PART_SIZE = 5 * ftp_v5.conf.common_config.GIGABYTE
UPLOAD_THREAD_NUM = CosFtpConfig().upload_thread_num
_lock = threading.Lock()
def __init__(self, cos_client, bucket_name, object_name=None):
self._cos_client = cos_client
self._bucket_name = bucket_name
self._key_name = object_name
if CosFtpConfig().single_file_max_size > 40 * 1000 * ftp_v5.conf.common_config.GIGABYTE:
logger.error("Max file size: %d is too big. Thread: %s" % (
CosFtpConfig().single_file_max_size, threading.currentThread().getName()))
raise ValueError("Max file size: %d is too big" % CosFtpConfig().single_file_max_size)
self._min_part_size = int(math.ceil(float(CosFtpConfig().single_file_max_size) / MultipartUpload.MaxiumPartNum))
if self._min_part_size < StreamUploader.MIN_PART_SIZE:
self._min_part_size = StreamUploader.MIN_PART_SIZE # part size 最小限制为1MB
logger.info("Min part size: %d" % self._min_part_size)
self._has_init = False
self._has_commit = False
self._buffer = FifoBuffer()
self._buffer_len = 0
self._multipart_uploader = None
self._part_num = 1 # 当前上传的分片号
self._uploaded_part = dict() # 已经成功上传的分片
self._uploaded_len = 0 # 已经上传字节数
self._upload_pool = None # 用于并发上传的线程池子
def write(self, data):
logger.debug(
"Receive string with length : {0} Thread: {1}".format(len(data), threading.currentThread().getName()))
self._buffer.write(data)
self._buffer_len += len(data)
if self._uploaded_len > CosFtpConfig().single_file_max_size:
logger.error("Uploading file: {0} exceeds the maximum file limit: {1}".format(self._key_name, str(
CosFtpConfig().single_file_max_size).encode("utf-8")))
raise IOError("Uploading file: {0} exceeds the maximum file limit: {1}".format(self._key_name, str(
CosFtpConfig().single_file_max_size).encode("utf-8")))
while self._buffer_len >= self._min_part_size:
if not self._has_init:
self._upload_pool = UploadPool()
response = self._cos_client.create_multipart_upload(Bucket=self._bucket_name, Key=self._key_name)
self._multipart_uploader = MultipartUpload(self._cos_client, response)
self._part_num = 1
self._isSuccess = None
self._has_init = True
def callback(part_num, result):
with StreamUploader._lock:
self._uploaded_part[str(part_num)] = result
self._isSuccess = result
def check_finish():
if self._isSuccess is not None and not self._isSuccess:
err_msg = "Uploading file:{0} failed.".format(self._key_name)
logger.error(err_msg)
raise IOError(err_msg)
check_finish()
self._uploaded_part[str(self._part_num)] = None
self._upload_pool.apply_task(self._multipart_uploader.upload_part,
(self._buffer.read(self._min_part_size), self._part_num, callback))
self._part_num += 1
self._buffer_len -= self._min_part_size # 只要提交到并发上传的线程池中，就可以减掉了
self._uploaded_len += self._min_part_size
logger.info("upload new part with length: {0} File: {1}".format(self._min_part_size, self._key_name))
def _wait_for_finish(self):
is_finish = False
while not is_finish:
if len(self._uploaded_part) == 0:
return
for part_num, result in self._uploaded_part.items():
if result is not None and not result:
logger.error("Uploading file failed. Failed part_num: %d " % int(part_num))
raise IOError("Uploading part_num: %d failed. " % int(part_num))
if result is None:
break
else:
is_finish = True
if not is_finish:
time.sleep(10 / 1000) # 休眠10毫秒
def close(self):
logger.info(
"Closing the stream upload... File: {}, Thread: {}".format(self._key_name, threading.currentThread()))
if self._buffer_len != 0:
if not self._has_init:
# 采用简单文件上传
logger.info("Simple file upload! File: {0}".format(self._key_name))
self._cos_client.put_object(Bucket=self._bucket_name,
Body=self._buffer.read(self._buffer_len),
Key=self._key_name)
else:
logger.info("Upload the last part File：{0}".format(self._key_name))
self._multipart_uploader.upload_part(self._buffer.read(self._buffer_len), self._part_num)
if self._has_init:
logger.info("Wait for all the tasks to finish.")
self._wait_for_finish()
self._multipart_uploader.complete_upload()
self._uploaded_len = 0
self._buffer.close()
def test():
cos_client = CosS3Client(
CosConfig(Appid=CosFtpConfig().appid,
Access_id=CosFtpConfig().secretid,
Access_key=CosFtpConfig().secretkey,
Region=CosFtpConfig().region))
def listdir(self, path):
logger.info("user invoke listdir for {0}".format(
str(path).encode("utf-8")))
logger.info("Current work directory {0}".format(
str(self.cwd).encode("utf-8")))
assert isinstance(path, unicode), path
ftp_path = self.fs2ftp(path)
logger.debug("ftp_path: {0}".format(str(ftp_path).encode("utf-8")))
dir_name = ftp_path
logger.debug("dir_name: {0}".format(str(dir_name).encode("utf-8")))
list_name = list()
max_list_file = CosFtpConfig().max_list_file
if dir_name == "/": # 如果是根目录
isTruncated = True
next_marker = str("")
while isTruncated and max_list_file > 0 and next_marker is not None:
try:
response = self._cos_client.list_objects(
Bucket=self._bucket_name,
Delimiter="/",
Marker=next_marker)
tmp_list = self._gen_list(response)
list_name.extend(tmp_list)
max_list_file -= len(tmp_list)
if response['IsTruncated'] == 'true':
isTruncated = True
next_marker = response['NextMarker']
else:
isTruncated = False
except CosClientError as e:
logger.exception("Dir path:" + dir_name, e)
raise FilesystemError(
"list dir:{0} failed.".format(ftp_path))
except CosServiceError as e:
logger.exception("Dir path:" + dir_name, e)
raise FilesystemError(
"list dir:{0} failed.".format(ftp_path))
except Exception as e:
logger.exception("Dir path:" + dir_name, e)
raise FilesystemError(
"list dir:{0} failed.".format(ftp_path))
return list_name
if len(dir_name.split("/")) >= 2: # 二级以上目录
isTruncated = True
next_marker = str("")
while isTruncated and max_list_file > 0 and next_marker is not None:
try:
response = self._cos_client.list_objects(
Bucket=self._bucket_name,
Prefix=(dir_name.strip("/") + "/"),
Delimiter="/",
Marker=next_marker)
tmp_list = self._gen_list(response)
list_name.extend(tmp_list)
max_list_file -= len(tmp_list)
if response['IsTruncated'] == 'true':
isTruncated = True
next_marker = response['NextMarker']
else:
isTruncated = False
except CosClientError as e:
logger.exception("Dir path:" + dir_name, e)
raise FilesystemError(
"list dir:{0} failed.".format(ftp_path))
except CosServiceError as e:
logger.exception("Dir path:" + dir_name, e)
raise FilesystemError(
"list dir:{0} failed.".format(ftp_path))
except Exception as e:
logger.exception("Dir path:" + dir_name, e)
raise FilesystemError(
"list dir:{0} failed.".format(ftp_path))
return list_name
import urllib
import logging
import sys
import copy
import xml.dom.minidom
import xml.etree.ElementTree
from requests import Request, Session
from streambody import StreamBody
from xml2dict import Xml2Dict
from cos_auth import CosS3Auth
from cos_exception import CosClientError
from cos_exception import CosServiceError
from ftp_v5.conf.ftp_config import CosFtpConfig
logging.basicConfig(
level=CosFtpConfig().log_level,
format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s',
datefmt='%a, %d %b %Y %H:%M:%S',
filename=CosFtpConfig().log_filename,
filemode='w')
logger = logging.getLogger(__name__)
reload(sys)
sys.setdefaultencoding('utf-8')
maplist = {
'ContentLength': 'Content-Length',
'ContentType': 'Content-Type',
'ContentMD5': 'Content-MD5',
'CacheControl': 'Cache-Control',
'ContentDisposition': 'Content-Disposition',
'ContentEncoding': 'Content-Encoding',
#!/usr/bin/env python
# -*- coding:utf-8 -*-
import logging
from pyftpdlib.servers import FTPServer, MultiprocessFTPServer
from ftp_v5.cos_authorizer import CosAuthorizer
from ftp_v5.cos_ftp_handler import CosFtpHandler
from ftp_v5.cos_file_system import CosFileSystem
from ftp_v5.conf.ftp_config import CosFtpConfig
logging.basicConfig(filename='pyftpd.log', level=CosFtpConfig().log_level)
def run(port=2121, passive_ports=range(60000, 65535), masquerade_address=None):
authorizer = CosAuthorizer()
for login_user, login_password, permission in CosFtpConfig().login_users:
perm = ""
if "R" in permission:
perm = perm + authorizer.read_perms
if "W" in permission:
perm = perm + authorizer.write_perms
authorizer.add_user(login_user,
login_password,
CosFtpConfig().homedir,
perm=perm)
handler = CosFtpHandler
handler.authorizer = authorizer
handler.abstracted_fs = CosFileSystem