def run(port=2121, passive_ports=range(60000, 65535), masquerade_address=None): authorizer = CosAuthorizer() for login_user, login_password, home_dir, permission in CosFtpConfig( ).login_users: perm = "" if "R" in permission: perm = perm + authorizer.read_perms if "W" in permission: perm = perm + authorizer.write_perms authorizer.add_user(login_user, login_password, home_dir, perm=perm) handler = CosFtpHandler handler.authorizer = authorizer handler.abstracted_fs = CosFileSystem handler.banner = "Welcome to COS FTP Service" handler.permit_foreign_addresses = True if masquerade_address is not None: handler.masquerade_address = masquerade_address handler.passive_ports = passive_ports server = FTPServer(("0.0.0.0", port), handler) server.max_cons = CosFtpConfig().max_connection_num print "starting ftp server..." try: server.serve_forever() finally: server.close_all()
def main(): port = CosFtpConfig().listen_port external_ip = CosFtpConfig().masquerade_address passive_ports = CosFtpConfig().passive_ports run(port=port, masquerade_address=external_ip, passive_ports=passive_ports)
def __init__(self, cos_client, bucket_name, object_name=None): self._cos_client = cos_client self._bucket_name = bucket_name self._key_name = object_name if CosFtpConfig().single_file_max_size > 40 * 1000 * ftp_v5.conf.common_config.GIGABYTE: logger.error("Max file size: %d is too big. Thread: %s" % ( CosFtpConfig().single_file_max_size, threading.currentThread().getName())) raise ValueError("Max file size: %d is too big" % CosFtpConfig().single_file_max_size) self._min_part_size = int(math.ceil(float(CosFtpConfig().single_file_max_size) / MultipartUpload.MaxiumPartNum)) if self._min_part_size < StreamUploader.MIN_PART_SIZE: self._min_part_size = StreamUploader.MIN_PART_SIZE # part size 最小限制为1MB logger.info("Min part size: %d" % self._min_part_size) self._has_init = False self._has_commit = False self._buffer = FifoBuffer() self._buffer_len = 0 self._multipart_uploader = None self._part_num = 1 # 当前上传的分片号 self._uploaded_part = dict() # 已经成功上传的分片 self._uploaded_len = 0 # 已经上传字节数 self._upload_pool = None # 用于并发上传的线程池子
def rename(self, src, dest): logger.info("User invoke rename for {0} to {1}".format( str(src).encode("utf-8"), str(dest).encode("utf-8"))) logger.info("Current work directory: {0}".format( str(self.cwd).encode("utf-8"))) assert isinstance(src, unicode), src assert isinstance(dest, unicode), dest if src == dest: return if self.isfile(src): src_key_name = self.fs2ftp(src)[1:] # 去除头部的/ dest_key_name = self.fs2ftp(dest)[1:] # 去除头部的/ copy_source = dict() copy_source["Bucket"] = self._bucket_name copy_source["Key"] = src_key_name # XXX 该不该带斜线 copy_source['Region'] = CosFtpConfig().get_user_info( self.root)['cos_region'] copy_source['Endpoint'] = CosFtpConfig().get_user_info( self.root)['cos_endpoint'] try: response = self._cos_client.copy_object( Bucket=self._bucket_name, Key=dest_key_name, CopySource=copy_source, CopyStatus='Copy') self._cos_client.delete_object(Bucket=self._bucket_name, Key=src_key_name) except CosClientError as e: logger.exception("Rename " + str(src).encode("utf-8") + " to " + str(dest).encode("utf-8") + "occurs an CosClientError.") raise FilesystemError( "Rename {0} to {1} failed.".format( str(src).encode("utf-8")), str(dest).encode("utf-8")) except CosServiceError as e: logger.exception("Rename " + str(src).encode("utf-8") + "to" + str(dest).encode("utf-8") + "occurs an CosServiceError.") raise FilesystemError("Rename {0} to {1} failed.".format( str(src).encode("utf-8"), str(dest).encode("utf-8"))) except Exception as e: logger.exception("Rename " + str(src).encode("utf-8") + "to" + str(dest).encode("utf-8") + "occurs an exception.") raise FilesystemError("Rename {0} to {1} failed.".format( str(src).encode("utf-8"), str(dest).encode("utf-8"))) elif self.isdir(src): raise FilesystemError("Directory renaming is not supported") else: raise FilesystemError("Invalid parameter!")
def __init__(self, *args, **kwargs): super(CosFileSystem, self).__init__(*args, **kwargs) self._cos_client = CosS3Client(CosConfig( Appid=CosFtpConfig().appid, Region=CosFtpConfig().region, Access_id=CosFtpConfig().secretid, Access_key=CosFtpConfig().secretkey), retry=3) self._bucket_name = CosFtpConfig().bucket
def __init__(self): if UploadPool._isInit: # 如果已经初始化就不再初始化了 return logger.info("init pool") self._thread_pool = ThreadPool(CosFtpConfig().upload_thread_num) # 固定线程池的大小 self._thread_num = CosFtpConfig().upload_thread_num # 线程数目 self._semaphore = threading.Semaphore(CosFtpConfig().upload_thread_num) # 控制线程数目 self._reference_threads = set() # 引用计数 UploadPool._isInit = True
def write(self, data): logger.debug("Receive string with length : {0} Thread: {1}".format( len(data), threading.currentThread().getName())) self._buffer.write(data) self._buffer_len += len(data) if self._uploaded_len > CosFtpConfig().single_file_max_size: logger.error( "Uploading file: {0} exceeds the maximum file limit: {1}". format( self._key_name, str(CosFtpConfig().single_file_max_size).encode("utf-8"))) raise IOError( "Uploading file: {0} exceeds the maximum file limit: {1}". format( self._key_name, str(CosFtpConfig().single_file_max_size).encode("utf-8"))) while self._buffer_len >= self._min_part_size: if not self._has_init: self._upload_pool = UploadPool() response = self._cos_client.create_multipart_upload( Bucket=self._bucket_name, Key=self._key_name) self._multipart_uploader = MultipartUpload( self._cos_client, response) self._part_num = 1 self._isSuccess = None self._has_init = True def callback(part_num, result): with StreamUploader._lock: self._uploaded_part[str(part_num)] = result self._isSuccess = result def check_finish(): if self._isSuccess is not None and not self._isSuccess: err_msg = "Uploading file:{0} failed.".format( self._key_name) logger.error(err_msg) raise IOError(err_msg) check_finish() self._uploaded_part[str(self._part_num)] = None self._upload_pool.apply_task( self._multipart_uploader.upload_part, (self._buffer.read( self._min_part_size), self._part_num, callback)) self._part_num += 1 self._buffer_len -= self._min_part_size # 只要提交到并发上传的线程池中,就可以减掉了 self._uploaded_len += self._min_part_size logger.info("upload new part with length: {0} File: {1}".format( self._min_part_size, self._key_name))
def __init__(self, *args, **kwargs): super(CosFileSystem, self).__init__(*args, **kwargs) self._cos_client = CosS3Client( CosConfig( Region=CosFtpConfig().get_user_info(self.root)['cos_region'], # Endpoint=CosFtpConfig().get_user_info(self.root)['cos_endpoint'], Scheme=CosFtpConfig().get_user_info(self.root)['cos_protocol'], Access_id=CosFtpConfig().get_user_info( self.root)["cos_secretid"], Access_key=CosFtpConfig().get_user_info( self.root)['cos_secretkey'], # UA=version.user_agent ), retry=3) self._bucket_name = CosFtpConfig().get_user_info(self.root)["bucket"]
def validate_authentication(self, user_name, password, handler): for login_user_name, login_password, login_permission in CosFtpConfig( ).login_users: if user_name == login_user_name and password == login_password: return True raise AuthenticationFailed
def remove(self, path): logger.info("user invoke remove for {0}".format( str(path).encode("utf-8"))) logger.info("Current work directory {0}".format( str(self.cwd).encode("utf-8"))) assert isinstance(path, unicode), path if not CosFtpConfig().get_user_info(self.root)['delete_enable']: raise FilesystemError("Current user is not allowed to delete.") if self.isfile(path): key_name = self.fs2ftp(path).strip("/") logger.debug("key_name: {0}".format(str(key_name).encode("utf-8"))) try: response = self._cos_client.delete_object( Bucket=self._bucket_name, Key=key_name) except CosClientError as e: logger.exception( "Remove file:{0} occurs a CosClientError.".format( str(key_name).encode("utf-8"))) raise FilesystemError("Remove file:{0} occurs error.".format( str(path).encode("utf-8"))) except CosServiceError as e: logger.exception( "Remove file:{0} occurs a CosServiceError.".format( str(key_name).encode("utf-8"))) raise FilesystemError("Remove file:{0} occurs error.".format( str(path).encode("utf-8"))) except Exception as e: logger.exception("Remove file:{0} occurs an exception.".format( str(key_name).encode("utf-8"))) raise FilesystemError("Remove file:{0} occurs error.".format( str(path).encode("utf-8"))) logger.debug("response: {0}".format(str(response).encode("utf-8")))
def __init__(self, *args, **kwargs): super(CosFileSystem, self).__init__(*args, **kwargs) if CosFtpConfig().host is not None: self._cos_client = CosS3Client(CosConfig( Appid=CosFtpConfig().appid, Region=None, Access_id=CosFtpConfig().secretid, Access_key=CosFtpConfig().secretkey, Host=CosFtpConfig().host), retry=3) else: self._cos_client = CosS3Client(CosConfig( Appid=CosFtpConfig().appid, Region=CosFtpConfig().region, Access_id=CosFtpConfig().secretid, Access_key=CosFtpConfig().secretkey, Host=None), retry=3) self._bucket_name = CosFtpConfig().bucket
def rmdir(self, path): logger.info("user invoke rmdir for {0}".format( str(path).encode("utf-8"))) logger.info("Current work directory {0}".format( str(self.cwd).encode("utf-8"))) assert isinstance(path, six.text_type), path if not CosFtpConfig().get_user_info(self.root)['delete_enable']: raise FilesystemError("Current user is not allowed to delete.") if self.isdir(path): dir_name = self.fs2ftp(path).strip("/") + "/" logger.debug("dir_name:{0}".format(str(dir_name).encode("utf-8"))) response = self._cos_client.delete_object(Bucket=self._bucket_name, Key=dir_name) logger.debug("response:{0}".format(str(response).encode("utf-8")))
def main(): # 首先校验配置的合理性 CosFtpConfig.check_config(CosFtpConfig()) port = CosFtpConfig().listen_port external_ip = CosFtpConfig().masquerade_address passive_ports = CosFtpConfig().passive_ports run(port=port, masquerade_address=external_ip, passive_ports=passive_ports)
def __init__(self, cos_client, bucket_name, object_name=None): self._cos_client = cos_client self._bucket_name = bucket_name self._key_name = object_name self._min_part_size = int( math.ceil( float(CosFtpConfig().single_file_max_size) / MultipartUpload.MaxiumPartNum)) if self._min_part_size < StreamUploader.MIN_PART_SIZE: self._min_part_size = StreamUploader.MIN_PART_SIZE # part size 最小限制为1MB logger.info("Min part size: %d" % self._min_part_size) self._has_init = False self._has_commit = False self._buffer = FifoBuffer() self._buffer_len = 0 self._multipart_uploader = None self._part_num = 1 # 当前上传的分片号 self._uploaded_part = dict() # 已经成功上传的分片 self._uploaded_len = 0 # 已经上传字节数 self._upload_pool = None # 用于并发上传的线程池子
class StreamUploader(object): MIN_PART_SIZE = CosFtpConfig().min_part_size MAX_PART_SIZE = 5 * ftp_v5.conf.common_config.GIGABYTE UPLOAD_THREAD_NUM = CosFtpConfig().upload_thread_num _lock = threading.Lock() def __init__(self, cos_client, bucket_name, object_name=None): self._cos_client = cos_client self._bucket_name = bucket_name self._key_name = object_name if CosFtpConfig().single_file_max_size > 40 * 1000 * ftp_v5.conf.common_config.GIGABYTE: logger.error("Max file size: %d is too big. Thread: %s" % ( CosFtpConfig().single_file_max_size, threading.currentThread().getName())) raise ValueError("Max file size: %d is too big" % CosFtpConfig().single_file_max_size) self._min_part_size = int(math.ceil(float(CosFtpConfig().single_file_max_size) / MultipartUpload.MaxiumPartNum)) if self._min_part_size < StreamUploader.MIN_PART_SIZE: self._min_part_size = StreamUploader.MIN_PART_SIZE # part size 最小限制为1MB logger.info("Min part size: %d" % self._min_part_size) self._has_init = False self._has_commit = False self._buffer = FifoBuffer() self._buffer_len = 0 self._multipart_uploader = None self._part_num = 1 # 当前上传的分片号 self._uploaded_part = dict() # 已经成功上传的分片 self._uploaded_len = 0 # 已经上传字节数 self._upload_pool = None # 用于并发上传的线程池子 def write(self, data): logger.debug( "Receive string with length : {0} Thread: {1}".format(len(data), threading.currentThread().getName())) self._buffer.write(data) self._buffer_len += len(data) if self._uploaded_len > CosFtpConfig().single_file_max_size: logger.error("Uploading file: {0} exceeds the maximum file limit: {1}".format(self._key_name, str( CosFtpConfig().single_file_max_size).encode("utf-8"))) raise IOError("Uploading file: {0} exceeds the maximum file limit: {1}".format(self._key_name, str( CosFtpConfig().single_file_max_size).encode("utf-8"))) while self._buffer_len >= self._min_part_size: if not self._has_init: self._upload_pool = UploadPool() response = self._cos_client.create_multipart_upload(Bucket=self._bucket_name, Key=self._key_name) self._multipart_uploader = MultipartUpload(self._cos_client, response) self._part_num = 1 self._isSuccess = None self._has_init = True def callback(part_num, result): with StreamUploader._lock: self._uploaded_part[str(part_num)] = result self._isSuccess = result def check_finish(): if self._isSuccess is not None and not self._isSuccess: err_msg = "Uploading file:{0} failed.".format(self._key_name) logger.error(err_msg) raise IOError(err_msg) check_finish() self._uploaded_part[str(self._part_num)] = None self._upload_pool.apply_task(self._multipart_uploader.upload_part, (self._buffer.read(self._min_part_size), self._part_num, callback)) self._part_num += 1 self._buffer_len -= self._min_part_size # 只要提交到并发上传的线程池中,就可以减掉了 self._uploaded_len += self._min_part_size logger.info("upload new part with length: {0} File: {1}".format(self._min_part_size, self._key_name)) def _wait_for_finish(self): is_finish = False while not is_finish: if len(self._uploaded_part) == 0: return for part_num, result in self._uploaded_part.items(): if result is not None and not result: logger.error("Uploading file failed. Failed part_num: %d " % int(part_num)) raise IOError("Uploading part_num: %d failed. " % int(part_num)) if result is None: break else: is_finish = True if not is_finish: time.sleep(10 / 1000) # 休眠10毫秒 def close(self): logger.info( "Closing the stream upload... File: {}, Thread: {}".format(self._key_name, threading.currentThread())) if self._buffer_len != 0: if not self._has_init: # 采用简单文件上传 logger.info("Simple file upload! File: {0}".format(self._key_name)) self._cos_client.put_object(Bucket=self._bucket_name, Body=self._buffer.read(self._buffer_len), Key=self._key_name) else: logger.info("Upload the last part File:{0}".format(self._key_name)) self._multipart_uploader.upload_part(self._buffer.read(self._buffer_len), self._part_num) if self._has_init: logger.info("Wait for all the tasks to finish.") self._wait_for_finish() self._multipart_uploader.complete_upload() self._uploaded_len = 0 self._buffer.close()
def test(): cos_client = CosS3Client( CosConfig(Appid=CosFtpConfig().appid, Access_id=CosFtpConfig().secretid, Access_key=CosFtpConfig().secretkey, Region=CosFtpConfig().region))
def listdir(self, path): logger.info("user invoke listdir for {0}".format( str(path).encode("utf-8"))) logger.info("Current work directory {0}".format( str(self.cwd).encode("utf-8"))) assert isinstance(path, unicode), path ftp_path = self.fs2ftp(path) logger.debug("ftp_path: {0}".format(str(ftp_path).encode("utf-8"))) dir_name = ftp_path logger.debug("dir_name: {0}".format(str(dir_name).encode("utf-8"))) list_name = list() max_list_file = CosFtpConfig().max_list_file if dir_name == "/": # 如果是根目录 isTruncated = True next_marker = str("") while isTruncated and max_list_file > 0 and next_marker is not None: try: response = self._cos_client.list_objects( Bucket=self._bucket_name, Delimiter="/", Marker=next_marker) tmp_list = self._gen_list(response) list_name.extend(tmp_list) max_list_file -= len(tmp_list) if response['IsTruncated'] == 'true': isTruncated = True next_marker = response['NextMarker'] else: isTruncated = False except CosClientError as e: logger.exception("Dir path:" + dir_name, e) raise FilesystemError( "list dir:{0} failed.".format(ftp_path)) except CosServiceError as e: logger.exception("Dir path:" + dir_name, e) raise FilesystemError( "list dir:{0} failed.".format(ftp_path)) except Exception as e: logger.exception("Dir path:" + dir_name, e) raise FilesystemError( "list dir:{0} failed.".format(ftp_path)) return list_name if len(dir_name.split("/")) >= 2: # 二级以上目录 isTruncated = True next_marker = str("") while isTruncated and max_list_file > 0 and next_marker is not None: try: response = self._cos_client.list_objects( Bucket=self._bucket_name, Prefix=(dir_name.strip("/") + "/"), Delimiter="/", Marker=next_marker) tmp_list = self._gen_list(response) list_name.extend(tmp_list) max_list_file -= len(tmp_list) if response['IsTruncated'] == 'true': isTruncated = True next_marker = response['NextMarker'] else: isTruncated = False except CosClientError as e: logger.exception("Dir path:" + dir_name, e) raise FilesystemError( "list dir:{0} failed.".format(ftp_path)) except CosServiceError as e: logger.exception("Dir path:" + dir_name, e) raise FilesystemError( "list dir:{0} failed.".format(ftp_path)) except Exception as e: logger.exception("Dir path:" + dir_name, e) raise FilesystemError( "list dir:{0} failed.".format(ftp_path)) return list_name
import urllib import logging import sys import copy import xml.dom.minidom import xml.etree.ElementTree from requests import Request, Session from streambody import StreamBody from xml2dict import Xml2Dict from cos_auth import CosS3Auth from cos_exception import CosClientError from cos_exception import CosServiceError from ftp_v5.conf.ftp_config import CosFtpConfig logging.basicConfig( level=CosFtpConfig().log_level, format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s', datefmt='%a, %d %b %Y %H:%M:%S', filename=CosFtpConfig().log_filename, filemode='w') logger = logging.getLogger(__name__) reload(sys) sys.setdefaultencoding('utf-8') maplist = { 'ContentLength': 'Content-Length', 'ContentType': 'Content-Type', 'ContentMD5': 'Content-MD5', 'CacheControl': 'Cache-Control', 'ContentDisposition': 'Content-Disposition', 'ContentEncoding': 'Content-Encoding',
#!/usr/bin/env python # -*- coding:utf-8 -*- import logging from pyftpdlib.servers import FTPServer, MultiprocessFTPServer from ftp_v5.cos_authorizer import CosAuthorizer from ftp_v5.cos_ftp_handler import CosFtpHandler from ftp_v5.cos_file_system import CosFileSystem from ftp_v5.conf.ftp_config import CosFtpConfig logging.basicConfig(filename='pyftpd.log', level=CosFtpConfig().log_level) def run(port=2121, passive_ports=range(60000, 65535), masquerade_address=None): authorizer = CosAuthorizer() for login_user, login_password, permission in CosFtpConfig().login_users: perm = "" if "R" in permission: perm = perm + authorizer.read_perms if "W" in permission: perm = perm + authorizer.write_perms authorizer.add_user(login_user, login_password, CosFtpConfig().homedir, perm=perm) handler = CosFtpHandler handler.authorizer = authorizer handler.abstracted_fs = CosFileSystem