def manage_multiple(src, dst, sport, dport, user, passwd, m, cmd_file):
commands = fetch_commands(cmd_file, m, user, passwd)
connections = []
for i in range(m):
connections.append(
FTPClient(src,
dst,
sport[i],
dport,
logfile='ftp-{}.log'.format(i)))
threads = []
for cmd in commands:
print('Running: ', cmd)
for i in range(m):
t = Thread(target=connections[i].run_command, args=(cmd[i], ))
t.start()
threads.append(t)
with open('ftp-{}.log'.format(i), 'a') as f:
f.write('>> {}\n'.format(cmd[i]))
for i in range(m):
threads[i].join()
print('Done')
threads = []
for i in range(m):
connections[i].quit()
return
class CreateWorkOrder(luigi.Task):
runDate = luigi.DateParameter(default=datetime.datetime.now())
ftp = FTPClient()
folder = FolderClient()
config = ConfigManager('cfg.ini')
def run(self):
with self.output().open('w') as wddump:
plist = {}
for p in PRODUCTLIST:
print('Getting file list for: ' + p)
plist[p] = self.ftp.listProductFiles(p)
plist['yearly'] = self.folder.listProductFiles(
'yearly', self.config.getAnnualFolder())
print('Writing file list')
json.dump(plist, wddump)
def output(self):
filePath = os.path.join(
os.path.join(S3_FILE_ROOT, self.runDate.strftime("%Y-%m-%d")),
'todo.json')
return S3Target(filePath)
def run(self):
while True:
address, path = self.queue.get()
if path == '/' and address in self.remaining:
continue
conn = FTPClient(address)
conn.login('username', 'mypass')
conn.create_data_connection()
list = conn.list(path)
conn.quit()
self.parse_list_and_send(list, path, address)
def get_file(path=None):
"""
this route and method is kept for backward compatibility.
"""
fpath = request.args.get("fpath")
if not fpath:
return abort(400, "Missing the mandatory parameter.")
protocol, host, username, password = get_connection_spec(
path, request.authorization)
if not host and not protocol:
return abort(400, "Cannot find the endpoint url for {}".format(path))
f_stream = None
client = None
try:
if protocol == "FTP":
client = FTPClient(username, password, host)
elif protocol == "FTPS":
client = FTPSClient(username, password, host)
elif protocol == "SFTP":
client = SFTPClient(username, password, host)
else:
return abort(400, "Not supported protocal.")
f_stream = client.get_stream(fpath)
f_stream.seek(0)
f_name = fpath.split("/")[-1]
client.quit()
return send_file(f_stream,
attachment_filename=f_name,
as_attachment=True)
except Exception as e:
logger.exception(e)
return abort(500, e)
def get_session(protocol, host, user, pwd):
session = None
if protocol == "FTP":
session = FTPClient(user, pwd, host)
elif protocol == "FTPS":
session = FTPSClient(user, pwd, host)
elif protocol == "SFTP":
session = SFTPClient(user, pwd, host)
return session
def main(args=None):
"""Entry point for chat_client"""
if len(sys.argv) < 5:
print("Invalid usage. Usage:")
print(
"{0} {{ write | read }} filename host:port {{ aes256 | aes128 | none }} [key]"
.format(str(sys.argv[0])))
elif len(sys.argv) > 6:
print("Invalid usage. Usage:")
print(
"{0} {{ write | read }} filename host:port {{ aes256 | aes128 | none }} [key]"
.format(str(sys.argv[0])))
else:
try:
command = str(sys.argv[1])
# make sure command is properly specified
if command != "read" and command != "write":
raise Exception("'command' must be 'read' or 'write'")
filename = str(sys.argv[2])
host_port = str(sys.argv[3])
cipher = str(sys.argv[4]).lower()
key = None
if len(sys.argv) == 6:
key = str(sys.argv[5])
# make sure cipher is properly specified
if cipher != "aes256" and cipher != "aes128" and cipher != "none":
raise Exception(
"cipher must be one of 'aes256', 'aes128', or 'none'")
# raise exception, key needs to be set if cipher is not none
if cipher != "none" and key is None:
raise Exception(
"key must be specified if cipher is not equal to 'none'")
(host_ip, port) = domain_resolution(host_port)
client = FTPClient(host_ip, port, command, filename, cipher, key)
except ValueError:
print("!! {0} does not specify a valid port number.".format(
sys.argv[3]))
print(traceback.format_exc())
except IPFormatError:
print("!! {0} does not specify a valid ip number.".format(
sys.argv[3]))
print(traceback.format_exc())
except socket.error as ex:
print("!! Error deriving domain name: {0}".format(
type(ex).__name__))
print(traceback.format_exc())
except:
print("!! Error: {0}".format(sys.exc_info()[1]))
print(traceback.format_exc())
def main():
"""
Takes in args, either creates and runs a server instance or client instance.
"""
ap = argparse.ArgumentParser()
ap.add_argument("-s", "--server", help="Create and run a FTP server instance", action="store_true")
ap.add_argument("-c", "--client", help="Create and run a FTP client instance", action="store_true")
args = ap.parse_args()
if args.server:
# create and run the server
server = FTPServer()
server.run_server()
if args.client:
# create and run client
client = FTPClient()
client.cmdloop()
if not args.server and not args.client:
# none specified so exit
sys.exit("You must specify to start and create either a client or server")
def manage_multiple(n, user, passwd, infile):
ports = []
for i in range(n):
p = random.randint(1024, 65535)
while p in ports:
p = random.randint(1024, 65535)
ports.append(p)
connections = []
for i in range(n):
logfile = 'ftp-client-{}.log'.format(i)
c = FTPClient(server_ip,
ports[i],
server_port,
interact=False,
logfile=logfile)
c.handshake()
connections.append(c)
run_cmd_multiple(connections, n, ['USER ' + user] * n)
run_cmd_multiple(connections, n, ['PASS ' + passwd] * n)
cmds = []
if infile:
with open(infile, 'r') as f:
cmds = f.read().split('\n')
cmd_list = []
for c in cmds:
if c.strip():
cmd_list.append(c.strip().split(','))
cmd_list = cmd_list[::-1]
while True:
if infile:
if not len(cmd_list):
c = []
else:
c = cmd_list.pop()
else:
c = fetch_commands(n)
# c += '\r\n'
print c
if len(c) != n:
if len(c):
print "Insufficient commands {} for {} connections".format(
','.join(c), n)
for i in range(n):
with open(connections[i].logfile, 'a') as f:
f.write('Exiting...\n\n')
connections[i].close()
return
run_cmd_multiple(connections, n, c)
from ftp_client import FTPClient
class BasicVar:
def __init__(self):
self.login_page = LoginPage()
self.addr_set_page = AddrSetPage()
self.rename_page = RenamePage()
self.download_page = DownloadPage()
self.mkdir_page = MkdirPage()
self.mode_setting_page = ModeSettingPage()
self.upload_page = UploadPage()
self.file_page = FilePage()
global_var = BasicVar()
client = FTPClient()
#确定按钮点击函数
def confirm_btn_clicked(main_window):
client.username = global_var.login_page.username_edit.text()
client.password = global_var.login_page.password_edit.text()
try:
client.cmd_connect()
except:
msg = QMessageBox(QMessageBox.Critical, "连接失败，检查ip和端口", "error")
msg.exec_()
return
if client.cmd_user() and client.cmd_pass():
client.cmd_syst()
global_var.file_page.setupUi(main_window)
msg = QMessageBox(QMessageBox.Information, "登录成功", "success")
class ProcessNetCDFFile(luigi.Task):
runDate = luigi.DateParameter(default=datetime.datetime.now())
product = luigi.Parameter()
srcFile = luigi.Parameter()
fileDate = luigi.Parameter()
ftp = FTPClient()
catalog = CatalogManager()
config = ConfigManager('cfg.ini')
metadata = NetCDFMetadata()
def run(self):
ncFile = os.path.join(
os.path.join(FILE_ROOT, self.runDate.strftime("%Y-%m-%d")),
self.product + '-' + self.fileDate + '.nc')
tiffFile = os.path.join(
os.path.join(FILE_ROOT, self.runDate.strftime("%Y-%m-%d")),
'UK-' + self.product + '-' + self.fileDate + '.tif')
s3DstFile = os.path.join(
os.path.join(
os.path.join(os.path.join('meo-ap/chlor_a', self.product),
self.fileDate[:4]), self.fileDate[4:6]),
'UK-' + self.product + '-' + self.fileDate + '.tif')
httpLoc = 'http://jncc-data.s3-eu-west-1.amazonaws.com/' + s3DstFile
# Get NetCDF file from FTP site
print('Retrieving ' + self.srcFile)
self.ftp.getFile(self.product, self.srcFile, ncFile)
# Get metadata from NetCDF (also acts as a validation check)
tc = self.metadata.getTimeCoverage(ncFile)
# Use GDAL to translate NetCDF to GeoTIFF and trim file to UK waters
os.system('gdal_translate NETCDF:' + ncFile +
':chlor_a -projwin -24 63 6 48 ' + tiffFile)
# Push file to S3
s3Helper.copy_file_to_s3(self.config.getAmazonKeyId(),
self.config.getAmazonKeySecret(),
self.config.getAmazonRegion(),
self.config.getAmazonBucketName(), tiffFile,
s3DstFile, True, None)
# Add entry to catalog
self.catalog.addEntry(
self.product, 'Chlorophyll-a Concentration for UK Waters - ' +
self.product + ' - ' + self.fileDate, self.srcFile, httpLoc,
tc['start'][:8], tc['end'][:8],
datetime.datetime.now().strftime("%Y-%m-%d"))
# Clean up intermediate files so we don't flood out /tmp
os.remove(ncFile)
os.remove(tiffFile)
return
def output(self):
filePath = os.path.join(
os.path.join(
os.path.join(os.path.join(S3_FILE_ROOT, self.product),
self.fileDate[:4]), self.fileDate[4:6]),
'UK-' + self.product + '-' + self.fileDate + '.tif')
return S3Target(filePath)
parser.add_argument('-m',
'--multiple',
help='Open multiple connections to FTP server',
nargs=1,
type=int)
parser.add_argument(
'-c',
'--command_file',
help='Run commands from given file for multi connections. \
Each line has n(number of multiple connections) comma seperated FTP commands.',
nargs=1,
type=str)
args = parser.parse_args()
server_ip = args.ipaddr[0]
server_port = args.port[0]
client_port = random.randint(1024, 65535)
if args.multiple:
print "Multiple mode"
num = args.multiple[0]
infile = None
if args.command_file:
infile = args.command_file[0]
manage_multiple(num, args.user[0], args.passwd[0], infile)
else:
client = FTPClient(server_ip, client_port, server_port)
client.handshake()
client.run_command('USER ' + args.user[0] + '\r\n')
client.run_command('PASS ' + args.passwd[0] + '\r\n')
client.interactive()
if args.source_port:
portaddr = args.source_port[0]
else:
portaddr = random.randint(1024, 65535)
src = ipaddr
dst = args.host[0]
sport = portaddr
dport = args.port[0]
username = args.user[0]
password = args.passwd[0]
if not args.multiple:
client = FTPClient(src, dst, sport, dport)
client.run_command("USER %s" % (username, ))
client.run_command("PASS %s" % (password, ))
# client.quit()
client.interactive()
else:
m = args.multiple[0]
if not args.cmd_file:
print('Give command file with -c argument')
sys.exit(-1)
cmd_file = args.cmd_file[0]
sport = []
while len(sport) < m: