def upload(self, local_path, remote_path, filter_regex=None, istree=False):
try:
# path prepare
local_path = self._local_path_transfor(local_path)
if os.path.isdir(local_path) and istree is False:
self.logger.warning("warning : use upload to upload tree")
istree = True
# osfs prepare
localfs, local_relative = self.build_osfs(local_path)
walker = None
# walk prepare
if filter_regex is not None:
if not isinstance(filter_regex, list):
filter_regex = list(filter_regex)
walker = Walker(filter=filter_regex)
# ftp prepare
ftp_args = self._ftp_path_transfor(remote_path)
ftpfs = FTPFS(host=ftp_args['host'],
port=ftp_args['port'],
passwd=ftp_args['password'],
user=ftp_args['user'])
if not istree:
ftp_local, ftp_file = self._parse_file_name(ftp_args['relative_path'])
try:
ftpfs.makedirs(ftp_local)
except Exception, error_msg:
self.logger.error(str(error_msg))
copy_file(localfs, local_relative, ftpfs, ftp_args['relative_path'])
else:
def __enter__(self) -> FTPFS:
self.base_path = self.conn_params.extra.get('base_path', '')
self.conn = FTPFS(
host=self.conn_params.host,
user=self.conn_params.login,
passwd=self.conn_params.password,
)
return self.conn
def test_parse_mlsx(self):
info = list(
FTPFS._parse_mlsx(["create=19740705000000;modify=19740705000000; /foo"])
)[0]
self.assertEqual(info["details"]["modified"], 142214400)
self.assertEqual(info["details"]["created"], 142214400)
info = list(FTPFS._parse_mlsx(["foo=bar; .."]))
self.assertEqual(info, [])
def test_parse_mlsx(self):
info = list(
FTPFS._parse_mlsx(
['create=19740705000000;modify=19740705000000; /foo']))[0]
self.assertEqual(info['details']['modified'], 142214400)
self.assertEqual(info['details']['created'], 142214400)
info = list(FTPFS._parse_mlsx(['foo=bar; ..']))
self.assertEqual(info, [])
def test_connection_error(self):
fs = FTPFS("ftp.not.a.chance", timeout=1)
with self.assertRaises(errors.RemoteConnectionError):
fs.listdir("/")
with self.assertRaises(errors.RemoteConnectionError):
fs.makedir("foo")
with self.assertRaises(errors.RemoteConnectionError):
fs.open("foo.txt")
def test_connection_error(self):
fs = FTPFS('ftp.not.a.chance', timeout=1)
with self.assertRaises(errors.RemoteConnectionError):
fs.listdir('/')
with self.assertRaises(errors.RemoteConnectionError):
fs.makedir('foo')
with self.assertRaises(errors.RemoteConnectionError):
fs.open('foo.txt')
def get_fs(cls, registry, fs_name, fs_name_params, fs_path, writeable, create_dir):
from fs.ftpfs import FTPFS
username, password, fs_path = _parse_credentials(fs_path)
scheme, _netloc, _path, _params, _query, _fragment = urlparse(fs_path)
if not scheme:
fs_path = 'ftp://' + fs_path
scheme, netloc, path, _params, _query, _fragment = urlparse(fs_path)
dirpath, resourcepath = pathsplit(path)
url = netloc
ftpfs = FTPFS(url, user=username or '', passwd=password or '', follow_symlinks=(fs_name_params == "symlinks"))
ftpfs.cache_hint(True)
if create_dir and path:
ftpfs.makedir(path, recursive=True, allow_recreate=True)
if dirpath:
ftpfs = ftpfs.opendir(dirpath)
if not resourcepath:
return ftpfs, None
else:
return ftpfs, resourcepath
class FTPHook(FileSystemHookInterface):
conn_type = 'ftp'
def __init__(self, conn_params: ConnectionParams):
self.conn_params = conn_params
def __enter__(self) -> FTPFS:
self.base_path = self.conn_params.extra.get('base_path', '')
self.conn = FTPFS(
host=self.conn_params.host,
user=self.conn_params.login,
passwd=self.conn_params.password,
)
return self.conn
def __exit__(self, exc_type, exc_val, exc_tb):
self.conn.close()
self.conn = None
def get_fs(cls, registry, fs_name, fs_name_params, fs_path, writeable, create_dir):
from fs.ftpfs import FTPFS
username, password, fs_path = _parse_credentials(fs_path)
scheme, _netloc, _path, _params, _query, _fragment = urlparse(fs_path)
if not scheme:
fs_path = 'ftp://' + fs_path
scheme, netloc, path, _params, _query, _fragment = urlparse(fs_path)
dirpath, resourcepath = pathsplit(path)
url = netloc
ftpfs = FTPFS(url, user=username or '', passwd=password or '')
ftpfs.cache_hint(True)
if create_dir and path:
ftpfs.makedir(path, recursive=True, allow_recreate=True)
if dirpath:
ftpfs = ftpfs.opendir(dirpath)
if not resourcepath:
return ftpfs, None
else:
return ftpfs, resourcepath
def make_fs(self):
from fs.ftpfs import FTPFS
global ftp_port_offset
temp_path = os.path.join(self._temp_dir, text_type(uuid.uuid4()))
_ftp_port = ftp_port + ftp_port_offset
ftp_port_offset += 1
os.mkdir(temp_path)
env = os.environ.copy()
server = subprocess.Popen(
[
sys.executable,
os.path.abspath(__file__),
temp_path,
text_type(_ftp_port)
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env
)
server.stdout.readline()
if _WINDOWS_PLATFORM:
# Don't know why this is necessary on Windows
time.sleep(0.1)
# Poll until a connection can be made
start_time = time.time()
while time.time() - start_time < 5:
try:
ftpurl = urlopen('ftp://127.0.0.1:{}'.format(_ftp_port))
except IOError:
time.sleep(0)
else:
ftpurl.read()
ftpurl.close()
break
else:
raise Exception("unable to start ftp server")
self.servers.append(server)
fs = FTPFS(
'127.0.0.1',
user='******',
passwd='12345',
port=_ftp_port,
timeout=5.0
)
return fs
def test_parse_mlsx_type(self):
lines = [
"Type=cdir;Modify=20180731114724;UNIX.mode=0755; /tmp",
"Type=pdir;Modify=20180731112024;UNIX.mode=0775; /",
"Type=file;Size=331523;Modify=20180731112041;UNIX.mode=0644; a.csv",
"Type=file;Size=368340;Modify=20180731112041;UNIX.mode=0644; b.csv",
]
expected = [
{
"basic": {
"name": "a.csv",
"is_dir": False
},
"ftp": {
"type": "file",
"size": "331523",
"modify": "20180731112041",
"unix.mode": "0644",
},
"details": {
"type": 2,
"size": 331523,
"modified": 1533036041
},
},
{
"basic": {
"name": "b.csv",
"is_dir": False
},
"ftp": {
"type": "file",
"size": "368340",
"modify": "20180731112041",
"unix.mode": "0644",
},
"details": {
"type": 2,
"size": 368340,
"modified": 1533036041
},
},
]
info = list(FTPFS._parse_mlsx(lines))
self.assertEqual(info, expected)
def __init__(self, creds_file):
creds = liteconfig.Config(creds_file)
self.worker = FTPFS(creds.host,
user=creds.user,
passwd=creds.passwd,
port=creds.port or 21)
def make_ftpfs(url):
"""Construct FTPFS from url."""
if url.scheme != "ftp":
raise ValueError("Scheme must be == 'ftp'")
return FTPFS(host=url.hostname, user=url.username, passwd=url.password, port=(url.port or 21))
def process_scan(req_tuple):
target_result = {}
req_data = dict(req_tuple)
target_result['name'] = req_data['name']
target_result['targets'] = []
targets = req_data['targets'] if 'targets' in req_data else []
arr_target = []
for target in targets:
sensitive_files_result = []
'''
==========================
| ftp |
==========================
'''
host = target['target']
arr_sensitive = []
if target['scanningToolOptions']['ftp']['enabled'] != False:
store_dir = []
store = []
try:
root_ftp = target['scanningToolOptions']['ftp']
username_ftp = root_ftp[
'username'] if 'username' in root_ftp else 'anonymous'
password_ftp = root_ftp[
'password'] if 'password' in root_ftp else ''
account_ftp = ''
timeout_ftp = 10
port_ftp = int(root_ftp['port']) if 'port' in root_ftp else 21
proxy_ftp = root_ftp['proxy'] if 'proxy' in root_ftp else None
tls_ftp = root_ftp['tls'] if 'tls' in root_ftp else False
home_fs = FTPFS(host=host,
user=username_ftp,
passwd=password_ftp,
acct=account_ftp,
timeout=timeout_ftp,
port=port_ftp,
proxy=proxy_ftp) #tls=tls_ftp
store_dir = home_fs.listdir('.')
for match in home_fs.glob("**/*"):
temp = match.path
if any(x in match.path.lower()
for x in list(keyword.split(","))):
store.append(temp)
store_arr = []
for res in store:
store_arr.append({'protocol': 'ftp', 'path': res})
for x in store_arr:
arr_sensitive.append(x)
except Exception as e:
print(e)
'''
==========================
| smb |
==========================
'''
if target['scanningToolOptions']['smb']['enabled'] != False:
store_share = []
store_path = []
try:
root_smb = target['scanningToolOptions']['smb']
username_smb = root_smb[
'username'] if 'username' in root_smb else ''
password_smb = root_smb[
'password'] if 'password' in root_smb else ''
domain_smb = root_smb['domain'] if 'domain' in root_smb else ''
port_smb = int(root_smb['port']) if 'port' in root_smb else 445
nameport_smb = int(
root_smb['namePort']) if 'namePort' in root_smb else 137
directtcp_smb = True
try:
ipaddress.ip_address(host)
ip_address_smb = host
except:
try:
ip_address_smb = socket.gethostbyname(host)
except:
ip_address_smb = host
conn = SMBConnection(username_smb,
password_smb,
'',
ip_address_smb,
domain=domain_smb,
is_direct_tcp=directtcp_smb)
assert conn.connect(ip_address_smb, port_smb)
filelist = conn.listShares()
if hidden == 'true':
for share in filelist:
store_share.append(share.name)
else:
for share in filelist:
if share.name[-1] != '$':
store_share.append(share.name)
# for share in store_share:
# try:
# smb_fs = fs.smbfs.SMBFS(host=(ip_address_smb,ip_address_smb), username=username_smb, passwd=password_smb, domain=domain_smb, port=port_smb, name_port=nameport_smb, direct_tcp=directtcp_smb)
# home_fs = smb_fs.opendir(share, factory=ClosingSubFS)
# except Exception as e:
# print(e)
# continue
# for match in home_fs.glob("**/*"):
# temp = match.path
# if any(x in match.path.lower() for x in list(keyword.split(','))):
# store_path.append(share + temp)
############new
try:
smb_fs = fs.smbfs.SMBFS(host=(ip_address_smb,
ip_address_smb),
username=username_smb,
passwd=password_smb,
domain=domain_smb,
port=port_smb,
name_port=nameport_smb,
direct_tcp=directtcp_smb)
# print(dir(smb_fs))
#store_path = []
def traverse(path, smb):
try:
fs = {
f: os.path.join(path, f)
for f in smb.listdir(path)
}
for f in fs:
# cur_p = fs[f]
cur_p = fs[f].replace('\\', '/')
if smb.isdir(cur_p) and not smb.islink(cur_p):
traverse(cur_p, smb)
elif not smb.islink(cur_p) and smb.isfile(
cur_p):
# store_path.append(cur_p)
if any(x in cur_p.lower()
for x in list(keyword.split(','))):
store_path.append(cur_p)
# print(cur_p)
except Exception as e:
pass
return
for share in store_share:
traverse(share, smb_fs)
#print(store_path)
except Exception as e:
print(e)
################new
store_arr = []
for path in store_path:
store_arr.append({'protocol': 'smb', 'path': path})
for x in store_arr:
arr_sensitive.append(x)
except Exception as e:
print(e)
arr_target.append({
'target': target['target'],
'sensitiveFiles': arr_sensitive
})
target_result['targets'] = arr_target
print(target_result)
with Api_Hackbot(HACKBOT_BASE_URL, HACKBOT_USERNAME,
HACKBOT_PASSWORD) as api:
print(api.post_to_hackbot(target_result))
def test_ftp(self):
base_test_dir = 'base_test_dir.{}'.format(uuid4().hex)
base_test_path = '/{}'.format(base_test_dir)
test_file = 'zxc.txt'
test_data = 'test_data.{}'.format(uuid4()).encode()
test_file_path = os.path.join(base_test_path, test_file)
def check_directory(client, should_exist):
# type: (FTPFileClient, bool)
result = client.list('/')
directory_list = result['directory_list']
for item in directory_list:
if item['name'] == base_test_dir:
if not should_exist:
raise ValueError('Directory `{}` should not exist'.format(
os.path.normpath(os.path.join(ftp_config.directory, base_test_dir))))
else:
self.assertTrue(item['is_dir'])
break
else:
if should_exist:
raise ValueError('Expected for directory `{}` to exist'.format(
os.path.normpath(os.path.join(ftp_config.directory, base_test_dir))))
# Create an embedded FTP server ..
server = FTPServer()
# .. start it in a new thread ..
server.start()
# .. wait a moment to make sure it is started ..
wait_until_port_taken(ftp_config.port)
# .. create an underlying FTP connection object ..
conn = FTPFS('localhost', ftp_config.username, ftp_config.password, port=ftp_config.port)
# .. create a higher-level FTP client ..
client = FTPFileClient(conn, {
'encoding': 'utf8'
})
# .. confirm we are connected ..
ping_response = client.ping()
self.assertTrue(ping_response)
# .. create a new directory ..
client.create_directory(base_test_dir)
# .. make sure the directory was created ..
check_directory(client, True)
# .. store a new file ..
client.store(test_file_path, test_data)
# .. download the uploaded file ..
received_data = client.get(test_file_path)
# .. compare the downloaded and uploaded data ..
self.assertEqual(received_data, test_data)
# .. delete the test directory ..
client.delete_directory(base_test_dir)
# .. make sure the directory was deleted ..
check_directory(client, False)
# .. stop the client ..
client.close()
# .. finally, stop the embedded server.
server.stop()
def test_parse_ftp_time(self):
self.assertIsNone(FTPFS._parse_ftp_time('notreallyatime'))
t = FTPFS._parse_ftp_time('19740705000000')
self.assertEqual(t, 142214400)
def _open_fs(self, user_context):
props = self._serialization_props(user_context)
handle = FTPFS(**props)
return handle
# ################################################################################################################################
# ################################################################################################################################
if __name__ == '__main__':
# pyfilesystem
from fs.ftpfs import FTPFS
host = 'localhost'
user = '******'
password = '******'
port = 11021
conn = FTPFS(host, user, password, port=port)
config = {'encoding': 'utf8'}
client = FTPFileClient(conn, config)
# client.create_directory('aaa2')
# client.delete_directory('aaa2')
path = '/aaa2/abc.txt2'
client.store(path, 'zzzz')
client.touch(path)
result = client.list('/aaa2')
def initialize_ftp(username, password, host, port):
"""Initializes FTP filesystem and returns associated instance"""
ftp_fs = FTPFS(host, user=username, passwd=password, port=port)
return ftp_fs
