def get_version(source):
host, port, username, password, encrypt = parse_source(source)
log.info('lookup "%s" ...', host)
ip = socket.gethostbyname(host)
localhost = socket.gethostname()
log.info('connect "%s" ...', ip)
conn = SMBConnection(username,
password,
localhost,
host,
use_ntlm_v2=True,
is_direct_tcp=True)
ready = conn.connect(ip, 445)
if not ready:
raise Exception("Connect failed, host, user or pass is not valid!")
# version magick
log.info('get version ...')
try:
attr = conn.getAttributes(ADMIN_SERVICE, MAGICK_FILENAME)
version = attr.last_write_time
except OperationFailure:
version = None
conn.close()
return format_version(version)
def samba_get(self, service_name, remotefile, localfile):
samba = SMBConnection(self.username, self.password, '', '')
samba.connect(self.ip, self.port)
def list_get(share_name, remotestuff, localstuff):
if os.path.isdir(localstuff) is False:
os.mkdir(localstuff)
for files in samba.listPath(share_name, remotestuff)[2:]:
localpath = localstuff + os.sep + files.filename
remotepath = remotestuff + '/' + files.filename
if not samba.getAttributes(share_name, remotepath).isDirectory:
handler = open(localpath, 'wb')
samba.retrieveFile(share_name, remotepath, handler)
handler.close()
else:
list_get(share_name, remotepath, localpath)
try:
if not samba.getAttributes(service_name, remotefile).isDirectory:
handle = open(localfile, 'wb')
samba.retrieveFile(service_name, remotefile, handle)
handle.close()
else:
list_get(service_name, remotefile, localfile)
except Exception:
print('download file failure! ')
finally:
samba.close()
def list_content(self, service_name, path):
samba = SMBConnection(self.username, self.password, '', '')
samba.connect(self.ip, self.port)
try:
content = 'type\tname\n----\t----\n'
for files in samba.listPath(service_name, path)[2:]:
if samba.getAttributes(service_name, path.rstrip('/') + '/' + files.filename).isDirectory:
content += ('d\t%s\n' % files.filename)
else:
content += ('f\t%s\n' % files.filename)
else:
return content[:-1]
except Exception:
print('retrieve content failure! ')
finally:
samba.close()
def sort_files_from_server(service_name, csv_name):
print("Files sorting process was initiated")
user_name = SMB_USER
password = SMB_PASSWORD
local_machine_name = "laptop"
server_machine_name = SMB_SERVER_NAME
server_ip = SMB_SERVER_IP
paths_df = pd.read_csv(csv_name)
paths_df = paths_df['Full Object Name'].map(lambda x: x.lstrip(service_name))
files = []
sizes = []
for path in paths_df:
print(".")
path = path.replace('\\', '/')
files.append(path)
try:
# print("\nFile information retrieval process was requested for file -> " + path)
conn = SMBConnection(user_name, password, local_machine_name, server_machine_name, use_ntlm_v2=True, is_direct_tcp=True)
assert conn.connect(server_ip, 445)
attributes = conn.getAttributes(service_name, path)
file_size = attributes.file_size
sizes.append(file_size)
except:
print("Information retrieval process failed for file " + path)
sizes.append(0)
continue
size_df = pd.DataFrame()
size_df['file_paths'] = pd.Series(files)
size_df['file_size'] = pd.Series(sizes)
# SORTING AND CONVERTING DATA
sorted_df = size_df.sort_values(by='file_size', ascending=True).reset_index(drop=True)
sorted_df['file_size'] = sorted_df['file_size'].map(lambda val: convert_size(val))
# SAVE DATAFRAME TO CSV
sorted_df.to_csv('Objects_sorted.csv')
print(sorted_df)
class SMB_client():
def __init__(self,username=None,password=None,smb_name=None,print_errors=True):
self.username = username
self.password = password
self.smb_name = smb_name
self.print_errors = print_errors
self.smb_ip = None
self.conn = None
self.service_name = None
self.my_name = None
self.error = None
self.tree = []
def getBIOSName(self, remote_smb_ip, timeout=5): # unused if dynamic IP
# ip -> smb name
try:
self.error = None
bios = NetBIOS()
srv_name = bios.queryIPForName(remote_smb_ip, timeout=timeout)
return srv_name[0]
except Exception as e:
if self.print_errors:
print('Error on line {}'.format(sys.exc_info()[-1].tb_lineno), type(e).__name__, e)
else:
self.error = 'Error on line {}'.format(sys.exc_info()[-1].tb_lineno) + str(type(e).__name__) + str(e)
return None
def getIP(self):
# smb name -> ip
try:
self.error = None
bios = NetBIOS()
ip = bios.queryName(self.smb_name)
return ip[0]
except Exception as e:
if self.print_errors:
print('Error on line {}'.format(sys.exc_info()[-1].tb_lineno), type(e).__name__, e)
else:
self.error = 'Error on line {}'.format(sys.exc_info()[-1].tb_lineno) + str(type(e).__name__) + str(e)
return None
def connect(self):
try:
self.error = None
self.my_name = gethostname() # iDevice name
self.smb_ip = self.getIP()
smb_structs.SUPPORT_SMB2 = True
self.conn = SMBConnection(self.username, self.password, self.my_name, self.smb_name, use_ntlm_v2 = True)
self.conn.connect(self.smb_ip, 139) #139=NetBIOS / 445=TCP
if self.conn:
shares = self.conn.listShares()
for share in shares:
if share.type == 0: # 0 = DISK_TREE
self.service_name = share.name
except Exception as e:
if self.print_errors:
print('Error on line {}'.format(sys.exc_info()[-1].tb_lineno), type(e).__name__, e)
else:
self.error = 'Error on line {}'.format(sys.exc_info()[-1].tb_lineno) + str(type(e).__name__) + str(e)
def close(self):
try:
self.error = None
self.conn.close()
except Exception as e:
if self.print_errors:
print('Error on line {}'.format(sys.exc_info()[-1].tb_lineno), type(e).__name__, e)
else:
self.error = 'Error on line {}'.format(sys.exc_info()[-1].tb_lineno) + str(type(e).__name__) + str(e)
def getRemoteDir(self, path, pattern):
try:
self.error = None
files = self.conn.listPath(self.service_name, path, pattern=pattern)
return files
except Exception as e:
if self.print_errors:
print('Error on line {}'.format(sys.exc_info()[-1].tb_lineno), type(e).__name__, e)
else:
self.error = 'Error on line {}'.format(sys.exc_info()[-1].tb_lineno) + str(type(e).__name__) + str(e)
return None
def getRemoteTree(self,path=''):
try:
self.error = None
if path == '':
w = ''
else:
w = path+'/'
files = self.getRemoteDir(path, '*')
if files:
for file in files:
if file.filename[0] == '.':
continue
self.tree.append({'name':w+file.filename, 'isdir':file.isDirectory, 'size':file.file_size})
if file.isDirectory:
self.getRemoteTree(path=w+file.filename)
return self.tree
except Exception as e:
if self.print_errors:
print('Error on line {}'.format(sys.exc_info()[-1].tb_lineno), type(e).__name__, e)
else:
self.error = 'Error on line {}'.format(sys.exc_info()[-1].tb_lineno) + str(type(e).__name__) + str(e)
return None
def download(self, path, filename,buffersize=None,callback=None, local_path=None):
try:
self.error = None
#print('Download = ' + path + filename)
attr = self.conn.getAttributes(self.service_name, path+filename)
#print('Size = %.1f kB' % (attr.file_size / 1024.0))
#print('start download')
file_obj = BytesIO()
if local_path:
fw = open(local_path+filename, 'wb')
else:
fw = open(filename, 'wb')
offset = 0
transmit =0
while True:
if not buffersize:
file_attributes, filesize = self.conn.retrieveFile(self.service_name, path+filename, file_obj)
else:
file_attributes, filesize = self.conn.retrieveFileFromOffset(self.service_name, path+filename, file_obj,offset=offset,max_length=buffersize)
if callback:
transmit = transmit + filesize
callback(transmit)
file_obj.seek(offset)
for line in file_obj:
fw.write(line)
offset = offset + filesize
if (not buffersize) or (filesize == 0):
break
fw.close()
#print('download finished')
except Exception as e:
if self.print_errors:
print('Error on line {}'.format(sys.exc_info()[-1].tb_lineno), type(e).__name__, e)
else:
self.error = 'Error on line {}'.format(sys.exc_info()[-1].tb_lineno) + str(type(e).__name__) + str(e)
def upload(self, path, filename,buffersize=None,callback=None, local_path=None):
try:
self.error = None
#print('Upload = ' + path + filename)
#print('Size = %.1f kB' % (os.path.getsize(filename) / 1024.0))
#print('start upload')
if local_path:
file_obj = open(local_path+filename, 'rb')
else:
file_obj = open(filename, 'rb')
offset = 0
while True:
if not buffersize:
filesize = self.conn.storeFile(self.service_name, path+filename, file_obj)
break
else:
buffer_obj = file_obj.read(buffersize)
if buffer_obj:
buffer_fileobj = BytesIO()
buffer_fileobj.write(buffer_obj)
buffer_fileobj.seek(0)
offset_new = self.conn.storeFileFromOffset(self.service_name, path+filename, buffer_fileobj, offset=offset, truncate=False)
#return the file position where the next byte will be written.
offset = offset_new
if callback:
callback(offset)
else:
break
file_obj.close()
#print('upload finished')
except Exception as e:
if self.print_errors:
print('Error on line {}'.format(sys.exc_info()[-1].tb_lineno), type(e).__name__, e)
else:
self.error = 'Error on line {}'.format(sys.exc_info()[-1].tb_lineno) + str(type(e).__name__) + str(e)
def delete_remote_file(self,path, filename):
try:
self.error = None
self.conn.deleteFiles(self.service_name, path+filename)
#print('Remotefile ' + path + filename + ' deleted')
except Exception as e:
if self.print_errors:
print('Error on line {}'.format(sys.exc_info()[-1].tb_lineno), type(e).__name__, e)
else:
self.error = 'Error on line {}'.format(sys.exc_info()[-1].tb_lineno) + str(type(e).__name__) + str(e)
def createRemoteDir(self, path):
try:
self.error = None
self.conn.createDirectory(self.service_name, path)
except Exception as e:
if self.print_errors:
print('Error on line {}'.format(sys.exc_info()[-1].tb_lineno), type(e).__name__, e)
else:
self.error = 'Error on line {}'.format(sys.exc_info()[-1].tb_lineno) + str(type(e).__name__) + str(e)
def removeRemoteDir(self,path):
try:
self.error = None
self.conn.deleteDirectory(self.service_name, path)
except Exception as e:
if self.print_errors:
print('Error on line {}'.format(sys.exc_info()[-1].tb_lineno), type(e).__name__, e)
else:
self.error = 'Error on line {}'.format(sys.exc_info()[-1].tb_lineno) + str(type(e).__name__) + str(e)
def renameRemoteFileOrDir(self,old_path, new_path):
try:
self.error = None
self.conn.rename(self.service_name, old_path, new_path)
except Exception as e:
if self.print_errors:
print('Error on line {}'.format(sys.exc_info()[-1].tb_lineno), type(e).__name__, e)
else:
self.error = 'Error on line {}'.format(sys.exc_info()[-1].tb_lineno) + str(type(e).__name__) + str(e)
def do_out(self, source, params, folder):
host, port, username, password, encrypt = parse_source(source)
file = params.get('file')
log.debug('file: "%s"', file)
if file is None:
raise ValueError("File name in params is mandatory!")
filename = os.path.basename(file)
filepath = os.path.join(folder, file)
if not os.path.exists(filepath):
raise Exception("File '%s' not found!" % filepath)
log.info('lookup "%s" ...', host)
ip = socket.gethostbyname(host)
localhost = socket.gethostname()
# connect smb share
log.info('connect "%s" ...', ip)
conn = SMBConnection(username,
password,
localhost,
host,
use_ntlm_v2=True,
is_direct_tcp=True)
ready = conn.connect(ip, 445)
if not ready:
raise Exception("Connect failed, host, user or pass is not valid!")
# create temp folder and move package to
log.info('prepare ...')
remote_dir = "temp\\%s" % temp_name()
conn.createDirectory(ADMIN_SERVICE, remote_dir)
# copy package remote
file_obj = open(filepath, 'r')
remote_filepath = "%s\\%s" % (remote_dir, filename)
conn.storeFile(ADMIN_SERVICE, remote_filepath, file_obj)
file_obj.close()
# install package remotely
log.info('install "%s" ...', filename)
psexec = Client(ip,
username=username,
password=password,
encrypt=encrypt)
psexec.connect()
remote_logpath = remote_dir + "\\msiexec.log"
try:
psexec.create_service()
msi_path = "%systemroot%\\" + remote_filepath
log_path = "%systemroot%\\" + remote_logpath
cmd = "msiexec /i %s /qn /norestart /L*v %s" % (msi_path, log_path)
log.debug(cmd)
stdout, stderr, rc = psexec.run_executable("cmd.exe",
arguments="/c " + cmd)
log.debug("exit code: %s", rc)
finally:
psexec.remove_service()
psexec.disconnect()
# dump msi log content
log.debug('read remote log "%s" content', remote_logpath)
try:
with tempfile.NamedTemporaryFile() as tmp_file:
conn.retrieveFile(ADMIN_SERVICE, remote_logpath, tmp_file)
tmp_file.seek(0)
log.debug(codecs.decode(tmp_file.read(), 'utf-16'))
except:
log.error(str(e)) # non fatal
if rc != 0: # sorry, fatal
raise Exception(stdout.decode('utf-16'))
# version magick
log.info('set version ...')
with tempfile.NamedTemporaryFile() as ver_obj:
conn.storeFile(ADMIN_SERVICE, MAGICK_FILENAME, ver_obj)
attr = conn.getAttributes(ADMIN_SERVICE, MAGICK_FILENAME)
version = attr.last_write_time
# clean and diconnect
conn.deleteFiles(ADMIN_SERVICE, remote_logpath)
conn.deleteFiles(ADMIN_SERVICE, remote_filepath)
conn.deleteDirectory(ADMIN_SERVICE, remote_dir)
conn.close()
return {"version": {"ref": format_version(version)}, "metadata": []}
'OXYA-JAD',
domain='JAD.com')
conn_status = conn.connect('172.16.128.150')
print(conn_status)
# list shares
share = conn.listShares()
share_list = []
for i in share:
share_list.append(i.name)
if share_service_name in share_list:
print('Share Available')
# store files
with open('c:/test/test.txt', 'rb') as fd:
file_size = conn.storeFile(share_service_name, share_path, fd)
# get file Attributes online
file_attribute_online = conn.getAttributes(share_service_name,
share_path,
timeout=30)
file_size_online = file_attribute_online.file_size
print('filesize on smb is {}'.format(file_size_online))
# retrieve files
#with open('c:/test/test.txt', 'wb') as fd:
# file_attributes, file_size = conn.retrieveFile(share_service_name, share_path, fd)
# verify
if file_size == file_size_online:
print('transfer ok')
class CommonCIFSShare(object):
"""
Handle CIFS shares
"""
def __init__(self):
self.smb_conn = None
def com_cifs_connect(self, ip_addr, user_name='guest', user_password=''):
"""
Connect to share
"""
server_name = 'Server'
client_name = 'My Computer'
self.smb_conn = SMBConnection(user_name, user_password, client_name, server_name,
use_ntlm_v2=True)
self.smb_conn.connect(ip_addr, 139)
def com_cifs_share_list_by_connection(self):
"""
List shares
"""
share_names = []
for row_data in self.smb_conn.listShares():
share_names.append(row_data.name)
return share_names
def com_cifs_share_file_list_by_share(self, share_name, path_text='/'):
"""
List files in share
"""
file_names = []
for row_data in self.smb_conn.listPath(share_name, path_text):
common_global.es_inst.com_elastic_index('info', {'stuff': row_data.filename})
file_names.append(row_data.filename)
return file_names
def com_cifs_share_directory_check(self, share_name, dir_path):
"""
Verify smb directory
"""
# try due to fact invalid file/path freaks out the connection
try:
return self.smb_conn.getAttributes(share_name, dir_path).isDirectory
except:
pass
return False
def com_cifs_share_file_dir_info(self, share_name, file_path):
"""
Get specific path/file info
"""
return self.smb_conn.getAttributes(share_name, file_path)
def com_cifs_share_file_upload(self, file_path):
"""
Upload file to smb
"""
self.smb_conn.storeFile(os.path.join(
self.sharename, file_path), open(file_path, 'rb'))
def com_cifs_share_file_download(self, file_path):
"""
Download from smb
"""
self.smb_conn.retrieveFile(self.sharename, open(file_path, 'wb'))
def com_cifs_share_file_delete(self, share_name, file_path):
"""
Delete from smb
"""
self.smb_conn.deleteFiles(os.path.join(share_name, file_path))
def com_cifs_close(self):
"""
Close connection
"""
self.smb_conn.close()
def com_cifs_walk(self, share_name, file_path='/'):
"""
cifs directory walk
"""
dirs, nondirs = [], []
for name in self.smb_conn.listPath(share_name, file_path):
if name.isDirectory:
if name.filename not in ['.', '..']:
dirs.append(name.filename)
else:
nondirs.append(name.filename)
yield file_path, dirs, nondirs
for name in dirs:
# new_path = file_path + '\\' + name
# for ndx in self.com_cifs_walk(share_name, new_path):
for ndx in self.com_cifs_walk(share_name, os.path.join(file_path, name)):
yield ndx
class SMBClient:
"""
SMB连接客户端
"""
status = False
samba = None
def __init__(self, username: str, password: str, ip: str, port: int = 139):
self.username = username
self.password = password
self.ip = ip
self.port = port
self.samba = SMBConnection(self.username, self.password, '', '', use_ntlm_v2=True)
self.samba.connect(self.ip, self.port)
self.status = self.samba.auth_result
def close(self):
if self.status:
self.samba.close()
def list_smb_dir(self, share_name, sub_dir=""):
"""列出文件夹内所有文件名
:param share_name: 共享文件夹名称
:param sub_dir: 相对共享文件夹的子目录
"""
file_names = list()
for e in self.samba.listPath(share_name, sub_dir):
if e.filename[0] != '.': # 过滤上级文件夹及影藏文件
file_names.append(e.filename)
return file_names
def download(self, filename, local_dir, share_name, sub_dir=""):
"""下载文件
:param filename: 文件名
:param share_name: 共享文件夹名称
:param sub_dir: 相对共享文件夹的子目录
:param local_dir: 本地保存文件夹路径
"""
assert isinstance(filename, str)
with open(os.path.join(local_dir, filename), 'wb') as fp:
self.samba.retrieveFile(share_name, os.path.join(sub_dir, filename), fp)
def download_bytes(self, share_name, sub_file_path):
"""直接下载共享文件的Bytes数据
:param share_name: 共享文件夹名称
:param sub_file_path: 共享文件路径
"""
fp = io.BytesIO()
self.samba.retrieveFile(share_name, sub_file_path, fp)
fp.seek(0)
data = fp.read()
fp.close()
return data
def upload(self, local_file_path, share_name, smb_save_path):
"""上传文件
:param local_file_path: 本地文件路径
:param share_name: 共享文件夹名称
:param smb_save_path: 在共享文件夹的存放路径
"""
with open(local_file_path, "rb") as fp:
self.samba.storeFile(share_name, smb_save_path, fp)
def upload_bytes(self, data, share_name, smb_save_path):
"""直接将Bytes类型的数据保存到共享文件
:param data: Bytes类型数据
:param share_name: 共享文件夹名称
:param smb_save_path: 在共享文件夹的存放路径
"""
fp = io.BytesIO(data)
self.samba.storeFile(share_name, smb_save_path, fp)
fp.close()
def create_dir(self, share_name, sub_dir):
"""创建文件夹
:param share_name: 共享文件夹名称
:param sub_dir: 相对共享文件夹的子目录
"""
self.samba.createDirectory(share_name, sub_dir)
def file_attrs(self, share_name, smb_file_path):
attrs = {}
try:
file = self.samba.getAttributes(share_name, smb_file_path)
for attr in ['alloc_size', 'create_time', 'file_attributes', 'file_id', 'file_size', 'filename',
'isDirectory', 'isNormal', 'isReadOnly', 'last_access_time', 'last_attr_change_time',
'last_write_time', 'short_name']:
attrs[attr] = getattr(file, attr)
except OperationFailure:
pass
return attrs
def delete_file(self, share_name, smb_file_pattern):
"""删除文件, 默认都把传入的smb_file_pattern转换为Unicode字符串
:param share_name: 共享文件夹名称
:param smb_file_pattern: 文件的相对路径, 可以使用通配符
"""
self.samba.deleteFiles(share_name, smb_file_pattern)
class SmbPlugin(syncplugin.AbstractSyncPlugin):
NAME: str = "smb"
def __init__(self) -> None:
self._connection: SMBConnection = None
self._info = _ConnectionInfo()
self._attributes: typing.Dict[pathlib.PurePath, SharedFile] = {}
def _password_identifier(self) -> str:
"""Get an unique identifier for the connection in self._info.
The identifier is used to request/store passwords. Thus, it contains all
information associated with a connection and a user account.
A newline is used as a separator because it shouldn't cause any trouble
in the password backend, but still should never be part of a
username/domain/hostname.
It does *not* contain the port, as it could be possible to access the
same SMB server via different protocol versions that way, and that's
probably more likely than having different SMB servers running on the
same host.
"""
domain: str = self._info.domain if self._info.domain else ''
return '\n'.join([self._info.username, domain, self._info.hostname])
def configure(self, info: typing.Dict[str, typing.Any]) -> None:
self._info.use_ntlm_v2 = info.get('use_ntlm_v2', False)
sign_options = info.get('sign_options', 'when_required')
self._info.sign_options = _SignOptions[sign_options]
self._info.is_direct_tcp = info.get('is_direct_tcp', True)
self._info.username = info['username']
self._info.domain = info.get('domain', 'HSR')
self._info.share = info.get('share', 'skripte')
self._info.hostname = info.get('hostname', 'svm-c213.hsr.ch')
prompt = f'{self._info.username}@{self._info.hostname}'
self._info.password = utils.get_password('smb',
self._password_identifier(),
prompt)
default_port = 445 if self._info.is_direct_tcp else 139
self._info.port = info.get('port', default_port)
if not info.get('debug', False):
# PySMB has too verbose logging, we don't want to see that.
logging.getLogger('SMB.SMBConnection').propagate = False
logger.debug(f'Configured: {self._info}')
def connect(self) -> None:
self._connection = SMBConnection(
username=self._info.username,
password=self._info.password,
domain=self._info.domain,
my_name=socket.gethostname(),
remote_name=self._info.hostname,
use_ntlm_v2=self._info.use_ntlm_v2,
sign_options=self._info.sign_options,
is_direct_tcp=self._info.is_direct_tcp)
try:
server_ip: str = socket.gethostbyname(self._info.hostname)
except socket.gaierror:
raise utils.PluginOperationError(
f'Could not find server {self._info.hostname}. '
'Maybe you need to open a VPN connection or the server is not available.'
)
logger.debug(
f'Connecting to {server_ip} ({self._info.hostname}) port {self._info.port}'
)
try:
success = self._connection.connect(server_ip, self._info.port)
except (ConnectionRefusedError, socket.timeout):
raise utils.PluginOperationError(
f'Could not connect to {server_ip}:{self._info.port}')
# FIXME Can be removed once https://github.com/miketeo/pysmb/issues/108 is fixed
except ProtocolError:
success = False
if not success:
raise utils.AuthenticationError(
f'Authentication failed for {server_ip}:{self._info.port}')
def disconnect(self) -> None:
self._connection.close()
def _create_digest(self, size: int, mtime: float) -> str:
"""Create a digest from a size and mtime.
We can't get a hash from the server, so we use the last modified time
(mtime) and file size as a substitute. Also see:
https://en.wikipedia.org/wiki/St_mtime
The mtime resolution can differ between client/server or on different
file systems, so we truncate it to full seconds.
"""
mtime = int(mtime)
return f'{size}-{mtime}'
def create_local_digest(self, path: pathlib.Path) -> str:
info = path.lstat()
return self._create_digest(size=info.st_size, mtime=info.st_mtime)
def create_remote_digest(self, path: pathlib.PurePath) -> str:
try:
attributes = self._connection.getAttributes(
self._info.share, str(path))
except OperationFailure:
raise utils.PluginOperationError(
f'Could not find remote file {path} in share "{self._info.share}"'
)
self._attributes[path] = attributes
return self._create_digest(size=attributes.file_size,
mtime=attributes.last_write_time)
def list_path(self,
path: pathlib.PurePath) -> typing.Iterable[pathlib.PurePath]:
try:
entries = self._connection.listPath(self._info.share, str(path))
except OperationFailure:
raise utils.PluginOperationError(f'Folder "{path}" not found')
for entry in entries:
if entry.isDirectory:
if entry.filename not in [".", ".."]:
yield from self.list_path(
pathlib.PurePath(path / entry.filename))
else:
yield pathlib.PurePath(path / entry.filename)
def retrieve_file(self, path: pathlib.PurePath,
fileobj: typing.IO[bytes]) -> typing.Optional[int]:
logger.debug(f'Retrieving file {path}')
try:
self._connection.retrieveFile(self._info.share, str(path), fileobj)
except OperationFailure:
raise utils.PluginOperationError(
f'Could not download {path} from share "{self._info.share}"')
mtime: int = self._attributes[path].last_write_time
return mtime
def connection_schema(self) -> utils.JsonType:
return {
'type': 'object',
'properties': {
'hostname': {
'type': 'string'
},
'port': {
'type': 'number'
},
'share': {
'type': 'string'
},
'domain': {
'type': 'string'
},
'username': {
'type': 'string'
},
'sign_options': {
'type': 'string',
'enum': ['never', 'when_supported', 'when_required'],
},
'use_ntlm_v2': {
'type': 'boolean'
},
'is_direct_tcp': {
'type': 'boolean'
},
'debug': {
'type': 'boolean'
},
},
'required': [
'username',
],
'additionalProperties': False,
}
class SmbStorage(Storage):
def __init__(self,
name,
host,
share,
path,
user='',
password='',
port=139):
super().__init__(name, path)
self.share = share
self.user = user
self.password = password
self.host = host
self.port = port
self.name_port = 137
self.my_hostname = socket.gethostname()
self.smbcon = SMBConnection(self.user, self.password, self.my_hostname,
self.host)
self.smbcon.connect(self.host)
def connect(self):
self.smbcon = SMBConnection(self.user, self.password, self.my_hostname,
self.host)
self.smbcon.connect(self.host)
def close(self):
self.smbcon.close()
def create_dir(self, dir_name):
smb_dir = self.path + '/' + dir_name
self.smbcon.createDirectory(self.share, smb_dir)
def put(self, dir_name, file):
print('File: ', file)
#self.shares = self.smbcon.listShares()
#for s in self.shares:
# print(s.name)
date_str = datetime.datetime.now().strftime("%d-%m-%Y")
try:
smb_dir = self.smbcon.getAttributes(self.share,
self.path + '/' + dir_name)
if smb_dir.isDirectory == False:
self.create_dir(dir_name)
except OperationFailure:
self.create_dir(dir_name)
try:
smb_dir = self.smbcon.getAttributes(
self.share, self.path + '/' + dir_name + '/' + date_str)
if smb_dir.isDirectory == False:
self.create_dir(dir_name + '/' + date_str)
except OperationFailure:
self.create_dir(dir_name + '/' + date_str)
data = open(file, 'rb')
file_name = file.split('/')
file_name = file_name[-1]
self.smbcon.storeFile(
self.share,
self.path + '/' + dir_name + '/' + date_str + '/' + file_name,
data)
data.close()
class SMBNetDriver(StorageDriver):
"Implement a SMB driver "
scheme = "smb"
#https://pythonhosted.org/pysmb/api/smb_SMBConnection.html
def __init__(self, mount_url=None, credentials = None, readonly = False, **kw):
""" initializae a storage driver
@param mount_url: optional full storeurl to mount
"""
self.conn = None
self.mount_url = posixpath.join(mount_url, '')
self.readonly = readonly
if credentials is None:
log.warn ("SMBMount Cannot proceed without credentials")
return
self.user, self.password = credentials.split (':')
self.localhost = socket.gethostname()
urlcomp = urlparse.urlparse (self.mount_url)
self.serverhost = urlcomp.netloc
self.server_ip = socket.gethostbyname(self.serverhost)
# New interface
def mount(self):
"""Mount the driver
@param mount_url: an smb prefix to be used to mount smb://smbhostserver/sharename/d1/d2/"
@param credentials: a string containing user:password for smb connection
"""
# I don't this this is the SMB hostname but am not sure
if self.conn is not None:
return
self.conn = SMBConnection (self.user, self.password, self.localhost, self.serverhost)
if not self.conn.connect(self.server_ip, 139):
self.conn = None
#except smb.base.NotReadyError:
# log.warn("NotReady")
#except smb.base.NotConnectedError:
# log.warn("NotReady")
#except smb.base.SMBTimeout:
# log.warn("SMBTimeout")
def unmount (self):
"""Unmount the driver """
if self.conn:
self.conn.close()
self.conn=None
def mount_status(self):
"""return the status of the mount: mounted, error, unmounted
"""
@classmethod
def split_smb(cls,storeurl):
"return a pair sharename, path suitable for operations"
smbcomp = urlparse.urlparse (storeurl)
# smb://host smbcomp.path = /sharenmae/path
_, sharename, path = smbcomp.path.split ('/', 2)
return sharename, '/' + path
# File interface
def valid (self, storeurl):
"Return validity of storeurl"
return storeurl.startswith (self.mount_url) and storeurl
def push(self, fp, storeurl, uniq=None):
"Push a local file (file pointer) to the store"
sharename, path = self.split_smb(storeurl)
uniq = uniq or make_uniq_code()
base,ext = os.path.splitext(path)
for x in xrange(len(uniq)-7):
try:
if not self.conn.getAttributes (sharename, path):
break
except smb.OperationError:
path = "%s-%s%s" % (base , uniq[3:7+x] , ext)
written = self.conn.storeFile (sharename, path, fp)
log.debug ("smb wrote %s bytes", written)
return "smb://%s/%s" % (sharename, path), None
def pull (self, storeurl, localpath=None, blocking=True):
"Pull a store file to a local location"
sharename, path = self.split_smb(storeurl)
if self.conn:
if localpath:
file_obj = open (localpath, 'wb')
else:
file_obj = tempfile.NamedTemporaryFile()
file_attributes, filesize = self.conn.retrieveFile(sharename, path, file_obj)
log.debug ("smb fetch of %s got %s bytes", storeurl, filesize)
def chmod(self, storeurl, permission):
"""Change permission of """
def delete(self, storeurl):
'delete an entry on the store'
sharename, path = self.split_smb(storeurl)
if self.conn:
self.conn.deleteFiles(sharename, path)
def isdir (self, storeurl):
"Check if a url is a container/directory"
def status(self, storeurl):
"return status of url: dir/file, readable, etc"
def list(self, storeurl):
"list contents of store url"
def reduce(self, events):
try:
app_config = cli.getConfStanza('ep_general', 'settings')
cmd_config = cli.getConfStanzas('ep_smb')
except BaseException as e:
raise Exception("Could not read configuration: " + repr(e))
# Facility info - prepended to log lines
facility = os.path.basename(__file__)
facility = os.path.splitext(facility)[0]
try:
logger = setup_logger(app_config["log_level"],
'export_everything.log', facility)
except BaseException as e:
raise Exception("Could not create logger: " + repr(e))
logger.info('SMB Export search command initiated')
logger.debug('search_ep_smb command: %s', self) # logs command line
# Enumerate proxy settings
http_proxy = os.environ.get('HTTP_PROXY')
https_proxy = os.environ.get('HTTPS_PROXY')
proxy_exceptions = os.environ.get('NO_PROXY')
if http_proxy is not None:
logger.debug("HTTP proxy: %s" % http_proxy)
if https_proxy is not None:
logger.debug("HTTPS proxy: %s" % https_proxy)
if proxy_exceptions is not None:
logger.debug("Proxy Exceptions: %s" % proxy_exceptions)
# Enumerate settings
app = self._metadata.searchinfo.app
user = self._metadata.searchinfo.username
dispatch = self._metadata.searchinfo.dispatch_dir
session_key = self._metadata.searchinfo.session_key
if self.target is None and 'target=' in str(self):
recover_parameters(self)
# Replace all tokenized parameter strings
replace_object_tokens(self)
# Use the random number to support running multiple outputs in a single search
random_number = str(random.randint(10000, 100000))
try:
target_config = get_config_from_alias(session_key, cmd_config,
self.target)
if target_config is None:
exit_error(
logger,
"Unable to find target configuration (%s)." % self.target,
100937)
except BaseException as e:
exit_error(logger,
"Error reading target server configuration: " + repr(e),
124812)
# Get the local client hostname
client_name = socket.gethostname()
# Delete any domain from the client hostname string
if '.' in client_name:
client_name = client_name[0:client_name.index('.')]
# Check to see if we have credentials
valid_settings = []
for l in list(target_config.keys()):
if len(target_config[l]) > 0:
valid_settings.append(l)
if 'host' in valid_settings:
# A target has been configured. Check for credentials.
try:
if 'credential_username' in valid_settings and 'credential_password' in valid_settings and 'share_name' in valid_settings:
domain = target_config[
'credential_realm'] if 'credential_realm' in list(
target_config.keys()) else target_config['host']
try:
# Try port 445 first
conn = SMBConnection(
target_config['credential_username'],
target_config['credential_password'],
client_name,
target_config['host'],
domain=domain,
use_ntlm_v2=True,
sign_options=SMBConnection.SIGN_WHEN_SUPPORTED,
is_direct_tcp=True)
connected = conn.connect(target_config['host'],
445,
timeout=5)
if target_config['share_name'] not in (
s.name for s in conn.listShares(timeout=10)):
exit_error(
logger,
"Unable to find the specified share name on the server",
553952)
'''
p445_error = repr(e445)
try:
# Try port 139 if that didn't work
conn = SMBConnection(target_config['credential_username'], target_config['credential_password'], client_name,
target_config['host'], domain=domain, use_ntlm_v2=True,
sign_options = SMBConnection.SIGN_WHEN_SUPPORTED)
connected = conn.connect(target_config['host'], 139, timeout=5)
except BaseException as e139:
p139_error = repr(e139)
raise Exception("Errors connecting to host: \\nPort 139: %s\\nPort 445: %s" % (p139_error, p445_error))
conn = SMBConnection(target_config['credential_username'], target_config['credential_password'], client_name,
target_config['host'], domain=domain, use_ntlm_v2=True,
sign_options = SMBConnection.SIGN_WHEN_SUPPORTED)
connected = conn.connect(target_config['host'], 139)
shares =
share_exists = False
for i in range(len(shares)):
if shares[i].name == target_config['share_name']:
share_exists = True
break
'''
except BaseException as e:
exit_error(
logger,
"Unable to setup SMB connection: " + repr(e),
921982)
else:
exit_error(logger, "Required settings not found", 101926)
except BaseException as e:
exit_error(logger,
"Error reading the configuration: " + repr(e),
230494)
else:
exit_error(logger,
"Could not find required configuration settings",
2823874)
file_extensions = {
'raw': '.log',
'kv': '.log',
'pipe': '.log',
'csv': '.csv',
'tsv': '.tsv',
'json': '.json'
}
if self.outputformat is None:
self.outputformat = 'csv'
# Create the default filename
default_filename = ('export_' + user + '___now__' +
file_extensions[self.outputformat]).strip("'")
folder, filename = event_file.parse_outputfile(self.outputfile,
default_filename,
target_config)
if self.compress is not None:
logger.debug('Compression: %s', self.compress)
else:
try:
self.compress = target_config.get('compress')
except:
self.compress = False
staging_filename = 'export_everything_staging_' + random_number + '.txt'
local_output_file = os.path.join(dispatch, staging_filename)
if self.compress:
local_output_file = local_output_file + '.gz'
# Append .gz to the output file if compress=true
if not self.compress and len(filename) > 3:
if filename[-3:] == '.gz':
# We have a .gz extension when compression was not specified. Enable compression.
self.compress = True
elif self.compress and len(filename) > 3:
if filename[-3:] != '.gz':
filename = filename + '.gz'
if conn is not None:
# Use the credential to connect to the SFTP server
try:
# Check to see if the folder exists
folder_attrs = conn.getAttributes(target_config['share_name'],
folder,
timeout=10)
except BaseException:
# Remote directory could not be loaded. It must not exist. Create it.
# Create the folders required to store the file
subfolders = ['/'] + folder.strip('/').split('/')
if '' in subfolders:
subfolders.remove('')
logger.debug("Folders list for dir creation: %s" %
str(subfolders))
current_folder = ''
folder_depth = len(subfolders) - 1
for i, subfolder_name in enumerate(subfolders):
current_folder = (current_folder + '/' +
subfolder_name).replace('//', '/')
logger.debug("Current folder = " + current_folder)
try:
conn.getAttributes(target_config['share_name'],
current_folder,
timeout=10)
except:
conn.createDirectory(target_config['share_name'],
current_folder,
timeout=10)
try:
folder_attrs = conn.getAttributes(
target_config['share_name'], folder, timeout=10)
except BaseException as e:
exit_error(
logger, "Could not load or create remote directory: " +
repr(e), 377890)
# This should always be true
if folder_attrs is not None:
if folder_attrs.isReadOnly or not folder_attrs.isDirectory:
exit_error(
logger,
"Could not access the remote directory: " + repr(e),
184772)
else:
try:
event_counter = 0
# Write the output file to disk in the dispatch folder
logger.debug(
"Writing events to dispatch file. file=\"%s\" format=%s compress=%s fields=%s",
local_output_file, self.outputformat,
self.compress, self.fields)
for event in event_file.write_events_to_file(
events, self.fields, local_output_file,
self.outputformat, self.compress):
yield event
event_counter += 1
except BaseException as e:
exit_error(logger,
"Error writing file to upload: " + repr(e),
296733)
# Write the file to the remote location
try:
with open(local_output_file, 'rb',
buffering=0) as local_file:
bytes_uploaded = conn.storeFile(
target_config['share_name'],
folder + '/' + filename, local_file)
except BaseException as e:
exit_error(
logger,
"Error uploading file to SMB server: " + repr(e),
109693)
if bytes_uploaded > 0:
message = "SMB Export Status: Success. File name: %s" % (
folder + '/' + filename)
eprint(message)
logger.info(message)
else:
exit_error(logger, "Zero bytes uploaded", 771293)
else:
exit_error(logger, "Could not connect to server.", 159528)
class SMB_client():
def __init__(self,
username=None,
password=None,
smb_name=None,
print_errors=True):
self.username = username
self.password = password
self.smb_name = smb_name
self.print_errors = print_errors
self.smb_ip = None
self.conn = None
self.service_name = None
self.my_name = None
self.error = None
self.tree = []
def getBIOSName(self, remote_smb_ip, timeout=5): # unused if dynamic IP
# ip -> smb name
try:
self.error = None
bios = NetBIOS()
srv_name = bios.queryIPForName(remote_smb_ip, timeout=timeout)
return srv_name[0]
except Exception as e:
if self.print_errors:
print('Error on line {}'.format(sys.exc_info()[-1].tb_lineno),
type(e).__name__, e)
else:
self.error = 'Error on line {}'.format(
sys.exc_info()[-1].tb_lineno) + str(
type(e).__name__) + str(e)
return None
def getIP(self):
# smb name -> ip
try:
self.error = None
bios = NetBIOS()
ip = bios.queryName(self.smb_name)
return ip[0]
except Exception as e:
if self.print_errors:
print('Error on line {}'.format(sys.exc_info()[-1].tb_lineno),
type(e).__name__, e)
else:
self.error = 'Error on line {}'.format(
sys.exc_info()[-1].tb_lineno) + str(
type(e).__name__) + str(e)
return None
def connect(self):
try:
self.error = None
self.my_name = gethostname() # iDevice name
self.smb_ip = self.getIP()
smb_structs.SUPPORT_SMB2 = True
self.conn = SMBConnection(self.username,
self.password,
self.my_name,
self.smb_name,
use_ntlm_v2=True)
self.conn.connect(self.smb_ip, 139) #139=NetBIOS / 445=TCP
if self.conn:
shares = self.conn.listShares()
for share in shares:
if share.type == 0: # 0 = DISK_TREE
self.service_name = share.name
except Exception as e:
if self.print_errors:
print('Error on line {}'.format(sys.exc_info()[-1].tb_lineno),
type(e).__name__, e)
else:
self.error = 'Error on line {}'.format(
sys.exc_info()[-1].tb_lineno) + str(
type(e).__name__) + str(e)
def close(self):
try:
self.error = None
self.conn.close()
except Exception as e:
if self.print_errors:
print('Error on line {}'.format(sys.exc_info()[-1].tb_lineno),
type(e).__name__, e)
else:
self.error = 'Error on line {}'.format(
sys.exc_info()[-1].tb_lineno) + str(
type(e).__name__) + str(e)
def getRemoteDir(self, path, pattern):
try:
self.error = None
files = self.conn.listPath(self.service_name,
path,
pattern=pattern)
return files
except Exception as e:
if self.print_errors:
print('Error on line {}'.format(sys.exc_info()[-1].tb_lineno),
type(e).__name__, e)
else:
self.error = 'Error on line {}'.format(
sys.exc_info()[-1].tb_lineno) + str(
type(e).__name__) + str(e)
return None
def getRemoteTree(self, path=''):
try:
self.error = None
if path == '':
w = ''
else:
w = path + '/'
files = self.getRemoteDir(path, '*')
if files:
for file in files:
if file.filename[0] == '.':
continue
self.tree.append({
'name': w + file.filename,
'isdir': file.isDirectory,
'size': file.file_size
})
if file.isDirectory:
self.getRemoteTree(path=w + file.filename)
return self.tree
except Exception as e:
if self.print_errors:
print('Error on line {}'.format(sys.exc_info()[-1].tb_lineno),
type(e).__name__, e)
else:
self.error = 'Error on line {}'.format(
sys.exc_info()[-1].tb_lineno) + str(
type(e).__name__) + str(e)
return None
def download(self,
path,
filename,
buffersize=None,
callback=None,
local_path=None):
try:
self.error = None
#print('Download = ' + path + filename)
attr = self.conn.getAttributes(self.service_name, path + filename)
#print('Size = %.1f kB' % (attr.file_size / 1024.0))
#print('start download')
file_obj = BytesIO()
if local_path:
fw = open(local_path + filename, 'wb')
else:
fw = open(filename, 'wb')
offset = 0
transmit = 0
while True:
if not buffersize:
file_attributes, filesize = self.conn.retrieveFile(
self.service_name, path + filename, file_obj)
else:
file_attributes, filesize = self.conn.retrieveFileFromOffset(
self.service_name,
path + filename,
file_obj,
offset=offset,
max_length=buffersize)
if callback:
transmit = transmit + filesize
callback(transmit)
file_obj.seek(offset)
for line in file_obj:
fw.write(line)
offset = offset + filesize
if (not buffersize) or (filesize == 0):
break
fw.close()
#print('download finished')
except Exception as e:
if self.print_errors:
print('Error on line {}'.format(sys.exc_info()[-1].tb_lineno),
type(e).__name__, e)
else:
self.error = 'Error on line {}'.format(
sys.exc_info()[-1].tb_lineno) + str(
type(e).__name__) + str(e)
def upload(self,
path,
filename,
buffersize=None,
callback=None,
local_path=None):
try:
self.error = None
#print('Upload = ' + path + filename)
#print('Size = %.1f kB' % (os.path.getsize(filename) / 1024.0))
#print('start upload')
if local_path:
file_obj = open(local_path + filename, 'rb')
else:
file_obj = open(filename, 'rb')
offset = 0
while True:
if not buffersize:
filesize = self.conn.storeFile(self.service_name,
path + filename, file_obj)
break
else:
buffer_obj = file_obj.read(buffersize)
if buffer_obj:
buffer_fileobj = BytesIO()
buffer_fileobj.write(buffer_obj)
buffer_fileobj.seek(0)
offset_new = self.conn.storeFileFromOffset(
self.service_name,
path + filename,
buffer_fileobj,
offset=offset,
truncate=False)
#return the file position where the next byte will be written.
offset = offset_new
if callback:
callback(offset)
else:
break
file_obj.close()
#print('upload finished')
except Exception as e:
if self.print_errors:
print('Error on line {}'.format(sys.exc_info()[-1].tb_lineno),
type(e).__name__, e)
else:
self.error = 'Error on line {}'.format(
sys.exc_info()[-1].tb_lineno) + str(
type(e).__name__) + str(e)
def delete_remote_file(self, path, filename):
try:
self.error = None
self.conn.deleteFiles(self.service_name, path + filename)
#print('Remotefile ' + path + filename + ' deleted')
except Exception as e:
if self.print_errors:
print('Error on line {}'.format(sys.exc_info()[-1].tb_lineno),
type(e).__name__, e)
else:
self.error = 'Error on line {}'.format(
sys.exc_info()[-1].tb_lineno) + str(
type(e).__name__) + str(e)
def createRemoteDir(self, path):
try:
self.error = None
self.conn.createDirectory(self.service_name, path)
except Exception as e:
if self.print_errors:
print('Error on line {}'.format(sys.exc_info()[-1].tb_lineno),
type(e).__name__, e)
else:
self.error = 'Error on line {}'.format(
sys.exc_info()[-1].tb_lineno) + str(
type(e).__name__) + str(e)
def removeRemoteDir(self, path):
try:
self.error = None
self.conn.deleteDirectory(self.service_name, path)
except Exception as e:
if self.print_errors:
print('Error on line {}'.format(sys.exc_info()[-1].tb_lineno),
type(e).__name__, e)
else:
self.error = 'Error on line {}'.format(
sys.exc_info()[-1].tb_lineno) + str(
type(e).__name__) + str(e)
def renameRemoteFileOrDir(self, old_path, new_path):
try:
self.error = None
self.conn.rename(self.service_name, old_path, new_path)
except Exception as e:
if self.print_errors:
print('Error on line {}'.format(sys.exc_info()[-1].tb_lineno),
type(e).__name__, e)
else:
self.error = 'Error on line {}'.format(
sys.exc_info()[-1].tb_lineno) + str(
type(e).__name__) + str(e)
