class TestFile(unittest.TestCase):
def setUp(self):
self.file = File(api_url = config.API_URL, token = config.TOKEN)
self.token = config.TOKEN
@unittest.skip("skip")
def test_get_files(self):
offset = 0
resp, content = self.file.get_files(offset = offset)
self.assertEqual(int(resp['status']), 200)
@unittest.skip("skip")
def test_download_files(self):
filenames = "test.txt"
resp, content = self.file.download_file(filenames = filenames)
self.assertEqual(int(resp['status']), 200)
@unittest.skip("skip")
def test_upload_files(self):
register_openers()
datagen, headers = multipart_encode({"files": open("test.txt", "rb")})
headers["Authorization"] = "bearer:%s" % self.token
request = urllib2.Request("%s/member/files/upload" % config.API_URL, datagen, headers)
try:
response = urllib2.urlopen(request)
code = response.getcode()
content = response.read()
except urllib2.HTTPError, e:
code = e.code
content = e.msg
self.assertEqual(code, 200)
def test_find(self):
path = "/home/test/downloads"
size_filter = core.algorithm.SizeFilter()
char_filter = core.algorithm.CharacterFilter()
filters = [
size_filter,
char_filter
]
dup_finder = core.dup_finder.DupFinder([path], filters)
file_a = File("/home/test/downloads/a")
file_b = File("/home/test/downloads/b")
file_insts = [file_a, file_b]
when(core.dup_finder.Walker).walk([path]).thenReturn(file_insts)
when(size_filter).find().thenReturn(None)
size_filter.filtered_files = file_insts
when(size_filter).set_files().thenReturn(None)
when(char_filter).set_files().thenReturn(None)
when(char_filter).find().thenReturn(None)
char_filter.filtered_files = []
dup_finder.find()
verify(core.dup_finder.Walker).walk([path])
verify(size_filter).set_files(file_insts)
verify(size_filter).find()
verify(char_filter).set_files(file_insts)
verify(char_filter).find()
self.assertEqual([], dup_finder.dup_files)
def put(self, path):
try:
context = g.context
connection = context.get_connection()
result = []
for f in request.files.getlist('file'):
file = File()
file.update_file(context, f, path)
result.append({
"status": "OK",
"file_id": file.get_file_id(),
"remote_path": path
})
return make_response(dict({
"results": result,
"status": "ok"
}), 200)
except FileNotFoundInDatabase as err:
logger.exception(f"FileNotFoundInDatabase: {err}")
return make_response({"status": "Err", "error": str(err)}, 404)
except Exception as err:
logger.exception(f"Exception: {err}")
return make_response({"status": "Err", "error": str(err)}, 500)
def get(self, path):
try:
context = g.context
connection = context.get_connection()
file = File()
result = file.read_file(context, path)
if result == None:
return make_response(
{
"status": "Err",
"error": f"File not found: {path}"
}, 404)
response = make_response(result['file'])
response.headers.set('Content-Type', result['mime_type'])
#response.headers.set('Content-Disposition', 'attachment', filename='test.jpg')
response.headers.set('Content-Disposition',
'inline',
filename=result['name'])
return response
except Exception as err:
logger.exception(f"Exception: {err}")
return make_response({"status": "Err", "error": str(err)}, 500)
def __init__(self, opts):
""" constructor"""
self.opts = opts # all nullscan options
self.logger = Logger()
self.log = self.logger.log
self.file = File()
self.misc = Misc()
# tmp stupid fix to strip bad chars out of options
self.bad_chars = '<>()[]{}:;.,="\'*&^%$#@!+-_/?°²³´`'
return
def __init__(self):
""" constructor """
# logger
self.logger = Logger()
self.log = self.logger.log
# file i/o
self.file = File()
# original terminal state
self.term_fd = sys.stdin.fileno()
self.term_state = termios.tcgetattr(self.term_fd)
return
def __path_walk(self, path):
file_instances = list()
for (root, dirs, files) in os.walk(path):
# LOG.debug("{0} {1} {2}".format(root, dirs, files))
for _file in files:
filepath = os.path.join(root, _file)
if os.path.exists(filepath):
file_instance = File(filepath)
file_instances.append(file_instance)
self.progress = self.progress + 1
return file_instances
def __init__(self, date, opts, template_dir, report_dir, logs_dir):
""" constructor """
self.date = date
self.opts = opts
self.res = {} # holds the results out of log dirs
self.template_dir = template_dir
self.report_dir = report_dir
self.logs_dir = logs_dir
self.file = File()
self.file.copy_dirs(self.template_dir, self.report_dir)
self.index_html = f'{self.report_dir}/index.html'
self.res_html = f'{self.report_dir}/results.html'
self.index_html_data = self.file.read_file(self.index_html)
self.res_html_data = self.file.read_file(self.res_html)
return
def __init__(self):
""" constructor """
# options and modules
self.opt = None
self.mod = Module(MOD_PATH)
# logger
self.logger = Logger()
self.log = self.logger.log
# rest we need
self.file = File()
self.check = Check()
self.misc = Misc()
self.parser = None
# nullscan working dir
self.nullscan_dir = None
return
def __init__(self, mod_path):
""" constructor """
# logger
self.logger = Logger()
self.log = self.logger.log
# file i/o
self.file = File()
# get all modules
self.mod_path = mod_path
self.mods = []
self.get_modules()
# all loaded module
self.lmod = {}
# docstrings for all tools
self.docstrings = {}
return
def main():
path = sys.argv[1]
LOG.info("Start to find duplicated files on {0}".format(path))
if os.path.isfile(path):
start_time = time.time()
print (File(path).md5sum)
print (File(path).size)
end_time = time.time()
print (end_time - start_time)
else:
start_time = time.time()
filters = [
core.algorithm.SizeFilter(),
core.algorithm.CharacterFilter()
]
dup_finder = core.dup_finder.DupFinder([path], filters)
dup_finder.find()
end_time = time.time()
#dup_finder.dump2file("output.txt")
dup_finder.dump2csv("output.csv")
print (end_time - start_time)
print (utils.size_renderer(dup_finder.dup_size))
def __init__(self, target, opts):
""" init """
Helper.__init__(self, target, opts)
ToolsHelper.__init__(self)
Tools.__init__(self, target, opts)
Parser.__init__(self)
self.logger = Logger()
self.log = self.logger.log
self.file = File()
self.logfile = None # logfile for command
return
def getresults(self,
filepath,
frame,
resolution=None,
surface=False,
center=False):
"""
Get cached results for the given parameters.
**Parameters:**
`filepath` :
absolute path of the input file
`frame` :
the frame number
`resolution` :
resolution of the used discretization
`surface` :
query calculation results for surface-based cavities
`center` :
query calculation results for center-based cavities
**Returns:**
A :class:`core.data.Results` object if cached results exist,
else `None`
"""
inputfile = File.open(filepath)
# TODO: error handling
resultfile = None
results = None
if isinstance(inputfile, core.file.ResultFile):
resultfile = inputfile
elif filepath in self.cache:
resultfile = self.cache[filepath]
if resultfile is not None:
if resolution is None:
resolutions = sorted(resultfile.info.resolutions())[::-1]
resolution = 64
for res in resolutions:
if resultfile.info[res].domains[frame] is not None:
resolution = res
break
results = resultfile.getresults(frame, resolution)
if results is None:
if resolution is None:
resolution = 64
results = data.Results(filepath, frame, resolution,
inputfile.getatoms(frame), None, None, None)
return results
def add_file(self, path):
bname = os.path.basename(path)
f = file.File.open(path)
n_frames = f.info.num_frames
if bname not in self.path_dict.keys() and File.exists(path):
self.path_dict[bname] = str(path)
root = bname
sib = ['frame {}'.format(i) for i in range(1, n_frames + 1)]
self.append_item(root, sib)
if path not in config.recent_files:
config.add_recent_file(path)
else:
index = config.recent_files.index(path)
config.recent_files.pop(index)
config.add_recent_file(path)
self.files_changed()
def getresults(self, filepath, frame, resolution=None, surface=False, center=False):
"""
Get cached results for the given parameters.
**Parameters:**
`filepath` :
absolute path of the input file
`frame` :
the frame number
`resolution` :
resolution of the used discretization
`surface` :
query calculation results for surface-based cavities
`center` :
query calculation results for center-based cavities
**Returns:**
A :class:`core.data.Results` object if cached results exist,
else `None`
"""
inputfile = File.open(filepath)
# TODO: error handling
resultfile = None
results = None
if isinstance(inputfile, core.file.ResultFile):
resultfile = inputfile
elif filepath in self.cache:
resultfile = self.cache[filepath]
if resultfile is not None:
if resolution is None:
resolutions = sorted(resultfile.info.resolutions())[::-1]
resolution = 64
for res in resolutions:
if resultfile.info[res].domains[frame] is not None:
resolution = res
break
results = resultfile.getresults(frame, resolution)
if results is None:
if resolution is None:
resolution = 64
results = data.Results(filepath, frame, resolution, inputfile.getatoms(frame), None, None, None)
return results
def calculatedframes(self,
filepath,
resolution,
surface=False,
center=False):
"""
Query the cache if it contains results for the given parameters.
**Parameters:**
`filepath` :
absolute path of the input file
`resolution` :
resolution of the used discretization
`surface` :
query calculation results for surface-based cavities
`center` :
query calculation results for center-based cavities
**Returns:**
A :class:`core.data.TimestampList` containing the dates
of the calculation or `None`.
"""
info = None
inputfile = File.open(filepath)
# TODO: error handling
if isinstance(inputfile, core.file.ResultFile):
info = inputfile.info
else:
if filepath in self.cache:
cf = self.cache[filepath]
info = cf.info
if info is not None:
if surface:
return info[resolution].surface_cavities
elif center:
return info[resolution].center_cavities
else:
return info[resolution].domains
else:
return data.TimestampList(inputfile.info.num_frames)
def calculatedframes(self, filepath, resolution, surface=False, center=False):
"""
Query the cache if it contains results for the given parameters.
**Parameters:**
`filepath` :
absolute path of the input file
`resolution` :
resolution of the used discretization
`surface` :
query calculation results for surface-based cavities
`center` :
query calculation results for center-based cavities
**Returns:**
A :class:`core.data.TimestampList` containing the dates
of the calculation or `None`.
"""
info = None
inputfile = File.open(filepath)
# TODO: error handling
if isinstance(inputfile, core.file.ResultFile):
info = inputfile.info
else:
if filepath in self.cache:
cf = self.cache[filepath]
info = cf.info
if info is not None:
if surface:
return info[resolution].surface_cavities
elif center:
return info[resolution].center_cavities
else:
return info[resolution].domains
else:
return data.TimestampList(inputfile.info.num_frames)
def calculateframe(self,
filepath,
frame,
resolution,
cutoff_radii=None,
domains=False,
surface=False,
center=False,
atoms=None,
gyration_tensor_parameters=False,
recalculate=False,
last_frame=True):
"""
Get results for the given parameters. They are either loaded from the
cache or calculated.
**Parameters:**
`filepath` :
absolute path of the input file
`frame` :
the frame number
`resolution` :
resolution of the used discretization
`domains` :
calculate cavitiy domains
`cutoff_radii` :
dict that maps element symbols to cutoff radii
`surface` :
calculate surface-based cavities
`center` :
calculate center-based cavities
`gyration_tensor_parameters` :
gyration tensor parameters will be calculated for cavities (they are always calculated for
cavity domains)
`recalculate` :
results will be calculated even if cached results exists
**Returns:**
A :class:`core.data.Results` object.
"""
# always recalculate if gyration tensor parameters shall be computed for center or surface based cavities
recalculate = recalculate or (gyration_tensor_parameters and
(center or surface))
message.progress(0)
inputfile = File.open(filepath)
# TODO: error handling
if isinstance(inputfile, core.file.ResultFile):
resultfile = inputfile
else:
resultfile = self.cache[filepath]
try:
results = resultfile.getresults(frame, resolution)
except Exception as e:
logger.debug("error in resultfile.getresults: {}".format(e))
results = None
if atoms is None:
atoms = inputfile.getatoms(frame)
atoms.radii = cutoff_radii
volume = atoms.volume
if results is None:
results = data.Results(filepath, frame, resolution, atoms, None,
None, None)
if recalculate:
results.domains = None
results.surface_cavities = None
results.center_cavities = None
if not ((domains and results.domains is None) or
(surface and results.surface_cavities is None) or
(center and results.center_cavities is None)):
message.print_message("Reusing results")
else:
cachepath = os.path.join(self.cachedir,
'discretization_cache.hdf5')
discretization_cache = DiscretizationCache(cachepath)
with DiscretizationCache(cachepath) as discretization_cache:
discretization = discretization_cache.get_discretization(
volume, resolution)
atom_discretization = AtomDiscretization(atoms, discretization)
message.progress(10)
if (domains and results.domains is None) \
or (surface and results.surface_cavities is None):
# CavityCalculation depends on DomainCalculation
message.print_message("Calculating domains")
domain_calculation = DomainCalculation(discretization,
atom_discretization)
if domain_calculation.critical_domains:
logger.warn(
'Found {:d} critical domains in file {}, frame {:d}. Domain indices: {}'
.format(len(domain_calculation.critical_domains),
os.path.basename(filepath), frame,
domain_calculation.critical_domains))
message.log(
'Found {:d} critical domains in file {}, frame {:d}'.
format(
len(domain_calculation.critical_domains),
os.path.basename(filepath),
frame + 1,
))
if results.domains is None:
results.domains = data.Domains(domain_calculation)
message.progress(40)
if surface and results.surface_cavities is None:
message.print_message("Calculating surface-based cavities")
cavity_calculation = CavityCalculation(
domain_calculation,
use_surface_points=True,
gyration_tensor_parameters=gyration_tensor_parameters)
results.surface_cavities = data.Cavities(cavity_calculation)
message.progress(70)
if center and results.center_cavities is None:
message.print_message("Calculating center-based cavities")
domain_calculation = FakeDomainCalculation(
discretization, atom_discretization, results)
cavity_calculation = CavityCalculation(
domain_calculation,
use_surface_points=False,
gyration_tensor_parameters=gyration_tensor_parameters)
results.center_cavities = data.Cavities(cavity_calculation)
resultfile.addresults(results, overwrite=recalculate)
message.progress(100)
message.print_message("Calculation finished")
if last_frame:
message.finish()
return results
class HTML:
""" class for HTML reporting """
def __init__(self, date, opts, template_dir, report_dir, logs_dir):
""" constructor """
self.date = date
self.opts = opts
self.res = {} # holds the results out of log dirs
self.template_dir = template_dir
self.report_dir = report_dir
self.logs_dir = logs_dir
self.file = File()
self.file.copy_dirs(self.template_dir, self.report_dir)
self.index_html = f'{self.report_dir}/index.html'
self.res_html = f'{self.report_dir}/results.html'
self.index_html_data = self.file.read_file(self.index_html)
self.res_html_data = self.file.read_file(self.res_html)
return
def make_results(self):
""" get all results from the targets log directory """
logs = f'{self.logs_dir}/targets/'
targets = os.listdir(logs)
for target in targets:
target_orig = target
if ':' in target:
target = target.replace(':', '-')
self.res[target] = {}
for moddir in os.listdir(logs + target_orig):
if moddir in ('tcp', 'udp', 'social'):
submods = os.listdir(f'{logs}{target_orig}/{moddir}')
for s in submods:
for modname in os.listdir(
f'{logs}{target_orig}/{moddir}/{s}'):
if moddir == 'tcp' or moddir == 'udp':
module = f'{moddir}.{s}.{modname}'
else:
module = f'{moddir}.{modname}'
self.res[target][module] = {}
tool_path = f'{logs}{target_orig}/{moddir}/{s}/{modname}/'
for tool in os.listdir(tool_path):
if '.log' in tool:
self.res[target][module][tool] = \
'\n'.join(self.file.read_file(tool_path + tool))
else:
for modname in os.listdir(f'{logs}{target_orig}/{moddir}'):
target = target.replace(':', '-') # url port
module = f'{moddir}.{modname}'
self.res[target][module] = {}
for tool in os.listdir(
f'{logs}{target_orig}/{moddir}/{modname}'):
if '.log' in tool and '.bin' not in tool:
tool_path = f'{logs}{target_orig}/{moddir}/{modname}/'
self.res[target][module][tool] = '\n'.join(
self.file.read_file(tool_path + tool))
return
def make_target_html(self):
""" create $target.html with its contents/results """
_mods = ''
panels = []
panel = '''
<div class="panel panel-default">
<div class="panel-heading">
<h3 class="panel-title">$TOOL</h3>
</div>
<div class="panel-body">
<pre>$RESULT</pre>
</div>
</div>
'''
for target, mods in self.res.items():
t_html = f'{self.report_dir}/{target}.html'
self.file.copy_files(self.res_html, t_html)
t_html_data = self.file.read_file(t_html)
t_html_data = [w.replace('$TARGET', target) for w in t_html_data]
self.file.write_file(t_html, ' '.join(t_html_data))
for mod, tools in mods.items():
_mods += f'<h5>{mod}</h5>'
for tool, data in tools.items():
tlink = tool.split('.log')[0]
link = f'{mod}.{tlink}'
_mods += f'<a href="#{link}" target="_blank">{tlink}</a><br />'
p1 = panel.replace(f'$TOOL',
f'<a name="{link}">{link}</a>')
p2 = p1.replace('$RESULT', html.escape(data))
panels.append(p2)
t_html_data = [w.replace('$ALLMODS', _mods) for w in t_html_data]
_mods = ''
t_html_data = [w.replace('$CONTENT', ' '.join(panels)) \
for w in t_html_data]
panels = []
self.file.write_file(t_html, ' '.join(t_html_data))
return
def make_index_html(self):
""" create the index.html """
num_social_targets = 0
patterns = {
'$DATE': self.date,
'$CMDLINE_ARGS': ' '.join(self.opts['cmdline'])
}
# index.html: date + cmdline args
for k, v in patterns.items():
self.index_html_data = [
w.replace(k, v) for w in self.index_html_data
]
# index.html: num target modes
for k, v in self.opts['targets'].items():
if k == 'social':
for s in self.opts['targets'][k]:
if self.opts['targets'][k][s]:
num_social_targets += len(self.opts['targets'][k][s])
self.index_html_data = [
w.replace('$NUM_SOCIAL', str(num_social_targets))
for w in self.index_html_data
]
self.index_html_data = [w.replace(f'$NUM_{k.upper()}', str(len(v))) \
for w in self.index_html_data]
return
def make_menu(self, mod, targets):
""" create the menu and links """
# targets menu items
if mod == 'tcp' or mod == 'udp':
l = []
for target in self.opts['targets'][mod]:
target = f"<li><a href='{target['host']}.html'>{target['host']}</a></li>"
l.append(target)
self.index_html_data = [
w.replace(f'$TARGETS_{mod.upper()}', ' '.join(l)) \
for w in self.index_html_data
]
self.res_html_data = [
w.replace(f'$TARGETS_{mod.upper()}', ' '.join(l)) \
for w in self.res_html_data
]
elif mod == 'social':
l = []
for part in self.opts['targets'][mod]:
for target in self.opts['targets'][mod][part]:
target = f'<li><a href="{target}.html">{target}</a></li>'
l.append(target)
self.index_html_data = [
w.replace(f'$TARGETS_{mod.upper()}', ' '.join(l)) \
for w in self.index_html_data
]
self.res_html_data = [
w.replace(f'$TARGETS_{mod.upper()}', ' '.join(l)) \
for w in self.res_html_data
]
else:
for target in targets:
if '://' in target:
target = target.split('://')[1].rstrip('/').split('/')[0]
if ':' in target:
target = target.replace(':', '-') # port
target = f'<li><a href="{target}.html">{target}</a></li>'
self.index_html_data = [
w.replace(f'$TARGETS_{mod.upper()}', target) \
for w in self.index_html_data
]
self.res_html_data = [
w.replace(f'$TARGETS_{mod.upper()}', target) \
for w in self.res_html_data
]
return
def make_report(self):
""" make the report """
self.make_results()
for mod, targets in self.opts['targets'].items():
self.make_menu(mod, targets)
self.make_index_html()
self.file.write_file(self.index_html, ' '.join(self.index_html_data))
self.file.write_file(self.res_html, ' '.join(self.res_html_data))
self.make_target_html()
self.file.del_file(f'{self.report_dir}/results.html')
return
def calculate(self, calcsettings):
"""
Calculate (or load from the cache) all results for the given settings.
**Parameters:**
`calcsettings` :
:class:`CalculationSettings` object
**Returns:**
A list of list of :class:`core.data.Results` objects. The outer list contains
an entry for each entry in `calcsettings.filenames`; the inner
list has a `Results` entry for each frame specified in
`calcsettings.frames`.
"""
allresults = []
for filename, frames in calcsettings.datasets.iteritems():
filepath = core.file.get_abspath(filename)
fileprefix = os.path.basename(filename).rsplit(".", 1)[0]
if calcsettings.exportdir is not None:
exportdir = core.file.get_abspath(calcsettings.exportdir)
# replace asterisks with directories
dirlist = os.path.dirname(filepath).split("/")
while "*" in exportdir:
i = exportdir.rindex("*")
exportdir = os.path.join(exportdir[:i],
dirlist.pop() + exportdir[i + 1:])
if (calcsettings.exporthdf5 or calcsettings.exporttext) \
and not os.path.exists(exportdir):
os.makedirs(exportdir)
else:
exportdir = os.path.dirname(filepath)
if calcsettings.exporthdf5:
efpath = os.path.join(exportdir, fileprefix + ".hdf5")
efpath = core.file.get_abspath(efpath)
# copy atoms into HDF5 file
exportfile = core.file.HDF5File.fromInputFile(efpath, filepath)
# use HDF5 file as input
filepath = efpath
fileresults = []
if frames[0] == -1:
inputfile = File.open(filepath)
frames = range(inputfile.info.num_frames)
last_frame = False
for frame in frames:
# calculate single frame
if frame is frames[-1]:
last_frame = True
frameresult = self.calculateframe(
filepath,
frame,
calcsettings.resolution,
calcsettings.cutoff_radii,
domains=calcsettings.domains,
surface=calcsettings.surface_cavities,
center=calcsettings.center_cavities,
gyration_tensor_parameters=calcsettings.gyration_tensor,
recalculate=calcsettings.recalculate,
last_frame=last_frame)
# export to text file
if calcsettings.exporttext:
fmt = os.path.join(
exportdir, fileprefix) + "-{property}-{frame:06d}.txt"
if frameresult.atoms is not None:
frameresult.atoms.totxt(
fmt.format(property='{property}', frame=frame + 1))
if frameresult.domains is not None:
try:
frameresult.domains.totxt(
fmt.format(property='domain_{property}',
frame=frame + 1))
except ValueError as e:
logger.warn(e.message)
logger.warn(
'The export of domain information could not be finished.'
)
if frameresult.surface_cavities is not None:
frameresult.surface_cavities.totxt(
fmt.format(property='surface_cavities_{property}',
frame=frame + 1))
if frameresult.center_cavities is not None:
frameresult.center_cavities.totxt(
fmt.format(property='center_cavities_{property}',
frame=frame + 1))
# TODO: try/except
# gather results
fileresults.append(frameresult)
allresults.append(fileresults)
return allresults
class Controller:
""" program controller class """
def __init__(self):
""" constructor """
# options and modules
self.opt = None
self.mod = Module(MOD_PATH)
# logger
self.logger = Logger()
self.log = self.logger.log
# rest we need
self.file = File()
self.check = Check()
self.misc = Misc()
self.parser = None
# nullscan working dir
self.nullscan_dir = None
return
def prepare(self):
""" preparation/initialization of opts and env: parsing & checks """
# declare nullscan options
self.opt = Option(sys.argv)
# check argc and argc (usage)
self.check.check_argc(len(sys.argv))
self.check.check_argv(sys.argv)
# check for missing libraries / deps / python modules
self.check.check_deps(self.file.read_file(PYDEPS))
# parse cmdline and config options, update final options dictionary
try:
self.parser = Parser(self.opt.opts)
self.parser.parse_cmdline()
self.parser.parse_config()
self.opt.opts = self.parser.opts
except:
self.log('usage', _type='err', end='\n')
# update final options dictionary
self.opt.update_opts()
# further checks for usage, options, env, etc.
self.check.check_opts(self.opt.opts)
# collect all py-files and grep the tools out of the py-files
tools = []
py_files = self.misc.find_py_files(MOD_PATH)
for py in py_files:
tools.append(self.misc.grep_tools(py))
tools = [x for sublist in tools for x in sublist]
# create the locks for each tool except for excluded ones
with ThreadPoolExecutor(50) as exe:
for tool in tools:
if tool not in self.opt.opts['tools']['ex_tools']:
exe.submit(self.file.create_lock, tool)
# copy debug flag to target_opts (for nullscan tools)
self.opt.opts['targets_opts']['debug'] = self.opt.opts['debug']
return
def run_misc(self):
""" run chosen misc options """
if self.opt.opts['check_tools']:
self.log('Checking for missing tools\n\n', _type='msg')
self.check.check_tools()
os._exit(SUCCESS)
if self.opt.opts['print_tools']:
self.log('Printing tools and information\n\n', _type='msg')
self.misc.print_tool(self.opt.opts['print_tools'])
os._exit(SUCCESS)
if self.opt.opts['add_module']:
self.log('Adding new module\n', _type='msg')
self.misc.add_mod_tool('mod', self.opt.opts['add_module'])
os._exit(SUCCESS)
if self.opt.opts['add_tool']:
self.log('Adding new tool\n', _type='msg')
self.misc.add_mod_tool('tool', self.opt.opts['add_tool'])
os._exit(SUCCESS)
return
def prepare_modules(self):
""" filter in-/excluded mods + return a unique list """
self.mod.filter_modules(self.opt.opts)
self.mod.mods = list(set(self.mod.mods))
return
def run_nmap_mode(self):
""" run nmap scan mode """
nmap = Nmap(self.opt.opts['targets']['nmap'])
# create nmap logdir based on scan type requested
logpath = f'{self.nullscan_dir}/logs/nmap/{nmap.get_protocol()}'
logfile = f'{logpath}/results'
self.file.make_dir(logpath)
nmap.set_logfile(logfile)
# build nmap command line
nmap.build_cmd()
# start scans
self.log('NMAP mode activated\n', _type='msg', color='blue')
self.log('Targets added: {}\n'.format(
len(self.opt.opts['targets']['nmap']['hosts'])),
_type='msg')
if self.opt.opts['verbose']:
self.log('\n')
for target in self.opt.opts['targets']['nmap']['hosts']:
self.log(f'{target}\n', _type='msg')
nmap.scan(debug=self.opt.opts['debug'])
return f'{logfile}.xml'
def run_social_mode(self, target):
""" run social mode """
if '.social.' in str(self.mod.mods):
# availabel social modules to import and load
mods = [i for i in self.mod.mods if '.social.' in i]
with ProcessPoolExecutor(self.opt.opts['m_workers']) as exe:
for key in target.keys():
if target[key]:
for t in target[key]:
# default module
rdir = f'{self.nullscan_dir}/logs/targets/'
wdir = f"{rdir}{t}/social/{key}/default"
exe.submit(self.mod.run_module,
'modules.social.default', t,
self.opt.opts, wdir)
# non-default modules
for m in mods:
if 'default' not in m:
splitted = m.split('.')
moddir = splitted[1]
modname = splitted[2]
wdir = f"{rdir}{t}/{moddir}/{key}/{modname}"
exe.submit(self.mod.run_module, m, t,
self.opt.opts, wdir)
return
def run_wifi_mode(self, target):
""" run wifi mode """
if '.wifi.' in str(self.mod.mods):
# available wifi modules to import and load
mods = [i for i in self.mod.mods if '.wifi.' in i]
# default module first
rdir = f'{self.nullscan_dir}/logs/targets/'
wdir = f'{rdir}{target}/wifi/default'
self.mod.run_module('modules.wifi.default', target, self.opt.opts,
wdir)
# non-default modules
with ProcessPoolExecutor(self.opt.opts['m_workers']) as exe:
for m in mods:
if 'default' not in m:
splitted = m.split('.')
moddir = splitted[1]
modname = splitted[2]
wdir = f'{rdir}{target}/{moddir}/{modname}'
exe.submit(self.mod.run_module, m, target,
self.opt.opts, wdir)
return
def run_lan_mode(self, target):
""" run lan mode """
if '.lan.' in str(self.mod.mods):
# available lan modules to import and load
mods = [i for i in self.mod.mods if '.lan.' in i]
# default module first
rdir = f'{self.nullscan_dir}/logs/targets/'
wdir = f'{rdir}{target}/lan/default'
self.mod.run_module('modules.lan.default', target, self.opt.opts,
wdir)
# non-default modules
with ProcessPoolExecutor(self.opt.opts['m_workers']) as exe:
for m in mods:
if 'default' not in m:
splitted = m.split('.')
moddir = splitted[1]
modname = splitted[2]
wdir = f'{rdir}{target}/{moddir}/{modname}'
exe.submit(self.mod.run_module, m, target,
self.opt.opts, wdir)
return
def run_web_mode(self, target):
""" run web mode """
if '.web.' in str(self.mod.mods):
# available web modules to import and load
mods = [i for i in self.mod.mods if '.web.' in i]
# we need host name for working directory
host = requests.utils.urlparse(target).netloc
# default module first
rdir = f'{self.nullscan_dir}/logs/targets/'
wdir = f'{rdir}{host}/web/default'
self.mod.run_module('modules.web.default', target, self.opt.opts,
wdir)
# non-default modules
with ProcessPoolExecutor(self.opt.opts['m_workers']) as exe:
for m in mods:
if 'default' not in m:
splitted = m.split('.')
moddir = splitted[1]
modname = splitted[2]
wdir = f'{rdir}{host}/{moddir}/{modname}'
exe.submit(self.mod.run_module, m, target,
self.opt.opts, wdir)
return
def run_udp_mode(self, target):
""" run udp mode """
# we need to run host modules before tcp and we need to run default
# tool first
self.run_host_mode(target)
# now tcp modules
if '.udp.' in str(self.mod.mods):
with ProcessPoolExecutor(self.opt.opts['m_workers']) as exe:
for p in target['ports']:
exe.submit(self.run_tcp_udp_mode, target, p, 'udp')
return
def run_tcp_mode(self, target):
""" run tcp mode """
# we need to run host modules before tcp and we need to run default
# module first
self.run_host_mode(target)
# now tcp modules
if '.tcp.' in str(self.mod.mods):
with ProcessPoolExecutor(self.opt.opts['m_workers']) as exe:
for p in target['ports']:
exe.submit(self.run_tcp_udp_mode, target, p, 'tcp')
return
def run_tcp_udp_mode(self, host, port, proto):
""" wrapper for tcp/udp mode """
# available modules
mod = f'modules.{proto}.{port[1]}'
# force default module for given port if module does not exist
if mod not in self.mod.mods:
mod = f'modules.{proto}.default'
# new target dict as we only need the corresponding port
t = {'host': host['host'], 'port': port[0]}
# default module first
rdir = f'{self.nullscan_dir}/logs/targets/'
wdir = f"{rdir}{t['host']}/{proto}/{port[0]}/default"
self.mod.run_module(f'modules.{proto}.default', t, self.opt.opts, wdir)
# now non-default module
if '.default' not in mod:
wdir = f"{rdir}{t['host']}/{proto}/{port[0]}/{port[1]}"
self.mod.run_module(mod, t, self.opt.opts, wdir)
return
def run_host_mode(self, target):
""" run host mode """
# available host modules to import and load
mods = [i for i in self.mod.mods if '.host.' in i]
# default module
rdir = f'{self.nullscan_dir}/logs/targets/'
wdir = f"{rdir}{target['host']}/host/default"
self.mod.run_module('modules.host.default', target, self.opt.opts,
wdir)
# non-default modules
with ProcessPoolExecutor(self.opt.opts['m_workers']) as exe:
for m in mods:
if 'default' not in m:
splitted = m.split('.')
moddir = splitted[1]
modname = splitted[2]
wdir = f"{rdir}{target['host']}/{moddir}/{modname}"
exe.submit(self.mod.run_module, m, target, self.opt.opts,
wdir)
return
def run_modes(self):
""" run chosen modes """
scans = []
modes = {
'tcp': self.run_tcp_mode,
'udp': self.run_udp_mode,
'wifi': self.run_wifi_mode,
'web': self.run_web_mode,
'social': self.run_social_mode,
'http': self.run_web_mode,
'https': self.run_web_mode,
}
# run lan mode first if requested
if self.opt.opts['targets']['lan']:
ifaces = []
self.log('LAN mode activated\n', color='blue', _type='msg')
self.log(
f"Targets added: {len(self.opt.opts['targets']['lan'])}\n\n",
_type='msg')
for iface in self.opt.opts['targets']['lan']:
if self.opt.opts['verbose']:
self.log(f'{iface}\n', _type='vmsg')
ifaces.append((self.run_lan_mode, iface))
if self.opt.opts['verbose']:
self.log('\n')
with ProcessPoolExecutor(self.opt.opts['t_workers']) as exe:
self.log('Shooting tools\n\n', color='green', _type='msg')
for iface in ifaces:
exe.submit(iface[0], iface[1])
self.log('\n')
if not self.opt.opts['verbose']:
self.log('\n')
self.log('Tools done\n\n', color='green', _type='msg')
for log in glob.glob('**/lan/portscan/*.xml', recursive=True):
if log:
self.parser.parse_nmap_logfile(log, lan=True)
# collect scan modes for each target
for k, v in self.opt.opts['targets'].items():
if self.opt.opts['targets'][k]:
if k == 'lan':
continue
else:
self.log(f'{k.upper()} mode activated\n',
color='blue',
_type='msg')
self.log(f'Targets added: {len(v)}\n\n', _type='msg')
if k == 'social':
if self.opt.opts['verbose']:
for targets in v.values():
for target in targets:
self.log(f'{target}\n', _type='vmsg')
self.log('\n')
scans.append((modes[k], v))
else:
for target in v:
scans.append((modes[k], target))
if self.opt.opts['verbose']:
if 'host' in target:
target_head = target['host']
else:
target_head = target
self.log(f'{target_head}\n', _type='vmsg')
if self.opt.opts['verbose']:
self.log('\n')
# start mode for each target
with ProcessPoolExecutor(self.opt.opts['t_workers']) as exe:
if scans:
self.log('Shooting tools\n\n', color='green', _type='msg')
for scan in scans:
exe.submit(scan[0], scan[1])
if scans:
self.log('\n')
if not self.opt.opts['verbose']:
self.log('\n')
self.log('Tools done\n\n', _type='msg', color='green')
return
def start(self):
""" nullscan starts here with actions """
self.check.check_uid()
self.log('Game Started\n\n', _type='msg')
# create nullscan working, targets and log dir
self.nullscan_dir = self.file.make_dir(self.opt.opts['nullscan_dir'],
incr=True)
self.file.make_dir(f'{self.nullscan_dir}/logs/targets')
self.opt.opts['targets_opts']['nullscan_logdir'] = \
f'{self.nullscan_dir}/logs/targets/'
# run nmap mode first if requested
if 'hosts' in self.opt.opts['targets']['nmap']:
self.parser.parse_nmap_logfile(self.run_nmap_mode())
self.log('\n')
# delete nmap key
del self.opt.opts['targets']['nmap']
# prepare modules for other modes
self.prepare_modules()
# run the nullscan modes now
self.run_modes()
return
def end(self):
""" program ends here. clean-ups, reporting, etc. """
# go back to root dir
os.chdir(ROOT_PATH)
# clean up empty directories and empty (log-)files or logfiles containing
# a singl ebyte (newline) (failed tools)
self.misc.remove_empty_files_dirs(f'{self.nullscan_dir}/logs/targets/')
# just to be safe - delete all locks, in case tools didn't
self.file.del_file('/tmp/nullscan', _dir=True)
# create report
if self.opt.opts['report']:
self.log('Creating report\n', _type='msg')
tmpl_dir = f'{ROOT_PATH}/src/report/template'
rep_dir = f'{self.nullscan_dir}/report'
logs_dir = f'{self.nullscan_dir}/logs'
self.report = HTML(TODAY, self.opt.opts, tmpl_dir, rep_dir,
logs_dir)
self.report.make_report()
self.log('Report done\n\n', _type='msg')
self.log('Game Over\n', _type='msg')
# reset terminal to original state. sometimes f**k up occurs because of
# color and other escape codes.
self.misc.reset_terminal()
return
class Module:
""" module handler class """
def __init__(self, mod_path):
""" constructor """
# logger
self.logger = Logger()
self.log = self.logger.log
# file i/o
self.file = File()
# get all modules
self.mod_path = mod_path
self.mods = []
self.get_modules()
# all loaded module
self.lmod = {}
# docstrings for all tools
self.docstrings = {}
return
def get_docstrings(self):
""" get a list of docstrings from all nullscan tools """
tools = [] # tools from sub classes
for mod in self.mods:
self.load_module(mod)
cls = next(iter(self.lmod[mod].keys()))
for tool in self.lmod[mod][cls]:
modsplit = mod.split('.')
moddir = modsplit[1]
module = modsplit[2]
if 'Base' not in str(cls):
t_attr = getattr(cls, tool)
docstr = ' '.join(t_attr.__doc__.split()).split()
try:
d_idx = docstr.index('DESCR:')
t_idx = docstr.index('TOOLS:')
except:
pass
descr = ' '.join((docstr[d_idx + 1:t_idx]))
tools.append(docstr[t_idx + 1:])
unique_tools = list(set(itertools.chain.from_iterable(tools)))
self.docstrings[tool] = {'moddir': moddir, 'module': module,
'descr': descr, 'tools': unique_tools }
return
def get_objects(self, mod, lmod):
""" get classes and their methods out of a loaded module """
methods = []
for d in dir(lmod):
if d[0].isupper() and 'Base' not in d:
attr = getattr(lmod, d) # class
for m in dir(attr): # public methods
if not m.startswith('_') and not m.startswith('__') and \
callable(getattr(attr, m)):
methods.append(m)
self.lmod = {mod: {attr: list(filter(None, methods))}}
return
def get_modules(self):
""" get all modules out of MOD_PATH directory """
for f in glob.glob(f'{self.mod_path}/**', recursive=True):
if '__' not in f and '/libs/' not in f and f.endswith('.py'):
self.mods.append('.'.join(f.split('/')[-3:]).split('.py')[0])
#for root, dirs, files in os.walk(self.mod_path, topdown=True):
# dirs[:] = [d for d in dirs if d != 'libs']
# for f in files:
# if f != '__init__.py' and f.endswith('.py'):
# mod = '.'.join(os.path.join(root, f).split('/')[-3:])
# self.mods.append((mod.replace('.py', '')))
return
def filter_modules(self, opts):
""" filter included or excluded modules """
# all default modules out of modules/ dir
def_mods = [
x.replace('/','.').replace('src.','') + '.default' \
for x in glob.glob('src/modules/**') if not '__' in x
]
tmp = []
# add/remove chosen module by user
if 'in_modules' in opts['modules'] and opts['modules']['in_modules']:
# list with default modules (force) + user chosed modules to import
self.mods = def_mods
for key, val in opts['modules']['in_modules'].items():
for v in val:
self.mods.append(f'modules.{key}.{v}')
elif 'ex_modules' in opts['modules'] and opts['modules']['ex_modules']:
for key, val in opts['modules']['ex_modules'].items():
for v in val:
if v == 'default':
self.log('mod_default', _type='err', end='\n')
tmp.append(f'modules.{key}.{v}')
for i in self.mods:
if i in tmp:
self.mods.remove(i)
return
def load_module(self, mod):
""" load desired module and get classes+methods from loaded modules """
try:
lmod = importlib.import_module(mod)
self.get_objects(mod, lmod)
except Exception as e:
self.log('mod_import', eargs=f"{mod} -> {e}", _type='err', end='\n')
return
def run_module(self, mod, target, opts, wdir):
""" load and run module with its tools """
# change temp working dir for tool logs
rootdir = os.getcwd()
os.chdir(self.file.make_dir(wdir))
# load module, get class and create object of
self.load_module(mod)
cls = next(iter(self.lmod[mod].keys()))
c = cls(target, opts['targets_opts'])
# for status line
cur_tool = 0
sum_tool = len(self.lmod[mod][cls])
# get method (tool) string, create object and run desired tool
with ProcessPoolExecutor(opts['p_workers']) as exe:
for t in self.lmod[mod][cls]:
cur_tool += 1
if 'host' in target:
target_head = target['host']
else:
target_head = target
stat_line = f"{target_head} | {'.'.join(mod.split('.')[1:])}.{t}" + \
f' ({cur_tool}/{sum_tool})' + ' ' * 25
tool = getattr(c, t)
# filter in-/ex-cluded tools by user
if '.default' not in repr(cls):
if opts['tools']['in_tools']:
if t not in opts['tools']['in_tools']:
continue
if t in opts['tools']['ex_tools']:
continue
# run tool
try:
if opts['verbose']:
self.log(stat_line, _type='vmsg', end='\n')
else:
self.log(stat_line, _type='vmsg', flush=True, end='\r')
exe.submit(tool)
except:
self.log('tool_failed', eargs=t, _type='err', end='\n')
# done, move bitch...
os.chdir(rootdir)
return
def setUp(self):
self.file = File(api_url = config.API_URL, token = config.TOKEN)
self.token = config.TOKEN
def get(self, **kwargs):
args = self.parser.parse_args()
path = args['path']
if os.path.isfile(path):
return File(path)
raise UnprocessableEntity
class Misc:
""" class for miscellaneous stuff """
def __init__(self):
""" constructor """
# logger
self.logger = Logger()
self.log = self.logger.log
# file i/o
self.file = File()
# original terminal state
self.term_fd = sys.stdin.fileno()
self.term_state = termios.tcgetattr(self.term_fd)
return
def reset_terminal(self):
""" reset terminal to original state """
termios.tcsetattr(self.term_fd, termios.TCSADRAIN, self.term_state)
return
def grep_tools(self, py_file):
""" Find all tools (method names) of given file """
tools = []
lines = self.file.read_file(py_file)
for line in lines:
if 'def ' in line:
method = line.split()[1].split('(')[0]
if not method.startswith('_'):
tools.append(method)
return sorted(list(set(tools)))
def find_py_files(self, root_path):
""" Find all py files with some exceptions of given root_path """
py_files = []
files = glob.glob(f'{root_path}/*/*.py', recursive=True)
[py_files.append(x) for x in files if '__ini' not in x and '/libs' not in x]
return sorted(list(set(py_files)))
def kill_process(self, pattern, signal='TERM'):
""" kill (all) processes matched by pattern """
cmd = ['pkill', f'-{signal}', '-f', pattern]
subprocess.run(cmd)
return
def remove_empty_files_dirs(self, rootpath):
""" delete empty (log-)files and directories """
threads = 20
files = glob.glob(f'{rootpath}/**', recursive=True)
# files
with ThreadPoolExecutor(max_workers=threads) as exe:
# files
for f in files:
if os.path.isfile(f):
fsize = os.path.getsize(f)
if fsize <= 3:
exe.submit(os.unlink, f)
# directories
for f in files:
if os.path.isdir(f) and not os.listdir(f):
exe.submit(os.rmdir, f)
self.remove_empty_files_dirs(rootpath)
return
def lookup_port_service(self, port, proto='tcp'):
""" return corresponding service from services.csv for a given port """
service = None
services = self.file.read_csv_file(f'{ROOT_PATH}/lists/services.csv')
for s in services:
if port in s and proto in s:
service = s[0]
return service
def print_tool(self, opts):
""" print tool and a description of it """
tools = []
m = Module(MOD_PATH)
m.get_docstrings()
if 'all' in opts.keys():
for tool in m.docstrings.keys():
tools.append(tool)
else:
for moddir, module in opts.items():
if module:
for mod in module:
for tool in m.docstrings:
if m.docstrings[tool]['moddir'] == moddir and \
m.docstrings[tool]['module'] == mod:
tools.append(tool)
else:
for k, v in m.docstrings.items():
if moddir in v['moddir']:
tools.append(k)
tools = sorted(list(set(tools)))
for tool in tools:
self.log(m.docstrings[tool]['moddir'] + '/' +
m.docstrings[tool]['module'] + '/' + tool + ' - ' +
m.docstrings[tool]['descr'], end='\n', _type='vmsg')
return
def add_mod_tool(self, form, opts):
""" create a new module and/or add tool to exisiting module """
template = f'{MOD_PATH}libs/template.py'
modpart = f"{MOD_PATH}{opts['moddir']}/"
moddir = modpart
module = f"{modpart}{opts['modname']}.py"
tmpmod = f'{modpart}tmpxxx.py'
if form == 'mod':
with open(template, 'r') as fin:
if not os.path.isdir(moddir):
self.file.make_dir(moddir)
with open(module, 'w') as fout:
for line in fin:
if '<template>.py' in line:
fout.write(line.replace('<template>', opts['modname']))
elif '<file>' in line:
fout.write(line.replace('<file>', opts['modname']))
elif '<class>' in line:
fout.write(line.replace('<class>',
opts['modname'].capitalize()))
elif '<mod>' in line:
fout.write(line.replace('<mod>', opts['modname']))
else:
fout.write(line)
self.log(f"Created module {opts['moddir']}/{opts['modname']}.py\n",
_type='msg')
# append new tools
try:
shutil.copyfile(module, tmpmod)
with open(tmpmod, 'a') as fout:
fout.write('\n\n @tool\n')
fout.write(f" def {opts['func']}(self):\n")
fout.write(' """\n')
fout.write(' DESCR: <REPLACE MANUALLY>\n')
fout.write(' TOOLS: <ADD MANUALLY>\n')
fout.write(' """\n\n')
fout.write(f" opts = \'{' '.join(opts['args'])}\'\n\n")
fout.write(f" self._run_tool(\'{opts['tool']}\', opts)\n\n")
fout.write(' return\n')
shutil.move(tmpmod, module)
self.log(f"Added tool {opts['func']} to {opts['moddir']}/" +
f"{opts['modname']}.py\n", _type='msg')
except:
self.log('add_tool', _type='err', end='\n')
return
def calculateframe(self, filepath, frame, resolution, cutoff_radii=None, domains=False, surface=False, center=False,
atoms=None, gyration_tensor_parameters=False, recalculate=False, last_frame=True):
"""
Get results for the given parameters. They are either loaded from the
cache or calculated.
**Parameters:**
`filepath` :
absolute path of the input file
`frame` :
the frame number
`resolution` :
resolution of the used discretization
`domains` :
calculate cavitiy domains
`cutoff_radii` :
dict that maps element symbols to cutoff radii
`surface` :
calculate surface-based cavities
`center` :
calculate center-based cavities
`gyration_tensor_parameters` :
gyration tensor parameters will be calculated for cavities (they are always calculated for
cavity domains)
`recalculate` :
results will be calculated even if cached results exists
**Returns:**
A :class:`core.data.Results` object.
"""
# always recalculate if gyration tensor parameters shall be computed for center or surface based cavities
recalculate = recalculate or (gyration_tensor_parameters and (center or surface))
message.progress(0)
inputfile = File.open(filepath)
# TODO: error handling
if isinstance(inputfile, core.file.ResultFile):
resultfile = inputfile
else:
resultfile = self.cache[filepath]
try:
results = resultfile.getresults(frame, resolution)
except Exception as e:
logger.debug("error in resultfile.getresults: {}".format(e))
results = None
if atoms is None:
atoms = inputfile.getatoms(frame)
atoms.radii = cutoff_radii
volume = atoms.volume
if results is None:
results = data.Results(filepath, frame, resolution, atoms, None, None, None)
if recalculate:
results.domains = None
results.surface_cavities = None
results.center_cavities = None
if not ((domains and results.domains is None)
or (surface and results.surface_cavities is None)
or (center and results.center_cavities is None)):
message.print_message("Reusing results")
else:
cachepath = os.path.join(self.cachedir, 'discretization_cache.hdf5')
discretization_cache = DiscretizationCache(cachepath)
discretization = discretization_cache.get_discretization(volume, resolution)
atom_discretization = AtomDiscretization(atoms, discretization)
message.progress(10)
if (domains and results.domains is None) \
or (surface and results.surface_cavities is None):
# CavityCalculation depends on DomainCalculation
message.print_message("Calculating domains")
domain_calculation = DomainCalculation(discretization, atom_discretization)
if domain_calculation.critical_domains:
logger.warn('Found {:d} critical domains in file {}, frame {:d}. Domain indices: {}'.format(
len(domain_calculation.critical_domains), os.path.basename(filepath), frame,
domain_calculation.critical_domains
))
message.log('Found {:d} critical domains in file {}, frame {:d}'.format(
len(domain_calculation.critical_domains), os.path.basename(filepath), frame + 1,
))
if results.domains is None:
results.domains = data.Domains(domain_calculation)
message.progress(40)
if surface and results.surface_cavities is None:
message.print_message("Calculating surface-based cavities")
cavity_calculation = CavityCalculation(domain_calculation, use_surface_points=True,
gyration_tensor_parameters=gyration_tensor_parameters)
results.surface_cavities = data.Cavities(cavity_calculation)
message.progress(70)
if center and results.center_cavities is None:
message.print_message("Calculating center-based cavities")
domain_calculation = FakeDomainCalculation(discretization, atom_discretization, results)
cavity_calculation = CavityCalculation(domain_calculation, use_surface_points=False,
gyration_tensor_parameters=gyration_tensor_parameters)
results.center_cavities = data.Cavities(cavity_calculation)
resultfile.addresults(results, overwrite=recalculate)
message.progress(100)
message.print_message("Calculation finished")
if last_frame:
message.finish()
return results
def calculate(self, calcsettings):
"""
Calculate (or load from the cache) all results for the given settings.
**Parameters:**
`calcsettings` :
:class:`CalculationSettings` object
**Returns:**
A list of list of :class:`core.data.Results` objects. The outer list contains
an entry for each entry in `calcsettings.filenames`; the inner
list has a `Results` entry for each frame specified in
`calcsettings.frames`.
"""
allresults = []
for filename, frames in calcsettings.datasets.iteritems():
filepath = core.file.get_abspath(filename)
fileprefix = os.path.basename(filename).rsplit(".", 1)[0]
if calcsettings.exportdir is not None:
exportdir = core.file.get_abspath(calcsettings.exportdir)
# replace asterisks with directories
dirlist = os.path.dirname(filepath).split("/")
while "*" in exportdir:
i = exportdir.rindex("*")
exportdir = os.path.join(exportdir[:i], dirlist.pop() + exportdir[i+1:])
if (calcsettings.exporthdf5 or calcsettings.exporttext) \
and not os.path.exists(exportdir):
os.makedirs(exportdir)
else:
exportdir = os.path.dirname(filepath)
if calcsettings.exporthdf5:
efpath = os.path.join(exportdir, fileprefix + ".hdf5")
efpath = core.file.get_abspath(efpath)
# copy atoms into HDF5 file
exportfile = core.file.HDF5File.fromInputFile(efpath, filepath)
# use HDF5 file as input
filepath = efpath
fileresults = []
if frames[0] == -1:
inputfile = File.open(filepath)
frames = range(inputfile.info.num_frames)
last_frame = False
for frame in frames:
# calculate single frame
if frame is frames[-1]:
last_frame = True
frameresult = self.calculateframe(
filepath,
frame,
calcsettings.resolution,
calcsettings.cutoff_radii,
domains=calcsettings.domains,
surface=calcsettings.surface_cavities,
center=calcsettings.center_cavities,
gyration_tensor_parameters=calcsettings.gyration_tensor,
recalculate=calcsettings.recalculate,
last_frame=last_frame)
# export to text file
if calcsettings.exporttext:
fmt = os.path.join(exportdir, fileprefix) + "-{property}-{frame:06d}.txt"
if frameresult.atoms is not None:
frameresult.atoms.totxt(fmt.format(property='{property}', frame=frame+1))
if frameresult.domains is not None:
try:
frameresult.domains.totxt(fmt.format(property='domain_{property}', frame=frame+1))
except ValueError as e:
logger.warn(e.message)
logger.warn('The export of domain information could not be finished.')
if frameresult.surface_cavities is not None:
frameresult.surface_cavities.totxt(fmt.format(property='surface_cavities_{property}', frame=frame+1))
if frameresult.center_cavities is not None:
frameresult.center_cavities.totxt(fmt.format(property='center_cavities_{property}', frame=frame+1))
# TODO: try/except
# gather results
fileresults.append(frameresult)
allresults.append(fileresults)
return allresults
class Parser:
""" class for parsing cmdline and config file from nullscan """
def __init__(self, opts):
""" constructor"""
self.opts = opts # all nullscan options
self.logger = Logger()
self.log = self.logger.log
self.file = File()
self.misc = Misc()
# tmp stupid fix to strip bad chars out of options
self.bad_chars = '<>()[]{}:;.,="\'*&^%$#@!+-_/?°²³´`'
return
def parse_config(self):
""" parse nullscan config file and get options+values as dictionary+list. """
try:
cfg = ConfigParser()
data = cfg.read(self.opts['config']['file'])
if not data:
raise Exception
for s in cfg.sections():
for o in cfg.options(s):
values = cfg.get(s, o).replace('$NULLSCAN_DIR/', ROOT_PATH)
if ',' in values and o != 'nmap':
self.opts['config']['opts'][o] = values.split(',')
else:
self.opts['config']['opts'][o] = values
except:
self.log('config',
eargs=f"{self.opts['config']['file']}",
_type='err',
end='\n')
return
def parse_add_module_tool(self, args, dest):
""" parse add module/tool option (-m/-a) """
s = args.split()
try:
m = s[0].split('/')
d = {
'moddir': m[0],
'modname': m[1],
'func': s[1],
'tool': s[2],
'args': s[3:]
}
except:
self.log('add_mod_tool', _type='err', end='\n')
self.opts[dest] = d
return
def parse_print_tools(self, args):
""" parse print tools option (-p) """
dicted = {}
splitted = list(filter(None, args.split(';')))
for item in splitted:
if item == 'all':
dicted['all'] = True
break
if '=' in item:
i = item.split('=')
for j in i:
if ',' in j:
dicted[i[0]] = j.split(',')
else:
dicted[i[0]] = [j]
else:
dicted[item] = []
self.opts['print_tools'] = dicted
return
def parse_tools(self, args, dest):
""" parse tools include/exclude option (-I/-X) """
self.opts['tools'][dest] = args.split(',')
return
def parse_modules(self, args, dest):
""" parse modules include/exclude options (-i/-x) """
# remove empty ones, if ';' was used at the end without further opts
if ';' in args:
mods = [x.strip(self.bad_chars) for x in args.split(';') if x]
elif '=' in args:
mods = args.split()
else:
self.log('inexmods', _type='err', end='\n')
# create dict out of list itmes
dicted = dict(d.split('=') for d in mods)
for key, val in dicted.items():
if ',' in val:
dicted[key] = val.split(',')
else:
dicted[key] = [val]
self.opts['modules'][dest] = dicted
return
def parse_extra_opts(self, args):
""" parse extra options option (-o) """
# remove empty ones, if ';' was used at the end without further opts
if ';' in args:
args = [x.strip(self.bad_chars) for x in args.split(';') if x]
else:
args = [args]
# create dict out of list items except for nmap
dicted = dict(d.split('=') for d in args)
for key in dicted.keys():
if ',' in dicted[key] and key != 'nmap':
dicted[key] = dicted[key].split(',')
self.opts['targets_opts'] = dicted
return
def parse_nmap_targets(self, targets):
""" parse nmap targets to scan from either ranges or from file (-t) """
hosts = []
if os.path.isfile(targets):
hosts = self.file.read_file(targets)
else:
try:
# host range format
if '-' in targets:
splitted = targets.split('-')
try:
start = ipaddress.IPv4Address(splitted[0])
end = ipaddress.IPv4Address(splitted[1])
for i in range(int(start), int(end) + 1):
ipaddr = str(ipaddress.IPv4Address(i))
hosts.append(ipaddr)
except:
# must be a hostname/domain then
if ',' in targets:
hosts = targets.split(',')
else:
hosts.append(targets)
# cidr range format
elif '/' in targets:
for ipaddr in ipaddress.IPv4Network(targets).hosts():
hosts.append(str(ipaddr))
# multiple single hosts
elif ',' in targets:
hosts = targets.split(',')
# single host
else:
hosts.append(targets)
except Exception as e:
self.log('hostrange',
eargs=repr(e.args[0]),
_type='err',
end='\n')
if hosts:
self.opts['targets']['nmap']['hosts'] = hosts
return
def parse_nmap_logfile(self, logfile, privip=False):
""" parse hosts and ports from nmap xml logfile (-l) """
nmap = NmapParser()
try:
parsed = nmap.parse_fromfile(logfile)
except:
self.log('nmap', eargs=f'{logfile}', _type='err', end='\n')
else:
for host in parsed.hosts:
if len(host.hostnames) >= 1:
_host = host.hostnames[0] # better work with hostname
else:
_host = host.address
try:
if ipaddress.ip_address(_host).is_private:
privip = True
except:
pass # hostname
ports = []
res = host.get_open_ports()
if res:
for port, proto in res:
tmp = str(host.get_service(port)).translate(
str.maketrans('', '', ':[])')).split()
service = self.misc.lookup_port_service(
str(port), proto)
ports.append([str(port), service])
self.opts['targets'][proto].append({
'host': _host,
'ports': ports,
'privip': privip
})
else:
# host up but no ports found
self.opts['targets']['tcp'].append({
'host': _host,
'ports': [],
'privip': privip
})
return
def parse_targets(self, args, privip=False):
""" parse all targets from line specified via URIs (-u) """
targets = []
# remove empty ones, if ';' was used at the end without further targets
if ';' in args:
targets = [x.strip() for x in args.split(';') if x]
elif 'person://' in args or 'company://' in args:
targets.append(args)
else:
targets = args.split()
for t in targets:
if re.search('^tcp://|^udp://', t):
ports = []
part = t.split('://')
proto = part[0].strip(self.bad_chars)
self.opts['targets_opts']['target_type'] = proto
host = part[1].split(':')[0].strip(self.bad_chars)
try:
if ipaddress.ip_address(host).is_private:
privip = True
except:
pass # hostname
if ':' in part[1]:
tmp_ports = part[1].split(':')[1]
tmp_ports = tmp_ports.split(',')
for p in tmp_ports:
if '=' in p:
ports.append(p.split('='))
else:
service = self.misc.lookup_port_service(p)
if service:
p = f'{p}={service}'
else:
# assign default service if not given
p = f'{p}=default'
ports.append(p.split('='))
self.opts['targets'][proto].append({
'host': host,
'ports': ports,
'privip': privip
})
elif re.search('^http://|^https://', t):
self.opts['targets_opts']['target_type'] = t.split(':')[0]
self.opts['targets']['web'] = t.split(',')
# add leading '/' to url if not given
#for u in t.split(','):
# if not u.endswith('/'):
# u = f'{u}/'
# self.opts['targets']['web'].append(u)
elif re.search(
'\
^mail://|^person://|^company://|^domain://|^lan://|^wifi://', t):
self.opts['targets_opts']['target_type'] = t.split(':')[0]
line = list(
filter(None,
[l.replace('://', ':') for l in t.split(';')]))
line = dict(l.split(':') for l in line)
for k in line.keys():
if k == 'mail' or k == 'person' or k == 'company' or k == 'domain':
self.opts['targets']['social'][k] = line[k].split(',')
else:
self.opts['targets'][k] = line[k].split(',')
else:
unknown_mode = t.split('://')[0]
self.log('mode', eargs=unknown_mode, _type='err', end='\n')
return
def parse_cmdline(self):
""" parse command line """
try:
opts, args = getopt.getopt(
self.opts['cmdline'][1:],
't:u:l:o:i:I:x:X:T:M:P:k:rR:c:vdCp:m:a:VH')
except getopt.GetoptError as err:
self.log('default', eargs=repr(err), _type='err', end='\n')
for o, a in opts:
if o == '-t':
if a == '?':
Usage.nmap_mode_usage()
os._exit(SUCCESS)
else:
self.parse_nmap_targets(a)
elif o == '-u':
if a == '?':
Usage.host_mode_usage()
os._exit(SUCCESS)
else:
self.parse_targets(a)
elif o == '-l':
self.parse_nmap_logfile(a)
elif o == '-o':
if a == '?':
Usage.extra_opts_usage()
os._exit(SUCCESS)
else:
self.parse_extra_opts(a)
elif o == '-i':
if a == '?':
Usage.modules_usage()
os._exit(SUCCESS)
else:
self.parse_modules(a, 'in_modules')
elif o == '-I':
if a == '?':
Usage.tools_usage()
os._exit(SUCCESS)
else:
self.parse_tools(a, 'in_tools')
elif o == '-x':
if a == '?':
Usage.modules_usage()
os._exit(SUCCESS)
else:
self.parse_modules(a, 'ex_modules')
elif o == '-X':
if a == '?':
Usage.tools_usage()
os._exit(SUCCESS)
else:
self.parse_tools(a, 'ex_tools')
elif o == '-T':
self.opts['t_workers'] = a
elif o == '-M':
self.opts['m_workers'] = a
elif o == '-P':
self.opts['p_workers'] = a
elif o == '-k':
self.opts['timeout'] = a
elif o == '-r':
self.opts['report'] = True
elif o == '-R':
self.opts['nullscan_dir'] = a + '-' + TODAY
elif o == '-c':
self.opts['config']['file'] = a
elif o == '-v':
self.opts['verbose'] = True
elif o == '-d':
self.opts['debug'] = True
elif o == '-C':
self.opts['check_tools'] = True
elif o == '-p':
if a == '?':
Usage.print_tools_usage()
os._exit(SUCCESS)
else:
self.parse_print_tools(a)
elif o == '-m':
if a == '?':
Usage.add_module_usage()
os._exit(SUCCESS)
else:
self.parse_add_module_tool(a, 'add_module')
elif o == '-a':
if a == '?':
Usage.add_tool_usage()
os._exit(SUCCESS)
else:
self.parse_add_module_tool(a, 'add_tool')
elif o == '-V':
self.log(VERSION, _type='msg', end='\n')
os._exit(SUCCESS)
elif o == '-H':
Usage.usage()
os._exit(SUCCESS)
else:
self.log('cmdopt', eargs=o, _type='err', end='\n')
return