示例#1
0
class TestFile(unittest.TestCase):
    def setUp(self):
        self.file = File(api_url = config.API_URL, token = config.TOKEN)
        self.token = config.TOKEN

    @unittest.skip("skip")
    def test_get_files(self):
        offset = 0
        resp, content = self.file.get_files(offset = offset)
        self.assertEqual(int(resp['status']), 200)

    @unittest.skip("skip")
    def test_download_files(self):
        filenames = "test.txt"
        resp, content = self.file.download_file(filenames = filenames)
        self.assertEqual(int(resp['status']), 200)

    @unittest.skip("skip")
    def test_upload_files(self):
        register_openers()
        datagen, headers = multipart_encode({"files": open("test.txt", "rb")})
        headers["Authorization"] = "bearer:%s" % self.token
        request = urllib2.Request("%s/member/files/upload" % config.API_URL, datagen, headers)
        try:
            response = urllib2.urlopen(request)
            code = response.getcode()
            content = response.read()
        except urllib2.HTTPError, e:
            code = e.code
            content = e.msg
        self.assertEqual(code, 200)
示例#2
0
 def test_find(self):
     path = "/home/test/downloads"
     size_filter = core.algorithm.SizeFilter()
     char_filter = core.algorithm.CharacterFilter()
     filters = [
         size_filter,
         char_filter
     ]
     dup_finder = core.dup_finder.DupFinder([path], filters)
     file_a = File("/home/test/downloads/a")
     file_b = File("/home/test/downloads/b")
     file_insts = [file_a, file_b]
     when(core.dup_finder.Walker).walk([path]).thenReturn(file_insts)
     when(size_filter).find().thenReturn(None)
     size_filter.filtered_files = file_insts
     when(size_filter).set_files().thenReturn(None)
     when(char_filter).set_files().thenReturn(None)
     when(char_filter).find().thenReturn(None)
     char_filter.filtered_files = []
     dup_finder.find()
     verify(core.dup_finder.Walker).walk([path])
     verify(size_filter).set_files(file_insts)
     verify(size_filter).find()
     verify(char_filter).set_files(file_insts)
     verify(char_filter).find()
     self.assertEqual([], dup_finder.dup_files)
示例#3
0
    def put(self, path):
        try:
            context = g.context
            connection = context.get_connection()
            result = []
            for f in request.files.getlist('file'):
                file = File()
                file.update_file(context, f, path)
                result.append({
                    "status": "OK",
                    "file_id": file.get_file_id(),
                    "remote_path": path
                })

            return make_response(dict({
                "results": result,
                "status": "ok"
            }), 200)

        except FileNotFoundInDatabase as err:
            logger.exception(f"FileNotFoundInDatabase: {err}")
            return make_response({"status": "Err", "error": str(err)}, 404)
        except Exception as err:
            logger.exception(f"Exception: {err}")
            return make_response({"status": "Err", "error": str(err)}, 500)
示例#4
0
    def get(self, path):
        try:
            context = g.context
            connection = context.get_connection()

            file = File()
            result = file.read_file(context, path)

            if result == None:
                return make_response(
                    {
                        "status": "Err",
                        "error": f"File not found: {path}"
                    }, 404)

            response = make_response(result['file'])
            response.headers.set('Content-Type', result['mime_type'])
            #response.headers.set('Content-Disposition', 'attachment', filename='test.jpg')
            response.headers.set('Content-Disposition',
                                 'inline',
                                 filename=result['name'])
            return response

        except Exception as err:
            logger.exception(f"Exception: {err}")
            return make_response({"status": "Err", "error": str(err)}, 500)
示例#5
0
文件: parser.py 项目: shamb0/nullscan
    def __init__(self, opts):
        """ constructor"""

        self.opts = opts  # all nullscan options
        self.logger = Logger()
        self.log = self.logger.log
        self.file = File()
        self.misc = Misc()

        # tmp stupid fix to strip bad chars out of options
        self.bad_chars = '<>()[]{}:;.,="\'*&^%$#@!+-_/?°²³´`'

        return
示例#6
0
文件: misc.py 项目: shamb0/nullscan
  def __init__(self):
    """ constructor """

    # logger
    self.logger = Logger()
    self.log = self.logger.log

    # file i/o
    self.file = File()

    # original terminal state
    self.term_fd = sys.stdin.fileno()
    self.term_state = termios.tcgetattr(self.term_fd)

    return
示例#7
0
 def __path_walk(self, path):
     file_instances = list()
     for (root, dirs, files) in os.walk(path):
         #            LOG.debug("{0} {1} {2}".format(root, dirs, files))
         for _file in files:
             filepath = os.path.join(root, _file)
             if os.path.exists(filepath):
                 file_instance = File(filepath)
                 file_instances.append(file_instance)
                 self.progress = self.progress + 1
     return file_instances
示例#8
0
文件: html.py 项目: shamb0/nullscan
    def __init__(self, date, opts, template_dir, report_dir, logs_dir):
        """ constructor """

        self.date = date
        self.opts = opts
        self.res = {}  # holds the results out of log dirs

        self.template_dir = template_dir
        self.report_dir = report_dir
        self.logs_dir = logs_dir

        self.file = File()

        self.file.copy_dirs(self.template_dir, self.report_dir)
        self.index_html = f'{self.report_dir}/index.html'
        self.res_html = f'{self.report_dir}/results.html'
        self.index_html_data = self.file.read_file(self.index_html)
        self.res_html_data = self.file.read_file(self.res_html)

        return
示例#9
0
    def __init__(self):
        """ constructor """

        # options and modules
        self.opt = None
        self.mod = Module(MOD_PATH)

        # logger
        self.logger = Logger()
        self.log = self.logger.log

        # rest we need
        self.file = File()
        self.check = Check()
        self.misc = Misc()
        self.parser = None

        # nullscan working dir
        self.nullscan_dir = None

        return
示例#10
0
  def __init__(self, mod_path):
    """ constructor """

    # logger
    self.logger = Logger()
    self.log = self.logger.log

    # file i/o
    self.file = File()

    # get all modules
    self.mod_path = mod_path
    self.mods = []
    self.get_modules()

    # all loaded module
    self.lmod = {}

    # docstrings for all tools
    self.docstrings = {}

    return
示例#11
0
def main():
    path = sys.argv[1]
    LOG.info("Start to find duplicated files on {0}".format(path))

    if os.path.isfile(path):
        start_time = time.time()
        print (File(path).md5sum)
        print (File(path).size)
        end_time = time.time()
        print (end_time - start_time)
    else:
        start_time = time.time()
        filters = [
            core.algorithm.SizeFilter(),
            core.algorithm.CharacterFilter()
        ]
        dup_finder = core.dup_finder.DupFinder([path], filters)
        dup_finder.find()
        end_time = time.time()
        #dup_finder.dump2file("output.txt")
        dup_finder.dump2csv("output.csv")
        print (end_time - start_time)
        print (utils.size_renderer(dup_finder.dup_size))
示例#12
0
文件: base.py 项目: shamb0/nullscan
    def __init__(self, target, opts):
        """ init """

        Helper.__init__(self, target, opts)
        ToolsHelper.__init__(self)
        Tools.__init__(self, target, opts)
        Parser.__init__(self)

        self.logger = Logger()
        self.log = self.logger.log
        self.file = File()
        self.logfile = None  # logfile for command

        return
示例#13
0
    def getresults(self,
                   filepath,
                   frame,
                   resolution=None,
                   surface=False,
                   center=False):
        """
        Get cached results for the given parameters.

        **Parameters:**
            `filepath` :
                absolute path of the input file
            `frame` :
                the frame number
            `resolution` :
                resolution of the used discretization
            `surface` :
                query calculation results for surface-based cavities
            `center` :
                query calculation results for center-based cavities

        **Returns:**
            A :class:`core.data.Results` object if cached results exist,
            else `None`
        """
        inputfile = File.open(filepath)
        # TODO: error handling
        resultfile = None
        results = None
        if isinstance(inputfile, core.file.ResultFile):
            resultfile = inputfile
        elif filepath in self.cache:
            resultfile = self.cache[filepath]
        if resultfile is not None:
            if resolution is None:
                resolutions = sorted(resultfile.info.resolutions())[::-1]
                resolution = 64
                for res in resolutions:
                    if resultfile.info[res].domains[frame] is not None:
                        resolution = res
                        break
            results = resultfile.getresults(frame, resolution)
        if results is None:
            if resolution is None:
                resolution = 64
            results = data.Results(filepath, frame, resolution,
                                   inputfile.getatoms(frame), None, None, None)
        return results
示例#14
0
    def add_file(self, path):
        bname = os.path.basename(path)
        f = file.File.open(path)
        n_frames = f.info.num_frames

        if bname not in self.path_dict.keys() and File.exists(path):
            self.path_dict[bname] = str(path)
            root = bname
            sib = ['frame {}'.format(i) for i in range(1, n_frames + 1)]
            self.append_item(root, sib)

            if path not in config.recent_files:
                config.add_recent_file(path)
            else:
                index = config.recent_files.index(path)
                config.recent_files.pop(index)
                config.add_recent_file(path)

        self.files_changed()
示例#15
0
    def getresults(self, filepath, frame, resolution=None, surface=False, center=False):
        """
        Get cached results for the given parameters.

        **Parameters:**
            `filepath` :
                absolute path of the input file
            `frame` :
                the frame number
            `resolution` :
                resolution of the used discretization
            `surface` :
                query calculation results for surface-based cavities
            `center` :
                query calculation results for center-based cavities

        **Returns:**
            A :class:`core.data.Results` object if cached results exist,
            else `None`
        """
        inputfile = File.open(filepath)
        # TODO: error handling
        resultfile = None
        results = None
        if isinstance(inputfile, core.file.ResultFile):
            resultfile = inputfile
        elif filepath in self.cache:
            resultfile = self.cache[filepath]
        if resultfile is not None:
            if resolution is None:
                resolutions = sorted(resultfile.info.resolutions())[::-1]
                resolution = 64
                for res in resolutions:
                    if resultfile.info[res].domains[frame] is not None:
                        resolution = res
                        break
            results = resultfile.getresults(frame, resolution)
        if results is None:
            if resolution is None:
                resolution = 64
            results = data.Results(filepath, frame, resolution, inputfile.getatoms(frame), None, None, None)
        return results
示例#16
0
    def calculatedframes(self,
                         filepath,
                         resolution,
                         surface=False,
                         center=False):
        """
        Query the cache if it contains results for the given parameters.

        **Parameters:**
            `filepath` :
                absolute path of the input file
            `resolution` :
                resolution of the used discretization
            `surface` :
                query calculation results for surface-based cavities
            `center` :
                query calculation results for center-based cavities

        **Returns:**
            A :class:`core.data.TimestampList` containing the dates
            of the calculation or `None`.
        """
        info = None
        inputfile = File.open(filepath)
        # TODO: error handling
        if isinstance(inputfile, core.file.ResultFile):
            info = inputfile.info
        else:
            if filepath in self.cache:
                cf = self.cache[filepath]
                info = cf.info
        if info is not None:
            if surface:
                return info[resolution].surface_cavities
            elif center:
                return info[resolution].center_cavities
            else:
                return info[resolution].domains
        else:
            return data.TimestampList(inputfile.info.num_frames)
示例#17
0
    def calculatedframes(self, filepath, resolution, surface=False, center=False):
        """
        Query the cache if it contains results for the given parameters.

        **Parameters:**
            `filepath` :
                absolute path of the input file
            `resolution` :
                resolution of the used discretization
            `surface` :
                query calculation results for surface-based cavities
            `center` :
                query calculation results for center-based cavities

        **Returns:**
            A :class:`core.data.TimestampList` containing the dates
            of the calculation or `None`.
        """
        info = None
        inputfile = File.open(filepath)
        # TODO: error handling
        if isinstance(inputfile, core.file.ResultFile):
            info = inputfile.info
        else:
            if filepath in self.cache:
                cf = self.cache[filepath]
                info = cf.info
        if info is not None:
            if surface:
                return info[resolution].surface_cavities
            elif center:
                return info[resolution].center_cavities
            else:
                return info[resolution].domains
        else:
            return data.TimestampList(inputfile.info.num_frames)
示例#18
0
    def calculateframe(self,
                       filepath,
                       frame,
                       resolution,
                       cutoff_radii=None,
                       domains=False,
                       surface=False,
                       center=False,
                       atoms=None,
                       gyration_tensor_parameters=False,
                       recalculate=False,
                       last_frame=True):
        """
        Get results for the given parameters. They are either loaded from the
        cache or calculated.

        **Parameters:**
            `filepath` :
                absolute path of the input file
            `frame` :
                the frame number
            `resolution` :
                resolution of the used discretization
            `domains` :
                calculate cavitiy domains
            `cutoff_radii` :
                dict that maps element symbols to cutoff radii
            `surface` :
                calculate surface-based cavities
            `center` :
                calculate center-based cavities
            `gyration_tensor_parameters` :
                gyration tensor parameters will be calculated for cavities (they are always calculated for
                cavity domains)
            `recalculate` :
                results will be calculated even if cached results exists

        **Returns:**
            A :class:`core.data.Results` object.
        """
        # always recalculate if gyration tensor parameters shall be computed for center or surface based cavities
        recalculate = recalculate or (gyration_tensor_parameters and
                                      (center or surface))
        message.progress(0)
        inputfile = File.open(filepath)
        # TODO: error handling
        if isinstance(inputfile, core.file.ResultFile):
            resultfile = inputfile
        else:
            resultfile = self.cache[filepath]
        try:
            results = resultfile.getresults(frame, resolution)
        except Exception as e:
            logger.debug("error in resultfile.getresults: {}".format(e))
            results = None

        if atoms is None:
            atoms = inputfile.getatoms(frame)
        atoms.radii = cutoff_radii
        volume = atoms.volume
        if results is None:
            results = data.Results(filepath, frame, resolution, atoms, None,
                                   None, None)

        if recalculate:
            results.domains = None
            results.surface_cavities = None
            results.center_cavities = None

        if not ((domains and results.domains is None) or
                (surface and results.surface_cavities is None) or
                (center and results.center_cavities is None)):
            message.print_message("Reusing results")
        else:
            cachepath = os.path.join(self.cachedir,
                                     'discretization_cache.hdf5')
            discretization_cache = DiscretizationCache(cachepath)
            with DiscretizationCache(cachepath) as discretization_cache:
                discretization = discretization_cache.get_discretization(
                    volume, resolution)
            atom_discretization = AtomDiscretization(atoms, discretization)
            message.progress(10)
            if (domains and results.domains is None) \
                    or (surface and results.surface_cavities is None):
                # CavityCalculation depends on DomainCalculation
                message.print_message("Calculating domains")
                domain_calculation = DomainCalculation(discretization,
                                                       atom_discretization)
                if domain_calculation.critical_domains:
                    logger.warn(
                        'Found {:d} critical domains in file {}, frame {:d}. Domain indices: {}'
                        .format(len(domain_calculation.critical_domains),
                                os.path.basename(filepath), frame,
                                domain_calculation.critical_domains))
                    message.log(
                        'Found {:d} critical domains in file {}, frame {:d}'.
                        format(
                            len(domain_calculation.critical_domains),
                            os.path.basename(filepath),
                            frame + 1,
                        ))
            if results.domains is None:
                results.domains = data.Domains(domain_calculation)
            message.progress(40)

            if surface and results.surface_cavities is None:
                message.print_message("Calculating surface-based cavities")
                cavity_calculation = CavityCalculation(
                    domain_calculation,
                    use_surface_points=True,
                    gyration_tensor_parameters=gyration_tensor_parameters)
                results.surface_cavities = data.Cavities(cavity_calculation)
            message.progress(70)

            if center and results.center_cavities is None:
                message.print_message("Calculating center-based cavities")
                domain_calculation = FakeDomainCalculation(
                    discretization, atom_discretization, results)
                cavity_calculation = CavityCalculation(
                    domain_calculation,
                    use_surface_points=False,
                    gyration_tensor_parameters=gyration_tensor_parameters)
                results.center_cavities = data.Cavities(cavity_calculation)
            resultfile.addresults(results, overwrite=recalculate)

        message.progress(100)
        message.print_message("Calculation finished")
        if last_frame:
            message.finish()
        return results
示例#19
0
文件: html.py 项目: shamb0/nullscan
class HTML:
    """ class for HTML reporting """
    def __init__(self, date, opts, template_dir, report_dir, logs_dir):
        """ constructor """

        self.date = date
        self.opts = opts
        self.res = {}  # holds the results out of log dirs

        self.template_dir = template_dir
        self.report_dir = report_dir
        self.logs_dir = logs_dir

        self.file = File()

        self.file.copy_dirs(self.template_dir, self.report_dir)
        self.index_html = f'{self.report_dir}/index.html'
        self.res_html = f'{self.report_dir}/results.html'
        self.index_html_data = self.file.read_file(self.index_html)
        self.res_html_data = self.file.read_file(self.res_html)

        return

    def make_results(self):
        """ get all results from the targets log directory """

        logs = f'{self.logs_dir}/targets/'

        targets = os.listdir(logs)
        for target in targets:
            target_orig = target
            if ':' in target:
                target = target.replace(':', '-')
            self.res[target] = {}
            for moddir in os.listdir(logs + target_orig):
                if moddir in ('tcp', 'udp', 'social'):
                    submods = os.listdir(f'{logs}{target_orig}/{moddir}')
                    for s in submods:
                        for modname in os.listdir(
                                f'{logs}{target_orig}/{moddir}/{s}'):
                            if moddir == 'tcp' or moddir == 'udp':
                                module = f'{moddir}.{s}.{modname}'
                            else:
                                module = f'{moddir}.{modname}'
                            self.res[target][module] = {}
                            tool_path = f'{logs}{target_orig}/{moddir}/{s}/{modname}/'
                            for tool in os.listdir(tool_path):
                                if '.log' in tool:
                                    self.res[target][module][tool] = \
                                      '\n'.join(self.file.read_file(tool_path + tool))
                else:
                    for modname in os.listdir(f'{logs}{target_orig}/{moddir}'):
                        target = target.replace(':', '-')  # url port
                        module = f'{moddir}.{modname}'
                        self.res[target][module] = {}
                        for tool in os.listdir(
                                f'{logs}{target_orig}/{moddir}/{modname}'):
                            if '.log' in tool and '.bin' not in tool:
                                tool_path = f'{logs}{target_orig}/{moddir}/{modname}/'
                                self.res[target][module][tool] = '\n'.join(
                                    self.file.read_file(tool_path + tool))

        return

    def make_target_html(self):
        """ create $target.html with its contents/results """

        _mods = ''
        panels = []
        panel = '''
    <div class="panel panel-default">
      <div class="panel-heading">
        <h3 class="panel-title">$TOOL</h3>
      </div>
      <div class="panel-body">
        <pre>$RESULT</pre>
      </div>
    </div>
    '''

        for target, mods in self.res.items():
            t_html = f'{self.report_dir}/{target}.html'
            self.file.copy_files(self.res_html, t_html)
            t_html_data = self.file.read_file(t_html)
            t_html_data = [w.replace('$TARGET', target) for w in t_html_data]
            self.file.write_file(t_html, ' '.join(t_html_data))
            for mod, tools in mods.items():
                _mods += f'<h5>{mod}</h5>'
                for tool, data in tools.items():
                    tlink = tool.split('.log')[0]
                    link = f'{mod}.{tlink}'
                    _mods += f'<a href="#{link}" target="_blank">{tlink}</a><br />'
                    p1 = panel.replace(f'$TOOL',
                                       f'<a name="{link}">{link}</a>')
                    p2 = p1.replace('$RESULT', html.escape(data))
                    panels.append(p2)
            t_html_data = [w.replace('$ALLMODS', _mods) for w in t_html_data]
            _mods = ''
            t_html_data = [w.replace('$CONTENT', ' '.join(panels)) \
              for w in t_html_data]
            panels = []

            self.file.write_file(t_html, ' '.join(t_html_data))

        return

    def make_index_html(self):
        """ create the index.html """

        num_social_targets = 0
        patterns = {
            '$DATE': self.date,
            '$CMDLINE_ARGS': ' '.join(self.opts['cmdline'])
        }

        # index.html: date + cmdline args
        for k, v in patterns.items():
            self.index_html_data = [
                w.replace(k, v) for w in self.index_html_data
            ]

        # index.html: num target modes
        for k, v in self.opts['targets'].items():
            if k == 'social':
                for s in self.opts['targets'][k]:
                    if self.opts['targets'][k][s]:
                        num_social_targets += len(self.opts['targets'][k][s])
                self.index_html_data = [
                    w.replace('$NUM_SOCIAL', str(num_social_targets))
                    for w in self.index_html_data
                ]
            self.index_html_data = [w.replace(f'$NUM_{k.upper()}', str(len(v))) \
              for w in self.index_html_data]

        return

    def make_menu(self, mod, targets):
        """ create the menu and links """

        # targets menu items
        if mod == 'tcp' or mod == 'udp':
            l = []
            for target in self.opts['targets'][mod]:
                target = f"<li><a href='{target['host']}.html'>{target['host']}</a></li>"
                l.append(target)
            self.index_html_data = [
              w.replace(f'$TARGETS_{mod.upper()}', ' '.join(l)) \
              for w in self.index_html_data
            ]
            self.res_html_data = [
              w.replace(f'$TARGETS_{mod.upper()}', ' '.join(l)) \
              for w in self.res_html_data
            ]
        elif mod == 'social':
            l = []
            for part in self.opts['targets'][mod]:
                for target in self.opts['targets'][mod][part]:
                    target = f'<li><a href="{target}.html">{target}</a></li>'
                    l.append(target)
            self.index_html_data = [
              w.replace(f'$TARGETS_{mod.upper()}', ' '.join(l)) \
              for w in self.index_html_data
            ]
            self.res_html_data = [
              w.replace(f'$TARGETS_{mod.upper()}', ' '.join(l)) \
              for w in self.res_html_data
            ]
        else:
            for target in targets:
                if '://' in target:
                    target = target.split('://')[1].rstrip('/').split('/')[0]
                if ':' in target:
                    target = target.replace(':', '-')  # port
                target = f'<li><a href="{target}.html">{target}</a></li>'
                self.index_html_data = [
                  w.replace(f'$TARGETS_{mod.upper()}', target) \
                  for w in self.index_html_data
                ]
                self.res_html_data = [
                  w.replace(f'$TARGETS_{mod.upper()}', target) \
                  for w in self.res_html_data
                ]

        return

    def make_report(self):
        """ make the report """

        self.make_results()

        for mod, targets in self.opts['targets'].items():
            self.make_menu(mod, targets)
        self.make_index_html()

        self.file.write_file(self.index_html, ' '.join(self.index_html_data))
        self.file.write_file(self.res_html, ' '.join(self.res_html_data))

        self.make_target_html()
        self.file.del_file(f'{self.report_dir}/results.html')

        return
示例#20
0
    def calculate(self, calcsettings):
        """
        Calculate (or load from the cache) all results for the given settings.

        **Parameters:**
            `calcsettings` :
                :class:`CalculationSettings` object

        **Returns:**
            A list of list of :class:`core.data.Results` objects. The outer list contains
            an entry for each entry in `calcsettings.filenames`; the inner
            list has a `Results` entry for each frame specified in
            `calcsettings.frames`.
        """
        allresults = []
        for filename, frames in calcsettings.datasets.iteritems():
            filepath = core.file.get_abspath(filename)
            fileprefix = os.path.basename(filename).rsplit(".", 1)[0]
            if calcsettings.exportdir is not None:
                exportdir = core.file.get_abspath(calcsettings.exportdir)
                # replace asterisks with directories
                dirlist = os.path.dirname(filepath).split("/")
                while "*" in exportdir:
                    i = exportdir.rindex("*")
                    exportdir = os.path.join(exportdir[:i],
                                             dirlist.pop() + exportdir[i + 1:])
                if (calcsettings.exporthdf5 or calcsettings.exporttext) \
                        and not os.path.exists(exportdir):
                    os.makedirs(exportdir)
            else:
                exportdir = os.path.dirname(filepath)
            if calcsettings.exporthdf5:
                efpath = os.path.join(exportdir, fileprefix + ".hdf5")
                efpath = core.file.get_abspath(efpath)
                # copy atoms into HDF5 file
                exportfile = core.file.HDF5File.fromInputFile(efpath, filepath)
                # use HDF5 file as input
                filepath = efpath

            fileresults = []
            if frames[0] == -1:
                inputfile = File.open(filepath)
                frames = range(inputfile.info.num_frames)
            last_frame = False
            for frame in frames:
                # calculate single frame
                if frame is frames[-1]:
                    last_frame = True
                frameresult = self.calculateframe(
                    filepath,
                    frame,
                    calcsettings.resolution,
                    calcsettings.cutoff_radii,
                    domains=calcsettings.domains,
                    surface=calcsettings.surface_cavities,
                    center=calcsettings.center_cavities,
                    gyration_tensor_parameters=calcsettings.gyration_tensor,
                    recalculate=calcsettings.recalculate,
                    last_frame=last_frame)
                # export to text file
                if calcsettings.exporttext:
                    fmt = os.path.join(
                        exportdir, fileprefix) + "-{property}-{frame:06d}.txt"
                    if frameresult.atoms is not None:
                        frameresult.atoms.totxt(
                            fmt.format(property='{property}', frame=frame + 1))
                    if frameresult.domains is not None:
                        try:
                            frameresult.domains.totxt(
                                fmt.format(property='domain_{property}',
                                           frame=frame + 1))
                        except ValueError as e:
                            logger.warn(e.message)
                            logger.warn(
                                'The export of domain information could not be finished.'
                            )
                    if frameresult.surface_cavities is not None:
                        frameresult.surface_cavities.totxt(
                            fmt.format(property='surface_cavities_{property}',
                                       frame=frame + 1))
                    if frameresult.center_cavities is not None:
                        frameresult.center_cavities.totxt(
                            fmt.format(property='center_cavities_{property}',
                                       frame=frame + 1))
                    # TODO: try/except
                # gather results
                fileresults.append(frameresult)
            allresults.append(fileresults)
        return allresults
示例#21
0
class Controller:
    """ program controller class """
    def __init__(self):
        """ constructor """

        # options and modules
        self.opt = None
        self.mod = Module(MOD_PATH)

        # logger
        self.logger = Logger()
        self.log = self.logger.log

        # rest we need
        self.file = File()
        self.check = Check()
        self.misc = Misc()
        self.parser = None

        # nullscan working dir
        self.nullscan_dir = None

        return

    def prepare(self):
        """ preparation/initialization of opts and env: parsing & checks """

        # declare nullscan options
        self.opt = Option(sys.argv)

        # check argc and argc (usage)
        self.check.check_argc(len(sys.argv))
        self.check.check_argv(sys.argv)

        # check for missing libraries / deps / python modules
        self.check.check_deps(self.file.read_file(PYDEPS))

        # parse cmdline and config options, update final options dictionary
        try:
            self.parser = Parser(self.opt.opts)
            self.parser.parse_cmdline()
            self.parser.parse_config()
            self.opt.opts = self.parser.opts
        except:
            self.log('usage', _type='err', end='\n')

        # update final options dictionary
        self.opt.update_opts()

        # further checks for usage, options, env, etc.
        self.check.check_opts(self.opt.opts)

        # collect all py-files and grep the tools out of the py-files
        tools = []
        py_files = self.misc.find_py_files(MOD_PATH)
        for py in py_files:
            tools.append(self.misc.grep_tools(py))
        tools = [x for sublist in tools for x in sublist]

        # create the locks for each tool except for excluded ones
        with ThreadPoolExecutor(50) as exe:
            for tool in tools:
                if tool not in self.opt.opts['tools']['ex_tools']:
                    exe.submit(self.file.create_lock, tool)

        # copy debug flag to target_opts (for nullscan tools)
        self.opt.opts['targets_opts']['debug'] = self.opt.opts['debug']

        return

    def run_misc(self):
        """ run chosen misc options """

        if self.opt.opts['check_tools']:
            self.log('Checking for missing tools\n\n', _type='msg')
            self.check.check_tools()
            os._exit(SUCCESS)
        if self.opt.opts['print_tools']:
            self.log('Printing tools and information\n\n', _type='msg')
            self.misc.print_tool(self.opt.opts['print_tools'])
            os._exit(SUCCESS)
        if self.opt.opts['add_module']:
            self.log('Adding new module\n', _type='msg')
            self.misc.add_mod_tool('mod', self.opt.opts['add_module'])
            os._exit(SUCCESS)
        if self.opt.opts['add_tool']:
            self.log('Adding new tool\n', _type='msg')
            self.misc.add_mod_tool('tool', self.opt.opts['add_tool'])
            os._exit(SUCCESS)

        return

    def prepare_modules(self):
        """ filter in-/excluded mods + return a unique list """

        self.mod.filter_modules(self.opt.opts)
        self.mod.mods = list(set(self.mod.mods))

        return

    def run_nmap_mode(self):
        """ run nmap scan mode """

        nmap = Nmap(self.opt.opts['targets']['nmap'])

        # create nmap logdir based on scan type requested
        logpath = f'{self.nullscan_dir}/logs/nmap/{nmap.get_protocol()}'
        logfile = f'{logpath}/results'
        self.file.make_dir(logpath)
        nmap.set_logfile(logfile)

        # build nmap command line
        nmap.build_cmd()

        # start scans
        self.log('NMAP mode activated\n', _type='msg', color='blue')
        self.log('Targets added: {}\n'.format(
            len(self.opt.opts['targets']['nmap']['hosts'])),
                 _type='msg')
        if self.opt.opts['verbose']:
            self.log('\n')
            for target in self.opt.opts['targets']['nmap']['hosts']:
                self.log(f'{target}\n', _type='msg')
        nmap.scan(debug=self.opt.opts['debug'])

        return f'{logfile}.xml'

    def run_social_mode(self, target):
        """ run social mode """

        if '.social.' in str(self.mod.mods):
            # availabel social modules to import and load
            mods = [i for i in self.mod.mods if '.social.' in i]

            with ProcessPoolExecutor(self.opt.opts['m_workers']) as exe:
                for key in target.keys():
                    if target[key]:
                        for t in target[key]:
                            # default module
                            rdir = f'{self.nullscan_dir}/logs/targets/'
                            wdir = f"{rdir}{t}/social/{key}/default"
                            exe.submit(self.mod.run_module,
                                       'modules.social.default', t,
                                       self.opt.opts, wdir)

                            # non-default modules
                            for m in mods:
                                if 'default' not in m:
                                    splitted = m.split('.')
                                    moddir = splitted[1]
                                    modname = splitted[2]
                                    wdir = f"{rdir}{t}/{moddir}/{key}/{modname}"
                                    exe.submit(self.mod.run_module, m, t,
                                               self.opt.opts, wdir)

        return

    def run_wifi_mode(self, target):
        """ run wifi mode """

        if '.wifi.' in str(self.mod.mods):
            # available wifi modules to import and load
            mods = [i for i in self.mod.mods if '.wifi.' in i]

            # default module first
            rdir = f'{self.nullscan_dir}/logs/targets/'
            wdir = f'{rdir}{target}/wifi/default'
            self.mod.run_module('modules.wifi.default', target, self.opt.opts,
                                wdir)

            # non-default modules
            with ProcessPoolExecutor(self.opt.opts['m_workers']) as exe:
                for m in mods:
                    if 'default' not in m:
                        splitted = m.split('.')
                        moddir = splitted[1]
                        modname = splitted[2]
                        wdir = f'{rdir}{target}/{moddir}/{modname}'
                        exe.submit(self.mod.run_module, m, target,
                                   self.opt.opts, wdir)

        return

    def run_lan_mode(self, target):
        """ run lan mode """

        if '.lan.' in str(self.mod.mods):
            # available lan modules to import and load
            mods = [i for i in self.mod.mods if '.lan.' in i]

            # default module first
            rdir = f'{self.nullscan_dir}/logs/targets/'
            wdir = f'{rdir}{target}/lan/default'
            self.mod.run_module('modules.lan.default', target, self.opt.opts,
                                wdir)

            # non-default modules
            with ProcessPoolExecutor(self.opt.opts['m_workers']) as exe:
                for m in mods:
                    if 'default' not in m:
                        splitted = m.split('.')
                        moddir = splitted[1]
                        modname = splitted[2]
                        wdir = f'{rdir}{target}/{moddir}/{modname}'
                        exe.submit(self.mod.run_module, m, target,
                                   self.opt.opts, wdir)

        return

    def run_web_mode(self, target):
        """ run web mode """

        if '.web.' in str(self.mod.mods):
            # available web modules to import and load
            mods = [i for i in self.mod.mods if '.web.' in i]

            # we need host name for working directory
            host = requests.utils.urlparse(target).netloc

            # default module first
            rdir = f'{self.nullscan_dir}/logs/targets/'
            wdir = f'{rdir}{host}/web/default'
            self.mod.run_module('modules.web.default', target, self.opt.opts,
                                wdir)

            # non-default modules
            with ProcessPoolExecutor(self.opt.opts['m_workers']) as exe:
                for m in mods:
                    if 'default' not in m:
                        splitted = m.split('.')
                        moddir = splitted[1]
                        modname = splitted[2]
                        wdir = f'{rdir}{host}/{moddir}/{modname}'
                        exe.submit(self.mod.run_module, m, target,
                                   self.opt.opts, wdir)

        return

    def run_udp_mode(self, target):
        """ run udp mode """

        # we need to run host modules before tcp and we need to run default
        # tool first
        self.run_host_mode(target)

        # now tcp modules
        if '.udp.' in str(self.mod.mods):
            with ProcessPoolExecutor(self.opt.opts['m_workers']) as exe:
                for p in target['ports']:
                    exe.submit(self.run_tcp_udp_mode, target, p, 'udp')

        return

    def run_tcp_mode(self, target):
        """ run tcp mode """

        # we need to run host modules before tcp and we need to run default
        # module first
        self.run_host_mode(target)

        # now tcp modules
        if '.tcp.' in str(self.mod.mods):
            with ProcessPoolExecutor(self.opt.opts['m_workers']) as exe:
                for p in target['ports']:
                    exe.submit(self.run_tcp_udp_mode, target, p, 'tcp')

        return

    def run_tcp_udp_mode(self, host, port, proto):
        """ wrapper for tcp/udp mode """

        # available modules
        mod = f'modules.{proto}.{port[1]}'

        # force default module for given port if module does not exist
        if mod not in self.mod.mods:
            mod = f'modules.{proto}.default'

        # new target dict as we only need the corresponding port
        t = {'host': host['host'], 'port': port[0]}

        # default module first
        rdir = f'{self.nullscan_dir}/logs/targets/'
        wdir = f"{rdir}{t['host']}/{proto}/{port[0]}/default"
        self.mod.run_module(f'modules.{proto}.default', t, self.opt.opts, wdir)

        # now non-default module
        if '.default' not in mod:
            wdir = f"{rdir}{t['host']}/{proto}/{port[0]}/{port[1]}"
            self.mod.run_module(mod, t, self.opt.opts, wdir)

        return

    def run_host_mode(self, target):
        """ run host mode """

        # available host modules to import and load
        mods = [i for i in self.mod.mods if '.host.' in i]

        # default module
        rdir = f'{self.nullscan_dir}/logs/targets/'
        wdir = f"{rdir}{target['host']}/host/default"
        self.mod.run_module('modules.host.default', target, self.opt.opts,
                            wdir)

        # non-default modules
        with ProcessPoolExecutor(self.opt.opts['m_workers']) as exe:
            for m in mods:
                if 'default' not in m:
                    splitted = m.split('.')
                    moddir = splitted[1]
                    modname = splitted[2]
                    wdir = f"{rdir}{target['host']}/{moddir}/{modname}"
                    exe.submit(self.mod.run_module, m, target, self.opt.opts,
                               wdir)

        return

    def run_modes(self):
        """ run chosen modes """

        scans = []
        modes = {
            'tcp': self.run_tcp_mode,
            'udp': self.run_udp_mode,
            'wifi': self.run_wifi_mode,
            'web': self.run_web_mode,
            'social': self.run_social_mode,
            'http': self.run_web_mode,
            'https': self.run_web_mode,
        }

        # run lan mode first if requested
        if self.opt.opts['targets']['lan']:
            ifaces = []
            self.log('LAN mode activated\n', color='blue', _type='msg')
            self.log(
                f"Targets added: {len(self.opt.opts['targets']['lan'])}\n\n",
                _type='msg')
            for iface in self.opt.opts['targets']['lan']:
                if self.opt.opts['verbose']:
                    self.log(f'{iface}\n', _type='vmsg')
                ifaces.append((self.run_lan_mode, iface))
            if self.opt.opts['verbose']:
                self.log('\n')
            with ProcessPoolExecutor(self.opt.opts['t_workers']) as exe:
                self.log('Shooting tools\n\n', color='green', _type='msg')
                for iface in ifaces:
                    exe.submit(iface[0], iface[1])
            self.log('\n')
            if not self.opt.opts['verbose']:
                self.log('\n')
            self.log('Tools done\n\n', color='green', _type='msg')
            for log in glob.glob('**/lan/portscan/*.xml', recursive=True):
                if log:
                    self.parser.parse_nmap_logfile(log, lan=True)

        # collect scan modes for each target
        for k, v in self.opt.opts['targets'].items():
            if self.opt.opts['targets'][k]:
                if k == 'lan':
                    continue
                else:
                    self.log(f'{k.upper()} mode activated\n',
                             color='blue',
                             _type='msg')
                self.log(f'Targets added: {len(v)}\n\n', _type='msg')
                if k == 'social':
                    if self.opt.opts['verbose']:
                        for targets in v.values():
                            for target in targets:
                                self.log(f'{target}\n', _type='vmsg')
                        self.log('\n')
                    scans.append((modes[k], v))
                else:
                    for target in v:
                        scans.append((modes[k], target))
                        if self.opt.opts['verbose']:
                            if 'host' in target:
                                target_head = target['host']
                            else:
                                target_head = target
                            self.log(f'{target_head}\n', _type='vmsg')
                    if self.opt.opts['verbose']:
                        self.log('\n')

        # start mode for each target
        with ProcessPoolExecutor(self.opt.opts['t_workers']) as exe:
            if scans:
                self.log('Shooting tools\n\n', color='green', _type='msg')
                for scan in scans:
                    exe.submit(scan[0], scan[1])
        if scans:
            self.log('\n')
            if not self.opt.opts['verbose']:
                self.log('\n')
            self.log('Tools done\n\n', _type='msg', color='green')

        return

    def start(self):
        """ nullscan starts here with actions """

        self.check.check_uid()
        self.log('Game Started\n\n', _type='msg')

        # create nullscan working, targets and log dir
        self.nullscan_dir = self.file.make_dir(self.opt.opts['nullscan_dir'],
                                               incr=True)
        self.file.make_dir(f'{self.nullscan_dir}/logs/targets')
        self.opt.opts['targets_opts']['nullscan_logdir'] = \
          f'{self.nullscan_dir}/logs/targets/'

        # run nmap mode first if requested
        if 'hosts' in self.opt.opts['targets']['nmap']:
            self.parser.parse_nmap_logfile(self.run_nmap_mode())
            self.log('\n')

        # delete nmap key
        del self.opt.opts['targets']['nmap']

        # prepare modules for other modes
        self.prepare_modules()

        # run the nullscan modes now
        self.run_modes()

        return

    def end(self):
        """ program ends here. clean-ups, reporting, etc. """

        # go back to root dir
        os.chdir(ROOT_PATH)

        # clean up empty directories and empty (log-)files or logfiles containing
        # a singl ebyte (newline) (failed tools)
        self.misc.remove_empty_files_dirs(f'{self.nullscan_dir}/logs/targets/')

        # just to be safe - delete all locks, in case tools didn't
        self.file.del_file('/tmp/nullscan', _dir=True)

        # create report
        if self.opt.opts['report']:
            self.log('Creating report\n', _type='msg')
            tmpl_dir = f'{ROOT_PATH}/src/report/template'
            rep_dir = f'{self.nullscan_dir}/report'
            logs_dir = f'{self.nullscan_dir}/logs'
            self.report = HTML(TODAY, self.opt.opts, tmpl_dir, rep_dir,
                               logs_dir)
            self.report.make_report()
            self.log('Report done\n\n', _type='msg')
        self.log('Game Over\n', _type='msg')

        # reset terminal to original state. sometimes f**k up occurs because of
        # color and other escape codes.
        self.misc.reset_terminal()

        return
示例#22
0
class Module:
  """ module handler class """


  def __init__(self, mod_path):
    """ constructor """

    # logger
    self.logger = Logger()
    self.log = self.logger.log

    # file i/o
    self.file = File()

    # get all modules
    self.mod_path = mod_path
    self.mods = []
    self.get_modules()

    # all loaded module
    self.lmod = {}

    # docstrings for all tools
    self.docstrings = {}

    return


  def get_docstrings(self):
    """ get a list of docstrings from all nullscan tools """

    tools = []    # tools from sub classes

    for mod in self.mods:
      self.load_module(mod)
      cls = next(iter(self.lmod[mod].keys()))
      for tool in self.lmod[mod][cls]:
        modsplit = mod.split('.')
        moddir = modsplit[1]
        module = modsplit[2]
        if 'Base' not in str(cls):
          t_attr = getattr(cls, tool)
        docstr = ' '.join(t_attr.__doc__.split()).split()
        try:
          d_idx = docstr.index('DESCR:')
          t_idx = docstr.index('TOOLS:')
        except:
          pass
        descr = ' '.join((docstr[d_idx + 1:t_idx]))
        tools.append(docstr[t_idx + 1:])
        unique_tools = list(set(itertools.chain.from_iterable(tools)))
        self.docstrings[tool] = {'moddir': moddir, 'module': module,
          'descr': descr, 'tools': unique_tools }

    return


  def get_objects(self, mod, lmod):
    """ get classes and their methods out of a loaded module """

    methods = []

    for d in dir(lmod):
      if d[0].isupper() and 'Base' not in d:
        attr = getattr(lmod, d) # class
        for m in dir(attr): # public methods
          if not m.startswith('_') and not m.startswith('__') and \
            callable(getattr(attr, m)):
              methods.append(m)
        self.lmod = {mod: {attr: list(filter(None, methods))}}

    return


  def get_modules(self):
    """ get all modules out of MOD_PATH directory """

    for f in glob.glob(f'{self.mod_path}/**', recursive=True):
      if '__' not in f and '/libs/' not in f and f.endswith('.py'):
        self.mods.append('.'.join(f.split('/')[-3:]).split('.py')[0])
    #for root, dirs, files in os.walk(self.mod_path, topdown=True):
    #  dirs[:] = [d for d in dirs if d != 'libs']
    #  for f in files:
    #    if f != '__init__.py' and f.endswith('.py'):
    #      mod = '.'.join(os.path.join(root, f).split('/')[-3:])
    #      self.mods.append((mod.replace('.py', '')))
    return


  def filter_modules(self, opts):
    """ filter included or excluded modules """

    # all default modules out of modules/ dir
    def_mods = [
      x.replace('/','.').replace('src.','') + '.default' \
        for x in glob.glob('src/modules/**') if not '__' in x
    ]
    tmp = []

    # add/remove chosen module by user
    if 'in_modules' in opts['modules'] and opts['modules']['in_modules']:
      # list with default modules (force) + user chosed modules to import
      self.mods = def_mods
      for key, val in opts['modules']['in_modules'].items():
        for v in val:
          self.mods.append(f'modules.{key}.{v}')
    elif 'ex_modules' in opts['modules'] and opts['modules']['ex_modules']:
      for key, val in opts['modules']['ex_modules'].items():
        for v in val:
          if v == 'default':
            self.log('mod_default', _type='err', end='\n')
          tmp.append(f'modules.{key}.{v}')
      for i in self.mods:
        if i in tmp:
          self.mods.remove(i)

    return


  def load_module(self, mod):
    """ load desired module and get classes+methods from loaded modules """

    try:
      lmod = importlib.import_module(mod)
      self.get_objects(mod, lmod)
    except Exception as e:
      self.log('mod_import', eargs=f"{mod} -> {e}", _type='err', end='\n')

    return


  def run_module(self, mod, target, opts, wdir):
    """ load and run module with its tools """

    # change temp working dir for tool logs
    rootdir = os.getcwd()
    os.chdir(self.file.make_dir(wdir))

    # load module, get class and create object of
    self.load_module(mod)
    cls = next(iter(self.lmod[mod].keys()))
    c = cls(target, opts['targets_opts'])

    # for status line
    cur_tool = 0
    sum_tool = len(self.lmod[mod][cls])

    # get method (tool) string, create object and run desired tool
    with ProcessPoolExecutor(opts['p_workers']) as exe:
      for t in self.lmod[mod][cls]:
        cur_tool += 1
        if 'host' in target:
          target_head = target['host']
        else:
          target_head = target
        stat_line = f"{target_head} | {'.'.join(mod.split('.')[1:])}.{t}" + \
          f' ({cur_tool}/{sum_tool})' + ' ' * 25
        tool = getattr(c, t)

        # filter in-/ex-cluded tools by user
        if '.default' not in repr(cls):
          if opts['tools']['in_tools']:
            if t not in opts['tools']['in_tools']:
              continue
          if t in opts['tools']['ex_tools']:
            continue

        # run tool
        try:
          if opts['verbose']:
            self.log(stat_line, _type='vmsg', end='\n')
          else:
            self.log(stat_line, _type='vmsg', flush=True, end='\r')
          exe.submit(tool)
        except:
          self.log('tool_failed', eargs=t, _type='err', end='\n')

    # done, move bitch...
    os.chdir(rootdir)

    return
示例#23
0
 def setUp(self):
     self.file = File(api_url = config.API_URL, token = config.TOKEN)
     self.token = config.TOKEN
示例#24
0
 def get(self, **kwargs):
     args = self.parser.parse_args()
     path = args['path']
     if os.path.isfile(path):
         return File(path)
     raise UnprocessableEntity
示例#25
0
文件: misc.py 项目: shamb0/nullscan
class Misc:
  """ class for miscellaneous stuff """


  def __init__(self):
    """ constructor """

    # logger
    self.logger = Logger()
    self.log = self.logger.log

    # file i/o
    self.file = File()

    # original terminal state
    self.term_fd = sys.stdin.fileno()
    self.term_state = termios.tcgetattr(self.term_fd)

    return


  def reset_terminal(self):
    """ reset terminal to original state """

    termios.tcsetattr(self.term_fd, termios.TCSADRAIN, self.term_state)

    return


  def grep_tools(self, py_file):
    """ Find all tools (method names) of given file """

    tools = []

    lines = self.file.read_file(py_file)
    for line in lines:
      if 'def ' in line:
        method = line.split()[1].split('(')[0]
        if not method.startswith('_'):
          tools.append(method)

    return sorted(list(set(tools)))


  def find_py_files(self, root_path):
    """ Find all py files with some exceptions of given root_path """

    py_files = []

    files = glob.glob(f'{root_path}/*/*.py', recursive=True)
    [py_files.append(x) for x in files if '__ini' not in x and '/libs' not in x]

    return sorted(list(set(py_files)))


  def kill_process(self, pattern, signal='TERM'):
    """ kill (all) processes matched by pattern """

    cmd = ['pkill', f'-{signal}', '-f', pattern]
    subprocess.run(cmd)

    return


  def remove_empty_files_dirs(self, rootpath):
    """ delete empty (log-)files and directories """

    threads = 20
    files = glob.glob(f'{rootpath}/**', recursive=True)

    # files
    with ThreadPoolExecutor(max_workers=threads) as exe:
      # files
      for f in files:
        if os.path.isfile(f):
          fsize = os.path.getsize(f)
          if fsize <= 3:
            exe.submit(os.unlink, f)

      # directories
      for f in files:
        if os.path.isdir(f) and not os.listdir(f):
          exe.submit(os.rmdir, f)
          self.remove_empty_files_dirs(rootpath)

    return


  def lookup_port_service(self, port, proto='tcp'):
    """ return corresponding service from services.csv for a given port """

    service = None
    services = self.file.read_csv_file(f'{ROOT_PATH}/lists/services.csv')

    for s in services:
      if port in s and proto in s:
        service = s[0]

    return service


  def print_tool(self, opts):
    """ print tool and a description of it """

    tools = []

    m = Module(MOD_PATH)
    m.get_docstrings()

    if 'all' in opts.keys():
      for tool in m.docstrings.keys():
        tools.append(tool)
    else:
      for moddir, module in opts.items():
        if module:
          for mod in module:
            for tool in m.docstrings:
              if m.docstrings[tool]['moddir'] == moddir and \
                m.docstrings[tool]['module'] == mod:
                  tools.append(tool)
        else:
          for k, v in m.docstrings.items():
            if moddir in v['moddir']:
              tools.append(k)

    tools = sorted(list(set(tools)))
    for tool in tools:
      self.log(m.docstrings[tool]['moddir'] + '/' +
        m.docstrings[tool]['module'] + '/' + tool + ' - ' +
        m.docstrings[tool]['descr'], end='\n', _type='vmsg')

    return


  def add_mod_tool(self, form, opts):
    """ create a new module and/or add tool to exisiting module """

    template = f'{MOD_PATH}libs/template.py'
    modpart = f"{MOD_PATH}{opts['moddir']}/"
    moddir = modpart
    module = f"{modpart}{opts['modname']}.py"
    tmpmod = f'{modpart}tmpxxx.py'

    if form == 'mod':
      with open(template, 'r') as fin:
        if not os.path.isdir(moddir):
          self.file.make_dir(moddir)
        with open(module, 'w') as fout:
          for line in fin:
            if '<template>.py' in line:
              fout.write(line.replace('<template>', opts['modname']))
            elif '<file>' in line:
              fout.write(line.replace('<file>', opts['modname']))
            elif '<class>' in line:
              fout.write(line.replace('<class>',
                opts['modname'].capitalize()))
            elif '<mod>' in line:
              fout.write(line.replace('<mod>', opts['modname']))
            else:
              fout.write(line)
      self.log(f"Created module {opts['moddir']}/{opts['modname']}.py\n",
        _type='msg')

    # append new tools
    try:
      shutil.copyfile(module, tmpmod)
      with open(tmpmod, 'a') as fout:
        fout.write('\n\n  @tool\n')
        fout.write(f"  def {opts['func']}(self):\n")
        fout.write('    """\n')
        fout.write('    DESCR: <REPLACE MANUALLY>\n')
        fout.write('    TOOLS: <ADD MANUALLY>\n')
        fout.write('    """\n\n')
        fout.write(f"    opts = \'{' '.join(opts['args'])}\'\n\n")
        fout.write(f"    self._run_tool(\'{opts['tool']}\', opts)\n\n")
        fout.write('    return\n')
      shutil.move(tmpmod, module)
      self.log(f"Added tool {opts['func']} to {opts['moddir']}/" +
        f"{opts['modname']}.py\n", _type='msg')
    except:
      self.log('add_tool', _type='err', end='\n')

    return
示例#26
0
    def calculateframe(self, filepath, frame, resolution, cutoff_radii=None, domains=False, surface=False, center=False,
                       atoms=None, gyration_tensor_parameters=False, recalculate=False, last_frame=True):
        """
        Get results for the given parameters. They are either loaded from the
        cache or calculated.

        **Parameters:**
            `filepath` :
                absolute path of the input file
            `frame` :
                the frame number
            `resolution` :
                resolution of the used discretization
            `domains` :
                calculate cavitiy domains
            `cutoff_radii` :
                dict that maps element symbols to cutoff radii
            `surface` :
                calculate surface-based cavities
            `center` :
                calculate center-based cavities
            `gyration_tensor_parameters` :
                gyration tensor parameters will be calculated for cavities (they are always calculated for
                cavity domains)
            `recalculate` :
                results will be calculated even if cached results exists

        **Returns:**
            A :class:`core.data.Results` object.
        """
        # always recalculate if gyration tensor parameters shall be computed for center or surface based cavities
        recalculate = recalculate or (gyration_tensor_parameters and (center or surface))
        message.progress(0)
        inputfile = File.open(filepath)
        # TODO: error handling
        if isinstance(inputfile, core.file.ResultFile):
            resultfile = inputfile
        else:
            resultfile = self.cache[filepath]
        try:
            results = resultfile.getresults(frame, resolution)
        except Exception as e:
            logger.debug("error in resultfile.getresults: {}".format(e))
            results = None

        if atoms is None:
            atoms = inputfile.getatoms(frame)
        atoms.radii = cutoff_radii
        volume = atoms.volume
        if results is None:
            results = data.Results(filepath, frame, resolution, atoms, None, None, None)

        if recalculate:
            results.domains = None
            results.surface_cavities = None
            results.center_cavities = None

        if not ((domains and results.domains is None)
                or (surface and results.surface_cavities is None)
                or (center and results.center_cavities is None)):
            message.print_message("Reusing results")
        else:
            cachepath = os.path.join(self.cachedir, 'discretization_cache.hdf5')
            discretization_cache = DiscretizationCache(cachepath)
            discretization = discretization_cache.get_discretization(volume, resolution)
            atom_discretization = AtomDiscretization(atoms, discretization)
            message.progress(10)
            if (domains and results.domains is None) \
                    or (surface and results.surface_cavities is None):
                # CavityCalculation depends on DomainCalculation
                message.print_message("Calculating domains")
                domain_calculation = DomainCalculation(discretization, atom_discretization)
                if domain_calculation.critical_domains:
                    logger.warn('Found {:d} critical domains in file {}, frame {:d}. Domain indices: {}'.format(
                        len(domain_calculation.critical_domains), os.path.basename(filepath), frame,
                        domain_calculation.critical_domains
                    ))
                    message.log('Found {:d} critical domains in file {}, frame {:d}'.format(
                        len(domain_calculation.critical_domains), os.path.basename(filepath), frame + 1,
                    ))
            if results.domains is None:
                results.domains = data.Domains(domain_calculation)
            message.progress(40)

            if surface and results.surface_cavities is None:
                message.print_message("Calculating surface-based cavities")
                cavity_calculation = CavityCalculation(domain_calculation, use_surface_points=True,
                                                       gyration_tensor_parameters=gyration_tensor_parameters)
                results.surface_cavities = data.Cavities(cavity_calculation)
            message.progress(70)

            if center and results.center_cavities is None:
                message.print_message("Calculating center-based cavities")
                domain_calculation = FakeDomainCalculation(discretization, atom_discretization, results)
                cavity_calculation = CavityCalculation(domain_calculation, use_surface_points=False,
                                                       gyration_tensor_parameters=gyration_tensor_parameters)
                results.center_cavities = data.Cavities(cavity_calculation)
            resultfile.addresults(results, overwrite=recalculate)

        message.progress(100)
        message.print_message("Calculation finished")
        if last_frame:
            message.finish()
        return results
示例#27
0
    def calculate(self, calcsettings):
        """
        Calculate (or load from the cache) all results for the given settings.

        **Parameters:**
            `calcsettings` :
                :class:`CalculationSettings` object

        **Returns:**
            A list of list of :class:`core.data.Results` objects. The outer list contains
            an entry for each entry in `calcsettings.filenames`; the inner
            list has a `Results` entry for each frame specified in
            `calcsettings.frames`.
        """
        allresults = []
        for filename, frames in calcsettings.datasets.iteritems():
            filepath = core.file.get_abspath(filename)
            fileprefix = os.path.basename(filename).rsplit(".", 1)[0]
            if calcsettings.exportdir is not None:
                exportdir = core.file.get_abspath(calcsettings.exportdir)
                # replace asterisks with directories
                dirlist = os.path.dirname(filepath).split("/")
                while "*" in exportdir:
                    i = exportdir.rindex("*")
                    exportdir = os.path.join(exportdir[:i], dirlist.pop() + exportdir[i+1:])
                if (calcsettings.exporthdf5 or calcsettings.exporttext) \
                        and not os.path.exists(exportdir):
                    os.makedirs(exportdir)
            else:
                exportdir = os.path.dirname(filepath)
            if calcsettings.exporthdf5:
                efpath = os.path.join(exportdir, fileprefix + ".hdf5")
                efpath = core.file.get_abspath(efpath)
                # copy atoms into HDF5 file
                exportfile = core.file.HDF5File.fromInputFile(efpath, filepath)
                # use HDF5 file as input
                filepath = efpath

            fileresults = []
            if frames[0] == -1:
                inputfile = File.open(filepath)
                frames = range(inputfile.info.num_frames)
            last_frame = False
            for frame in frames:
                # calculate single frame
                if frame is frames[-1]:
                    last_frame = True
                frameresult = self.calculateframe(
                    filepath,
                    frame,
                    calcsettings.resolution,
                    calcsettings.cutoff_radii,
                    domains=calcsettings.domains,
                    surface=calcsettings.surface_cavities,
                    center=calcsettings.center_cavities,
                    gyration_tensor_parameters=calcsettings.gyration_tensor,
                    recalculate=calcsettings.recalculate,
                    last_frame=last_frame)
                # export to text file
                if calcsettings.exporttext:
                    fmt = os.path.join(exportdir, fileprefix) + "-{property}-{frame:06d}.txt"
                    if frameresult.atoms is not None:
                        frameresult.atoms.totxt(fmt.format(property='{property}', frame=frame+1))
                    if frameresult.domains is not None:
                        try:
                            frameresult.domains.totxt(fmt.format(property='domain_{property}', frame=frame+1))
                        except ValueError as e:
                            logger.warn(e.message)
                            logger.warn('The export of domain information could not be finished.')
                    if frameresult.surface_cavities is not None:
                        frameresult.surface_cavities.totxt(fmt.format(property='surface_cavities_{property}', frame=frame+1))
                    if frameresult.center_cavities is not None:
                        frameresult.center_cavities.totxt(fmt.format(property='center_cavities_{property}', frame=frame+1))
                    # TODO: try/except
                # gather results
                fileresults.append(frameresult)
            allresults.append(fileresults)
        return allresults
示例#28
0
文件: parser.py 项目: shamb0/nullscan
class Parser:
    """ class for parsing cmdline and config file from nullscan """
    def __init__(self, opts):
        """ constructor"""

        self.opts = opts  # all nullscan options
        self.logger = Logger()
        self.log = self.logger.log
        self.file = File()
        self.misc = Misc()

        # tmp stupid fix to strip bad chars out of options
        self.bad_chars = '<>()[]{}:;.,="\'*&^%$#@!+-_/?°²³´`'

        return

    def parse_config(self):
        """ parse nullscan config file and get options+values as dictionary+list. """

        try:
            cfg = ConfigParser()
            data = cfg.read(self.opts['config']['file'])
            if not data:
                raise Exception
            for s in cfg.sections():
                for o in cfg.options(s):
                    values = cfg.get(s, o).replace('$NULLSCAN_DIR/', ROOT_PATH)
                    if ',' in values and o != 'nmap':
                        self.opts['config']['opts'][o] = values.split(',')
                    else:
                        self.opts['config']['opts'][o] = values
        except:
            self.log('config',
                     eargs=f"{self.opts['config']['file']}",
                     _type='err',
                     end='\n')

        return

    def parse_add_module_tool(self, args, dest):
        """ parse add module/tool option (-m/-a) """

        s = args.split()

        try:
            m = s[0].split('/')
            d = {
                'moddir': m[0],
                'modname': m[1],
                'func': s[1],
                'tool': s[2],
                'args': s[3:]
            }
        except:
            self.log('add_mod_tool', _type='err', end='\n')

        self.opts[dest] = d

        return

    def parse_print_tools(self, args):
        """ parse print tools option (-p) """

        dicted = {}
        splitted = list(filter(None, args.split(';')))

        for item in splitted:
            if item == 'all':
                dicted['all'] = True
                break
            if '=' in item:
                i = item.split('=')
                for j in i:
                    if ',' in j:
                        dicted[i[0]] = j.split(',')
                    else:
                        dicted[i[0]] = [j]
            else:
                dicted[item] = []

        self.opts['print_tools'] = dicted

        return

    def parse_tools(self, args, dest):
        """ parse tools include/exclude option (-I/-X) """

        self.opts['tools'][dest] = args.split(',')

        return

    def parse_modules(self, args, dest):
        """ parse modules include/exclude options (-i/-x) """

        # remove empty ones, if ';' was used at the end without further opts
        if ';' in args:
            mods = [x.strip(self.bad_chars) for x in args.split(';') if x]
        elif '=' in args:
            mods = args.split()
        else:
            self.log('inexmods', _type='err', end='\n')

        # create dict out of list itmes
        dicted = dict(d.split('=') for d in mods)

        for key, val in dicted.items():
            if ',' in val:
                dicted[key] = val.split(',')
            else:
                dicted[key] = [val]

        self.opts['modules'][dest] = dicted

        return

    def parse_extra_opts(self, args):
        """ parse extra options option (-o) """

        # remove empty ones, if ';' was used at the end without further opts
        if ';' in args:
            args = [x.strip(self.bad_chars) for x in args.split(';') if x]
        else:
            args = [args]

        # create dict out of list items except for nmap
        dicted = dict(d.split('=') for d in args)
        for key in dicted.keys():
            if ',' in dicted[key] and key != 'nmap':
                dicted[key] = dicted[key].split(',')

        self.opts['targets_opts'] = dicted

        return

    def parse_nmap_targets(self, targets):
        """ parse nmap targets to scan from either ranges or from file (-t) """

        hosts = []

        if os.path.isfile(targets):
            hosts = self.file.read_file(targets)
        else:
            try:
                # host range format
                if '-' in targets:
                    splitted = targets.split('-')
                    try:
                        start = ipaddress.IPv4Address(splitted[0])
                        end = ipaddress.IPv4Address(splitted[1])
                        for i in range(int(start), int(end) + 1):
                            ipaddr = str(ipaddress.IPv4Address(i))
                            hosts.append(ipaddr)
                    except:
                        # must be a hostname/domain then
                        if ',' in targets:
                            hosts = targets.split(',')
                        else:
                            hosts.append(targets)
                # cidr range format
                elif '/' in targets:
                    for ipaddr in ipaddress.IPv4Network(targets).hosts():
                        hosts.append(str(ipaddr))
                # multiple single hosts
                elif ',' in targets:
                    hosts = targets.split(',')
                # single host
                else:
                    hosts.append(targets)
            except Exception as e:
                self.log('hostrange',
                         eargs=repr(e.args[0]),
                         _type='err',
                         end='\n')

        if hosts:
            self.opts['targets']['nmap']['hosts'] = hosts

        return

    def parse_nmap_logfile(self, logfile, privip=False):
        """ parse hosts and ports from nmap xml logfile (-l) """

        nmap = NmapParser()

        try:
            parsed = nmap.parse_fromfile(logfile)
        except:
            self.log('nmap', eargs=f'{logfile}', _type='err', end='\n')
        else:
            for host in parsed.hosts:
                if len(host.hostnames) >= 1:
                    _host = host.hostnames[0]  # better work with hostname
                else:
                    _host = host.address
                    try:
                        if ipaddress.ip_address(_host).is_private:
                            privip = True
                    except:
                        pass  # hostname
                ports = []
                res = host.get_open_ports()
                if res:
                    for port, proto in res:
                        tmp = str(host.get_service(port)).translate(
                            str.maketrans('', '', ':[])')).split()
                        service = self.misc.lookup_port_service(
                            str(port), proto)
                        ports.append([str(port), service])
                    self.opts['targets'][proto].append({
                        'host': _host,
                        'ports': ports,
                        'privip': privip
                    })
                else:
                    # host up but no ports found
                    self.opts['targets']['tcp'].append({
                        'host': _host,
                        'ports': [],
                        'privip': privip
                    })

        return

    def parse_targets(self, args, privip=False):
        """ parse all targets from line specified via URIs (-u) """

        targets = []

        # remove empty ones, if ';' was used at the end without further targets
        if ';' in args:
            targets = [x.strip() for x in args.split(';') if x]
        elif 'person://' in args or 'company://' in args:
            targets.append(args)
        else:
            targets = args.split()

        for t in targets:
            if re.search('^tcp://|^udp://', t):
                ports = []
                part = t.split('://')
                proto = part[0].strip(self.bad_chars)
                self.opts['targets_opts']['target_type'] = proto
                host = part[1].split(':')[0].strip(self.bad_chars)
                try:
                    if ipaddress.ip_address(host).is_private:
                        privip = True
                except:
                    pass  # hostname
                if ':' in part[1]:
                    tmp_ports = part[1].split(':')[1]
                    tmp_ports = tmp_ports.split(',')
                    for p in tmp_ports:
                        if '=' in p:
                            ports.append(p.split('='))
                        else:
                            service = self.misc.lookup_port_service(p)
                            if service:
                                p = f'{p}={service}'
                            else:
                                # assign default service if not given
                                p = f'{p}=default'
                            ports.append(p.split('='))
                self.opts['targets'][proto].append({
                    'host': host,
                    'ports': ports,
                    'privip': privip
                })
            elif re.search('^http://|^https://', t):
                self.opts['targets_opts']['target_type'] = t.split(':')[0]
                self.opts['targets']['web'] = t.split(',')
                # add leading '/' to url if not given
                #for u in t.split(','):
                #  if not u.endswith('/'):
                #    u = f'{u}/'
                #  self.opts['targets']['web'].append(u)
            elif re.search(
                    '\
        ^mail://|^person://|^company://|^domain://|^lan://|^wifi://', t):
                self.opts['targets_opts']['target_type'] = t.split(':')[0]
                line = list(
                    filter(None,
                           [l.replace('://', ':') for l in t.split(';')]))
                line = dict(l.split(':') for l in line)
                for k in line.keys():
                    if k == 'mail' or k == 'person' or k == 'company' or k == 'domain':
                        self.opts['targets']['social'][k] = line[k].split(',')
                    else:
                        self.opts['targets'][k] = line[k].split(',')
            else:
                unknown_mode = t.split('://')[0]
                self.log('mode', eargs=unknown_mode, _type='err', end='\n')

        return

    def parse_cmdline(self):
        """ parse command line """

        try:
            opts, args = getopt.getopt(
                self.opts['cmdline'][1:],
                't:u:l:o:i:I:x:X:T:M:P:k:rR:c:vdCp:m:a:VH')
        except getopt.GetoptError as err:
            self.log('default', eargs=repr(err), _type='err', end='\n')

        for o, a in opts:
            if o == '-t':
                if a == '?':
                    Usage.nmap_mode_usage()
                    os._exit(SUCCESS)
                else:
                    self.parse_nmap_targets(a)
            elif o == '-u':
                if a == '?':
                    Usage.host_mode_usage()
                    os._exit(SUCCESS)
                else:
                    self.parse_targets(a)
            elif o == '-l':
                self.parse_nmap_logfile(a)
            elif o == '-o':
                if a == '?':
                    Usage.extra_opts_usage()
                    os._exit(SUCCESS)
                else:
                    self.parse_extra_opts(a)
            elif o == '-i':
                if a == '?':
                    Usage.modules_usage()
                    os._exit(SUCCESS)
                else:
                    self.parse_modules(a, 'in_modules')
            elif o == '-I':
                if a == '?':
                    Usage.tools_usage()
                    os._exit(SUCCESS)
                else:
                    self.parse_tools(a, 'in_tools')
            elif o == '-x':
                if a == '?':
                    Usage.modules_usage()
                    os._exit(SUCCESS)
                else:
                    self.parse_modules(a, 'ex_modules')
            elif o == '-X':
                if a == '?':
                    Usage.tools_usage()
                    os._exit(SUCCESS)
                else:
                    self.parse_tools(a, 'ex_tools')
            elif o == '-T':
                self.opts['t_workers'] = a
            elif o == '-M':
                self.opts['m_workers'] = a
            elif o == '-P':
                self.opts['p_workers'] = a
            elif o == '-k':
                self.opts['timeout'] = a
            elif o == '-r':
                self.opts['report'] = True
            elif o == '-R':
                self.opts['nullscan_dir'] = a + '-' + TODAY
            elif o == '-c':
                self.opts['config']['file'] = a
            elif o == '-v':
                self.opts['verbose'] = True
            elif o == '-d':
                self.opts['debug'] = True
            elif o == '-C':
                self.opts['check_tools'] = True
            elif o == '-p':
                if a == '?':
                    Usage.print_tools_usage()
                    os._exit(SUCCESS)
                else:
                    self.parse_print_tools(a)
            elif o == '-m':
                if a == '?':
                    Usage.add_module_usage()
                    os._exit(SUCCESS)
                else:
                    self.parse_add_module_tool(a, 'add_module')
            elif o == '-a':
                if a == '?':
                    Usage.add_tool_usage()
                    os._exit(SUCCESS)
                else:
                    self.parse_add_module_tool(a, 'add_tool')
            elif o == '-V':
                self.log(VERSION, _type='msg', end='\n')
                os._exit(SUCCESS)
            elif o == '-H':
                Usage.usage()
                os._exit(SUCCESS)
            else:
                self.log('cmdopt', eargs=o, _type='err', end='\n')

        return