Esempio n. 1
0
def main(workflow):
    query = "" if len(workflow.args) == 0 else workflow.args[0]

    config = workflow.stored_data("configuration")
    if config == None:
        Options.warning(
            "Didn't find your configuration",
            "Please supply your cheat sheet path using 'cf ~/your/path'",
            workflow)
        workflow.send_feedback()
        return -1

    parser = Parser(config.getPath())
    options = Options(parser, workflow)

    tokens = query.strip().split(" ", 1)
    tokens = [i.strip() for i in tokens if i != ""]

    if len(tokens) < 2:
        sheetName = "" if len(tokens) == 0 else tokens[0]
        handler = options.showAvailable if sheetName not in parser.availableSheets(
        ) else options.list
        handler(sheetName)
    else:
        sheetName = tokens[0]
        searchTerm = tokens[1]
        if sheetName == "--search":
            options.search(None, searchTerm)
        else:
            options.search(sheetName, searchTerm)

    workflow.send_feedback()
    return None
Esempio n. 2
0
def run_compiler(source, target, debug=False):
    """Run Compiler

    Executes the compilation process given a source file path.

    Arguments:
        source: The source file to compile.
        target: The destination binary executable file.
        debug: If True, verbose parsing details are shown. (Default: False)

    Returns:
        True on success, False otherwise.
    """
    # Define a temporary location for the intermediate C code
    TMP_CODE_FILE = './ir.c'

    # Create a Parser object to parse the inputted source file
    parser = Parser(debug)

    # Parse the source file to the temporary code file
    if not parser.parse(source, TMP_CODE_FILE):
        print('Error while parsing "%s"' % source)
        return False

    # Set up gcc compilation command
    gcc_cmd = ['gcc', '-m32', '-o', target, TMP_CODE_FILE]

    # Compile the temporary file with gcc. Output to the target location
    if subprocess.call(gcc_cmd) != 0:
        print('Error while compiling "%s"' % target)
        return False

    return True
Esempio n. 3
0
    def _parse_calendars_process(input_tuple: (int, int, (int, str, str),
                                               datetime, dict)) -> dict:
        input_index, total_length, calendar_tuple, timestamp, website_base = input_tuple
        calendar_id, calendar_url, calendar_html_file_path = calendar_tuple
        file = os.path.basename(calendar_html_file_path)

        debug_output = "{}/{} | {}/{}".format(input_index, total_length,
                                              website_base["domain"], file)

        if not os.path.isfile(calendar_html_file_path):
            debug_output += " | 0 (File '{}' does not exist!)".format(
                calendar_html_file_path)
            print(debug_output)
            return {calendar_id: []}

        with open(calendar_html_file_path, encoding="utf-8") as html_file:
            dom = etree.parse(html_file, etree.HTMLParser())

        parser = Parser(website_base["parser"])
        parser.set_dom(dom)
        event_urls = parser.get_event_urls()

        events_to_insert = []
        for index, url_path in enumerate(event_urls):
            event_url = urllib.urljoin(calendar_url, url_path)
            events_to_insert.append((event_url, timestamp))

        debug_output += " | {}".format(len(events_to_insert))
        print(debug_output)

        return {calendar_id: events_to_insert}
Esempio n. 4
0
def run_compiler(input_name, output_name):
    parser = Parser()
    analyzer = StaticAnalyzer()
    flow_generator = FlowGraph()
    code_generator = CodeGenerator()

    source_code = ''

    try:
        with open(input_name, 'r') as f:
            source_code = f.read()
    except FileNotFoundError:
        logging.error("File not found")
        exit(1)

    try:
        # group tokens into syntactical units using parser
        parse_tree = parser.parse(source_code)
        # perform semantic analyze
        symtab, ast = analyzer.analyze(parse_tree)
        # generate flow graph
        flow_graph = flow_generator.generate(ast)
        # generate code
        code = code_generator.generate(flow_graph, symtab)

        with open(output_name, 'w') as f:
            f.write(str(code))

    except CompilerError as error:
        if str(error):
            logging.error("COMPILER_ERROR: {0}".format(str(error)))
        exit(1)
Esempio n. 5
0
def run_compiler(source, target, debug=False):
    """Run Compiler

    Executes the compilation process given a source file path.

    Arguments:
        source: The source file to compile.
        target: The destination binary executable file.
        debug: If True, verbose parsing details are shown. (Default: False)

    Returns:
        True on success, False otherwise.
    """
    # Define a temporary location for the intermediate C code
    TMP_CODE_FILE = "./ir.c"

    # Create a Parser object to parse the inputted source file
    parser = Parser(debug)

    # Parse the source file to the temporary code file
    if not parser.parse(source, TMP_CODE_FILE):
        print('Error while parsing "%s"' % source)
        return False

    # Set up gcc compilation command
    gcc_cmd = ["gcc", "-m32", "-o", target, TMP_CODE_FILE]

    # Compile the temporary file with gcc. Output to the target location
    if subprocess.call(gcc_cmd) != 0:
        print('Error while compiling "%s"' % target)
        return False

    return True
    def setUp(self):
        yaml_files = [
            'lib/configs/ogc_identifier.yaml',
            'lib/configs/iso_identifier.yaml'
        ]

        # set up for the known csw getcapabilities
        with open('tests/test_data/cwic_csw_v2_0_2.xml', 'r') as f:
            csw_content = f.read()
        csw_url = 'http://www.mapserver.com/cgi?SERVICE=WCS&VERSION=2.0.2&REQUEST=GETCAPABILITIES'

        csw_content = csw_content.replace('\\n', '')
        csw_parser = Parser(csw_content)

        self.csw_identifier = Identify(yaml_files, csw_content, csw_url,
                                       **{'parser': csw_parser})

        # set up for the geonetwork mismatched namespacing iso issue
        with open('tests/test_data/geonetwork_iso_NOT_csw.xml', 'r') as f:
            iso_content = f.read()
        iso_url = 'http://catalog.data.gov/harvest/object/d5de6dde-3042-4daf-b4ba-95e21e3ab343'

        iso_content = iso_content.replace('\\n', '')
        iso_parser = Parser(iso_content)

        self.iso_identifier = Identify(yaml_files, iso_content, iso_url,
                                       **{'parser': iso_parser})
Esempio n. 7
0
def output_pharser_result(format_file_input, data_file_input):
    format_file = mockFileName(format_file_input)
    data_file = mockFileName(data_file_input)

    parser = Parser(format_file)
    parser.loadData(data_file)

    return parser.asObject()
Esempio n. 8
0
 def _build_all_tasks(self):
     parser = Parser()
     apps = parser.parse_folder(self._folder)
     first, nonfirst, columns = self._get_task_records(apps)
     return (pd.DataFrame.from_records(first,
                                       columns=columns),
             pd.DataFrame.from_records(nonfirst,
                                       columns=columns))
Esempio n. 9
0
 def test_get_pages(self):
     page = PageLoader()
     response = page.get_page()
     parser = Parser(response.text)
     links = parser.get_pages()
     result = False
     if len(links) > 0:
         result = True
     self.assertTrue(result)
Esempio n. 10
0
 def _build(self):
     all_durs, all_sizes, all_workers = [], [], []
     parser = Parser()
     apps = parser.parse_folder(self._folder)
     sizes = self._get_app_sizes()
     for app, size in zip(apps, sizes):
         duration = self._get_app_duration(app)
         all_durs.append(duration)
         all_sizes.append(size)
         all_workers.append(app.slaves)
     return _get_df(all_workers, all_sizes, all_durs)
Esempio n. 11
0
File: uws.py Progetto: elcodigok/uws
	def do_get(self, person):
		if person and person in SET_OPTIONS:
			greeting = 'hi, %s!' % person
		elif person:
			greeting = "hello, " + person
		else:
			greeting = 'hello'

		p = Parser()
		print p.checkParameter(self.lastcmd.split())
		print greeting
Esempio n. 12
0
File: uws.py Progetto: elcodigok/uws
	def do_show(self, line):
		if line and line in SHOW_OPTIONS:
			greeting = 'hi, %s!' % line
		elif line:
			greeting = self.uws.getUrl()
		else:
			greeting = 'URL\t=>\t' + self.uws.getUrl()

		p = Parser()
		p.checkParameter(self.lastcmd.split())
		print greeting
Esempio n. 13
0
def main():
    parser = Parser()

    # ms.printDB('lpmdb.bin')
    # ms.json_to_lpmdb_bin('lpmdb.json', 'lpmdb.bin')
    # mv = ms.readMovieByPos('lpmdb.bin', 11616)
    # print(mv.title)

    # parser.parse(':: from title filter the')
    while parser.parse(input()):  # returns false if input() returns "exit"
        pass
    def __init__(self):
        self.arguments = Arguments()
        self.blender = Blender()
        self.parser = Parser()

        self.botfile = self.arguments.getBotFile()
        self.blender.unselectEverything()
        print("\nNow building " + self.botfile + "...")
        self.cubeData = self.parser.parseBotFile(self.botfile)
        self.cubedatabase = self.parser.parseCSVFile("assets/cubes.csv")
        self.blender.build(self.cubeData, self.cubedatabase)
        print("done!")
Esempio n. 15
0
class TestParser(unittest.TestCase):
    def setUp(self):
        '''
        we are assuming input from the solr sample parser
        so this is the encoded, cdata-removed input

        except i have blended the two a bit (handling that \\n issue
            in the parser vs the cdata issue in the solr response)

        so init the parser with a file that reflects that
        '''
        with open('tests/test_data/basic_osdd_c1576284036448b5ef3d16b2cd37acbc.txt', 'r') as f:
            data = f.read()

        data = data.replace('\\n', ' ')

        self.parser = Parser(data)

    def test_parse_xml(self):
        '''
        the _parse is called from init
        '''

        self.assertTrue(self.parser.xml is not None)
        self.assertTrue(len(self.parser._namespaces) > 0)

    def test_remap_namespaced_paths(self):
        # basic check
        in_xpath = '{http://a9.com/-/spec/opensearch/1.1/}OpenSearchDescription/{http://a9.com/-/spec/opensearch/1.1/}Image'
        out_xpath = 'default:OpenSearchDescription/default:Image'

        test_xpath = self.parser._remap_namespaced_xpaths(in_xpath)

        self.assertTrue(test_xpath == out_xpath)

    def test_find_nodes(self):
        nodes = self.parser.find_nodes()

        self.assertTrue(len(nodes) == 5)
        self.assertTrue(len(nodes[2]) == 3)
        self.assertTrue(nodes[1]['text'] == 'UTF-8')

        # run with excludes
        excludes = [
            '{http://a9.com/-/spec/opensearch/1.1/}OpenSearchDescription/{http://a9.com/-/spec/opensearch/1.1/}InputEncoding',
            '{http://a9.com/-/spec/opensearch/1.1/}OpenSearchDescription/{http://a9.com/-/spec/opensearch/1.1/}Image/@width'
        ]
        nodes = self.parser.find_nodes(excludes)
        self.assertTrue(len(nodes) == 4)
        self.assertTrue(len(nodes[1]) == 3)
        self.assertTrue(nodes[0]['text'] == 'CEOS')
        self.assertTrue(nodes[1]['attributes'][1]['text'] == '16')
        self.assertTrue(nodes[1]['attributes'][0]['xpath'] == '{http://a9.com/-/spec/opensearch/1.1/}OpenSearchDescription/{http://a9.com/-/spec/opensearch/1.1/}Image/@type')
Esempio n. 16
0
def test_without_typedef():
    testcase = """
    //comment
    struct FirstName {
        int foo;
        float bar[1];
    } NameAlias;
    """
    lexer = Lexer(testcase)
    parser = Parser(lexer)
    structure = parser.parse()
    assert structure is not None
Esempio n. 17
0
def test_multiple_names():
    testcase = """
    //comment
    typedef struct StructName {
        int foo;
        float bar[MAX_SIZE];
    } StructNameAlias;
    """
    lexer = Lexer(testcase)
    parser = Parser(lexer)
    structure = parser.parse()
    assert structure.name == 'StructNameAlias'
Esempio n. 18
0
def calculate_column(dataset, dframe, formula, name):
    """
    For calculating new columns.
    Get necessary data given a calculation ID, execute calculation formula,
    store results in dataset the calculation refers to.
    """
    # parse formula into function and variables
    parser = Parser()
    func = parser.parse_formula(formula)

    new_column = dframe.apply(func, axis=1, args=(parser, ))
    new_column.name = name
    return Observation.update(dframe.join(new_column), dataset)
Esempio n. 19
0
class Analytics:
    def __init__(self, config):
        self.config = config

        self.downloader = Downloader(config)
        self.parser = Parser(config)
        self.analyser = Analyser(config)

    def generate_report(self):
        if self.config.download:
            self.downloader.download()
        self.parser.parse()
        self.analyser.analyse()
        return self.config
Esempio n. 20
0
 def setUp(self):
     data = '''<xml>
     <node>Does it parse?  &lt;br/&gt; It &lt;em&gt;should&lt;/em&gt;!</node>
     <nextnode>Wow, that's a typography sin right there, but &lt;a href="#anchor"&gt;Nope&lt;/a&gt; and &lt;span&gt;Many XML&lt;/span&gt;.</nextnode>
     </xml>
     '''
     self.parser = Parser(data)
Esempio n. 21
0
    def setUp(self):
        '''
        we are assuming input from the solr sample parser
        so this is the encoded, cdata-removed input

        except i have blended the two a bit (handling that \\n issue
            in the parser vs the cdata issue in the solr response)

        so init the parser with a file that reflects that
        '''
        with open('tests/test_data/basic_osdd_c1576284036448b5ef3d16b2cd37acbc.txt', 'r') as f:
            data = f.read()

        data = data.replace('\\n', ' ')

        self.parser = Parser(data)
Esempio n. 22
0
class ExperimentInfo:
    def __init__(self, threads, block_size, path=None):
        self.threads = threads
        self.block_size = block_size
        self._path = path
        self._parser = Parser()

    def get_n_blocks(self, input_size):
        return np.ceil(input_size / self.block_size)

    def get_n_tasks(self, stage):
        task_amounts = self._get_ns_tasks(stage)
        assert len(task_amounts) == 1, 'More than 1 value: {}'.format(
            task_amounts)
        return task_amounts[0]

    def _get_ns_tasks(self, stage):
        amounts = set()
        for app in self._get_apps():
            tasks = app.stages[stage].tasks
            n = sum(1 for t in tasks if not t.failed)
            amounts.add(len(app.stages[stage].tasks))
        return list(amounts)

    def _get_apps(self):
        return self._parser.parse_folder(self._path)
Esempio n. 23
0
    def run_analysis(self, string_list):
        """
            This method is used by the core to run analyze and extract strings
            matching with the "elf symbols" type.

            :param string_list: All strings to analyse.
            :type string_list: List
            :return: A list of string without strings previously matched.
            :rtype: List
        """

        epur_string_list = []
        ret = subprocess.Popen(["readelf", "-s", self.malware_path],
                                stdout=subprocess.PIPE)
        readelf_output = ret.communicate()[0].split('\n')[3:]

        self.bar.init(string_list, self.info_msg + "is running")
        for string in string_list:
            sym_found, specific_type = Parser.getSymbol(string, readelf_output)
            if sym_found != None:
                self.db.createEntry(string, sym_found,
                                    self.type + "-" + specific_type)
            else:
                epur_string_list.append(string)
            self.bar.update()
        self.bar.close(self.info_msg + "is complete", True)
        return epur_string_list
Esempio n. 24
0
def test_comment_before_bracket():
    testcase = """
    //comment
    typedef struct // comment before opening bracket
    {
        int a;
        unsigned long long int b;
    /* comment
        before
        closing bracket */
    } StructName;
    """
    lexer = Lexer(testcase)
    parser = Parser(lexer)
    structure = parser.parse()
    assert structure is not None
Esempio n. 25
0
    def run_analysis(self, string_list):
        """
            This method is used by the core to run analyze and extract strings
            matching with the "elf symbols" type.

            :param string_list: All strings to analyse.
            :type string_list: List
            :return: A list of string without strings previously matched.
            :rtype: List
        """

        epur_string_list = []
        ret = subprocess.Popen(["readelf", "-s", self.malware_path],
                               stdout=subprocess.PIPE)
        readelf_output = ret.communicate()[0].split('\n')[3:]

        self.bar.init(string_list, self.info_msg + "is running")
        for string in string_list:
            sym_found, specific_type = Parser.getSymbol(string, readelf_output)
            if sym_found != None:
                self.db.createEntry(string, sym_found,
                                    self.type + "-" + specific_type)
            else:
                epur_string_list.append(string)
            self.bar.update()
        self.bar.close(self.info_msg + "is complete", True)
        return epur_string_list
Esempio n. 26
0
 def _write_set(self, sset, apps_sizes):
     for app, size in apps_sizes:
         stage_stats = StageStats.get_stats(app.stages)
         # Bug in bytes read metrics reported by Spark
         stage_stats[1] = size
         self._writer.writerow([len(app.slaves), sset, size,
                                app.records_read, app.duration,
                                Parser.fits_in_memory(app)] + stage_stats)
Esempio n. 27
0
 def _build_first_nonfirst_tasks(self):
     durations = {'first': [], 'nonfirst': []}
     sizes = {'first': [], 'nonfirst': []}
     workers = {'first': [], 'nonfirst': []}
     parser = Parser()
     apps = parser.parse_folder(self._folder)
     apps_sizes = self._get_app_sizes()
     for app, size in zip(apps, apps_sizes):
         first, nonfirst = self._get_first_nonfirst_tasks_durations(app)
         durations['first'].extend(first)
         durations['nonfirst'].extend(nonfirst)
         sizes['first'].extend([size] * len(first))
         sizes['nonfirst'].extend([size] * len(nonfirst))
         workers['first'].extend([app.slaves] * len(first))
         workers['nonfirst'].extend([app.slaves] * len(nonfirst))
     first = _get_df(workers, sizes, durations, 'first')
     nonfirst = _get_df(workers, sizes, durations, 'nonfirst')
     return first, nonfirst
Esempio n. 28
0
def main(grammarFile, tokenizer, testString):

    parser = Parser(grammarFile, tokenizer)

    testString = parser.process_input(testString)

    print('tokenization: ' +
          ' '.join(map(lambda s: s.replace('\'', ''), (testString))))

    parser.cfg.convertToCNF(allow_recursive_start=True,
                            allow_mixed_terms=False,
                            allow_epsilon_rules=True,
                            allow_unit_rules=True)

    if parser.cfg.findMember() == None:
        return None, INF

    def runTest(k):
        lev = levenshtein_automata(testString, k)
        inter = intersect(parser.cfg, lev)
        return inter.findMember()

    if runTest(0) != None:
        return testString, 0

    mn = 0  # exclusive
    mx = 1  # inclusive
    match = runTest(mx)
    while match == None:
        mn = mx
        mx *= 2
        match = runTest(mx)

    maxMatch = match
    while mx - mn > 1:
        h = (mx + mn) // 2
        match = runTest(h)
        if match == None:
            mn = h
        else:
            mx = h
            maxMatch = match

    return (maxMatch, mx)
Esempio n. 29
0
def main(workflow):
    # Try to read configuration from local disk
    config = workflow.stored_data("configuration")
    if config is None:
        Options.warning("Didn't find your configuration", "Please supply your cheat sheet path using 'cf ~/your/path'", workflow)
        workflow.send_feedback()
        return -1

    parser = Parser(config.getPath())
    # Note: by pasing workflow as a variable, its state is changed in Options.py logic
    options = Options(parser, workflow)

    # Query is whatever comes after "cheat". Stored in one single variable
    query = "" if len(workflow.args) == 0 else workflow.args[0]
    tokens = query.strip().split(" ", 1)  # len 2 list
    tokens = [i.strip() for i in tokens if i != ""]

    if len(tokens) == 0:
        options.showAvailable()
        workflow.send_feedback()
        return None

    if len(tokens) == 1 and tokens[0] == "--search":
        Options.hint("Globally searching for ...?", "In global mode", workflow)
        workflow.send_feedback()
        return None

    if len(tokens) == 1 and tokens[0] not in parser.availableSheets():
        options.showAvailable(tokens[0])
        workflow.send_feedback()
        return None

    if len(tokens) == 1:
        options.list(tokens[0])
        workflow.send_feedback()
        return None

    sheetName = None if tokens[0] == "--search" else tokens[0]
    searchTerm = tokens[1]
    options.searchInSheetByKeyword(sheetName, searchTerm)

    workflow.send_feedback()
    return None
Esempio n. 30
0
class Analytics:
    def __init__(self, config):
        self.config = config

        self.downloader = Downloader(config)
        self.parser = Parser(config)
        self.analyser = Analyser(config)

    def download_logs(self):
        print("here")
        if self.config.download:
            self.downloader.download()

    def generate_report(self):
        #if self.config.download:
        #    self.downloader.download()
        self.parser.parse()
        self.analyser.analyse()
        return self.config
Esempio n. 31
0
def main():
    text = ""
    with open("src/sample-source-codes/test5.cfpl", "r") as file:
        for line in file.readlines():
            text += line.lstrip() + "\\n"

    lexer = Lexer(text)
    parser = Parser(lexer)
    interpreter = Interpreter(parser)
    interpreter.interpret()
Esempio n. 32
0
    def _parse_calendars_process(input_tuple: (int, int, (int, str, str),
                                               datetime, dict)) -> dict:
        simple_logger = logging.getLogger(SIMPLE_LOGGER_PREFIX + __file__)

        input_index, total_length, calendar_tuple, timestamp, website_base = input_tuple
        calendar_id, calendar_url, calendar_html_file_path = calendar_tuple
        file = os.path.basename(calendar_html_file_path)

        info_output = "{}/{} | {}/{}".format(input_index, total_length,
                                             website_base["domain"], file)

        if not os.path.isfile(calendar_html_file_path):
            simple_logger.warning(info_output +
                                  " | 0 (File '{}' does not exist!)".format(
                                      calendar_html_file_path))
            return {calendar_id: []}

        with open(calendar_html_file_path, encoding="utf-8") as html_file:
            dom = etree.parse(html_file, etree.HTMLParser())

        parser = Parser(website_base["parser"])
        parser.set_dom(dom)

        events_to_insert = []
        for index, url_path in enumerate(parser.get_event_urls()):
            event_url = url_path
            if not bool(urllib.urlparse(event_url).netloc):
                event_url = urllib.urljoin(calendar_url, url_path)
            events_to_insert.append((event_url, timestamp))

        simple_logger.info(info_output + " | {}".format(len(events_to_insert)))

        if len(events_to_insert) == 0:
            if len(parser.error_messages) != 0:
                simple_logger.debug("Parser's errors: {}".format(
                    json.dumps(parser.error_messages, indent=4)))
        else:
            simple_logger.debug("Found URLs: {}".format(
                json.dumps([event_url for event_url, _ in events_to_insert],
                           indent=4)))

        return {calendar_id: events_to_insert}
Esempio n. 33
0
def get_row(log):
    """Return a row using only one LogParser instance."""
    parser = LogParser()
    app = parser.parse_file(log)
    input_bytes = app.bytes_read
    input_records = app.records_read
    sset = HBKmeansParser.get_set(input_records)

    stage_stats = StageStats.get_stats(app.stages)
    return [len(app.slaves), sset, input_bytes, input_records, app.duration,
            Parser.fits_in_memory(app)] + stage_stats
Esempio n. 34
0
    def run(self):
        """
            Entry point of the MSA functions.

            :param self: blabla
            :type self: object
            :return: If the function success or not.
            :rtype: Boolean
        """
        self.print_msa_logo()
        print "File information:"
        print "-----------------"
        print "-> path:\t\t" + self.malware_path \
                + "\n-> db table:\t\t" + self.db.malware_name
        print "-> Strings number:\t" + str(len(self.string_list)) + "\n"

        if self.vt.isActivate() == True:
            print "Virus total file analysis:"
            print "--------------------------"
            self.vt.file_analysis()

        isValid, error_msg = Parser.isValidBin(self.malware_path)
        if isValid == False:
            print error_msg
            sys.exit(2)

        print "\nStrings analysis:"
        print "-----------------"
        self.string_list = self.symbol.run_analysis(self.string_list)
        self.string_list = self.url.run_analysis(self.string_list)
        self.string_list = self.ipAddr.run_analysis(self.string_list)
        self.string_list = self.cmd.run_analysis(self.string_list)
        self.string_list = self.id.run_analysis(self.string_list)
        self.string_list = self.path.run_analysis(self.string_list)
        self.string_list = self.section.run_analysis(self.string_list)
        self.string_list = self.formatStr.run_analysis(self.string_list)
        self.string_list = self.string_list = self.msg.run_analysis(self.string_list)
        self.undef.run_analysis(self.string_list)

        print "\nVirus total URLs information:"
        print "-----------------------------"
        urls = list(set(self.db.getUrls()))
        for url in urls:
            self.vt.url_analysis(url)


        print "\nVirus total IP addresses information:"
        print "-------------------------------------"
        ips = list(set(self.db.getIpAddresses()))
        for ip in ips:
            self.vt.ip_analysis(ip)

        self.db.close()
        return True
Esempio n. 35
0
def compilation(file_path, out_path):
    parser = Parser()
    analyser = CodeAnalysis()
    flow_graph = FlowGraph()
    machine_code = MachineCode()

    with open(file_path, 'r') as f:
        content = f.read()

    try:
        ptree = parser.parse(content)
        symtab, ast = analyser.check(ptree)
        graph = flow_graph.convert(ast)
        code = machine_code.gen(graph, symtab)
    except YamcError:
        exit(1)

    with open(out_path, 'w') as f:
        for line in code:
            f.write(line + '\n')
Esempio n. 36
0
    def run(self):
        """
            Entry point of the MSA functions.

            :param self: blabla
            :type self: object
            :return: If the function success or not.
            :rtype: Boolean
        """
        self.print_msa_logo()
        print "File information:"
        print "-----------------"
        print "-> path:\t\t" + self.malware_path \
                + "\n-> db table:\t\t" + self.db.malware_name
        print "-> Strings number:\t" + str(len(self.string_list)) + "\n"

        if self.vt.isActivate() == True:
            print "Virus total file analysis:"
            print "--------------------------"
            self.vt.file_analysis()

        isValid, error_msg = Parser.isValidBin(self.malware_path)
        if isValid == False:
            print error_msg
            sys.exit(2)

        print "\nStrings analysis:"
        print "-----------------"
        self.string_list = self.symbol.run_analysis(self.string_list)
        self.string_list = self.url.run_analysis(self.string_list)
        self.string_list = self.ipAddr.run_analysis(self.string_list)
        self.string_list = self.cmd.run_analysis(self.string_list)
        self.string_list = self.id.run_analysis(self.string_list)
        self.string_list = self.path.run_analysis(self.string_list)
        self.string_list = self.section.run_analysis(self.string_list)
        self.string_list = self.formatStr.run_analysis(self.string_list)
        self.string_list = self.string_list = self.msg.run_analysis(
            self.string_list)
        self.undef.run_analysis(self.string_list)

        print "\nVirus total URLs information:"
        print "-----------------------------"
        urls = list(set(self.db.getUrls()))
        for url in urls:
            self.vt.url_analysis(url)

        print "\nVirus total IP addresses information:"
        print "-------------------------------------"
        ips = list(set(self.db.getIpAddresses()))
        for ip in ips:
            self.vt.ip_analysis(ip)

        self.db.close()
        return True
Esempio n. 37
0
def test_correct_syntax():
    testcase = """
//comment
typedef struct {
    int a;
    float b;
    /* some "multiline" comment */
    struct {
        long double c;
    } inner_struct_name;
    signed short d[123];
    unsigned char e;
} StructName;
    """
    lexer = Lexer(testcase)
    parser = Parser(lexer)
    structure = parser.parse()
    assert structure.name == "StructName"
    assert structure.description == "comment"
    assert len(structure.variables) == 5
Esempio n. 38
0
    def _parse_events_process(input_tuple: (int, int, (int, str, str, str), datetime, dict)) -> (dict, datetime, tuple):
        simple_logger = logging.getLogger(SIMPLE_LOGGER_PREFIX + __file__)

        input_index, total_length, event_tuple, timestamp, website_base = input_tuple
        event_html_id, event_html_file_path, event_url, _ = event_tuple

        info_output = "{}/{} | Parsing event: {}".format(input_index, total_length, event_url)

        if not event_html_file_path:
            simple_logger.error(info_output + " | NOK - (Filepath is None!)".format(event_url))
            return {"error": "Filepath is None!"}, timestamp, event_tuple

        elif not os.path.isfile(event_html_file_path):
            simple_logger.error(info_output + " | NOK - (File '{}' does not exist!)".format(event_html_file_path))
            return {"error": "File does not exist!"}, timestamp, event_tuple

        with open(event_html_file_path, encoding="utf-8") as html_file:
            dom = etree.parse(html_file, etree.HTMLParser(encoding="utf-8"))

        parser = Parser(website_base["parser"])
        parser.set_dom(dom)

        try:
            parsed_event_data = parser.get_event_data()
        except Exception as e:
            simple_logger.error(info_output + " | NOK (Exception: {})".format(str(e)))
            if len(parser.error_messages) != 0:
                simple_logger.debug("Parser's errors: {}".format(json.dumps(parser.error_messages, indent=4)))
            return {"error": "Exception occurred during parsing!"}, timestamp, event_tuple

        if len(parsed_event_data) == 0:
            simple_logger.error(info_output + " | NOK")
            if len(parser.error_messages) != 0:
                simple_logger.debug("Parser's errors: {}".format(json.dumps(parser.error_messages, indent=4)))
            return {"error": "No data were parsed!"}, timestamp, event_tuple

        simple_logger.info(info_output + " | OK")
        if len(parser.error_messages) != 0:
            simple_logger.debug("Parser's errors: {}".format(json.dumps(parser.error_messages, indent=4)))

        return parsed_event_data, timestamp, event_tuple
    def setUp(self):
        yaml_file = 'lib/configs/thredds_identifier.yaml'

        with open('tests/test_data/mod_stellwagen.xml', 'r') as f:
            content = f.read()
        url = 'http://stellwagen.er.usgs.gov/thredds/catalog/TSdata/catalog.xml'

        content = content.replace('\\n', '')
        parser = Parser(content)

        self.identifier = Identify([yaml_file], content, url,
                                   **{'parser': parser})
Esempio n. 40
0
def main():
    args = arg_parser()

    parser = Parser()
    flow = ControlFlowGraph()
    analyser = Analyser()
    codegen = MachineCode()

    with open(args.input_file, 'r') as file_in:
        content = file_in.read()

    try:
        tree = parser.parse(content)
        symtab, commands = analyser.analyse(tree)
        g = flow.convert(commands, symtab)
        code = codegen.start(g, symtab)
    except CompilerError:
        exit(1)

    with open(args.output_file, 'w') as file_out:
        file_out.write(code)
Esempio n. 41
0
def main():
    files = os.listdir(f"{ BASE_DIR }/src/sample-source-codes")
    for file_name in files:
        text = ""
        with open(f"src/sample-source-codes/{file_name}", "r") as file:
            for line in file.readlines():
                text += line.lstrip() + "\\n"

        lexer = Lexer(text)
        parser = Parser(lexer)
        interpreter = Interpreter(parser)
        interpreter.interpret()
    def setUp(self):
        yaml_file = 'tests/test_data/complex_identifier_test.yaml'

        with open('tests/test_data/wfs_v1_1_0.xml', 'r') as f:
            content = f.read()
        url = 'http://www.mapserver.com/cgi?SERVICE=WFS&VERSION=1.1.0&REQUEST=GETCAPABILITIES'

        content = content.replace('\\n', '')
        parser = Parser(content)

        self.identifier = Identify([yaml_file], content, url,
                                   **{'parser': parser})
Esempio n. 43
0
def test_unexpected_end_of_lexemes():
    testcase = """
    //comment
        typedef struct {
            int a;
            float b;
            /* some "multiline" comment */
            struct {
                long double c;
            } inner_struct_name[5];
            signed short d[123];
            unsigned char e;
        }
    """
    lexer = Lexer(testcase)
    parser = Parser(lexer)
    try:
        parser.parse()
    except ParserError as err:
        assert err.message.endswith('end of lexemes found')
    else:
        assert False
Esempio n. 44
0
class TestParser(TestBase):

    def setUp(self):
        self.parser = Parser()
        self.row = {'VAR': 1}
        TestBase.setUp(self)

    def _check_func(self, parse_result):
        func = parse_result
        self.assertEqual(func.func_name, '_eval')
        return parse_result

    def test_parse_formula(self):
        func = self._check_func(
                self.parser.parse_formula('VAR'))
        self.assertEqual(func(self.row, self.parser), 1)

    def test_bnf(self):
        result = self.parser.BNF()
        print type(self.parser.bnf)
        self.assertNotEqual(self.parser.bnf, None)

    def test_parse_formula_with_var(self):
        func = self._check_func(
                self.parser.parse_formula('VAR + 1'))
        self.assertEqual(func(self.row, self.parser), 2)

    def test_parse_formula_bad_formula(self):
        bad_formulas = [
            '=BAD +++ FOR',
            #'2 +>+ 1',
            #'1 ** 2',
        ]

        for bad_formula in bad_formulas:
            self.assertRaises(ParseError, self.parser.parse_formula,
                    bad_formula)
Esempio n. 45
0
    def load_strings(self):
        if not os.path.isfile(self.malware_path):
            print self.err_bad_path
            return False
        try:
            self.string_list = Parser.strings(self.malware_path)
        except:
            print self.err_string_recovery
            return False

        if self.string_list is None:
            print self.err_empty_bin
            return False

        return True
Esempio n. 46
0
File: uws.py Progetto: elcodigok/uws
	def do_set(self, person):
		if person and person in SET_OPTIONS:
			greeting = 'hi, %s!' % person
		elif person:
			greeting = "hello, " + person
		else:
			greeting = 'hello'

		p = Parser()
		print p.checkParameter(self.lastcmd.split())
		if (p.checkParameter(self.lastcmd.split()) == True):
			self.uws.setUrl(p.returnParameter(2))
		print p.countParameter(self.lastcmd.split())
		print greeting
Esempio n. 47
0
    def run_analysis(self, string_list):
        """
            This method is used by the core to run analyze and extract strings
            matching with the "command line" type.

            :param string_list: All strings to analyse.
            :type string_list: List
            :return: A list of string without strings previously matched.
            :rtype: List
        """
        epur_string_list = []

        self.bar.init(string_list, self.info_msg + "is running")
        for string in string_list:
            cmd_found = Parser.getCmd(string)
            if cmd_found != None:
                self.db.createEntry(string, cmd_found, self.type)
            else:
                epur_string_list.append(string)
            self.bar.update()
        self.bar.close(self.info_msg + "is complete", True)
        return epur_string_list
Esempio n. 48
0
    def run(self):
        """Parse logs and extract relevant information."""
        self.start()

        # CSV files
        csv_gen = CSVGen()
        app = HBSortParser.get_app()
        stage_titles = StageStats.get_titles(app.stages)
        header = ['workers', 'set', 'input_bytes', 'input_records',
                  'duration_ms', 'in_memory'] + stage_titles
        writer = csv_gen.get_writer(header, self.filename)

        for app in HBSortParser.get_apps():
            size = app.bytes_read
            sset = HBSortParser.get_set(size)
            stage_stats = StageStats.get_stats(app.stages)
            row = [len(app.slaves), sset, size, app.records_read, app.duration,
                   Parser.fits_in_memory(app)] + stage_stats
            writer.writerow(row)

        csv_gen.close()
        self.finish()
Esempio n. 49
0
    def main(self):
        """The main method.
        
        Returns the code to OS (used in sys.exit()).
        
        """
        
        # Iterates over all parameters.
        for parameter in range(1, len(sys.argv)):
            
            if (sys.argv[parameter] == "-c" or
                sys.argv[parameter] == "--csv"):
                
                # Check if destination directory exists. 
                # If not, create it.
                try:
                    if not exists(dirname(BASE_DIR_PATH)):
                        makedirs(dirname(BASE_DIR_PATH), mode=0755)
                
                except OSError:
                    print("LotoParser: Could not create working directory.")
                    return 1
                
                lf = LotoFile()
                files_with_errors = []
                
                print("Downloading files from Caixa's website...")
                for game in ZIP_FILES:
                    try:
                        lf.download(game, game)
                    
                    except (IOError, ContentTooShortError):
                        files_with_errors.append(game)
                
                if files_with_errors:
                    print("LotoParser: A download problem occurred with \
these files: {0}.".format(files_with_errors))
                
                files_with_errors = []  # Restarts errors.
                
                print("Unzipping downloaded files...")
                for game in ZIP_FILES:
                    try:
                        lf.unzip(game)
                    except (IOError, BadZipfile):
                        files_with_errors.append(game)
                    
                if files_with_errors:
                    print("LotoParser: A unzip problem occurred with these \
files: {0}.".format(files_with_errors))
                
                files_with_errors = []  # Restarts errors.
                p = Parser()
                index = -1
                
                print("Parsing HTML files...")
                for game in HTM_FILES:
                    try:
                        index += 1
                        p.csv_parser(BASE_DIR_PATH + game, 
                            BASE_DIR_PATH + OUT_FILES[index] + ".csv")
                    except IOError:
                        files_with_errors.append(game)
                
                if files_with_errors:
                    print("LotoParser: A parse problem occurred with these \
files: {0}.".format(files_with_errors))
                
                files_with_errors = []  # Restarts errors.
                
                print("Removing junk files...")
                for index in range(9):
                    try:
                        lf.remove(ZIP_FILES[index])
                    except OSError:
                        files_with_errors.append(ZIP_FILES[index])
                    
                    try:
                        lf.remove(HTM_FILES[index])
                    except OSError:
                        files_with_errors.append(HTM_FILES[index])
                    
                    try:
                        lf.remove(PIC_FILES[index])
                    except OSError:
                        files_with_errors.append(PIC_FILES[index])
                    
                if files_with_errors:
                    print("LotoParser: A removing problem occurred with \
these files: {0}.".format(files_with_errors))
                
                return 0
            
            elif (sys.argv[parameter] == "-x" or
                sys.argv[parameter] == "--xml"):
                
                print("LotoParser: CSV files must be generated first.")
                
                print("Generating MegaSena's XML file...")
                Parser().xml_parser("./loto/megasena.csv", 
                    "./loto/megasena.xml")
                print("Generating LotoFacil's XML file...")
                Parser().xml_parser("./loto/lotofacil.csv", 
                    "./loto/lotofacil.xml")
                print("Generating Quina's XML file...")
                Parser().xml_parser("./loto/quina.csv", 
                    "./loto/quina.xml")
                print("Generating LotoMania's XML file...")
                Parser().xml_parser("./loto/lotomania.csv", 
                    "./loto/lotomania.xml")
                print("Generating DuplaSena's XML file...")
                Parser().xml_parser("./loto/duplasena.csv", 
                    "./loto/duplasena.xml")
                print("Generating Federal's XML file...")
                Parser().xml_parser("./loto/federal.csv", 
                    "./loto/federal.xml")
                print("Generating LotoGol's XML file...")
                Parser().xml_parser("./loto/lotogol.csv", 
                    "./loto/lotogol.xml")
                print("Generating Loteca's XML file...")
                Parser().xml_parser("./loto/loteca.csv", 
                    "./loto/loteca.xml")
                print("Generating TimeMania's XML file...")
                Parser().xml_parser("./loto/timemania.csv", 
                    "./loto/timemania.xml")
                
                return 0
            
            elif (sys.argv[parameter] == "-h" or
                sys.argv[parameter] == "--help"):
                print(self.help())
                return 0
            
            elif (sys.argv[parameter] == "-v" or 
                sys.argv[parameter] == "--version"):
                print(self.version())
                return 0
            
            else:
                # Invalid parameter.
                # Finnishes loop to display help.
                break
            
        # No parameters were given.
        print(self.help())
        return 1    
Esempio n. 50
0
#   Read in file for processing...
files = glob.glob("data/" + str(sys.argv[1]) + "/*.txt")
no_errors = True
for filename in files:

    file = open(filename, "r")
    if not file:
        util.error("File could not be loaded... Exiting...", True)

    lexical_analyzer = LexicalAnalyzer()
    tokens = lexical_analyzer.process_file(file)

    if len(tokens) > 0:
        tokens.reverse()

    parser = Parser(tokens)
    parse_result = parser.parse()

    should_fail = "-fail" in filename
    error = False

    if should_fail and parse_result == "ACCEPT":
        error = True
    elif not should_fail and parse_result == "REJECT":
        error = True

    if error:
        if no_errors:
            print("Invalid Tests:")

        no_errors = False
Esempio n. 51
0
 def __init__(self, threads, block_size, path=None):
     self.threads = threads
     self.block_size = block_size
     self._path = path
     self._parser = Parser()
Esempio n. 52
0
	def __init__(self, screen_name):
		Parser.__init__(self)
		
		self.screen_name = screen_name
		self.belle = Belle()
Esempio n. 53
0
import sys
import time
import urllib2

from lib.parser import Parser

if __name__ == "__main__":

    CUR_DIR = os.path.dirname(os.path.realpath(__file__))

    # Namespaces for XML
    ATOM_NS = "{http://www.w3.org/2005/Atom}"
    CAP_NS = "{urn:oasis:names:tc:emergency:cap:1.1}"

    # Instantiate the parser
    parser = Parser(CUR_DIR)

    # Try to load the previous alerts
    previous_alerts_filepath = os.path.join(parser.output_dir, 'alerts.json')
    if os.path.exists(previous_alerts_filepath):
        previous_alerts_dict = parser.load_json(previous_alerts_filepath)
        parser.previous_alerts_list = previous_alerts_dict['alerts']
    else:
        parser.previous_alerts_list = []

    # Parse the XML
    try:
        noaa_url = "http://alerts.weather.gov/cap/us.php?x=1"
        f = urllib2.urlopen(noaa_url, timeout=10)
        request_data = f.read()
        tree = parser.load_xml_from_url_contents(request_data)
Esempio n. 54
0
 def get_apps():
     """Return a generator for all apps."""
     parser = Parser()
     apps = parser.get_apps('hibench', 'sort', '*')
     return (app for app in apps if len(app.slaves) not in [12, 123])
Esempio n. 55
0
 def setUp(self):
     self.parser = Parser()
     self.row = {'VAR': 1}
     TestBase.setUp(self)