def main(workflow):
query = "" if len(workflow.args) == 0 else workflow.args[0]
config = workflow.stored_data("configuration")
if config == None:
Options.warning(
"Didn't find your configuration",
"Please supply your cheat sheet path using 'cf ~/your/path'",
workflow)
workflow.send_feedback()
return -1
parser = Parser(config.getPath())
options = Options(parser, workflow)
tokens = query.strip().split(" ", 1)
tokens = [i.strip() for i in tokens if i != ""]
if len(tokens) < 2:
sheetName = "" if len(tokens) == 0 else tokens[0]
handler = options.showAvailable if sheetName not in parser.availableSheets(
) else options.list
handler(sheetName)
else:
sheetName = tokens[0]
searchTerm = tokens[1]
if sheetName == "--search":
options.search(None, searchTerm)
else:
options.search(sheetName, searchTerm)
workflow.send_feedback()
return None
def run_compiler(source, target, debug=False):
"""Run Compiler
Executes the compilation process given a source file path.
Arguments:
source: The source file to compile.
target: The destination binary executable file.
debug: If True, verbose parsing details are shown. (Default: False)
Returns:
True on success, False otherwise.
"""
# Define a temporary location for the intermediate C code
TMP_CODE_FILE = './ir.c'
# Create a Parser object to parse the inputted source file
parser = Parser(debug)
# Parse the source file to the temporary code file
if not parser.parse(source, TMP_CODE_FILE):
print('Error while parsing "%s"' % source)
return False
# Set up gcc compilation command
gcc_cmd = ['gcc', '-m32', '-o', target, TMP_CODE_FILE]
# Compile the temporary file with gcc. Output to the target location
if subprocess.call(gcc_cmd) != 0:
print('Error while compiling "%s"' % target)
return False
return True
def _parse_calendars_process(input_tuple: (int, int, (int, str, str),
datetime, dict)) -> dict:
input_index, total_length, calendar_tuple, timestamp, website_base = input_tuple
calendar_id, calendar_url, calendar_html_file_path = calendar_tuple
file = os.path.basename(calendar_html_file_path)
debug_output = "{}/{} | {}/{}".format(input_index, total_length,
website_base["domain"], file)
if not os.path.isfile(calendar_html_file_path):
debug_output += " | 0 (File '{}' does not exist!)".format(
calendar_html_file_path)
print(debug_output)
return {calendar_id: []}
with open(calendar_html_file_path, encoding="utf-8") as html_file:
dom = etree.parse(html_file, etree.HTMLParser())
parser = Parser(website_base["parser"])
parser.set_dom(dom)
event_urls = parser.get_event_urls()
events_to_insert = []
for index, url_path in enumerate(event_urls):
event_url = urllib.urljoin(calendar_url, url_path)
events_to_insert.append((event_url, timestamp))
debug_output += " | {}".format(len(events_to_insert))
print(debug_output)
return {calendar_id: events_to_insert}
def run_compiler(input_name, output_name):
parser = Parser()
analyzer = StaticAnalyzer()
flow_generator = FlowGraph()
code_generator = CodeGenerator()
source_code = ''
try:
with open(input_name, 'r') as f:
source_code = f.read()
except FileNotFoundError:
logging.error("File not found")
exit(1)
try:
# group tokens into syntactical units using parser
parse_tree = parser.parse(source_code)
# perform semantic analyze
symtab, ast = analyzer.analyze(parse_tree)
# generate flow graph
flow_graph = flow_generator.generate(ast)
# generate code
code = code_generator.generate(flow_graph, symtab)
with open(output_name, 'w') as f:
f.write(str(code))
except CompilerError as error:
if str(error):
logging.error("COMPILER_ERROR: {0}".format(str(error)))
exit(1)
def run_compiler(source, target, debug=False):
"""Run Compiler
Executes the compilation process given a source file path.
Arguments:
source: The source file to compile.
target: The destination binary executable file.
debug: If True, verbose parsing details are shown. (Default: False)
Returns:
True on success, False otherwise.
"""
# Define a temporary location for the intermediate C code
TMP_CODE_FILE = "./ir.c"
# Create a Parser object to parse the inputted source file
parser = Parser(debug)
# Parse the source file to the temporary code file
if not parser.parse(source, TMP_CODE_FILE):
print('Error while parsing "%s"' % source)
return False
# Set up gcc compilation command
gcc_cmd = ["gcc", "-m32", "-o", target, TMP_CODE_FILE]
# Compile the temporary file with gcc. Output to the target location
if subprocess.call(gcc_cmd) != 0:
print('Error while compiling "%s"' % target)
return False
return True
def setUp(self):
yaml_files = [
'lib/configs/ogc_identifier.yaml',
'lib/configs/iso_identifier.yaml'
]
# set up for the known csw getcapabilities
with open('tests/test_data/cwic_csw_v2_0_2.xml', 'r') as f:
csw_content = f.read()
csw_url = 'http://www.mapserver.com/cgi?SERVICE=WCS&VERSION=2.0.2&REQUEST=GETCAPABILITIES'
csw_content = csw_content.replace('\\n', '')
csw_parser = Parser(csw_content)
self.csw_identifier = Identify(yaml_files, csw_content, csw_url,
**{'parser': csw_parser})
# set up for the geonetwork mismatched namespacing iso issue
with open('tests/test_data/geonetwork_iso_NOT_csw.xml', 'r') as f:
iso_content = f.read()
iso_url = 'http://catalog.data.gov/harvest/object/d5de6dde-3042-4daf-b4ba-95e21e3ab343'
iso_content = iso_content.replace('\\n', '')
iso_parser = Parser(iso_content)
self.iso_identifier = Identify(yaml_files, iso_content, iso_url,
**{'parser': iso_parser})
def output_pharser_result(format_file_input, data_file_input):
format_file = mockFileName(format_file_input)
data_file = mockFileName(data_file_input)
parser = Parser(format_file)
parser.loadData(data_file)
return parser.asObject()
def _build_all_tasks(self):
parser = Parser()
apps = parser.parse_folder(self._folder)
first, nonfirst, columns = self._get_task_records(apps)
return (pd.DataFrame.from_records(first,
columns=columns),
pd.DataFrame.from_records(nonfirst,
columns=columns))
def test_get_pages(self):
page = PageLoader()
response = page.get_page()
parser = Parser(response.text)
links = parser.get_pages()
result = False
if len(links) > 0:
result = True
self.assertTrue(result)
def _build(self):
all_durs, all_sizes, all_workers = [], [], []
parser = Parser()
apps = parser.parse_folder(self._folder)
sizes = self._get_app_sizes()
for app, size in zip(apps, sizes):
duration = self._get_app_duration(app)
all_durs.append(duration)
all_sizes.append(size)
all_workers.append(app.slaves)
return _get_df(all_workers, all_sizes, all_durs)
def do_get(self, person): if person and person in SET_OPTIONS: greeting = 'hi, %s!' % person elif person: greeting = "hello, " + person else: greeting = 'hello' p = Parser() print p.checkParameter(self.lastcmd.split()) print greeting
def do_show(self, line): if line and line in SHOW_OPTIONS: greeting = 'hi, %s!' % line elif line: greeting = self.uws.getUrl() else: greeting = 'URL\t=>\t' + self.uws.getUrl() p = Parser() p.checkParameter(self.lastcmd.split()) print greeting
def main():
parser = Parser()
# ms.printDB('lpmdb.bin')
# ms.json_to_lpmdb_bin('lpmdb.json', 'lpmdb.bin')
# mv = ms.readMovieByPos('lpmdb.bin', 11616)
# print(mv.title)
# parser.parse(':: from title filter the')
while parser.parse(input()): # returns false if input() returns "exit"
pass
def __init__(self):
self.arguments = Arguments()
self.blender = Blender()
self.parser = Parser()
self.botfile = self.arguments.getBotFile()
self.blender.unselectEverything()
print("\nNow building " + self.botfile + "...")
self.cubeData = self.parser.parseBotFile(self.botfile)
self.cubedatabase = self.parser.parseCSVFile("assets/cubes.csv")
self.blender.build(self.cubeData, self.cubedatabase)
print("done!")
class TestParser(unittest.TestCase):
def setUp(self):
'''
we are assuming input from the solr sample parser
so this is the encoded, cdata-removed input
except i have blended the two a bit (handling that \\n issue
in the parser vs the cdata issue in the solr response)
so init the parser with a file that reflects that
'''
with open('tests/test_data/basic_osdd_c1576284036448b5ef3d16b2cd37acbc.txt', 'r') as f:
data = f.read()
data = data.replace('\\n', ' ')
self.parser = Parser(data)
def test_parse_xml(self):
'''
the _parse is called from init
'''
self.assertTrue(self.parser.xml is not None)
self.assertTrue(len(self.parser._namespaces) > 0)
def test_remap_namespaced_paths(self):
# basic check
in_xpath = '{http://a9.com/-/spec/opensearch/1.1/}OpenSearchDescription/{http://a9.com/-/spec/opensearch/1.1/}Image'
out_xpath = 'default:OpenSearchDescription/default:Image'
test_xpath = self.parser._remap_namespaced_xpaths(in_xpath)
self.assertTrue(test_xpath == out_xpath)
def test_find_nodes(self):
nodes = self.parser.find_nodes()
self.assertTrue(len(nodes) == 5)
self.assertTrue(len(nodes[2]) == 3)
self.assertTrue(nodes[1]['text'] == 'UTF-8')
# run with excludes
excludes = [
'{http://a9.com/-/spec/opensearch/1.1/}OpenSearchDescription/{http://a9.com/-/spec/opensearch/1.1/}InputEncoding',
'{http://a9.com/-/spec/opensearch/1.1/}OpenSearchDescription/{http://a9.com/-/spec/opensearch/1.1/}Image/@width'
]
nodes = self.parser.find_nodes(excludes)
self.assertTrue(len(nodes) == 4)
self.assertTrue(len(nodes[1]) == 3)
self.assertTrue(nodes[0]['text'] == 'CEOS')
self.assertTrue(nodes[1]['attributes'][1]['text'] == '16')
self.assertTrue(nodes[1]['attributes'][0]['xpath'] == '{http://a9.com/-/spec/opensearch/1.1/}OpenSearchDescription/{http://a9.com/-/spec/opensearch/1.1/}Image/@type')
def test_without_typedef():
testcase = """
//comment
struct FirstName {
int foo;
float bar[1];
} NameAlias;
"""
lexer = Lexer(testcase)
parser = Parser(lexer)
structure = parser.parse()
assert structure is not None
def test_multiple_names():
testcase = """
//comment
typedef struct StructName {
int foo;
float bar[MAX_SIZE];
} StructNameAlias;
"""
lexer = Lexer(testcase)
parser = Parser(lexer)
structure = parser.parse()
assert structure.name == 'StructNameAlias'
def calculate_column(dataset, dframe, formula, name):
"""
For calculating new columns.
Get necessary data given a calculation ID, execute calculation formula,
store results in dataset the calculation refers to.
"""
# parse formula into function and variables
parser = Parser()
func = parser.parse_formula(formula)
new_column = dframe.apply(func, axis=1, args=(parser, ))
new_column.name = name
return Observation.update(dframe.join(new_column), dataset)
class Analytics:
def __init__(self, config):
self.config = config
self.downloader = Downloader(config)
self.parser = Parser(config)
self.analyser = Analyser(config)
def generate_report(self):
if self.config.download:
self.downloader.download()
self.parser.parse()
self.analyser.analyse()
return self.config
def setUp(self):
data = '''<xml>
<node>Does it parse? <br/> It <em>should</em>!</node>
<nextnode>Wow, that's a typography sin right there, but <a href="#anchor">Nope</a> and <span>Many XML</span>.</nextnode>
</xml>
'''
self.parser = Parser(data)
def setUp(self):
'''
we are assuming input from the solr sample parser
so this is the encoded, cdata-removed input
except i have blended the two a bit (handling that \\n issue
in the parser vs the cdata issue in the solr response)
so init the parser with a file that reflects that
'''
with open('tests/test_data/basic_osdd_c1576284036448b5ef3d16b2cd37acbc.txt', 'r') as f:
data = f.read()
data = data.replace('\\n', ' ')
self.parser = Parser(data)
class ExperimentInfo:
def __init__(self, threads, block_size, path=None):
self.threads = threads
self.block_size = block_size
self._path = path
self._parser = Parser()
def get_n_blocks(self, input_size):
return np.ceil(input_size / self.block_size)
def get_n_tasks(self, stage):
task_amounts = self._get_ns_tasks(stage)
assert len(task_amounts) == 1, 'More than 1 value: {}'.format(
task_amounts)
return task_amounts[0]
def _get_ns_tasks(self, stage):
amounts = set()
for app in self._get_apps():
tasks = app.stages[stage].tasks
n = sum(1 for t in tasks if not t.failed)
amounts.add(len(app.stages[stage].tasks))
return list(amounts)
def _get_apps(self):
return self._parser.parse_folder(self._path)
def run_analysis(self, string_list):
"""
This method is used by the core to run analyze and extract strings
matching with the "elf symbols" type.
:param string_list: All strings to analyse.
:type string_list: List
:return: A list of string without strings previously matched.
:rtype: List
"""
epur_string_list = []
ret = subprocess.Popen(["readelf", "-s", self.malware_path],
stdout=subprocess.PIPE)
readelf_output = ret.communicate()[0].split('\n')[3:]
self.bar.init(string_list, self.info_msg + "is running")
for string in string_list:
sym_found, specific_type = Parser.getSymbol(string, readelf_output)
if sym_found != None:
self.db.createEntry(string, sym_found,
self.type + "-" + specific_type)
else:
epur_string_list.append(string)
self.bar.update()
self.bar.close(self.info_msg + "is complete", True)
return epur_string_list
def test_comment_before_bracket():
testcase = """
//comment
typedef struct // comment before opening bracket
{
int a;
unsigned long long int b;
/* comment
before
closing bracket */
} StructName;
"""
lexer = Lexer(testcase)
parser = Parser(lexer)
structure = parser.parse()
assert structure is not None
def run_analysis(self, string_list):
"""
This method is used by the core to run analyze and extract strings
matching with the "elf symbols" type.
:param string_list: All strings to analyse.
:type string_list: List
:return: A list of string without strings previously matched.
:rtype: List
"""
epur_string_list = []
ret = subprocess.Popen(["readelf", "-s", self.malware_path],
stdout=subprocess.PIPE)
readelf_output = ret.communicate()[0].split('\n')[3:]
self.bar.init(string_list, self.info_msg + "is running")
for string in string_list:
sym_found, specific_type = Parser.getSymbol(string, readelf_output)
if sym_found != None:
self.db.createEntry(string, sym_found,
self.type + "-" + specific_type)
else:
epur_string_list.append(string)
self.bar.update()
self.bar.close(self.info_msg + "is complete", True)
return epur_string_list
def _write_set(self, sset, apps_sizes):
for app, size in apps_sizes:
stage_stats = StageStats.get_stats(app.stages)
# Bug in bytes read metrics reported by Spark
stage_stats[1] = size
self._writer.writerow([len(app.slaves), sset, size,
app.records_read, app.duration,
Parser.fits_in_memory(app)] + stage_stats)
def _build_first_nonfirst_tasks(self):
durations = {'first': [], 'nonfirst': []}
sizes = {'first': [], 'nonfirst': []}
workers = {'first': [], 'nonfirst': []}
parser = Parser()
apps = parser.parse_folder(self._folder)
apps_sizes = self._get_app_sizes()
for app, size in zip(apps, apps_sizes):
first, nonfirst = self._get_first_nonfirst_tasks_durations(app)
durations['first'].extend(first)
durations['nonfirst'].extend(nonfirst)
sizes['first'].extend([size] * len(first))
sizes['nonfirst'].extend([size] * len(nonfirst))
workers['first'].extend([app.slaves] * len(first))
workers['nonfirst'].extend([app.slaves] * len(nonfirst))
first = _get_df(workers, sizes, durations, 'first')
nonfirst = _get_df(workers, sizes, durations, 'nonfirst')
return first, nonfirst
def main(grammarFile, tokenizer, testString):
parser = Parser(grammarFile, tokenizer)
testString = parser.process_input(testString)
print('tokenization: ' +
' '.join(map(lambda s: s.replace('\'', ''), (testString))))
parser.cfg.convertToCNF(allow_recursive_start=True,
allow_mixed_terms=False,
allow_epsilon_rules=True,
allow_unit_rules=True)
if parser.cfg.findMember() == None:
return None, INF
def runTest(k):
lev = levenshtein_automata(testString, k)
inter = intersect(parser.cfg, lev)
return inter.findMember()
if runTest(0) != None:
return testString, 0
mn = 0 # exclusive
mx = 1 # inclusive
match = runTest(mx)
while match == None:
mn = mx
mx *= 2
match = runTest(mx)
maxMatch = match
while mx - mn > 1:
h = (mx + mn) // 2
match = runTest(h)
if match == None:
mn = h
else:
mx = h
maxMatch = match
return (maxMatch, mx)
def main(workflow):
# Try to read configuration from local disk
config = workflow.stored_data("configuration")
if config is None:
Options.warning("Didn't find your configuration", "Please supply your cheat sheet path using 'cf ~/your/path'", workflow)
workflow.send_feedback()
return -1
parser = Parser(config.getPath())
# Note: by pasing workflow as a variable, its state is changed in Options.py logic
options = Options(parser, workflow)
# Query is whatever comes after "cheat". Stored in one single variable
query = "" if len(workflow.args) == 0 else workflow.args[0]
tokens = query.strip().split(" ", 1) # len 2 list
tokens = [i.strip() for i in tokens if i != ""]
if len(tokens) == 0:
options.showAvailable()
workflow.send_feedback()
return None
if len(tokens) == 1 and tokens[0] == "--search":
Options.hint("Globally searching for ...?", "In global mode", workflow)
workflow.send_feedback()
return None
if len(tokens) == 1 and tokens[0] not in parser.availableSheets():
options.showAvailable(tokens[0])
workflow.send_feedback()
return None
if len(tokens) == 1:
options.list(tokens[0])
workflow.send_feedback()
return None
sheetName = None if tokens[0] == "--search" else tokens[0]
searchTerm = tokens[1]
options.searchInSheetByKeyword(sheetName, searchTerm)
workflow.send_feedback()
return None
class Analytics:
def __init__(self, config):
self.config = config
self.downloader = Downloader(config)
self.parser = Parser(config)
self.analyser = Analyser(config)
def download_logs(self):
print("here")
if self.config.download:
self.downloader.download()
def generate_report(self):
#if self.config.download:
# self.downloader.download()
self.parser.parse()
self.analyser.analyse()
return self.config
def main():
text = ""
with open("src/sample-source-codes/test5.cfpl", "r") as file:
for line in file.readlines():
text += line.lstrip() + "\\n"
lexer = Lexer(text)
parser = Parser(lexer)
interpreter = Interpreter(parser)
interpreter.interpret()
def _parse_calendars_process(input_tuple: (int, int, (int, str, str),
datetime, dict)) -> dict:
simple_logger = logging.getLogger(SIMPLE_LOGGER_PREFIX + __file__)
input_index, total_length, calendar_tuple, timestamp, website_base = input_tuple
calendar_id, calendar_url, calendar_html_file_path = calendar_tuple
file = os.path.basename(calendar_html_file_path)
info_output = "{}/{} | {}/{}".format(input_index, total_length,
website_base["domain"], file)
if not os.path.isfile(calendar_html_file_path):
simple_logger.warning(info_output +
" | 0 (File '{}' does not exist!)".format(
calendar_html_file_path))
return {calendar_id: []}
with open(calendar_html_file_path, encoding="utf-8") as html_file:
dom = etree.parse(html_file, etree.HTMLParser())
parser = Parser(website_base["parser"])
parser.set_dom(dom)
events_to_insert = []
for index, url_path in enumerate(parser.get_event_urls()):
event_url = url_path
if not bool(urllib.urlparse(event_url).netloc):
event_url = urllib.urljoin(calendar_url, url_path)
events_to_insert.append((event_url, timestamp))
simple_logger.info(info_output + " | {}".format(len(events_to_insert)))
if len(events_to_insert) == 0:
if len(parser.error_messages) != 0:
simple_logger.debug("Parser's errors: {}".format(
json.dumps(parser.error_messages, indent=4)))
else:
simple_logger.debug("Found URLs: {}".format(
json.dumps([event_url for event_url, _ in events_to_insert],
indent=4)))
return {calendar_id: events_to_insert}
def get_row(log):
"""Return a row using only one LogParser instance."""
parser = LogParser()
app = parser.parse_file(log)
input_bytes = app.bytes_read
input_records = app.records_read
sset = HBKmeansParser.get_set(input_records)
stage_stats = StageStats.get_stats(app.stages)
return [len(app.slaves), sset, input_bytes, input_records, app.duration,
Parser.fits_in_memory(app)] + stage_stats
def run(self):
"""
Entry point of the MSA functions.
:param self: blabla
:type self: object
:return: If the function success or not.
:rtype: Boolean
"""
self.print_msa_logo()
print "File information:"
print "-----------------"
print "-> path:\t\t" + self.malware_path \
+ "\n-> db table:\t\t" + self.db.malware_name
print "-> Strings number:\t" + str(len(self.string_list)) + "\n"
if self.vt.isActivate() == True:
print "Virus total file analysis:"
print "--------------------------"
self.vt.file_analysis()
isValid, error_msg = Parser.isValidBin(self.malware_path)
if isValid == False:
print error_msg
sys.exit(2)
print "\nStrings analysis:"
print "-----------------"
self.string_list = self.symbol.run_analysis(self.string_list)
self.string_list = self.url.run_analysis(self.string_list)
self.string_list = self.ipAddr.run_analysis(self.string_list)
self.string_list = self.cmd.run_analysis(self.string_list)
self.string_list = self.id.run_analysis(self.string_list)
self.string_list = self.path.run_analysis(self.string_list)
self.string_list = self.section.run_analysis(self.string_list)
self.string_list = self.formatStr.run_analysis(self.string_list)
self.string_list = self.string_list = self.msg.run_analysis(self.string_list)
self.undef.run_analysis(self.string_list)
print "\nVirus total URLs information:"
print "-----------------------------"
urls = list(set(self.db.getUrls()))
for url in urls:
self.vt.url_analysis(url)
print "\nVirus total IP addresses information:"
print "-------------------------------------"
ips = list(set(self.db.getIpAddresses()))
for ip in ips:
self.vt.ip_analysis(ip)
self.db.close()
return True
def compilation(file_path, out_path):
parser = Parser()
analyser = CodeAnalysis()
flow_graph = FlowGraph()
machine_code = MachineCode()
with open(file_path, 'r') as f:
content = f.read()
try:
ptree = parser.parse(content)
symtab, ast = analyser.check(ptree)
graph = flow_graph.convert(ast)
code = machine_code.gen(graph, symtab)
except YamcError:
exit(1)
with open(out_path, 'w') as f:
for line in code:
f.write(line + '\n')
def run(self):
"""
Entry point of the MSA functions.
:param self: blabla
:type self: object
:return: If the function success or not.
:rtype: Boolean
"""
self.print_msa_logo()
print "File information:"
print "-----------------"
print "-> path:\t\t" + self.malware_path \
+ "\n-> db table:\t\t" + self.db.malware_name
print "-> Strings number:\t" + str(len(self.string_list)) + "\n"
if self.vt.isActivate() == True:
print "Virus total file analysis:"
print "--------------------------"
self.vt.file_analysis()
isValid, error_msg = Parser.isValidBin(self.malware_path)
if isValid == False:
print error_msg
sys.exit(2)
print "\nStrings analysis:"
print "-----------------"
self.string_list = self.symbol.run_analysis(self.string_list)
self.string_list = self.url.run_analysis(self.string_list)
self.string_list = self.ipAddr.run_analysis(self.string_list)
self.string_list = self.cmd.run_analysis(self.string_list)
self.string_list = self.id.run_analysis(self.string_list)
self.string_list = self.path.run_analysis(self.string_list)
self.string_list = self.section.run_analysis(self.string_list)
self.string_list = self.formatStr.run_analysis(self.string_list)
self.string_list = self.string_list = self.msg.run_analysis(
self.string_list)
self.undef.run_analysis(self.string_list)
print "\nVirus total URLs information:"
print "-----------------------------"
urls = list(set(self.db.getUrls()))
for url in urls:
self.vt.url_analysis(url)
print "\nVirus total IP addresses information:"
print "-------------------------------------"
ips = list(set(self.db.getIpAddresses()))
for ip in ips:
self.vt.ip_analysis(ip)
self.db.close()
return True
def test_correct_syntax():
testcase = """
//comment
typedef struct {
int a;
float b;
/* some "multiline" comment */
struct {
long double c;
} inner_struct_name;
signed short d[123];
unsigned char e;
} StructName;
"""
lexer = Lexer(testcase)
parser = Parser(lexer)
structure = parser.parse()
assert structure.name == "StructName"
assert structure.description == "comment"
assert len(structure.variables) == 5
def _parse_events_process(input_tuple: (int, int, (int, str, str, str), datetime, dict)) -> (dict, datetime, tuple):
simple_logger = logging.getLogger(SIMPLE_LOGGER_PREFIX + __file__)
input_index, total_length, event_tuple, timestamp, website_base = input_tuple
event_html_id, event_html_file_path, event_url, _ = event_tuple
info_output = "{}/{} | Parsing event: {}".format(input_index, total_length, event_url)
if not event_html_file_path:
simple_logger.error(info_output + " | NOK - (Filepath is None!)".format(event_url))
return {"error": "Filepath is None!"}, timestamp, event_tuple
elif not os.path.isfile(event_html_file_path):
simple_logger.error(info_output + " | NOK - (File '{}' does not exist!)".format(event_html_file_path))
return {"error": "File does not exist!"}, timestamp, event_tuple
with open(event_html_file_path, encoding="utf-8") as html_file:
dom = etree.parse(html_file, etree.HTMLParser(encoding="utf-8"))
parser = Parser(website_base["parser"])
parser.set_dom(dom)
try:
parsed_event_data = parser.get_event_data()
except Exception as e:
simple_logger.error(info_output + " | NOK (Exception: {})".format(str(e)))
if len(parser.error_messages) != 0:
simple_logger.debug("Parser's errors: {}".format(json.dumps(parser.error_messages, indent=4)))
return {"error": "Exception occurred during parsing!"}, timestamp, event_tuple
if len(parsed_event_data) == 0:
simple_logger.error(info_output + " | NOK")
if len(parser.error_messages) != 0:
simple_logger.debug("Parser's errors: {}".format(json.dumps(parser.error_messages, indent=4)))
return {"error": "No data were parsed!"}, timestamp, event_tuple
simple_logger.info(info_output + " | OK")
if len(parser.error_messages) != 0:
simple_logger.debug("Parser's errors: {}".format(json.dumps(parser.error_messages, indent=4)))
return parsed_event_data, timestamp, event_tuple
def setUp(self):
yaml_file = 'lib/configs/thredds_identifier.yaml'
with open('tests/test_data/mod_stellwagen.xml', 'r') as f:
content = f.read()
url = 'http://stellwagen.er.usgs.gov/thredds/catalog/TSdata/catalog.xml'
content = content.replace('\\n', '')
parser = Parser(content)
self.identifier = Identify([yaml_file], content, url,
**{'parser': parser})
def main():
args = arg_parser()
parser = Parser()
flow = ControlFlowGraph()
analyser = Analyser()
codegen = MachineCode()
with open(args.input_file, 'r') as file_in:
content = file_in.read()
try:
tree = parser.parse(content)
symtab, commands = analyser.analyse(tree)
g = flow.convert(commands, symtab)
code = codegen.start(g, symtab)
except CompilerError:
exit(1)
with open(args.output_file, 'w') as file_out:
file_out.write(code)
def main():
files = os.listdir(f"{ BASE_DIR }/src/sample-source-codes")
for file_name in files:
text = ""
with open(f"src/sample-source-codes/{file_name}", "r") as file:
for line in file.readlines():
text += line.lstrip() + "\\n"
lexer = Lexer(text)
parser = Parser(lexer)
interpreter = Interpreter(parser)
interpreter.interpret()
def setUp(self):
yaml_file = 'tests/test_data/complex_identifier_test.yaml'
with open('tests/test_data/wfs_v1_1_0.xml', 'r') as f:
content = f.read()
url = 'http://www.mapserver.com/cgi?SERVICE=WFS&VERSION=1.1.0&REQUEST=GETCAPABILITIES'
content = content.replace('\\n', '')
parser = Parser(content)
self.identifier = Identify([yaml_file], content, url,
**{'parser': parser})
def test_unexpected_end_of_lexemes():
testcase = """
//comment
typedef struct {
int a;
float b;
/* some "multiline" comment */
struct {
long double c;
} inner_struct_name[5];
signed short d[123];
unsigned char e;
}
"""
lexer = Lexer(testcase)
parser = Parser(lexer)
try:
parser.parse()
except ParserError as err:
assert err.message.endswith('end of lexemes found')
else:
assert False
class TestParser(TestBase):
def setUp(self):
self.parser = Parser()
self.row = {'VAR': 1}
TestBase.setUp(self)
def _check_func(self, parse_result):
func = parse_result
self.assertEqual(func.func_name, '_eval')
return parse_result
def test_parse_formula(self):
func = self._check_func(
self.parser.parse_formula('VAR'))
self.assertEqual(func(self.row, self.parser), 1)
def test_bnf(self):
result = self.parser.BNF()
print type(self.parser.bnf)
self.assertNotEqual(self.parser.bnf, None)
def test_parse_formula_with_var(self):
func = self._check_func(
self.parser.parse_formula('VAR + 1'))
self.assertEqual(func(self.row, self.parser), 2)
def test_parse_formula_bad_formula(self):
bad_formulas = [
'=BAD +++ FOR',
#'2 +>+ 1',
#'1 ** 2',
]
for bad_formula in bad_formulas:
self.assertRaises(ParseError, self.parser.parse_formula,
bad_formula)
def load_strings(self):
if not os.path.isfile(self.malware_path):
print self.err_bad_path
return False
try:
self.string_list = Parser.strings(self.malware_path)
except:
print self.err_string_recovery
return False
if self.string_list is None:
print self.err_empty_bin
return False
return True
def do_set(self, person): if person and person in SET_OPTIONS: greeting = 'hi, %s!' % person elif person: greeting = "hello, " + person else: greeting = 'hello' p = Parser() print p.checkParameter(self.lastcmd.split()) if (p.checkParameter(self.lastcmd.split()) == True): self.uws.setUrl(p.returnParameter(2)) print p.countParameter(self.lastcmd.split()) print greeting
def run_analysis(self, string_list):
"""
This method is used by the core to run analyze and extract strings
matching with the "command line" type.
:param string_list: All strings to analyse.
:type string_list: List
:return: A list of string without strings previously matched.
:rtype: List
"""
epur_string_list = []
self.bar.init(string_list, self.info_msg + "is running")
for string in string_list:
cmd_found = Parser.getCmd(string)
if cmd_found != None:
self.db.createEntry(string, cmd_found, self.type)
else:
epur_string_list.append(string)
self.bar.update()
self.bar.close(self.info_msg + "is complete", True)
return epur_string_list
def run(self):
"""Parse logs and extract relevant information."""
self.start()
# CSV files
csv_gen = CSVGen()
app = HBSortParser.get_app()
stage_titles = StageStats.get_titles(app.stages)
header = ['workers', 'set', 'input_bytes', 'input_records',
'duration_ms', 'in_memory'] + stage_titles
writer = csv_gen.get_writer(header, self.filename)
for app in HBSortParser.get_apps():
size = app.bytes_read
sset = HBSortParser.get_set(size)
stage_stats = StageStats.get_stats(app.stages)
row = [len(app.slaves), sset, size, app.records_read, app.duration,
Parser.fits_in_memory(app)] + stage_stats
writer.writerow(row)
csv_gen.close()
self.finish()
def main(self):
"""The main method.
Returns the code to OS (used in sys.exit()).
"""
# Iterates over all parameters.
for parameter in range(1, len(sys.argv)):
if (sys.argv[parameter] == "-c" or
sys.argv[parameter] == "--csv"):
# Check if destination directory exists.
# If not, create it.
try:
if not exists(dirname(BASE_DIR_PATH)):
makedirs(dirname(BASE_DIR_PATH), mode=0755)
except OSError:
print("LotoParser: Could not create working directory.")
return 1
lf = LotoFile()
files_with_errors = []
print("Downloading files from Caixa's website...")
for game in ZIP_FILES:
try:
lf.download(game, game)
except (IOError, ContentTooShortError):
files_with_errors.append(game)
if files_with_errors:
print("LotoParser: A download problem occurred with \
these files: {0}.".format(files_with_errors))
files_with_errors = [] # Restarts errors.
print("Unzipping downloaded files...")
for game in ZIP_FILES:
try:
lf.unzip(game)
except (IOError, BadZipfile):
files_with_errors.append(game)
if files_with_errors:
print("LotoParser: A unzip problem occurred with these \
files: {0}.".format(files_with_errors))
files_with_errors = [] # Restarts errors.
p = Parser()
index = -1
print("Parsing HTML files...")
for game in HTM_FILES:
try:
index += 1
p.csv_parser(BASE_DIR_PATH + game,
BASE_DIR_PATH + OUT_FILES[index] + ".csv")
except IOError:
files_with_errors.append(game)
if files_with_errors:
print("LotoParser: A parse problem occurred with these \
files: {0}.".format(files_with_errors))
files_with_errors = [] # Restarts errors.
print("Removing junk files...")
for index in range(9):
try:
lf.remove(ZIP_FILES[index])
except OSError:
files_with_errors.append(ZIP_FILES[index])
try:
lf.remove(HTM_FILES[index])
except OSError:
files_with_errors.append(HTM_FILES[index])
try:
lf.remove(PIC_FILES[index])
except OSError:
files_with_errors.append(PIC_FILES[index])
if files_with_errors:
print("LotoParser: A removing problem occurred with \
these files: {0}.".format(files_with_errors))
return 0
elif (sys.argv[parameter] == "-x" or
sys.argv[parameter] == "--xml"):
print("LotoParser: CSV files must be generated first.")
print("Generating MegaSena's XML file...")
Parser().xml_parser("./loto/megasena.csv",
"./loto/megasena.xml")
print("Generating LotoFacil's XML file...")
Parser().xml_parser("./loto/lotofacil.csv",
"./loto/lotofacil.xml")
print("Generating Quina's XML file...")
Parser().xml_parser("./loto/quina.csv",
"./loto/quina.xml")
print("Generating LotoMania's XML file...")
Parser().xml_parser("./loto/lotomania.csv",
"./loto/lotomania.xml")
print("Generating DuplaSena's XML file...")
Parser().xml_parser("./loto/duplasena.csv",
"./loto/duplasena.xml")
print("Generating Federal's XML file...")
Parser().xml_parser("./loto/federal.csv",
"./loto/federal.xml")
print("Generating LotoGol's XML file...")
Parser().xml_parser("./loto/lotogol.csv",
"./loto/lotogol.xml")
print("Generating Loteca's XML file...")
Parser().xml_parser("./loto/loteca.csv",
"./loto/loteca.xml")
print("Generating TimeMania's XML file...")
Parser().xml_parser("./loto/timemania.csv",
"./loto/timemania.xml")
return 0
elif (sys.argv[parameter] == "-h" or
sys.argv[parameter] == "--help"):
print(self.help())
return 0
elif (sys.argv[parameter] == "-v" or
sys.argv[parameter] == "--version"):
print(self.version())
return 0
else:
# Invalid parameter.
# Finnishes loop to display help.
break
# No parameters were given.
print(self.help())
return 1
# Read in file for processing...
files = glob.glob("data/" + str(sys.argv[1]) + "/*.txt")
no_errors = True
for filename in files:
file = open(filename, "r")
if not file:
util.error("File could not be loaded... Exiting...", True)
lexical_analyzer = LexicalAnalyzer()
tokens = lexical_analyzer.process_file(file)
if len(tokens) > 0:
tokens.reverse()
parser = Parser(tokens)
parse_result = parser.parse()
should_fail = "-fail" in filename
error = False
if should_fail and parse_result == "ACCEPT":
error = True
elif not should_fail and parse_result == "REJECT":
error = True
if error:
if no_errors:
print("Invalid Tests:")
no_errors = False
def __init__(self, threads, block_size, path=None):
self.threads = threads
self.block_size = block_size
self._path = path
self._parser = Parser()
def __init__(self, screen_name): Parser.__init__(self) self.screen_name = screen_name self.belle = Belle()
import sys
import time
import urllib2
from lib.parser import Parser
if __name__ == "__main__":
CUR_DIR = os.path.dirname(os.path.realpath(__file__))
# Namespaces for XML
ATOM_NS = "{http://www.w3.org/2005/Atom}"
CAP_NS = "{urn:oasis:names:tc:emergency:cap:1.1}"
# Instantiate the parser
parser = Parser(CUR_DIR)
# Try to load the previous alerts
previous_alerts_filepath = os.path.join(parser.output_dir, 'alerts.json')
if os.path.exists(previous_alerts_filepath):
previous_alerts_dict = parser.load_json(previous_alerts_filepath)
parser.previous_alerts_list = previous_alerts_dict['alerts']
else:
parser.previous_alerts_list = []
# Parse the XML
try:
noaa_url = "http://alerts.weather.gov/cap/us.php?x=1"
f = urllib2.urlopen(noaa_url, timeout=10)
request_data = f.read()
tree = parser.load_xml_from_url_contents(request_data)
def get_apps():
"""Return a generator for all apps."""
parser = Parser()
apps = parser.get_apps('hibench', 'sort', '*')
return (app for app in apps if len(app.slaves) not in [12, 123])
def setUp(self):
self.parser = Parser()
self.row = {'VAR': 1}
TestBase.setUp(self)