def test_common_and_host_file_collision(self):
'''
What happens if a file with the same name exists in the common dir and
in the "hostname" dir?
Desired behavior is that common is linked first and hostname overwrites
it, so that common can have a config that applies to all machines, but
it gets overwritten on a machine by machine basis.
'''
real_bothfile = path.join(self.indir, gethostname(), 'bothfile')
real_bothfile_back = path.join(self.indir, 'common', 'bothfile')
self.touch(real_bothfile)
self.touch(real_bothfile_back)
linker = Linker(self.indir, self.outdir)
linker.make_links()
bothfile=path.join(self.outdir, 'bothfile')
bothfile_back=path.join(self.outdir, 'bothfile.back')
try:
self.assertTrue(path.exists(bothfile))
self.assertTrue(path.exists(bothfile_back))
self.assertTrue(path.islink(bothfile))
self.assertTrue(path.islink(bothfile_back))
self.assertEqual(real_bothfile, path.realpath(bothfile))
self.assertEqual(real_bothfile_back, path.realpath(bothfile_back))
finally:
remove(real_bothfile)
remove(real_bothfile_back)
def __init__(self, drawer):
"""Initialization commands class."""
self.drawer = drawer
self.phrases = self.drawer.phrases
self.message = Message(self.drawer)
self.linker = Linker()
self.player = Player()
self.config = self.drawer.config
self.config.get_outputs = self.player.get_outputs
self.player.config = self.config
self.wildcard = 'Wave files (*.wav)|*.wav|' \
'All files (*.*)|*.*'
self.__mods = [
wx.WXK_CONTROL,
wx.WXK_SHIFT,
wx.WXK_ALT,
wx.WXK_WINDOWS_LEFT,
wx.WXK_WINDOWS_RIGHT,
wx.WXK_WINDOWS_MENU,
]
self.set_window()
def test_exclude_common(self):
linker = Linker(target=self.indir, destination=self.outdir,
exclude_common=True)
want = ['hostnamefile1']
linker.make_links()
have = listdir(self.outdir)
self.assertEqual(want, have)
def parse(self, fileName,entitySet,vectorMap):
#unpickling dict for cheap link
with open("entityDictionary.dat", "rb") as r:
entDict=pickle.load(r)
with open(fileName) as f:
for line in f:
if 'inv idx' in line:
break
elif 'predicate' in line:
predicate=self.extractPredicate(line)
currentVector = []
vectorMap.put(predicate,currentVector)
elif ': ' in line and 'num preds' not in line:
#if linking:extract single words
#if linking: link single words
#if linking: concatenate the entity links
origEntities=self.extractEntities(line)
l = Linker()
entities=l.cheapLink(origEntities, entDict)
count=self.extractCount(line)
index = entitySet.getIndex(entities)
pair = (index,count)
currentVector.append(pair)
print("done parsing")
return vectorMap, entitySet
def compile_buffer(input, output, resolver):
""" Compile code from input buffer and output to another buffer """
compiled_main = compile_module_stream('main', input)
# Link modules
linker = Linker(compiled_main, resolver)
compiled_whole = linker.link()
compiled_whole.save(output)
def setUp(self):
Linker.configure({
"protocol_classes": {
"http": None,
'ftp': 'ftp',
"*": "unknown"
}
})
def test_relative_paths(self):
linker = Linker(target='./tests/input', destination='./tests/output')
want = ['commonfile1', '.commonfile3', 'common_file5', 'hostnamefile1',
'/tmp/commonfile4']
linker.make_links()
have = listdir(path.abspath(path.expanduser('./tests/output')))
if path.exists(want[-1]): have.append(want[-1])
want.sort()
have.sort()
self.assertEqual(want, have)
def test_vanilla(self):
linker = Linker(target=self.indir, destination=self.outdir)
want = ['commonfile1', '.commonfile3', 'common_file5', 'hostnamefile1',
'/tmp/commonfile4']
linker.make_links()
have = listdir(self.outdir)
if path.exists(want[-1]): have.append(want[-1])
want.sort()
have.sort()
self.assertEqual(want, have)
def __init__(self, someprocess=None, logit=True, ticks=sys.maxint):
super(Executor, self).__init__()
if someprocess:
self.process = someprocess
else:
if ticks < sys.maxint:
ticks += 1
self.process = Scheduler(ticks=ticks, name="")
self.logit = logit
self.linker = Linker()
def test_move_existing(self):
want = ['commonfile1', '.commonfile3', 'common_file5', 'hostnamefile1',
'common_file5.back']
self.touch(self.outdir, 'common_file5')
linker = Linker(self.indir, self.outdir)
linker.make_links()
have = listdir(self.outdir)
want.sort()
have.sort()
self.assertEqual(want, have)
def test_delete_existing(self):
want = ['commonfile1', '.commonfile3', 'common_file5', 'hostnamefile1']
self.touch(self.outdir, 'common_file5')
self.assertTrue(not path.islink(path.join(self.outdir, 'common_file5')))
linker = Linker(self.indir, self.outdir, delete_existing=True)
linker.make_links()
have = listdir(self.outdir)
want.sort()
have.sort()
self.assertEqual(want, have)
self.assertTrue(path.islink(path.join(self.outdir, 'common_file5')))
def mainf():
im = img.open("screenshot.png").convert("RGB")
_head,_blocks = buildmap(im)
l = Linker(W,H)
l.sethead(_head)
for blk in _blocks:
l.setblock(blk)
end = l.link()
l.show()
print(end)
return view(im,l,end)
class Executor(object):
def __init__(self, someprocess=None, logit=True, ticks=sys.maxint):
super(Executor, self).__init__()
if someprocess:
self.process = someprocess
else:
if ticks < sys.maxint:
ticks += 1
self.process = Scheduler(ticks=ticks, name="")
self.logit = logit
self.linker = Linker()
def schedule(self, components):
if type(components) is not list:
components = components['components']
self.process.send(('activate', components), 'control')
def kill(self, components):
if type(components) is not list:
components = components['components']
self.process.send(('deactivate', components), 'control')
def build(self, links):
self.graph = self.linker.link(links)
self.schedule(self.graph)
def run(self):
for _ in self.process.run():
if self.logit:
print utils.COLOR.cyan,
print "\tExecd: ", _,
print utils.COLOR.white
class Executor(object):
def __init__(self, someprocess=None, logit=True, ticks=sys.maxint):
super(Executor,self).__init__()
if someprocess:
self.process = someprocess
else:
if ticks < sys.maxint:
ticks += 1
self.process = Scheduler(ticks=ticks, name="")
self.logit = logit
self.linker = Linker()
def schedule(self, components):
if type(components) is not list:
components = components['components']
self.process.send(('activate',components), 'control')
def kill(self, components):
if type(components) is not list:
components = components['components']
self.process.send(('deactivate',components), 'control')
def build(self, links):
self.graph = self.linker.link(links)
self.schedule(self.graph)
def run(self):
for _ in self.process.run():
if self.logit:
print utils.COLOR.cyan,
print "\tExecd: ", _,
print utils.COLOR.white
def compose_message(self, data: dict):
linker = Linker(self.images_path)
html_content = ''
with open(self.html_content_path) as html_content_file:
html_content = html_content_file.read()
html_content, used_images = linker.link_html(html_content, data)
message = MIMEMultipart('related')
message['Subject'] = self.subject
message['From'] = self.sender_email
message['To'] = data[EMAIL]
message.preamble = 'This is a multi-part message in MIME format.'
message = linker.attach_html_content(message, html_content)
message = linker.attach_images(message, used_images)
return message
def __init__(self, path):
self.path = path
self.fileobj = open(path, 'rb')
self.elf = elffile.open(fileobj=self.fileobj)
self.linker = Linker(self)
self.final_hook = []
self.asm_hook = []
self.c_hook = []
self.verbose = False
autolink.declare(self.linker)
start = 0xFFFFFFFFFFFFFFFF
end = 0
# find the end of current segments
# TODO: doesn't handle new mem being mapped or unmapped
for ph in reversed(self.elf.progs):
if ph.isload:
start = min(start, ph.vaddr)
end = max(ph.vaddr + ph.vsize, end)
def new_segment(addr):
"""adds a patch segment
Args:
addr (int): address to create segment
Returns:
ph: added program header
"""
align = 0x1000
ph = self.elf.programHeaderClass()
ph.data = bytearray()
ph.type = PT['PT_LOAD'].code
ph.vaddr = (addr + align - 1) & ~(align - 1)
ph.paddr = ph.vaddr
# TODO: default is RWX?!
ph.flags = 7
ph.align = align
ph.memsz = 0
ph.filesz = 0
self.elf.progs.append(ph)
return ph
# patch, nxpatch, linkpatch, jitpatch is new segments of different
# semantics, adds them to binary first, if we don't need any of them,
# just remove it when saving
self.patch = new_segment(end)
self.nxpatch = new_segment(end + 0x800000)
self.nxpatch.flags = 6 # RW
self.linkpatch = new_segment(end + 0x1600000)
self.jitpatch = new_segment(end + 0x2400000)
self.entry_hooks = []
def __init__(self, someprocess=None, logit=True, ticks=sys.maxint): super(Executor,self).__init__() if someprocess: self.process = someprocess else: if ticks < sys.maxint: ticks += 1 self.process = Scheduler(ticks=ticks, name="") self.logit = logit self.linker = Linker()
def on_start_button_clicked(self, widget):
finalfile = self.x[0].split('.')[0] + '.8085'
info = self.shell_ui.get_object("start_info_label")
# str1 = ''
# str1 += 'Interpreting...........\nRunning Assembler \n'
# info.set_text(str1)
self.assembler_instance = Assembler(self.shell_ui)
self.assembler_instance.test(self.x)
# str1 = str1 + 'Assembler Completed \n' + 'Running Linker \n'
# info.set_text(str1)
self.linker_instance = Linker(self.shell_ui,
self.assembler_instance.symTable,
self.assembler_instance.globTable,
self.assembler_instance.filelen)
self.linker_instance.linker(self.x)
# str1 = str1 + 'Linker Completed \n' + 'Set offset and run loader\n'
# info.set_text(str1)
self.loader_instance = Loader(self.shell_ui)
self.loader_instance.loader(self.x)
def run(self):
"""
Keeps the graph database and continously running in the terminal
and parses the input.
"""
while True:
if verbose:
# Prints out the graph structure for verbose output
self.gs.display()
commands = []
#sys.stdout.write(">>> ")
command = raw_input(">>> ")
while True:
#command = raw_input(">>> ")
if (len(command) != 0 and command[-1] == ";"):
commands.append(command)
break
commands.append(command)
command = raw_input("... ")
command_str = " ".join(commands)
# Start the parser and parse the commands
if (command_str[-1] == ";"):
real_command = command_str[:-1] + " ;" # need to add space for parser
parser = Parser(real_command)
parser.run()
else:
print "ERROR INVALID QUERY"
# Check if user entered any errors in query.
# If there are no errors, then create linker
if (not(self.has_Errors(parser))):
linker = Linker(parser.get_object_list(), self.gs)
linker.execute()
# Else, print the error
else:
print "Invalid Query"
def run(self):
"""
Keeps the graph database and continously running in the terminal
and parses the input.
"""
while True:
if verbose:
# Prints out the graph structure for verbose output
self.gs.display()
commands = []
#sys.stdout.write(">>> ")
command = raw_input(">>> ")
while True:
#command = raw_input(">>> ")
if (len(command) != 0 and command[-1] == ";"):
commands.append(command)
break
commands.append(command)
command = raw_input("... ")
command_str = " ".join(commands)
# Start the parser and parse the commands
if (command_str[-1] == ";"):
real_command = command_str[:-1] + " ;" # need to add space for parser
parser = Parser(real_command)
parser.run()
else:
print "ERROR INVALID QUERY"
# Check if user entered any errors in query.
# If there are no errors, then create linker
if (not (self.has_Errors(parser))):
linker = Linker(parser.get_object_list(), self.gs)
linker.execute()
# Else, print the error
else:
print "Invalid Query"
def __init__(self, gs, filename):
"""
Constructor takes a L{GraphStructure} object, a parser object, and
a file name as arguments. It will execute all of the commands that are
in the file.
"""
self.gs = gs
f = open(filename, 'r')
# Every line in the file is a command
for line in f:
# Skips blank lines and commented lines in text file
if (len(line.split()) == 0) or (line[0] == '#'):
continue
# Assert that line ends with semicolon
if line.split()[-1][-1] != ';':
print "Invalid file format! Every line must end with a semicolon."
break
# Add a space before semicolon for parser. This ensures that
# commands can end with a semicolon right after the last word
# in the command.
arr = line.split()
arr[-1] = arr[-1][:-1] + ' ;'
command = " ".join(arr)
# Run the parser.
parser = Parser(command)
parser.run()
# Extract the created objects from the parser and execute the linker.
linker = Linker(parser.get_object_list(), self.gs)
linker.execute()
f.close()
def __init__(self, gs, filename):
"""
Constructor takes a L{GraphStructure} object, a parser object, and
a file name as arguments. It will execute all of the commands that are
in the file.
"""
self.gs = gs
f = open(filename, 'r')
# Every line in the file is a command
for line in f:
# Skips blank lines and commented lines in text file
if (len(line.split()) == 0) or (line[0] == '#'):
continue
# Assert that line ends with semicolon
if line.split()[-1][-1] != ';':
print "Invalid file format! Every line must end with a semicolon."
break
# Add a space before semicolon for parser. This ensures that
# commands can end with a semicolon right after the last word
# in the command.
arr = line.split()
arr[-1] = arr[-1][:-1] + ' ;'
command = " ".join(arr)
# Run the parser.
parser = Parser(command)
parser.run()
# Extract the created objects from the parser and execute the linker.
linker = Linker(parser.get_object_list(), self.gs)
linker.execute()
f.close()
def __init__(self,
align_file_name="",
output_folder="",
output_name="",
obl_ratio_limit="0.5",
min_obl_inst_num=2,
**kwargs):
""" overriden block method """
super().__init__(**kwargs)
self.align_file = None
self.output_folder = output_folder
self.output_name = output_name
self.obl_ratio_limit = float(obl_ratio_limit)
self.min_obl_inst_num = int(min_obl_inst_num)
self.linker = Linker() # !!!
if align_file_name != "":
try:
self.align_file = open(align_file_name, 'r')
except FileNotFoundError:
#print( "Cesta: " + os.path.dirname(os.path.realpath(__file__)))
print("ERROR: Alignment file " + align_file_name +
" not found.")
exit()
#self.a_and_b = 0
#self.a_only = 0
#self.b_only = 0
#self.direction = 0 # 0 .. both, 1 .. a -> b, 2 .. b -> a
# to be overloaded
self.a_frame_extractor = Frame_extractor()
self.b_frame_extractor = Frame_extractor()
self.a_lang_mark = ""
self.b_lang_mark = ""
self.examiner = Modal_examiner()
def test_move_first(self):
test_file = '/tmp/test_file'
self.touch(self.outdir, test_file)
self.linker = Linker(self.indir, test_file)
test_link = path.join(self.indir, gethostname(),
self.linker.generate_target(test_file))
try:
self.linker.move_to_target()
self.assertTrue(path.exists(test_file))
self.assertTrue(path.islink(test_file))
self.assertTrue(path.exists(test_link))
self.assertFalse(path.islink(test_link))
self.assertEqual(path.realpath(test_file), test_link)
finally:
remove(test_file)
def __init__(self, path, backend):
self.path = path
self.backend = backend
self.fileobj = open(path, 'rb')
self.elf = elffile.open(fileobj=self.fileobj)
self.linker = Linker(self)
self.final_hook = []
self.asm_hook = []
self.c_hook = []
self.verbose = False
autolink.declare(self.linker)
start = 0xFFFFFFFFFFFFFFFF
end = 0
# TODO: doesn't handle new mem being mapped or unmapped
for ph in reversed(self.elf.progs):
if ph.isload:
start = min(start, ph.vaddr)
end = max(ph.vaddr + ph.vsize, end)
# add patch segment
def new_segment(addr):
align = 0x1000
ph = self.elf.programHeaderClass()
ph.data = bytearray()
ph.type = PT['PT_LOAD'].code
ph.vaddr = (addr + align - 1) & ~(align - 1)
ph.paddr = ph.vaddr
# TODO: default is RWX?!
ph.flags = 7
ph.align = align
ph.memsz = 0
ph.filesz = 0
self.elf.progs.append(ph)
return ph
# RX
self.code = new_segment(end)
self.code.flags = 5
# RW
self.data = new_segment(end + 0x800000)
self.data.flags = 6
self.entry_hooks = []
self.arch = arch.create_arch(self, backend)
def main(mode, discord_token, discord_channel, asana_token, asana_workspace,
sentry_url):
"""
Starts AsanaBot and AsanaListener in separated threads.
Arguments:\n
MODE: Mode to run the script. It just changes the logger. "production" or "testing" \n
DISCORD_TOKEN: Your Discord token (str)\n
DISCORD_CHANNEL: Target channel to post the announcements (int)\n
ASANA_TOKEN: Your Asana account token (str)\n
ASANA_WORKSPACE: The target Asana workspace to search tasks from (str)\n
SENTRY_URL: Your Sentry application url token (str)\n
"""
logger = logger_pick(mode, sentry_url)
linker = Linker(
) # A class that links the discord bot with the asana listeners
# Start project_starter thread
project_starter_thread = threading.Thread(target=project_starter,
kwargs={
'linker':
linker,
'logger':
logger,
'asana_token':
asana_token,
'asana_workspace':
asana_workspace
})
project_starter_thread.start()
# Start AsanaBot thread
bot_thread = threading.Thread(target=retry_wrapper,
kwargs={
'target': AsanaBot.start,
'target_type': "bot",
'linker': linker,
'logger': logger,
'discord_token': discord_token,
'discord_channel': discord_channel,
'asana_token': asana_token,
'asana_workspace': asana_workspace
})
bot_thread.start()
logger.info('All the threads are running.')
def on_start_button_clicked(self, widget):
finalfile = self.x[0].split('.')[0] + '.8085'
info = self.shell_ui.get_object("start_info_label")
# str1 = ''
# str1 += 'Interpreting...........\nRunning Assembler \n'
# info.set_text(str1)
self.assembler_instance = Assembler(self.shell_ui)
self.assembler_instance.test(self.x)
# str1 = str1 + 'Assembler Completed \n' + 'Running Linker \n'
# info.set_text(str1)
self.linker_instance = Linker(self.shell_ui, self.assembler_instance.symTable,
self.assembler_instance.globTable,
self.assembler_instance.filelen)
self.linker_instance.linker(self.x)
# str1 = str1 + 'Linker Completed \n' + 'Set offset and run loader\n'
# info.set_text(str1)
self.loader_instance = Loader(self.shell_ui)
self.loader_instance.loader(self.x)
class Commands:
"""Helper class, contains command for bind events, menu and buttons."""
def __init__(self, drawer):
"""Initialization commands class."""
self.drawer = drawer
self.phrases = self.drawer.phrases
self.message = Message(self.drawer)
self.linker = Linker()
self.player = Player()
self.config = self.drawer.config
self.config.get_outputs = self.player.get_outputs
self.player.config = self.config
self.wildcard = 'Wave files (*.wav)|*.wav|' \
'All files (*.*)|*.*'
self.__mods = [
wx.WXK_CONTROL,
wx.WXK_SHIFT,
wx.WXK_ALT,
wx.WXK_WINDOWS_LEFT,
wx.WXK_WINDOWS_RIGHT,
wx.WXK_WINDOWS_MENU,
]
self.set_window()
def set_window(self):
"""Set size and position window from saving data."""
self.drawer.SetPosition(self.config.get_pos())
self.drawer.SetSize(self.config.get_size())
self.drawer.Layout()
def donate(self, event):
"""Run donate hyperlink in browser."""
webbrowser.open(self.config.donate_url)
def about(self, event):
"""Run about dialog."""
About(self.drawer, self.phrases.about.title, self.phrases.about.name,
version.VERSION, self.phrases.about.author).ShowModal()
def close(self, event):
"""Close event for button close."""
self.drawer.Close(True)
def close_window(self, event):
"""Close window event."""
self.config.set_pos(self.drawer.GetScreenPosition())
self.config.set_size(self.drawer.GetSize())
self.config.close()
self.linker.close()
self.player.close()
self.drawer.Destroy()
def options(self, event):
"""Run settings dialog."""
self.config.open_settings(self.drawer)
def get_path_file(self):
"""Return path wave file."""
path = ''
file_dlg = wx.FileDialog(self.drawer,
self.phrases.titles.choice_file,
'',
'',
self.wildcard,
style=wx.FD_OPEN | wx.FD_FILE_MUST_EXIST)
if wx.ID_OK == file_dlg.ShowModal():
path = file_dlg.GetPath()
file_dlg.Destroy()
return path
def process(self, event):
"""Main process eventer for button."""
keycode = event.GetKeyCode()
if not keycode in self.__mods:
if event.CmdDown() and event.ShiftDown():
self.linker.del_link(keycode)
self.drawer.data.SetValue(self.linker.get_all_links())
self.drawer.Layout()
elif event.CmdDown():
path = self.get_path_file()
if path != '':
self.linker.set_link(keycode, path)
self.drawer.data.SetValue(self.linker.get_all_links())
self.drawer.Layout()
else:
path = self.linker.get_link(keycode)
if path is not None:
self.player.play(path)
event.Skip()
def test_dry_run(self):
linker = Linker(self.indir, self.outdir, dry_run=True)
want = []
linker.make_links()
have = listdir(self.outdir)
self.assertEqual(want, have)
def testNamed(self):
s = ur"abc http://d.e.f|DEF ghi"
f = Linker(s, 0)
frags, next = f.apply()
self.assertTrue(u"".join(frags).startswith(u'abc <a href="http://d.e.f">DEF</a>'))
self.assertEqual(next, s.index(" ghi"))
class AllApp(Gtk.Application):
# constructor of the Gtk Application
def __init__(self):
Gtk.Application.__init__(self)
self.shell_ui = Gtk.Builder()
self.shell_ui.add_from_file("shell.glade")
self.handler_dict = {
"on_start_file_chooser_button_clicked":
self.on_start_file_chooser_button_clicked,
"on_start_button_clicked": self.on_start_button_clicked,
"on_simulator_open_file_button_clicked":
self.on_simulator_open_file_button_clicked,
"on_simulate_pass_button_clicked":
self.on_simulate_pass_button_clicked,
"on_run_button_clicked": self.on_run_button_clicked,
"on_quit_image_menu_item_activate": self.on_quit_activate,
"on_offset_button_clicked": self.on_offset_button_clicked
}
self.shell_ui.connect_signals(self.handler_dict)
self.x = []
self.z = []
self.assembler_instance = None
self.loader_instance = None
self.linker_instance = None
self.simulator_instance = None
# self.simulator=Simulator()
def on_offset_button_clicked(self, widget):
self.loader_instance.loader2(self.x)
def do_activate(self):
window = self.shell_ui.get_object("all_window")
self.add_window(window)
window.show_all()
def on_start_file_chooser_button_clicked(self, widget):
window = self.shell_ui.get_object("all_window")
dialog = Gtk.FileChooserDialog(
title="Please choose a file",
parent=window,
action=Gtk.FileChooserAction.OPEN,
buttons=(Gtk.STOCK_CANCEL, Gtk.ResponseType.CANCEL, Gtk.STOCK_OPEN,
Gtk.ResponseType.OK))
response = dialog.run()
if response == Gtk.ResponseType.OK:
selected_file_path = dialog.get_filename()
relative_path = os.path.basename(selected_file_path)
inputfile = open(relative_path, "r")
code = inputfile.read()
lines = code.split('\n')
finalfile = lines[0].split('.')[0] + '.8085'
print(lines[0].split('.')[0])
print(finalfile)
entries_box = self.shell_ui.get_object("start_entries_box")
wids = entries_box.get_children()
for widget in wids:
widget.destroy()
i = 0
print(lines)
for line in lines:
if line != '':
self.z.append(line)
label = Gtk.Label("Code" + str(i))
tv = Gtk.TextView()
tb = tv.get_buffer()
entries_box.add(label)
entries_box.add(tv)
i += 1
with open(line, "r") as file:
s = file.read()
tb.set_text(s)
print(s)
self.shell_ui.get_object("start_entry_number_entry").set_text(
str(i))
entries_box.show_all()
self.x = preprocess(self.z)
processed_box = self.shell_ui.get_object("processed_box")
i = 0
for file_name in self.x:
if file_name != '':
label = Gtk.Label("Code" + str(i))
tv = Gtk.TextView()
tb = tv.get_buffer()
processed_box.add(label)
processed_box.add(tv)
i += 1
with open(file_name, "r") as file:
s = file.read()
tb.set_text(s)
print(s)
processed_box.show_all()
elif response == Gtk.ResponseType.CANCEL:
print("Cancel clicked")
dialog.destroy()
def on_start_button_clicked(self, widget):
finalfile = self.x[0].split('.')[0] + '.8085'
info = self.shell_ui.get_object("start_info_label")
# str1 = ''
# str1 += 'Interpreting...........\nRunning Assembler \n'
# info.set_text(str1)
self.assembler_instance = Assembler(self.shell_ui)
self.assembler_instance.test(self.x)
# str1 = str1 + 'Assembler Completed \n' + 'Running Linker \n'
# info.set_text(str1)
self.linker_instance = Linker(self.shell_ui,
self.assembler_instance.symTable,
self.assembler_instance.globTable,
self.assembler_instance.filelen)
self.linker_instance.linker(self.x)
# str1 = str1 + 'Linker Completed \n' + 'Set offset and run loader\n'
# info.set_text(str1)
self.loader_instance = Loader(self.shell_ui)
self.loader_instance.loader(self.x)
# str1 = str1 + 'Loading Complete \n' + '\t\tFile ready to simulate.\n' + '\t\tFile name is : ' + finalfile + '\n'
# info.set_text(str1)
def on_simulator_open_file_button_clicked(self, widget):
pass
def on_simulate_pass_button_clicked(self, widget):
finalfile = self.x[0].split('.')[0] + '.8085'
self.simulator_instance = Simulator(
self.shell_ui, self.shell_ui.get_object("all_window"))
self.simulator_instance.load(finalfile)
def on_run_button_clicked(self, widget):
self.simulator_instance.callbackf()
def do_startup(self):
Gtk.Application.do_startup(self)
def on_quit_activate(self, menu_item):
sys.exit()
def testNamedEscaped(self):
s = ur'abc http://d.e.f|"D\"E\"F" ghi'
f = Linker(s, 0)
frags, next = f.apply()
self.assertTrue(u"".join(frags).startswith(u'abc <a href="http://d.e.f">D"E"F</a>'))
self.assertEqual(next, s.index(" ghi"))
from bs4 import BeautifulSoup
import requests
from linker import Linker
import re
from urllib.request import Request, urlopen
from urllib.error import HTTPError
link = Linker()
current_row = 0
def format_url():
base_url = "https://web.archive.org/web/"
date = "20001017050155/"
return base_url + date + link.getNextLink()
def format_title_to_url(result):
a = result.text
b = re.sub('-', ' ', a).strip()
c = re.sub('\s+', ' ', b).strip()
d = remove_non_ascii(c)
e = d.replace(" ", "-")
return e
def get_title_from_wayback(url):
page = requests.get(url)
soup = BeautifulSoup(page.text, 'html.parser')
result = soup.find('span', class_='H1')
def setUp(self):
super(FunctionalTests, self).setUp()
self.linker = Linker(self.indir, self.outdir)
def main():
# Info
system = platform.system()
release = platform.release()
print("Crawl 'n' Scrape - {} {}".format(system, release))
# Parse the CLI arguments
args = parse_cli_args()
# Read the configuration
try:
config = Config(args.definition_dir)
except Exception as error:
msg.error(traceback.format_exc())
sys.exit(1)
# --------------------- INITIALIZATION ---------------------
# Create a directory for each category
for category in config.categories:
os.makedirs(os.path.join(config.directory, category.name),
exist_ok=True)
# Read robots.txt and sitemap
website = Website(config.base_url, args.time_delay, args.sitemap)
# Linker
linker = Linker(config)
# --------------------- CRAWL AND SCRAPE ---------------------
try:
for link, category in linker:
# Obey the time delay
time.sleep(website.time_delay)
# Retrieve the content from the web
content, links = url.content_and_links(config.base_url + link)
if content is None:
msg.warning(
f"Unable to reach {link} (no internet connection?)")
continue
msg.info("Visited {}".format(link))
# Update the linker
linker.add_links(links)
# Parse the content
filename, data = config.scraper.scrape(link, content)
# Save the file
file_path = os.path.join(config.directory, category.name,
f"{filename}.{args.file_format}")
if os.path.exists(file_path): # File already exists
continue
with open(file_path, "wt", encoding="utf-8") as f:
f.write(data)
# Save the linker's state
linker.save_state()
except KeyboardInterrupt: # Ctrl + C
pass
except Exception as e:
msg.error(traceback.format_exc())
def testPunctuationAndName(self):
s = ur"abc http://d.e.f?ghi!|foo. ghi"
f = Linker(s, 0)
frags, next = f.apply()
self.assertEqual(u"".join(frags), u'abc <a href="http://d.e.f?ghi!">foo.</a>')
self.assertEqual(next, s.index(" ghi"))
def testInnerParens(self):
s = ur"abc http://wiki/Foo_(Bar) ghi"
f = Linker(s, 0)
frags, next = f.apply()
self.assertEqual(u"".join(frags), u'abc <a href="http://wiki/Foo_(Bar)">http://wiki/Foo_(Bar)</a>')
self.assertEqual(next, s.index(" ghi"))
class FunctionalTests(BaseCase):
def setUp(self):
super(FunctionalTests, self).setUp()
self.linker = Linker(self.indir, self.outdir)
def test_move_first(self):
test_file = '/tmp/test_file'
self.touch(self.outdir, test_file)
self.linker = Linker(self.indir, test_file)
test_link = path.join(self.indir, gethostname(),
self.linker.generate_target(test_file))
try:
self.linker.move_to_target()
self.assertTrue(path.exists(test_file))
self.assertTrue(path.islink(test_file))
self.assertTrue(path.exists(test_link))
self.assertFalse(path.islink(test_link))
self.assertEqual(path.realpath(test_file), test_link)
finally:
remove(test_file)
def test_move_first_common(self):
test_file = '/tmp/test_file'
self.touch(self.outdir, test_file)
self.linker = Linker(self.indir, test_file)
test_link = path.join(self.indir, 'common',
self.linker.generate_target(test_file))
try:
self.linker.move_to_target(common=True)
self.assertTrue(path.exists(test_file))
self.assertTrue(path.islink(test_file))
self.assertTrue(path.exists(test_link))
self.assertFalse(path.islink(test_link))
self.assertEqual(path.realpath(test_file), test_link)
finally:
remove(test_file)
remove(test_link)
def test_fetch_targets(self):
want_output = [path.join(self.indir, 'common', f) for f in \
['commonfile1', '.commonfile3', '_tmp_commonfile4',
'common__file5']]
have_output = self.linker.fetch_targets(path.join(self.indir, 'common'))
want_output.sort()
have_output.sort()
self.assertListEqual(want_output, have_output)
def test_find_targets(self):
want_output = [path.join(self.indir, 'common', f) for f in \
['commonfile1', '.commonfile3', '_tmp_commonfile4',
'common__file5']]
want_output.append(path.join(self.indir, gethostname(),
'hostnamefile1'))
have_output = self.linker.find_targets(self.indir)
want_output.sort()
have_output.sort()
self.assertEqual(want_output, have_output)
def test_mkdir_p(self):
want_dir = path.join(self.outdir, 'mkdirp', 'test')
self.linker.mkdir_p(want_dir)
self.assertTrue(path.exists(want_dir))
self.assertTrue(path.isdir(want_dir))
class AllApp(Gtk.Application):
# constructor of the Gtk Application
def __init__(self):
Gtk.Application.__init__(self)
self.shell_ui = Gtk.Builder()
self.shell_ui.add_from_file("shell.glade")
self.handler_dict = {
"on_start_file_chooser_button_clicked": self.on_start_file_chooser_button_clicked,
"on_start_button_clicked": self.on_start_button_clicked,
"on_simulator_open_file_button_clicked": self.on_simulator_open_file_button_clicked,
"on_simulate_pass_button_clicked": self.on_simulate_pass_button_clicked,
"on_run_button_clicked": self.on_run_button_clicked,
"on_quit_image_menu_item_activate": self.on_quit_activate,
"on_offset_button_clicked": self.on_offset_button_clicked
}
self.shell_ui.connect_signals(self.handler_dict)
self.x = []
self.z = []
self.assembler_instance = None
self.loader_instance = None
self.linker_instance = None
self.simulator_instance = None
# self.simulator=Simulator()
def on_offset_button_clicked(self, widget):
self.loader_instance.loader2(self.x)
def do_activate(self):
window = self.shell_ui.get_object("all_window")
self.add_window(window)
window.show_all()
def on_start_file_chooser_button_clicked(self, widget):
window = self.shell_ui.get_object("all_window")
dialog = Gtk.FileChooserDialog(title="Please choose a file", parent=window, action=Gtk.FileChooserAction.OPEN,
buttons=(
Gtk.STOCK_CANCEL, Gtk.ResponseType.CANCEL, Gtk.STOCK_OPEN,
Gtk.ResponseType.OK))
response = dialog.run()
if response == Gtk.ResponseType.OK:
selected_file_path = dialog.get_filename()
relative_path = os.path.basename(selected_file_path)
inputfile = open(relative_path, "r")
code = inputfile.read()
lines = code.split('\n')
finalfile = lines[0].split('.')[0] + '.8085'
print(lines[0].split('.')[0])
print(finalfile)
entries_box = self.shell_ui.get_object("start_entries_box")
wids = entries_box.get_children()
for widget in wids:
widget.destroy()
i = 0
print (lines)
for line in lines:
if line != '':
self.z.append(line)
label = Gtk.Label("Code" + str(i))
tv = Gtk.TextView()
tb = tv.get_buffer()
entries_box.add(label)
entries_box.add(tv)
i += 1
with open(line, "r") as file:
s = file.read()
tb.set_text(s)
print(s)
self.shell_ui.get_object("start_entry_number_entry").set_text(str(i))
entries_box.show_all()
self.x = preprocess(self.z)
processed_box = self.shell_ui.get_object("processed_box")
i = 0
for file_name in self.x:
if file_name != '':
label = Gtk.Label("Code" + str(i))
tv = Gtk.TextView()
tb = tv.get_buffer()
processed_box.add(label)
processed_box.add(tv)
i += 1
with open(file_name, "r") as file:
s = file.read()
tb.set_text(s)
print(s)
processed_box.show_all()
elif response == Gtk.ResponseType.CANCEL:
print("Cancel clicked")
dialog.destroy()
def on_start_button_clicked(self, widget):
finalfile = self.x[0].split('.')[0] + '.8085'
info = self.shell_ui.get_object("start_info_label")
# str1 = ''
# str1 += 'Interpreting...........\nRunning Assembler \n'
# info.set_text(str1)
self.assembler_instance = Assembler(self.shell_ui)
self.assembler_instance.test(self.x)
# str1 = str1 + 'Assembler Completed \n' + 'Running Linker \n'
# info.set_text(str1)
self.linker_instance = Linker(self.shell_ui, self.assembler_instance.symTable,
self.assembler_instance.globTable,
self.assembler_instance.filelen)
self.linker_instance.linker(self.x)
# str1 = str1 + 'Linker Completed \n' + 'Set offset and run loader\n'
# info.set_text(str1)
self.loader_instance = Loader(self.shell_ui)
self.loader_instance.loader(self.x)
# str1 = str1 + 'Loading Complete \n' + '\t\tFile ready to simulate.\n' + '\t\tFile name is : ' + finalfile + '\n'
# info.set_text(str1)
def on_simulator_open_file_button_clicked(self, widget):
pass
def on_simulate_pass_button_clicked(self, widget):
finalfile = self.x[0].split('.')[0] + '.8085'
self.simulator_instance = Simulator(self.shell_ui, self.shell_ui.get_object("all_window"))
self.simulator_instance.load(finalfile)
def on_run_button_clicked(self, widget):
self.simulator_instance.callbackf()
def do_startup(self):
Gtk.Application.do_startup(self)
def on_quit_activate(self, menu_item):
sys.exit()
def testOuterParensAndDot(self):
s = ur"abc (http://d.e.f). ghi"
f = Linker(s, 0)
frags, next = f.apply()
self.assertEqual(u"".join(frags), u'abc (<a href="http://d.e.f">http://d.e.f</a>')
self.assertEqual(next, s.index("). ghi"))
def testComma(self):
s = ur"abc http://d.e.f, ghi"
f = Linker(s, 0)
frags, next = f.apply()
self.assertEqual(u"".join(frags), u'abc <a href="http://d.e.f">http://d.e.f</a>')
self.assertEqual(next, s.index(", ghi"))
def testBadCharsInUrl(self):
s = ur'abc http://d.e.f?<a>+"b"|foo ghi'
f = Linker(s, 0)
frags, next = f.apply()
self.assertEqual(u"".join(frags), u'abc <a href="http://d.e.f?<a>+"b"">foo</a>')
self.assertEqual(next, s.index(" ghi"))
def testUnknownProtocolClass(self):
s = ur"abc zox://wiki/Foo ghi"
f = Linker(s, 0)
frags, next = f.apply()
self.assertEqual(u"".join(frags), u'abc <a href="zox://wiki/Foo" class="unknown">zox://wiki/Foo</a>')
self.assertEqual(next, s.index(" ghi"))
logic_board_thread.start()
panel_thread = ConnectorThread(
ConnectorFactory(device=cfg['panel']['device'],
baudrate=cfg['panel']['baudrate']), panel_in, panel_out)
panel_thread.start()
fanout_thread = Fanout(cmd_queue, fanout_queues_list)
fanout_thread.start()
ctl_thread = ControlThread(logic_board_out, raw_bypass_queue,
cmd_to_control_queue)
ctl_thread.start()
if 'link' in cfg and cfg['link']['url']:
linker_thread = Linker(cfg['link']['url'], cmd_to_linker_queue)
linker_thread.start()
if 'mqtt' in cfg and cfg['mqtt']['url']:
mqtt_thread = Mqtt(cfg['mqtt']['url'], cfg['mqtt']['port'],
cfg['mqtt']['use_ssl'], cfg['mqtt']['validate_cert'],
cfg['mqtt']['user'], cfg['mqtt']['passwd'],
cmd_to_mqtt_queue, cmd_queue)
mqtt_thread.start()
panel_status_thread = PanelStatus(panel_in, cmd_queue, raw_bypass_queue)
panel_status_thread.start()
cron_thread = CronCommands(cmd_queue)
cron_thread.start()
class Frame_aligner(Block):
def __init__(self,
align_file_name="",
output_folder="",
output_name="",
obl_ratio_limit="0.5",
min_obl_inst_num=2,
**kwargs):
""" overriden block method """
super().__init__(**kwargs)
self.align_file = None
self.output_folder = output_folder
self.output_name = output_name
self.obl_ratio_limit = float(obl_ratio_limit)
self.min_obl_inst_num = int(min_obl_inst_num)
self.linker = Linker() # !!!
if align_file_name != "":
try:
self.align_file = open(align_file_name, 'r')
except FileNotFoundError:
#print( "Cesta: " + os.path.dirname(os.path.realpath(__file__)))
print("ERROR: Alignment file " + align_file_name +
" not found.")
exit()
#self.a_and_b = 0
#self.a_only = 0
#self.b_only = 0
#self.direction = 0 # 0 .. both, 1 .. a -> b, 2 .. b -> a
# to be overloaded
self.a_frame_extractor = Frame_extractor()
self.b_frame_extractor = Frame_extractor()
self.a_lang_mark = ""
self.b_lang_mark = ""
self.examiner = Modal_examiner()
def process_bundle(self, bundle): # void
""" overriden Block method """
#logging.info( "bundle id: " + str( bundle.bundle_id))
a_frame_insts = []
b_frame_insts = []
for tree_root in bundle.trees:
if tree_root.zone == self.a_lang_mark:
a_frame_insts = \
self.a_frame_extractor.process_tree( tree_root)
self.examiner.examine_sentence(tree_root, self.a_lang_mark)
elif tree_root.zone == self.b_lang_mark:
b_frame_insts = \
self.b_frame_extractor.process_tree( tree_root)
self.examiner.examine_sentence(tree_root, self.b_lang_mark)
word_alignments = self.align_file.readline().split()
frame_pairs = self.linker.find_frame_pairs(a_frame_insts,
b_frame_insts,
word_alignments)
print(len(frame_pairs))
for frame_pair in frame_pairs:
# linking frame types
# if the frame type link does not exist yet, create one
a_frame_type = frame_pair.a_frame_type
b_frame_type = frame_pair.b_frame_type
a_b_frame_type_link = a_frame_type.find_link_with(b_frame_type)
# could be done the other way around: b_frame.find( frst_frame)
if a_b_frame_type_link is None:
a_b_frame_type_link = Frame_type_link(a_frame_type,
b_frame_type)
# linking frame instances
a_frame_inst = frame_pair.a_frame_inst
b_frame_inst = frame_pair.b_frame_inst
a_b_frame_type_link.link_frame_insts(a_frame_inst, b_frame_inst)
# ADDED FOR MODALS, DELETE
return a_frame_insts, b_frame_insts, frame_pairs
def after_process_document(self, doc): # void
""" overriden block method """
self.examiner.print_stats()
a_dict_of_verbs = self.a_frame_extractor.get_dict_of_verbs()
b_dict_of_verbs = self.b_frame_extractor.get_dict_of_verbs()
#self.end_obl( a_dict_of_verbs)
#self.end_obl( b_dict_of_verbs)
#self._finalize_dictionary( a_dict_of_verbs, self.a_lang_mark)
#self._finalize_dictionary( b_dict_of_verbs, self.b_lang_mark)
#self._output_control()
self._pickle_dict(a_dict_of_verbs, b_dict_of_verbs)
#return
#print( "=== pocty slovies ===")
#print( self.a_lang_mark, len( self._a_dict_of_verbs))
#print( self.b_lang_mark, len( self._b_dict_of_verbs))
#print( self.a_and_b, self.a_only, self.b_only)
super().after_process_document(doc)
def end_obl(self, dict_of_verbs):
suma = 0
for vr in dict_of_verbs.values():
for ft in vr.frame_types:
for fta in ft.args:
if fta.deprel == "obl":
suma += len(fta.insts)
print(suma)
def _finalize_dictionary(self, dict_of_verbs, lang_mark):
#obl_ratio_limit = 0#.5
extraction_finalizer = \
Extraction_finalizer( dict_of_verbs, self.obl_ratio_limit,
self.min_obl_inst_num, lang_mark)
for verb_record in dict_of_verbs.values():
verb_record.finalize_extraction(extraction_finalizer)
extraction_finalizer.finalize_extraction()
def _pickle_dict(self, a_dict_of_verbs, b_dict_of_verbs): # void
""" called from after_process_document """
a_b_dicts_of_verbs = a_dict_of_verbs, b_dict_of_verbs
logging.info(sys.getrecursionlimit())
logging.info(sys.getsizeof(a_b_dicts_of_verbs))
sys.setrecursionlimit(50000)
logging.info(sys.getrecursionlimit())
#a_output_name = self.output_folder + self.a_lang_mark + \
# "_" + self.b_lang_mark + "_" + self.output_name
#b_output_name = self.output_folder + self.b_lang_mark + \
# "_" + self.a_lang_mark + "_" + self.output_name
a_b_output_name = self.output_folder + "_" + self.output_name
pickle.dump(a_b_dicts_of_verbs, open(a_b_output_name, 'wb'))
#pickle.dump( a_dict_of_verbs, open( a_output_name, 'wb'))
#pickle.dump( b_dict_of_verbs, open( b_output_name, 'wb'))
def _output_control(self): # void
""" called from after_process_document """
a_dict_of_verbs = self.a_frame_extractor.dict_of_verbs
for verb_record in list(a_dict_of_verbs.values()):
for ft in verb_record.frame_types:
for fi in ft.insts:
for fia in fi.args:
fial = fia.frame_inst_arg_link
print(fia.node.form, fial)
