def __init__(self, drawer):
"""Initialization commands class."""
self.drawer = drawer
self.phrases = self.drawer.phrases
self.message = Message(self.drawer)
self.linker = Linker()
self.player = Player()
self.config = self.drawer.config
self.config.get_outputs = self.player.get_outputs
self.player.config = self.config
self.wildcard = 'Wave files (*.wav)|*.wav|' \
'All files (*.*)|*.*'
self.__mods = [
wx.WXK_CONTROL,
wx.WXK_SHIFT,
wx.WXK_ALT,
wx.WXK_WINDOWS_LEFT,
wx.WXK_WINDOWS_RIGHT,
wx.WXK_WINDOWS_MENU,
]
self.set_window()
def parse(self, fileName,entitySet,vectorMap):
#unpickling dict for cheap link
with open("entityDictionary.dat", "rb") as r:
entDict=pickle.load(r)
with open(fileName) as f:
for line in f:
if 'inv idx' in line:
break
elif 'predicate' in line:
predicate=self.extractPredicate(line)
currentVector = []
vectorMap.put(predicate,currentVector)
elif ': ' in line and 'num preds' not in line:
#if linking:extract single words
#if linking: link single words
#if linking: concatenate the entity links
origEntities=self.extractEntities(line)
l = Linker()
entities=l.cheapLink(origEntities, entDict)
count=self.extractCount(line)
index = entitySet.getIndex(entities)
pair = (index,count)
currentVector.append(pair)
print("done parsing")
return vectorMap, entitySet
def compile_buffer(input, output, resolver):
""" Compile code from input buffer and output to another buffer """
compiled_main = compile_module_stream('main', input)
# Link modules
linker = Linker(compiled_main, resolver)
compiled_whole = linker.link()
compiled_whole.save(output)
def __init__(self, path):
self.path = path
self.fileobj = open(path, 'rb')
self.elf = elffile.open(fileobj=self.fileobj)
self.linker = Linker(self)
self.final_hook = []
self.asm_hook = []
self.c_hook = []
self.verbose = False
autolink.declare(self.linker)
start = 0xFFFFFFFFFFFFFFFF
end = 0
# find the end of current segments
# TODO: doesn't handle new mem being mapped or unmapped
for ph in reversed(self.elf.progs):
if ph.isload:
start = min(start, ph.vaddr)
end = max(ph.vaddr + ph.vsize, end)
def new_segment(addr):
"""adds a patch segment
Args:
addr (int): address to create segment
Returns:
ph: added program header
"""
align = 0x1000
ph = self.elf.programHeaderClass()
ph.data = bytearray()
ph.type = PT['PT_LOAD'].code
ph.vaddr = (addr + align - 1) & ~(align - 1)
ph.paddr = ph.vaddr
# TODO: default is RWX?!
ph.flags = 7
ph.align = align
ph.memsz = 0
ph.filesz = 0
self.elf.progs.append(ph)
return ph
# patch, nxpatch, linkpatch, jitpatch is new segments of different
# semantics, adds them to binary first, if we don't need any of them,
# just remove it when saving
self.patch = new_segment(end)
self.nxpatch = new_segment(end + 0x800000)
self.nxpatch.flags = 6 # RW
self.linkpatch = new_segment(end + 0x1600000)
self.jitpatch = new_segment(end + 0x2400000)
self.entry_hooks = []
def __init__(self, someprocess=None, logit=True, ticks=sys.maxint):
super(Executor, self).__init__()
if someprocess:
self.process = someprocess
else:
if ticks < sys.maxint:
ticks += 1
self.process = Scheduler(ticks=ticks, name="")
self.logit = logit
self.linker = Linker()
def mainf():
im = img.open("screenshot.png").convert("RGB")
_head,_blocks = buildmap(im)
l = Linker(W,H)
l.sethead(_head)
for blk in _blocks:
l.setblock(blk)
end = l.link()
l.show()
print(end)
return view(im,l,end)
def __init__(self, path, backend):
self.path = path
self.backend = backend
self.fileobj = open(path, 'rb')
self.elf = elffile.open(fileobj=self.fileobj)
self.linker = Linker(self)
self.final_hook = []
self.asm_hook = []
self.c_hook = []
self.verbose = False
autolink.declare(self.linker)
start = 0xFFFFFFFFFFFFFFFF
end = 0
# TODO: doesn't handle new mem being mapped or unmapped
for ph in reversed(self.elf.progs):
if ph.isload:
start = min(start, ph.vaddr)
end = max(ph.vaddr + ph.vsize, end)
# add patch segment
def new_segment(addr):
align = 0x1000
ph = self.elf.programHeaderClass()
ph.data = bytearray()
ph.type = PT['PT_LOAD'].code
ph.vaddr = (addr + align - 1) & ~(align - 1)
ph.paddr = ph.vaddr
# TODO: default is RWX?!
ph.flags = 7
ph.align = align
ph.memsz = 0
ph.filesz = 0
self.elf.progs.append(ph)
return ph
# RX
self.code = new_segment(end)
self.code.flags = 5
# RW
self.data = new_segment(end + 0x800000)
self.data.flags = 6
self.entry_hooks = []
self.arch = arch.create_arch(self, backend)
def main(mode, discord_token, discord_channel, asana_token, asana_workspace,
sentry_url):
"""
Starts AsanaBot and AsanaListener in separated threads.
Arguments:\n
MODE: Mode to run the script. It just changes the logger. "production" or "testing" \n
DISCORD_TOKEN: Your Discord token (str)\n
DISCORD_CHANNEL: Target channel to post the announcements (int)\n
ASANA_TOKEN: Your Asana account token (str)\n
ASANA_WORKSPACE: The target Asana workspace to search tasks from (str)\n
SENTRY_URL: Your Sentry application url token (str)\n
"""
logger = logger_pick(mode, sentry_url)
linker = Linker(
) # A class that links the discord bot with the asana listeners
# Start project_starter thread
project_starter_thread = threading.Thread(target=project_starter,
kwargs={
'linker':
linker,
'logger':
logger,
'asana_token':
asana_token,
'asana_workspace':
asana_workspace
})
project_starter_thread.start()
# Start AsanaBot thread
bot_thread = threading.Thread(target=retry_wrapper,
kwargs={
'target': AsanaBot.start,
'target_type': "bot",
'linker': linker,
'logger': logger,
'discord_token': discord_token,
'discord_channel': discord_channel,
'asana_token': asana_token,
'asana_workspace': asana_workspace
})
bot_thread.start()
logger.info('All the threads are running.')
def compose_message(self, data: dict):
linker = Linker(self.images_path)
html_content = ''
with open(self.html_content_path) as html_content_file:
html_content = html_content_file.read()
html_content, used_images = linker.link_html(html_content, data)
message = MIMEMultipart('related')
message['Subject'] = self.subject
message['From'] = self.sender_email
message['To'] = data[EMAIL]
message.preamble = 'This is a multi-part message in MIME format.'
message = linker.attach_html_content(message, html_content)
message = linker.attach_images(message, used_images)
return message
def on_start_button_clicked(self, widget):
finalfile = self.x[0].split('.')[0] + '.8085'
info = self.shell_ui.get_object("start_info_label")
# str1 = ''
# str1 += 'Interpreting...........\nRunning Assembler \n'
# info.set_text(str1)
self.assembler_instance = Assembler(self.shell_ui)
self.assembler_instance.test(self.x)
# str1 = str1 + 'Assembler Completed \n' + 'Running Linker \n'
# info.set_text(str1)
self.linker_instance = Linker(self.shell_ui,
self.assembler_instance.symTable,
self.assembler_instance.globTable,
self.assembler_instance.filelen)
self.linker_instance.linker(self.x)
# str1 = str1 + 'Linker Completed \n' + 'Set offset and run loader\n'
# info.set_text(str1)
self.loader_instance = Loader(self.shell_ui)
self.loader_instance.loader(self.x)
def run(self):
"""
Keeps the graph database and continously running in the terminal
and parses the input.
"""
while True:
if verbose:
# Prints out the graph structure for verbose output
self.gs.display()
commands = []
#sys.stdout.write(">>> ")
command = raw_input(">>> ")
while True:
#command = raw_input(">>> ")
if (len(command) != 0 and command[-1] == ";"):
commands.append(command)
break
commands.append(command)
command = raw_input("... ")
command_str = " ".join(commands)
# Start the parser and parse the commands
if (command_str[-1] == ";"):
real_command = command_str[:-1] + " ;" # need to add space for parser
parser = Parser(real_command)
parser.run()
else:
print "ERROR INVALID QUERY"
# Check if user entered any errors in query.
# If there are no errors, then create linker
if (not (self.has_Errors(parser))):
linker = Linker(parser.get_object_list(), self.gs)
linker.execute()
# Else, print the error
else:
print "Invalid Query"
def __init__(self, gs, filename):
"""
Constructor takes a L{GraphStructure} object, a parser object, and
a file name as arguments. It will execute all of the commands that are
in the file.
"""
self.gs = gs
f = open(filename, 'r')
# Every line in the file is a command
for line in f:
# Skips blank lines and commented lines in text file
if (len(line.split()) == 0) or (line[0] == '#'):
continue
# Assert that line ends with semicolon
if line.split()[-1][-1] != ';':
print "Invalid file format! Every line must end with a semicolon."
break
# Add a space before semicolon for parser. This ensures that
# commands can end with a semicolon right after the last word
# in the command.
arr = line.split()
arr[-1] = arr[-1][:-1] + ' ;'
command = " ".join(arr)
# Run the parser.
parser = Parser(command)
parser.run()
# Extract the created objects from the parser and execute the linker.
linker = Linker(parser.get_object_list(), self.gs)
linker.execute()
f.close()
def __init__(self,
align_file_name="",
output_folder="",
output_name="",
obl_ratio_limit="0.5",
min_obl_inst_num=2,
**kwargs):
""" overriden block method """
super().__init__(**kwargs)
self.align_file = None
self.output_folder = output_folder
self.output_name = output_name
self.obl_ratio_limit = float(obl_ratio_limit)
self.min_obl_inst_num = int(min_obl_inst_num)
self.linker = Linker() # !!!
if align_file_name != "":
try:
self.align_file = open(align_file_name, 'r')
except FileNotFoundError:
#print( "Cesta: " + os.path.dirname(os.path.realpath(__file__)))
print("ERROR: Alignment file " + align_file_name +
" not found.")
exit()
#self.a_and_b = 0
#self.a_only = 0
#self.b_only = 0
#self.direction = 0 # 0 .. both, 1 .. a -> b, 2 .. b -> a
# to be overloaded
self.a_frame_extractor = Frame_extractor()
self.b_frame_extractor = Frame_extractor()
self.a_lang_mark = ""
self.b_lang_mark = ""
self.examiner = Modal_examiner()
logic_board_thread.start()
panel_thread = ConnectorThread(
ConnectorFactory(device=cfg['panel']['device'],
baudrate=cfg['panel']['baudrate']), panel_in, panel_out)
panel_thread.start()
fanout_thread = Fanout(cmd_queue, fanout_queues_list)
fanout_thread.start()
ctl_thread = ControlThread(logic_board_out, raw_bypass_queue,
cmd_to_control_queue)
ctl_thread.start()
if 'link' in cfg and cfg['link']['url']:
linker_thread = Linker(cfg['link']['url'], cmd_to_linker_queue)
linker_thread.start()
if 'mqtt' in cfg and cfg['mqtt']['url']:
mqtt_thread = Mqtt(cfg['mqtt']['url'], cfg['mqtt']['port'],
cfg['mqtt']['use_ssl'], cfg['mqtt']['validate_cert'],
cfg['mqtt']['user'], cfg['mqtt']['passwd'],
cmd_to_mqtt_queue, cmd_queue)
mqtt_thread.start()
panel_status_thread = PanelStatus(panel_in, cmd_queue, raw_bypass_queue)
panel_status_thread.start()
cron_thread = CronCommands(cmd_queue)
cron_thread.start()
from bs4 import BeautifulSoup
import requests
from linker import Linker
import re
from urllib.request import Request, urlopen
from urllib.error import HTTPError
link = Linker()
current_row = 0
def format_url():
base_url = "https://web.archive.org/web/"
date = "20001017050155/"
return base_url + date + link.getNextLink()
def format_title_to_url(result):
a = result.text
b = re.sub('-', ' ', a).strip()
c = re.sub('\s+', ' ', b).strip()
d = remove_non_ascii(c)
e = d.replace(" ", "-")
return e
def get_title_from_wayback(url):
page = requests.get(url)
soup = BeautifulSoup(page.text, 'html.parser')
result = soup.find('span', class_='H1')
def main():
# Info
system = platform.system()
release = platform.release()
print("Crawl 'n' Scrape - {} {}".format(system, release))
# Parse the CLI arguments
args = parse_cli_args()
# Read the configuration
try:
config = Config(args.definition_dir)
except Exception as error:
msg.error(traceback.format_exc())
sys.exit(1)
# --------------------- INITIALIZATION ---------------------
# Create a directory for each category
for category in config.categories:
os.makedirs(os.path.join(config.directory, category.name),
exist_ok=True)
# Read robots.txt and sitemap
website = Website(config.base_url, args.time_delay, args.sitemap)
# Linker
linker = Linker(config)
# --------------------- CRAWL AND SCRAPE ---------------------
try:
for link, category in linker:
# Obey the time delay
time.sleep(website.time_delay)
# Retrieve the content from the web
content, links = url.content_and_links(config.base_url + link)
if content is None:
msg.warning(
f"Unable to reach {link} (no internet connection?)")
continue
msg.info("Visited {}".format(link))
# Update the linker
linker.add_links(links)
# Parse the content
filename, data = config.scraper.scrape(link, content)
# Save the file
file_path = os.path.join(config.directory, category.name,
f"{filename}.{args.file_format}")
if os.path.exists(file_path): # File already exists
continue
with open(file_path, "wt", encoding="utf-8") as f:
f.write(data)
# Save the linker's state
linker.save_state()
except KeyboardInterrupt: # Ctrl + C
pass
except Exception as e:
msg.error(traceback.format_exc())
