def send_external(self, csv_file, write_output=True):
message = self.build()
to_filter = ClientFilter(csv_file)
to_contact = to_filter.filter_emails()
output = Output(to_contact)
print("\nClient list to be emailed:\n---------")
for i in to_contact:
print(i.name)
print("-------")
if write_output:
output.write()
proceed = input("Proceed with emailing? y/n ... ")
if proceed == 'y':
with smtplib.SMTP_SSL("smtp.gmail.com", 465,
context=self.context) as server:
server.login(self.sender_email, self.password)
for each in to_contact:
try:
server.sendmail(self.sender_email, each.email,
message.as_string())
except smtplib.SMTPRecipientsRefused as e:
print(e)
to_contact.pop(each)
pass
print("\n")
else:
print("Aborting the mission...")
def block_receive(self, recipient_id, timestamp):
if self.broadcast_status[recipient_id] == False:
self.broadcast_status[recipient_id] = True
self.broadcast_counter += 1
ratio = self.broadcast_counter / p.Nn
Output.add(timestamp, self.id, recipient_id, self.broadcast_counter, ratio)
def stop_process(plugin, notify=True):
id = plugin.get("config", "plugin_id")
process = plugin.get("config", "process")
process_aux = plugin.get("config", "process_aux")
name = plugin.get("config", "name")
command = plugin.get("config", "shutdown")
# stop service
if command:
logger.info("Stopping service %s (%s): %s.." % (id, name, command))
logger.debug(commands.getstatusoutput(command))
# notify result to server
if notify:
time.sleep(1)
if not process:
logger.debug("plugin (%s) has an unknown state" % (name))
Output.plugin_state(Watchdog.PLUGIN_UNKNOWN_STATE_MSG % (id))
elif Watchdog.pidof(process, process_aux) is None:
logger.info(WATCHDOG_PROCESS_STOPPED % (process, id))
Output.plugin_state(Watchdog.PLUGIN_STOP_STATE_MSG % (id))
Watchdog.__pluginID_stoppedByServer.append(id)
logger.info("Plugin %s process :%s stopped by server..." % (id,name))
else:
logger.warning(WATCHDOG_ERROR_STOPPING_PROCESS % (process, id))
def extractTables(self, path, target):
"""
Starts the table extraction. Using only this method, nothing will be returned,
but the HTML output Files will be created in the specified output folder.
"""
try:
os.mkdir(target)
except OSError:
pass
os.chdir(target)
self.__dtdFile = open(target + "/pdf2xml.dtd", "w")
self.buildDtd()
self.__cmdLine = "pdftohtml -xml " + path
print(self.__cmdLine)
os.system(self.__cmdLine)
xmlFile = os.path.basename(path).rstrip(".pdf") + ".xml"
fileMover.moveXmlFile(path=path, target=target)
#starting the extraction
firstClassification = FirstClassification(target)
self.__resultTuple = firstClassification.run(target + "/" + xmlFile)
tableList = self.__resultTuple[0]
fontsList = self.__resultTuple[1]
path = self.__resultTuple[2]
self.__outputObj = Output(tableList, fontsList, path)
self.__outputObj.createOutput()
class Spider(object):
def __init__(self):
print 'init'
self.urlManager = UrlManager()
self.downloader = Downloader()
self.praser = HtmlPraser()
self.outputer = Output()
def craw(self, rootUrl):
self.urlManager.addUrl(rootUrl)
count = 1
while self.urlManager.hasNewUrl():
newUrl = self.urlManager.getNewUrl()
print '爬取第', count, '个url,url是：', newUrl
htmlContent = self.downloader.download(newUrl)
newUrls, newData = self.praser.praser(newUrl, htmlContent)
self.urlManager.addUrls(newUrls)
self.outputer.collect(newData)
if count == 10:
break
count = count + 1
self.outputer.output()
def main():
for i in range(p.Runs):
from Models.Trias.Node import Node
p.NODES = []
for i in range(p.Nn):
p.NODES.append(Node(id=i, hashPower=100 / p.Nn))
clock = 0 # set clock to 0 at the start of the simulation
if p.hasTrans:
if p.Ttechnique == "Light":
LT.create_transactions() # generate pending transactions
elif p.Ttechnique == "Full":
FT.create_transactions() # generate pending transactions
Node.generate_gensis_block(
p.NODES) # generate the gensis block for all miners
BlockCommit.generate_initial_events(
) # initiate initial events >= 1 to start with
if p.attack:
Security.generate_initial_events()
while not Queue.isEmpty() and clock <= p.simTime:
next_event = Queue.pop_event()
clock = next_event.time # move clock to the time of the event
BlockCommit.handle_event(next_event)
Output.output_to_xlsx("data_v5.xlsx")
# Output.calculate()
Output.reset()
def send_message(self, event):
if self._event_os_cached(event):
return
if self._exclude_event(event):
return
# use default values for some empty attributes
event = self._plugin_defaults(event)
Output.event(event)
Stats.new_event(event)
return
# check for consolidation
if self.conn is not None:
try:
self.conn.send(str(event))
except:
id = self._plugin.get("config", "plugin_id")
c = ServerConnPro(self._conf, id)
self.conn = c.connect(0, 10)
try:
self.conn.send(str(event))
except:
return
logger.info(str(event).rstrip())
elif not self.consolidation.insert(event):
Output.event(event)
Stats.new_event(event)
def stop_process(plugin, notify=True):
id = plugin.get("config", "plugin_id")
process = plugin.get("config", "process")
process_aux = plugin.get("config", "process_aux")
name = plugin.get("config", "name")
command = plugin.get("config", "shutdown")
# stop service
if command:
logger.info("Stopping service %s (%s): %s.." % (id, name, command))
logger.debug(commands.getstatusoutput(command))
# notify result to server
if notify:
time.sleep(1)
if not process:
logger.debug("plugin (%s) has an unknown state" % (name))
Output.plugin_state(Watchdog.PLUGIN_UNKNOWN_STATE_MSG % (id))
elif Watchdog.pidof(process, process_aux) is None:
logger.info(WATCHDOG_PROCESS_STOPPED % (process, id))
Output.plugin_state(Watchdog.PLUGIN_STOP_STATE_MSG % (id))
Watchdog.__pluginID_stoppedByServer.append(id)
logger.info("Plugin %s process :%s stopped by server..." %
(id, name))
else:
logger.warning(WATCHDOG_ERROR_STOPPING_PROCESS % (process, id))
def send_message(self, event):
if self._event_os_cached(event):
return
if self._exclude_event(event):
return
# use default values for some empty attributes
event = self._plugin_defaults(event)
Output.event(event)
Stats.new_event(event)
return
# check for consolidation
if self.conn is not None:
try:
self.conn.send(str(event))
except:
id = self._plugin.get("config", "plugin_id")
c = ServerConnPro(self._conf, id)
self.conn = c.connect(0, 10)
try:
self.conn.send(str(event))
except:
return
logger.info(str(event).rstrip())
elif not self.consolidation.insert(event):
Output.event(event)
Stats.new_event(event)
def __init__(self):
"""
creates a new program
"""
self.input = Input()
self.output = Output()
self.inventory = Inventory()
self.choice = None
def setupOutput(self):
path = u'standard_test_cases/Unit Testing/Unit Test Output/Analysis/Grade 0/Experiment 1/test'
InputData.Set_Start_Freq(1)
InputData.Set_Stop_Freq(2)
InputData.Set_Step_Freq(1)
TransformedSignalData.Set_Original_Transformed_Data(numpy.asarray([[2,4,6,8,10],[3,6,9,12,15]]))
TransformedSignalData.Set_Filtered_Transformed_Data(numpy.asarray([[1,2,3,4,5],[10,20,30,40,50]]))
TransformedSignalData.Set_Original_Frequency_Data(numpy.asarray([[5,10,15,20,25],[4,8,12,16,20]]))
TransformedSignalData.Set_Filtered_Frequency_Data(numpy.asarray([[6,12,18,24,30],[7,14,21,28,35]]))
output = Output()
output.output(path)
def disable_process(plugin, notify=True):
id = plugin.get("config", "plugin_id")
name = plugin.get("config", "name")
# disable plugin
plugin.set("config", "enable", "no")
# notify to server
if notify:
logger.info("plugin (%s) is now disabled" % (name))
Output.plugin_state(Watchdog.PLUGIN_DISABLE_STATE_MSG % (id))
def disable_process(plugin, notify=True):
id = plugin.get("config", "plugin_id")
name = plugin.get("config", "name")
# disable plugin
plugin.set("config", "enable", "no")
# notify to server
if notify:
logger.info("plugin (%s) is now disabled" % (name))
Output.plugin_state(Watchdog.PLUGIN_DISABLE_STATE_MSG % (id))
def disable_process(plugin, notify=True):
id = plugin.get("config", "plugin_id")
name = plugin.get("config", "name")
# disable plugin
plugin.set("config", "enable", "no")
# notify to server
if notify:
logger.info("plugin (%s) is now disabled" % (name))
Output.plugin_state(PluginDisableState(id))
def disable_process(plugin, notify=True):
id = plugin.get("config", "plugin_id")
name = plugin.get("config", "name")
# disable plugin
plugin.set("config", "enable", "no")
# notify to server
if notify:
logger.info("plugin (%s) is now disabled" % (name))
Output.plugin_state(PluginDisableState(id))
def clear(self):
events_to_remove = []
for event in self.__event_list:
Output.event(event)
events_to_remove.append(event)
Stats.consolidation['consolidated'] += 1
for e in events_to_remove:
self.__event_list.removeRule(e)
del events_to_remove
def send_message(self, event):
if self._event_os_cached(event):
return
if self._exclude_event(event):
return
# use default values for some empty attributes
event = self._plugin_defaults(event)
Output.event(event)
Stats.new_event(event)
return
def send_message(self, event):
if self._event_os_cached(event):
return
if self._exclude_event(event):
return
# use default values for some empty attributes
event = self._plugin_defaults(event)
Output.event(event)
Stats.new_event(event)
return
def clear(self):
events_to_remove = []
for event in self.__event_list:
Output.event(event)
events_to_remove.append(event)
Stats.consolidation['consolidated'] += 1
for e in events_to_remove:
self.__event_list.removeRule(e)
del events_to_remove
def executeFunction(self,
output=True,
parsingCriteria=None,
default=False,
unpack=True,
**kargs):
params = " ".join(list(map(lambda x: f"-{x[0]} {x[1]}",
kargs.items())))
if parsingCriteria is None:
parsingCriteria = self.defaultParsingCriteria
parsingOp = ParsingCriteria().generateCriteria(parsingCriteria)
if parsingOp is None:
parsingOp = ""
code = f"""{self.funcDef}
$result = {self.funcName} {params}
if($result -ne $null){{
$result = $result {parsingOp}
return $result
}}
"""
if output:
powershell = sp.Popen(["powershell", code],
stderr=sp.STDOUT,
stdout=sp.PIPE)
result = powershell.communicate()
try:
result = result[0].decode('utf-8')
except:
result = str(result[0])
result = result[2:-1]
result = Output(result)
if default:
if self.defaultParsing is None:
return result
try:
parser = result.feed(self.defaultParsing)
parser.parsingArgs = self.parsingArgs
output = parser.parse()
if unpack:
return output.unpack()
else:
return output
except:
return result
else:
return result
else:
sp.call(["powershell", code])
return None
def runProgram(self):
# try:
self.output = Output(sleepChar=self.sleepChar,
sleepLine=self.sleepLine)
self.write = self.output.write
self.writeln = self.output.writeln
self.canWrite = self.output.canWrite
self.printHello()
if not self.tryConnectToRemoteServer():
print "Closing.. bye."
self.dbManager = DbManager(self.output)
self.tryConnectDB()
self.mainBOT()
self.newEntry()
class ExecInstruction:
output = Output()
def __init__(self):
pass
def run(self, option):
if option:
cmd = ""
for name, value in option.env.items():
cmd += 'export ' + name + '=' + '`python -c \'print "' + value + '"\'`;'
if option.execMode == Mode.PIPED:
cmd += "(python -c 'print(" + option.payload + ")'; cat) | " + option.binPath
elif option.execMode == Mode.ARG:
cmd += option.binPath + " `python -c 'import sys; sys.stdout.write(" + option.payload + ")'`"
if option.bruteforce:
option.bruteforce = False
try:
while True:
self.output.write_message(Color.yellow + "executing command: " + cmd + Color.reset)
except KeyboardInterrupt:
pass
self.output.write_message(Color.pink + "executing command: " + cmd + Color.reset)
system(cmd)
def __exec(self, command, switches, args):
cwd = os.path.abspath(os.curdir)
if (not os.path.exists(self.path)):
if (not command == "branch"):
raise ProjectPathError(
"Error executing bzr %s. The project path %s does not exist."
% (command, self.path))
else:
os.chdir(self.path)
switchString = [it + "=" + switches[it] for it in switches]
finalCommand = (" ").join([self.cmd, command] + switchString + args)
p = subprocess.Popen(finalCommand,
shell=True,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=True)
returnValue = Output(p.stdout, p.stderr)
p.wait()
os.chdir(cwd)
if p.returncode != 0:
raise BzrException(p.returncode, str(returnValue), finalCommand)
return returnValue
def get_points(self):
final_score = 0
try:
print("Raw input: " + str(self.raw_input))
print("Raw output: " + str(self.raw_output))
self.obj_in = Input(self.raw_input)
self.obj_out = Output(self.raw_output)
#self.obj_in.print()
#self.obj_out.print()
print("==================================================")
for vehicle in self.obj_out.vehicles:
print("----------------------------------------------")
print("Calculating score for vehicle: {0}".format(vehicle.index))
for ride_index in vehicle.rides_indexes:
ride = self.obj_in.rides[int(ride_index)]
print("Taking ride {0}".format(ride_index))
ride_score = self.calculate_points(vehicle, ride, int(self.obj_in.bonus))
print("Adding {0} to the total score of {1}".format(ride_score, final_score))
final_score += ride_score
print("Current step time {0}".format(vehicle.step_time))
print("------")
except Exception as e:
print("Something crashed: " + str(e))
#raise e
return 0
return final_score
def __lint(self, user_id, case_id):
# 使用pylint检查代码质量
file_name = '.\\codes\\' + str(user_id) + '\\' + str(
case_id) + '\\main.py'
buff = StringIO()
reporter = TextReporter(output=buff)
results = pylint.lint.Run([file_name], reporter=reporter, exit=False)
try:
score = results.linter.stats['global_note']
out_str = buff.getvalue()
Output.write_code_lint(out_str, user_id, case_id)
# print(score)
return score
except Exception as e:
# 总有莫名其妙 有得分却没提交记录的人，返回0分
return 0
class GetEnvVarAddrCommand(Command):
command = "geva"
usage = "[variable name]"
output = Output()
help = "\tGet address of an environment variable.\n\tex:\tgeva SHELL"
def set_option(self, option, cmd):
cm = ""
for name, value in option.env.items():
cm += 'export ' + name + '=' + '`python -c \'print "' + value + '"\'`;'
if option.binType == BinType.THIRTY_TWO:
getenv = "../utils/getenv32"
else:
getenv = "../utils/getenv64"
try:
addr = int(
subprocess.check_output(cm + getenv + ' ' + cmd[1],
shell=True), 16)
except Exception:
self.output.write_error_message("Env variable " + cmd[1] +
" not Found")
return
addr += len(getenv) - len(option.binPath)
self.output.write_message("Env variable " + cmd[1] + " Found at " +
Color.blue + hex(addr) + Color.reset)
def extractTables(self, path, target):
"""
Starts the table extraction. Using only this method, nothing will be returned,
but the HTML output Files will be created in the specified output folder.
"""
try:
os.mkdir(target)
except OSError:
pass
os.chdir(target)
self.__dtdFile = open(target + "/pdf2xml.dtd", "w")
self.buildDtd()
self.__cmdLine = "pdftohtml -xml " + path
print(self.__cmdLine)
os.system(self.__cmdLine)
xmlFile = os.path.basename(path).rstrip(".pdf") + ".xml"
fileMover.moveXmlFile(path = path, target = target)
#starting the extraction
firstClassification = FirstClassification(target)
self.__resultTuple = firstClassification.run(target + "/" + xmlFile)
tableList = self.__resultTuple[0]
fontsList = self.__resultTuple[1]
path = self.__resultTuple[2]
self.__outputObj = Output(tableList, fontsList, path)
self.__outputObj.createOutput()
def transactionFromByteArray(trans_data):
offset = 0
in_arr = []
out_arr = []
no_of_input = int.from_bytes(trans_data[:4], 'big')
offset += 4
for i in range(no_of_input):
trans_ID = trans_data[offset:offset + 32].hex()
offset += 32
index = int.from_bytes(trans_data[offset:offset + 4], 'big')
offset += 4
sign_len = int.from_bytes(trans_data[offset:offset + 4], 'big')
offset += 4
sign = trans_data[offset:offset + sign_len].hex()
offset += sign_len
inp_obj = Input(trans_ID, index, sign)
in_arr.append(inp_obj)
no_of_output = int.from_bytes(trans_data[offset:offset + 4], 'big')
offset += 4
for i in range(no_of_output):
coins = int.from_bytes(trans_data[offset:offset + 8], 'big')
offset += 8
key_len = int.from_bytes(trans_data[offset:offset + 4], 'big')
offset += 4
key = trans_data[offset:offset + key_len].decode()
offset += key_len
out_obj = Output(coins, key)
out_arr.append(out_obj)
return [in_arr, out_arr]
class MainLoop:
option = Option()
quit = False
userInput = UserInput()
commandHandler = CommandHandler()
output = Output()
execInstrution = ExecInstruction()
def __init__(self, option):
self.option = option
self.start_loop()
def start_loop(self):
while not self.quit:
self.output.load_prompt()
try:
cmd = self.userInput.read_input()
if cmd:
self.option.payload = None
self.commandHandler.command_handler(cmd, self.option)
if self.option.payload:
self.execInstrution.run(self.option)
except KeyboardInterrupt:
self.output.write_message("use " + Color.pink + "\"quit\"" + Color.reset + " to exit")
def __init__(self, yml=None, variables=None, switch_expect=None):
if yml is not None:
self.__variables = variables
self.__variables.set_vars(yml.get('vars', []))
self.__opts = Opts(yml.get('opts', []), variables)
self.__set_input(yml['input'])
self.__transform = Transform(
yml.get('transform', ''), variables,
self.__opts.get('delimiter', None))
self.__query = yml.get('query', '')
self.__preview = yml.get('preview', '')
self.__bind = Bind(yml.get('bind', {}), self.__variables, self.__transform)
self.__output = Output(yml.get('output', {}), variables)
self.__switch_expect = switch_expect
class Programa:
def __init__(self, name):
self.name = name
self.instrucciones = []
self.output = Output()
def agregarInstruccion(self, instruccion):
self.instrucciones.append(instruccion)
def ejecutar(self):
for instr in self.instrucciones:
instr.run(self.output)
def size(self):
return len(self.instrucciones)
def getInstrucciones(self):
return self.instrucciones
def getElemento(self, posicion):
return self.instrucciones[posicion]
def getInstruccionEjecutada(self, indice):
return self.output.get(indice)
def send_message(self, event):
if self._event_os_cached(event):
return
if self._exclude_event(event):
return
# use default values for some empty attributes
# check_data =True
# if event["event_type"] != EventIdm.EVENT_TYPE:
# check_data =False
event = self._plugin_defaults(event)
Output.event(event)
Stats.new_event(event)
return
def __init__(self):
self.input = Input()
pygame.init()
pygame.display.set_mode(
(Constants.SCREEN_WIDTH, Constants.SCREEN_HEIGHT))
pygame.display.set_caption("color-based-multi-person-id-tracker")
screen = pygame.display.get_surface()
self.output = Output(screen, self.input)
def from_json(self, data):
data_inp = data['input']
data_out = data['output']
for i in data_inp:
self.inp_arr.append(
Input(i['transactionID'], i['index'], i['signature']))
for i in data_out:
self.out_arr.append(Output(i['amount'], i['recepient']))
def __init__(self, config, window):
self._l = logging.getLogger("plugin.builder")
self._l.debug("Entered")
self._window = window
self._config = config
# Store last build/compile command run for each document
# during this session
self._prev_commands = {}
# Map to record location of items found in the Output buffer
# _item_locations[item number] ->
# if file loaded : a GtkTextBuffer mark
# else: a tuple of (path, line, col)
self._item_locations = {}
ui_builder = gtk.Builder()
ui_builder.add_from_file(
os.path.join(config.get_data_dir(), 'Builder.glade'))
self._ui = ui_builder.get_object
self._ui('dlg_unsaved_file').set_transient_for(window)
self._ui('dlg_run_command').set_transient_for(window)
# Add bottom panel console
self._console = Output(config)
self._console.set_item_selected_cb(self._goto)
self._console.set_item_found_cb(self._record)
bottom = self._window.get_bottom_panel()
bottom.add_item(self._console.widget(), _('Build Output'),
gtk.STOCK_EXECUTE)
# Insert menu item Tools->Compile
manager = self._window.get_ui_manager()
self._action_group = gtk.ActionGroup("plugin_builder")
self._action_group.add_actions([
("plugin_builder_compile", None, _("_Compile"), "<control>F5",
_("Compile the current document"), self._compile_doc),
("plugin_builder_build", None, _("_Build"), "<control>F6",
_("Run the build command"), self._build)
])
manager.insert_action_group(self._action_group, -1)
self._ui_id = manager.add_ui_from_string(ui_str)
self._l.info(manager.get_ui())
self._save_action = manager.get_action(
'/ui/MenuBar/FileMenu/FileSaveMenu')
self._l.info(self._save_action)
def lint(self):
statusoutput = self.status()
lintoutput = ""
for line in str(statusoutput).split("\n"):
if line.startswith("?"):
lintoutput += line + "\n"
lintoutput += str(self.__exec("conflicts", {}, []))
tmp = Output(StringIO.StringIO(lintoutput), StringIO.StringIO())
return tmp
def start_process(plugin, notify=True):
id = plugin.get("config", "plugin_id")
process = plugin.get("config", "process")
process_aux = plugin.get("config", "process_aux")
name = plugin.get("config", "name")
command = plugin.get("config", "startup")
# start service
if command:
logger.info("Starting service %s (%s): %s.." % (id, name, command))
task = Task(command)
task.Run(1, 0)
timeout = 300
start = datetime.now()
plugin.start_time = float(time.time())
while not task.Done():
time.sleep(0.1)
now = datetime.now()
if (now - start).seconds > timeout:
task.Kill()
logger.warning(
"Could not start %s, returning after %s second(s) wait time."
% (command, timeout))
# notify result to server
if notify:
if not process:
logger.debug("plugin (%s) has an unknown state" % (name))
Output.plugin_state(Watchdog.PLUGIN_UNKNOWN_STATE_MSG % (id))
elif Watchdog.pidof(process, process_aux) is not None:
logger.info(WATCHDOG_PROCESS_STARTED % (process, id))
Output.plugin_state(Watchdog.PLUGIN_START_STATE_MSG % (id))
for pid in Watchdog.__pluginID_stoppedByServer:
if pid == id:
Watchdog.__pluginID_stoppedByServer.remove(id)
break
else:
logger.warning(WATCHDOG_ERROR_STARTING_PROCESS % (process, id))
def set_option(self, option, cmd):
output = Output()
if option.buffSize == 0:
output.write_error_message(
"Buffer size must be set to execute this command")
return
value = format_hex_value(cmd[1], option)
if len(cmd) > 2 and cmd[2] == "after":
option.payload = '"\\x90" * ' + str(
option.buffSize
) + ' + "' + value + '" + "' + option.shellcode + '"'
else:
size = option.buffSize - (len(option.shellcode) / 4)
option.payload = '"' + option.shellcode + '" + "\\x90" * ' + str(
int(size)) + ' + "' + value + '"'
def evaluate(self, rule_name):
if self.first_value is None:
logger.debug(
"Can not extract value (arg1) from monitor response or no initial value to compare with"
)
return True
value = None
monitor_response = self.get_data(rule_name)
if not monitor_response:
logger.warning("No data received from monitor")
return True
else:
value = self.get_value(monitor_response, rule_name)
if value is None:
return True
#if not value:
# continue
if self.eval_condition(cond=self.watch_rule["condition"],
arg1=self.first_value,
arg2=value,
value=int(self.watch_rule["value"])):
self.watch_rule["type"] = "monitor"
try:
cond = self.watch_rule["condition"]
arg1 = self.first_value
arg2 = value
value = int(self.watch_rule["value"])
comm = self.queries
log = "Monitor Command: %s , Monitor expresion evaluation: %s(arg2) <%s> %s(arg1) + %s(value)? , Command Response: %s" % (
str(comm), str(arg2), str(cond), str(arg1), str(value),
monitor_response.replace("\n", "\r"))
except:
log = "Monitor Exception"
self.watch_rule = self._plugin_defaults(self.watch_rule, log)
Output.event(self.watch_rule)
Stats.new_event(self.watch_rule)
return True
logger.debug("No data matching the watch-rule received from monitor")
return False
def start_process(plugin, notify=True):
id = plugin.get("config", "plugin_id")
process = plugin.get("config", "process")
process_aux = plugin.get("config", "process_aux")
name = plugin.get("config", "name")
command = plugin.get("config", "startup")
# start service
if command:
logger.info("Starting service %s (%s): %s.." % (id, name, command))
task = Task(command)
task.Run(1, 0)
timeout = 300
start = datetime.now()
plugin.start_time = float(time.time())
while not task.Done():
time.sleep(0.1)
now = datetime.now()
if (now - start).seconds > timeout:
task.Kill()
logger.warning("Could not start %s, returning after %s second(s) wait time." % (command, timeout))
# notify result to server
if notify:
if not process:
logger.debug("plugin (%s) has an unknown state" % (name))
Output.plugin_state(Watchdog.PLUGIN_UNKNOWN_STATE_MSG % (id))
elif Watchdog.pidof(process, process_aux) is not None:
logger.info(WATCHDOG_PROCESS_STARTED % (process, id))
Output.plugin_state(Watchdog.PLUGIN_START_STATE_MSG % (id))
for pid in Watchdog.__pluginID_stoppedByServer:
if pid == id:
Watchdog.__pluginID_stoppedByServer.remove(id)
break
else:
logger.warning(WATCHDOG_ERROR_STARTING_PROCESS % (process, id))
class Program():
"""
Creates a blueprint for a program
"""
def __init__(self):
"""
creates a new program
"""
self.input = Input()
self.output = Output()
self.inventory = Inventory()
self.choice = None
def run(self):
"""
runs the program
"""
print("\nWelcome!")
while self.choice != 4:
# Prints the main menu
print(self.output.get_main_menu())
self.choice = self.input.get_choice_in_range(4)
if self.choice == 1:
new_product = self.inventory.create_product()
self.input.set_product_properties(new_product)
self.inventory.add_product(new_product)
elif self.choice == 2:
products = self.inventory.get_products()
print(self.output.get_products_list(products))
elif self.choice == 3:
product_name = self.input.get_input_as_str("Enter product name: ")
product = self.inventory.get_product(product_name)
print(self.output.get_product_information(product))
def test_property_handling(self):
output = Output()
propertyDict = {'fw1': {'prop1': 'val1', 'prop2': 'not val2'}, 'fw2': {'propfw2': 'something'}}
output.add_property(propertyDict)
self.assertEqual(output.get_properties(),propertyDict)
updateDict = {'fw1': {'prop2': 'val2', 'prop3': None}, 'fw3': {'propfw3': 'foo'}}
output.add_property(updateDict)
propertyDict['fw1'].update(updateDict['fw1'])
propertyDict['fw3'] = updateDict['fw3']
# print(propertyDict)
self.assertEqual(output.get_properties(),propertyDict)
def evaluate(self, rule_name):
if self.first_value is None:
logger.debug("Can not extract value (arg1) from monitor response or no initial value to compare with")
return True
value = None
monitor_response = self.get_data(rule_name)
if not monitor_response:
logger.warning("No data received from monitor")
return True
else:
value = self.get_value(monitor_response, rule_name)
if value is None:
return True
#if not value:
# continue
if self.eval_condition(cond=self.watch_rule["condition"],
arg1=self.first_value,
arg2=value,
value=int(self.watch_rule["value"])):
self.watch_rule["type"] = "monitor"
try:
cond = self.watch_rule["condition"]
arg1 = self.first_value
arg2 = value
value = int(self.watch_rule["value"])
comm = self.queries
log = "Monitor Command: %s , Monitor expresion evaluation: %s(arg2) <%s> %s(arg1) + %s(value)? , Command Response: %s" % (str(comm), str(arg2), str(cond), str(arg1), str(value), monitor_response.replace("\n", "\r"))
except:
log = "Monitor Exception"
self.watch_rule = self._plugin_defaults(self.watch_rule, log)
Output.event(self.watch_rule)
Stats.new_event(self.watch_rule)
return True
logger.debug("No data matching the watch-rule received from monitor")
return False
def main():
ticker = raw_input("\n\n\n----------------------------------------------\nWelcome. Ready to trade? Pick a stock ticker: ")
reuterObj = ReutersQuery()
reuterVector = reuterObj.getQuery(ticker)
sentimentObj = Sentiment()
sentiments = sentimentObj.sentimentVectorize(reuterVector)
yahooObj = YahooQuery()
yahooVector = yahooObj.doYahooQuery(ticker, reuterVector)
reuterDates = DateFormat()
dates = reuterDates.fixDates(reuterVector)
mergeObj = Merge()
merged = mergeObj.mergeEverything(sentiments, yahooVector, dates)
strategyObj = Strategy()
metrics = strategyObj.runStrategy(ticker, merged)
outputObj = Output()
outputObj.putOutput(ticker, metrics, yahooVector, merged)
print '\nThanks for trading with Vivek! Get money, get paid!'
def __init__(self, disco_config, params):
"""
creating a Disco class instance which will provide the regular interface to access the cluster lifecycle
:param disco_config: data needed for Disco class
:param params: possible further parameters (not used currently)
"""
self.deployer = disco_config['deployer']
self.framework_directory = disco_config['framework_directory']
self.root_component = disco_config['root_component']
self.root_component_state = disco_config['root_component_state']
self.params = params
self.components = {}
self.output = FinalOutput()
self.load_components()
def __init__(self, config, window):
self._l = logging.getLogger("plugin.builder")
self._l.debug("Entered")
self._window = window
self._config = config
# Store last build/compile command run for each document
# during this session
self._prev_commands = {}
# Map to record location of items found in the Output buffer
# _item_locations[item number] ->
# if file loaded : a GtkTextBuffer mark
# else: a tuple of (path, line, col)
self._item_locations = {}
ui_builder = gtk.Builder()
ui_builder.add_from_file(os.path.join(config.get_data_dir(), 'Builder.glade'))
self._ui = ui_builder.get_object
self._ui('dlg_unsaved_file').set_transient_for(window)
self._ui('dlg_run_command').set_transient_for(window)
# Add bottom panel console
self._console = Output(config);
self._console.set_item_selected_cb(self._goto)
self._console.set_item_found_cb(self._record)
bottom = self._window.get_bottom_panel()
bottom.add_item(self._console.widget(), _('Build Output'), gtk.STOCK_EXECUTE)
# Insert menu item Tools->Compile
manager = self._window.get_ui_manager()
self._action_group = gtk.ActionGroup("plugin_builder")
self._action_group.add_actions([("plugin_builder_compile", None,
_("_Compile"), "<control>F5",
_("Compile the current document"),
self._compile_doc),
("plugin_builder_build", None,
_("_Build"), "<control>F6",
_("Run the build command"),
self._build)])
manager.insert_action_group(self._action_group, -1)
self._ui_id = manager.add_ui_from_string(ui_str)
self._l.info(manager.get_ui())
self._save_action = manager.get_action('/ui/MenuBar/FileMenu/FileSaveMenu')
self._l.info(self._save_action)
def test_output_handling(self):
output = Output()
teststring = "test this output"
output.set_output(teststring)
output.append_output(teststring)
self.assertEqual(output.get_output(), 2*teststring)
def run(self):
first_run = True
while not Watchdog.__shutdown_running:
tzone = str(self.conf.get("plugin-defaults", "tzone"))
if tzone in all_timezones:
agent_tz = timezone(tzone)
local_dt = agent_tz.localize(datetime.now())
str_tzone = time.strftime("%z")
matches = self.patternlocalized.match(str_tzone)
if type(matches) is not None:
tzone_symbol = matches.group("tzone_symbol")
tzone_hour = matches.group("tzone_hour")
tzone_min = matches.group("tzone_min")
tzone = (float(tzone_hour) * 60 + float(tzone_min)) / 60
if tzone_symbol == "-":
tzone = -1 * tzone
else:
tzone = 0
logger.info("Warning: TimeZone doesn't match: %s --set to 0" % tzone)
else:
logger.info("Warning: Agent invalid agent tzone: %s --set to 0" % tzone)
tzone = 0
t = datetime.now()
Output.plugin_state(self.AGENT_DATE % (str(mktime(t.timetuple())), tzone))
logger.info(self.AGENT_DATE % (str(time.mktime(t.timetuple())), tzone))
for plugin in self.plugins:
id = plugin.get("config", "plugin_id")
process = plugin.get("config", "process")
process_aux = plugin.get("config", "process_aux")
name = plugin.get("config", "name")
logger.debug("Checking process %s for plugin %s." \
% (process, name))
sttopedBySrv = False
for pid in Watchdog.__pluginID_stoppedByServer:
if pid == id:
sttopedBySrv = True
break
# 1) unknown process to monitoring
if not process:
logger.debug("plugin (%s) has an unknown state" % (name))
Output.plugin_state(self.PLUGIN_UNKNOWN_STATE_MSG % (id))
# 2) process is running
elif self.pidof(process, process_aux) is not None:
logger.debug("plugin (%s) is running" % (name))
Output.plugin_state(self.PLUGIN_START_STATE_MSG % (id))
# check for for plugin restart
self._restart_services(plugin)
# 3) process is not running
else:
logger.debug("plugin (%s) is not running" % (name))
Output.plugin_state(self.PLUGIN_STOP_STATE_MSG % (id))
# restart services (if start=yes in plugin
# configuration and plugin is enabled)
if plugin.getboolean("config", "start") and \
plugin.getboolean("config", "enable") and not sttopedBySrv:
self.start_process(plugin)
if self.pidof(process, process_aux) is not None and not first_run:
Stats.watchdog_restart(process)
# send plugin enable/disable state
if plugin.getboolean("config", "enable"):
logger.debug("plugin (%s) is enabled" % (name))
Output.plugin_state(self.PLUGIN_ENABLE_STATE_MSG % (id))
else:
logger.debug("plugin (%s) is disabled" % (name))
Output.plugin_state(self.PLUGIN_DISABLE_STATE_MSG % (id))
time.sleep(float(self.interval))
first_run = False
def __init__(self, name):
self.name = name
self.instrucciones = []
self.output = Output()
class Builder:
_READY = 0
_SAVING = 1
_EXECUTE = 2
def __init__(self, config, window):
self._l = logging.getLogger("plugin.builder")
self._l.debug("Entered")
self._window = window
self._config = config
# Store last build/compile command run for each document
# during this session
self._prev_commands = {}
# Map to record location of items found in the Output buffer
# _item_locations[item number] ->
# if file loaded : a GtkTextBuffer mark
# else: a tuple of (path, line, col)
self._item_locations = {}
ui_builder = gtk.Builder()
ui_builder.add_from_file(os.path.join(config.get_data_dir(), 'Builder.glade'))
self._ui = ui_builder.get_object
self._ui('dlg_unsaved_file').set_transient_for(window)
self._ui('dlg_run_command').set_transient_for(window)
# Add bottom panel console
self._console = Output(config);
self._console.set_item_selected_cb(self._goto)
self._console.set_item_found_cb(self._record)
bottom = self._window.get_bottom_panel()
bottom.add_item(self._console.widget(), _('Build Output'), gtk.STOCK_EXECUTE)
# Insert menu item Tools->Compile
manager = self._window.get_ui_manager()
self._action_group = gtk.ActionGroup("plugin_builder")
self._action_group.add_actions([("plugin_builder_compile", None,
_("_Compile"), "<control>F5",
_("Compile the current document"),
self._compile_doc),
("plugin_builder_build", None,
_("_Build"), "<control>F6",
_("Run the build command"),
self._build)])
manager.insert_action_group(self._action_group, -1)
self._ui_id = manager.add_ui_from_string(ui_str)
self._l.info(manager.get_ui())
self._save_action = manager.get_action('/ui/MenuBar/FileMenu/FileSaveMenu')
self._l.info(self._save_action)
def deactivate(self):
self._l.debug("Entered")
# Remove menu item Tools->Compile File
manager = self._window.get_ui_manager()
manager.remove_ui(self._ui_id)
manager.remove_action_group(self._action_group)
manager.ensure_update()
# Remove bottom panel console
bottom = self._window.get_bottom_panel()
bottom.remove_item(self._console.widget())
# Tidy up
self._window = None
self._config = None
self._action_group = None
def _compile_doc(self, action):
self._l.debug("Entered")
rootdir = None
doc = self._window.get_active_document()
if doc:
rootdir = os.path.dirname(doc.get_uri_for_display())
self._generate_output(self._config.compile_cmd, rootdir)
def _build(self, action):
self._l.debug("Entered")
rootdir = None
doc = self._window.get_active_document()
rootdir = self._config.build_root(doc)
self._generate_output(self._config.build_cmd, rootdir)
def _generate_output(self, get_cmd_func, rootdir):
self._l.debug("Entered")
bp = self._window.get_bottom_panel()
bp.activate_item(self._console.widget())
bp.set_property("visible", True)
doc = self._window.get_active_document()
active_tab = self._window.get_active_tab()
unsaved = self._window.get_unsaved_documents()
self._l.info("There are %d unsaved documents" % len(unsaved))
self._doc_handler_ids = {}
self._state = self._EXECUTE
if unsaved:
self._state = self._SAVING
md = self._ui('dlg_unsaved_file')
resp = 0
for us_doc in unsaved:
self._l.info('Makeing %s active' % us_doc.get_uri_for_display())
tab = self._window.get_tab_from_uri(us_doc.get_uri())
# Make doc the active doc because the only reliable way to save
# a document is to activate the save action attached to the File
# menu which, of course, works on the active document. Plus it
# is always good to show the doc you are asking if they want to
# save.
self._window.set_active_tab(tab)
if resp != 3: # yes_to_all
md.format_secondary_text(us_doc.get_uri_for_display())
resp = md.run()
self._l.debug("User responded with %d" % resp)
if resp == 1 or resp == 3: # yes OR yes_to_all
self._doc_handler_ids[us_doc] = us_doc.connect(
'saved', self._execute_command, get_cmd_func, rootdir, doc)
#us_doc.save(0) generates critical errors
self._l.info('Saving active doc')
self._save_action.activate()
elif resp == 2: # no
self._l.info('Not saving %s' % us_doc.get_uri_for_display())
pass
elif resp == 4: # no_to_all
self._l.info('No more docs to save before compile/build')
break
else: # cancel (or closed)
self._l.info('User canceled compile/build')
self._state = self._READY
break
self._window.set_active_tab(active_tab)
md.hide()
if self._state == self._SAVING:
self._state = self._EXECUTE
if len(self._doc_handler_ids) == 0:
self._execute_command(doc, None, get_cmd_func, rootdir, doc)
else:
self._execute_command(doc, None, get_cmd_func, rootdir, doc)
def _execute_command(self, us_doc, arg1, get_cmd_func, rootdir, doc):
""" Executes the compile or build command once all documents have been saved.
doc - the active documents when the build/compile was initiated
get_cmd_func - returns the build or compile command
rootdir - for compile this is the doc dir, for build it comes from config
"""
self._l.debug('Entered')
# First check that all documents have finished saving
if us_doc in self._doc_handler_ids.keys():
us_doc.disconnect(self._doc_handler_ids[us_doc])
del(self._doc_handler_ids[us_doc])
if self._doc_handler_ids:
self._l.info("Still waiting for documents to save")
elif self._state != self._EXECUTE:
self._l.info("Build/compile cancelled after saves")
else:
# Allow user to edit command before running
md = self._ui('dlg_run_command')
button_revert = self._ui('button_revert')
buffer = self._ui('command_text').get_buffer()
buffer.set_text(self._get_prev_command(doc, get_cmd_func))
resp = 1
while resp == 1:
resp = md.run()
if resp == 1:
buffer.set_text(get_cmd_func(doc))
md.hide()
command = buffer.get_text(buffer.get_start_iter(),
buffer.get_end_iter())
if resp == gtk.RESPONSE_OK:
self._set_prev_command(doc, get_cmd_func, command)
saved_cwd = os.getcwd()
if rootdir:
os.chdir(rootdir)
for k, v in self._item_locations.iteritems():
if isinstance(v, gtk.TextMark):
v.get_buffer().delete_mark(v)
self._item_locations.clear()
self._console.execute(command)
os.chdir(saved_cwd)
return
def _get_prev_command(self, doc, get_cmd_func):
if doc in self._prev_commands.keys():
if get_cmd_func in self._prev_commands[doc]:
return self._prev_commands[doc][get_cmd_func]
return get_cmd_func(doc)
def _set_prev_command(self, doc, get_cmd_func, command):
if doc not in self._prev_commands.keys():
self._prev_commands[doc] = {}
self._prev_commands[doc][get_cmd_func] = command
def _record(self, item_no, file_name, line_no, col_no):
self._l.debug("Entered: item[%d], filename[%s], line[%d], col[%d]" % (item_no, file_name, line_no, col_no))
# Find the document for this file_name
open_docs = self._window.get_documents()
self._l.debug("Looking to see if [%s] is already open" % os.path.basename(file_name))
for doc in open_docs:
self._l.debug("Found open doc [%s]" % doc.get_short_name_for_display())
if doc.get_uri_for_display() == file_name:
self._l.debug("Found!")
# this seems a bit around the houses
tab = self._window.get_tab_from_uri(doc.get_uri())
self._item_locations[item_no] = self._mark_document(doc, line_no, col_no)
return
self._item_locations[item_no] = (file_name, line_no, col_no)
def _get_tab_for_uri(self, uri):
"""Return tab for given uri, or None"""
self._l.debug("Entered(%s)" % uri)
open_docs = self._window.get_documents()
for doc in open_docs:
self._l.debug("Found open doc [%s]" % doc.get_short_name_for_display())
if doc.get_uri_for_display() == uri:
self._l.debug("Found!")
# this seems a bit around the houses
return self._window.get_tab_from_uri(doc.get_uri())
return None
def _set_cursor_pos(self, where):
self._l.debug("Entered")
doc = where.get_buffer()
doc.place_cursor(doc.get_iter_at_mark(where))
tab = self._window.get_tab_from_uri(doc.get_uri())
self._window.set_active_tab(tab)
tab.get_view().scroll_to_cursor()
tab.get_view().grab_focus()
def _goto(self, item_no):
self._l.debug("Entered: item[%d]" % item_no)
where = self._item_locations[item_no]
if isinstance(where, gtk.TextMark):
self._set_cursor_pos(where)
else:
(file_name, line_no, col_no) = where
self._l.info("Document no currently loaded %s:%d:%d" % (file_name, line_no, col_no))
#uri = "file://" + file_name
uri = gnomevfs.get_uri_from_local_path(file_name)
if gedit.utils.uri_exists(uri):
# First check that we are not in the middle of opening this document
if self._get_tab_for_uri(uri):
self._l.info(" but is loading...")
return
tab = self._window.create_tab_from_uri(uri, None, 0, False, True)
doc = tab.get_document()
self._l.debug("Now opening [%s]" % doc.get_uri_for_display())
doc.connect("loaded", self._doc_loaded_cb, item_no)
else:
md = gtk.MessageDialog(
self._window,
gtk.DIALOG_DESTROY_WITH_PARENT | gtk.DIALOG_MODAL,
gtk.MESSAGE_ERROR,
gtk.BUTTONS_CLOSE,
"Could not find '%s'" % file_name)
md.run()
md.destroy()
def _doc_loaded_cb(self, doc, arg1, item_no):
doc_uri = doc.get_uri_for_display()
self._l.debug("Entered('%s', arg1, %d" % (doc_uri, item_no))
# Now we have loaded the document we might as well convert all relevant
# items to marks
for k, v in self._item_locations.iteritems():
if isinstance(v, tuple):
(file_name, line_no, col_no) = v
if file_name == doc_uri:
self._l.debug("Setting mark for %d:%d" % (line_no, col_no))
self._item_locations[k] = self._mark_document(doc, line_no, col_no)
self._set_cursor_pos(self._item_locations[item_no])
return False
def _mark_document(self, doc, line_no, col_no):
self._l.debug("Entered(doc, %d, %d)" % (line_no, col_no))
# line_no and col_no come from an external source so they can't be trusted.
# Unfortunately, gtk.TextBuffer.get_iter_at_line_offset just aborts the app
# if values are out of range. Top banana!
lines = doc.get_line_count()
if line_no < 0:
line_no = 0
if line_no > lines:
line_no = lines
if col_no < 0:
col_no = 0
cur_iter = doc.get_iter_at_line(line_no-1)
if col_no > cur_iter.get_chars_in_line():
col_no = cur_iter.get_chars_in_line()
if line_no == lines:
col_no += 1
self._l.debug("Creating mark at %d:%d" % (line_no, col_no))
cur_iter = doc.get_iter_at_line_offset(line_no-1, col_no-1)
return doc.create_mark(None, cur_iter, False)
def update_ui(self):
#self._l.debug("Entered")
pass
def run(path, film_thickness, logger):
manager = MeasurementManager(path, Geometry(film_thickness))
# Calculate the resist R from the U/I-Plot
for meas in manager.get_all():
x, y = meas.get_measured_values()
slope, intercept, std_err = linear_fit(x, y)
meas.resist = 1 / slope
# Calculate the contact resists
for temp in manager.temp_keys:
manager.contact_resist_dict[temp] = get_contact_resist(
manager.get_by_temp(temp))
# Correct the resists with the contact resist and the geometry
for meas in manager.get_all():
temp_key = meas.temp_celcius
meas.contact_resist = manager.contact_resist_dict[temp_key]
meas.film_resist = get_film_resist(meas, manager.geometry)
if meas.film_resist < 0:
logger.warning("Negative film resist in {0}".format(meas.name))
# Get activation energy and sigma_0 from an linearized Arrhenius plot
for dist in manager.dist_keys:
measurements = manager.get_by_dist(dist)
film_resists = [meas.film_resist for meas in measurements]
temps = array([meas.temp_kelvin for meas in measurements])
actv_energy, sigma_0 = arrhenius_fit(temps, film_resists)
manager.arrhenius_dict[dist] = (actv_energy, sigma_0)
output = Output(path)
output.summary(manager)
output.show_summary()
output.measurements_raw(manager)
output.arrhenius_raw(manager)
output.contact_resist_raw(manager)
output.film_resist_raw(manager)
def send_message(self,data):
event = self._setdefaults(data)
Output.event(event)
class Disco:
# instance of Deployer class for deploying, retrieving and deleting clusters
deployer = None
# store for given parameters during construction phase
params = None
def __init__(self, disco_config, params):
"""
creating a Disco class instance which will provide the regular interface to access the cluster lifecycle
:param disco_config: data needed for Disco class
:param params: possible further parameters (not used currently)
"""
self.deployer = disco_config['deployer']
self.framework_directory = disco_config['framework_directory']
self.root_component = disco_config['root_component']
self.root_component_state = disco_config['root_component_state']
self.params = params
self.components = {}
self.output = FinalOutput()
self.load_components()
def get_component_names(self):
"""
read names of available components
:return: available component names as array
"""
datadir = os.path.join(self.framework_directory)
print( os.path.dirname(os.path.realpath(__file__)))
return [name for name in os.listdir(datadir)
if os.path.isdir(os.path.join(datadir, name))]
def deploy(self):
"""
A new stack will be deployed here.
:return: return value of the deployer's deploy() function
"""
self.deploy_string = self.generate_output()
self.stackInfo = self.deployer.deploy(self.deploy_string)
return self.stackInfo
def load_components(self):
"""
each component will be instantiated and saved within the Disco instance
"""
component_names = self.get_component_names()
for component in component_names:
self.components[component] = Component(os.path.join(self.framework_directory,component), self.output)
def inject_requested_properties(self, properties):
"""
properties given by the end user over HTTP or default properties inserted by the framework
properties are of type {'componentname': {'componentproperty': 'value', ... }, ...}
:param properties: properties as dictionary of dictionary of values
"""
for component, props in properties.iteritems():
if component in self.components:
self.components[component].set_requested_properties(props)
def retrieve(self, parameter):
"""
forward the retrieve request to the deployer instance
:param parameter: parameter to retrieve the requested entity (probably a cluster)
:return: return value of the deployer instance
"""
return self.deployer.retrieve(parameter)
def delete(self, parameter):
"""
forward the delete request to the deployer instance
:param parameter: parameter to delete the requested entity (probably a cluster)
:return: return value of the deployer instance
"""
return self.deployer.delete(parameter)
def inject_dependency(self, component_name, dependencies):
'''
dependencies as {'dependency1': {'state1': {'value1': 'possible value','value2': None},
...},
'dependency2': {}
}
:param component_name: string of the componen
t's name to be injected into
:param dependencies: which dependencies (formatting see above)
:return:
'''
if component_name in self.components:
self.components[component_name].add_dependencies(dependencies)
def dep_resolve(self, component, resolved, unresolved):
"""
recursive dependency resolving algorithm based on 2 lists, one with the already resolved dependencies and one with the dependencies to be resolved
each component is described as a dictionary:
{ "name": <name of the component>,
"component": <the actual component's instance>,
"state": <the needed state of the requested component> }
:param component: name of the component that needs all dependencies
:param resolved: already resolved components in a list
:param unresolved: currently unresolved components in a list
"""
# the component to be resolved is obviously unresolved yet and needs to
# be added to the unresolved list
unresolved.append(component)
# get each dependency of the new component which has to be resolved
for comp, deps in component['component'].get_dependencies(component['state']).iteritems(): # {depname: {state: {dep1, dep2, ...},...},...}
# e.g. comp='shell', deps={'start': {}}
# each state of the depending components have to be handled
for state, dep in deps.iteritems():
# e.g. state='start', dep={}
# check whether a component with name saved in comp exists
if not comp in self.components:
raise Exception('No component %s registered' % (comp))
# check whether current dependent component with requested
# state is resolved already
if not self.list_contains_comp(resolved, {'name': self.components[comp].get_name(), 'component': self.components[comp], 'state': state}):
# if the found dependent component is not just in the
# resolved list but also in the unresolved list, something
# is wrong: then, a circular dependency is present
if self.list_contains_comp(unresolved, {'name': self.components[comp].get_name(), 'component': self.components[comp], 'state': state}):
raise Exception('Circular reference detected: %s:%s -> %s:%s' % (component['component'].get_name(), component['state'], comp, state))
# if the dependent component is not within the locally
# instantiated components, then there is something wrong as
# well
if not comp in self.components:
raise Exception('No component %s registered' % (comp))
# else, a new component dictionary can be "created" for the
# next recursive call of dep_resolve
new_comp = self.components[comp]
component_to_resolve = {'name': new_comp.get_name(), 'component': new_comp, 'state': state}
self.dep_resolve(component_to_resolve, resolved, unresolved)
elif len(deps) is not 0:
# here, the new requirements have to be inserted into existing framework instance
# if no new attributes are to be inserted, no call necessary as the framework will be installed anyway
#TODO: comment this in and write method
# self.add_attributes_to_comp(deps, comp, resolved)
pass
# finally, the component has been found and can be switched from the
# unresolved list to the resolved list
resolved.append(component)
unresolved.remove(component)
def list_contains_comp(self, compList, compName):
"""
list_contains_comp checks whether a component is within a given list
this is a slightly more complicated call because the component also has
a state
:param compList: haystack; list containing the components, can be the resolved or unresolved list
:param compName: dictionary of needle
:return: True if component is within list, else False
"""
for fw in iter(compList):
if fw['component'].get_name()==compName['component'].get_name() and fw['state']==compName['state']:
return True
return False
def resolve_dependencies(self):
"""
initialise the component resolving (i.e. dependency handling) of the
components
"""
self.resolved_dependencies = []
self.dep_resolve(
{
'name': self.components[self.root_component].get_name(),
'component': self.components[self.root_component],
'state':self.root_component_state
},
self.resolved_dependencies,
[])
def generate_output(self):
'''
generate the entire output within the Output (FinalOutput) class
:return:
'''
for current_component in self.resolved_dependencies:
current_component['component'].set_property('state', current_component['state'])
current_component['component'].handle_output()
return self.output.get_output()
class ExecuteConverter:
def __init__(self):
self.__resultTuple = None
def extractTables(self, path, target):
"""
Starts the table extraction. Using only this method, nothing will be returned,
but the HTML output Files will be created in the specified output folder.
"""
try:
os.mkdir(target)
except OSError:
pass
os.chdir(target)
self.__dtdFile = open(target + "/pdf2xml.dtd", "w")
self.buildDtd()
self.__cmdLine = "pdftohtml -xml " + path
print(self.__cmdLine)
os.system(self.__cmdLine)
xmlFile = os.path.basename(path).rstrip(".pdf") + ".xml"
fileMover.moveXmlFile(path = path, target = target)
#starting the extraction
firstClassification = FirstClassification(target)
self.__resultTuple = firstClassification.run(target + "/" + xmlFile)
tableList = self.__resultTuple[0]
fontsList = self.__resultTuple[1]
path = self.__resultTuple[2]
self.__outputObj = Output(tableList, fontsList, path)
self.__outputObj.createOutput()
def getTableList(self, outputTypeObj = GetOutputStringIOList()):
"""
Return a list. The output type depends of the parameter.
The default return type is a list of stringIO's containing the content of
the generate HTML output files
"""
self.__outputObj.setOutputType(outputTypeObj)
outputFilesList = self.__outputObj.getOutputList()
return outputFilesList
def buildDtd(self):
dtd = "<?xml version=\"1.0\" encoding=\"iso-8859-1\"?>\n" + \
"<!ELEMENT pdf2xml (page+,line*,fontspec*)>\n" + \
"<!ELEMENT page (fontspec*, text*)>\n" + \
"<!ATTLIST page\n" + \
"number CDATA #REQUIRED\n" + \
"position CDATA #REQUIRED\n" + \
"top CDATA #REQUIRED\n" + \
"left CDATA #REQUIRED\n" + \
"height CDATA #REQUIRED\n" + \
"width CDATA #REQUIRED\n" + \
">\n" + \
"<!ELEMENT fontspec EMPTY>\n" + \
"<!ATTLIST fontspec\n" + \
"id CDATA #REQUIRED\n" + \
"size CDATA #REQUIRED\n" + \
"family CDATA #REQUIRED\n" + \
"color CDATA #REQUIRED\n" + \
">\n" + \
"<!ELEMENT text (#PCDATA | b | i)*>\n" + \
"<!ATTLIST text\n" + \
"top CDATA #REQUIRED\n" + \
"left CDATA #REQUIRED\n" + \
"width CDATA #REQUIRED\n" + \
"height CDATA #REQUIRED\n" + \
"font CDATA #REQUIRED\n" + \
">\n" + \
"<!ELEMENT b (#PCDATA)>\n" + \
"<!ELEMENT i (#PCDATA)>\n" + \
"<!ELEMENT line (text+)>\n" + \
"<!ATTLIST line\n" + \
"typ CDATA #REQUIRED\n" + \
"top CDATA #REQUIRED\n" + \
"left CDATA #REQUIRED\n" + \
"font CDATA #REQUIRED\n" + \
">"
self.__dtdFile.write(dtd)
self.__dtdFile.close()
