def get_object_from_dict(values): direction = Direction() direction.type = XmlParser.get_direction_type(values) direction.name = XmlParser.get_title(values) direction.tag = XmlParser.get_tag(values) return direction
def main(): parser = argparse.ArgumentParser(description='Stackoverflow data dump XML parser.') parser.add_argument('--file', help='pass the file to parse the xml') parser.add_argument('--delimiter', help='delimiter in the file') parser.add_argument('--cassandraIP', help='Cassandra DB IP address') parser.add_argument('--cassandraKeyspace', help='Cassandra Keyspace') args = parser.parse_args() if args.file is None: print(args.file) sys.exit(0) xmlParser = XmlParser(args.file) list = xmlParser.parseXml() #print(list.count()) cassandraIP = args.cassandraIP cassandraKeyspace = args.cassandraKeyspace if cassandraIP is not None and cassandraKeyspace is not None: connector = CassandraConnector(cassandraIP, cassandraKeyspace) connector.connect() connector.writeData() connector._ip else: print(args.file) sys.exit(0)
def loadXML(self): if self.needsToLoadXML: self.XMLParser = XmlParser(self.filePath, self.GUICreator) returnData = self.XMLParser.getConfigData() self.setTheme(returnData[Constants.THEME_ATTRIBUTE]) self.tabsByXMLFIle[self.filePath] = returnData["tabNames"] self.needsToLoadXML = False
def get_object_from_dict(values): location = Location() latitude = XmlParser.get_latitude(values) longitude = XmlParser.get_longitude(values) location = Location(latitude, longitude) return location
def parse_file(path, name): xml_file_path = "./xmlResults/" + name + ".xml" print "parsing by WCRFT ..." XmlParser.produce_xml_with_morphological_data(path, xml_file_path) print "parsed by WCRFT ..." clear_author_characteristics(name) print "saving in db..." add_new_record(path, name, xml_file_path)
def parse_file(path, name): xml_file_path = './xmlResults/' + name + '.xml' print "parsing by WCRFT ..." XmlParser.produce_xml_with_morphological_data(path, xml_file_path) print "parsed by WCRFT ..." clear_author_characteristics(name) print "saving in db..." add_new_record(path, name, xml_file_path)
def get_object_from_dict(values): vehicle_state = VehicleState() latitude = XmlParser.get_latitude(values) longitude = XmlParser.get_longitude(values) location = Location(latitude=latitude, longitude=longitude) vehicle_state.location = location vehicle_state.heading = XmlParser.get_heading(values) vehicle_state.speed = XmlParser.get_speed(values) return vehicle_state
def get_object_from_dict(values): route = Route() route.name = XmlParser.get_title(values) route.tag = XmlParser.get_tag(values) latitude_min = RouteXmlParser.get_latitude_min(values) latitude_max = RouteXmlParser.get_latitude_max(values) longitude_min = RouteXmlParser.get_longitude_min(values) longtitude_max = RouteXmlParser.get_longitude_max(values) route.bounding_box = [ [latitude_min, latitude_max], [longitude_min, longtitude_max] ] return route
def parse_all_texts(texts_path): author_text_files = [f for f in listdir(texts_path) if isfile(join(texts_path, f))] clear_characteristics_collection() # todo: uncomment when VB with morphological parsing will be enabled shutil.rmtree("./xmlResults/") for author_text in author_text_files: author_name = author_text.split("_")[0] xml_file_path = "./xmlResults/" + author_name + ".xml" # todo: uncomment when VB with morphological parsing will be enabled XmlParser.produce_xml_with_morphological_data(texts_path + "/" + author_text, xml_file_path) add_new_record(texts_path + "/" + author_text, author_name, xml_file_path)
def read_label_from_xml(label_path, care_types): labels = XmlParser.parseXML(label_path) label_dic = {} tracklet_counter = 0 for label in labels: obj_type = label.objectType care = False for care_type in care_types: if obj_type == care_type: care = True break if care: tracklet_counter += 1 first_frame = label.firstFrame nframes = label.nFrames size = label.size for index, place, rotate in zip( range(first_frame, first_frame + nframes), label.trans, label.rots): if index in label_dic.keys(): # array merged using vertical stack label_dic[index]["place"] = np.vstack( (label_dic[index]["place"], place)) label_dic[index]["size"] = np.vstack( (label_dic[index]["size"], np.array(size))) label_dic[index]["rotate"] = np.vstack( (label_dic[index]["rotate"], rotate)) else: # inited as array label_dic[index] = {} label_dic[index]["place"] = place label_dic[index]["rotate"] = rotate label_dic[index]["size"] = np.array(size) return label_dic, tracklet_counter
def parse_all_texts(texts_path): author_text_files = [ f for f in listdir(texts_path) if isfile(join(texts_path, f)) ] clear_characteristics_collection() # todo: uncomment when VB with morphological parsing will be enabled shutil.rmtree('./xmlResults/') for author_text in author_text_files: author_name = author_text.split('_')[0] xml_file_path = './xmlResults/' + author_name + '.xml' # todo: uncomment when VB with morphological parsing will be enabled XmlParser.produce_xml_with_morphological_data( texts_path + '/' + author_text, xml_file_path) add_new_record(texts_path + '/' + author_text, author_name, xml_file_path)
def parse_article(filename): title, text = XmlParser.parse_article(filename) intro, unnamed_sections, named_sections = TextParser.parse_text(text) uncited_intro, citations = ContentParser.parse_content(intro.content) section_metas = {} uncited_sections = [] for section in unnamed_sections: # Map section level to other section meta information section_meta = section.meta section_meta_level = section_meta.level if not section_meta_level in section_metas: section_metas[section_meta_level] = [] section_metas[section_meta_level].append(section_meta) # Split section content into citations and actual content uncited_section, section_citations = ContentParser.parse_content( section.content) uncited_sections.append(uncited_section) citations = citations + section_citations return Article.Article(title, intro.meta, uncited_intro, section_metas, uncited_sections, named_sections, citations)
def create_new_record(xml_string, text_classes, raw_text): t0 = time() xml_tree = XmlParser(xml_string) print "XML tree generated in %fs" % (time() - t0) text = xml_tree.get_base_words() t0 = time() numerical_characteristics = count_numerical_statistics(text) print "Numerical characteristics counted in %fs" % (time() - t0) xml_tree.set_number_of_words(numerical_characteristics["words_count"]) t0 = time() author_position_characteristics = { "raw_text": raw_text, "parts_of_speech_frequencies": xml_tree.get_parts_of_speech_frequency(), "classes": text_classes } print "Parts of speech characteristics counted in %fs" % (time() - t0) author_position_characteristics.update(numerical_characteristics) t0 = time() author_position_characteristics.update( xml_tree.get_top_and_rarest_used_words()) print "Top and rarest words counted in %fs" % (time() - t0) return author_position_characteristics
def get_object_from_xml(xml, vehicles, current_time): root_xml = xmlLib.fromstring(xml) vehicles_state_xml = root_xml.findall(VehicleStateXmlParser.VEHICLE_TAG) for vehicle_state_xml in vehicles_state_xml: vehicle_state = VehicleStateXmlParser.get_object_from_dict(vehicle_state_xml.attrib) vehicle_state.time = current_time vehicle_id = XmlParser.get_vehicle_id(vehicle_state_xml.attrib) current_vehicles = filter(lambda x: x.id == vehicle_id, vehicles) if len(current_vehicles) != 0: current_vehicles[0].states.append(vehicle_state) else: route_tag = XmlParser.get_route_tag(vehicle_state_xml.attrib) vehicle = Vehicle(vehicle_id=vehicle_id, route=route_tag) vehicle.states.append(vehicle_state) vehicles.append(vehicle) return vehicles
def get_object_from_xml(xml, stops): directions_xml = xml.findall(DirectionXmlParser.DIRECTION_TAG) directions = [] for direction_xml in directions_xml: direction = DirectionXmlParser.get_object_from_dict(direction_xml.attrib) stops_xml = direction_xml.findall(DirectionXmlParser.STOP_TAG) for stop_xml in stops_xml: tag = XmlParser.get_tag(stop_xml.attrib) stop = filter(lambda x: x.tag == int(tag), stops)[0] direction.stops.append(stop) directions.append(direction) return directions
def create_new_record(xml_string, text_classes, raw_text): t0 = time() xml_tree = XmlParser(xml_string) print "XML tree generated in %fs" % (time() - t0) text = xml_tree.get_base_words() t0 = time() numerical_characteristics = count_numerical_statistics(text) print "Numerical characteristics counted in %fs" % (time() - t0) xml_tree.set_number_of_words(numerical_characteristics["words_count"]) t0 = time() author_position_characteristics = { "raw_text": raw_text, "parts_of_speech_frequencies": xml_tree.get_parts_of_speech_frequency(), "classes": text_classes } print "Parts of speech characteristics counted in %fs" % (time() - t0) author_position_characteristics.update(numerical_characteristics) t0 = time() author_position_characteristics.update(xml_tree.get_top_and_rarest_used_words()) print "Top and rarest words counted in %fs" % (time() - t0) return author_position_characteristics
class CoreGUI(threading.Thread): """This class handles the underlying functionality of updating widgets, running, and creating the GUI""" CustomWidgetList = [] themes = {} def __init__(self, filePath, createSettings=False, loadXMLFirst=True): self.filePath = filePath self.GUIStarted = False self.GUIDone = False self.stopCommanded = False self.GUICreator = None self.mainWindow = None self.XMLParser = None self.createSettingsTab = createSettings self.loadXMLFirst = loadXMLFirst self.needsToLoadXML = loadXMLFirst self.tabsByXMLFIle = {} self.activeClickedWidget = None self.activeOffset = [0, 0] self.dataPassDict = {} self.consoleDict = {} self.returnDict = {} self.callbackQueue = [] self.rainbow = False self.hideOnClick = False self.hue = 0 self.theme = "none" # Add new hardcoded themes here # Can't use the names "Dark", "Light", and "Blue" # Theme colors are defined as Background, WidgetBackground, Default Text, Header Text, Border self.themes["Better Dark"] = [ "rgb[13, 17, 23]", "rgb[13, 17, 23]", "rgb[139,148,158]", "rgb[88,166,255]", "rgb[139,148,158]" ] self.themes["Dark 2"] = [ "rgb[20, 20, 20]", "rgb[25, 25, 25]", "rgb[230,230,255]", "rgb[230,230,255]", "rgb[50,50,50]" ] self.themes["Green Text"] = [ "rgb[20, 20, 20]", "rgb[25, 25, 25]", "rgb[150,150,150]", "rgb[10,200,10]", "rgb[50,50,50]" ] # Start the GUI threading.Thread.__init__(self) self.start() # Wait for the gui to actually start running while not self.GUIStarted: time.sleep(0.001) def run(self): self.GUICreator = GUIMaker() self.mainWindow = self.GUICreator.getMainWindow() # Create menu bar menuBar = self.mainWindow.menuBar() fileMenu = menuBar.addMenu("File") fileMenu.addAction("Open") # Reload GUI from scratch??? OUCH! fileMenu.addAction("Save", self.saveGUI) # Save to XML fileMenu.addAction("Save As", self.saveGUIAs) fileMenu.addSeparator() fileMenu.addAction("Quit", self.stop) # Menu bar for widgets widgetMenu = menuBar.addMenu("Widgets") colorSubMenu = widgetMenu.addMenu("Set Color") colorSubMenu.addAction( "White", lambda color="white": self.setColorOnALlWidgets(color)) colorSubMenu.addAction( "Blue", lambda color="blue": self.setColorOnALlWidgets(color)) colorSubMenu.addAction( "Red", lambda color="rgb[255,0,0]": self.setColorOnALlWidgets(color)) colorSubMenu.addAction( "Green", lambda color="rgb[0,100,0]": self.setColorOnALlWidgets(color)) colorSubMenu.addAction( "Gray", lambda color="rgb[50,50,50]": self.setColorOnALlWidgets(color)) colorSubMenu.addAction( "Default", lambda color="default": self.setColorOnALlWidgets(color)) colorSubMenu.addSeparator() colorSubMenu.addAction("Toggle Rainbow", self.toggleRainbow) widgetMenu.addSeparator() # Menu to create new widget newWidgetSubMenu = widgetMenu.addMenu("New") for item in self.GUICreator.getAvailableWidgets(): newWidgetSubMenu.addAction( item, lambda name=item: self.makeNewWidgetInCurrentTab(name)) widgetMenu.addSeparator() widgetMenu.addAction( "Lock all widgets", lambda draggable=False: self.setDraggingOnALlWidgets(draggable)) widgetMenu.addAction( "Unlock all widgets", lambda draggable=True: self.setDraggingOnALlWidgets(draggable)) widgetMenu.addAction( "Disable hide on click", lambda enabled=False: self.setHideOnClick(enabled)) widgetMenu.addAction("Enable hide on click", lambda enabled=True: self.setHideOnClick(enabled)) widgetMenu.addAction("Show all widgets", self.showAllWidgets) # Menu bar for themes themeMenu = menuBar.addMenu("Theme") for theme in self.themes: themeMenu.addAction( theme, lambda themeName=theme: self.setTheme(themeName)) themeMenu.addSection("Default") themeMenu.addAction("Dark", lambda themeName="Dark": self.setTheme(themeName)) themeMenu.addAction("Blue", lambda themeName="Blue": self.setTheme(themeName)) themeMenu.addAction("Light", lambda themeName="Light": self.setTheme(themeName)) themeMenu.addSection("Stupid") themeMenu.addAction("Red", lambda theme="rgb[100,0,0]": self.setTheme(theme)) themeMenu.addAction("Black", lambda theme="rgb[0,0,0]": self.setTheme(theme)) helpMenu = menuBar.addMenu("Help") helpMenu.addAction("Whack Patrick", self.toggleRainbow) if self.createSettingsTab: # Widgets that only run in test mode. Used for testing stuff before it's completely done self.GUICreator.createTab("Settings") self.GUICreator.createTextBoxDropDownWidget("Settings", 100, 100) self.GUICreator.createButton( "Settings", 100, 300, {Constants.TITLE_ATTRIBUTE: "Whack Patrick"}) self.GUICreator.createSimpleDropDown("Settings", 400, 100) self.GUICreator.createAnnunciatorPanelWidget("Settings", 500, 500) self.GUICreator.createSimpleConsoleWidget("Settings", 600, 500) self.GUICreator.createCompleteConsoleWidget( "Settings", 700, 500, {Constants.SOURCE_ATTRIBUTE: "complete_console_test"}) self.GUICreator.createBrowse("Settings", 500, 100) self.GUICreator.createWidgetFromName("FullFlightDisplay", "Settings", 900, 100) self.GUICreator.createWidgetFromName("MultiBarGraph", "Settings", 200, 200) self.GUICreator.createWidgetFromName("ROVStatusWidget", "Settings", 600, 300) self.GUICreator.createWidgetFromName("StatusEventWidget", "Settings", 600, 100) self.GUICreator.createWidgetFromName("MapWidget", "Settings", 50, 500) if self.loadXMLFirst: self.loadXML() self.setupEventHandler() # QTimer to run the update method timer = QTimer() timer.timeout.connect(self.updateGUI) timer.start(10) # Start GUI thread self.GUIStarted = True self.GUICreator.start() self.GUIDone = True def loadXML(self): if self.needsToLoadXML: self.XMLParser = XmlParser(self.filePath, self.GUICreator) returnData = self.XMLParser.getConfigData() self.setTheme(returnData[Constants.THEME_ATTRIBUTE]) self.tabsByXMLFIle[self.filePath] = returnData["tabNames"] self.needsToLoadXML = False def stop(self): self.stopCommanded = True def updateDataPassDict(self, dataPassDict): self.dataPassDict = dataPassDict def updateConsole(self, name, value): """HI""" if name not in self.consoleDict: self.consoleDict[name] = [] self.consoleDict[name] = ([value] + self.consoleDict[name])[:20] def getReturnDict(self): return self.returnDict def getCallbackQueue(self): return self.callbackQueue def clearCallbackQueue(self): self.callbackQueue = [] def setTheme(self, theme: str): """Function to set a theme for the whole GUI""" self.theme = theme # Legacy themes if theme == "Dark": self.setColorOnALlWidgets("rgb[40,40,40]") self.GUICreator.setGUIColor(30, 30, 30) elif theme == "Light": self.setColorOnALlWidgets("default") self.GUICreator.setGUIColor(250, 250, 250) elif theme == "Blue": self.setColorOnALlWidgets("rgb[0,0,50]") self.GUICreator.setGUIColor(0, 0, 40) elif "rgb[" in theme: self.setColorOnALlWidgets(theme) [red, green, blue] = theme.split("[")[1].split("]")[0].split(",") self.GUICreator.setGUIColor(int(float(red)), int(float(green)), int(float(blue))) elif theme in self.themes: themeData = self.themes[theme] [red, green, blue] = themeData[0].split("[")[1].split("]")[0].split(",") self.setColorOnALlWidgets(themeData[1], themeData[2], themeData[3], themeData[4]) self.GUICreator.setGUIColor(int(float(red)), int(float(green)), int(float(blue))) def updateGUI(self): if self.stopCommanded: self.GUICreator.stop() if self.needsToLoadXML: self.loadXML() for key in self.consoleDict: self.dataPassDict[key] = self.consoleDict[key] listOfWidgets = self.GUICreator.getWidgetList() returnDict = {} for widget in listOfWidgets: widget.update(self.dataPassDict, listOfWidgets) if widget.returnsData(): returnDict[widget.getReturnKey()] = widget.getData() returnEvents = widget.getReturnEvents() for event in returnEvents: self.callbackQueue.append(event) self.returnDict = copy.deepcopy(returnDict) if self.rainbow: # Somehow this is important enough to go here! self.hue += 0.01 if self.hue > 1: self.hue = 0 [red, green, blue] = colorsys.hsv_to_rgb(self.hue, 1, 1) self.setColorOnALlWidgets("rgb[{0},{1},{2}]".format( red * 255, green * 255, blue * 255)) def setupEventHandler(self): """Overwrites mainWindow's event handlers to ones in this class""" self.mainWindow.mouseMoveEvent = self.mouseMoveEvent self.mainWindow.mouseReleaseEvent = self.mouseReleaseEvent self.mainWindow.mousePressEvent = self.mousePressEvent def mouseMoveEvent(self, e): """Moves the active widget to the position of the mouse if we are currently clicked""" if self.activeClickedWidget is not None and self.activeClickedWidget.isDraggable( ): x = clamp( e.x() - self.activeOffset[0], 0, float(self.mainWindow.width()) - self.activeClickedWidget.width - 5) y = clamp( e.y() - self.activeOffset[1], 0, float(self.mainWindow.height()) - self.activeClickedWidget.height - 55) self.activeClickedWidget.setPosition(x, y) def mousePressEvent(self, e: QMouseEvent): """Determines if we clicked on a widget""" currentTabIndex = self.mainWindow.centralWidget().currentIndex() tabList = self.GUICreator.getTabNames() if self.activeClickedWidget is None and e.button() == 1: listOfWidgets = self.GUICreator.getWidgetList() for widget in reversed(listOfWidgets): if widget.isPointInWidget(float(e.x()), float(e.y( ))) and tabList[currentTabIndex] == widget.getTabName(): if self.hideOnClick: widget.hide() else: self.activeClickedWidget = widget self.activeOffset = [ float(e.x()) - widget.x, float(e.y()) - widget.y ] return def mouseReleaseEvent(self, e): if self.activeClickedWidget is not None: self.activeClickedWidget = None def makeNewWidgetInCurrentTab(self, widgetName): currentTab = self.GUICreator.getTabNames()[ self.mainWindow.centralWidget().currentIndex()] self.GUICreator.createWidgetFromName(widgetName, currentTab, 300, 300) if self.theme not in self.themes: self.GUICreator.widgetList[-1].setColor( self.getColorFromTheme(self.theme)) else: themeData = self.themes[self.theme] self.GUICreator.widgetList[-1].setColor(themeData[1], themeData[2], themeData[3], themeData[4]) def getColorFromTheme(self, theme): if theme == "dark": return "grey" elif theme == "light": return "default" elif theme == "blue": return "rgb[0,0,50]" elif "rgb" in theme: return theme elif theme in self.themes: return self.themes[theme][0] return None def setColorOnALlWidgets(self, color, textColor=None, headerTextColor=None, borderColor=None): """Sets colors on all widgets""" listOfWidgets = self.GUICreator.getWidgetList() for widget in listOfWidgets: if textColor is not None and headerTextColor is not None and borderColor is not None: widget.setColor(color, textColor, headerTextColor, borderColor) elif textColor is not None: widget.setColor(color, textColor) else: widget.setColor(color) def setDraggingOnALlWidgets(self, draggable): """Sets colors on all widgets""" listOfWidgets = self.GUICreator.getWidgetList() for widget in listOfWidgets: widget.setDraggable(draggable) def toggleRainbow(self): self.rainbow = not self.rainbow def setHideOnClick(self, enabled: bool): self.hideOnClick = enabled def showAllWidgets(self): """Shows all widgets""" listOfWidgets = self.GUICreator.getWidgetList() for widget in listOfWidgets: widget.show() def saveGUI(self, fileName=None): """Generates output XML, and writes that to a file""" if fileName is not None: tabInfo = self.GUICreator.getTabNames() windowInfo = [ self.mainWindow.windowTitle(), self.mainWindow.width(), self.mainWindow.height(), self.theme ] XMLOutput(windowInfo, tabInfo, self.GUICreator.getWidgetList(), fileName, self.createSettingsTab) else: for file in self.tabsByXMLFIle: tabInfo = self.tabsByXMLFIle[file] windowInfo = [ self.mainWindow.windowTitle(), self.mainWindow.width(), self.mainWindow.height(), self.theme ] XMLOutput(windowInfo, tabInfo, self.GUICreator.getWidgetList(), file, self.createSettingsTab) def saveGUIAs(self): """Opens a file dialog, then calls the save function with the full filename""" name = QFileDialog.getSaveFileName(self.mainWindow, 'Save File') self.saveGUI(name[0])
from XmlParser import XmlParser from textCharacteristics import average_word_length, type_token_ratio, hapax_legomana_ratio, average_sentence_length, \ avg_sentence_complexity def print_results_in_file(path, content): print "Writing results to file results" with open(path, "w") as result_file: for result in content: result_file.write("%s\n" % result) def save_text_as_file(name, text): file = open(name, "w") file.write(text) file.write("\n") file.close() if __name__ == '__main__': text = open('./texts/toDetectFile', 'r').readlines() resultFileContent = ['average word length: ', average_word_length(text), 'type token ratio: ', type_token_ratio(text), 'hapax legomana ratio: ', hapax_legomana_ratio(text), 'average sentence length: ', average_sentence_length(text), 'average sentence complexity: ', avg_sentence_complexity(text)] XmlParser.produce_xml_with_morphological_data('./numericalResults', './xmlResults/toDetect.xml') xml_tree = XmlParser('./xmlResults/toDetect.xml')
avg_sentence_complexity def print_results_in_file(path, content): print "Writing results to file results" with open(path, "w") as result_file: for result in content: result_file.write("%s\n" % result) def save_text_as_file(name, text): file = open(name, "w") file.write(text) file.write("\n") file.close() if __name__ == '__main__': text = open('./texts/toDetectFile', 'r').readlines() resultFileContent = [ 'average word length: ', average_word_length(text), 'type token ratio: ', type_token_ratio(text), 'hapax legomana ratio: ', hapax_legomana_ratio(text), 'average sentence length: ', average_sentence_length(text), 'average sentence complexity: ', avg_sentence_complexity(text) ] XmlParser.produce_xml_with_morphological_data('./numericalResults', './xmlResults/toDetect.xml') xml_tree = XmlParser('./xmlResults/toDetect.xml')