Esempio n. 1
0
    def run_search(self):
        '''Master: creates workers to do the graph search.
        '''
        # initialize bfs
        self._todo_queue.put((self._start, None))
        self._reached[self._start] = self._start

        # apply BFS with the help of threads
        threads = []
        for _ in range(self._NUM_WORKER_THREADS):
            thr = threading.Thread(target=self.worker)
            # make sure the thread will eventually exit!
            thr.deamon = True
            threads.append(thr)
            thr.start()

        # block until all workers are done
        for thr in threads:
            if settings.debug:
                debug_log('join thread [{}]'.format(thr.name))
            thr.join()

        # program goes here iff graph search is done
        if self.found_target:
            try:
                if self._reached[self._end]:
                    self.generate_path()
            except KeyError:
                log('Cannot reach {} while generating path'.format(self._end))
Esempio n. 2
0
    def data(self, index, role=QtCore.Qt.DisplayRole):
        if not index.isValid() or \
           not (0 <= index.row() < self.rowCount()):
            return None
        link_list = sorted(self.network_model.linksFrom(self.xy))
        #log.debug_log("FWLM data: " + str(link_list))
        link_data = link_list[index.row()]
        link = link_data[2]
        if link_data[0] == self.xy:
            linked_world_xy = link_data[1]
        elif link[1] == self.xy:
            linked_world_xy = link_data[0]
        else:
            log.debug_log("Link does not contain orriginating hex: " + \
                          self.xy + " " + link_data)
        column = index.column()

        if role == QtCore.Qt.DisplayRole \
        or role == QtCore.Qt.EditRole \
        or role == ColorRole:
            
            if column == LINKED_WORLD_NAME:
                return self.world_model.getWorldAt(linked_world_xy).name

            if column == LINK_TYPE_NAME:
                return link.name

            if column == LINK_LINE:
                if role == ColorRole:
                    return link.color
                else:
                    return link.line_style

        return None
Esempio n. 3
0
 def __init__(self, network_index, parent=None):
     super(NetworkLineStyleComboBox, self).__init__(parent)
     log.debug_log("Set up Line Style Combo Box")
     self.network_index = network_index
     # Note self.model is a list model of line styles, and is not the network model
     self.setModel(QStringListModel(NWM.LINE_STYLE_LIST_STRINGS, parent))
     self.setItemDelegate(ComboLineStyleDelegate(self.network_index, self))
Esempio n. 4
0
    def projectManager(self):
        projectManager = ProjectManager.ProjectManager(self.model, self)
        # Run the project manager dialog to select a project
        # If valid project was selected and loaded into model, returns 1
        result = projectManager.exec_()
        app.processEvents()

        if result == 1:
            self.worldItemView = MapScene.WorldItemView(self.model)
            self.scene = self.worldItemView.scene

            self.view = MapView(self.scene)
            self.view.setRenderHints(QtGui.QPainter.Antialiasing)
            
            self.view.setScene(self.scene)
            self.view.changeZoomLevel.connect(self.wheelScrollZoom)
            self.pointerGroupClicked(1)
            self.sceneScaleCombo.setCurrentIndex(self.model.initialZoomLevel)
            self.sceneScaleChanged(self.sceneScaleCombo.currentText())
            self.view.horizontalScrollBar().setValue(self.model.horizontalScroll)
            self.view.verticalScrollBar().setValue(self.model.verticalScroll)
            self.selectionTypeCombo.setCurrentIndex(0)
            self.quickLinkCheckBox.setChecked(self.scene.quickLink)
            self.displayLinksCheckBox.setChecked(True)
            self.selectionTypeChanged(self.selectionTypeCombo.currentText())
            self.refreshRegionComboBox()
            
        elif result == 0:
            log.debug_log('Application closing. No project opened.')
            sys.exit()
Esempio n. 5
0
    def get_all_links(self):
        '''get all the links in one page, aka get all edges of one node
        Return: a list of tuples that the first element is the number of
        links, the second element is the name and
        the third element its the RELATIVE url.
        O(n) where n is approximately the size of the HTML tree.
        '''
        assert self.has_soup(), 'self._soup is None'

        # valid urls need to be in this format: start with /wiki/ and
        # not contain `:`.
        all_link_tags = self._soup.find(id='bodyContent')\
                            .find_all('a',
                                      href=re.compile('^(/wiki/)((?!:).)*$'))
        # invalid: aaa/wiki/
        # /wiki/aaa:bbb:
        # valid : /wiki/apple_pie
        # valie : /wiki/fruit/apple (not likely to occur)

        links = []
        for link_tag in all_link_tags:
            if 'href' in link_tag.attrs and link_tag.get_text() != 'ISBN':
                links.append(
                    (link_tag.get_text().lower(), link_tag.get('href')))
        if settings.debug:
            debug_log('number of links: {}'.format(len(links)))
        return links
Esempio n. 6
0
def make_export_dir(save_path, date_tag):
    try:
        #创健目录
        make_dir_p(save_path + date_tag + "/" + "new")
        make_dir_p(save_path + date_tag + "/" + "old")
    except Exception, e:
        debug_log(str(e))
        return False
Esempio n. 7
0
 def renameLinks(self, old_name, new_name):
     # Possibly not needed anymore as setData does the job
     found = False
     for link_type in self.link_types:
         if link_type.name == old_name:
             found = True
             link_type.name = new_name
             self.linksChanged.emit()
     if not found:
         log.debug_log("Link type not found: " + old_name)
Esempio n. 8
0
def export(commit_lst, save_path, date_tag, file_lst):
    for commit in commit_lst:
        local_file_path = save_path + date_tag + ("/new/" if commit == commit_lst[0] else "/old/")
        debug_log("export:local_file_path is %s" % local_file_path)

        # 拷贝到指定目录去
        for file_name in file_lst:
            debug_log("export:downloading %s..." % file_name)
            if not download_file(commit, file_name, local_file_path):
                return False
    return True
Esempio n. 9
0
    def __init__(self, start=None, relurl=None):
        '''`relurl` can be None if the url is not known yet. Crawler
        will automatically generate the url for you.
        start: name of lemma at the starting point
        relurl: @NOTE:RELATIVE url
        '''
        self._start = start
        self._url = self._BASE_RUL + relurl if relurl is not None else None
        if settings.debug:
            debug_log(self._url)

        self._soup = self._make_soup()
Esempio n. 10
0
def download_file(commit, path, local_path):
    ex_path = REMOTE_PATH + "/" + USER_NAME + "/" + PROJECT_NAME
    commit_path = "/" + commit + "/" + path
    local_file_path = local_path + path

    try:
        #创健目录
        make_dir_p(os.path.dirname(local_file_path))
    except Exception, e:
        debug_log("download_file:make dir %s error" % local_file_path)
        debug_log(str(e))
        return False
Esempio n. 11
0
    def __init__(self, network_model, scene, parent=None):
        super(NetworkManager, self).__init__(parent)
        log.debug_log('Started Network Manager init.')
        self.setAttribute(Qt.WA_DeleteOnClose)
        self.setWindowTitle("Network Manager")
        self.network_model = network_model
        self.scene = scene

        log.debug_log('Setting up Network Editor controlls')
        #self.networkTable = QTableView()
        self.networkTable = NetworkTableView(self.network_model)
        #self.networkTable.setModel(self.network_model)
        #self.networkTable.setItemDelegate(NetworkDelegate(self))
        self.networkTable.setItemDelegateForColumn(NWM.NET_CHECKED,
                                                   CheckBoxDelegate(self))
        self.networkTable.setItemDelegateForColumn(NWM.NET_COLOR,
                                                   NetworkColorDelegate(self))
        self.networkTable.setItemDelegateForColumn(NWM.NET_STYLE,
                                                   NetworkLineStyleDelegate(self))
        self.networkTable.setSelectionMode(QTableView.SingleSelection)
        self.networkTable.setSelectionBehavior(QTableView.SelectRows)
        self.networkTable.resizeColumnsToContents()

        self.addButton = QPushButton('Add')
        self.mergeButton = QPushButton('Merge')
        self.colorButton = QPushButton('Change Colour')
        self.deleteButton = QPushButton('Delete')
        self.okButton = QPushButton('OK')

        buttonLayout =  QVBoxLayout()
        buttonLayout.addWidget(self.addButton)
        buttonLayout.addWidget(self.mergeButton)
        buttonLayout.addWidget(self.colorButton)
        buttonLayout.addWidget(self.deleteButton)
        buttonLayout.addStretch(10)
        buttonLayout.addWidget(self.okButton)

        hLayout = QHBoxLayout()
        hLayout.addWidget(self.networkTable, 1)
        hLayout.addLayout(buttonLayout)
        self.setLayout(hLayout)

        self.addButton.clicked.connect(self.addButtonClicked)
        self.deleteButton.clicked.connect(self.deleteButtonClicked)
        self.colorButton.clicked.connect(self.colorButtonClicked)
        self.okButton.clicked.connect(self.okButtonClicked)

        self.network_model.dataChanged.connect(self.checkStateChanged)
        self.network_model.dataChanged.connect(
            self.networkTable.resizeColumnsToContents)
        #self.resetCheckStates()
        self.checkStateChanged()
Esempio n. 12
0
 def addLink(self, xy1, xy2, network_name):
     for node in (xy1, xy2):
         if node not in self.nodes:
             log.debug_log("Error: node not found in graph: " + str(node))
             return
         
     log.debug_log("addLink: "+ str(xy1) +' '+ str(xy2) +' '+ str(network_name))
     for lt in self.linkTypes:
         if lt.name == network_name:
             if xy1 in self.graph[xy2]:
                 for link in self.graph[xy1][xy2]['links']:
                     if link.name == network_name:
                         log.debug_log("Link already exists: " + \
                                       xy1 + ", " + xy2 + ", " + \
                                       network_name)
                         return
                 # Edge exists, but link doesn't
                 self.graph[xy1][xy2]['links'].append(Link(link_type=lt))
             else:
                 # Edge doesn't exist
                 self.graph.add_edge(xy1, xy2)
                 #print 'graph:', self.graph.edges() #, xy1, xy2 
                 self.graph[xy1][xy2]['links'] = [Link(link_type=lt)]
             self.linksChanged.emit()
             return
     log.debug_log("network does not exist: " + network_name)
Esempio n. 13
0
    def _make_soup(self):
        '''send request, handle response, and get soup object
        @NOTE: It will automatically generate url if url was
        originally passed in as `None`.
        Return:
            soup object if no Exception is raised.
            Otherwise return `None`.
        '''
        try:
            if self._url is None:
                self._url = self.generate_full_url()
                if settings.debug:
                    debug_log('get request...')
                    debug_log('url is {}'.format(self._url))

            req = Request(self._url, headers=self._HEADERS)
            response = urlopen(req)
            soup = BeautifulSoup(response, "html5lib")

        except URLError as e:
            err_msg = 'Cannot open the page with name {}: {}'.format(
                self._start, e)
            if settings.debug:
                debug_log(err_msg)
            log(err_msg)
            return None

        except UnicodeDecodeError as e:
            # example: https://en.wikipedia.org/wiki/Kronkåsa
            # 'Kronkåsa' contains non-ascii characters
            # We don't do encode & decode trick simply because it will
            # not be a reasonable keyword to search.
            err_msg = 'Cannot open the page with name {}: {}'.format(
                self._start, e)
            if settings.debug:
                debug_log(err_msg)
            log(err_msg)
            return None

        except http.client.BadStatusLine as e:
            err_msg = 'Cannot open the page with name {}: {}'.format(
                self._start, e)
            if settings.debug:
                debug_log(err_msg)
            log(err_msg)
            return None

        else:
            return soup
Esempio n. 14
0
    def removeLink(self, xy1, xy2, network_name):
        links = self.graph[xy1][xy2]['links']
        removed = False
        for link in links:
            if link.name == network_name:
                links.remove(link)
                removed = True
        if removed:
            self.graph[xy1][xy2]['links'] = links
        else:
            log.debug_log("removeLink - Link not found: " + \
                          str(xy1) + " " + str(xy2) + \
                          " " + str(network_name))

        self.linksChanged.emit()
Esempio n. 15
0
 def closeEvent(self, event=QtGui.QCloseEvent()):
     saveDialog = QtGui.QMessageBox()
     saveDialog.setText('Do you want to save any changes before exiting?')
     saveDialog.setStandardButtons(QtGui.QMessageBox.Save | QtGui.QMessageBox.Discard | QtGui.QMessageBox.Cancel)
     saveDialog.setDefaultButton(QtGui.QMessageBox.Save)
     ret = saveDialog.exec_()
     if ret == QtGui.QMessageBox.Save:
         self.save()
         log.debug_log('If built using PySide it takes a while to exit, be patient.')
         event.accept()
     elif ret == QtGui.QMessageBox.Discard:
         log.debug_log('If built using PySide it takes a while to exit, be patient.')
         event.accept()
     elif ret == QtGui.QMessageBox.Cancel:
         event.ignore()
         app.processEvents()
Esempio n. 16
0
 def createRegion(self):
     # Temporarily disable clearing of currently selected hexes
     #self.beginCreateRegion = True
     #ok = 0
     region_name, name_ok = QtGui.QInputDialog.getText(self, 'Crate New Region',
                                             'New Region Name:',
                                             QtGui.QLineEdit.Normal)
     if name_ok and region_name:
         added_ok = self.model.addNewHexRegion(str(region_name))
         #debug_log('App: New group "' + str(group_name) + '" added')
         if added_ok:
             self.refreshRegionComboBox()
             last_item = self.regionSelectionCombo.count() - 1
             self.regionSelectionCombo.setCurrentIndex(last_item)
     else:
         log.debug_log('App: Adding new region failed. Name: ' + str(region_name))
 def looking_for_path(common, num):
     '''trace back to find the path.
     '''
     path = []
     while common != self._start[num]:
         old_vertex = common
         if settings.debug:
             debug_log(common)
         path.append(common)
         common = self._reached[num][old_vertex]
         # this is aimed to prevent from loop in the graph
         if common == old_vertex:
             if settings.debug:
                 debug_log('current vertex[{}] == \
                        old vertex[{}]'.format(common, old_vertex))
             break
     return path
Esempio n. 18
0
  def updateData(self, headerInfo):
    """1. adjust the order of the header
       2. add the new data of the inserted header and save
    """
    if len(self.__data) == 1 or len(self.__data[0]) == 0:
      self.__headerInfo = headerInfo
      if len(self.__data) == 1:
        for h in headerInfo:
          self.__data.insert(0, [])
      return [KDataStatusError.STATUS_OK]

    orderInfo = []
    for h in headerInfo:
      f = False
      for idx in range(len(self.__headerInfo)):
        f = self.__headerInfo[idx].canConvert2(h)
        if f:
          orderInfo.append(idx)
          break
      if not f:
        orderInfo.append([self.getDefaultValue(h.dataType) \
          for i in range(len(self.__data[0]))])
    data = []
# debug--------------
    if __debug__:
      log.debug_log("org header: %s now header: %s\n org data: %s\n" 
          % ([unicode(h) for h in self.__headerInfo],
            [unicode(h) for h in headerInfo],
            self.__data))
# debug--------------
    for o in orderInfo:
      data.append(self.__data[o] if type(o) == int else o)
    data.append(self.__data[-1])

    hbk = self.__headerInfo
    dbk = self.__data
    self.__headerInfo = headerInfo
    self.__data = data
    ret = self.save()
    if ret[0] == KDataStatusError.STATUS_OK:
      KDataWarehouse.updateData(self.__filename, self.__data)
    else:
      self.__data = dbk
      self.__headerInfo = hbk
    return ret
Esempio n. 19
0
def do_export(begin_commit, end_commit, git_path, save_path, check_status):
    #获取当前目录
    cur_cwd = os.getcwd()

    # 做一下反斜替换
    git_path = git_path.replace("\\",'/')
    save_path = save_path.replace("\\",'/')

    # 读取配置数据
    load_config()

    # 获取真实的commit,end_commit可能没有填写
    if check_status:
        commit_lst = get_real_commit(begin_commit, git_path)
    else:
        commit_lst = [begin_commit, end_commit]

    # 获取要到处的文件列表
    file_lst = get_export_file(begin_commit, git_path)

    #切换会当前路径
    os.chdir(cur_cwd)

    # 对文件和commit做一个校验
    if len(commit_lst) != 2 or not check_valid_commit(commit_lst[0]) or not check_valid_commit(commit_lst[1]):
        return GET_COMMIT_ERROR

    if len(file_lst) == 0:
        return GET_FILE_ERROR

    # 获取时间标记
    date_tag = datetime.now().strftime('%Y-%m-%d-%H-%M-%S')

    # 创建指定的目录
    if not make_export_dir(save_path, date_tag):
        debug_log("do_export:make dir %s error" % save_path)
        return MAKE_DIR_ERROR

    if not export(commit_lst, save_path, date_tag, file_lst):
        debug_log("do_export:export file error")
        return EXPORT_FILE_ERROR

    return EXEC_SUCCESS
Esempio n. 20
0
    def removeRows(self, position, rows=1, index=QtCore.QModelIndex()):
        log.debug_log("WorldLinkTypes:About to remove row(s).")
        self.beginRemoveRows(QtCore.QModelIndex(), position, position + rows - 1)
        log.debug_log("WorldLinkTypes:Removing row(s).")
        dead_link_types = self.linkTypes[position:(position + rows)]

        edge_list = self.links
        # edges are like [xy1, xy2, link]
        for edge in edge_list:
            link = edge[2]
            if link.link_type in dead_link_types:
                self.removeLink(edge[0], edge[1], link.name)

        self.linkTypes = self.linkTypes[:position] + \
                         self.linkTypes[position + rows:]

        self.endRemoveRows()
        self.linksChanged.emit()
        self.dirty = True
        return True
Esempio n. 21
0
def show_result(result):
    '''result: returned value from searcher.result()
    '''
    log('path: ' + str(result['path']))
    log('degree: ' + str(result['degree']))
    #################
    # draw  graph####
    #################
    log('Rendering graph...')
    nx.draw_spectral(result['graph'], node_size=50)
    if settings.debug:
        debug_log('call plt to show the graph...')
    # save graph as .png
    img_suffix = datetime.datetime.now().strftime('%y%m%d_%H%M%S')
    plt.savefig('graph_img/graph_img' + img_suffix + '.png')

    # write the graph to a file
    if settings.debug:
        debug_log('storing graph...')
    fileio.write_graph(
        str(result['graph'].nodes()) + '\n' + str(result['graph'].edges()))
Esempio n. 22
0
def show_result(result):
    '''result: returned value from searcher.result()
    '''
    log('path1: ' + str(result['path'][0]))
    log('path2: ' + str(result['path'][1]))
    log('degree: ' + str(result['degree']))

    #################
    # draw  graph####
    #################
    log('Rendering graphs...')
    nx.draw(result['graph1'], node_size=50)
    if settings.debug:
        debug_log('call plt to save the graph1...')
    # save graph as .png
    img_suffix = datetime.datetime.now().strftime('%y%m%d_%H%M%S')
    plt.savefig('graph_img/graph_common_img1_' + img_suffix + '.png')

    nx.draw(result['graph2'], node_size=50)
    if settings.debug:
        debug_log('call plt to save the graph2...')
    # save graph as .png
    plt.savefig('graph_img/graph_common_img2_' + img_suffix + '.png')

    # write the graph to a file
    if settings.debug:
        debug_log('storing graph...')
    tmp_graphs = (str(result['graph1'].nodes()) + '\n' +
                  str(result['graph1'].edges()) + '\n' +
                  str(result['graph2'].nodes()) + '\n' +
                  str(result['graph2'].edges()))
    fileio.write_graph(tmp_graphs)
    def run_search(self):
        '''Master: creates workers to do the graph search.
            running time: O(n) where n is the number of thread.
        '''
        # initialize bfs
        self._todo_queues[0].put((self._start[0], None))
        self._reached[0][self._start[0]] = self._start[0]

        self._todo_queues[1].put((self._start[1], None))
        self._reached[1][self._start[1]] = self._start[1]

        # apply BFS with the help of threads
        threads1 = []
        threads2 = []
        for _ in range(self._NUM_WORKER_THREADS // 2):
            thr = threading.Thread(target=self.worker, args=(0, ))
            # make sure the thread will eventually exit!
            thr.deamon = True
            threads1.append(thr)
            thr.start()

        for _ in range(self._NUM_WORKER_THREADS // 2):
            thr = threading.Thread(target=self.worker, args=(1, ))
            # make sure the thread will eventually exit!
            thr.deamon = True
            threads2.append(thr)
            thr.start()

        # block until all workers are done
        for thr in threads1 + threads2:
            if settings.debug:
                debug_log('join thread [{}]'.format(thr.name))
            thr.join()

        # program goes here iff graph search is done
        if self.found_common_ancestor:
            log('common ancestor(s) found: {}'.format(
                str(self._common_ancestors)))
            self.generate_path()
Esempio n. 24
0
    def get_node_name(self):
        '''get the node name for the as the start name.
        O(n) where n is the approximately the size of HTML tree.
        '''
        try:
            title_node = self._soup.find(id="firstHeading")
            title = title_node.get_text().strip()
            if not title:
                raise ValueError('Cannot find title')

        except ValueError as e:
            log('Wikipedia change their DOM!')
            if settings.debug:
                debug_log(e)
            return None
        except Exception as e:
            log('iternal error detected')
            if settings.debug:
                debug_log(e)
            return None
        else:
            return title
Esempio n. 25
0
 def generate_path(self):
     '''Trace back to generate path
     Will raise a KeyError if a bad graph is parsed.
     '''
     curr_vertex = self._end
     path = []
     while curr_vertex != self._start:
         old_vertex = curr_vertex
         if settings.debug:
             debug_log(curr_vertex)
         path.append(curr_vertex)
         # change edge color of the graph such that the path is different
         # from other edges.
         # TODO
         curr_vertex = self._reached[old_vertex]
         # this is aimed to prevent from loop in the graph
         if curr_vertex == old_vertex:
             if settings.debug:
                 debug_log('current vertex[{}] == old vertex[{}]'.format(
                     curr_vertex, old_vertex))
             break
     path.append(self._start)
     self._path = path[::-1]
Esempio n. 26
0
	def check_config(self):
		#转换为整数
		convert_int(self.config)

		for path, value in self.config['backup'].items():
			if not os.path.exists(path):
				del self.config['backup'][path]
				debug_log('path %s not exist.' % path)
				continue
			if os.path.isfile(path):
				self.config['backup'][path]['type'] = 'file'
			elif os.path.isdir(path):
				self.config['backup'][path]['type'] = 'dir'
			else:
				del self.config['backup'][path]
				debug_log('path %s type not support.' % path)
				continue
			self.config['backup'][path]['dir'] = path

		#判断是否存在可备份文件
		if not len(self.config['backup']): return False

		return True
Esempio n. 27
0
def load_config():
    global REMOTE_PATH
    global USER_NAME
    global PROJECT_NAME

    REMOTE_PATH = DEFAULT_REMOTE_PATH
    USER_NAME = DEFAULT_USER_NAME
    PROJECT_NAME = DEFAULT_PROJECT_NAME

    f = open("config.ini")
    for line in f.readlines():
        info_lst = line.split("=")
        if len(info_lst) != 2:
            debug_log("load_config:not = in opt config")
            continue
        if info_lst[0].strip() == "REMOTE_PATH":
            REMOTE_PATH = info_lst[1].strip()
        elif info_lst[0].strip() == "USER_NAME":
            USER_NAME = info_lst[1].strip()
        elif info_lst[0].strip() == "PROJECT_NAME":
            PROJECT_NAME = info_lst[1].strip()
        else:
            debug_log("load_config:Error opt config %s" % info_lst[0])
Esempio n. 28
0
def write_graph(graph):
    '''write the graph to a file.
    Argument:
        graph: a dictiction representation of a
               graph.
    Return:
        None
    '''
    latest_file_name = 'path_dict_result.txt'

    # write results to a file.
    # latest_file_name is the file that stores the lastest
    # result.
    with open(latest_file_name, 'w') as result_file:
        if settings.debug:
            debug_log('writing path dict to file')
        result_file.write(str(graph))

    # create a unique name for each file in the graph_bank
    basename = BANK_PATH + 'graph'
    suffix = datetime.datetime.now().strftime('%y%m%d%H%M%S')
    filename = '_'.join([basename, suffix]) + '.txt'
    # copy the lastest result file to the graph_bank
    subprocess.run(['cp', latest_file_name, filename])
Esempio n. 29
0
 def __init__(self, network_index, parent=None):
     super(ComboLineStyleDelegate, self).__init__(parent)
     log.debug_log("Set up combo style delegate")
     self.network_index = network_index
Esempio n. 30
0
 def sizeHint(self, option, index):
     if index.column() == NWM.NET_COLOR:
         return QSize(64, 32)
     else:
         log.debug_log("NetworkColorDelegate: Size Hint Error")
Esempio n. 31
0
 def __init__(self, parent=None):
     super(NetworkColorDelegate, self).__init__(parent)
     log.debug_log("Set up color delegate")
    def worker(self, num):
        '''worker which implements DFS on the graph of wikipedia.
            running time: technically O(|E|+|V|) where |E| is the length
            of all edges and |V| is the length of all vertices, and thus
            the todo_queue are accumulated faster than the speed of graph
            parsing.
        '''
        while (not self.found_common_ancestor and not self.max_limit_reached
               and not self.invalid_start_point):
            # {
            # get a task
            curr_vertex_info = self._todo_queues[num].get()

            LOCK.acquire()
            try:
                self.nodes_counter += 1
            finally:
                LOCK.release()

            if self.nodes_counter > self._max_page:
                self._todo_queues[num].task_done()
                LOCK.acquire()
                try:
                    self.max_limit_reached = True
                finally:
                    LOCK.release()
                log('Maximum page limit reached. Terminating...')
                return

            if settings.debug:
                debug_log('##now at ({}), total {}, queue size {}##'.format(
                    curr_vertex_info, self.nodes_counter,
                    self._todo_queues[num].qsize()))

            # check if the common ancestors are found
            common_ancestors = self.common_ancestor_found()
            if common_ancestors:
                LOCK.acquire()
                try:
                    self.found_common_ancestor = True
                    self._common_ancestors = common_ancestors
                finally:
                    LOCK.release()
                self._todo_queues[num].task_done()
                return

            # do work
            crawler = Crawler(start=curr_vertex_info[0],
                              relurl=curr_vertex_info[1])
            if not crawler.has_soup():
                if self._todo_queues[num].empty():
                    LOCK.acquire()
                    try:
                        self.invalid_start_point = True
                    finally:
                        LOCK.release()
                    return
                # be tolerant, go to next iteration
                continue

            # get all link information; each element in neighbours
            # contains (name, url)
            neighbours = crawler.get_all_links()

            # put all neighbours into _todo_queue except ones already
            # visited
            for neighbour in neighbours:
                if (neighbour[0] not in self._reached[num]
                        and neighbour[0] not in curr_vertex_info[0]):
                    if self._reached[num][curr_vertex_info[0]] != neighbour[0]:
                        # ADD LOCK #######
                        LOCK.acquire()
                        try:
                            #################
                            # update graph###
                            #################
                            self._nx_digraphs[num].add_edge(
                                curr_vertex_info[0], neighbour[0])
                            self._reached[num][
                                neighbour[0]] = curr_vertex_info[0]
                        finally:
                            LOCK.release()
                    self._todo_queues[num].put(neighbour)
            # finish current task
            self._todo_queues[num].task_done()
        # } end while loop
        # Target has been found or max limit reached if goes here.
        if settings.debug:
            debug_log('killing worker...')
Esempio n. 33
0
            debug_log("load_config:Error opt config %s" % info_lst[0])

def download_file(commit, path, local_path):
    ex_path = REMOTE_PATH + "/" + USER_NAME + "/" + PROJECT_NAME
    commit_path = "/" + commit + "/" + path
    local_file_path = local_path + path

    try:
        #创健目录
        make_dir_p(os.path.dirname(local_file_path))
    except Exception, e:
        debug_log("download_file:make dir %s error" % local_file_path)
        debug_log(str(e))
        return False

    debug_log("download_file:download path is %s" % (ex_path + commit_path))
    try:
        urllib.urlretrieve(ex_path + commit_path, local_path + path, Schedule)
    except Exception, e:
        debug_log(str(e))
        return False
    return True

def export(commit_lst, save_path, date_tag, file_lst):
    for commit in commit_lst:
        local_file_path = save_path + date_tag + ("/new/" if commit == commit_lst[0] else "/old/")
        debug_log("export:local_file_path is %s" % local_file_path)

        # 拷贝到指定目录去
        for file_name in file_lst:
            debug_log("export:downloading %s..." % file_name)
Esempio n. 34
0
 def __init__(self, parent=None):
     super(NetworkLineStyleDelegate, self).__init__(parent)
     log.debug_log("Set up line style delegate")
Esempio n. 35
0
    def worker(self):
        '''worker which implements DFS on the graph of wikipedia.
        '''
        def target_found(tar):
            '''Notify user that the target is found and prepare
            for killing this worker.
            '''
            log('$$found target {}({})!$$'.format(self._end, tar))
            # modify self._end:
            LOCK.acquire()
            try:
                self._end = tar
                self.found_target = True
                self._todo_queue.task_done()
            finally:
                LOCK.release()

        while (not self.found_target and not self._max_limit_reached
               and not self._invalid_start_point):
            # {
            # get a task
            curr_vertex_info = self._todo_queue.get()
            LOCK.acquire()
            try:
                self.nodes_counter += 1
            finally:
                LOCK.release()
            if self.nodes_counter > self._max_page:
                self._todo_queue.task_done()
                self._max_limit_reached = True
                log('Maximum page limit reached. Terminating...')
                return

            if settings.debug:
                debug_log('##now at ({}), total {}, queue size {}##'.format(
                    curr_vertex_info, self.nodes_counter,
                    self._todo_queue.qsize()))
            if self._end in curr_vertex_info[0]:
                target_found(curr_vertex_info[0])
                return

            # do work
            crawler = Crawler(start=curr_vertex_info[0],
                              relurl=curr_vertex_info[1])
            if not crawler.has_soup():
                if self._todo_queue.empty():
                    LOCK.acquire()
                    try:
                        self._invalid_start_point = True
                    finally:
                        LOCK.release()
                    return
                # else, be tolerant, go to next iteration
                continue

            # get all link information; each element in neighbours
            # contains (name, url)
            neighbours = crawler.get_all_links()

            # put all neighbours into _todo_queue except ones already
            # visited
            for neighbour in neighbours:
                if (neighbour[0] not in self._reached
                        and neighbour[0] not in curr_vertex_info[0]):
                    if self._reached[curr_vertex_info[0]] != neighbour[0]:
                        # ADD LOCK ##########
                        LOCK.acquire()
                        try:
                            #################
                            # update graph###
                            #################
                            self._nx_digraph.add_edge(curr_vertex_info[0],
                                                      neighbour[0])
                            self._reached[neighbour[0]] = curr_vertex_info[0]
                        finally:
                            LOCK.release()
                    self._todo_queue.put(neighbour)
                    if self._end in neighbour[0]:
                        target_found(neighbour[0])
                        return
            # finish current task
            self._todo_queue.task_done()
        # } end while loop
        # Target has been found or max limit reached if the thread goes here.
        if settings.debug:
            debug_log('killing worker...')
Esempio n. 36
0
 def __init__(self, parent=None):
     super(CheckBoxDelegate, self).__init__(parent)
     log.debug_log("Set up CheckBoxDelegate delegate")
Esempio n. 37
0
    def render(self, nx_graph, path_list, figure=plt):
        '''Draw graph, store dot file, convert dot to pdf.
        '''
        #################
        # draw  graph####
        #################
        img_suffix = datetime.datetime.now().strftime('%y%m%d_%H%M%S')
        img_name = 'graph_img/graph_img' + img_suffix
        dot_name = img_name + '.dot'
        if settings.debug:
            debug_log('generating {} file...'.format(dot_name))
        write_dot(nx_graph, dot_name)

        if settings.debug:
            debug_log('generating {}.pdf file from .dot'
                      'file...'.format(dot_name))
        # save graph as .png
        subprocess.run(['dot', '-Tpdf', '-O', dot_name])
        subprocess.run(['open', dot_name+'.pdf'])

        log('Rendering graph...')
        # nx.draw_spring(nx_graph, node_size=50)
        ###################################################
        # change color of node[ok] and path###
        ###################################################
        start_node = path_list[0]
        end_node = path_list[-1]
        nx_graph_pos = nx.spring_layout(nx_graph)
        nx.draw_networkx(nx_graph, nx_graph_pos,
                         with_labels=False,
                         node_size=50)
        # change color of nodes along the path
        nx.draw_networkx_nodes(nx_graph,
                               pos=nx_graph_pos,
                               nodelist=path_list,
                               node_color='g',
                               node_size=200)
        # change color of end-point nodes
        nx.draw_networkx_nodes(nx_graph,
                               pos=nx_graph_pos,
                               nodelist=[start_node, end_node],
                               node_color='b',
                               node_size=200)
        # change path color
        path_edge_list = [(path_list[i], path_list[i+1])
                          for i in range(len(path_list)-1)]
        nx.draw_networkx_edges(nx_graph,
                               pos=nx_graph_pos,
                               edgelist=path_edge_list,
                               width=1.0,
                               edge_color='b')

        # write the graph to a file
        if settings.debug:
            debug_log('storing graph...')
        fileio.write_graph(nx_graph.nodes())
        fileio.write_graph(nx_graph.edges())

        log('Done')
        plt.axis('off')
        figure.show()