def run_search(self):
'''Master: creates workers to do the graph search.
'''
# initialize bfs
self._todo_queue.put((self._start, None))
self._reached[self._start] = self._start
# apply BFS with the help of threads
threads = []
for _ in range(self._NUM_WORKER_THREADS):
thr = threading.Thread(target=self.worker)
# make sure the thread will eventually exit!
thr.deamon = True
threads.append(thr)
thr.start()
# block until all workers are done
for thr in threads:
if settings.debug:
debug_log('join thread [{}]'.format(thr.name))
thr.join()
# program goes here iff graph search is done
if self.found_target:
try:
if self._reached[self._end]:
self.generate_path()
except KeyError:
log('Cannot reach {} while generating path'.format(self._end))
def data(self, index, role=QtCore.Qt.DisplayRole):
if not index.isValid() or \
not (0 <= index.row() < self.rowCount()):
return None
link_list = sorted(self.network_model.linksFrom(self.xy))
#log.debug_log("FWLM data: " + str(link_list))
link_data = link_list[index.row()]
link = link_data[2]
if link_data[0] == self.xy:
linked_world_xy = link_data[1]
elif link[1] == self.xy:
linked_world_xy = link_data[0]
else:
log.debug_log("Link does not contain orriginating hex: " + \
self.xy + " " + link_data)
column = index.column()
if role == QtCore.Qt.DisplayRole \
or role == QtCore.Qt.EditRole \
or role == ColorRole:
if column == LINKED_WORLD_NAME:
return self.world_model.getWorldAt(linked_world_xy).name
if column == LINK_TYPE_NAME:
return link.name
if column == LINK_LINE:
if role == ColorRole:
return link.color
else:
return link.line_style
return None
def __init__(self, network_index, parent=None):
super(NetworkLineStyleComboBox, self).__init__(parent)
log.debug_log("Set up Line Style Combo Box")
self.network_index = network_index
# Note self.model is a list model of line styles, and is not the network model
self.setModel(QStringListModel(NWM.LINE_STYLE_LIST_STRINGS, parent))
self.setItemDelegate(ComboLineStyleDelegate(self.network_index, self))
def projectManager(self):
projectManager = ProjectManager.ProjectManager(self.model, self)
# Run the project manager dialog to select a project
# If valid project was selected and loaded into model, returns 1
result = projectManager.exec_()
app.processEvents()
if result == 1:
self.worldItemView = MapScene.WorldItemView(self.model)
self.scene = self.worldItemView.scene
self.view = MapView(self.scene)
self.view.setRenderHints(QtGui.QPainter.Antialiasing)
self.view.setScene(self.scene)
self.view.changeZoomLevel.connect(self.wheelScrollZoom)
self.pointerGroupClicked(1)
self.sceneScaleCombo.setCurrentIndex(self.model.initialZoomLevel)
self.sceneScaleChanged(self.sceneScaleCombo.currentText())
self.view.horizontalScrollBar().setValue(self.model.horizontalScroll)
self.view.verticalScrollBar().setValue(self.model.verticalScroll)
self.selectionTypeCombo.setCurrentIndex(0)
self.quickLinkCheckBox.setChecked(self.scene.quickLink)
self.displayLinksCheckBox.setChecked(True)
self.selectionTypeChanged(self.selectionTypeCombo.currentText())
self.refreshRegionComboBox()
elif result == 0:
log.debug_log('Application closing. No project opened.')
sys.exit()
def get_all_links(self):
'''get all the links in one page, aka get all edges of one node
Return: a list of tuples that the first element is the number of
links, the second element is the name and
the third element its the RELATIVE url.
O(n) where n is approximately the size of the HTML tree.
'''
assert self.has_soup(), 'self._soup is None'
# valid urls need to be in this format: start with /wiki/ and
# not contain `:`.
all_link_tags = self._soup.find(id='bodyContent')\
.find_all('a',
href=re.compile('^(/wiki/)((?!:).)*$'))
# invalid: aaa/wiki/
# /wiki/aaa:bbb:
# valid : /wiki/apple_pie
# valie : /wiki/fruit/apple (not likely to occur)
links = []
for link_tag in all_link_tags:
if 'href' in link_tag.attrs and link_tag.get_text() != 'ISBN':
links.append(
(link_tag.get_text().lower(), link_tag.get('href')))
if settings.debug:
debug_log('number of links: {}'.format(len(links)))
return links
def make_export_dir(save_path, date_tag):
try:
#创健目录
make_dir_p(save_path + date_tag + "/" + "new")
make_dir_p(save_path + date_tag + "/" + "old")
except Exception, e:
debug_log(str(e))
return False
def renameLinks(self, old_name, new_name):
# Possibly not needed anymore as setData does the job
found = False
for link_type in self.link_types:
if link_type.name == old_name:
found = True
link_type.name = new_name
self.linksChanged.emit()
if not found:
log.debug_log("Link type not found: " + old_name)
def export(commit_lst, save_path, date_tag, file_lst):
for commit in commit_lst:
local_file_path = save_path + date_tag + ("/new/" if commit == commit_lst[0] else "/old/")
debug_log("export:local_file_path is %s" % local_file_path)
# 拷贝到指定目录去
for file_name in file_lst:
debug_log("export:downloading %s..." % file_name)
if not download_file(commit, file_name, local_file_path):
return False
return True
def __init__(self, start=None, relurl=None):
'''`relurl` can be None if the url is not known yet. Crawler
will automatically generate the url for you.
start: name of lemma at the starting point
relurl: @NOTE:RELATIVE url
'''
self._start = start
self._url = self._BASE_RUL + relurl if relurl is not None else None
if settings.debug:
debug_log(self._url)
self._soup = self._make_soup()
def download_file(commit, path, local_path):
ex_path = REMOTE_PATH + "/" + USER_NAME + "/" + PROJECT_NAME
commit_path = "/" + commit + "/" + path
local_file_path = local_path + path
try:
#创健目录
make_dir_p(os.path.dirname(local_file_path))
except Exception, e:
debug_log("download_file:make dir %s error" % local_file_path)
debug_log(str(e))
return False
def __init__(self, network_model, scene, parent=None):
super(NetworkManager, self).__init__(parent)
log.debug_log('Started Network Manager init.')
self.setAttribute(Qt.WA_DeleteOnClose)
self.setWindowTitle("Network Manager")
self.network_model = network_model
self.scene = scene
log.debug_log('Setting up Network Editor controlls')
#self.networkTable = QTableView()
self.networkTable = NetworkTableView(self.network_model)
#self.networkTable.setModel(self.network_model)
#self.networkTable.setItemDelegate(NetworkDelegate(self))
self.networkTable.setItemDelegateForColumn(NWM.NET_CHECKED,
CheckBoxDelegate(self))
self.networkTable.setItemDelegateForColumn(NWM.NET_COLOR,
NetworkColorDelegate(self))
self.networkTable.setItemDelegateForColumn(NWM.NET_STYLE,
NetworkLineStyleDelegate(self))
self.networkTable.setSelectionMode(QTableView.SingleSelection)
self.networkTable.setSelectionBehavior(QTableView.SelectRows)
self.networkTable.resizeColumnsToContents()
self.addButton = QPushButton('Add')
self.mergeButton = QPushButton('Merge')
self.colorButton = QPushButton('Change Colour')
self.deleteButton = QPushButton('Delete')
self.okButton = QPushButton('OK')
buttonLayout = QVBoxLayout()
buttonLayout.addWidget(self.addButton)
buttonLayout.addWidget(self.mergeButton)
buttonLayout.addWidget(self.colorButton)
buttonLayout.addWidget(self.deleteButton)
buttonLayout.addStretch(10)
buttonLayout.addWidget(self.okButton)
hLayout = QHBoxLayout()
hLayout.addWidget(self.networkTable, 1)
hLayout.addLayout(buttonLayout)
self.setLayout(hLayout)
self.addButton.clicked.connect(self.addButtonClicked)
self.deleteButton.clicked.connect(self.deleteButtonClicked)
self.colorButton.clicked.connect(self.colorButtonClicked)
self.okButton.clicked.connect(self.okButtonClicked)
self.network_model.dataChanged.connect(self.checkStateChanged)
self.network_model.dataChanged.connect(
self.networkTable.resizeColumnsToContents)
#self.resetCheckStates()
self.checkStateChanged()
def addLink(self, xy1, xy2, network_name):
for node in (xy1, xy2):
if node not in self.nodes:
log.debug_log("Error: node not found in graph: " + str(node))
return
log.debug_log("addLink: "+ str(xy1) +' '+ str(xy2) +' '+ str(network_name))
for lt in self.linkTypes:
if lt.name == network_name:
if xy1 in self.graph[xy2]:
for link in self.graph[xy1][xy2]['links']:
if link.name == network_name:
log.debug_log("Link already exists: " + \
xy1 + ", " + xy2 + ", " + \
network_name)
return
# Edge exists, but link doesn't
self.graph[xy1][xy2]['links'].append(Link(link_type=lt))
else:
# Edge doesn't exist
self.graph.add_edge(xy1, xy2)
#print 'graph:', self.graph.edges() #, xy1, xy2
self.graph[xy1][xy2]['links'] = [Link(link_type=lt)]
self.linksChanged.emit()
return
log.debug_log("network does not exist: " + network_name)
def _make_soup(self):
'''send request, handle response, and get soup object
@NOTE: It will automatically generate url if url was
originally passed in as `None`.
Return:
soup object if no Exception is raised.
Otherwise return `None`.
'''
try:
if self._url is None:
self._url = self.generate_full_url()
if settings.debug:
debug_log('get request...')
debug_log('url is {}'.format(self._url))
req = Request(self._url, headers=self._HEADERS)
response = urlopen(req)
soup = BeautifulSoup(response, "html5lib")
except URLError as e:
err_msg = 'Cannot open the page with name {}: {}'.format(
self._start, e)
if settings.debug:
debug_log(err_msg)
log(err_msg)
return None
except UnicodeDecodeError as e:
# example: https://en.wikipedia.org/wiki/Kronkåsa
# 'Kronkåsa' contains non-ascii characters
# We don't do encode & decode trick simply because it will
# not be a reasonable keyword to search.
err_msg = 'Cannot open the page with name {}: {}'.format(
self._start, e)
if settings.debug:
debug_log(err_msg)
log(err_msg)
return None
except http.client.BadStatusLine as e:
err_msg = 'Cannot open the page with name {}: {}'.format(
self._start, e)
if settings.debug:
debug_log(err_msg)
log(err_msg)
return None
else:
return soup
def removeLink(self, xy1, xy2, network_name):
links = self.graph[xy1][xy2]['links']
removed = False
for link in links:
if link.name == network_name:
links.remove(link)
removed = True
if removed:
self.graph[xy1][xy2]['links'] = links
else:
log.debug_log("removeLink - Link not found: " + \
str(xy1) + " " + str(xy2) + \
" " + str(network_name))
self.linksChanged.emit()
def closeEvent(self, event=QtGui.QCloseEvent()):
saveDialog = QtGui.QMessageBox()
saveDialog.setText('Do you want to save any changes before exiting?')
saveDialog.setStandardButtons(QtGui.QMessageBox.Save | QtGui.QMessageBox.Discard | QtGui.QMessageBox.Cancel)
saveDialog.setDefaultButton(QtGui.QMessageBox.Save)
ret = saveDialog.exec_()
if ret == QtGui.QMessageBox.Save:
self.save()
log.debug_log('If built using PySide it takes a while to exit, be patient.')
event.accept()
elif ret == QtGui.QMessageBox.Discard:
log.debug_log('If built using PySide it takes a while to exit, be patient.')
event.accept()
elif ret == QtGui.QMessageBox.Cancel:
event.ignore()
app.processEvents()
def createRegion(self):
# Temporarily disable clearing of currently selected hexes
#self.beginCreateRegion = True
#ok = 0
region_name, name_ok = QtGui.QInputDialog.getText(self, 'Crate New Region',
'New Region Name:',
QtGui.QLineEdit.Normal)
if name_ok and region_name:
added_ok = self.model.addNewHexRegion(str(region_name))
#debug_log('App: New group "' + str(group_name) + '" added')
if added_ok:
self.refreshRegionComboBox()
last_item = self.regionSelectionCombo.count() - 1
self.regionSelectionCombo.setCurrentIndex(last_item)
else:
log.debug_log('App: Adding new region failed. Name: ' + str(region_name))
def looking_for_path(common, num):
'''trace back to find the path.
'''
path = []
while common != self._start[num]:
old_vertex = common
if settings.debug:
debug_log(common)
path.append(common)
common = self._reached[num][old_vertex]
# this is aimed to prevent from loop in the graph
if common == old_vertex:
if settings.debug:
debug_log('current vertex[{}] == \
old vertex[{}]'.format(common, old_vertex))
break
return path
def updateData(self, headerInfo):
"""1. adjust the order of the header
2. add the new data of the inserted header and save
"""
if len(self.__data) == 1 or len(self.__data[0]) == 0:
self.__headerInfo = headerInfo
if len(self.__data) == 1:
for h in headerInfo:
self.__data.insert(0, [])
return [KDataStatusError.STATUS_OK]
orderInfo = []
for h in headerInfo:
f = False
for idx in range(len(self.__headerInfo)):
f = self.__headerInfo[idx].canConvert2(h)
if f:
orderInfo.append(idx)
break
if not f:
orderInfo.append([self.getDefaultValue(h.dataType) \
for i in range(len(self.__data[0]))])
data = []
# debug--------------
if __debug__:
log.debug_log("org header: %s now header: %s\n org data: %s\n"
% ([unicode(h) for h in self.__headerInfo],
[unicode(h) for h in headerInfo],
self.__data))
# debug--------------
for o in orderInfo:
data.append(self.__data[o] if type(o) == int else o)
data.append(self.__data[-1])
hbk = self.__headerInfo
dbk = self.__data
self.__headerInfo = headerInfo
self.__data = data
ret = self.save()
if ret[0] == KDataStatusError.STATUS_OK:
KDataWarehouse.updateData(self.__filename, self.__data)
else:
self.__data = dbk
self.__headerInfo = hbk
return ret
def do_export(begin_commit, end_commit, git_path, save_path, check_status):
#获取当前目录
cur_cwd = os.getcwd()
# 做一下反斜替换
git_path = git_path.replace("\\",'/')
save_path = save_path.replace("\\",'/')
# 读取配置数据
load_config()
# 获取真实的commit,end_commit可能没有填写
if check_status:
commit_lst = get_real_commit(begin_commit, git_path)
else:
commit_lst = [begin_commit, end_commit]
# 获取要到处的文件列表
file_lst = get_export_file(begin_commit, git_path)
#切换会当前路径
os.chdir(cur_cwd)
# 对文件和commit做一个校验
if len(commit_lst) != 2 or not check_valid_commit(commit_lst[0]) or not check_valid_commit(commit_lst[1]):
return GET_COMMIT_ERROR
if len(file_lst) == 0:
return GET_FILE_ERROR
# 获取时间标记
date_tag = datetime.now().strftime('%Y-%m-%d-%H-%M-%S')
# 创建指定的目录
if not make_export_dir(save_path, date_tag):
debug_log("do_export:make dir %s error" % save_path)
return MAKE_DIR_ERROR
if not export(commit_lst, save_path, date_tag, file_lst):
debug_log("do_export:export file error")
return EXPORT_FILE_ERROR
return EXEC_SUCCESS
def removeRows(self, position, rows=1, index=QtCore.QModelIndex()):
log.debug_log("WorldLinkTypes:About to remove row(s).")
self.beginRemoveRows(QtCore.QModelIndex(), position, position + rows - 1)
log.debug_log("WorldLinkTypes:Removing row(s).")
dead_link_types = self.linkTypes[position:(position + rows)]
edge_list = self.links
# edges are like [xy1, xy2, link]
for edge in edge_list:
link = edge[2]
if link.link_type in dead_link_types:
self.removeLink(edge[0], edge[1], link.name)
self.linkTypes = self.linkTypes[:position] + \
self.linkTypes[position + rows:]
self.endRemoveRows()
self.linksChanged.emit()
self.dirty = True
return True
def show_result(result):
'''result: returned value from searcher.result()
'''
log('path: ' + str(result['path']))
log('degree: ' + str(result['degree']))
#################
# draw graph####
#################
log('Rendering graph...')
nx.draw_spectral(result['graph'], node_size=50)
if settings.debug:
debug_log('call plt to show the graph...')
# save graph as .png
img_suffix = datetime.datetime.now().strftime('%y%m%d_%H%M%S')
plt.savefig('graph_img/graph_img' + img_suffix + '.png')
# write the graph to a file
if settings.debug:
debug_log('storing graph...')
fileio.write_graph(
str(result['graph'].nodes()) + '\n' + str(result['graph'].edges()))
def show_result(result):
'''result: returned value from searcher.result()
'''
log('path1: ' + str(result['path'][0]))
log('path2: ' + str(result['path'][1]))
log('degree: ' + str(result['degree']))
#################
# draw graph####
#################
log('Rendering graphs...')
nx.draw(result['graph1'], node_size=50)
if settings.debug:
debug_log('call plt to save the graph1...')
# save graph as .png
img_suffix = datetime.datetime.now().strftime('%y%m%d_%H%M%S')
plt.savefig('graph_img/graph_common_img1_' + img_suffix + '.png')
nx.draw(result['graph2'], node_size=50)
if settings.debug:
debug_log('call plt to save the graph2...')
# save graph as .png
plt.savefig('graph_img/graph_common_img2_' + img_suffix + '.png')
# write the graph to a file
if settings.debug:
debug_log('storing graph...')
tmp_graphs = (str(result['graph1'].nodes()) + '\n' +
str(result['graph1'].edges()) + '\n' +
str(result['graph2'].nodes()) + '\n' +
str(result['graph2'].edges()))
fileio.write_graph(tmp_graphs)
def run_search(self):
'''Master: creates workers to do the graph search.
running time: O(n) where n is the number of thread.
'''
# initialize bfs
self._todo_queues[0].put((self._start[0], None))
self._reached[0][self._start[0]] = self._start[0]
self._todo_queues[1].put((self._start[1], None))
self._reached[1][self._start[1]] = self._start[1]
# apply BFS with the help of threads
threads1 = []
threads2 = []
for _ in range(self._NUM_WORKER_THREADS // 2):
thr = threading.Thread(target=self.worker, args=(0, ))
# make sure the thread will eventually exit!
thr.deamon = True
threads1.append(thr)
thr.start()
for _ in range(self._NUM_WORKER_THREADS // 2):
thr = threading.Thread(target=self.worker, args=(1, ))
# make sure the thread will eventually exit!
thr.deamon = True
threads2.append(thr)
thr.start()
# block until all workers are done
for thr in threads1 + threads2:
if settings.debug:
debug_log('join thread [{}]'.format(thr.name))
thr.join()
# program goes here iff graph search is done
if self.found_common_ancestor:
log('common ancestor(s) found: {}'.format(
str(self._common_ancestors)))
self.generate_path()
def get_node_name(self):
'''get the node name for the as the start name.
O(n) where n is the approximately the size of HTML tree.
'''
try:
title_node = self._soup.find(id="firstHeading")
title = title_node.get_text().strip()
if not title:
raise ValueError('Cannot find title')
except ValueError as e:
log('Wikipedia change their DOM!')
if settings.debug:
debug_log(e)
return None
except Exception as e:
log('iternal error detected')
if settings.debug:
debug_log(e)
return None
else:
return title
def generate_path(self):
'''Trace back to generate path
Will raise a KeyError if a bad graph is parsed.
'''
curr_vertex = self._end
path = []
while curr_vertex != self._start:
old_vertex = curr_vertex
if settings.debug:
debug_log(curr_vertex)
path.append(curr_vertex)
# change edge color of the graph such that the path is different
# from other edges.
# TODO
curr_vertex = self._reached[old_vertex]
# this is aimed to prevent from loop in the graph
if curr_vertex == old_vertex:
if settings.debug:
debug_log('current vertex[{}] == old vertex[{}]'.format(
curr_vertex, old_vertex))
break
path.append(self._start)
self._path = path[::-1]
def check_config(self):
#转换为整数
convert_int(self.config)
for path, value in self.config['backup'].items():
if not os.path.exists(path):
del self.config['backup'][path]
debug_log('path %s not exist.' % path)
continue
if os.path.isfile(path):
self.config['backup'][path]['type'] = 'file'
elif os.path.isdir(path):
self.config['backup'][path]['type'] = 'dir'
else:
del self.config['backup'][path]
debug_log('path %s type not support.' % path)
continue
self.config['backup'][path]['dir'] = path
#判断是否存在可备份文件
if not len(self.config['backup']): return False
return True
def load_config():
global REMOTE_PATH
global USER_NAME
global PROJECT_NAME
REMOTE_PATH = DEFAULT_REMOTE_PATH
USER_NAME = DEFAULT_USER_NAME
PROJECT_NAME = DEFAULT_PROJECT_NAME
f = open("config.ini")
for line in f.readlines():
info_lst = line.split("=")
if len(info_lst) != 2:
debug_log("load_config:not = in opt config")
continue
if info_lst[0].strip() == "REMOTE_PATH":
REMOTE_PATH = info_lst[1].strip()
elif info_lst[0].strip() == "USER_NAME":
USER_NAME = info_lst[1].strip()
elif info_lst[0].strip() == "PROJECT_NAME":
PROJECT_NAME = info_lst[1].strip()
else:
debug_log("load_config:Error opt config %s" % info_lst[0])
def write_graph(graph):
'''write the graph to a file.
Argument:
graph: a dictiction representation of a
graph.
Return:
None
'''
latest_file_name = 'path_dict_result.txt'
# write results to a file.
# latest_file_name is the file that stores the lastest
# result.
with open(latest_file_name, 'w') as result_file:
if settings.debug:
debug_log('writing path dict to file')
result_file.write(str(graph))
# create a unique name for each file in the graph_bank
basename = BANK_PATH + 'graph'
suffix = datetime.datetime.now().strftime('%y%m%d%H%M%S')
filename = '_'.join([basename, suffix]) + '.txt'
# copy the lastest result file to the graph_bank
subprocess.run(['cp', latest_file_name, filename])
def __init__(self, network_index, parent=None):
super(ComboLineStyleDelegate, self).__init__(parent)
log.debug_log("Set up combo style delegate")
self.network_index = network_index
def sizeHint(self, option, index):
if index.column() == NWM.NET_COLOR:
return QSize(64, 32)
else:
log.debug_log("NetworkColorDelegate: Size Hint Error")
def __init__(self, parent=None):
super(NetworkColorDelegate, self).__init__(parent)
log.debug_log("Set up color delegate")
def worker(self, num):
'''worker which implements DFS on the graph of wikipedia.
running time: technically O(|E|+|V|) where |E| is the length
of all edges and |V| is the length of all vertices, and thus
the todo_queue are accumulated faster than the speed of graph
parsing.
'''
while (not self.found_common_ancestor and not self.max_limit_reached
and not self.invalid_start_point):
# {
# get a task
curr_vertex_info = self._todo_queues[num].get()
LOCK.acquire()
try:
self.nodes_counter += 1
finally:
LOCK.release()
if self.nodes_counter > self._max_page:
self._todo_queues[num].task_done()
LOCK.acquire()
try:
self.max_limit_reached = True
finally:
LOCK.release()
log('Maximum page limit reached. Terminating...')
return
if settings.debug:
debug_log('##now at ({}), total {}, queue size {}##'.format(
curr_vertex_info, self.nodes_counter,
self._todo_queues[num].qsize()))
# check if the common ancestors are found
common_ancestors = self.common_ancestor_found()
if common_ancestors:
LOCK.acquire()
try:
self.found_common_ancestor = True
self._common_ancestors = common_ancestors
finally:
LOCK.release()
self._todo_queues[num].task_done()
return
# do work
crawler = Crawler(start=curr_vertex_info[0],
relurl=curr_vertex_info[1])
if not crawler.has_soup():
if self._todo_queues[num].empty():
LOCK.acquire()
try:
self.invalid_start_point = True
finally:
LOCK.release()
return
# be tolerant, go to next iteration
continue
# get all link information; each element in neighbours
# contains (name, url)
neighbours = crawler.get_all_links()
# put all neighbours into _todo_queue except ones already
# visited
for neighbour in neighbours:
if (neighbour[0] not in self._reached[num]
and neighbour[0] not in curr_vertex_info[0]):
if self._reached[num][curr_vertex_info[0]] != neighbour[0]:
# ADD LOCK #######
LOCK.acquire()
try:
#################
# update graph###
#################
self._nx_digraphs[num].add_edge(
curr_vertex_info[0], neighbour[0])
self._reached[num][
neighbour[0]] = curr_vertex_info[0]
finally:
LOCK.release()
self._todo_queues[num].put(neighbour)
# finish current task
self._todo_queues[num].task_done()
# } end while loop
# Target has been found or max limit reached if goes here.
if settings.debug:
debug_log('killing worker...')
debug_log("load_config:Error opt config %s" % info_lst[0])
def download_file(commit, path, local_path):
ex_path = REMOTE_PATH + "/" + USER_NAME + "/" + PROJECT_NAME
commit_path = "/" + commit + "/" + path
local_file_path = local_path + path
try:
#创健目录
make_dir_p(os.path.dirname(local_file_path))
except Exception, e:
debug_log("download_file:make dir %s error" % local_file_path)
debug_log(str(e))
return False
debug_log("download_file:download path is %s" % (ex_path + commit_path))
try:
urllib.urlretrieve(ex_path + commit_path, local_path + path, Schedule)
except Exception, e:
debug_log(str(e))
return False
return True
def export(commit_lst, save_path, date_tag, file_lst):
for commit in commit_lst:
local_file_path = save_path + date_tag + ("/new/" if commit == commit_lst[0] else "/old/")
debug_log("export:local_file_path is %s" % local_file_path)
# 拷贝到指定目录去
for file_name in file_lst:
debug_log("export:downloading %s..." % file_name)
def __init__(self, parent=None):
super(NetworkLineStyleDelegate, self).__init__(parent)
log.debug_log("Set up line style delegate")
def worker(self):
'''worker which implements DFS on the graph of wikipedia.
'''
def target_found(tar):
'''Notify user that the target is found and prepare
for killing this worker.
'''
log('$$found target {}({})!$$'.format(self._end, tar))
# modify self._end:
LOCK.acquire()
try:
self._end = tar
self.found_target = True
self._todo_queue.task_done()
finally:
LOCK.release()
while (not self.found_target and not self._max_limit_reached
and not self._invalid_start_point):
# {
# get a task
curr_vertex_info = self._todo_queue.get()
LOCK.acquire()
try:
self.nodes_counter += 1
finally:
LOCK.release()
if self.nodes_counter > self._max_page:
self._todo_queue.task_done()
self._max_limit_reached = True
log('Maximum page limit reached. Terminating...')
return
if settings.debug:
debug_log('##now at ({}), total {}, queue size {}##'.format(
curr_vertex_info, self.nodes_counter,
self._todo_queue.qsize()))
if self._end in curr_vertex_info[0]:
target_found(curr_vertex_info[0])
return
# do work
crawler = Crawler(start=curr_vertex_info[0],
relurl=curr_vertex_info[1])
if not crawler.has_soup():
if self._todo_queue.empty():
LOCK.acquire()
try:
self._invalid_start_point = True
finally:
LOCK.release()
return
# else, be tolerant, go to next iteration
continue
# get all link information; each element in neighbours
# contains (name, url)
neighbours = crawler.get_all_links()
# put all neighbours into _todo_queue except ones already
# visited
for neighbour in neighbours:
if (neighbour[0] not in self._reached
and neighbour[0] not in curr_vertex_info[0]):
if self._reached[curr_vertex_info[0]] != neighbour[0]:
# ADD LOCK ##########
LOCK.acquire()
try:
#################
# update graph###
#################
self._nx_digraph.add_edge(curr_vertex_info[0],
neighbour[0])
self._reached[neighbour[0]] = curr_vertex_info[0]
finally:
LOCK.release()
self._todo_queue.put(neighbour)
if self._end in neighbour[0]:
target_found(neighbour[0])
return
# finish current task
self._todo_queue.task_done()
# } end while loop
# Target has been found or max limit reached if the thread goes here.
if settings.debug:
debug_log('killing worker...')
def __init__(self, parent=None):
super(CheckBoxDelegate, self).__init__(parent)
log.debug_log("Set up CheckBoxDelegate delegate")
def render(self, nx_graph, path_list, figure=plt):
'''Draw graph, store dot file, convert dot to pdf.
'''
#################
# draw graph####
#################
img_suffix = datetime.datetime.now().strftime('%y%m%d_%H%M%S')
img_name = 'graph_img/graph_img' + img_suffix
dot_name = img_name + '.dot'
if settings.debug:
debug_log('generating {} file...'.format(dot_name))
write_dot(nx_graph, dot_name)
if settings.debug:
debug_log('generating {}.pdf file from .dot'
'file...'.format(dot_name))
# save graph as .png
subprocess.run(['dot', '-Tpdf', '-O', dot_name])
subprocess.run(['open', dot_name+'.pdf'])
log('Rendering graph...')
# nx.draw_spring(nx_graph, node_size=50)
###################################################
# change color of node[ok] and path###
###################################################
start_node = path_list[0]
end_node = path_list[-1]
nx_graph_pos = nx.spring_layout(nx_graph)
nx.draw_networkx(nx_graph, nx_graph_pos,
with_labels=False,
node_size=50)
# change color of nodes along the path
nx.draw_networkx_nodes(nx_graph,
pos=nx_graph_pos,
nodelist=path_list,
node_color='g',
node_size=200)
# change color of end-point nodes
nx.draw_networkx_nodes(nx_graph,
pos=nx_graph_pos,
nodelist=[start_node, end_node],
node_color='b',
node_size=200)
# change path color
path_edge_list = [(path_list[i], path_list[i+1])
for i in range(len(path_list)-1)]
nx.draw_networkx_edges(nx_graph,
pos=nx_graph_pos,
edgelist=path_edge_list,
width=1.0,
edge_color='b')
# write the graph to a file
if settings.debug:
debug_log('storing graph...')
fileio.write_graph(nx_graph.nodes())
fileio.write_graph(nx_graph.edges())
log('Done')
plt.axis('off')
figure.show()
