def test_site(site):
# Create graph
graph = graphs.Manager()
# Add site to graph
graph.add_site(site)
# Show graph pages
print('-' * 80)
for page in graph.pages:
print(page, page.status)
# Show single page
a_page = graph.get_page("A")
print(a_page.url, [link.url for link in a_page.links])
def get_sequence(self,
site_list,
max_next_requests,
downloader_simulator=BaseDownloaderSimulator(),
frontier_tester=FrontierTester):
"""
Returns an Frontera iteration sequence from a site list
:param list site_list: A list of sites to use as frontier seeds.
:param int max_next_requests: Max next requests for the frontier.
"""
# Graph
graph_manager = graphs.Manager()
graph_manager.add_site_list(site_list)
# Tester
tester = frontier_tester(frontier=self.get_frontier(),
graph_manager=graph_manager,
max_next_requests=max_next_requests,
downloader_simulator=downloader_simulator)
tester.run()
return tester.sequence
"""
Graph manager reading data from database
"""
from frontera.utils import graphs
if __name__ == '__main__':
# Create graph with sqlite db
graph = graphs.Manager('sqlite:///data/graph.db')
# Show graph pages
for page in graph.pages:
print(page)
"""
Graph manager example with site list
"""
from frontera.utils import graphs
SITE_LIST = [
[
("A1", ["A2", "A3"]),
("A2", ["A4", "A5"]),
("A3", ["A6", "A7"]),
],
[
("B1", ["B2", "B3"]),
("B2", ["B4", "B5"]),
("B3", ["B6", "B7"]),
],
]
if __name__ == '__main__':
# Create graph
graph = graphs.Manager()
# Add site list to graph
graph.add_site_list(SITE_LIST)
# Show graph pages
for page in graph.pages:
print(page)
def generate_graph(site_list, filename, title, use_urls=False):
print('Generating diagram "%s"...' % title)
graph = graphs.Manager()
graph.add_site_list(site_list)
graph.render(filename=filename, label=title, use_urls=use_urls)
"""
Graph manager with database
"""
from frontera.utils import graphs
SITE_LIST = [
[
("A1", ["A2", "A3"]),
("A2", ["A4", "A5"]),
("A3", ["A6", "A7"]),
],
[
("B1", ["B2", "B3"]),
("B2", ["B4", "B5"]),
("B3", ["B6", "B7"]),
],
]
if __name__ == '__main__':
# Create graph with sqlite db
graph = graphs.Manager('sqlite:///data/graph.db', drop_all_tables=True)
# Add site list to graph
graph.add_site_list(SITE_LIST)
# Show graph pages
for page in graph.pages:
print(page)