def load_node(self, x, y, prevnode=None, message=None):
"""Load nodes from dict"""
# border blocking- all created nodes must be fully shown on screen : create a border of 50 around the graph space
# if x < 50 or x > self.width-150:
# return
# if y < 50 or y > self.height-50:
# return
# for first node
# if self.nodes == []:
# newnode = Node(x, y, prevnode)
# self.graph.create_rectangle(x-50, y-20, x+50, y+20, fill='red')
# # for other nodes
# else:
# newnode = Node(x, y, self.nodes[-1])
# self.graph.create_rectangle(x-50, y-20, x+50, y+20, fill='red')
# newnode.create_link(self.graph)
newnode = Node(x, y, prevnode, message)
self.graph.create_rectangle(x - 50, y - 20, x + 50, y + 20, fill='red')
if prevnode is not None: newnode.create_link(self.graph)
if message is not None: self.graph.create_text((x, y), text=message)
self.nodes.append(newnode)
return newnode
def __init__(self, handler):
Node.__init__(self, handler, 0x0000)
#self.setAddress(0xFFFF, True)
self.addressCount = 0x1000 #1000 - FFFE
self.lastheartbeattime = 0.0
self.startKeepAlive()
self.heartbeats = set()
async def main(self):
try:
sites = open(self.options['source'], 'r')
lines = sites.readlines()
spid = Spider(self.options)
for site in lines:
site = site.rstrip()
try:
crawler = spid.crawl(site)
print(f'Crawling : {site}')
data = await crawler
# Améliorable : on peut set la valeur du filtre personnalisé (pas pour les liens), à la valeur assignée en options
# sinon, on garde la valeur None
ufilter = self.options['other_data'] if len(
self.options['other_data']) > 0 else None
user_data = spid.filter(data, ufilter)
hyperlinks = spid.filter(data)
print(f'Parsing data')
if (hyperlinks):
if (ufilter and user_data):
self.nodelist.push(
Node(site, hyperlinks, user_data))
else:
self.nodelist.push(Node(site, hyperlinks))
print(f'Found links in {site}')
else:
print(
f'No links were found in {site}, continuing crawling'
)
continue
data.close()
except Exception as e:
raise e
print(e)
print(f"An error occured whilst crawling : {site}")
await spid.destroySession()
nodes = self.nodelist.nodes_table
edges = self.nodelist.edges_table
self.filemanager.export_to_csv(nodes, edges)
except Exception as e:
raise e
print(f"Input error : no website has been carwled")
def depth_first_search(initial_node: Node, count):
initial_node.visited = True
count[0] += 1
for neighbor in initial_node.neighbors:
if not hasattr(neighbor.node,
'visited') or neighbor.node.visited == False:
neighbor.node.visited = True
depth_first_search(neighbor.node, count)
count[0] += 1
# print(initial_node.value, 'count: ', count[0])
initial_node.topological_count = count[0]
return initial_node
def start_node(self, node_number):
# Use a temp copy of stakepool_config
stakepool_config_temp = copy.deepcopy(
yaml.safe_load(open(self.conf.stakepool_config_path, 'r')))
tmp_config_file_path = 'tmp_config.json'
# Create a temporarily file, as jormungandr require this as input
with open(tmp_config_file_path, 'w') as tmp_config_file:
# Increase port number for specific node
ip_address, port = stakepool_config_temp['rest']['listen'].split(
':')
stakepool_config_temp['rest']['listen'] = ip_address + ':' + str(
int(port) + node_number)
# Listen_address
x, ip, ip_address, comm_p, port = stakepool_config_temp['p2p'][
'listen_address'].split('/')
stakepool_config_temp['p2p'][
'listen_address'] = f'/{ip}/{ip_address}/{comm_p}/{int(port) + node_number}'
# Public_address
x, ip, ip_address, comm_p, port = stakepool_config_temp['p2p'][
'public_address'].split('/')
stakepool_config_temp['p2p'][
'public_address'] = f'/{ip}/{ip_address}/{comm_p}/{int(port) + node_number}'
if self.conf.use_seperate_storage:
storage = stakepool_config_temp['storage']
stakepool_config_temp['storage'] = f'{storage}_{node_number}'
# Save in temp config file
json.dump(stakepool_config_temp, tmp_config_file)
print(f'Starting node {node_number}...')
return Node(node_number, self.conf.jcli_call_format,
self.conf.jormungandr_call_format, self.conf.genesis_hash,
tmp_config_file_path, self.conf.node_secret_path)
def create_nodes(self, values=[]):
nodes = []
for value in values:
nodes.append(Node(value))
self.make_nodes_screen(nodes)
self.nodes.extend(nodes)
return nodes
def __init__(self, serial):
'''
Constructor
Configures module by calling AT commands
'''
logger.info("##### INITIALIZING #####")
threading.Thread.__init__(self)
self.serial = serial
self._stop_event = threading.Event()
self.coordLock = threading.Lock()
self.hasCoordinator = False
self.lastCoordinatorAlive = 0
commands = [
'AT+CFG=433000000,20,9,10,1,1,0,0,0,0,3000,8,4',
'AT+DEST=FFFF', # BROADCAST EVERYTHING
'AT+SAVE'
]
self.serial.start(self.onSerialInput)
# GENERAL MODULE CONFIGURATION
logger.debug("Configuring module")
for cmd in commands:
if not self.serial.write(cmd):
raise RuntimeError("Configuration failed")
# NODE
self.node = Node(self)
def check(node_val): if node_val not in [i.value for i in self.open_nodes]: node = Node(node_val) else: for i in self.open_nodes: if i.value == node_val: node = i return node
def make_grid(rows, width):
grid = []
gap = width // rows
for i in range(rows):
grid.append([])
for j in range(rows):
node = Node(i, j, gap, rows)
grid[i].append(node)
return grid
def make_node(self, x, y, message=None, prevnode=None):
"""Creates a node given x and y coordinates"""
# border blocking- all created nodes must be fully shown on screen : create a border of 50 around the graph space
if x < 50 or x > self.width - 150:
return
if y < 50 or y > self.height - 50:
return
# for first node
if self.nodes == []:
newnode = Node(x, y, prevnode)
self.graph.create_rectangle(x - 50,
y - 20,
x + 50,
y + 20,
fill='red')
# for other nodes
else:
newnode = Node(x, y, self.nodes[-1])
self.graph.create_rectangle(x - 50,
y - 20,
x + 50,
y + 20,
fill='red')
newnode.create_link(self.graph)
self.nodes.append(newnode)
return newnode
def __init__(self, geoJson, resolution=9):
self.geoJson = geoJson
self.resolution = resolution
self.polyline = self.geoJson['coordinates'][0]
self.polyline.append(self.polyline[0])
self.hexagons = list(h3.polyfill(geoJson, resolution))
self.lookup_table = pd.read_pickle(
os.path.abspath('data/lookup_table.pkl'))
self.valid_nodes = list(self.lookup_table.index.levels[0]) #[50:60]
self.nodes = [Node(node_id) for node_id in self.valid_nodes] #[50:60]
self.city_time = 0
self.orders = np.load(os.path.abspath('data/prep_data.npy'))
self.max_timesteps = 143
self.days = 2
#self.num_taxis = 5
self.taxis = None #[Taxi(_id) for _id in range(self.num_taxis)]
self.set_node_neighbors()
def get_open_nodes(self): p = self.maze.layout for x in range(len(p)): for y in range(len(p[x])): if p[x][y] != self.maze.wall_char: self.open_nodes.append(Node((x, y)))
def _reset(self):
self.node = Node(self)
self._resetCoordinator()
def push(self, item):
"""Adds item to the top of the stack."""
self._items = Node(item, self._items)
self._size += 1