Esempio n. 1
0
def load_config_module():
	"""
	Load the configuration module specified as a command line argument to this 
	script. 

	@return: a tuple of route, crawler_config, agent_config
	@rtype: tuple (of three dict)
	"""
	import sys
	if len(sys.argv) < 2:
		print "No config module specified."
		sys.exit(-1)
	config_module = __import__(sys.argv[1])

	if not hasattr(config_module, 'AGENT_CONFIG'):
		sys.stderr('Config module requires an "AGENT_CONFIG" dictionary')
		sys.exit(-1)
	if not hasattr(config_module, 'CRAWLER_CONFIG'):
		sys.stderr('Config module requires an "CRAWLER_CONFIG" dictionary')
		sys.exit(-1)
	if not hasattr(config_module, 'ROUTE'):
		sys.stderr('Config module requires an "ROUTE" dictionary')
		sys.exit(-1)

	route = config_module.ROUTE or []

	if hasattr(config_module, 'ROUTING_DIR') and config_module.ROUTING_DIR:
		for group in RoutingGroup.load_from_dir(config_module.ROUTING_DIR):
			for command in group.route:
				route.append(command)

	return (route, config_module.CRAWLER_CONFIG, config_module.AGENT_CONFIG)
Esempio n. 2
0
	def test_load_from_dir(self):
		groups = RoutingGroup.load_from_dir('./conf/routes')
		self.assertEquals(6, len(groups))