def main():
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger()
ap = argparse.ArgumentParser(description='A miner server.')
ap.add_argument('--server_addr', type=str, nargs='?', default='localhost',
help='Address the server should bind to.')
ap.add_argument('--server_port', type=int, nargs='?', default=8993,
help='The port the server listen to.')
ap.add_argument('--board', type=str, nargs='?', default='Gossiping',
help='The Board you want to mine.')
ap.add_argument('--number', type=int, nargs='?', default=2,
help='The Pages you want to mine.')
ap.add_argument('--fetch_path', type=str, nargs='?', default='./',
help='The Place you want to put your mining file')
ap.add_argument('--mining', type=bool, nargs='?', default=False,
help='Mining or not')
ap.add_argument('--storage_path', type=str, nargs='?', default='./storage',
help='Path to storeage the mirror of the ptt server.')
ap.add_argument('--cache_size', type=int, nargs='?', default=5000,
help='Number of documents should be cached in the RAM.')
ap.add_argument('--miner2', type=bool, nargs='?', default=False,
help='Use the version 2 miner or not.')
opts = ap.parse_args()
if opts.miner2:
backend = L2Miner(logging.getLogger('miner'),
opts.cache_size, opts.storage_path)
else:
current_dir = os.getcwd()
if opts.mining == True:
miner = BBSCrawler(opts.board,opts.number,opts.fetch_path)
miner.getAllPagesInTheBoard()
miner.getContent()
os.chdir(current_dir)
backend = BuildData(opts.fetch_path + opts.board)
os.chdir(current_dir)
server = Server(backend, opts.server_addr, opts.server_port)
try:
server.run()
except KeyboardInterrupt as e:
logger.info('Keyboard interrupt.')
except SystemExit as _:
logger.info('System exit.')
except Exception as e:
logger.info(get_exception_msg(e))
sys.exit(1)
finally:
logger.info('Cleanup')
server.stop()
backend.destroy()
sys.exit(0)
def run(self):
try:
# Iterately asks the cache to add an document from new to old.
for url in self._url_generator:
succ = self._cache.add_doc(self._curr_id, url)
self._curr_id -= 1
if not succ or self._curr_id < 0 or self._stop_flag:
break
except Exception as e:
self._logger.warning(get_exception_msg(e))
self._died_callback(self)
def run(self):
try:
# Iterately asks the cache to add an document from new to old.
for url in self._url_generator:
succ = self._cache.add_doc(self._curr_id, url)
self._curr_id -= 1
if not succ or self._curr_id < 0 or self._stop_flag:
break
except Exception as e:
self._logger.warning(get_exception_msg(e))
self._died_callback(self)
def run(self):
while not self._stop_flag:
try:
newest_doc = self._cache.newest_doc
oldest_doc = self._cache.oldest_doc
max_page_idx = self._get_max_page_idx()
self._run_fetch_new_docs(newest_doc, max_page_idx)
self._run_fill_old_docs(oldest_doc, max_page_idx)
except Exception as e:
self._logger.warn(get_exception_msg(e))
if not sleep_if(self._update_timestamp,
self._clean_and_check_should_continue):
break
def run(self):
while not self._stop_flag:
try:
newest_doc = self._cache.newest_doc
oldest_doc = self._cache.oldest_doc
max_page_idx = self._get_max_page_idx()
self._run_fetch_new_docs(newest_doc, max_page_idx)
self._run_fill_old_docs(oldest_doc, max_page_idx)
except Exception as e:
self._logger.warn(get_exception_msg(e))
if not sleep_if(self._update_timestamp,
self._clean_and_check_should_continue):
break
def run(self):
'''Routine of this handler thread.'''
try:
while not self._stop_flag:
(typee, buf) = self._recv_package()
ret = self._delegator.handle_package(typee, buf)
if ret is not None:
(typee, buf) = ret
self._send_package(typee, buf)
except self._StopException as e:
pass
except Exception as e:
self._logger.warning(utils.get_exception_msg(e))
self._logger.info('Stop the handler.')
self._end_notify_func(self._connection)
finally:
if self._auto_close:
self._logger.info('Close the connection.')
self._connection.close()
def run(self):
'''Routine of this handler thread.'''
try:
while not self._stop_flag:
(typee, buf) = self._recv_package()
ret = self._delegator.handle_package(typee, buf)
if ret is not None:
(typee, buf) = ret
self._send_package(typee, buf)
except self._StopException as e:
pass
except Exception as e:
self._logger.warning(utils.get_exception_msg(e))
self._logger.info('Stop the handler.')
self._end_notify_func(self._connection)
finally:
if self._auto_close:
self._logger.info('Close the connection.')
self._connection.close()
def do_POST(self):
if self.path != '/' or \
self.headers['content-type'] != 'application/json':
self._logger.warning('Bad url path or content-type.')
self.send_response(400)
return
try:
length = int(self.headers['content-length'])
data = self.rfile.read(length).decode('utf-8')
obj = json.loads(data)
ret = self._real_handler.handle_json(obj)
data = json.dumps(ret).encode('utf-8')
except Exception as e:
self._logger.warning(get_exception_msg(e))
self.send_response(400)
return
self.send_response(200)
self.send_header('content-type', 'application/json')
self.end_headers()
self.wfile.write(data)
def main():
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger()
ap = argparse.ArgumentParser(description='The server side of the app')
ap.add_argument('--miner_server_addr',
type=str,
nargs='?',
default='localhost',
help='Address of the miner server')
ap.add_argument('--miner_server_port',
type=int,
nargs='?',
default=8993,
help='Port of the miner server')
ap.add_argument('--miner_cache_size',
type=int,
nargs='?',
default=1000,
help='Cached size of the miner proxy.')
ap.add_argument('--article_analysis_server_addr',
type=str,
nargs='?',
default='localhost',
help='Address of the article analysis server')
ap.add_argument('--article_analysis_server_port',
type=int,
nargs='?',
default=8997,
help='Port of the article analysis server')
ap.add_argument('--article_analysis_cache_size',
type=int,
nargs='?',
default=1000,
help='Cached size of the miner proxy.')
ap.add_argument('--echo_debug',
type=bool,
nargs='?',
default=False,
help='Just become an echo server or not.')
ap.add_argument('--server_addr',
type=str,
nargs='?',
default='localhost',
help='Server bind address.')
ap.add_argument('--server_port',
type=int,
nargs='?',
default=8085,
help='Server bind port.')
opts = ap.parse_args()
try:
if not opts.echo_debug:
m = modules.miner.Miner(opts.miner_server_addr,
opts.miner_server_port,
opts.miner_cache_size,
logger.getChild('Miner'))
a = modules.article_analysis.ArticleAnalysis(
opts.article_analysis_server_addr,
opts.article_analysis_server_port,
opts.article_analysis_cache_size,
logger.getChild('ArticleAnalysis'))
h = modules.main_handler.MainHandler(
m, a, logger.getChild('MainHandler'))
else:
h = modules.main_handler.EchoMainHandler()
def handler(*args, **kwargs):
return modules.http_request_handler.HTTPRequestHandler(
h, logger.getChild('HTTPRequestHandler'), *args, **kwargs)
s = HTTPServer((opts.server_addr, opts.server_port), handler)
s.socket = ssl.wrap_socket(s.socket,
keyfile=os.path.join(SHARE_PATH, 'key.pem'),
certfile=os.path.join(
SHARE_PATH, 'cert.pem'),
server_side=True)
logger.info('Starts the server.')
s.serve_forever()
except KeyboardInterrupt as e:
logger.info('Keyboard interrupt.')
except SystemExit as _:
logger.info('System exit.')
except Exception as e:
logger.error(get_exception_msg(e))
sys.exit(1)
finally:
try:
s.server_close()
except Exception as _:
pass
sys.exit(0)
def main():
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger()
ap = argparse.ArgumentParser(description='The server side of the app')
ap.add_argument('--miner_server_addr', type=str, nargs='?',
default='localhost', help='Address of the miner server')
ap.add_argument('--miner_server_port', type=int, nargs='?',
default=8993, help='Port of the miner server')
ap.add_argument('--miner_cache_size', type=int, nargs='?',
default=1000, help='Cached size of the miner proxy.')
ap.add_argument('--article_analysis_server_addr', type=str, nargs='?',
default='localhost',
help='Address of the article analysis server')
ap.add_argument('--article_analysis_server_port', type=int, nargs='?',
default=8997, help='Port of the article analysis server')
ap.add_argument('--article_analysis_cache_size', type=int, nargs='?',
default=1000, help='Cached size of the miner proxy.')
ap.add_argument('--echo_debug', type=bool, nargs='?',
default=False, help='Just become an echo server or not.')
ap.add_argument('--server_addr', type=str, nargs='?',
default='localhost', help='Server bind address.')
ap.add_argument('--server_port', type=int, nargs='?',
default=8085, help='Server bind port.')
opts = ap.parse_args()
try:
if not opts.echo_debug:
m = modules.miner.Miner(opts.miner_server_addr,
opts.miner_server_port,
opts.miner_cache_size,
logger.getChild('Miner'))
a = modules.article_analysis.ArticleAnalysis(
opts.article_analysis_server_addr,
opts.article_analysis_server_port,
opts.article_analysis_cache_size,
logger.getChild('ArticleAnalysis'))
h = modules.main_handler.MainHandler(
m, a, logger.getChild('MainHandler'))
else:
h = modules.main_handler.EchoMainHandler()
def handler(*args, **kwargs):
return modules.http_request_handler.HTTPRequestHandler(
h, logger.getChild('HTTPRequestHandler'), *args, **kwargs)
s = HTTPServer((opts.server_addr, opts.server_port), handler)
s.socket = ssl.wrap_socket(
s.socket,
keyfile=os.path.join(SHARE_PATH, 'key.pem'),
certfile=os.path.join(SHARE_PATH,'cert.pem'),
server_side=True)
logger.info('Starts the server.')
s.serve_forever()
except KeyboardInterrupt as e:
logger.info('Keyboard interrupt.')
except SystemExit as _:
logger.info('System exit.')
except Exception as e:
logger.error(get_exception_msg(e))
sys.exit(1)
finally:
try:
s.server_close()
except Exception as _:
pass
sys.exit(0)