def handle(self, *args, **options):
# logging
logger = logging.getLogger('autologin')
log_file = options['log']
if log_file:
log_file_dir = os.path.dirname(log_file)
if log_file_dir != '' and not os.path.exists(log_file_dir):
if options['parents']:
os.makedirs(log_file_dir)
else:
raise argparse.ArgumentError(
None,
"Log file's directory does not exist. "
"You can use set -p flag to create necessary directories as needed."
)
formatter = logging.Formatter(
fmt='%(asctime)s [%(name)s] %(levelname)s: %(message)s',
datefmt='%Y-%m-%d %H:%M:%S',
)
log_file_handler = logging.FileHandler(log_file)
log_file_handler.setFormatter(formatter)
logger.setLevel(logging.INFO)
logger.addHandler(log_file_handler)
else:
formatter = logging.Formatter('%(name)-12s %(levelname)-8s %(message)s')
console = logging.StreamHandler()
console.setFormatter(formatter)
logger.setLevel(logging.ERROR)
logger.addHandler(console)
# check network status
buptnet.autologin(options['username'], options['password'], logger)
def handle(self, *args, **options):
# logging
logger = logging.getLogger('autologin')
log_file = options['log']
if log_file:
log_file_dir = os.path.dirname(log_file)
if log_file_dir != '' and not os.path.exists(log_file_dir):
if options['parents']:
os.makedirs(log_file_dir)
else:
raise argparse.ArgumentError(
None, "Log file's directory does not exist. "
"You can use set -p flag to create necessary directories as needed."
)
formatter = logging.Formatter(
fmt='%(asctime)s [%(name)s] %(levelname)s: %(message)s',
datefmt='%Y-%m-%d %H:%M:%S',
)
log_file_handler = logging.FileHandler(log_file)
log_file_handler.setFormatter(formatter)
logger.setLevel(logging.INFO)
logger.addHandler(log_file_handler)
else:
formatter = logging.Formatter(
'%(name)-12s %(levelname)-8s %(message)s')
console = logging.StreamHandler()
console.setFormatter(formatter)
logger.setLevel(logging.ERROR)
logger.addHandler(console)
# check network status
buptnet.autologin(options['username'], options['password'], logger)
def handle(self, *args, **options):
# Decide save path
root_dir = options['dest']
root_dir = os.path.expanduser(root_dir)
if root_dir != '' and not os.path.exists(root_dir):
if not options['parents']:
raise argparse.ArgumentError(
None,
"dest's directory does not exist. "
"You can use set -p flag to create necessary directories as needed."
)
else:
os.makedirs(root_dir)
if not options['absolute']:
root_dir = os.path.join(root_dir, datetime.now(pytz.utc).astimezone().strftime("%Y-%m-%d-%H-%M-%S"))
os.makedirs(root_dir)
# Decide logging path
log_file_path = options['log']
if not log_file_path:
log_file_path = os.path.join(root_dir, 'log.txt')
else:
log_file_path = os.path.expanduser(log_file_path)
log_file_dir = os.path.dirname(log_file_path)
if log_file_dir != '' and not os.path.exists(log_file_dir):
if not options['parents']:
raise argparse.ArgumentError(
None,
"log files's directory does not exist. "
"You can use set -p flag to create necessary directories as needed."
)
else:
os.makedirs(log_file_dir)
# Setup logger for autologin
log_file_handler = logging.FileHandler(log_file_path)
formatter = logging.Formatter(
fmt='%(asctime)s [%(name)s] %(levelname)s: %(message)s',
datefmt='%Y-%m-%d %H:%M:%S',
)
log_file_handler.setFormatter(formatter)
login_logger = logging.getLogger('autologin')
login_logger.setLevel(logging.INFO)
login_logger.addHandler(log_file_handler)
# auto login
buptnet.autologin(options['username'], options['password'], login_logger)
# Setup logger for crawler
crawler_logger = logging.getLogger('pm25in')
crawler_logger.setLevel(logging.INFO)
# Redirect ERROR or higher level logging to stderr
console = logging.StreamHandler()
console.setLevel(logging.ERROR)
console.setFormatter(logging.Formatter("[%(name)s] %(levelname)s: %(message)s"))
logging.getLogger().addHandler(console)
def engine_stopped_handler():
crawler_logger.info("Stop crawling.")
def engine_started_handler():
crawler_logger.info("Start crawling.")
dispatcher.connect(engine_started_handler, engine_started)
dispatcher.connect(engine_stopped_handler, engine_stopped)
proj_settings = Settings()
module = os.environ.get(ENVVAR)
proj_settings.setmodule(module, priority='project')
proj_settings.set('LOG_FILE', log_file_path)
process = CrawlerProcess(proj_settings)
process.crawl(AQISpider, res_dir=root_dir, logger=crawler_logger, page_num=options['page_num'])
process.start()
def handle(self, *args, **options):
# Decide save path
root_dir = options['dest']
root_dir = os.path.expanduser(root_dir)
if root_dir != '' and not os.path.exists(root_dir):
if not options['parents']:
raise argparse.ArgumentError(
None, "dest's directory does not exist. "
"You can use set -p flag to create necessary directories as needed."
)
else:
os.makedirs(root_dir)
if not options['absolute']:
root_dir = os.path.join(
root_dir,
datetime.now(
pytz.utc).astimezone().strftime("%Y-%m-%d-%H-%M-%S"))
os.makedirs(root_dir)
# Decide logging path
log_file_path = options['log']
if not log_file_path:
log_file_path = os.path.join(root_dir, 'log.txt')
else:
log_file_path = os.path.expanduser(log_file_path)
log_file_dir = os.path.dirname(log_file_path)
if log_file_dir != '' and not os.path.exists(log_file_dir):
if not options['parents']:
raise argparse.ArgumentError(
None, "log files's directory does not exist. "
"You can use set -p flag to create necessary directories as needed."
)
else:
os.makedirs(log_file_dir)
# Setup logger for autologin
log_file_handler = logging.FileHandler(log_file_path)
formatter = logging.Formatter(
fmt='%(asctime)s [%(name)s] %(levelname)s: %(message)s',
datefmt='%Y-%m-%d %H:%M:%S',
)
log_file_handler.setFormatter(formatter)
login_logger = logging.getLogger('autologin')
login_logger.setLevel(logging.INFO)
login_logger.addHandler(log_file_handler)
# auto login
buptnet.autologin(options['username'], options['password'],
login_logger)
# Setup logger for crawler
crawler_logger = logging.getLogger('pm25in')
crawler_logger.setLevel(logging.INFO)
# Redirect ERROR or higher level logging to stderr
console = logging.StreamHandler()
console.setLevel(logging.ERROR)
console.setFormatter(
logging.Formatter("[%(name)s] %(levelname)s: %(message)s"))
logging.getLogger().addHandler(console)
def engine_stopped_handler():
crawler_logger.info("Stop crawling.")
def engine_started_handler():
crawler_logger.info("Start crawling.")
dispatcher.connect(engine_started_handler, engine_started)
dispatcher.connect(engine_stopped_handler, engine_stopped)
proj_settings = Settings()
module = os.environ.get(ENVVAR)
proj_settings.setmodule(module, priority='project')
proj_settings.set('LOG_FILE', log_file_path)
process = CrawlerProcess(proj_settings)
process.crawl(AQISpider,
res_dir=root_dir,
logger=crawler_logger,
page_num=options['page_num'])
process.start()
