class Base(object):
logger = get_logger()
def get_html(self, url, headers=None, cookies=None, **kwargs):
required_cookie = cookies
required_headers = headers or {'User-Agent': choice(USER_AGENT)}
for _ in range(3):
try:
response = requests.get(url,
headers=required_headers,
cookies=required_cookie,
**kwargs).content
return response
except Exception as e:
self.logger.info("Get html error: type <{}>, msg <{}>".format(
e.__class__, e))
return ''
def get_raw_html(self, url, data=None):
for i in range(1, 4):
req = urllib2.Request(url) if not data else urllib2.Request(
url, data)
req.add_header('User-Agent', choice(USER_AGENT))
try:
response = urllib2.urlopen(req, timeout=30)
feed_data = response.read()
response.close()
return feed_data
except Exception as e:
self.logger.info(
'Web open error: type <{}>, msg <{}>, time <{}>'.format(
e.__class__, e, i))
time.sleep(3)
return '<html></html>'
@staticmethod
def md5(value):
if not isinstance(value, basestring):
raise ValueError('md5 must string!')
m = hashlib.md5()
try:
m.update(value)
except UnicodeEncodeError:
m.update(value.encode('u8'))
return m.hexdigest()
@staticmethod
def trim(text):
return re.compile(r'\s+', re.S).sub(' ', text)
def main(args):
os.chdir(args['working-directory'])
logger = base.get_logger(__name__, args['log-level'])
build_site(logger, load_plugins(logger=logger, directories=_directories), args)
def server_watchdog_callback():
build_site(logger, load_plugins(logger=logger, directories=_directories), args)
server_callback = None
if args['server-refresh'] == 'true':
server_callback = server_watchdog_callback
if args['server'] == 'true':
server_thread, addr = httpserver.start_server(_directories['output'], args['server-ip'], int(args['server-port']), server_callback, (_directories['src'], _directories['data'], _directories['templates'], _directories['plugins']))
logger.info(f'Started server at {addr[0]}:{str(addr[1])}')
try:
while server_thread.is_alive():
sleep(1)
except KeyboardInterrupt:
logger.debug('KeyboardInterrupt received')
logger.info('Finished')
# -*- coding: utf-8 -*-
import urllib, hashlib, httplib, time, mimetypes
from cgi import FieldStorage
import json
import sys
import os
import binascii
import eleme_application_set
import base
reload(sys)
sys.setdefaultencoding("utf-8")
logger = base.get_logger(os.path.basename(__file__))
text_geo = [
{
"geometry": {
"type": "Polygon",
"coordinates": [
[
[121.381303, 31.243521],
[121.380938, 31.242778],
[121.380735, 31.242421],
[121.380627, 31.242196],
[121.380541, 31.24204],
[121.38037, 31.241664],
[121.380284, 31.241499],
[121.38023, 31.241389],
[121.380166, 31.241269],
[121.380134, 31.241178],
parser.add_argument("--whoami",
help=argparse.SUPPRESS,
default=pwd.getpwuid(os.geteuid()).pw_name)
cmdline_arg_dict = parser.parse_args().__dict__
delimiter_char = cmdline_arg_dict["delimiter"]
header_flag = cmdline_arg_dict["first_line_is_header"]
reg_exp = cmdline_arg_dict["regular_expression"]
whoami = cmdline_arg_dict["whoami"]
if cmdline_arg_dict["verbose"]:
base.set_loglevel(logging.DEBUG)
elif cmdline_arg_dict["terse"]:
base.set_loglevel(logging.WARNING)
else:
base.set_loglevel(logging.INFO)
my_logger = base.get_logger()
base.record_usage(sys.argv[0], whoami, cmdline_arg_dict)
error_message = ""
try:
pattern = re.compile(reg_exp, re.IGNORECASE)
except sre_constants.error:
error_message += "'{reg_exp}' is not a valid regular expression.".format(
**locals())
if error_message:
base.get_logger().critical(error_message)
sys.exit(1)
data = cmdline_arg_dict["input-file"].readlines()
# Evaulate column headers