def generate_entries(self):
logging.debug('Rendering entries')
userinfo = ObjectDict(self.storage.userinfo)
self.storage.ensure_dir(self.html_entries_path)
for entry_id, entry in self.storage.entries.items():
comments = self.storage.comments[entry_id]
comments_tree = CommentsTree(userinfo.username)
for comment in sorted(comments, key=lambda x: x['comment_id']):
comments_tree.add_comment(comment)
# ensure we have all basic entry attributes
for attr in ('subject', 'props'):
if attr not in entry:
entry[attr] = None
# tags
if 'taglist' in entry['props']:
taglist = entry['props']['taglist'].strip()
tags = [t.strip() for t in taglist.split(',')]
else:
tags = None
meta = ObjectDict(
entry_id=entry_id,
comments=entry['reply_count'],
image_links=0,
music_links=0,
document_links=0,
title=entry['subject'],
tags=tags,
year=int(entry['eventtime'][:4]),
)
self.entries_meta[entry_id] = meta
# remove <lj-cut>
entry['event'] = re.sub('</?lj-cut[^>]*>', '', entry['event'])
formatted = True if 'opt_preformatted' in entry['props'] and entry[
'props']['opt_preformatted'] == 1 else False
if not formatted:
tmp = []
for line in entry['event'].splitlines(False):
tmp.append('%s<br>' % line)
entry['event'] = '\n'.join(tmp)
html = self._render('entry.html',
entry=ObjectDict(entry),
meta=meta,
comments=comments_tree.render(),
userinfo=userinfo)
self.storage.save_file(self.html_entries_path,
'%d.html' % entry['ditemid'], html)
def run_cmd(self, payload):
host = payload['host']
request = ObjectDict(host=host)
request.track = ObjectDict(host=host)
uri = urlparse(payload['cmd'])
cmd = uri.path
if cmd == 'delete/module':
result = delete_module(request, parse_qs(uri.query)['name'])
log.debug('result: {0}'.format(result))
else:
raise HTTPServerError(
title="command '{0}' is not supported".format(cmd))
log.info('background processed command: {0}'.format(payload['cmd']))
def crawl_lagou_cites():
"""
获取拉勾上所有城市的信息
:return [
{
'id': 111,
'name': '广州'
},
.....
]
"""
logger.info(u'begin crawl cities info......')
response_html = etree.HTML(requests.get(constants.ALL_CITY_URL).text)
cities_html_list = response_html.xpath("//ul[@class='city_list']/li/a")
cities_dicts = []
for city_html in cities_html_list:
city_name = city_html.xpath('./text()')[0]
city_id = re.findall(pattern=r'/(\d+)-\d+-\d+',
string=city_html.xpath('./@href')[0])[0]
cities_dicts.append(ObjectDict(id=city_id, name=city_name))
logger.info(
u'crawl cities info finished! cites quantity is {cities_count}'.format(
cities_count=len(cities_dicts)))
return cities_dicts
def wrapper(*args, **kwargs):
try:
return func(*args, **kwargs)
except Exception as e:
logger.info(args)
logger.info(kwargs)
logger.info("====[EXIT]====")
# svc = args[0]
# # from code import interact
# # interact(local=locals())
# if isinstance(e, Exception):
# svc.db.rollback()
# logger.info("====[ROLLBACK]====")
# else:
# svc.db.commit()
# # svc.db.flush()
# logger.info("====[COMMIT]====")
# svc.db.remove()
# svc.db.close()
logger.info("====[CLOSE]====")
# svc = args[0]
# svc.db.close()
# svc.db.remove()
# logger.info(svc.db)
logThrown()
data = ObjectDict()
data.return_code = ERROR.system_err.errcode
data.return_message = e.__unicode__()
data.data = None
return data
def on_message(self, message):
if not self.current_user.is_authenticated():
self.write_message({"error": "logged_out"})
return False
obj = loads(message)
data = ObjectDict()
if settings.DEBUG:
print(obj)
data.debug = True
if obj.get("competition"):
if obj["competition"] == "global":
self.competition = "global"
IOLoop.current().add_callback(load_competition_scores,
self.competition)
else:
self.competition = obj["competition"]
self._competition = Competition.objects.get(
pk=self.competition)
listeners.competitions[self.competition][
self.current_user.id].append(self)
if load_competition(self.competition):
IOLoop.current().add_callback(send_scores,
self.competition)
listeners.scoreboards[self.competition].append(self)
if obj.get("clock") and (self._competition.get_role(self.current_user)
!= "compete"):
competition_clock_operation(obj["clock"], self._competition)
self.write_message(data)
def load_config(filename='config.py'):
catsup_path = os.path.dirname(__file__)
config = ObjectDict(catsup_path=catsup_path,
posts_path=os.path.join(catsup_path, '_posts'),
common_template_path=os.path.join(
catsup_path, 'template'),
deploy_path=os.path.join(catsup_path, 'deploy'),
disqus_shortname='catsup',
feed='feed.xml',
post_per_page=3,
gzip=True,
static_url='static',
theme_name='sealscript',
google_analytics='')
execfile(filename, {}, config)
if 'theme_path' not in config:
config.theme_path = os.path.join(catsup_path, 'themes',
config.theme_name)
if 'template_path' not in config:
config.template_path = os.path.join(config.theme_path, 'template')
if 'static_path' not in config:
config.static_path = os.path.join(config.theme_path, 'static')
if config.site_url.endswith('/'):
config.site_url = config.site_url[:-1]
if config.static_url.endswith('/'):
config.static_url = config.static_url[:-1]
return config
def run_job(self):
"""lagou job页面的信息任务"""
for doc_dict in self.from_col.find({
'_id': {
'$gte': self.last_id
}
}).sort('_id', 1):
if 'job' in doc_dict['url']: # job url
doc = ObjectDict(doc_dict)
assert doc.url and doc.html
if LagouCrawler.is_deleted_html(doc.html, False):
self.from_col.delete_one({'url': doc.url})
continue
job_parser = LagouHtmlParser(doc.url, doc.html)
data_dict = job_parser.parse_job()
if data_dict is None:
self.from_col.delete_one({'url': doc.url})
continue
self.logger.info('handle url: %s %s:%s', doc.url,
data_dict['source'], data_dict['job'])
if not DEBUG:
self.to_col.update({
'_id': doc._id,
}, {'$set': data_dict},
upsert=True)
self.set_id(doc._id)
def __init__(self,
app,
handler,
store,
initializer,
session_config=None,
session_object=None):
self.handler = handler
self.app = app
self.store = store or DiskStore(app_config.store_config.diskpath)
self._config = deepcopy(app_config.session_config)
if session_config:
self._config.update(session_config)
self._session_name = self._config.name
self._session_object = (session_object
or CookieObject)(app, handler, self.store,
self._config)
self._data = ObjectDict()
self._initializer = initializer
self.session_id = None
self._dirty = False
self._processor()
def generate(self, **kwargs: Any) -> bytes:
"""Generate this template with the given arguments."""
namespace = {
"escape": escape.xhtml_escape,
"xhtml_escape": escape.xhtml_escape,
"url_escape": escape.url_escape,
"json_encode": escape.json_encode,
"squeeze": escape.squeeze,
"linkify": escape.linkify,
"datetime": datetime,
"_tt_utf8": escape.utf8, # for internal use
"_tt_string_types": (unicode_type, bytes),
# __name__ and __loader__ allow the traceback mechanism to find
# the generated source code.
"__name__": self.name.replace(".", "_"),
"__loader__": ObjectDict(get_source=lambda name: self.code),
}
namespace.update(self.namespace)
namespace.update(kwargs)
exec_in(self.compiled, namespace)
execute = typing.cast(Callable[[], bytes], namespace["_tt_execute"])
# Clear the traceback module's cache of source data now that
# we've generated a new template (mainly for this module's
# unittests, where different tests reuse the same name).
linecache.clearcache()
return execute()
def format(cls, data):
if isinstance(data, list):
return [cls.format(item) for item in data]
else:
formatter = cls.get_formatter(data)
if not formatter:
raise ValueError(
u'Can not find the formatter by model {}'.format(
type(data)))
format_result = ObjectDict()
for field in formatter.FIELDS:
if not isinstance(field, Field):
raise ValueError('formatter field must be Field instance')
try:
value = getattr(data, field.name)
# 可再次渲染
if isinstance(value, list) or cls.get_formatter(value):
value = cls.format(value)
if field.converter:
value = field.converter(value)
except Exception:
# Field 设置了降级
if field.downgrade:
value = field.downgrade.value
else:
raise
format_result[field.name] = value
return format_result
class ThreadlocalLikeRequestContext(object):
"""
通过request_context.data,可以在任意位置访问当前请求的request对象，
以达到类似多线程wsgi程序中全局threadlocal request的目的。
eg：current_request = request_context.request
"""
_state = threading.local()
_state.data = ObjectDict({})
class __metaclass__(type):
@property
def data(cls):
if not hasattr(cls._state, 'data'):
return ObjectDict({})
return ObjectDict(cls._state.data)
@property
def request(cls):
return cls.data.get('request')
def __init__(self, **data):
self._data = data
def __enter__(self):
self._prev_data = self.__class__.data
self.__class__._state.data = self._data
def __exit__(self, *exc):
self.__class__._state.data = self._prev_data
del self._prev_data
return False
def __getitem__(self, key):
path = self._get_path(key)
if os.path.exists(path):
pickled = open(path).read()
return self.decode(pickled)
else:
return ObjectDict()
def on_message(self, message):
"""Handshake with livereload.js
1. client send 'hello'
2. server reply 'hello'
3. client send 'info'
"""
message = ObjectDict(escape.json_decode(message))
if message.command == 'hello':
handshake = {}
handshake['command'] = 'hello'
handshake['protocols'] = [
'http://livereload.com/protocols/official-7',
]
handshake['serverName'] = 'livereload-tornado'
self.send_message(handshake)
if message.command == 'info' and 'url' in message:
logging.info('Browser Connected: %s' % message.url)
LiveReloadHandler.waiters.add(self)
if not LiveReloadHandler._last_reload_time:
if not self.watcher._tasks:
logging.info('Watch current working directory')
self.watcher.watch(os.getcwd())
LiveReloadHandler._last_reload_time = time.time()
logging.info('Start watching changes')
if not self.watcher.start(self.poll_tasks):
ioloop.PeriodicCallback(self.poll_tasks, 800).start()
def get_json_arguments(self, raise_error=True):
try:
return ObjectDict(json_decode(self.request.body))
except Exception, e:
print traceback.format_exc()
if raise_error:
raise JsonDecodeError()
def get_current_user(self):
sid = self.get_secure_cookie(COOKIE_NAME)
if not sid:
return None
#这个因为在 log_function 中, 所以比 init 还要早, 数据库的连接可能出问题
try:
session = self.session.query(Session).filter_by(id=sid).first()
except:
self.session.rollback()
session = self.session.query(Session).filter_by(id=sid).first()
if not session:
return None
#if self.request.remote_ip != session.ip:
# return None
if session.user is None:
return None
user = self.session.query(User).filter_by(id=session.user).first()
if not user:
return None
p = user.dict()
p['sid'] = sid
return ObjectDict(p)
def get_cites_from_lagou():
"""
爬取拉勾城市数据
返回的 dict 组成:
id:
type: int
meaning: 城市 id
eg: 1
name:
type: str
meaning: 城市名
eg: 北京
:return: 拉勾城市数据集合
:rtype: List[tornado.util.ObjectDict]
"""
logger.info(u'begin crawl cities info......')
response_html = etree.HTML(requests.get(constants.ALL_CITY_URL).text)
cities_html_list = response_html.xpath("//ul[@class='city_list']/li/a")
cities_dicts = []
for city_html in cities_html_list:
city_name = city_html.xpath('./text()')[0]
city_id = re.findall(pattern=r'/(\d+)-\d+-\d+',
string=city_html.xpath('./@href')[0])[0]
cities_dicts.append(ObjectDict(id=city_id, name=city_name))
logger.info(
u'crawl cities info finished! cites quantity is {cities_count}'.format(
cities_count=len(cities_dicts)))
return cities_dicts
def __init__(self, handler):
self['__dict'] = ObjectDict(handler=handler)
if not hasattr(SessionManager, 'db'):
raise SetupError('Please call SessionManager.setup() first.')
self._setup_session_id()
def todict(self):
return ObjectDict(
id=self.id,
merchant_id=self.merchant_id,
hotel_id=self.hotel_id,
roomtype_id=self.roomtype_id,
base_hotel_id=self.base_hotel_id,
base_roomtype_id=self.base_roomtype_id,
name=self.name,
pay_type=self.pay_type,
stay_days=self.stay_days,
ahead_days=self.ahead_days,
cancel_type=self.cancel_type,
cancel_days=self.cancel_days,
cancel_time=self.cancel_time,
punish_type=self.punish_type,
punish_value=self.punish_value,
guarantee_start_time=self.guarantee_start_time,
guarantee_type=self.guarantee_type,
guarantee_count=self.guarantee_count,
start_date=self.start_date,
end_date=self.end_date,
is_online=self.is_online,
is_delete=self.is_delete,
)
def on_message(self, message):
"""Handshake with livereload.js
1. client send 'hello'
2. server reply 'hello'
3. client send 'info'
http://feedback.livereload.com/knowledgebase/articles/86174-livereload-protocol
"""
message = ObjectDict(escape.json_decode(message))
if message.command == 'hello':
handshake = {
'command': 'hello',
'protocols': [
'http://livereload.com/protocols/official-7',
],
'serverName': 'livereload-tornado',
}
self.send_message(handshake)
if message.command == 'info' and 'url' in message:
# print '- Client connected for url {0}'.format(message.url)
self.url = message.url
LiveReloadHandler.waiters.add(self)
if hasattr(LiveReloadHandler, 'update_status'):
LiveReloadHandler.update_status()
def generate(self, **kwargs):
"""Generate this template with the given arguments."""
namespace = {
"escape": escape.xhtml_escape,
"xhtml_escape": escape.xhtml_escape,
"url_escape": escape.url_escape,
"json_encode": escape.json_encode,
"squeeze": escape.squeeze,
"linkify": escape.linkify,
"datetime": datetime,
"_utf8": escape.utf8, # for internal use
"_string_types": (unicode, bytes_type),
# __name__ and __loader__ allow the traceback mechanism to find
# the generated source code.
"__name__": self.name.replace('.', '_'),
"__loader__": ObjectDict(get_source=lambda name: self.code),
}
namespace.update(self.namespace)
namespace.update(kwargs)
exec self.compiled in namespace
execute = namespace["_execute"]
# Clear the traceback module's cache of source data now that
# we've generated a new template (mainly for this module's
# unittests, where different tests reuse the same name).
linecache.clearcache()
try:
return execute()
except Exception:
formatted_code = _format_code(self.code).rstrip()
logging.error("%s code:\n%s", self.name, formatted_code)
raise
def execute_shell(self, command):
'''
异步执行外部shell命令
@gen.coroutine
def digest(self, message):
result = yield self.execute_shell('sleep 1;echo hello')
print result.returncode
print result.output
'''
process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)
pipe = process.stdout
shell_result = ObjectDict()
def handle_events(shell_result, io_loop, fd, events):
'''
ioloop异步回调
'''
io_loop.remove_handler(fd)
output = ''.join(pipe)
process.wait()
shell_result['returncode'] = process.returncode
shell_result['output'] = output
event.set()
event = toro.Event()
self.io_loop.add_handler(
pipe.fileno(),
partial(handle_events, shell_result, self.io_loop),
self.io_loop.READ)
yield event.wait()
raise gen.Return(shell_result)
def get_stations(self):
regions = yield self.get_regions()
for region in regions:
for q in self.const.BAIDU_POI_Q:
for page_num in range(0, 40):
baidu_poi_list = yield self.infra_ps.get_city_poi(
q, region.pname, region.rname, page_num, coord_type=2)
for item in baidu_poi_list.results:
if not item.get("location"):
continue
data_list = yield self.infra_ps.get_xian_nearby(
item.get("location", {}).get("lng", ""),
item.get("location", {}).get("lat", ""))
if data_list.status == 0 and data_list.data:
for item in data_list.data:
station = ObjectDict()
station['code'] = str(item.get('id'))
station['status'] = self.const.STATUS_INUSE
# station['type'] = ""
station['total'] = int(item['lock'])
station['name'] = item['name']
station['address'] = item['addr']
# station['district'] = ""
station['longitude'] = item["longitude"]
station['latitude'] = item["latitude"]
# station['service_time'] = ""
# station['is_24'] = ""
# station['is_duty'] = ""
print(station)
yield self.update_station(station)
x = int(random.random() * 10)
yield self.async_sleep(x)
raise gen.Return(True)
def on_message(self, message):
"""Handshake with livereload.js
1. client send 'hello'
2. server reply 'hello'
3. client send 'info'
http://help.livereload.com/kb/ecosystem/livereload-protocol
"""
message = ObjectDict(escape.json_decode(message))
if message.command == 'hello':
handshake = {}
handshake['command'] = 'hello'
protocols = message.protocols
protocols.append(
'http://livereload.com/protocols/2.x-remote-control')
handshake['protocols'] = protocols
handshake['serverName'] = 'livereload-tornado'
self.send_message(handshake)
if message.command == 'info' and 'url' in message:
logging.info('Browser Connected: %s' % message.url)
LiveReloadHandler.waiters.add(self)
if not LiveReloadHandler._last_reload_time:
if os.path.exists('Guardfile'):
logging.info('Reading Guardfile')
execfile('Guardfile', {})
else:
logging.info('No Guardfile')
Task.add(os.getcwd())
LiveReloadHandler._last_reload_time = time.time()
logging.info('Start watching changes')
if not Task.start(self.watch_tasks):
ioloop.PeriodicCallback(self.poll_tasks, 800).start()
def to_dict(self):
"""
只有当class 有 __init__ 方法的时候, instance的self.__dict__ 才不为空
"""
#_ = ObjectDict()
#for name, value in self.__dict__.iteritems():
# if not name.startswith('_'):
# if isinstance(value, datetime.datetime):
# value = str(value)
# _[name] = value
#return _
_ = ObjectDict()
for name in dir(self):
if not name.startswith('_'):
value = self.__getattribute__(name)
# TODO 需要将所有的输入输出都要转化为同一个字符类型
#print(name, value, type(value))
if isinstance(value, (unicode, str, int, long, dict)):
_[name] = value
if isinstance(value, datetime.datetime):
_[name] = str(value)
if isinstance(value, decimal.Decimal):
_[name] = str(value)
elif isinstance(value, ModelMixin):
_[name] = value.to_dict()
return _
def fetch(s, url, **kwargs):
self.assertEqual(url, expected_url)
fut = asyncio.Future()
response = ObjectDict()
response.code = 200
response.body = content
fut.set_result(response)
return fut
def get(self, hid, iid):
http = tornado.httpclient.AsyncHTTPClient()
host = self.db.query("SELECT * FROM host WHERE id= %s", int(hid))[0]
response = yield http.fetch("http://%s:%d/images/%s/json" %
(host.hostname, host.port, iid))
if response.error: raise tornado.web.HTTPError(500)
obj = json.loads(response.body)
self.render('image/detail.html', image=ObjectDict(obj), hid=hid)
def update_nested_dict(a, b):
for k, v in b.items():
if isinstance(v, dict):
d = a.setdefault(k, ObjectDict())
update_nested_dict(d, v)
else:
a[k] = v
return a
def add_comment(self, comment):
comment['comments'] = []
comment['_skip'] = False
obj = ObjectDict(comment)
place = self.find_place_by_parent_id(comment['comment_parent_id'])
place.append(obj)
if comment['state'] in ('D', 'B'):
self.deleted_comments.append(obj)
def failures(self, failure_list, data=None):
result = ObjectDict()
result.return_code = ERROR.database_save_err.errcode
# result.return_message = u",".join(["(%s)%s" % (f.return_code, f.return_message) for f in failure_list])
result.return_message = u"\n,".join([f for f in failure_list])
result.return_messages = failure_list
result.data = data
return result
def todict(self):
return ObjectDict(
id=self.id,
merchant_id=self.merchant_id,
base_hotel_id=self.base_hotel_id,
is_online=self.is_online,
is_suspend=self.is_suspend,
)
