def test():
c = Config()
mongo = MongoClient(c.get('mongo', 'host'), int(c.get('mongo', 'port')))
mdb = mongo.zhihu
s = mdb.sub_topic
info = ('19848294', '脑波游戏')
mdb.sub_topic.insert_one({'sub_tid': info[0], 'sub_name': info[1]})
def filter(self, data):
# common strategy
# 1. hasn't been found before
# 2. not exceed max size
max_size = Config.get("seed_max_size_mb")
data = list(
filter(lambda x: x.size < max_size and Cache().get(x.id) is None,
data))
# customized strategy
final_seeds = []
if Config.get("mteam_strategy") == "easy":
final_seeds = self.easy_strategy(data)
elif Config.get("mteam_strategy") == "medium":
final_seeds = self.medium_strategy(data)
elif Config.get("mteam_strategy") == "hard":
final_seeds = self.hard_strategy(data)
# white list
white_lists = Config.get("putao_white_list").split("|")
for seed in data:
for white_list in white_lists:
if re.search(white_list, seed.title):
final_seeds.append(seed)
break
for seed in final_seeds:
print("Find valuable seed: " + str(seed))
return final_seeds
def get_weather_code(self):
"""
Attempts to fetch and parses the forecast data
:return: weather type string or None on error
"""
realtime = Config.get(Config.FORECAST_OFFSET_KEY, default_config.FORECAST_OFFSET) == 0
if realtime:
response = requests.get("https://api.openweathermap.org/data/2.5/weather?id={0}&mode=json&appid={1}"
.format(Config.get(Config.CITY_ID, default_config.CITY_ID), self.API_KEY))
else:
response = requests.get("https://api.openweathermap.org/data/2.5/forecast?id={0}&mode=json&appid={1}"
.format(Config.get(Config.CITY_ID, default_config.CITY_ID), self.API_KEY))
if response.status_code != 200:
Logger.get_logger().warn("Failed request to forecast endpoint.")
Logger.get_logger().debug(response.text)
return None
try:
data = response.json()
if not realtime and type(data.get("list")) is list:
for forecast_index, forecast in enumerate(data.get("list")):
ignore_offset = forecast_index == len(data.get("list")) - 1
valid, weather_id = self.parse_weather_id(forecast, ignore_offset)
if not valid:
continue
return weather_id
elif realtime and type(data) is dict:
valid, weather_id = self.parse_weather_id(data, True)
if valid:
return weather_id
else:
Logger.get_logger().debug("Forecast/weather response invalid")
except ValueError as e:
Logger.get_logger().exception(e)
return None
return None
def from_template(cls, section, level, pos=None, vel=None):
cfg = Config("effects", section)
num_particles = cfg.get("num_particles")
pos_irange = cfg.get("position_irange")
pos = pos if (pos is not None) else cfg.get("position")
vel_irange = cfg.get("velocity_irange")
vel = vel if (vel is not None) else cfg.get("velocity")
duration = cfg.get("duration")
duration_irange = cfg.get("duration_irange")
color_seq = cfg.get("color_sequence")
quad_sz = cfg.get("quad_size")
img_path = cfg.get("image_path")
return cls(
level,
num_particles,
pos,
pos_irange,
vel,
vel_irange,
duration,
duration_irange,
color_seq,
quad_sz,
"resources/images/" + img_path,
)
def run(self):
"""
Internal main loop
:return: None
"""
while self.running:
time_since_last_update = time.time() - self.last_update_time
# Update every three hours
city_id = Config.get(Config.CITY_ID)
forecast_offset = Config.get(Config.FORECAST_OFFSET_KEY)
if time_since_last_update >= self.UPDATE_INTERVAL or city_id != self.city_id \
or forecast_offset != self.forecast_offset:
self.city_id = Config.get(Config.CITY_ID)
self.forecast_offset = Config.get(Config.FORECAST_OFFSET_KEY)
weather_code = self.get_weather_code()
if weather_code is not None:
self.cached_weather_code = weather_code
try:
self.handle_weather_code(weather_code)
except Exception as e:
Logger.get_logger().exception(e)
self.last_update_time = time.time()
elif self.cached_weather_code is not None:
self.last_update_time = time.time() - self.RETRY_INTERVAL
elif self.cached_weather_code is not None:
try:
self.handle_weather_code(self.cached_weather_code)
except Exception as e:
Logger.get_logger().exception(e)
time.sleep(10)
def requestUpdate(self):
"""This method contacts the sensor, and asks it to do an update of its ruleset."""
logger = logging.getLogger(__name__)
port = int(Config.get("sensor", "port"))
timeout = int(Config.get("sensor", "pingTimeout"))
sensor = xmlrpclib.Server("https://%s:%s" % (self.ipAddress, port))
try:
with Timeout(timeout):
result = sensor.startUpdate(self.name)
except Timeout.Timeout:
logger.warning("Ping to sensor timed out")
return {'status': False, 'message': "Ping to sensor timed out"}
except socket.gaierror:
logger.warning("Could not ping sensor. Address is malformed")
return {
'status': False,
'message': "Could not ping sensor. Address is malformed"
}
except socket.error as e:
logger.warning("Could not ping sensor. %s" % str(e))
return {
'status': False,
'message': "Could not ping sensor. %s" % str(e)
}
return result
def __init__(self, level_number):
Level.instance = self
self.number = level_number
self.camera = None
self.controller = None
self.objects = set()
self.asteroids = set()
self.shots = set()
self.planets = set()
self.portals = set()
self.particles = set()
self.missiles = set()
self.ship = None
self.models = {}
self.enemy_shots = set()
self.enemy_ships = set()
self.has_skybox = False
cfg = Config('levels', str(level_number))
level_name = cfg.get('name')
self.load_file(level_name)
skybox = cfg.get('skybox')
if (self.has_skybox):
self.setup_skybox('resources/' + skybox)
self.first = True
def access_token_for_id(cls, id, callback):
"""Returns the access token for an id, acquiring a new one if necessary."""
token = Cache.get(cls.auth_cache_key_template % id)
if token:
return IOLoop.instance().add_callback(lambda: callback(token))
# If we don't have an access token cached, see if we have a refresh token
token = TokenIdMapping.lookup_refresh_token(id)
if token:
post_body = urllib.urlencode({
'client_id': Config.get('oauth', 'client-id'),
'client_secret': Config.get('oauth', 'client-secret'),
'refresh_token': token,
'grant_type': 'refresh_token',
})
http_client = AsyncHTTPClient()
return http_client.fetch(
'https://accounts.google.com/o/oauth2/token',
lambda response: cls.on_refresh_complete(response, id, callback),
method='POST',
body=post_body,
request_timeout=20.0,
connect_timeout=15.0,
)
else:
logging.error("Unable to update access token for %s, no refresh token stored.", id)
return IOLoop.instance().add_callback(lambda: callback(None))
def initialize():
messages = []
levels = {'NOTSET': logging.NOTSET, 'DEBUG': logging.DEBUG, 'INFO': logging.INFO, 'WARNING': logging.WARNING, 'ERROR': logging.ERROR, 'CRITICAL': logging.CRITICAL}
loglevel = Config.get("logging", "severity")
logfile = Config.get("logging", "logfile")
# If the configfile lists a loglevel that is not valid, assume info.
if(loglevel not in levels):
# Since the logger is not yet initialized, add the logging-message to the messagelist, so that we can
# log it whenever the logger is initialized.
messages.append(("LogLevel is not correctly set in the config-file. Assuming INFO", logging.ERROR))
print "A"
loglevel = "INFO"
rootlogger = logging.getLogger()
formatter = logging.Formatter('%(asctime)s: %(name)s: %(levelname)s - %(message)s')
fh = logging.FileHandler(logfile)
fh.setFormatter(formatter)
rootlogger.addHandler(fh)
rootlogger.setLevel(levels[loglevel])
messages.append(("Logger initialized", logging.INFO))
# Now that the logger is initialized, log the messages that appared during the initialization of the module
logger = logging.getLogger(__name__)
for m in messages:
logger.log(m[1], m[0])
def get(self):
"""Initial request handler for receiving auth code."""
err = self.request.arguments.get('error', [None])[0]
if err is not None:
if err == 'access_denied':
return self.redirect(self.reverse_url('auth_denied'))
return self.send_error(500)
self.http_client = AsyncHTTPClient()
code = self.request.arguments.get('code', [None])[0]
if code is not None:
self.gplus_auth_code = code
# OAuth step #2: Receive authorization code, POST it
# back to Google to get an access token and a refresh token.
post_body = urllib.urlencode({
'code': code,
'client_id': Config.get('oauth', 'client-id'),
'client_secret': Config.get('oauth', 'client-secret'),
'redirect_uri': 'http://%s/oauth2callback' % self.request.host,
'grant_type': 'authorization_code',
})
return self.http_client.fetch(
'https://accounts.google.com/o/oauth2/token',
self.on_token_request_complete,
method='POST',
body=post_body,
request_timeout=20.0,
connect_timeout=15.0,
)
# If we got here, we don't recognize why this endpoint was called.
self.send_error(501) # 501 Not Implemented
def __init__(self, level_number):
Level.instance = self
self.number = level_number
self.camera = None
self.controller = None
self.objects = set()
self.asteroids = set()
self.shots = set()
self.planets = set()
self.portals = set()
self.particles = set()
self.missiles = set()
self.ship = None
self.models = {}
self.enemy_shots = set()
self.enemy_ships = set()
self.has_skybox = False
cfg = Config('levels',str(level_number))
level_name = cfg.get('name')
self.load_file(level_name)
skybox = cfg.get('skybox')
if (self.has_skybox):
self.setup_skybox('resources/'+skybox)
self.first = True
def from_template(cls, section, level, pos = None, vel = None):
cfg = Config('effects', section)
num_particles = cfg.get('num_particles')
pos_irange = cfg.get('position_irange')
pos = pos if (pos is not None) else cfg.get('position')
vel_irange = cfg.get('velocity_irange')
vel = vel if (vel is not None) else cfg.get('velocity')
duration = cfg.get('duration')
duration_irange = cfg.get('duration_irange')
color_seq = cfg.get('color_sequence')
quad_sz = cfg.get('quad_size')
img_path = cfg.get('image_path')
return cls(
level, num_particles, pos, pos_irange,
vel, vel_irange, duration, duration_irange,
color_seq, quad_sz, 'resources/images/'+img_path
)
def generate_site(self):
site = Site()
site.home_page = "https://totheglory.im/browse.php?c=M"
site.login_page = "https://totheglory.im/takelogin.php"
# encoding/compression is disabled
site.login_headers = {
"user-agent":
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36",
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
# "accept-encoding": "gzip, deflate, br",
"accept-language": "en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7,zh-TW;q=0.6,ja;q=0.5",
"dnt": "1",
"origin": "https://totheglory.im",
"referer": "https://totheglory.im/login.php?returnto=",
"upgrade-insecure-requests": "1",
"cache-control": "max-age=0",
"cookie": "__cfduid=d38205529894764cfe4019d151d9c5ba41512649384; cf_clearance=df03bb3d6d563412e8c8fc778a67f4725aa522e1-1512649390-7200"
}
site.login_needed = True
site.login_verify_css_selector = "table td span.smallfont b a"
site.login_verify_str = Config.get(self.get_site_name() + "_username")
site.login_username = site.login_verify_str
site.login_password = Config.get(self.get_site_name() + "_password")
site.stat_page = "https://totheglory.im/mybonus.php"
self.site = site
def pingSensor(self):
"""This method checks the status of the sensor, to see if the snowman-clientd is running. It returns a dictionary,
where 'status' contains a boolean value if the ping was successful, and 'message' contains a textual message of
what happened."""
logger = logging.getLogger(__name__)
port = int(Config.get("sensor", "port"))
timeout = int(Config.get("sensor", "pingTimeout"))
sensor = xmlrpclib.Server("https://%s:%s" % (self.ipAddress, port))
try:
with Timeout(timeout):
result = sensor.ping(self.name)
except Timeout.Timeout:
logger.warning("Ping to sensor %s timed out" % self.name)
return {'status': False, 'message': "Ping to sensor timed out"}
except socket.gaierror:
logger.warning("Could not ping sensor %s. Address is malformed" %
self.name)
return {
'status': False,
'message': "Could not ping sensor. Address is malformed"
}
except socket.error as e:
logger.warning("Could not ping sensor %s. %s" %
(self.name, str(e)))
return {
'status': False,
'message': "Could not ping sensor. %s" % str(e)
}
return result
def __init__(self):
if (Player.instance is None):
Player.instance = self
cfg = Config('game', 'Player')
self.max_hp = cfg.get('max_hp')
self.initial_lifes = cfg.get('initial_lifes')
self.reset()
def __init__(self):
if (Player.instance is None):
Player.instance = self
cfg = Config('game', 'Player')
self.max_hp = cfg.get('max_hp')
self.initial_lifes = cfg.get('initial_lifes')
self.reset()
class HelpScout:
def __init__(self):
"""
The connection with the HelpScout API.
"""
# get the helpscout config parameters
self.config = Config('helpscout')
self.data = {
'grant_type': self.config.get('grant_type'),
'client_id': self.config.get('client_id'),
'client_secret': self.config.get('client_secret')
}
self.token = requests.post(URL_AUTH, data=self.data)
self.headers = {'Authorization': 'Bearer {}'.format(self.token.json()['access_token'])}
def get_helpscout_data(self, url, start='', end=''):
"""
Get the data for a given HelpScout API url
:param url: the url to query
:param start: start date in question
:param end: end date in question
:return: the helpscout data in json format
"""
params = {
'start': '{}T00:00:00Z'.format(start),
'end': '{}T00:00:00Z'.format(end)
}
helpscout_data = requests.get(url, headers=self.headers, params=params)
time.sleep(1)
return helpscout_data.json()
def get_pages_breakdown(self, url, page, start='', end=''):
"""
Get the data for a given HelpScout API url if there are multiple pages
:param url: the url to query
:param page: the page in the multiple pages of results
:param start: start date in question
:param end: end date in question
:return: the helpscout data in json format
"""
params = {
'start': '{}T00:00:00Z'.format(start),
'end': '{}T00:00:00Z'.format(end),
'page': '{}'.format(page)
}
pages_breakdown = requests.get(url, headers=self.headers, params=params)
time.sleep(1)
return pages_breakdown.json()
def __init__(self, parent):
QGLWidget.__init__(self, parent)
cfg = Config('game','OpenGL')
self.fps = cfg.get('fps')
self.clearColor = cfg.get('clear_color')
self.adjust_widget()
self.adjust_timer()
def __init__(self, parent):
QGLWidget.__init__(self, parent)
cfg = Config('game', 'OpenGL')
self.fps = cfg.get('fps')
self.clearColor = cfg.get('clear_color')
self.adjust_widget()
self.adjust_timer()
def __init__(self, message, str=None):
cfg = Config('chats', message)
if (str is None):
self.str = cfg.get('message')
else:
self.str = str
self.str = self.str.replace('\\\n', '').replace('\n', '\n\n')
self.duration = cfg.get('duration')
self.font = FontManager.getFont(cfg.get('font'))
self.font.setPointSize(cfg.get('font_size'))
self.font_color = QColor.fromRgb(*cfg.get('font_color'))
self.image = QImage(cfg.get('image_path'))
p = cfg.get('image_pos')
self.image_rect = QRect(0., 0., self.image.width(),
self.image.height())
self.image_rect.moveCenter(QPoint(p[0], p[1]))
self.text_rect = QRect(*cfg.get('text_rect'))
self.has_cursor = True
self.blink_elapsed = 0.
self.blink_time = cfg.get('blink_time')
self.elapsed = 0.
self.message_sz = len(self.str)
def __init__(self, message, str = None):
cfg = Config('chats', message)
if (str is None):
self.str = cfg.get('message')
else:
self.str = str
self.str = self.str.replace('\\\n', '').replace('\n','\n\n')
self.duration = cfg.get('duration')
self.font = FontManager.getFont(cfg.get('font'))
self.font.setPointSize(cfg.get('font_size'))
self.font_color = QColor.fromRgb(*cfg.get('font_color'))
self.image = QImage(cfg.get('image_path'))
p = cfg.get('image_pos')
self.image_rect = QRect(0.,0.,self.image.width(),self.image.height())
self.image_rect.moveCenter(QPoint(p[0],p[1]))
self.text_rect = QRect(*cfg.get('text_rect'))
self.has_cursor = True
self.blink_elapsed = 0.
self.blink_time = cfg.get('blink_time')
self.elapsed = 0.
self.message_sz = len(self.str)
def __init__(self, model, ship, level):
self.model = model
self.ship = ship
self.level = level
cfg = Config('game','SimpleMissile')
self.pos = Vector3d(*cfg.get('pos'))
self.initial_velocity = cfg.get('initial_velocity')
self.attraction_vel = cfg.get('attraction_velocity')
self.damage = cfg.get('damage')
self.num_rockets = cfg.get('num_rockets')
def generate_site(cls):
site = Site()
site.home_page = "http://u.ishuhui.com/"
site.login_page = "http://u.ishuhui.com/login"
site.login_needed = True
site.login_verify_str = Config.get("shuhui_nickname")
site.login_username = Config.get("shuhui_username")
site.login_password = Config.get("shuhui_password")
return site
def __init__(self, message, str = None, pos = None):
cfg = Config('messages', message)
if (pos is None):
self.pos = QPoint(*cfg.get('position'))
else:
self.pos = QPoint(*pos)
self.velocity = cfg.get('velocity')
FadeMessage.__init__(self, message, str)
def __init__(self, message, str=None, pos=None):
cfg = Config('messages', message)
if (pos is None):
self.pos = QPoint(*cfg.get('position'))
else:
self.pos = QPoint(*pos)
self.velocity = cfg.get('velocity')
FadeMessage.__init__(self, message, str)
def __init__(self, model, shape, element, level):
Object.__init__(self, model, shape, element)
cfg = Config('physics','Ship')
self.move_force_sz = cfg.get('move_force')
self.spin_velocity = cfg.get('spin_velocity')
self.strafe_force = cfg.get('strafe_force')
self.shape.forces_res.append(cfg.get('vacuum_resistance'))
self.breake_rate = cfg.get('breake_rate')
self.mouse_sensivity = Config('game','Mouse').get('sensivity')
self.level = level
self.rotation = Quaternion.from_axis_rotations(0.,0.,0.)
self.ship_dir = None
self.up_dir = None
self.spinning = {
'up' : False,
'down' : False,
'left' : False,
'right' : False
}
self.vectors = {
'up' : (Vector3d.x_axis(), 1.),
'down' : (Vector3d.x_axis(), -1.),
'left' : (Vector3d.y_axis(), 1.),
'right' : (Vector3d.y_axis(), -1.)
}
self.strafe = {
'forward': False,
'left' : False,
'right' : False,
'breake' : False
}
self.strafe_vectors = {
'forward':Vector3d(0.,0.,-0.7),
'left' : Vector3d(-0.9,0.,0.),
'right' : Vector3d(0.9,0.,0.),
'breake' : Vector3d(0.,0.,1.)
}
self.angles = [0.,0.]
self.mouse_target = [0.,0.]
self.collision_set = set()
self.keep_colliding = set()
def __init__(self, model, shape, element, level):
Object.__init__(self, model, shape, element)
cfg = Config('physics', 'Ship')
self.move_force_sz = cfg.get('move_force')
self.spin_velocity = cfg.get('spin_velocity')
self.strafe_force = cfg.get('strafe_force')
self.shape.forces_res.append(cfg.get('vacuum_resistance'))
self.breake_rate = cfg.get('breake_rate')
self.mouse_sensivity = Config('game', 'Mouse').get('sensivity')
self.level = level
self.rotation = Quaternion.from_axis_rotations(0., 0., 0.)
self.ship_dir = None
self.up_dir = None
self.spinning = {
'up': False,
'down': False,
'left': False,
'right': False
}
self.vectors = {
'up': (Vector3d.x_axis(), 1.),
'down': (Vector3d.x_axis(), -1.),
'left': (Vector3d.y_axis(), 1.),
'right': (Vector3d.y_axis(), -1.)
}
self.strafe = {
'forward': False,
'left': False,
'right': False,
'breake': False
}
self.strafe_vectors = {
'forward': Vector3d(0., 0., -0.7),
'left': Vector3d(-0.9, 0., 0.),
'right': Vector3d(0.9, 0., 0.),
'breake': Vector3d(0., 0., 1.)
}
self.angles = [0., 0.]
self.mouse_target = [0., 0.]
self.collision_set = set()
self.keep_colliding = set()
def main():
c = Config()
mongo = MongoClient(c.get('mongo', 'host'), int(c.get('mongo', 'port')))
am = AccountManager(c.get('zhihu', 'email'), c.get('zhihu', 'password'))
cookie = am.load_cookie()
requestHeader.update(cookie)
mdb = mongo.zhihu
for topic in mdb.topic.find():
tid = topic['tid']
logger.info('get sub topics of {0}'.format(tid))
for subtopic in getSubTopics(tid):
mdb.sub_topic.insert_one({'sub_tid': subtopic[0], 'sub_name': subtopic[1]})
def dump_questions():
c = Config()
mongo = MongoClient(c.get('mongo', 'host'), int(c.get('mongo', 'port')))
cursor = mongo.zhihu.top_answers.find({}, {
'title': 1,
'href': 1,
'_id': 0
})
file_name = os.path.join(file_dir, '2017-12-01-questions-1.md')
with open(file_name, 'a') as md:
md.write("---\nlayout: post\ntitle: 'questions'\ntags: [zhihu]\n---\n")
for q in cursor:
md.write("### [%s](%s)\n" % (q['title'], q['href']))
def resizeGL(self, width, height):
QGLWidget.resizeGL(self,width,height)
glViewport(0,0,width,height)
glMatrixMode(GL_PROJECTION)
glLoadIdentity()
cfg = Config('game','OpenGL')
fovy = cfg.get('y_field_of_view')
z_near = cfg.get('z_near')
z_far = cfg.get('z_far')
gluPerspective(fovy,float(width)/height,z_near,z_far)
def resizeGL(self, width, height):
QGLWidget.resizeGL(self, width, height)
glViewport(0, 0, width, height)
glMatrixMode(GL_PROJECTION)
glLoadIdentity()
cfg = Config('game', 'OpenGL')
fovy = cfg.get('y_field_of_view')
z_near = cfg.get('z_near')
z_far = cfg.get('z_far')
gluPerspective(fovy, float(width) / height, z_near, z_far)
def dump():
c = Config()
mongo = MongoClient(c.get('mongo', 'host'), int(c.get('mongo', 'port')))
for i in range(570):
cursor = mongo.zhihu.top_answers.find({}).skip(i * 100).limit(100)
file_name = os.path.join(file_dir, '2017-12-01-answers-%d.md' % i)
with open(file_name, 'a') as md:
md.write(
"---\nlayout: post\ntitle: 'answers %d'\ntags: [zhihu]\n---\n"
% i)
for answer in cursor:
md.write("## [%s](%s)\n" % (answer['title'], answer['href']))
md.write("%s\n" % answer['answer'])
def __init__(self, registerInstance, server_address, logRequests=True):
"""Secure Documenting XML-RPC server.
It it very similar to DocXMLRPCServer but it uses HTTPS for transporting XML data.
"""
DocXMLRPCServer.__init__(self, server_address, RPCHandler, logRequests)
self.logRequests = logRequests
# stuff for doc server
try:
self.set_server_title(registerInstance.title)
except AttributeError:
self.set_server_title('default title')
try:
self.set_server_name(registerInstance.name)
except AttributeError:
self.set_server_name('default name')
if registerInstance.__doc__:
self.set_server_documentation(registerInstance.__doc__)
else:
self.set_server_documentation('default documentation')
self.register_introspection_functions()
# init stuff, handle different versions:
try:
SimpleXMLRPCServer.SimpleXMLRPCDispatcher.__init__(self)
except TypeError:
# An exception is raised in Python 2.5 as the prototype of the __init__
# method has changed and now has 3 arguments (self, allow_none, encoding)
SimpleXMLRPCServer.SimpleXMLRPCDispatcher.__init__(
self, False, None)
SocketServer.BaseServer.__init__(self, server_address, RPCHandler)
self.register_instance(
registerInstance
) # for some reason, have to register instance down here!
# SSL socket stuff
ctx = SSL.Context(SSL.SSLv23_METHOD)
keyfile = os.path.join(Config.djangoroot,
Config.get("xmlrpc-server", "keyfile"))
certfile = os.path.join(Config.djangoroot,
Config.get("xmlrpc-server", "certfile"))
ctx.use_privatekey_file(keyfile)
ctx.use_certificate_file(certfile)
self.socket = SSL.Connection(
ctx, socket.socket(self.address_family, self.socket_type))
self.server_bind()
self.server_activate()
# requests count and condition, to allow for keyboard quit via CTL-C
self.requests = 0
self.rCondition = Condition()
def __init__(self, model, ship, level):
self.model = model
self.ship = ship
self.level = level
cfg = Config('game', 'SimpleGun')
self.duration = cfg.get('duration')
self.pos = Vector3d(*cfg.get('pos'))
self.shoot_period = 1. / cfg.get('shoot_rate')
self.shoot_velocity_sz = cfg.get('shoot_velocity')
self.damage = cfg.get('damage')
self.shooting = False
self.since_last_shoot = 0.0
def __init__(self, model, ship, level):
self.model = model
self.ship = ship
self.level = level
cfg = Config('game','SimpleGun')
self.duration = cfg.get('duration')
self.pos = Vector3d(*cfg.get('pos'))
self.shoot_period = 1. / cfg.get('shoot_rate')
self.shoot_velocity_sz = cfg.get('shoot_velocity')
self.damage = cfg.get('damage')
self.shooting = False
self.since_last_shoot = 0.0
def initialize(logDict=None, toConsole=False):
messages = []
levels = {
'NOTSET': logging.NOTSET,
'DEBUG': logging.DEBUG,
'INFO': logging.INFO,
'WARNING': logging.WARNING,
'ERROR': logging.ERROR,
'CRITICAL': logging.CRITICAL
}
loglevel = Config.get("logging", "severity")
logfile = Config.get("logging", "logfile")
# If the configfile lists a loglevel that is not valid, assume info.
if (loglevel not in levels):
# Since the logger is not yet initialized, add the logging-message to the messagelist, so that we can
# log it whenever the logger is initialized.
messages.append(
("LogLevel is not correctly set in the config-file. Assuming INFO",
logging.ERROR))
loglevel = "INFO"
rootlogger = logging.getLogger()
formatter = logging.Formatter(
'%(asctime)s: %(name)s %(lineno)s: %(levelname)s - %(message)s')
# Open the logfile, and set its logging-level
fh = logging.FileHandler(logfile)
fh.setFormatter(formatter)
rootlogger.addHandler(fh)
rootlogger.setLevel(levels[loglevel])
# Set the django logging-level
djangologlevel = Config.get("logging", "djangoSeverity")
djangologger = logging.getLogger("django")
djangologger.setLevel(levels[djangologlevel])
# If we want to log to console, initialize that handler.
if (toConsole):
ch = logging.StreamHandler()
ch.setLevel(logging.INFO)
ch.setFormatter(formatter)
rootlogger.addHandler(ch)
# Now that the logger is initialized, log the messages that appared during the initialization of the module
messages.append(("Logger initialized", logging.INFO))
logger = logging.getLogger(__name__)
for m in messages:
logger.log(m[1], m[0])
def __init__(self):
cfg = Config()
opencv_home = cfg.get("face_detection", "opencv_home")
haarcascade = cfg.get("face_detection", "haarcascade")
self.haarcascade = cv2.CascadeClassifier('{0}/{1}'.format(opencv_home,
haarcascade))
def __init__(self):
cfg = Config()
opencv_home = cfg.get("face_detection", "opencv_home")
haarcascade = cfg.get("face_detection", "haarcascade")
self.haarcascade = cv2.CascadeClassifier('{0}/{1}'.format(
opencv_home, haarcascade))
def save_cookie(cls):
if cls.session is not None:
with open(Config.get(cls.KEY_COOKIE_LOCATION), "w") as f:
f.write(
json.dumps(
requests.utils.dict_from_cookiejar(
cls.session.cookies)))
def generate_site(self):
site = Site()
site.home_page = "http://hdhome.org/torrents.php"
site.login_page = "http://hdhome.org/takelogin.php"
site.login_headers = {
"User-Agent":
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36",
"Accept":
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
"Accept-Language":
"en-US,en;q=0.8,zh-CN;q=0.6,zh;q=0.4,zh-TW;q=0.2,ja;q=0.2",
"Accept-Encoding":
"gzip, deflate, br",
"Connection":
"keep-alive",
"DNT":
"1",
"Host":
"hdhome.org",
"Referer":
"http://hdhome.org/index.php",
"Upgrade-Insecure-Requests":
"1",
"Cookie":
"__cfduid=d485315600280be35365e84ee5ec16d651512109005; c_secure_uid=ODU1MzM%3D; c_secure_pass=847ef9da672b8f7ef43c2952b06473b7; c_secure_ssl=bm9wZQ%3D%3D; c_secure_tracker_ssl=bm9wZQ%3D%3D; c_secure_login=bm9wZQ%3D%3D"
}
site.login_needed = True
site.login_verify_css_selector = "#nav_block a.User_Name b"
site.login_verify_str = Config.get("hdh_username")
self.site = site
return site
def index(request):
"""The default view for the update section."""
data = {}
# Create a list over sources.
data['sources'] = createSourceList()
# If something is posted:
if (request.POST):
# Create the form based on the posted data
data['manualUpdateForm'] = ManualUpdateForm(request.POST,
request.FILES)
# If the form is considered valid:
if (data['manualUpdateForm'].is_valid()):
# Construct some path where we can work.
workarea = Config.get("storage", "inputFiles")
create = Config.get("storage", "createIfNotExists")
filename = os.path.join(workarea, request.FILES['file'].name)
# Create the working-directories, if needed and wanted.
if (os.path.isdir(workarea) == False and create == "true"):
os.makedirs(workarea)
# Store the uploaded file.
upload = open(filename, "wb+")
for chunk in request.FILES['file'].chunks():
upload.write(chunk)
upload.close()
# Generate a message for the user
source = Source.objects.get(pk=request.POST['source'])
if (source.locked):
data[
'uploadMessage'] = "There is already an update going for this source!"
else:
data[
'uploadMessage'] = "The ruleset is now uploaded, and the processing of the file is started. This might take a while however, depending on the size of the file."
# Call the background-update script.
subprocess.call(
['/usr/bin/snowman-manualUpdate', filename, source.name])
# If nothing is posted, create an empty form
else:
data['manualUpdateForm'] = ManualUpdateForm()
return render(request, "update/index.tpl", data)
def from_config(cls, section, interface):
cfg = Config('interface', section)
img_path = cfg.get('image_path')
img_pos = cfg.get('image_pos')
img_scale = cfg.get('image_scale')
img = QImage('resources/images/'+ img_path)
img_w, img_h = img.width(), img.height()
img_rect = QRect(
img_pos[0], img_pos[1],
int(img_w*img_scale), int(img_h*img_scale)
)
view_rect = QRect(*cfg.get('view_rect'))
return cls(img, img_rect, view_rect, interface)
def main():
c = Config()
mongo = MongoClient(c.get('mongo', 'host'), int(c.get('mongo', 'port')))
#am = AccountManager()
#cookie = am.load_cookie()
#requestHeader.update(cookie)
rQ = Queue(connection=Redis())
#cursor = mongo.zhihu.sub_topic.find({}, {'sub_tid': 1, '_id': 0}, no_cursor_timeout=True).skip(32).limit(200)
cursor = mongo.zhihu.sub_topic.find({}, {'sub_tid': 1, '_id': 0}).skip(150).limit(20000).batch_size(10)
for subtopic in cursor:
stid = subtopic['sub_tid']
#mongo.zhihu.sub_topic.update_one({'sub_tid': stid}, {'$set': {'max_page': page_no}})
#rQ.enqueue(questions_per_page, stid, requestHeader)
questions_per_topic(stid, requestHeader, rQ)
logger.info('done')
def index(request):
"""The default view for the update section."""
data = {}
# Create a list over sources.
data["sources"] = createSourceList()
# If something is posted:
if request.POST:
# Create the form based on the posted data
data["manualUpdateForm"] = ManualUpdateForm(request.POST, request.FILES)
# If the form is considered valid:
if data["manualUpdateForm"].is_valid():
# Construct some path where we can work.
workarea = Config.get("storage", "inputFiles")
create = Config.get("storage", "createIfNotExists")
filename = os.path.join(workarea, request.FILES["file"].name)
# Create the working-directories, if needed and wanted.
if os.path.isdir(workarea) == False and create == "true":
os.makedirs(workarea)
# Store the uploaded file.
upload = open(filename, "wb+")
for chunk in request.FILES["file"].chunks():
upload.write(chunk)
upload.close()
# Generate a message for the user
source = Source.objects.get(pk=request.POST["source"])
if source.locked:
data["uploadMessage"] = "There is already an update going for this source!"
else:
data[
"uploadMessage"
] = "The ruleset is now uploaded, and the processing of the file is started. This might take a while however, depending on the size of the file."
# Call the background-update script.
subprocess.call(["/usr/bin/snowman-manualUpdate", filename, source.name])
# If nothing is posted, create an empty form
else:
data["manualUpdateForm"] = ManualUpdateForm()
return render(request, "update/index.tpl", data)
def __init__(self, message, str = None):
cfg = Config('messages', message)
if (str is None):
self.str = cfg.get('message')
else:
self.str = str
self.duration = cfg.get('duration')
self.fade_duration = cfg.get('fade_duration')
self.color = QColor.fromRgb(*cfg.get('color'))
self.alpha_final = self.color.alpha()
self.color.setAlpha(0)
self.font = FontManager.getFont(cfg.get('font'))
self.font.setPointSize(cfg.get('font_size'))
self.font_color = QColor.fromRgb(*cfg.get('font_color'))
self.font_alpha_final = self.font_color.alpha()
self.font_color.setAlpha(0)
self.elapsed = 0.0
self.state = 0
self.tick_funcs = [self.tick_fade_in, self.tick_message, self.tick_fade_out]
def __init__(self, level_number, level):
self.level_number = level_number
self.info = {}
self.fields = set()
self.level = level
self.player_state = Player.get_instance()
cfg = Config('interface', 'Settings')
font_name = cfg.get('field_font')
font_size = cfg.get('field_font_sz')
self.field_font = FontManager.getFont(font_name)
self.field_font.setPointSize(font_size)
self.field_color = QColor.fromRgb(*cfg.get('field_color'))
for f_name in ConfigManager.getOptions('interface', 'Fields'):
s = ConfigManager.getVal('interface', 'Fields', f_name)
s = map(str.strip, s.split('||'))
img = QImage('resources/images/'+s[0])
img_pos = QPoint(*eval(s[1]))
info_rect = QRect(*eval(s[2]))
scale = float(s[3])
if (len(s) >= 5):
font = QFont(self.field_font)
font.setPointSize(int(s[4]))
else:
font = self.field_font
img_w, img_h = img.width(), img.height()
img_rect = QRect(
img_pos.x(), img_pos.y(),
int(img_w*scale), int(img_h*scale)
)
self.info[f_name] = ''
self.fields.add(Field(f_name, img, img_rect, info_rect, font))
self.radar = Radar.from_config('E-Radar', self)
self.missile = GuidedMissile.from_config('GuidedMissile', self)
def get(self):
redir_uri = "https://accounts.google.com/o/oauth2/auth?%s" % urllib.urlencode({
'client_id': Config.get('oauth', 'client-id'),
'redirect_uri': 'http://%s/oauth2callback' % self.request.host,
'scope': 'https://www.googleapis.com/auth/plus.me', # G+ API
'response_type': 'code', # server-side webapp
'access_type': 'offline', # needed to get refresh tokens
'approval_prompt': 'force', # needed to get refresh tokens
})
self.redirect(redir_uri)
def __init__(self):
"""Initializes internal data-structure, and makes sure that the folder
where we are going to store the configurationfiles actually exists."""
logger = logging.getLogger(__name__)
self.configlocation = Config.get("configfiles", "location")
self.configfiles = []
if(os.path.exists(self.configlocation) == False):
logger.warning("Location for the configfiles does not exist. Creating the folders.")
os.makedirs(self.configlocation, 0755)
def requestUpdate(self):
"""This method contacts the sensor, and asks it to do an update of its ruleset."""
logger = logging.getLogger(__name__)
port = int(Config.get("sensor", "port"))
timeout = int(Config.get("sensor", "pingTimeout"))
sensor = xmlrpclib.Server("https://%s:%s" % (self.ipAddress, port))
try:
with Timeout(timeout):
result = sensor.startUpdate(self.name)
except Timeout.Timeout:
logger.warning("Ping to sensor timed out")
return {'status': False, 'message': "Ping to sensor timed out"}
except socket.gaierror:
logger.warning("Could not ping sensor. Address is malformed")
return {'status': False, 'message': "Could not ping sensor. Address is malformed"}
except socket.error as e:
logger.warning("Could not ping sensor. %s" % str(e))
return {'status': False, 'message': "Could not ping sensor. %s" % str(e)}
return result
def __init__(self):
cfg = Config()
# set up face detection models
opencv_home = cfg.get("face_detection", "opencv_home")
haarcascade = cfg.get("face_detection", "haarcascade")
cascadePath = "/usr/local/share/OpenCV/haarcascades/haarcascade_frontalface_default.xml"
self.faceCascade = cv2.CascadeClassifier('{0}/{1}'.format(opencv_home, haarcascade))
self.recognizer = cv2.face.createLBPHFaceRecognizer()
#self.recognizer = cv2.face.createEigenFaceRecognizer()
#self.recognizer = cv2.face.createFisherFaceRecognizer()
# the faces and Raspberry Pi locations we'll use
self.names = ["james", "juanjo", "sayantan", "vineet"]
self.rasp_names = ["FrontDoor", "Entrance", "Garage"]
access = cfg.get("aws", "access_key_id")
secret = cfg.get("aws", "secret_access_key")
# connect to dynamo
self.conn = boto.dynamodb2.connect_to_region('us-west-1', aws_access_key_id=access, aws_secret_access_key=secret)
self.sc = Table('SMARTCAM', connection=self.conn)
def initialize(logDict = None, toConsole=False):
messages = []
levels = {'NOTSET': logging.NOTSET, 'DEBUG': logging.DEBUG, 'INFO': logging.INFO, 'WARNING': logging.WARNING, 'ERROR': logging.ERROR, 'CRITICAL': logging.CRITICAL}
loglevel = Config.get("logging", "severity")
logfile = Config.get("logging", "logfile")
# If the configfile lists a loglevel that is not valid, assume info.
if(loglevel not in levels):
# Since the logger is not yet initialized, add the logging-message to the messagelist, so that we can
# log it whenever the logger is initialized.
messages.append(("LogLevel is not correctly set in the config-file. Assuming INFO", logging.ERROR))
loglevel = "INFO"
rootlogger = logging.getLogger()
formatter = logging.Formatter('%(asctime)s: %(name)s %(lineno)s: %(levelname)s - %(message)s')
# Open the logfile, and set its logging-level
fh = logging.FileHandler(logfile)
fh.setFormatter(formatter)
rootlogger.addHandler(fh)
rootlogger.setLevel(levels[loglevel])
# Set the django logging-level
djangologlevel = Config.get("logging", "djangoSeverity")
djangologger = logging.getLogger("django")
djangologger.setLevel(levels[djangologlevel])
# If we want to log to console, initialize that handler.
if(toConsole):
ch = logging.StreamHandler()
ch.setLevel(logging.INFO)
ch.setFormatter(formatter)
rootlogger.addHandler(ch)
# Now that the logger is initialized, log the messages that appared during the initialization of the module
messages.append(("Logger initialized", logging.INFO))
logger = logging.getLogger(__name__)
for m in messages:
logger.log(m[1], m[0])
def _initialize():
"""This method reads the database-configuration from the configuration-files, and
initializes the connection to the databse. It also makes sure that the db-schema
is created and present."""
# Read the configuration from file:
dbType = Config.get("localdb", "type")
dbName = Config.get("localdb", "name")
dbHost = Config.get("localdb", "hostname")
dbUser = Config.get("localdb", "username")
dbPass = Config.get("localdb", "password")
# Construct the dbPath string, or rais an exception if the dbtype is unknown.
if(dbType == "sqlite"):
dbpath = 'sqlite:///' + dbName
elif(dbType == "mysql"):
dbpath = dbType + "://" + dbUser + ":" + dbPass + "@" + dbHost + "/" + dbName
else:
raise Exception("DatabaseConfiguration is not correct")
# Create a dbengine, and depending on the configfile maybe turn on the debug.
if(Config.get("localdb", "debug") == "0"):
Session.engine = create_engine(dbpath)
else:
Session.engine = create_engine(dbpath, echo=True)
# Create a session, and bind it to the engine.
Session.session = sessionmaker(bind=Session.engine)
# Making sure that the dbSchema is created.
Base.metadata.create_all(Session.engine)
def __init__(self, registerInstance, server_address, logRequests=True):
"""Secure Documenting XML-RPC server.
It it very similar to DocXMLRPCServer but it uses HTTPS for transporting XML data.
"""
DocXMLRPCServer.__init__(self, server_address, RPCHandler, logRequests)
self.logRequests = logRequests
# stuff for doc server
try: self.set_server_title(registerInstance.title)
except AttributeError: self.set_server_title('default title')
try: self.set_server_name(registerInstance.name)
except AttributeError: self.set_server_name('default name')
if registerInstance.__doc__: self.set_server_documentation(registerInstance.__doc__)
else: self.set_server_documentation('default documentation')
self.register_introspection_functions()
# init stuff, handle different versions:
try:
SimpleXMLRPCServer.SimpleXMLRPCDispatcher.__init__(self)
except TypeError:
# An exception is raised in Python 2.5 as the prototype of the __init__
# method has changed and now has 3 arguments (self, allow_none, encoding)
SimpleXMLRPCServer.SimpleXMLRPCDispatcher.__init__(self, False, None)
SocketServer.BaseServer.__init__(self, server_address, RPCHandler)
self.register_instance(registerInstance) # for some reason, have to register instance down here!
# SSL socket stuff
ctx = SSL.Context(SSL.SSLv23_METHOD)
keyfile = os.path.join(Config.djangoroot, Config.get("xmlrpc-server", "keyfile"))
certfile = os.path.join(Config.djangoroot, Config.get("xmlrpc-server", "certfile"))
ctx.use_privatekey_file(keyfile)
ctx.use_certificate_file(certfile)
self.socket = SSL.Connection(ctx, socket.socket(self.address_family, self.socket_type))
self.server_bind()
self.server_activate()
# requests count and condition, to allow for keyboard quit via CTL-C
self.requests = 0
self.rCondition = Condition()
def pingSensor(self):
"""This method checks the status of the sensor, to see if the snowman-clientd is running. It returns a dictionary,
where 'status' contains a boolean value if the ping was successful, and 'message' contains a textual message of
what happened."""
logger = logging.getLogger(__name__)
port = int(Config.get("sensor", "port"))
timeout = int(Config.get("sensor", "pingTimeout"))
sensor = xmlrpclib.Server("https://%s:%s" % (self.ipAddress, port))
try:
with Timeout(timeout):
result = sensor.ping(self.name)
except Timeout.Timeout:
logger.warning("Ping to sensor %s timed out" % self.name)
return {'status': False, 'message': "Ping to sensor timed out"}
except socket.gaierror:
logger.warning("Could not ping sensor %s. Address is malformed" % self.name)
return {'status': False, 'message': "Could not ping sensor. Address is malformed"}
except socket.error as e:
logger.warning("Could not ping sensor %s. %s" % (self.name, str(e)))
return {'status': False, 'message': "Could not ping sensor. %s" % str(e)}
return result
def __init__(self, parent):
if (GLController.instance is None):
GLController.instance = self
QGLWidget.__init__(self, parent)
self.painter = QPainter()
cfg = Config('game','OpenGL')
self.clearColor = cfg.get('clear_color')
self.fps = cfg.get('fps')
self.fps_sum = 0.
self.fps_count = 0
self.show_fps = 0.
self.fps_elapsed = 0.
self.adjust_widget()
self.adjust_timer()
self.to_hook = None
self.hooks = {}
def connect(self):
"""Method which tries to connect to the central snowman-server, and authenticate with it.
SnowmanServer.ConnectionError is thrown if the server is unreachable.
returns True if successfully authenticated, False othervise."""
logger = logging.getLogger(__name__)
logger.info("Connecting to the SRM-Server")
# Grab configuration from configfile
serveraddress = "https://" + Config.get("srm-server", "address") + ":" + Config.get("srm-server","port")
self.sensorname = Config.get("general", "sensorname")
self.secret = Config.get("general", "secret")
# Try to connect and authenticate with the server. If a socket error happens, the
# server is considered unreachable.
try:
self.server = xmlrpclib.Server(serveraddress)
response = self.server.authenticate(self.sensorname, self.secret)
except socket.error as e:
logger.error("Could not connect to %s!" % serveraddress)
logger.error(str(e))
raise SnowmanServer.ConnectionError("Could not connect to %s!" % serveraddress)
# If the server-response is positive, save the session-info before returning True to caller.
if(response['status']):
self.connected = True
self.token = response['token']
self.id = response['sensorID']
logger.info("Successfully authenticated with the server")
return True
# If the authentication failed, log the error-message befor False is returned.
else:
self.connected = False
logger.info("Could not connect to snowman-server. The server states: %s" % response['message'])
return False
def __init__(self, update):
# Get config from the configfile, and if the config is not valid,
# just set it to be "first"
self.msgsource = Config.get("update", "ruleMessageSource")
if(self.msgsource != "sidmsg" and self.msgsource != "rule"):
self.msgsource = "first"
self.update = update
self.comment = None
# Initialize the dictionaries to store the data in.
self.generators = {}
self.ruleSets = {}
self.rules = {}
self.classes = {}
self.references = {}
self.referenceTypes = {}
self.suppress = {}
self.filters = {}
import datetime as dt
import csv
import pprint
import re
import pandas
from pymongo import MongoClient, GEOSPHERE
from util import log
from util.config import Config
from bson.code import Code
from bson.son import SON
logger = log.getLogger(__name__)
# MongoDB Client & DB
cfg = Config()
client = MongoClient(cfg.get("mongo", "uri"))
db = client[cfg.get("mongo", "db_strava")]
segments_collection = db[cfg.get("mongo", "coll_segment")]
leaderboard_collection = db[cfg.get("mongo", "coll_leaderboards")]
zip_data_collection = db[cfg.get("mongo", "coll_zip")]
weather_collection = db[cfg.get("mongo","coll_weather")]
wban_collection = db[cfg.get("mongo","coll_wban")]
wban_date_format = cfg.get("weather","date_format")
wban_time_format = cfg.get("weather","time_format")
strava_datetime_format = cfg.get("strava","date_time_format")
def merge_segments_and_weather():
#iterate through segments in segment collection
for segment in segments_collection.find():
leaderboard_header = leaderboard_collection.find_one({},{'_id':0}).keys()#fieldnames of effort records
