def get(self):
"""
Handles the /register endpoint and PUBLIC
user registration.
"""
if self.user:
self.redirect("/dashboard")
else:
self.tv['page_register'] = True
self.tv["data"] = {}
if get_cookie(self, name="_rm_"):
self.tv["data"] = json.loads(base64.b64decode(get_cookie(self, name="_rm_")))
clear_cookie(self, name="_rm_")
if USER_REGISTER:
if self.GET('ea'):
self.tv['user_email'] = base64.b64decode(self.GET('ea'))
self.tv['agencies'] = Agency.query().fetch(100)
self.render("register.html")
else:
url = "/admin/register"
if self.GET("redirect"):
url += "?redirect="
url += self.GET("redirect")
self.redirect(url)
def get_auth():
username = cookie.get_cookie("username", None)
token = cookie.get_cookie("token", None)
if username is not None and token is not None and _check_token(username, token):
return username
else:
return "wegas_tmp" # TODODODODO
def login_handler(environ, start_response):
"""
ログインハンドラメソッド
/loginへリクエストを受けた際に呼び出される。
Sessionの有無やGETかPOSTかを判断して、HTML生成か認証を行うかを判断する
:param envrion:
:param start_response:
:return:
"""
# クッキーの情報を取得
cookie = get_cookie(environ)
if cookie is None:
""" セッションを確立していない場合 """
method = environ['REQUEST_METHOD']
if method == 'GET':
""" リクエストがGETメソッドである場合 """
# ログイン画面HTML生成
return html(environ, start_response, '/html/login.html')
elif method == 'POST':
""" リクエストがPOSTメソッドである場合 """
# ログイン処理
return login(environ, start_response)
else:
""" GETでもPOSTでもない場合 """
# 400 Bad Request
return bad_request(environ, start_response)
else:
""" すでにセッションを確立している場合 """
# 302 Found
return found('/menu', environ, start_response)
def parse(self, response):
cookie = get_cookie()
print cookie
page = requests.get('http://pravda.com.ua/articles', cookies=cookie)
assert page.ok
# create new resposne with new text for xpathing
res = scrapy.http.HtmlResponse(url='http://pravda.com.ua/articles',
body=page.text,
encoding='utf8')
all_links = res.xpath(
".//div[@class='summary sec']/h4/a/@href").extract()
all_names = res.xpath(
".//div[@class='summary sec']/h4/a/text()").extract()
filtered_links = [(all_links.index(link), link) for link in all_links
if "http" not in link]
# import ipdb; ipdb.set_trace()
filtered_names_and_links = [(all_names[i[0]], i[1])
for i in filtered_links]
# other_links = [link for link in all_links if "http" in link]
for name, link in filtered_names_and_links:
full_url = response.url + link
item = ArticleItem()
item['title'] = name
item['link'] = full_url
item['datetime'] = datetime.datetime.now() # int(time.time())
item['comments'] = 0
item['shares'] = json.loads(
requests.get(
"""https://graph.facebook.com/fql?q=select%20%20share_count%20from%20link_stat%20where%20url=%22{}%22"""
.format(full_url)).text)['data'][0]['share_count']
yield item
def logout_handler(environ, start_response):
"""
ログアウトメソッド
:param environ:
:param start_response:
:return:
"""
cookie = get_cookie(environ)
if cookie is None:
""" セッションを確立していない場合の処理 """
print ("forbbiden", environ['PATH_INFO'])
return forbidden(environ, start_response)
else:
""" セッションを確立している場合の処理 """
result = session_disconnect(cookie)
if result:
cookie['session']['expires'] = "Thu, 01-Jan-1970 00:00:00 GMT"
cookie['user_id']['expires'] = "Thu, 01-Jan-1970 00:00:00 GMT"
status = '302 Found'
response_headers = [('Location', '/login'),
('Set-Cookie', cookie['session'].OutputString()),
('Set-Cookie', cookie['user_id'].OutputString())]
start_response(status, response_headers)
return []
else:
status = '302 Found'
response_headers = [('Location', '/menu')]
start_response(status, response_headers)
return []
def checkuser(phonenumber):
url = 'http://weibo.com/signup/v5/formcheck'
headers = {
'Accept': '*/*',
'Accept-Encoding': 'gzip, deflate, sdch',
'Accept-Language': 'en-US,en;q=0.8,zh-CN;q=0.6,zh;q=0.4',
'Connection': 'keep-alive',
'Content-Type': 'application/x-www-form-urlencoded',
'Host': 'weibo.com',
'Referer': 'http://weibo.com/signup/signup.php?lang=zh-cn',
'User-Agent':
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.124 Safari/537.36',
'X-Requested-With': 'XMLHttpRequest',
}
data = {
'type': 'mobilesea',
'zone': '0086',
'value': '{}'.format(phonenumber),
'from': '',
'__rnd': '1438414785889',
}
s = requests.Session()
res = s.get(url=url,
params=data,
headers=headers,
cookies=get_cookie(
url='http://weibo.com/signup/signup.php?lang=zh-cn'),
timeout=3)
return res
def prg_insert_handler(environ, start_response):
"""
作業データをプログラムデータテーブルに挿入するメソッド
"""
cookie = get_cookie(environ)
if cookie is None:
""" セッションを確立していない場合 """
return forbidden(environ, start_response)
else:
""" セッションを確立している場合 """
usrId = cookie['user_id'].value
form = cgi.FieldStorage(environ=environ, fp=environ['wsgi.input'])
prgName = form.getfirst('prgName')
prgData = form.getfirst('workSpaceData')
comment = form.getfirst('comment')
prgId = prgInsert(usrId, prgName, prgData, comment)
result = '1'
output = '{},{}'.format(result, prgId)
status = '200 OK'
response_headers = [('Content-type', 'text/html'), ('Content-Length', str(len(output)))]
start_response(status, response_headers)
return [output.encode()]
def exp_update_handler(environ, start_response):
"""
呼び出しメソッド
"""
cookie = get_cookie(environ)
if cookie is None:
""" セッションを確立していない場合 """
return forbidden(environ, start_response)
else:
""" セッションを確立している場合 """
form = cgi.FieldStorage(environ=environ, fp=environ['wsgi.input'])
exp_id = form.getfirst('expId')
exp_data = form.getfirst('workSpaceData')
comment = form.getfirst('comment')
help_menu = form.getfirst('helpMenu')
result = form.getfirst('result')
limit_blocks = form.getfirst('limitedBlocks')
ret = expUpdate(exp_id, exp_data, comment, help_menu, result, limit_blocks)
output = '{}'.format(ret)
status = '200 OK'
response_headers = [('Content-type', 'text/html'), ('Content-Length', str(len(output)))]
start_response(status, response_headers)
return [output.encode()]
def users_handler(environ, start_response):
cookie = get_cookie(environ)
if cookie is None:
""" セッションを確立していない場合 """
return forbidden(environ, start_response)
else:
output = userApp(cookie['user_id'].value)
status = "200 OK"
headers = [('Content-type', 'text/html'), ('Content-Length', str(len(output)))]
start_response(status, headers)
return [output]
def make_header(referer=False, explorar=False):
"""
Hace una aplicacion parcial de la cookie, y agrega el referer de donde deberia venir la peticion
:param referer: opcional: si existe, lo agrega
:param explorar: opcional: si se desea explorar datos, el header es distinto
:return: headers correspondientes para hacer una peticion
"""
headers = GET_INFO_ENEMY_HEADERS if explorar else HEADERS
headers['Cookie'] = get_cookie()
if referer:
headers['Referer'] = "http://{}/{}".format(DOMAIN, referer)
return headers
def getheader_dict(target_domain):
# add UA
header = {
'User-Agent':'Mozilla/2.0 (X11; Linux x86_64) AppleWebKit/237.36 (KHTML, like Gecko) Chrome/62.0.3322.146 Safari/237.36',
}
# add cookie
cookie = get_cookie(target_domain)
# if cookie is in date,add it
if cookie:
header['Cookie']= cookie
# add referer
header['Referer']='https://'+target_domain+'/'
return header
def expProgHTML_handler(environ, start_response):
cookie = get_cookie(environ)
if cookie is None:
""" セッションを確立していない場合 """
return forbidden(environ, start_response)
else:
""" セッションを確立している場合 """
query = dict(urllib.parse.parse_qsl(environ.get('QUERY_STRING')))
expId = None
if 'expId' in query:
expId = query['expId']
output = expProgHTML(expId, cookie['user_id'].value)
status = "200 OK"
headers = [('Content-type', 'text/html'), ('Content-Length', str(len(output)))]
start_response(status, headers)
return [output]
def gen_traffic(self, url):
domain = get_domain_from_url(url)
# add cookie to DEFAULT_HEADER
cookie = get_cookie(domain)
self.DEFAULT_HEADER['Cookie'] = cookie
# add referer
self.DEFAULT_HEADER['Referer'] = 'https"//' + domain + '/'
request = HttpRequest(method='GET', url=url, headers=self.DEFAULT_HEADER, body='')
req = urllib2.Request(url=url, headers=self.DEFAULT_HEADER)
with gevent.Timeout(10, False)as t:
try:
resp = urllib2.urlopen(req)
except urllib2.URLError, e:
REQUEST_ERROR.append(('gen_traffic()', url, e.reason))
except CertificateError:
REQUEST_ERROR.append(('gen_traffic()', url, 'ssl.CertificateError'))
def menu_handler(environ, start_response):
"""
メニューハンドラメソッド
/menuへリクエストが送られた場合に呼び出されるメソッド
:param environ:
:param start_response:
:return:
"""
cookie = get_cookie(environ)
if 'session' in cookie:
""" セッションを確立している場合 """
# htmlを生成する
return html_response(environ, start_response, cookie)
else:
""" セッションを確立していない場合 """
# 403 forbidden
return forbidden(environ, start_response)
def getheader(target_domain):
# add UA
header = [
('User-Agent','Mozilla/2.0 (X11; Linux x86_64) AppleWebKit/237.36 (KHTML, like Gecko) Chrome/62.0.3322.146 Safari/237.36'),
]
# add cookie
if is_ip(target_domain):
ip=target_domain
cookie=get_cookie_ip(ip)
else:
cookie = get_cookie(target_domain)
# if cookie is in date,add it
if cookie:
header.append(('Cookie', cookie))
# referer bypass
header.append(('Referer','https://'+target_domain+'/'))
return header
def prg_delete_handler(environ, start_response):
"""
作業データを作業データテーブルから削除するメソッド
"""
cookie = get_cookie(environ)
if cookie is None:
""" セッションを確立していない場合 """
return forbidden(environ, start_response)
else:
""" セッションを確立している場合 """
form = cgi.FieldStorage(environ=environ, fp=environ['wsgi.input'])
prgId = form.getfirst('prgID')
result = prgDelete(prgId)
output = '{}'.format(result)
status = '200 OK'
response_headers = [('Content-type', 'text/html'), ('Content-Length', str(len(output)))]
start_response(status, response_headers)
return [output.encode()]
def checkuser(phonenumber):
url = 'http://weibo.com/signup/v5/formcheck'
headers = {
'Accept':'*/*',
'Accept-Encoding':'gzip, deflate, sdch',
'Accept-Language':'en-US,en;q=0.8,zh-CN;q=0.6,zh;q=0.4',
'Connection':'keep-alive',
'Content-Type':'application/x-www-form-urlencoded',
'Host':'weibo.com',
'Referer':'http://weibo.com/signup/signup.php?lang=zh-cn',
'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.124 Safari/537.36',
'X-Requested-With':'XMLHttpRequest',
}
data = {
'type':'mobilesea',
'zone':'0086',
'value':'{}'.format(phonenumber),
'from':'',
'__rnd':'1438414785889',
}
s = requests.Session()
res = s.get(url=url, params=data, headers=headers, cookies=get_cookie(url='http://weibo.com/signup/signup.php?lang=zh-cn'), timeout=3)
return res
def _get_cookies(self):
cookie_str = self._environ.get('HTTP_COOKIE', None)
if not cookie_str:
return None
cookie_dict = cookie.get_cookie(cookie_str)
return cookie_dict
def get_accepted_language_codes(firstshot=False):
"""
Findet heraus, welche Sprache eingestellt wurde und gibt eine Liste mit
möglichen Ländercodes, sortiert nach Priorität, zurück.
Zuerst wird in der URL nachgesehen, ob eine Sprache angefordert wird.
Wenn nicht, dann wird das Language-Cookie ausgewertet.
Dann wird der Request nach der Spracheinstellung des Browsers durchsucht.
Wird die Sprache über die URL übergeben, dann wird diese zusätzlich in das
Language-Cookie geschrieben.
:param firstshot: Wenn True, dann wird die erste mögliche Sprache zurück
geliefert. Es wird nicht lange nach anderen Sprachen gesucht.
"""
LANGUAGE_CODES = config.LANGUAGES.value
language_code = None
lang_items = []
# Path Info
path_list = [item for item in cherrypy.request.path_info.split("/") if item]
if path_list:
if path_list[0] in LANGUAGE_CODES:
language_code = path_list[0]
if firstshot:
return [language_code]
lang_items.append([3.0, language_code])
# Neue Sprache in das Language-Cookie eintragen
cookie.set_cookie(cookie.LANGUAGE, language_code)
# Language-Cookie
if not language_code:
lang_cookie = cookie.get_cookie(cookie.LANGUAGE)
if lang_cookie in LANGUAGE_CODES:
language_code = lang_cookie
if firstshot:
return [language_code]
lang_items.append([2.0, language_code])
# Browser
accept_language = cherrypy.request.headers.get("ACCEPT-LANGUAGE")
if accept_language:
accept_language = accept_language.replace("-", "_").lower()
# Möglichkeit 1: da, en-gb;q=0.8, en;q=0.7
# Möglichkeit 2: de-de;q=1.0,de;q=0.8,en-us;q=0.5,en;q=0.3
raw_items = [item.strip() for item in accept_language.split(",") if item.strip()]
for raw_item in raw_items:
if ";" in raw_item:
try:
lang, q = [item.strip() for item in raw_item.split(";", 1) if item.strip()]
lang_items.append([float(q.split("=")[1].strip()), lang])
except IndexError:
pass
else:
lang_items.append([1.0, raw_item])
# Standardsprache anhängen
lang_items.append([0.0, LANGUAGE_CODES[0]])
# Einträge nach Priorität sortieren und zurück geben
lang_items.sort(reverse=True)
ret_items = []
for lang_item in lang_items:
if lang_item[1] in LANGUAGE_CODES:
if lang_item[1] not in ret_items:
ret_items.append(lang_item[1])
return ret_items
import requests
import bs4,time,re,chardet,random
import cookie,json,csv
cookies=cookie.get_cookie()
headers=cookie.headers
def get_id():
with open("./data/game_id.csv","r",encoding="utf-8") as file:
id_list=file.read().split("\n")
id_list.remove("")
return id_list
def get_info_page(id):
url="http://store.steampowered.com/app/"+id+"/"
while True:
try:
page=requests.get(url,headers=headers,cookies=cookies)
return page.text
except:
time.sleep(40)
continue
def get_review_page(id,num):
url="http://store.steampowered.com/curators/ajaxgetcurators/render/?query=&start="+str(num)+"&count=10&filter=top_curators_reviewing&appid="+str(id)+"&new_browse=0"
page=requests.get(url,headers=headers,cookies=cookies)
#content=re.findall('results_html":"(.+?)"}',page.text)[0]
content=json.loads(page.text)["results_html"]
return content
def _get_cookies(self):
cookie_str = self._environ.get('HTTP_COOKIE', None)
if not cookie_str:
return None
cookie_dict = cookie.get_cookie(cookie_str)
return cookie_dict
