def GET(self, format, topic_id, point_id):
i = web.input(ol="")
topics, topic, points, point = model.t_p_helper(topic_id, point_id)
if not topics:
redirect(5, _("No such topic. Delete from your bookmarks: ") + request_uri(),
links.index)
return
if not points:
redirect(5, _("No such point. Delete from your bookmarks: ") + request_uri(),
links("topic", topic.id))
return
points.annotate_by_comments()
comments = point.comments
comments.annotate_by_profiles(default=DEFAULT_COMMENT_PROFILE)
context = Storage(
title=_("Point: ") + point.title,
)
comments.last_active = point.added
for c in comments:
comments.last_active = max(comments.last_active, c.added)
c.title = util.first_line(c.text, length=64)
c.summary = c.text
c.tags = []
uri_pref = pc_links('point', topic.id, point.id) + "#comment"
get_feed_helper(context, comments,
summary=lambda c: c.summary, #???
link=lambda c: uri_pref + str(c.id),
format=format,
with_content=(not i.ol))
def index(req, nav='main', **values):
req.content_type = "text/html"
# language = localize(req)
# level = authorize(req)
return display(req,nav).toxml()
return main(req).toxml()
if nav == "links":
resp += links()
def best_match_links():
try:
query = request.json['query']
return links(query)
except:
return 'Query not submitted'
def get_today():
setting = settings()
linksobj = links()
seasonsobj = season()
today = datetime.datetime.today()
today_date = '/'.join([today.month, today.day, today.year])
start_date, end_date = today_date, today_date
get_robots_txt('https://www.basketball-reference.com/robots.txt', setting)
scrape_by_dates(start_date, end_date, setting, linksobj)
scrape_games_by_links(setting, linksobj, seasonsobj, True)
def POST(self):
presets = i = web.input(file={})
topics, topic = model.t_helper(presets.topic_id)
user = get_user()
# this is security loophole: topic owner
# can allow contribution without checking captcha
if not acl.authorize.add_point(topic): # no login needed
print "ERROR: user not authorized to add points"
return
if "create": # == True. in case other commands will be added
user_loc = user_location()
if not user_loc[0]:
MobileMap().GET() #!!! should never happen
return
presets.origin = ''
presets.address = ''
presets.url = ''
tags = presets.get("tags", "")
presets.lat = float(user_loc[0])
presets.lon = float(user_loc[1])
model.encode_coordinates(presets)
point = model.Points.create(
user=user,
name=presets.name,
lat=presets.lat,
lon=presets.lon,
url=presets.url,
origin=presets.origin,
tags=tags,
description="<p>" +
presets.description.replace("\n", "</p><p>") + "</p>",
address=presets.address,
project_id=topic.id,
fileinfo=i.file,
)
web.seeother(links("point", t_id=topic.id, p_id=point.id))
return
def POST(self):
presets = i = web.input(file={})
topics, topic = model.t_helper(presets.topic_id)
user = get_user()
# this is security loophole: topic owner
# can allow contribution without checking captcha
if not acl.authorize.add_point(topic): # no login needed
print "ERROR: user not authorized to add points"
return
if "create": # == True. in case other commands will be added
user_loc = user_location()
if not user_loc[0]:
MobileMap().GET() #!!! should never happen
return
presets.origin = ''
presets.address = ''
presets.url = ''
tags = presets.get("tags", "")
presets.lat = float(user_loc[0])
presets.lon = float(user_loc[1])
model.encode_coordinates(presets)
point = model.Points.create(
user=user,
name=presets.name,
lat=presets.lat,
lon=presets.lon,
url=presets.url,
origin=presets.origin,
tags=tags,
description="<p>" + presets.description.replace("\n", "</p><p>") + "</p>",
address=presets.address,
project_id=topic.id,
fileinfo=i.file,
)
web.seeother(links("point", t_id=topic.id, p_id=point.id))
return
def index(lang, degree, loc):
r = StrictRedis(host='localhost', port=6379, db=0)
hash_sum = "h" + md5("{0}/{1}/{2}".format(lang, degree, loc)).hexdigest()
if r.exists(hash_sum):
return r.get(hash_sum)
weather, weather_dict = weather_data(lang, degree, loc)
temp = temp_now = float(weather_dict["weather_now"]["temp"])
if degree == "F":
temp_now = (temp_now - 32)*5.0/9
result = dumps({
"weather": weather,
"links": links(lang),
"surfaces": surfaces(lang, temp_now),
"feedback": {"current_temp": str(int(temp)),
"sensation_desc": vote_sensation_list(lang),
"list_surfaces": all_surface(lang)}})
r.set(hash_sum, result)
r.expire(hash_sum, 600)
return result
def crawl(self):
tocrawl = self.urls
self.visited = set()
while not len(tocrawl) == 0:
toadd = set()
for url in tocrawl:
self.visited.add(url)
resp, content = http.request(url, 'GET')
if resp['status'] == '404' or 'text/html' not in resp['content-type']:
continue
self.page_handler(resp, content)
h = html5.document_fromstring(content, guess_charset=False)
for link in links.links(h, url=url):
if self.isurlvalid(link.dest):
toadd.add(link.dest[:link.dest.rindex('#')] if '#' in link.dest else link.dest)
self.link_handler(link)
for url in toadd: self.urls.add(url)
tocrawl = (self.urls ^ self.visited)
def main():
setting = settings()
linksobj = links()
seasonsobj = season()
args = sys.argv[1:]
if (len(args) != 2):
print(
"There should be exactly 2 arguments, the start and end date in MM/DD/YYYY"
)
return
print("\nDates will be output in YYYY/MM/DD for ordering purposes\n")
start_date = args[0]
end_date = args[1]
start = time.time()
get_robots_txt('https://www.basketball-reference.com/robots.txt', setting)
scrape_by_dates(start_date, end_date, setting, linksobj)
#linksobj.print_links()
scrape_games_by_links(setting, linksobj, seasonsobj, False)
end = time.time()
print("Done. Time elapsed: " + str(timedelta(seconds=int(end - start))))
def index(lang, degree, loc):
r = StrictRedis(host='localhost', port=6379, db=0)
hash_sum = "h" + md5("{0}/{1}/{2}".format(lang, degree, loc)).hexdigest()
if r.exists(hash_sum):
return r.get(hash_sum)
weather, weather_dict = weather_data(lang, degree, loc)
temp = temp_now = float(weather_dict["weather_now"]["temp"])
if degree == "F":
temp_now = (temp_now - 32) * 5.0 / 9
result = dumps({
"weather": weather,
"links": links(lang),
"surfaces": surfaces(lang, temp_now),
"feedback": {
"current_temp": str(int(temp)),
"sensation_desc": vote_sensation_list(lang),
"list_surfaces": all_surface(lang)
}
})
r.set(hash_sum, result)
r.expire(hash_sum, 600)
return result
def GET(self, format, topic_id, point_id):
i = web.input(ol="")
topics, topic, points, point = model.t_p_helper(topic_id, point_id)
if not topics:
redirect(
5,
_("No such topic. Delete from your bookmarks: ") +
request_uri(), links.index)
return
if not points:
redirect(
5,
_("No such point. Delete from your bookmarks: ") +
request_uri(), links("topic", topic.id))
return
points.annotate_by_comments()
comments = point.comments
comments.annotate_by_profiles(default=DEFAULT_COMMENT_PROFILE)
context = Storage(title=_("Point: ") + point.title, )
comments.last_active = point.added
for c in comments:
comments.last_active = max(comments.last_active, c.added)
c.title = util.first_line(c.text, length=64)
c.summary = c.text
c.tags = []
uri_pref = pc_links('point', topic.id, point.id) + "#comment"
get_feed_helper(
context,
comments,
summary=lambda c: c.summary, #???
link=lambda c: uri_pref + str(c.id),
format=format,
with_content=(not i.ol))
def display(req, page = "main", selected = "home"):
req.content_type = "text/html"
# language = localize(req)
# level = authorize(req)
f = open(DOCROOT + 'structure.html')
docContent = f.read()
f.close()
# document = parse(DOCROOT+'structure.html')
# kostyl before moving to the new site
document = parseString(docContent.replace('ROOTDIRDELETE', '/smdc/'))
getElem(document.getElementsByTagName("div"), "id", "menu").appendChild(menu(selected).documentElement)
if page[:5] == "links":
page = parseString("<div>" + links() + "</div>").documentElement#links())
elif page[:4] == "sat/":
page,hdr = showsatpage(page[4:].split(".")[0])
for elem in hdr.childNodes:
document.getElementsByTagName("head")[0].appendChild(deepcopy(elem))
else:
p = open(DOCROOT + page + '.html')
page = parseString(p.read())
page = page.documentElement
p.close()
getElem(document.getElementsByTagName("div"), "id", "pages").appendChild(page)
return document
from itertools import chain
from utils import makedirs
from pprint import pprint
import codecs
from os.path import join
from markup import markup
from re import compile as re, escape
import regions, names, labels, links, locations, books
from names import language_index
REGIONS = regions.regions2()
NAMES = names.names()
LANGUAGES = names.languages
LANGUAGE_NAMES = names.language_names()
LINKS = links.links()
LOCATIONS = locations.locations()
BOOKS_LIST = books.books_iter()
BOOK_ABBRS = list(zip(*BOOKS_LIST)[0])
BOOKS = dict(BOOKS_LIST)
DIR = join('build', 'articles')
parts_re = re(r'[^\w\d\.]')
def parse_citation(citation):
if citation.startswith("http://"):
return {
"link": '<a href="%s">†</a>' % (
citation,
)
def display(self):
d = self.dict
id = d.get('id','')
p = ''
sort_keys = d.keys()
sort_keys.sort()
sort_keys = ' '.join(sort_keys)
if d.has_key('mrauthid') and sort_keys == 'id mrauthid name type': ### just an MR author record
p = '<b>' + d.get('name','') + ': </b> '
p += 'MathSciNet ' + d['id'] + ' '
for link in links.links('mrauthid', d['mrauthid']):
href,anchor = link.split(None,1)
anchor = anchor.replace('MR ','')
anchor = '[' + anchor.replace('[Subscribers only]','') + ']'
p += '<a href="' + href + '" target="link_out">' + anchor + '</a> '
p += '<br>'
#if not sort_keys == 'id mrauthid name type': ### more than just an MR author record
# p += '<table>'
# for k in d.keys():
# if 1:
# v = Value(d[k])
# #p += '<tr valign=top><td align="right">' + k + ': </td><td>' + '<pre>' + simplejson.dumps(d[k],sort_keys=True, indent=4) + '</pre></td></tr>'
# p += '<tr valign=top><td align="right">' + k + ': </td><td>' + v.str() + '</td></tr>'
# p += '<tr valign=top><td align="right">************</td><td>*****************</td></tr>'
# p += '</table>'
elif d.has_key('mgpWebPage'):
mg_id = d['mgpWebPage'].split('=',1)[1]
p += '<b>' + d.get('name',mg_id) + ': </b>'
p += 'Mathematics Genealogy: [<a href = "' + d['mgpWebPage'] + '" target = "MG">' + mg_id + '</a>]'
p += '<br>'
elif id.find('datasets/106/person/') >= 0:
owp_id = id.split('datasets/106/person/',1)[1]
p += '<b>' + d.get('name',owp_id) + ': </b>'
p += 'Oberwolfach Photos: [<a href = "http://owpdb.mfo.de/person_detail?id=' + owp_id + '" target = "Oberwolfach">' + owp_id+ '</a>]'
p += '<br>'
elif d.get('type') == 'Image': ### assumes its from OWP
img = '''<table style="border-top:0;border-bottom:0;"><tr><td>
<a href="$url_detail$"> <img title="$prefLabel$" alt="$prefLabel" src="$url$" target="images", height="150" /><br /><br /></td>
<td valign="top">$right_column$</td></tr></table>'''
d['url_detail'] = d['url'].replace('photoNormal?id=','detail?photo_id=')
for k in 'url url_detail prefLabel'.split(): img = img.replace('$' + k + '$',d.get(k,''))
d['description'] = d.get('localityName','') + ', ' + d.get('date','')
if d.get('creator', ''): d['prefLabel'] += ' by ' + d.get('creator', '')
d['copyright'] = 'Source: ' + d.get('publisher','')
d['prefLabel'] = d['prefLabel'].replace('of:','of ')
d['text'] = 'Click on the photo for a larger version with copyright details.'
right_column = '<table>'
for k in 'prefLabel description copyright text'.split():
v = Value(d[k])
right_column += '<tr valign=top><td align="right"> </td><td>' + v.str() + '</td></tr>'
right_column += '</table>'
p += img.replace('$right_column$',right_column)
elif id.find('datasets/130/resource/') >= 0:
p += '<b>' + d.get('name','') + '</b>: <a href="http://imstat.org/awards/honored_fellows.htm">IMS Fellow </a><br>'
p += '<table>'
for k in d.keys():
if not k in ['name', 'mrauthid','prefLabel','type','id']:
v = Value(d[k])
p += '<tr valign=top><td align="right">' + k + ': </td><td>' + v.str() + '</td></tr>'
p += '</table>'
else:
p += '<table>'
for k in d.keys():
##if not k in [', 'mrauthid','prefLabel']:
v = Value(d[k])
p += '<tr valign=top><td align="right">' + k + ': </td><td>' + v.str() + '</td></tr>'
p += '</table>'
#p = '<pre>' + self.json_indented(4) + '</pre>'
return p
keyword_density(hash, query, soup, check_query)
#print('title_h1')
title_h1(hash, tree)
#print('viewport')
viewport(hash, code)
#print('description')
description(hash, tree)
#print('title')
title(hash, tree)
#print('links')
links(hash, result_main, html_source)
#print('plugins')
plugins(hash, html_source, html_comments)
#print('https')
https(result_url, hash)
#print('micros')
micros(hash, html_comments, html_source)
#print('og')
og(hash, code)
#print('sitemap')
sitemap(hash, code)
def index(req, nav='public/main', switchdiv='Overview', switchsubdiv='', mark='', **values):
selnav = "x"+nav
if selnav == "xcoronasi" or selnav == "xcosmos1686" or selnav == "xryabina" or selnav == "xcoronasf" or selnav == "xmeteor" or selnav == "xglonass" or selnav == "xtatyana":
selnav = "xdata"
elif selnav == "xflares" or selnav == "xstorms" or selnav == "xmpause" or selnav == "xcoronas-pics" or selnav == "xomni" or selnav == "xcoho" or selnav == "xservices":
selnav = "xservice"
elif selnav == "xmodel-para" or selnav == "xmodel-ap8ae8" or selnav == "xmodel-igrf" or selnav == "xmodel-shieldose2" or selnav == "xmodel-cosrad":
selnav = "xmodels"
req.content_type = "text/html"
language = localize(req)
level = authorize(req)
f = open(DOCROOT+level+'index.html','r')
resp = f.read().replace(
'class="menu" id="%s"' % selnav,
'class="currentmenu"').replace(
"_COPYRIGHT_YEAR_",
str(datetime.now().year))
f.close()
if switchsubdiv == '':
subdiv = "javascript:switchdiv('%s');"%switchdiv
else:
subdiv = "javascript:switchsubdiv('%s', '%s');"%(switchdiv, switchsubdiv)
if mark != "":
subdiv += "javascript:markswitch('%s')"%mark
resp = resp.replace("javascript:init();", "javascript:init();%s"%subdiv)
if nav == "links":
resp += links()
elif nav.find("exp-") == 0:
try:
exp = nav.split("_")[0].split("-")[1]
if exp not in experiments:
f = open(DOCROOT+'404.html','r')
resp += ('<div style="overflow:auto;border:0px solid #666;padding: 8px" >')
resp += f.read()
resp += '</div>'
f.close()
else:
# resp += '<p>%s/%s/%s.html</p>' % (EXPURL, exp, nav.split("_")[1])
f = urlopen("%s/%s/%s.html"%(EXPURL, exp, nav.split("_")[1]))
resp += ('<div style="overflow:auto;border:0px solid #666;padding: 8px" >')
resp += f.read()
resp += '</div>'
f.close()
except:
f = open(DOCROOT+'404.html','r')
resp += ('<div style="overflow:auto;border:0px solid #666;padding: 8px" >')
resp += f.read()
resp += '</div>'
f.close()
else:
try:
f = open(DOCROOT + level + nav + '.html', 'r')
except:
f = open(DOCROOT+'404.html','r')
if selnav == "xmain":
resp+=('<div style="overflow:auto;border:0px solid #666;padding: 4px;background:url(\'design/'+level +'bg.jpg\') no-repeat center">')
else:
resp+=('<div style="overflow:auto;border:0px solid #666;padding: 4px">')
resp += f.read()
#req.write(translate(, language))
resp += '</div>'
f.close()
resp = resp.replace("_HDRTITLE_", mkTitle(nav) + TITLE)
# Parse other arguments and set optional checks
if nav == "model-igrf":
try:
resp = resp.replace("_IGRFWEB_", urlopen("http://cosrad.sinp.msu.ru/cgi-bin/igrf/igrfcmdline.pl?notable=1").read() + "<div id='igrfresult'> </div>")
resp = resp.replace('onclick="igrf_calc"', 'onclick="igrf_calc()"')
except:
resp = resp.replace("_IGRFWEB_", "Unable to contact IGRF model server")
if nav == "model-ap8ae8":
try:
resp = resp.replace("_AP8AE8WEB_", urlopen("http://cosrad.sinp.msu.ru/cgi-bin/ap8ae8/ap8ae8cmdline.pl?notable=1").read() + "<div id='ap8ae8result'> </div>")
resp = resp.replace('onclick="ap8ae8_calc"', 'onclick="ap8ae8_calc()"')
except:
resp = resp.replace("_AP8AE8WEB_", "Unable to contact IGRF model server")
for x in values:
if x != "output":
resp = resp.replace('name="%s%s"'%(mark, x), 'name="%s%s"'%(mark, x) + ' checked="checked"')
# Costyl!!!
resp = resp.replace('name="x%s"'%x, 'name="x%s"'%x + ' checked="checked"')
resp = resp.replace('name="j%s"'%x, 'name="j%s"'%x + ' checked="checked"')
else:
resp = resp.replace('name="%s%s"'%(mark, x) + ' value="%s"'%req.form.get(x, ''), 'name="%s%s"'%(mark, x) + ' value="%s"'%req.form.get(x, '') + ' checked="checked"')
# Costyl!!!
resp = resp.replace('name="x%s"'%x + ' value="%s"'%req.form.get(x, ''), 'name="x%s"'%x + ' value="%s"'%req.form.get(x, '') + ' checked="checked"')
#resp += 'name="%s%s"'%(mark, x) + " " + 'name="%s%s"'%(mark, x) + ' checked="checked"'
resp += """<h3 style="font-weight:normal;padding:5px;font-size:12px" align="right" class="separator">
<i>Developed in 2007.
<a href="http://www.gnu.org/copyleft/gpl.html" style="background-color:#d9e5f2">
Copyright © </a>
2007 - %s <a href="http://www.sinp.msu.ru" style="background-color:#d9e5f2">SINP</a>
<a href="http://www.msu.ru" style="background-color:#d9e5f2">MSU</a></i>
</h3>
</td></tr></table></body></html>""" % str(datetime.now().year)
return translate(resp, language)
def GET(self, topic_id, onlycode=False, webinput=None):
presets = webinput or web.input()
try:
presets.referer = web.ctx.environ["HTTP_REFERER"]
except KeyError:
presets.referer = 0
context = Storage(
presets=presets,
offset = 10,
framed = 1,
onlycode=onlycode,
title=presets.get("title", _("Points")),
desturl=pc_links("topic", int(topic_id), **presets),
# submitform=pc_links("topic_newpoint", int(topic_id)),
)
topics, topic = model.t_helper(topic_id)
topics.annotate_by_datasources()
topics.annotate_by_points()
number_of_points = int(presets.get("number_of_points",
config.points_per_page))
points = model.Points(project=topic, external=None)
points.limit_by_page(1, length=number_of_points)
points.annotate_by_comments()
points.annotate_by_tags()
points.annotate_by_profiles(default=DEFAULT_POINT_PROFILE)
feed_link = feed_links("topic_points",
topic.id, presets, page=None)
if "zoom" in topic.profile and topic.profile.zoom != "auto":
c_lat, c_lon, c_zoom = \
topic.lat, topic.lon, topic.profile.zoom
c_zoom = int(c_zoom)
else:
c_lat, c_lon, c_zoom = points.central_point(
config.center_lat, config.center_lon, 14)
map_context = Storage(
getmap_url=pc_links.getmap,
getmap_layers=config.getmap_layers,
getmap_layers1=config.getmap_layers1,
getmap_layers2=config.getmap_layers2,
map_params=config.getmap_params,
getmap_custom=config.getmap_custom,
getmap_custom_init=config.getmap_custom_init,
lat=c_lat,
lon=c_lon,
zoom=c_zoom + config.getmap_zoomshift,
initial_feed=feed_link,
has_new_point="false",
getmap_zoom1=config.getmap_zoom1 + config.getmap_zoomshift,
getmap_zoom2=config.getmap_zoom2 + config.getmap_zoomshift,
)
model.encode_coordinates(map_context)
context.map_context = map_context
context.page_specific_js = macros.map_js(map_context)
if onlycode:
presets.referer = ''
# !!! map widget can't be part of combo
self_link = links('map', topic_id, **presets)
return """<iframe src="%s" style="height:%spx;width:%spx;border:none;"></iframe>""" % \
(self_link, presets.height, presets.width)
get_widget('map', context, topic, presets)
from streamango import Streamango
app = Flask(__name__)
URL = "http://server.full-stream.nu"
DATABASE = 'safe_links'
USERNAME = '******'
PASSWORD = '******'
PORT = 3306
user_ins = user(username=USERNAME,
password=PASSWORD,
database=DATABASE,
port=PORT)
links_ins = links(username=USERNAME,
password=PASSWORD,
database=DATABASE,
port=PORT)
secret = b"Cool Links!!"
def haaash(w):
h = hashlib.md5(w)
x = h.hexdigest().encode('utf-8')
h = hashlib.md5(x)
return h.hexdigest()
def make_secure_val(val):
return '%s|%s' % (val, hmac.new(
secret, val.encode('utf-8')).hexdigest().encode('utf-8'))
