Exemplo n.º 1
0
    def GET(self, format, topic_id, point_id):
        i = web.input(ol="")
        topics, topic, points, point = model.t_p_helper(topic_id, point_id)
        if not topics:
            redirect(5, _("No such topic. Delete from your bookmarks: ") + request_uri(), 
                links.index)
            return

        if not points:
            redirect(5, _("No such point. Delete from your bookmarks: ") + request_uri(), 
                links("topic", topic.id))
            return

        points.annotate_by_comments()

        comments = point.comments
        comments.annotate_by_profiles(default=DEFAULT_COMMENT_PROFILE)

        context = Storage(
            title=_("Point: ") + point.title,
        )

        comments.last_active = point.added
        for c in comments:
            comments.last_active = max(comments.last_active, c.added)
            c.title = util.first_line(c.text, length=64)
            c.summary = c.text
            c.tags = []

        uri_pref = pc_links('point', topic.id, point.id) + "#comment"
        get_feed_helper(context, comments,
            summary=lambda c: c.summary,  #???
            link=lambda c: uri_pref + str(c.id),
            format=format,
            with_content=(not i.ol))
Exemplo n.º 2
0
def index(req, nav='main', **values):
    req.content_type = "text/html"
#    language = localize(req)
#    level = authorize(req)
    return display(req,nav).toxml()
    return main(req).toxml()
    if nav == "links":
        resp += links()
Exemplo n.º 3
0
def best_match_links():

    try:
        query = request.json['query']

        return links(query)

    except:
        return 'Query not submitted'
Exemplo n.º 4
0
def get_today():
    setting = settings()
    linksobj = links()
    seasonsobj = season()
    today = datetime.datetime.today()
    today_date = '/'.join([today.month, today.day, today.year])
    start_date, end_date = today_date, today_date
    get_robots_txt('https://www.basketball-reference.com/robots.txt', setting)
    scrape_by_dates(start_date, end_date, setting, linksobj)
    scrape_games_by_links(setting, linksobj, seasonsobj, True)
Exemplo n.º 5
0
    def POST(self):
        presets = i = web.input(file={})

        topics, topic = model.t_helper(presets.topic_id)

        user = get_user()

        # this is security loophole: topic owner
        # can allow contribution without checking captcha
        if not acl.authorize.add_point(topic):  # no login needed
            print "ERROR: user not authorized to add points"
            return

        if "create":  # == True. in case other commands will be added
            user_loc = user_location()
            if not user_loc[0]:
                MobileMap().GET()  #!!! should never happen
                return

            presets.origin = ''
            presets.address = ''
            presets.url = ''
            tags = presets.get("tags", "")
            presets.lat = float(user_loc[0])
            presets.lon = float(user_loc[1])
            model.encode_coordinates(presets)

            point = model.Points.create(
                user=user,
                name=presets.name,
                lat=presets.lat,
                lon=presets.lon,
                url=presets.url,
                origin=presets.origin,
                tags=tags,
                description="<p>" +
                presets.description.replace("\n", "</p><p>") + "</p>",
                address=presets.address,
                project_id=topic.id,
                fileinfo=i.file,
            )

            web.seeother(links("point", t_id=topic.id, p_id=point.id))
            return
Exemplo n.º 6
0
    def POST(self):
        presets = i = web.input(file={})

        topics, topic = model.t_helper(presets.topic_id)

        user = get_user()

        # this is security loophole: topic owner
        # can allow contribution without checking captcha
        if not acl.authorize.add_point(topic):  # no login needed
            print "ERROR: user not authorized to add points"
            return


        if "create":  # == True. in case other commands will be added
            user_loc = user_location()
            if not user_loc[0]:
                MobileMap().GET()  #!!! should never happen
                return

            presets.origin = ''
            presets.address = ''
            presets.url = ''
            tags = presets.get("tags", "")
            presets.lat = float(user_loc[0])
            presets.lon = float(user_loc[1])
            model.encode_coordinates(presets)

            point = model.Points.create(
                user=user,
                name=presets.name,
                lat=presets.lat,
                lon=presets.lon,
                url=presets.url,
                origin=presets.origin,
                tags=tags,
                description="<p>" + presets.description.replace("\n", "</p><p>") + "</p>",
                address=presets.address,
                project_id=topic.id,
                fileinfo=i.file,
            )

            web.seeother(links("point", t_id=topic.id, p_id=point.id))
            return
def index(lang, degree, loc):
    r = StrictRedis(host='localhost', port=6379, db=0)
    hash_sum = "h" + md5("{0}/{1}/{2}".format(lang, degree, loc)).hexdigest()
    if r.exists(hash_sum):
        return r.get(hash_sum)
    weather, weather_dict = weather_data(lang, degree, loc)
    temp = temp_now = float(weather_dict["weather_now"]["temp"])
    if degree == "F":
        temp_now = (temp_now - 32)*5.0/9
    result = dumps({
        "weather": weather,
        "links": links(lang),
        "surfaces": surfaces(lang, temp_now),
        "feedback": {"current_temp": str(int(temp)), 
                     "sensation_desc": vote_sensation_list(lang),
                     "list_surfaces": all_surface(lang)}})
    r.set(hash_sum, result)
    r.expire(hash_sum, 600)
    return result
Exemplo n.º 8
0
 def crawl(self):
   tocrawl = self.urls
   self.visited = set()
   while not len(tocrawl) == 0:
     toadd = set()
     for url in tocrawl:
       self.visited.add(url)
       resp, content = http.request(url, 'GET')
       if resp['status'] == '404' or 'text/html' not in resp['content-type']:
         continue
       
       self.page_handler(resp, content)
       h = html5.document_fromstring(content, guess_charset=False)
       for link in links.links(h, url=url):
         if self.isurlvalid(link.dest):
           toadd.add(link.dest[:link.dest.rindex('#')] if '#' in link.dest else link.dest)
         self.link_handler(link)
     
     for url in toadd: self.urls.add(url)
     tocrawl = (self.urls ^ self.visited)
Exemplo n.º 9
0
def main():
    setting = settings()
    linksobj = links()
    seasonsobj = season()
    args = sys.argv[1:]
    if (len(args) != 2):
        print(
            "There should be exactly 2 arguments, the start and end date in MM/DD/YYYY"
        )
        return
    print("\nDates will be output in YYYY/MM/DD for ordering purposes\n")
    start_date = args[0]
    end_date = args[1]
    start = time.time()
    get_robots_txt('https://www.basketball-reference.com/robots.txt', setting)
    scrape_by_dates(start_date, end_date, setting, linksobj)
    #linksobj.print_links()
    scrape_games_by_links(setting, linksobj, seasonsobj, False)
    end = time.time()
    print("Done. Time elapsed: " + str(timedelta(seconds=int(end - start))))
Exemplo n.º 10
0
def index(lang, degree, loc):
    r = StrictRedis(host='localhost', port=6379, db=0)
    hash_sum = "h" + md5("{0}/{1}/{2}".format(lang, degree, loc)).hexdigest()
    if r.exists(hash_sum):
        return r.get(hash_sum)
    weather, weather_dict = weather_data(lang, degree, loc)
    temp = temp_now = float(weather_dict["weather_now"]["temp"])
    if degree == "F":
        temp_now = (temp_now - 32) * 5.0 / 9
    result = dumps({
        "weather": weather,
        "links": links(lang),
        "surfaces": surfaces(lang, temp_now),
        "feedback": {
            "current_temp": str(int(temp)),
            "sensation_desc": vote_sensation_list(lang),
            "list_surfaces": all_surface(lang)
        }
    })
    r.set(hash_sum, result)
    r.expire(hash_sum, 600)
    return result
Exemplo n.º 11
0
    def GET(self, format, topic_id, point_id):
        i = web.input(ol="")
        topics, topic, points, point = model.t_p_helper(topic_id, point_id)
        if not topics:
            redirect(
                5,
                _("No such topic. Delete from your bookmarks: ") +
                request_uri(), links.index)
            return

        if not points:
            redirect(
                5,
                _("No such point. Delete from your bookmarks: ") +
                request_uri(), links("topic", topic.id))
            return

        points.annotate_by_comments()

        comments = point.comments
        comments.annotate_by_profiles(default=DEFAULT_COMMENT_PROFILE)

        context = Storage(title=_("Point: ") + point.title, )

        comments.last_active = point.added
        for c in comments:
            comments.last_active = max(comments.last_active, c.added)
            c.title = util.first_line(c.text, length=64)
            c.summary = c.text
            c.tags = []

        uri_pref = pc_links('point', topic.id, point.id) + "#comment"
        get_feed_helper(
            context,
            comments,
            summary=lambda c: c.summary,  #???
            link=lambda c: uri_pref + str(c.id),
            format=format,
            with_content=(not i.ol))
Exemplo n.º 12
0
def display(req, page = "main", selected = "home"):
    req.content_type = "text/html"
#    language = localize(req)
#    level = authorize(req)
    f = open(DOCROOT + 'structure.html')
    docContent = f.read()
    f.close()
#    document = parse(DOCROOT+'structure.html')
    # kostyl before moving to the new site
    document = parseString(docContent.replace('ROOTDIRDELETE', '/smdc/'))
    getElem(document.getElementsByTagName("div"), "id", "menu").appendChild(menu(selected).documentElement)
    if page[:5] == "links":
        page = parseString("<div>" + links() + "</div>").documentElement#links())
    elif page[:4] == "sat/":
        page,hdr = showsatpage(page[4:].split(".")[0])
        for elem in hdr.childNodes:
            document.getElementsByTagName("head")[0].appendChild(deepcopy(elem))
    else:
        p = open(DOCROOT + page + '.html')
        page = parseString(p.read())
        page = page.documentElement
        p.close()
    getElem(document.getElementsByTagName("div"), "id", "pages").appendChild(page)
    return document
Exemplo n.º 13
0
from itertools import chain
from utils import makedirs
from pprint import pprint
import codecs
from os.path import join
from markup import markup
from re import compile as re, escape

import regions, names, labels, links, locations, books
from names import language_index
REGIONS = regions.regions2()
NAMES = names.names()
LANGUAGES = names.languages
LANGUAGE_NAMES = names.language_names()
LINKS = links.links()
LOCATIONS = locations.locations()
BOOKS_LIST = books.books_iter()
BOOK_ABBRS = list(zip(*BOOKS_LIST)[0])
BOOKS = dict(BOOKS_LIST)

DIR = join('build', 'articles')

parts_re = re(r'[^\w\d\.]')

def parse_citation(citation):
    if citation.startswith("http://"):
        return {
            "link": '<a href="%s">&dagger;</a>' % (
                citation,
            )
Exemplo n.º 14
0
    def display(self): 
        d = self.dict
        id = d.get('id','')
        p = ''
        sort_keys = d.keys()
        sort_keys.sort()
        sort_keys = ' '.join(sort_keys)
        
        if d.has_key('mrauthid') and sort_keys == 'id mrauthid name type': ### just an MR author record
            p = '<b>' + d.get('name','') + ': </b> '
            p += 'MathSciNet ' + d['id'] + ' ' 
            for link in links.links('mrauthid', d['mrauthid']):
                href,anchor = link.split(None,1)
                anchor = anchor.replace('MR ','')
                anchor = '[' + anchor.replace('[Subscribers only]','') + ']'
                p += '<a href="' + href + '" target="link_out">'  +  anchor + '</a> '
            p += '<br>'
            #if not sort_keys == 'id mrauthid name type': ### more than just an MR author record
            #    p += '<table>'
            #    for k in d.keys():
            #        if 1:
            #            v = Value(d[k])
            #            #p += '<tr valign=top><td align="right">' + k + ': </td><td>' + '<pre>' + simplejson.dumps(d[k],sort_keys=True, indent=4) + '</pre></td></tr>'
            #            p += '<tr valign=top><td align="right">' + k + ': </td><td>' + v.str() + '</td></tr>'
            #    p += '<tr valign=top><td align="right">************</td><td>*****************</td></tr>'
            #    p += '</table>'
        elif d.has_key('mgpWebPage'):
             mg_id =  d['mgpWebPage'].split('=',1)[1]
             p += '<b>' +  d.get('name',mg_id)  + ': </b>'
             p += 'Mathematics Genealogy: [<a href = "' + d['mgpWebPage'] + '" target = "MG">' + mg_id + '</a>]'
             p += '<br>'
        elif id.find('datasets/106/person/') >= 0: 
             owp_id =  id.split('datasets/106/person/',1)[1]
             p += '<b>' +  d.get('name',owp_id)  + ': </b>'
             p += 'Oberwolfach Photos: [<a href = "http://owpdb.mfo.de/person_detail?id=' + owp_id + '" target = "Oberwolfach">' + owp_id+ '</a>]'
             p += '<br>'


        elif d.get('type') == 'Image':   ### assumes its from OWP

            img = '''<table style="border-top:0;border-bottom:0;"><tr><td>
                <a href="$url_detail$"> <img title="$prefLabel$" alt="$prefLabel" src="$url$" target="images", height="150" /><br /><br /></td>
                <td valign="top">$right_column$</td></tr></table>'''
            d['url_detail'] = d['url'].replace('photoNormal?id=','detail?photo_id=')
            for k in 'url url_detail prefLabel'.split(): img = img.replace('$' + k + '$',d.get(k,''))
            d['description'] = d.get('localityName','') + ', ' + d.get('date','') 
            if d.get('creator', ''): d['prefLabel'] += ' by ' + d.get('creator', '')
            d['copyright'] = 'Source:  ' + d.get('publisher','') 
            d['prefLabel'] = d['prefLabel'].replace('of:','of ')
            d['text'] = 'Click on the photo for a larger version with copyright details.'
            right_column = '<table>'

            for k in 'prefLabel description copyright text'.split():
                    v = Value(d[k])
                    right_column += '<tr valign=top><td align="right">&nbsp;&nbsp;&nbsp;</td><td>' + v.str() + '</td></tr>'
            right_column += '</table>'
            p += img.replace('$right_column$',right_column)
        elif id.find('datasets/130/resource/') >= 0: 
            p += '<b>' +  d.get('name','')  + '</b>: <a href="http://imstat.org/awards/honored_fellows.htm">IMS Fellow </a><br>'
            p += '<table>'
            for k in d.keys():
                if not k in ['name', 'mrauthid','prefLabel','type','id']:
                     v = Value(d[k])
                     p += '<tr valign=top><td align="right">' + k + ': </td><td>' + v.str() + '</td></tr>'
            p += '</table>'
        else: 
            p += '<table>'
            for k in d.keys():
                ##if not k in [', 'mrauthid','prefLabel']:
                     v = Value(d[k])
                     p += '<tr valign=top><td align="right">' + k + ': </td><td>' + v.str() + '</td></tr>'
            p += '</table>'
            #p = '<pre>' + self.json_indented(4) + '</pre>'
        return p
Exemplo n.º 15
0
            keyword_density(hash, query, soup, check_query)

            #print('title_h1')
            title_h1(hash, tree)

            #print('viewport')
            viewport(hash, code)

            #print('description')
            description(hash, tree)

            #print('title')
            title(hash, tree)

            #print('links')
            links(hash, result_main, html_source)

            #print('plugins')
            plugins(hash, html_source, html_comments)

            #print('https')
            https(result_url, hash)

            #print('micros')
            micros(hash, html_comments, html_source)

            #print('og')
            og(hash, code)

            #print('sitemap')
            sitemap(hash, code)
Exemplo n.º 16
0
def index(req, nav='public/main', switchdiv='Overview', switchsubdiv='', mark='', **values):
    selnav = "x"+nav
    if selnav == "xcoronasi" or selnav == "xcosmos1686" or selnav == "xryabina" or selnav == "xcoronasf" or selnav == "xmeteor" or selnav == "xglonass" or selnav == "xtatyana":
        selnav = "xdata"
    elif selnav == "xflares" or selnav == "xstorms" or selnav == "xmpause" or selnav == "xcoronas-pics" or selnav == "xomni" or selnav == "xcoho" or selnav == "xservices":
        selnav = "xservice"
    elif selnav == "xmodel-para" or selnav == "xmodel-ap8ae8" or selnav == "xmodel-igrf" or selnav == "xmodel-shieldose2" or selnav == "xmodel-cosrad":
        selnav = "xmodels"

    req.content_type = "text/html"
    language = localize(req)
    level = authorize(req)
    f = open(DOCROOT+level+'index.html','r')
    resp = f.read().replace(
           'class="menu" id="%s"' % selnav,
           'class="currentmenu"').replace(
           "_COPYRIGHT_YEAR_",
           str(datetime.now().year))
    f.close()
    if switchsubdiv == '':
        subdiv = "javascript:switchdiv('%s');"%switchdiv
    else:
        subdiv = "javascript:switchsubdiv('%s', '%s');"%(switchdiv, switchsubdiv)
    if mark != "":
        subdiv += "javascript:markswitch('%s')"%mark
    resp = resp.replace("javascript:init();", "javascript:init();%s"%subdiv)

    if nav == "links":
        resp += links()
    elif nav.find("exp-") == 0:
        try:
            exp = nav.split("_")[0].split("-")[1]
            if exp not in experiments:
                f = open(DOCROOT+'404.html','r')
                resp += ('<div style="overflow:auto;border:0px solid #666;padding: 8px" >')
                resp += f.read()
                resp += '</div>'
                f.close()
            else:
#                resp += '<p>%s/%s/%s.html</p>' % (EXPURL, exp, nav.split("_")[1])
                f = urlopen("%s/%s/%s.html"%(EXPURL, exp, nav.split("_")[1]))
                resp += ('<div style="overflow:auto;border:0px solid #666;padding: 8px" >')
                resp += f.read()
                resp += '</div>'
                f.close()
        except:
            f = open(DOCROOT+'404.html','r')
            resp += ('<div style="overflow:auto;border:0px solid #666;padding: 8px" >')
            resp += f.read()
            resp += '</div>'
            f.close()
    else:
        try:
            f = open(DOCROOT + level + nav + '.html', 'r')
        except:
            f = open(DOCROOT+'404.html','r')
        if selnav == "xmain":
            resp+=('<div style="overflow:auto;border:0px solid #666;padding: 4px;background:url(\'design/'+level +'bg.jpg\') no-repeat center">')
        else:
            resp+=('<div style="overflow:auto;border:0px solid #666;padding: 4px">')
        resp += f.read()
        #req.write(translate(, language))
        resp += '</div>'
        f.close()

    resp = resp.replace("_HDRTITLE_", mkTitle(nav) + TITLE)

    # Parse other arguments and set optional checks
    if nav == "model-igrf":
        try:
            resp = resp.replace("_IGRFWEB_", urlopen("http://cosrad.sinp.msu.ru/cgi-bin/igrf/igrfcmdline.pl?notable=1").read() + "<div id='igrfresult'> </div>")
            resp = resp.replace('onclick="igrf_calc"', 'onclick="igrf_calc()"')
        except:
            resp = resp.replace("_IGRFWEB_", "Unable to contact IGRF model server")
    if nav == "model-ap8ae8":
        try:
            resp = resp.replace("_AP8AE8WEB_", urlopen("http://cosrad.sinp.msu.ru/cgi-bin/ap8ae8/ap8ae8cmdline.pl?notable=1").read() + "<div id='ap8ae8result'> </div>")
            resp = resp.replace('onclick="ap8ae8_calc"', 'onclick="ap8ae8_calc()"')
        except:
            resp = resp.replace("_AP8AE8WEB_", "Unable to contact IGRF model server")
    for x in values:
        if x != "output":
            resp = resp.replace('name="%s%s"'%(mark, x), 'name="%s%s"'%(mark, x) + ' checked="checked"')
            # Costyl!!!
            resp = resp.replace('name="x%s"'%x, 'name="x%s"'%x + ' checked="checked"')
            resp = resp.replace('name="j%s"'%x, 'name="j%s"'%x + ' checked="checked"')
        else:
            resp = resp.replace('name="%s%s"'%(mark, x) + ' value="%s"'%req.form.get(x, ''), 'name="%s%s"'%(mark, x) + ' value="%s"'%req.form.get(x, '') + ' checked="checked"')
            # Costyl!!!
            resp = resp.replace('name="x%s"'%x + ' value="%s"'%req.form.get(x, ''), 'name="x%s"'%x + ' value="%s"'%req.form.get(x, '') + ' checked="checked"')
        #resp += 'name="%s%s"'%(mark, x) + "  " + 'name="%s%s"'%(mark, x) + ' checked="checked"'
    resp += """<h3 style="font-weight:normal;padding:5px;font-size:12px" align="right" class="separator">
               <i>Developed in 2007. 
                   <a href="http://www.gnu.org/copyleft/gpl.html" style="background-color:#d9e5f2">
                       Copyright © </a>
                  2007 - %s <a href="http://www.sinp.msu.ru" style="background-color:#d9e5f2">SINP</a>
                  <a href="http://www.msu.ru" style="background-color:#d9e5f2">MSU</a></i>
               </h3>
               </td></tr></table></body></html>""" % str(datetime.now().year)
    return translate(resp, language)
Exemplo n.º 17
0
    def GET(self, topic_id, onlycode=False, webinput=None):

        presets = webinput or web.input()
        try:
            presets.referer = web.ctx.environ["HTTP_REFERER"]
        except KeyError:
            presets.referer = 0

        context = Storage(
            presets=presets,
            offset = 10,
            framed = 1,
            onlycode=onlycode,
            title=presets.get("title", _("Points")),
            desturl=pc_links("topic", int(topic_id), **presets),
#            submitform=pc_links("topic_newpoint", int(topic_id)),
        )

        topics, topic = model.t_helper(topic_id)

        topics.annotate_by_datasources()
        topics.annotate_by_points()

        number_of_points = int(presets.get("number_of_points",
            config.points_per_page))

        points = model.Points(project=topic, external=None)
        points.limit_by_page(1, length=number_of_points)
        points.annotate_by_comments()
        points.annotate_by_tags()
        points.annotate_by_profiles(default=DEFAULT_POINT_PROFILE)

        feed_link = feed_links("topic_points",
            topic.id, presets, page=None)

        if "zoom" in topic.profile and topic.profile.zoom != "auto":
            c_lat, c_lon, c_zoom = \
                topic.lat, topic.lon, topic.profile.zoom
            c_zoom = int(c_zoom)
        else:
            c_lat, c_lon, c_zoom = points.central_point(
                config.center_lat, config.center_lon, 14)

        map_context = Storage(
            getmap_url=pc_links.getmap,
            getmap_layers=config.getmap_layers,
            getmap_layers1=config.getmap_layers1,
            getmap_layers2=config.getmap_layers2,
            map_params=config.getmap_params,
            getmap_custom=config.getmap_custom,
            getmap_custom_init=config.getmap_custom_init,
            lat=c_lat,
            lon=c_lon,
            zoom=c_zoom + config.getmap_zoomshift,
            initial_feed=feed_link,
            has_new_point="false",
            getmap_zoom1=config.getmap_zoom1 + config.getmap_zoomshift,
            getmap_zoom2=config.getmap_zoom2 + config.getmap_zoomshift,
        )
        model.encode_coordinates(map_context)

        context.map_context = map_context
        context.page_specific_js = macros.map_js(map_context)

        if onlycode:
            presets.referer = ''
            # !!! map widget can't be part of combo
            self_link = links('map', topic_id, **presets)
            return """<iframe src="%s" style="height:%spx;width:%spx;border:none;"></iframe>""" % \
                (self_link, presets.height, presets.width)


        get_widget('map', context, topic, presets)
Exemplo n.º 18
0
from streamango import Streamango

app = Flask(__name__)

URL = "http://server.full-stream.nu"
DATABASE = 'safe_links'
USERNAME = '******'
PASSWORD = '******'
PORT = 3306

user_ins = user(username=USERNAME,
                password=PASSWORD,
                database=DATABASE,
                port=PORT)
links_ins = links(username=USERNAME,
                  password=PASSWORD,
                  database=DATABASE,
                  port=PORT)
secret = b"Cool Links!!"


def haaash(w):
    h = hashlib.md5(w)
    x = h.hexdigest().encode('utf-8')
    h = hashlib.md5(x)
    return h.hexdigest()


def make_secure_val(val):
    return '%s|%s' % (val, hmac.new(
        secret, val.encode('utf-8')).hexdigest().encode('utf-8'))