コード例 #1
0
        def do_GET ( self ):
            self.send_response( 200 )
            self.send_header("Content-type", "text/html")
            self.end_headers()
            data = urlparse.parse_qs( self.path[2:] ).get( 'input' ) or [""]
            shortest = urlparse.parse_qs( self.path[2:] ).get( 'shortest' )
            html = """
<html>
<script>
function update(){
    document.getElementById('output').value = Array.prototype.slice.call(
        document.getElementsByTagName('select')).map(function (s) {
            return s.options[s.selectedIndex].value }).join('');
    document.getElementById('length').textContent = document.getElementById('output').value.length
}
function submit() { document.getElementById('form').submit(); }
</script>
<body onload=update()>
<tt>%s</tt> encoding from <tt>%s</tt> (%s tokens, %s unique)<hr>
<form id=form>Text to encode: <input name=input type=text value="%s"><br>
<small><input type=checkbox name=shortest %s onchange=submit()>Sort by length (instead of frequency)</small><hr>
Select alternates:
            """ % (encodingSpec, corpusFilename, len(corpusTokens), len(set(corpusTokens)), data[0], 'checked' if shortest else '')
            if data != None:
                for word in changeWordSize( 8, bits ) < map( ord, data[0] ):
                    keyFunc = len if shortest else model[word].count
                    html += '<select onchange=update() title="%s words hash to value %s">' % (len(set(model[word])),word)
                    for token in sorted( set(model[ word ]), key=keyFunc, reverse=(not shortest) ):
                        html += "<option value='%s'>%s</option>" % ( token, token )
                    html += "</select>"
            self.wfile.write( html + "<hr>Result (<span id=length></span> bytes):<br>" +
                                     "<textarea id=output cols=80 rows=5></textarea><br>(pipe through <tt>python -mbananaphone pipeline 'rh_decoder(\"%s\")'</tt> to decode it)" % (encodingSpec,))
コード例 #2
0
ファイル: tests.py プロジェクト: fossabot/tumbo-server
    def test_execute_async(self, call_rpc_client_mock):
        with patch.object(ResponseUnavailableViewMixing, 'verify', return_value=None) as mock_method:
            call_rpc_client_mock.return_value = True
            self.client1.login(username='******', password='******')
            from urllib2 import urlparse

            # get redirect
            response = self.client1.get(self.base1_apy1.get_exec_url()+"&async")
            self.assertEqual(301, response.status_code)
            queries = urlparse.urlparse(response['Location'])[4]
            rid = int(urlparse.parse_qs(queries)['rid'][0])
            transaction = Transaction.objects.get(pk=rid)

            # get state (RUNNING)
            #response = self.client1.get(self.base1_apy1.get_exec_url()+"&rid=%s" % rid, HTTP_ACCEPT='application/xml')
            response = self.client1.get(self.base1_apy1.get_exec_url()+"&rid=%s" % rid)
            self.assertEqual(200, response.status_code)
            tout = {u'status': u'RUNNING', "url": "/fastapp/base/base1/exec/base1_apy1/?json=&rid="+str(rid), 'rid': rid, 'id': u'base1_apy1'}
            self.assertEqual(json.loads(response.content)['rid'], tout['rid'])

            # mock creation of response
            tout = {u'status': u'OK', u'exception': None, u'returned': u'{"content": "{\\"aaa\\": \\"aaa\\"}", "class": "XMLResponse", "content_type": "application/json"}', u'response_class': u'JSONResponse', 'time_ms': '74', 'rid': rid, 'id': u'base1_apy1'}
            transaction.tout = tout
            transaction.save()
            self.assertEqual(transaction.apy, self.base1_apy1)

            # get response
            response = self.client1.get(self.base1_apy1.get_exec_url()+"&rid=%s" % rid)
            self.assertEqual(200, response.status_code)

            # check transaction duration
            transaction = Transaction.objects.get(pk=rid)
            self.assertEqual(int, type(transaction.duration))
コード例 #3
0
def get_details_from_url(url_portion):
    res = urlparse.parse_qs(url_portion.partition('?')[2])
    if not res: # might happen in the case of someone not submitting a URL with "details.php?" at the start
        res = url_portion 
    for key in res.keys():
        res[key] = res[key][0]
    return res
コード例 #4
0
def get_details_from_url(url_portion):
    res = urlparse.parse_qs(url_portion.partition('?')[2])
    if not res:  # might happen in the case of someone not submitting a URL with "details.php?" at the start
        res = url_portion
    for key in res.keys():
        res[key] = res[key][0]
    return res
コード例 #5
0
ファイル: simplesender.py プロジェクト: rodders/darklaunch
def victimise(victim, uri):
    raw_url = victim + uri
    scheme, netloc, path, raw_query, fragment = urlparse.urlsplit(raw_url)
    query = urlparse.parse_qs(raw_query)
    url = urlparse.urlunsplit((scheme, netloc, path, urlencode(query, True), fragment))
    print url
    http_client.fetch(url, fetch, use_gzip=False)
コード例 #6
0
def process_waze_message(token):
    connection=httplib.HTTPConnection("waze.to")
    connection.request("GET","/"+token)
    response=connection.getresponse()
    meetingUrl=response.getheader("location")
    logging.debug(meetingUrl)

    querystring=urlparse.urlparse(meetingUrl).query
    token= urlparse.parse_qs(querystring)["token"][0]
    logging.debug(token)

    meetingInfoUrl="http://mobile-web.waze.com/SocialMediaServer/internal/getMeetingInfo?event_id="+ token
    logging.debug(meetingInfoUrl)
    meetingInfo=json.loads(urllib2.urlopen(meetingInfoUrl).read())
    logging.debug(meetingInfo)

    driveUrl="http://mobile-web.waze.com/rtserver/web/PickUpGetDriverInfo?clientID=70a8b694c7&routeTimestamp=0&getUserInfo=true&token=" + token
    logging.debug(driveUrl)
    driverInfo=json.loads(urllib2.urlopen(driveUrl).read())
    logging.debug(driverInfo)
    if driverInfo['status']=='ok':
        eta = driverInfo['eta']
        insert_waze_request(token,eta)
    else:
        deactivate_waze_request(token)
コード例 #7
0
ファイル: album.py プロジェクト: dynamicguy/photomatic
    def fetch_photos_from_msg(self, album, msg=None):
        u = album.user
        token = get_access_token(u)
        graph = facebook.GraphAPI(token)

        if msg.status == 'awaiting':
            parts = urlparse.urlparse(msg.next_page)
            qs = urlparse.parse_qs(parts.query)
            after = qs.get('after')[0]
            photos = graph.get_object(album.fb_album_id + "/photos", fields='id,source', limit=2, after=after)
            new_next_page = photos.get('paging').get('next')
            new_msg = Message.objects.create(next_page=new_next_page, user=u, status='awaiting')
            for photo in photos.get('data'):
                img_temp = NamedTemporaryFile(delete=True)
                img_temp.write(urlopen(photo.get('source')).read())
                img_temp.flush()
                photo_object = Photo.objects.create(title=photo.get('id'),
                    description=photo.get('created_time'),
                    album=album,
                    file=File(img_temp))
                pprint(photo_object.filename)
                self.stdout.write('Successfully fetched photo for source "%s"\n' % photo.get('source'))
            msg.status = 'done'
            msg.save()
            self.stdout.write('Finished this queue "%s"\n' % new_msg.next_page)
コード例 #8
0
ファイル: tests.py プロジェクト: sahlinet/fastapp
    def test_execute_async(self, call_rpc_client_mock, send_client_mock):
        with patch.object(ResponseUnavailableViewMixing, 'verify', return_value=None) as mock_method:
            call_rpc_client_mock.return_value = True
            send_client_mock.return_value = True
            self.client1.login(username='******', password='******')
            from urllib2 import urlparse

            # get redirect
            response = self.client1.get(self.base1_apy1.get_exec_url()+"&async")
            self.assertEqual(301, response.status_code)
            queries = urlparse.urlparse(response['Location'])[4]
            rid = int(urlparse.parse_qs(queries)['rid'][0])
            transaction = Transaction.objects.get(pk=rid)

            # get state (RUNNING)
            response = self.client1.get(self.base1_apy1.get_exec_url()+"&rid=%s" % rid, HTTP_ACCEPT='application/xml')
            self.assertEqual(200, response.status_code)
            tout = {u'status': u'RUNNING', "url": "/fastapp/base/base1/exec/base1_apy1/?json=&rid="+str(rid), 'rid': rid, 'id': u'base1_apy1'}
            self.assertEqual(json.loads(response.content)['rid'], tout['rid'])

            # mock creation of response
            tout = {u'status': u'OK', u'exception': None, u'returned': u'{"content": "{\\"aaa\\": \\"aaa\\"}", "class": "XMLResponse", "content_type": "application/json"}', u'response_class': u'JSONResponse', 'time_ms': '74', 'rid': rid, 'id': u'base1_apy1'}
            transaction.tout = tout
            transaction.save()
            self.assertEqual(transaction.apy, self.base1_apy1)

            # get response
            response = self.client1.get(self.base1_apy1.get_exec_url()+"&rid=%s" % rid)
            self.assertEqual(200, response.status_code)

            # check transaction duration
            transaction = Transaction.objects.get(pk=rid)
            self.assertEqual(int, type(transaction.duration))
コード例 #9
0
ファイル: fullcontact.py プロジェクト: closeio/fullcontact
def batch_lookup(data_list, webhook=None, debug=False):
    """
    Send requests to the Full Contact API and return the status logs of those requests.
    If debug is True, print the logs.

    Arguments:
    data_list -- the list of tuples of contact data in the form (type, data)
    webhook -- url with the callback function that handles responses from Full Contact
    debug -- boolean specifying whether some debug information should be printed to the console.
    """
    # divide the data into chunks of 20 (max number for a single batch request)
    data_chunks = []
    counter = 0
    while counter < len(data_list):
        data_chunks.append(data_list[counter:counter+20])
        counter += 20
    # send the request for each chunk
    process_log = []
    for chunk in data_chunks:
        request_urls = []
        for data in chunk:
            # escape params
            if data:
                data = (quote(data[0].encode('utf-8')), quote(data[1].encode('utf-8')))
                url = 'https://api.fullcontact.com/v2/person.json?%s=%s' % (data)
                if webhook:
                    url += '&webhookUrl=' + webhook + '&webhookId=%s:%s' % (data)
            request_urls.append(url)
        post_data = simplejson.dumps({'requests' : request_urls})
        data = requests.post(
            'https://api.fullcontact.com/v2/batch.json',
            params={'apiKey': FULL_CONTACT_API_KEY},
            headers={'content-type': 'application/json'},
            data=post_data
        ).json
        for person_url, person_json in data['responses'].items():
            log = {}
            params = urlparse.parse_qs(urlparse.urlparse(person_url).query)
            if params.get('email'):
                log['type'] = 'email'
                log['data'] = params['email'][0]
            elif params.get('phone'):
                log['type'] = 'phone'
                log['data'] = params['phone'][0]
            elif params.get('twitter'):
                log['type'] = 'twitter'
                log['data'] = params['twitter'][0]
            elif params.get('facebookUsername'):
                log['type'] = 'facebookUsername'
                log['data'] = params['facebookUsername'][0]
            else:
                log['type'] = 'Wrong data'
                log['data'] = person_url
            log['status'] =  '%s - %s' % (person_json.get('status'),
                                          person_json.get('message'))
            process_log.append(log)
            if debug:
                print log
    return process_log
コード例 #10
0
def get_qs(url):
    """
    >>> get_qs("http://abc.com/a?a=123&b=%2b9")
    {'a': ['123'], 'b': ['+9']}
    """
    try:
        return urlparse.parse_qs(urlparse.urlparse(url).query)
    except:
        return {}
コード例 #11
0
def fetch_and_parse(url):
    """parses out the review and the spotify id, if present"""
    r = requests.get(url)
    soup = BeautifulSoup(r.content)
    review = soup.select('div.editorial')[0].text
    spotify_iframe = soup.select('div.spotify')[0].iframe
    spotify_embed_url = urlparse.urlparse(spotify_iframe['src'])
    spotify_item_uri = urlparse.parse_qs(spotify_embed_url.query)['uri'][0]
    return review, spotify_item_uri
コード例 #12
0
ファイル: urlutils.py プロジェクト: qiaohui/pygaga
def get_qs(url):
    """
    >>> get_qs("http://abc.com/a?a=123&b=%2b9")
    {'a': ['123'], 'b': ['+9']}
    """
    try:
        return urlparse.parse_qs(urlparse.urlparse(url).query)
    except:
        return {}
コード例 #13
0
ファイル: urlutils.py プロジェクト: qiaohui/pygaga
def get_query_arg(qs, arg):
    """
    >>> get_query_arg("a=123&b=%2b9", "b")
    '+9'
    """
    try:
        return safe_get(urlparse.parse_qs(qs), arg)
    except:
        return ""
コード例 #14
0
def get_query_arg(qs, arg):
    """
    >>> get_query_arg("a=123&b=%2b9", "b")
    '+9'
    """
    try:
        return safe_get(urlparse.parse_qs(qs), arg)
    except:
        return ""
コード例 #15
0
def fetch_and_parse(url):
    """parses out the review and the spotify id, if present"""
    r = requests.get(url)
    soup = BeautifulSoup(r.content)
    review = soup.select('div.editorial')[0].text
    spotify_iframe = soup.select('div.spotify')[0].iframe
    spotify_embed_url = urlparse.urlparse(spotify_iframe['src'])
    spotify_item_uri = urlparse.parse_qs(spotify_embed_url.query)['uri'][0]
    return review, spotify_item_uri
コード例 #16
0
ファイル: linktags.py プロジェクト: althalus/knotcrafters
def format_link(link):
    url_parts = urlparse.urlparse(link.link)
    if re.match(r'(www\.)?facebook\.com', url_parts[1]):
        return '<div class="fb-post" data-href="%s"></div>' % link.link
    elif re.match(r'(www\.)?youtu(\.be|be\.com)', url_parts[1]):
        qs = urlparse.parse_qs(url_parts[4])
        v = qs.get('v', ['', ])[0]
        return '<iframe title="YouTube video player" width="480" height="390" src="http://www.youtube.com/embed/%s" frameborder="0" allowfullscreen></iframe>' % v
    else:
        return '<a href="%s">%s</a>' % (link.link, link.name)
コード例 #17
0
ファイル: views.py プロジェクト: cgsv/djangoSite
def getparams(request):
    from urllib2 import urlparse
    print type(request)
    print dir(request)
    print request.get_full_path()
    res = urlparse.urlparse(request.GET.get('params','Not found'))
    query = urlparse.parse_qs(res.query)
    #data = plotp.picToBase64(query['p'][0])
    #return HttpResponse('<img src="' + "data:image/jpg;base64," + data + '"/>')
    img = open(plotp.path_prefix + query['p'][0],'rb').read()
    return HttpResponse(img, mimetype='image/jpg')
コード例 #18
0
ファイル: GlobalLib.py プロジェクト: ZhihongJiang/Catalyst
def parseCookie(cookie):
    parsed = urlparse.parse_qs(cookie)
    if(parsed.has_key('user_id') and parsed.has_key('token')):
        uid = parsed['user_id'][0]
        token = parsed['token'][0]
        userQuery = Users.get_by_id(int(uid))
        if(userQuery != None):
            pword = userQuery.pword
            if(token==encodeCookie(uid, pword)):
                return True
    return False
コード例 #19
0
ファイル: bananaphone.py プロジェクト: abeluck/bananaphone
 def do_GET ( self ):
     self.send_response( 200 )
     self.send_header("Content-type", "text/html")
     self.end_headers()
     html = "<html><form><input name=input type=text><br>"
     data = urlparse.parse_qs( self.path[2:] ).get( 'input' )
     if data != None:
         for word in changeWordSize( map( ord, data[0] ), 8, bits ):
             html += "<select>"
             for token in sorted( set(model[ word ]), key=model[word].count, reverse=True ):
                 html += "<option>%s</option>" % ( token, )
             html += "</select>"
     self.wfile.write( html + "\n" )
コード例 #20
0
def victimise(victim, request):
    try:
        lines = request.split('\n')
        uri = lines[0].split(' ')[1]
        body = lines[-1]

        raw_url = victim + uri
        scheme, netloc, path, raw_query, fragment = urlparse.urlsplit(raw_url)
        query = urlparse.parse_qs(raw_query)
        url = urlparse.urlunsplit((scheme, netloc, path, urlencode(query, True), fragment))
        if body:
            http_client.fetch(url, fetch, method="POST", body=body, use_gzip=False)
    except:
        pass
コード例 #21
0
def serve(env, start_response):
    path = env['PATH_INFO']
    args = urlparse.parse_qs(env['QUERY_STRING'])
    key = args['key'][0]

    start_response('200 OK', [('Content-type', 'text/plain')])

    if path == '/get':
        return data[key]
    if path == '/set':
        value = args['value'][0]
        data[key] = value
        return ['ok']
    return ['unknown error']
コード例 #22
0
 def do_GET(self):
     self.send_response(200)
     self.send_header("Content-type", "text/html")
     self.end_headers()
     html = "<html><form><input name=input type=text><br>"
     data = urlparse.parse_qs(self.path[2:]).get('input')
     if data != None:
         for word in changeWordSize(map(ord, data[0]), 8, bits):
             html += "<select>"
             for token in sorted(set(model[word]),
                                 key=model[word].count,
                                 reverse=True):
                 html += "<option>%s</option>" % (token, )
             html += "</select>"
     self.wfile.write(html + "\n")
コード例 #23
0
def parse_homepage(html):
    soup = BeautifulSoup(html)
    websites = soup.find_all('dd', 'websites')
    if websites and len(websites) > 0:
        websites = websites[0]
        sites = websites.find_all('li')
        if sites and len(sites) > 0:
            result = {}
            for site in sites:
                site_name = site.text.strip()
                original = site.a.get('href')
                url_parse = urlparse.urlparse(original).query
                query_parse = urlparse.parse_qs(url_parse)
                if 'url' in query_parse:
                    result[site_name] = query_parse['url']
            return result
    return None
コード例 #24
0
ファイル: LinkedinParser.py プロジェクト: 668/scrapy-linkedin
def parse_homepage(html):
    soup = BeautifulSoup(html)
    websites = soup.find_all('dd', 'websites')
    if websites and len(websites) > 0:
        websites = websites[0]
        sites = websites.find_all('li')
        if sites and len(sites) > 0:
            result = {}
            for site in sites:
                site_name = site.text.strip()
                original = site.a.get('href')
                url_parse = urlparse.urlparse(original).query
                query_parse = urlparse.parse_qs(url_parse)
                if 'url' in query_parse:
                    result[site_name] = query_parse['url']
            return result
    return None
コード例 #25
0
def get_details_from_url(url_portion):
    """
    >>> res = get_details_from_url("details.php?airport=SXF&amp;id=102191022&amp;flightno=4U+8129&amp;direction=WB")
    >>> res['direction']
    'WB'
    >>> res['airport']
    'SXF'
    >>> res['id']
    '102191022
    >>> res['flightno']
    '4U 8129'
    """
    res = urlparse.parse_qs(url_portion.partition('?')[2])
    if not res:  # might happen in the case of someone not submitting a URL with "details.php?" at the start
        res = url_portion
    for key in res.keys():
        res[key] = res[key][0]
    return res
コード例 #26
0
def get_details_from_url(url_portion):
    """
    >>> res = get_details_from_url("details.php?airport=SXF&amp;id=102191022&amp;flightno=4U+8129&amp;direction=WB")
    >>> res['direction']
    'WB'
    >>> res['airport']
    'SXF'
    >>> res['id']
    '102191022
    >>> res['flightno']
    '4U 8129'
    """
    res = urlparse.parse_qs(url_portion.partition("?")[2])
    if not res:  # might happen in the case of someone not submitting a URL with "details.php?" at the start
        res = url_portion
    for key in res.keys():
        res[key] = res[key][0]
    return res
コード例 #27
0
ファイル: youtube.py プロジェクト: rogerhil/heybaldock
def youtube_id_by_url(url):
    validator = URLValidator()
    try:
        validator(url)
    except ValidationError:
        msg = _("Please provide a valid url")
        raise ValidationError(msg)
    parsed = urlparse.urlparse(url)
    if 'youtube' not in parsed.netloc and \
       'y2u.be' not in parsed.netloc:
        msg = _("Only youtube videos are allowed.")
        raise ValidationError(msg)

    qs = urlparse.parse_qs(parsed.query)
    if 'v' not in qs and not qs['v']:
        msg = _("Invalid youtube video url.")
        raise ValidationError(msg)
    id = qs['v'][0]
    return id
コード例 #28
0
 def _parse(self, html, url, encoding, status_code):
     parse_result = urlparse.urlparse(url)
     params = urlparse.parse_qs(parse_result.query)
     item_type = params['types'][0]
     proName = params.get('proName', '')
     province = proName[0] if proName else 'all'
     city = params['industry'][0]
     response_data = json.loads(html)
     error = int(response_data['error'])
     if error != 0:
         yield None
     else:
         show_html = response_data['data']
         tree = BeautifulSoup(show_html, 'lxml')
         tr_list = tree.find_all('tr')
         for tr in tr_list:
             td_list = tr.find_all('td')
             overall_data = td_list[-1].a.get('onclick')
             read_num = td_list[2].get_text()
             read_num = self._get_abstract_num(read_num)
             like_num = td_list[3].get_text()
             like_num = self._get_abstract_num(like_num)
             try:
                 tag_wxarc = QingboWX.TAG_WXARC.search(
                     overall_data).groups()[0].split(',')[1:]
                 item_url = tag_wxarc[0][1:-1]
                 title = util.get_utf8_str(tag_wxarc[1][1:-1], encoding)
                 source = tag_wxarc[2][1:-1]
                 source_detail = util.get_utf8_str(tag_wxarc[3][1:-1],
                                                   encoding)
                 original_time = tag_wxarc[4][1:-1]
                 #tag = QingboWX.TYPE_TAG_TABLE[item_type]
                 tag = item_type
                 key = "{}#{}".format(title, source)
                 md5_key = md5(key).hexdigest()
                 item = self._get_item(title, source, source_detail,
                                       item_url, original_time, province,
                                       city, like_num, read_num, tag,
                                       md5_key)
                 yield item
             except Exception:
                 yield None
コード例 #29
0
    def __call__(self, url, timeout=15):
        """ Get data and convert it to TSV if possible
        """
        data = u''
        if not self.test(url):
            return data

        ourl = urlparse.urlparse(url)
        query = ourl.query
        query = urlparse.parse_qs(query)
        site = getSite()
        request = getattr(site, 'REQUEST', None)
        if getattr(request, 'form', None) is not None:
            request.form.update(query)

        try:
            view = site.unrestrictedTraverse(ourl.path)
        except Exception as err:
            logger.warn("Invalid data url '%s'", url)
            return data

        if IATBlob.providedBy(view):
            query = {}
            view = view.getFile

        if getattr(view, 'meta_type', None) == 'File':
            query = {}
            oldView = view
            view = lambda: oldView.data

        try:
            try:
                data = view(**query)
            except TypeError: # got an unexpected keyword argument
                query = {}
                data = view()
        except Unauthorized:
            data = self._data(view, query)
        except Exception, err:
            logger.exception(err)
コード例 #30
0
    def img(self):
        """ Image
        """
        if self._img:
            return self._img

        objectIds = getattr(self.context, 'objectIds', lambda: [])
        if 'cover.png' in objectIds():
            self._img = self.context.restrictedTraverse('cover.png')
            return self._img

        view = queryMultiAdapter((self.context, self.request),
                                 name='daviz-view.html')

        for tab in view.tabs:
            fallback = tab.get('fallback-image', None)
            if not fallback:
                continue

            url = urlparse.urlparse(fallback)
            query = urlparse.parse_qs(url.query)

            img = url.path.split('/')[-1]
            if isinstance(img, unicode):
                img = img.encode('utf-8')

            if img.startswith('embed-chart.svg'):
                img = query['chart'][0] + '.svg'

            img = self.context.restrictedTraverse(img, None)
            if not img:
                continue

            self._img = img
            break

        return self._img
コード例 #31
0
    def img(self):
        """ Image
        """
        if self._img:
            return self._img

        objectIds = getattr(self.context, 'objectIds', lambda: [])
        if 'cover.png' in objectIds():
            self._img = self.context.restrictedTraverse('cover.png')
            return self._img

        view = queryMultiAdapter((self.context, self.request),
                                 name='daviz-view.html')

        for tab in view.tabs:
            fallback = tab.get('fallback-image', None)
            if not fallback:
                continue

            url = urlparse.urlparse(fallback)
            query = urlparse.parse_qs(url.query)

            img = url.path.split('/')[-1]
            if isinstance(img, unicode):
                img = img.encode('utf-8')

            if img.startswith('embed-chart.svg'):
                img = query['chart'][0] + '.svg'

            img = self.context.restrictedTraverse(img, None)
            if not img:
                continue

            self._img = img
            break

        return self._img
コード例 #32
0
def sso_authorize(request):
    """
    Authorizes specific web sites to utilize an existing My.jobs account

    Required on HTTP GET:
    :auth_callback: GET parameter - Desired return url when authorization
        succeeds

    Required on HTTP POST:
    :auth_callback: POST parameter, copy of :auth_callback: GET parameter
    """
    # Common between GET and POST, callback is required.
    auth_callback = request.GET.get('auth_callback') or \
        request.POST.get('auth_callback')
    data = {'auth_callback': auth_callback}

    if auth_callback:
        auth_callback = unquote(auth_callback)
        auth_callback = urlparse.urlparse(auth_callback)
        if not auth_callback.netloc:
            # If the base url of the callback is not truthy, the url
            # must be malformed somehow
            raise Http404
    else:
        raise Http404

    if request.method == 'GET':
        # Initial view after being redirected from an external site
        data['auth_callback_short'] = auth_callback.netloc

        if not request.user.is_anonymous():
            # Process logged in users first; Certain criteria may cause the
            # user to be logged out.
            good_key = request.session.get('key')
            test_key = request.GET.get('key')
            if good_key:
                # The current user already has a key available.
                if test_key:
                    # The remote site has provided a key; the user has
                    # potentially already authorized this site.
                    if test_key == good_key:
                        if request.user.authorizedclient_set.filter(
                                site=auth_callback.netloc):
                            # The user has authorized this site; Reset the
                            # current session expiry, add the key to the
                            # callback url, and redirect to it.
                            request.session.set_expiry(None)

                            q = urlparse.parse_qs(auth_callback.query)
                            q.update({'key': good_key})
                            auth_callback = auth_callback._replace(
                                query=urlencode(q))
                            return redirect(urlparse.urlunparse(auth_callback))
                        else:
                            # The user at one time authorized this site but it
                            # was revoked (potential future functionality?).
                            # Ask for authorization again.
                            return render_to_response('mysignon/sso_auth.html',
                                                      data,
                                                      RequestContext(request))
                    else:
                        # The key provided does not match the user's key; Log
                        # the user out. It may be a different user's key.
                        logout(request)
                else:
                    # No key was provided; Proceed to authorization normally.
                    return render_to_response('mysignon/sso_auth.html', data,
                                              RequestContext(request))
            else:
                # The user has no key; Create one.
                request.session['key'] = AuthorizedClient.create_key(
                    request.user)
                if test_key:
                    # A key was provided, but the current user did not have one
                    # until now. Log out the user.
                    logout(request)
                else:
                    # No key was provided; Proceed to authorization.
                    return render_to_response('mysignon/sso_auth.html', data,
                                              RequestContext(request))

        # Only anonymous users can reach this point. This is not inside an else
        # block so that it can catch users who were logged out above.
        login_form = CustomAuthForm(auto_id=True)
        login_form.fields.pop('remember_me')
        data['login_form'] = login_form
        return render_to_response('mysignon/sso_auth.html', data,
                                  RequestContext(request))

    else:
        # Form was posted.
        action = request.POST.get('action')
        if action == 'login':
            login_form = CustomAuthForm(data=request.POST, auto_id=False)
            login_form.fields.pop('remember_me')
            if login_form.is_valid():
                user = authenticate(
                    username=login_form.cleaned_data['username'],
                    password=login_form.cleaned_data['password'])
                login(request, user)
                request.session.set_expiry(None)
                # User was logged in. Fall through to code common to
                # preauthenticated users
            else:
                if request.is_ajax():
                    return HttpResponse(
                        json.dumps({'errors': login_form.errors.items()}))
                else:
                    data['login_form'] = login_form
                    data['auth_callback_short'] = auth_callback.netloc

                    return render_to_response('mysignon/sso_auth.html', data,
                                              RequestContext(request))

        # Ensure that an AuthorizedClient instance exists for the current user
        # and the site that is requesting authorization.
        request.user.authorizedclient_set.get_or_create(
            site=auth_callback.netloc)

        # Ensure that the current user has a key.
        if not request.session.get('key'):
            request.session['key'] = AuthorizedClient.create_key(request.user)

        # Add the user's key to the callback url and redirect to it.
        q = urlparse.parse_qs(auth_callback.query)
        q.update({'key': request.session.get('key')})
        auth_callback = auth_callback._replace(query=urlencode(q))
        auth_callback = urlparse.urlunparse(auth_callback)
        if request.is_ajax():

            return HttpResponse(json.dumps({'url': auth_callback}))
        else:
            return redirect(auth_callback)
コード例 #33
0
    def _do_COMMON(self, data={}):
        handlers = [
            {
                "url": r"/terminate",
                "action": self._do_terminate,
                "require_auth": False,
            },
            {
                "url": r"/rest/usermanagement/1/authentication$",
                "action": self._auth_user,
                "require_auth": True,
                "method": "POST",
            },
            {
                "url": r"/rest/usermanagement/1/session$",
                "action": self._get_session,
                "require_auth": True,
                "method": "POST",
            },
            {
                "url": r"/rest/usermanagement/1/session/[A-Za-z0-9]{24}$",
                "action": self._validate_session,
                "require_auth": True,
                "method": "POST",
            },
            {
                "url": r"/rest/usermanagement/1/session/[A-Za-z0-9]{24}$",
                "action": self._delete_session,
                "require_auth": True,
                "method": "DELETE",
            },
            {
                "url": r"/rest/usermanagement/1/user/group/direct$",
                "action": self._get_groups,
                "require_auth": True,
                "method": "GET",
            },
            {
                "url": r"/rest/usermanagement/1/user/group/nested$",
                "action": self._get_groups,
                "require_auth": True,
                "method": "GET",
            },
            {
                "url": r"/rest/usermanagement/1/group/user/nested$",
                "action": self._get_group_users,
                "require_auth": True,
                "method": "GET",
            },
            {
                "url": r"/rest/usermanagement/1/user$",
                "action": self._get_user,
                "require_auth": True,
                "method": "GET",
            },
            {
                "url": r"/rest/usermanagement/1/user$",
                "action": self._add_user,
                "require_auth": True,
                "method": "POST",
            },


            # Default handler for unmatched requests
            {
                "url": r".*",
                "action": self._default_handler,
                "require_auth": True,
            },
        ]

        p = urlparse.urlparse(self.path)

        self.json_data = data
        self.get_params = urlparse.parse_qs(p.query)

        for handler in handlers:
            method = handler.get('method')
            if (re.match(handler['url'], p.path)
                and (not method or method == self.command)):

                # Authenticate application if required
                require_auth = handler.get('require_auth')
                if require_auth and not check_app_auth(self.headers):
                    self._do_app_failed_auth()
                    return

                # Run the handler's action
                handler['action']()
                return

        # An unhandled path was encountered.
        self.send_response(500)
        self.send_header("Content-type", "text/plain")
        self.end_headers()
        self.wfile.write('Oops, should not be here for {}'.format(self.path).encode('ascii'))
コード例 #34
0
ファイル: views.py プロジェクト: kepinq/MyJobs
def sso_authorize(request):
    """
    Authorizes specific web sites to utilize an existing My.jobs account

    Required on HTTP GET:
    :auth_callback: GET parameter - Desired return url when authorization
        succeeds

    Required on HTTP POST:
    :auth_callback: POST parameter, copy of :auth_callback: GET parameter
    """
    # Common between GET and POST, callback is required.
    auth_callback = request.GET.get('auth_callback') or \
        request.POST.get('auth_callback')
    data = {'auth_callback': auth_callback}

    if auth_callback:
        auth_callback = unquote(auth_callback)
        auth_callback = urlparse.urlparse(auth_callback)
        if not auth_callback.netloc:
            # If the base url of the callback is not truthy, the url
            # must be malformed somehow
            raise Http404("mysignon.views.sso_authorize: bad callback")
    else:
        raise Http404("mysignon.views.sso_authorize: no callback")

    if request.method == 'GET':
        # Initial view after being redirected from an external site
        data['auth_callback_short'] = auth_callback.netloc

        if not request.user.is_anonymous():
            # Process logged in users first; Certain criteria may cause the
            # user to be logged out.
            good_key = request.session.get('key')
            test_key = request.GET.get('key')
            if good_key:
                # The current user already has a key available.
                if test_key:
                    # The remote site has provided a key; the user has
                    # potentially already authorized this site.
                    if test_key == good_key:
                        if request.user.authorizedclient_set.filter(
                                site=auth_callback.netloc):
                            # The user has authorized this site; Reset the
                            # current session expiry, add the key to the
                            # callback url, and redirect to it.
                            request.session.set_expiry(None)

                            q = urlparse.parse_qs(auth_callback.query)
                            q.update({'key': good_key})
                            auth_callback = auth_callback._replace(
                                query=urlencode(q))
                            return redirect(urlparse.urlunparse(auth_callback))
                        else:
                            # The user at one time authorized this site but it
                            # was revoked (potential future functionality?).
                            # Ask for authorization again.
                            return render_to_response('mysignon/sso_auth.html',
                                                      data,
                                                      RequestContext(request))
                    else:
                        # The key provided does not match the user's key; Log
                        # the user out. It may be a different user's key.
                        logout(request)
                else:
                    # No key was provided; Proceed to authorization normally.
                    return render_to_response('mysignon/sso_auth.html',
                                              data,
                                              RequestContext(request))
            else:
                # The user has no key; Create one.
                request.session['key'] = AuthorizedClient.create_key(
                    request.user)
                if test_key:
                    # A key was provided, but the current user did not have one
                    # until now. Log out the user.
                    logout(request)
                else:
                    # No key was provided; Proceed to authorization.
                    return render_to_response('mysignon/sso_auth.html',
                                              data,
                                              RequestContext(request))

        # Only anonymous users can reach this point. This is not inside an else
        # block so that it can catch users who were logged out above.
        login_form = CustomAuthForm(auto_id=True)
        login_form.fields.pop('remember_me')
        data['login_form'] = login_form
        return render_to_response('mysignon/sso_auth.html',
                                  data,
                                  RequestContext(request))

    else:
        # Form was posted.
        action = request.POST.get('action')
        if action == 'login':
            login_form = CustomAuthForm(data=request.POST, auto_id=False)
            login_form.fields.pop('remember_me')
            if login_form.is_valid():
                user = authenticate(
                    username=login_form.cleaned_data['username'],
                    password=login_form.cleaned_data['password'])
                login(request, user)
                request.session.set_expiry(None)
                # User was logged in. Fall through to code common to
                # preauthenticated users
            else:
                if request.is_ajax():
                    return HttpResponse(json.dumps(
                        {'errors': login_form.errors.items()}))
                else:
                    data['login_form'] = login_form
                    data['auth_callback_short'] = auth_callback.netloc

                    return render_to_response('mysignon/sso_auth.html', data,
                                              RequestContext(request))

        # Ensure that an AuthorizedClient instance exists for the current user
        # and the site that is requesting authorization.
        request.user.authorizedclient_set.get_or_create(site=auth_callback.netloc)

        # Ensure that the current user has a key.
        if not request.session.get('key'):
            request.session['key'] = AuthorizedClient.create_key(request.user)

        # Add the user's key to the callback url and redirect to it.
        q = urlparse.parse_qs(auth_callback.query)
        q.update({'key': request.session.get('key')})
        auth_callback = auth_callback._replace(query=urlencode(q))
        auth_callback = urlparse.urlunparse(auth_callback)
        if request.is_ajax():

            return HttpResponse(json.dumps({'url': auth_callback}))
        else:
            return redirect(auth_callback)
コード例 #35
0
 def prepare(self):
     self.url = parse_path(self.path)
     self.url_params = urlparse.parse_qs(self.url.query)
     pass
コード例 #36
0
def request(method, url, content_type, data=None, params=None, headers=None, cookies=None,
            auth=None, redirection=True, timeout=60):
    """Creates a new HTTP request and processes it.

    :param method: the type of request to be created (``GET`` or ``POST``)
    :type method: ``str``.
    :param url: the url of the request.
    :type url: ``str``.
    :param content_type: the content type to be returned (``raw`` or ``json``)
    :type content_type: ``str``.
    :param data: the data to be posted.
    :type data: ``any``.
    :param params: mapping of url parameters.
    :type params: :class:`dict`.
    :param headers: the headers of the request.
    :type headers: :class:`dict`.
    :param cookies: the cookies of the request.
    :type cookies: :class:`dict`.
    :param auth: the authentication information to be used.
    :type auth: :class:`dict`.
    :param redirection: a flag indicating whether redirection is allowed or not.
    :type redirection: ``boolean``.
    :param timeout: a timeout for the request.
    :type timeout: ``int``.
    :return: the content obtained from executing the request.
    :rtype: ``str`` or ``json``.
    """

    openers = []
    if not redirection:
        openers.append(NoRedirectHttpHandler())

    if auth:
        manager = HTTPPasswordMgrWithDefaultRealm()
        manager.add_password(None, url, auth['username'], auth['password'])
        openers.append(HTTPBasicAuthHandler(manager))

    opener = build_opener(*openers)
    install_opener(opener)

    headers = headers or {}
    if cookies:
        for cookie in cookies.keys():
            headers['Cookie'] = "{0}={1}".format(cookie, cookies[cookie])

    if 'user-agent' not in headers:
        headers['user-agent'] = 'Alfred-Workflow/1.17'

    encodings = [s.strip() for s in headers.get('accept-encoding', '').split(',')]
    if 'gzip' not in encodings:
        encodings.append('gzip')

    headers['accept-encoding'] = ', '.join(encodings)

    if method == 'POST' and not data:
        data = ''

    if data and isinstance(data, dict):
        data = urlencode(format_headers(data))

    headers = format_headers(headers)

    if isinstance(url, unicode):
        url = url.encode('utf-8')

    if params:
        scheme, netloc, path, query, fragment = urlparse.urlsplit(url)

        if query:
            url_params = urlparse.parse_qs(query)
            url_params.update(params)
            params = url_params

        query = urlencode(format_headers(params), doseq=True)
        url = urlparse.urlunsplit((scheme, netloc, path, query, fragment))

    try:
        response = urlopen(Request(url, data, headers), timeout=timeout)
        response_headers = response.info()

        content = response.read()
        if 'gzip' in response_headers.get('content-encoding', '') \
                or 'gzip' in response_headers.get('transfer-encoding', ''):
            content = unzip(content)

        if content_type.lower() == 'json':
            return json.loads(content, 'utf-8')

        return content
    except (HTTPError, URLError):
        send_notification('Workflow', 'Error while calling {0}'.format(url))
        if content_type.lower() == 'json':
            return {}

        return ''
コード例 #37
0
    def _handle(self, is_get):

        debug('#\n' * 30)
        q = urlparse.parse_qs(urlparse.urlparse(self.path).query)
        url = q["url"][0]
        host = urlparse.urlsplit(url)

        headers = {
            "user-agent": USER_AGENT,
            "accept": "*/*",
            "host": host.netloc,
            "connection": "keep-alive",
            "keep-alive": "timeout=5, max=1000",
        }
        for h in self.headers:
            if h.lower() in ["icy-metadata", "range"] or h in headers:
                continue
            headers[h] = self.headers[h]

        debug("HEADERS: %s\n" % headers)
        debug("%s\n" % url)

        s = cache[url] if url in cache else requests.Session()
        res = s.get(url, headers=headers)

        debug("Response length: %d\n" % len(res.content))
        debug("Content Type: %s\n" % res.headers["Content-Type"])
        debug("Reponse Code: %s\n" % res.status_code)

        body = res.content
        url = "%s://%s/" % (host.scheme, host.netloc)
        if "application/vnd.apple.mpegurl" in res.headers['Content-Type'] or \
                "audio/x-mpegurl" in res.headers["Content-Type"]:
            # it's a playlist.  Route to our proxy server.
            out = []
            for line in body.splitlines(False):
                if line.startswith("http"):
                    line = "http://localhost:1704/?url=" + urllib.quote_plus(
                        line)
                elif line and not line.startswith("#EXT"):
                    line = "http://localhost:1704/?url=" + urllib.quote_plus(
                        url + "/" + line)
                elif line.startswith("#EXT-X-KEY"):
                    # reroute the encryption key url
                    m = uri_pat.search(line)
                    if m:
                        line = "%sURI=\"http://localhost:1704/?url=%s\"" % (
                            m.group(1), urllib.quote_plus(m.group(2)))

                elif line.startswith("#EXT-X-MEDIA"):
                    m = uri_pat.search(line)
                    if m:
                        uri = urllib.quote_plus(url + "/" + m.group(2))
                        line = "%sURI=\"http://localhost:1704/?url=%s\"" % (
                            m.group(1), uri)
                out.append(line)
            self.send(res, '\n'.join(out), is_get)
        else:
            self.send(res, body, is_get)

        debug("EXIT\n")
コード例 #38
0
def process_batch(sm_account_id,
                  graph,
                  interactions,
                  batch_requests,
                  p_session,
                  processed_interactions=None,
                  cutoff=None):
    """
    A function that sends batch requests to FB, collects the results and prepares the
    next set of batch requests for data corresponding to pagination.
    Call itself recursively until all posts in the given period are fetched.
    :param sm_account_id:
    :param graph:
    :param interactions:
    :param batch_requests:
    :param p_session:
    :param processed_interactions: Number of interactions already processed
    :param cutoff: stop collection if processed_interactions exceeds cutoff
    :return:
    """
    with transaction.manager:
        for interaction in interactions:
            p_session.merge(interaction)

    if len(batch_requests) == 0 or (processed_interactions
                                    and processed_interactions >= cutoff):
        return

    # process batch requests
    # Number of max items in a batch request is 50
    MAX_BATCH_SIZE = 50
    batch_requests_p = [{
        'method': req.get('method'),
        'relative_url': req.get('relative_url')
    } for req in batch_requests]
    batch_data = []

    interactions_new = set()
    batch_requests_new = []

    for i in range(math.ceil(len(batch_requests_p) / MAX_BATCH_SIZE)):
        # TODO handle connection error. attempt retries
        try:
            batch_req = json.dumps(
                batch_requests_p[i * MAX_BATCH_SIZE:(i * MAX_BATCH_SIZE) +
                                 (MAX_BATCH_SIZE - 1)],
                indent=1)
            batch_data += graph.request("", post_args={'batch': batch_req})

        except ConnectionError as e:
            logger.exception(
                'unable to process batch request \n:{}'.format(batch_req))
    for req, batch_response in zip(batch_requests, batch_data):
        parent_id = req.get('parent_id')
        if 'body' in batch_response:
            batch_response_data = json.loads(batch_response['body'])
            if 'error' in batch_response_data and batch_response_data[
                    'error'].get('code') == 1:
                # handle request failure - 'Please reduce the amount of data you are asking for, then retry your request'
                error_url = req.get('relative_url')
                parse_result = urlparse(error_url)
                query_data = urlparse.parse_qs(parse_result.query)
                old_limit = query_data.get('limit')[0]
                sm_account_id = parse_result.path.split("/")[2]
                new_limit = int(float(old_limit) / 2)
                new_req = get_feed_request(sm_account_id, limit=new_limit)
                batch_requests_new.append(new_req)

            if 'data' in batch_response_data:
                for interaction_raw in batch_response_data['data']:
                    Interactions.get_nested_interactions(
                        sm_account_id, interaction_raw, interactions_new,
                        batch_requests_new, parent_id)
            if 'paging' in batch_response_data and 'next' in batch_response_data[
                    'paging']:
                next_url = urlparse(batch_response_data['paging']['next'])
                relative_url = next_url.path + '?' + next_url.query + '&include_headers=false'
                req = {
                    'method': 'GET',
                    'relative_url': relative_url,
                    'parent_id': parent_id
                }
                batch_requests_new.append(req)
        else:
            logger.info(
                'Exception occurred while collecting posts for {} skipping this..'
                .format(sm_account_id))

    process_batch(sm_account_id, graph, interactions_new, batch_requests_new,
                  p_session, processed_interactions + len(interactions),
                  cutoff)
コード例 #39
0
 def query(self):
     return urlparse.parse_qs(self.parsed_uri.query)
コード例 #40
0
    def _do_COMMON(self, data={}):
        handlers = [
            {
                "url": r"/terminate",
                "action": self._do_terminate,
                "require_auth": False,
            },
            {
                "url": r"/rest/usermanagement/1/authentication$",
                "action": self._auth_user,
                "require_auth": True,
                "method": "POST",
            },
            {
                "url": r"/rest/usermanagement/1/session$",
                "action": self._get_session,
                "require_auth": True,
                "method": "POST",
            },
            {
                "url": r"/rest/usermanagement/1/session/[A-Za-z0-9]{24}$",
                "action": self._validate_session,
                "require_auth": True,
                "method": "POST",
            },
            {
                "url": r"/rest/usermanagement/1/session/[A-Za-z0-9]{24}$",
                "action": self._delete_session,
                "require_auth": True,
                "method": "DELETE",
            },
            {
                "url": r"/rest/usermanagement/1/user/group/direct$",
                "action": self._get_groups,
                "require_auth": True,
                "method": "GET",
            },
            {
                "url": r"/rest/usermanagement/1/user/group/nested$",
                "action": self._get_groups,
                "require_auth": True,
                "method": "GET",
            },
            {
                "url": r"/rest/usermanagement/1/group/membership$",
                "action": self._get_memberships,
                "require_auth": True,
                "method": "GET",
            },
            {
                "url": r"/rest/usermanagement/1/group/user/nested$",
                "action": self._get_group_users,
                "require_auth": True,
                "method": "GET",
            },
            {
                "url": r"/rest/usermanagement/1/user$",
                "action": self._get_user,
                "require_auth": True,
                "method": "GET",
            },
            {
                "url": r"/rest/usermanagement/1/user$",
                "action": self._add_user,
                "require_auth": True,
                "method": "POST",
            },
            {
                "url": r"/rest/usermanagement/1/user/password$",
                "action": self._change_password,
                "require_auth": True,
                "method": "PUT",
            },

            # Default handler for unmatched requests
            {
                "url": r".*",
                "action": self._default_handler,
                "require_auth": True,
            },
        ]

        p = urlparse.urlparse(self.path)

        self.json_data = data
        self.get_params = urlparse.parse_qs(p.query)

        for handler in handlers:
            method = handler.get('method')
            if (re.match(handler['url'], p.path)
                    and (not method or method == self.command)):

                # Authenticate application if required
                require_auth = handler.get('require_auth')
                if require_auth and not check_app_auth(self.headers):
                    self._do_app_failed_auth()
                    return

                # Run the handler's action
                handler['action']()
                return

        # An unhandled path was encountered.
        self.send_response(500)
        self.send_header("Content-type", "text/plain")
        self.end_headers()
        self.wfile.write('Oops, should not be here for {}'.format(
            self.path).encode('ascii'))