Example #1
0
def cleanup(url):
    url = _follow(url)
    # remove trackers params
    try:
        urlp = urlparse(url)
        # cleanup query param
        query = parse_qsl(urlp.query)
        # only if query is non empty and we manage to parse fragment as
        # key/value
        if urlp.query and query:
            for annoying in ANNOYING_PARAMS:
                query = [(x, y) for x, y in query if not x.startswith(annoying)]
            urlp = urlp._replace(
                query=urlencode(query),
            )

        # cleanup fragment param
        fragment = parse_qsl(urlp.fragment)
        # only if fragments is non empty and we manage to parse fragment as
        # key/value
        if urlp.fragment and fragment:
            for annoying in ANNOYING_PARAMS:
                fragment = [(x, y) for x, y in fragment if not x.startswith(annoying)]
            urlp = urlp._replace(
                fragment=urlencode(fragment),
            )
        url = urlp.geturl()
    except Exception:
        app.logger.exception("Problem cleaning url %s", url)

    app.logger.info("Final url %s", url)
    return url
Example #2
0
    def facebook_request(self, path, callback, access_token=None,
                         post_args=None, **args):
        """Fetches the given relative API path, e.g., "/btaylor/picture"

        If the request is a POST, ``post_args`` should be provided. Query
        string arguments should be given as keyword arguments.

        An introduction to the Facebook Graph API can be found at
        http://developers.facebook.com/docs/api

        Many methods require an OAuth access token which you can
        obtain through `~OAuth2Mixin.authorize_redirect` and
        `get_authenticated_user`. The user returned through that
        process includes an ``access_token`` attribute that can be
        used to make authenticated requests via this method.

        Example usage:

        ..testcode::

            class MainHandler(tornado.web.RequestHandler,
                              tornado.auth.FacebookGraphMixin):
                @tornado.web.authenticated
                @tornado.gen.coroutine
                def get(self):
                    new_entry = yield self.facebook_request(
                        "/me/feed",
                        post_args={"message": "I am posting from my Tornado application!"},
                        access_token=self.current_user["access_token"])

                    if not new_entry:
                        # Call failed; perhaps missing permission?
                        yield self.authorize_redirect()
                        return
                    self.finish("Posted a message!")

        .. testoutput::
           :hide:

        The given path is relative to ``self._FACEBOOK_BASE_URL``,
        by default "https://graph.facebook.com".

        .. versionchanged:: 3.1
           Added the ability to override ``self._FACEBOOK_BASE_URL``.
        """
        url = self._FACEBOOK_BASE_URL + path
        all_args = {}
        if access_token:
            all_args["access_token"] = access_token
            all_args.update(args)

        if all_args:
            url += "?" + urllib_parse.urlencode(all_args)
        callback = functools.partial(self._on_facebook_request, callback)
        http = self.get_auth_http_client()
        if post_args is not None:
            http.fetch(url, method="POST", body=urllib_parse.urlencode(post_args),
                       callback=callback)
        else:
            http.fetch(url, callback=callback)
Example #3
0
    def test_login_failed_empty(self):
        post_data = {'action': 'Login',
                     'email': '',
                     'password': '******'}
        body = urlencode(post_data)
        self.http_client.fetch(self.get_url('/login/email'),
                               self.stop,
                               method='POST',
                               body=body,
                               follow_redirects=False)
        response = self.wait()
        self.assertEqual(json_decode(response.body)['status'], 'failed')
        self.assertEqual(json_decode(response.body)['error'], 'Email and password are mandatory')

        post_data = {'action': 'Login',
                     'email': 'foo',
                     'password': ''}
        body = urlencode(post_data)
        self.http_client.fetch(self.get_url('/login/email'),
                               self.stop,
                               method='POST',
                               body=body,
                               follow_redirects=False)
        response = self.wait()
        self.assertEqual(json_decode(response.body)['status'], 'failed')
        self.assertEqual(json_decode(response.body)['error'], 'Email and password are mandatory')
Example #4
0
    def WriteData(self, data):
        """ Write observation data to server

            :param data: dictionary with observation data
            :returns: server answer or None (if error)
        """
        par = {}
        if data is None or self.DropData(data):
            logging.warning(" empty or inappropiate data not written")
            return
        # add datetime and/or id
        data = self.ExtendData(data)
        for key, val in data.items():
            if self.filt is None or key in self.filt:
                par[key] = self.StrVal(val)
        if self.mode == 'GET':
            #res = urlopen(self.url + '?' + urlencode(par)).read()
            print(self.url + '?' + urlencode(par))
            try:
                res = urlopen(self.url + '?' + urlencode(par))
                print(res)
                res = res.read()
                print(res)
            except:
                res = None
        else:
            try:
                d = urlencode(par)
                req = Request(self.url, d)
                print(req)
                res = urlopen(req).read()
                print(res)
            except:
                res = None
        return res
Example #5
0
 def get_user_work_page(self):
     self.userId = input('Enter User id: ')
     self.userDirName = join(self.imgStoreDirName, self.pixiv, self.userId)
     if self.userId.isdigit():
         if not exists(self.userDirName):
             makedirs(self.userDirName)
     else:
         print('Wrong id.')
         raise SystemExit(1)
     get_value = {'id': self.userId,
                  'type': 'all',
                  'p': 1
                  }
     get_data = urlencode(get_value)
     work_url = self.memIllUrl + get_data
     print("Load the page 1...")
     work_page = self.url_open(work_url)
     pattern = compile('class="count-badge">(.*?)</span>')
     count_num = search(pattern, work_page)
     count_num = int(count_num.group(1)[:-1])
     total_page = ceil(count_num / 20)
     yield work_page
     for i in range(2, total_page + 1):
         get_value['p'] = i
         get_data = urlencode(get_value)
         work_url = self.memIllUrl + get_data
         print("Load the page %d..." % i)
         work_page = self.url_open(work_url)
         yield work_page
Example #6
0
 def request_page(self, p, action="GET", **kvargs):
     conn = http.client.HTTPConnection(self.serv+".pokemon-gl.com")
     url = "/api/?"
     headers = {"Cookie": "PMDSUSSID={}; locale=en".format(self.PMDSUSSID)}
     get = [("p", p)]
     #if member_id != None:
     #    get.append(("member_savedata_id", member_id))
     if self.token != None:
         get.append(("token", self.token))
     get += kvargs.items()
     if debug: print(get)
     if action == "GET":
         url += urlencode(get)
         data = None
     elif action == "POST":
         headers["Content-type"]='application/x-www-form-urlencoded'
         data = urlencode(get)
     conn.request(action, url, data, headers)
     r = conn.getresponse()
     data = r.read().decode("utf-8")
     tree = json.loads(data)
     if 'error' in tree:
         error = tree['error']
         raise RuntimeError(error['code'], error['mess'], error['details'])
     if debug: print(tree)
     return tree
Example #7
0
def _photo_item(users, user, api_item):
    """Parses a photo item."""

    if api_item["type"] == "photo":
        title = "новые фотографии"
        photos = api_item["photos"]
        get_photo_url = lambda photo: _vk_url("feed?" + urlencode({
            "section": "photos",
            "z": "photo{owner_id}_{photo_id}/feed1_{source_id}_{timestamp}".format(
                owner_id=photo["owner_id"], photo_id=photo["pid"],
                source_id=api_item["source_id"], timestamp=api_item["date"])}))
    elif api_item["type"] == "photo_tag":
        title = "новые отметки на фотографиях"
        photos = api_item["photo_tags"]
        get_photo_url = lambda photo: _vk_url("feed?" + urlencode({
            "z": "photo{owner_id}_{photo_id}/feed3_{source_id}_{timestamp}".format(
                owner_id=photo["owner_id"], photo_id=photo["pid"],
                source_id=api_item["source_id"], timestamp=api_item["date"])}))
    else:
        raise Error("Logical error.")

    item = {
        "title": user["name"] + ": " + title,
        "text":  "",
    }

    for photo in photos[1:]:
        url = get_photo_url(photo)
        item.setdefault("url", url)
        item["text"] += _block(_link(url, _image(photo["src_big"])))

    if photos[0] > len(photos) - 1:
        item["text"] += _block("[показаны не все фотографии]")

    return item
    def method(self, url, method="GET", parameters=None, timeout=None):
        log.info('Making {0} request to {1} with parameters {2}'.format(method, url, parameters))
        method_url = urljoin(self.url, url)

        if method == "GET":
            if not parameters:
                parameters = dict()

            parameters['format'] = self.format
            parameters['auth_token'] = self.token

            query_string = urlencode(parameters)
            request_data = None
        else:
            query_parameters = dict()
            query_parameters['auth_token'] = self.token

            query_string = urlencode(query_parameters)

            if parameters:
                request_data = urlencode(parameters).encode('utf-8')
            else:
                request_data = None

        method_url = method_url + '?' + query_string
        log.debug('Method URL: {0}'.format(method_url))

        req = self.RequestWithMethod(method_url, http_method=method, data=request_data)
        response = self.opener.open(req, None, timeout)

        # TODO Check response status (failures, throttling)

        return json.loads(response.read().decode('utf-8'))
def total_file_size(slack_token, verbose=False):
    """
    Finds the total size of all files on the slack server
    :param slack_token:
    :param verbose:
    :return:
    """
    params = {
        'token': slack_token,
        'count': 500,
    }
    response = reader(urlopen('https://slack.com/api/files.list?' + urlencode(params)))
    size = 0

    file_ids = [f['id'] for f in load(response)['files']]
    for file_id in file_ids:
        params = {
            'token': token,
            'file': file_id
        }

        response = reader(urlopen('https://slack.com/api/files.info?' + urlencode(params)))
        size += load(response)['file']['size']
        mb = size / 1048576
        if verbose:
            print('{0:.2f} MB total'.format(mb))

    mb = size / 1048576
    return '{0:.2f} MB'.format(mb)
Example #10
0
 def auth_user(email, password, client_id, scope, opener):
     request_params = {
         'redirect_uri': 'https://oauth.vk.com/blank.html',
         'response_type': 'token',
         'client_id': client_id,
         'display': 'mobile',
         'scope': ','.join(scope),
         'v': API_VERSION
     }
     base_auth_url = 'https://oauth.vk.com/authorize'
     params = list(request_params.items())
     params = urlencode(params).encode('utf-8')
     response = opener.open(base_auth_url, params)
     doc = response.read().decode(encoding='utf-8', errors='replace')
     parser = FormParser()
     parser.feed(doc)
     parser.close()
     if (not parser.form_parsed or
             parser.url is None or
             "pass" not in parser.params or
             "email" not in parser.params):
         raise RuntimeError("Something wrong")
     parser.params["email"] = email
     parser.params["pass"] = password
     if parser.method == "POST":
         params = urlencode(parser.params).encode('utf-8')
         response = opener.open(parser.url, params)
     else:
         raise NotImplementedError("Method '%s'" % parser.method)
     doc = response.read().decode(encoding='utf-8', errors='replace')
     return doc, response.geturl()
def delete_files(file_ids, slack_token, verbose=False):
    """
    Deletes all files with IDs matching the given list
    :param file_ids:
    :param slack_token:
    :param verbose:
    """
    size = 0
    count = 0
    num_files = len(file_ids)

    for file_id in file_ids:
        count += 1
        params = {
            'token': slack_token,
            'file': file_id
        }

        response = reader(urlopen('https://slack.com/api/files.info?' + urlencode(params)))
        size += load(response)['file']['size']

        response = reader(urlopen('https://slack.com/api/files.delete?' + urlencode(params)))
        ok = load(response)['ok']
        mb = size / 1048576

        if verbose:
            print("{0} of {1} - {2} {3} ... {4:.2f} MB saved".format(count, num_files, file_id, ok, mb))
 def MsgAddCard(self, chat_id):
     self.p_sendMessage['text'] = "Enter card number [xxxxxxxxxxx]:"
     self.p_sendMessage['chat_id'] = str(chat_id)
     udata = urlencode(self.p_sendMessage).encode()
     req = Request(self.sendMessage, udata)
     resp = urlopen(req)
     bNum = False
     while not bNum:
         udata = urlencode(self.p_getUpdates).encode()
         req = Request(self.getUpdates, udata)
         resp = urlopen(req).read().decode()
         data = json.loads(resp).get('result')[0]
         if data.get('update_id') > self.update_id:
             self.update_id = data.get('update_id')
             bNum = True
     if self.CheckCardNum(data.get('message').get('text')):
         if chat_id in self.cards:
             self.cards[chat_id].append(data.get('message').get('text'))
         else:
             self.cards[chat_id] = []
             self.cards[chat_id].append(data.get('message').get('text'))
         self.p_sendMessage['text'] = "Card " + data.get('message').get('text') + " successfully added!"
         self.p_sendMessage['chat_id'] = str(chat_id)
         udata = urlencode(self.p_sendMessage).encode()
         req = Request(self.sendMessage, udata)
         resp = urlopen(req)
     else:
         self.p_sendMessage['text'] = "This doesn't look like card number!"
         self.p_sendMessage['chat_id'] = str(chat_id)
         udata = urlencode(self.p_sendMessage).encode()
         req = Request(self.sendMessage, udata)
         resp = urlopen(req)
Example #13
0
    def set(self, pagepath, text, comment=None):
        """Inputs text into the wiki input text box.

        Parameters
        ----------
        pagepath : :class:`str`
            Wiki page to update.
        text : :class:`str`
            The wiki text.
        comment : :class:`str`, optional
            A comment on the change.
        """
        response = self.opener.open(self.url+"/wiki/"+pagepath+"?action=edit")
        if self._debug:
            print(response.info())
        parser = SimpleWikiHTMLParser('version')
        parser.feed(response.read().decode('utf-8'))
        response.close()
        postdata = {'__FORM_TOKEN': self._form_token,
                    'from_editor': '1',
                    'action': 'edit',
                    'version': parser.search_value,
                    'save': 'Submit changes',
                    'text': CRLF(text)}
        if comment is not None:
            postdata['comment'] = CRLF(comment)
        if self._debug:
            print(urlencode(postdata))
        response = self.opener.open(self.url+"/wiki/"+pagepath,
                                    urlencode(postdata).encode('utf-8'))
        response.close()
        return
Example #14
0
def send_feedback(dsn, event_id, name, email, comment, timeout):
    """Send feedback, blocking.

    Args:
        dsn (str): The DSN
        event_id (str): The event ID this feedback should be attached to
        name (str): The user name
        email (str): The user email
        comment (str): The feedback text
        timeout (float): The timeout for this request
    Raises:
        SentryError: In case of timeout or other errors
    """

    name = str(name).encode("utf-8")
    email = str(email).encode("utf-8")
    comment = str(comment).encode("utf-8")

    data = urlencode(
        [('name', name), ('email', email), ('comments', comment)])
    if not isinstance(data, bytes):
        # py3
        data = data.encode("utf-8")

    headers = {"Referer": "https://quodlibet.github.io"}
    params = urlencode([("dsn", dsn), ("eventId", event_id)])

    try:
        req = Request(
            "https://sentry.io/api/embed/error-page/?" + params,
            data=data, headers=headers)

        urlopen(req, timeout=timeout).close()
    except EnvironmentError as e:
        raise SentryError(e)
Example #15
0
    def tag_text(self, text):
        """Tag the text with proper named entities token-by-token.

        :param text: raw text strig to tag
        :returns: tagged text in given output format
        """
        for s in ("\f", "\n", "\r", "\t", "\v"):  # strip whitespaces
            text = text.replace(s, "")
        text += "\n"  # ensure end-of-line
        with http_connection(self.host, self.port) as c:
            headers = {"Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain"}
            if self.classifier:
                params = urlencode(
                    {
                        "input": text,
                        "outputFormat": self.oformat,
                        "preserveSpacing": self.spacing,
                        "classifier": self.classifier,
                    }
                )
            else:
                params = urlencode({"input": text, "outputFormat": self.oformat, "preserveSpacing": self.spacing})
            try:
                c.request("POST", self.location, params, headers)
                response = c.getresponse()
                tagged_text = response.read()
            except httplib.HTTPException as e:
                print("Failed to post HTTP request.")
                raise e
        return tagged_text
Example #16
0
    def tag_text(self, text):
        """Tag the text with proper named entities token-by-token.

        :param text: raw text strig to tag
        :returns: tagged text in given output format
        """
        for s in ('\f', '\n', '\r', '\t', '\v'): #strip whitespaces
            text = text.replace(s, '')
        text += '\n' #ensure end-of-line
        with http_connection(self.host, self.port) as c:
            headers = {'Content-type': 'application/x-www-form-urlencoded', 'Accept' : 'text/plain'}
            if self.classifier:
                params = urlencode(
                    {'input': text, 'outputFormat': self.oformat,
                    'preserveSpacing': self.spacing,
                    'classifier': self.classifier})
            else:
                params = urlencode(
                    {'input': text, 'outputFormat': self.oformat,
                    'preserveSpacing': self.spacing})
            try:
                c.request('POST', self.location, params, headers)
                response = c.getresponse()
                tagged_text = response.read()
            except httplib.HTTPException as e:
                print("Failed to post HTTP request.")
                raise e
        return tagged_text
    def method(self, url, method="GET", parameters=None, timeout=None):
        method_url = urljoin(self.url, url)

        if method == "GET":
            if not parameters:
                parameters = dict()

            parameters['format'] = self.format
            parameters['auth_token'] = self.token

            query_string = urlencode(parameters)
            request_data = None
        else:
            query_parameters = dict()
            query_parameters['auth_token'] = self.token

            query_string = urlencode(query_parameters)

            if parameters:
                request_data = urlencode(parameters)
            else:
                request_data = None

        method_url = method_url + '?' + query_string

        if method == "POST":
            req = self.RequestWithMethod(method_url, http_method=method, data=request_data.encode('utf-8'))
        if method == "GET":
            req = self.RequestWithMethod(method_url, http_method=method, data=request_data)
        response = self.opener.open(req, None, timeout).read()

        return json.loads(response.decode('utf-8'))
Example #18
0
 def request(self, uri, *,
         method='GET',
         query=None,
         headers={},
         body=None):
     conn = self.connection()
     assert method.isidentifier(), method
     assert uri.startswith('/'), uri
     if query:
         if '?' in uri:
             uri += '&' + urlencode(query)
         else:
             uri += '?' + urlencode(query)
     headers = headers.copy()
     statusline = '{} {} HTTP/1.1'.format(method.upper(), uri)
     lines = [statusline]
     if isinstance(body, dict):
         body = urlencode(body)
     if isinstance(body, str):
         body = body.encode('utf-8')  # there are no other encodings, right?
     if body is not None:
         clen = len(body)
     else:
         clen = 0
         body = b''
     headers['Content-Length'] = clen
     for k, v in headers.items():
         lines.append('{}: {}'.format(k, str(v)))
     lines.append('')
     lines.append('')
     buf = '\r\n'.join(lines).encode('ascii')
     return self.response_class(*conn.request(buf + body).get())
Example #19
0
 def post(self, url, data, accept = "*/*", charset = "UTF-8", referer = ""):
     curl = pycurl.Curl()
     print("post: %s", url)
     print("data: %s", urlencode(data))
     buf = BytesIO()
     try:
         curl.setopt(pycurl.COOKIEFILE, CookieFile)
         curl.setopt(pycurl.COOKIEJAR, CookieFile)
         curl.setopt(pycurl.FOLLOWLOCATION, 1)    # 允许跟踪来源
         curl.setopt(pycurl.MAXREDIRS, 5)    # 设置最大重定向次数
         curl.setopt(pycurl.TIMEOUT, 80)    # 连接超时设置
         # curl.setopt(pycurl.VERBOSE, 1)    # verbose
         curl.setopt(pycurl.WRITEFUNCTION, buf.write)
         curl.setopt(pycurl.POSTFIELDS, urlencode(data))
         curl.setopt(pycurl.URL, url)
         curl.setopt(pycurl.HTTPHEADER, ["Accept: {0}", "Accept-Charset: {1}", "Referer: {2}".format(accept, charset, referer)])
         curl.perform()
         for line in curl.getinfo(pycurl.INFO_COOKIELIST):
             self._parseCookie(line)
         result = buf.getvalue()    # 得到缓冲区的数据
         buf.close()
         curl.close()
         del buf
         del curl
         return self.bytesToStr(result)
     except Exception as e:
         print(e)
         del buf
         del curl
         return ""
Example #20
0
    def __build_params(self):
        # See https://developers.facebook.com/docs/reference/api/batch/
        # for documentation on how the batch api is supposed to work.

        batch = []
        all_files = []
        for request in self.__api_calls:
            payload = {'method': request.method}
            if not request.ignore_result:
                payload['omit_response_on_success'] = False
            if request.name:
                payload['name'] = request.name
            if request.method in ['GET', 'DELETE']:
                payload['relative_url'] = request.path+'?'+urlencode(request.params)
            elif request.method == 'POST':
                payload['relative_url'] = request.path
                files = []
                params = {}
                for key, value in request.params.iteritems():
                    if isinstance(value, FileType):
                        all_files.append(value)
                        files.append('file%s' % (len(all_files) - 1))
                    else:
                        params[key] = value
                payload['body'] = urlencode(params)
                payload['attached_files'] = ','.join(files)
            batch.append(payload)

        params = {'batch':json.dumps(batch)}
        for i, f in enumerate(all_files):
            params['file%s' % i] = f

        return params
Example #21
0
 def load_posts(self, *, web_driver: WebDriver = None, **params) -> List[Post]:
     params.setdefault('owner_id', -self.group_id)
     if web_driver is None:
         raw_posts = self.get_all_objects('wall.get', **params)
     else:
         open_url('https://vk.com', web_driver)
         login = web_driver.find_element_by_xpath('//*[@id="index_email"]')
         login.clear()
         login.send_keys(self.user_login)
         password = web_driver.find_element_by_xpath('//*[@id="index_pass"]')
         password.clear()
         password.send_keys(self.user_password)
         web_driver.find_element_by_xpath('//*[@id="index_login_button"]').click()
         url_parts = list(urlparse('https://vk.com/dev/wall.get'))
         count = 100
         query = {'params[owner_id]': params['owner_id'],
                  'params[count]': count,
                  'params[offset]': params.get('offset', 0),
                  'params[filter]': params.get('filter', 'owner'),
                  'params[fields]': params.get('fields', ''),
                  'params[v]': self.api_version}
         url_parts[4] = urlencode(query)
         url = urlunparse(url_parts)
         response = parse_from_vk_dev(url, web_driver)['response']
         total_count = response['count']
         raw_posts = response['items']
         while len(raw_posts) < total_count:
             query['params[offset]'] += count
             url_parts[4] = urlencode(query)
             url = urlunparse(url_parts)
             response = parse_from_vk_dev(url, web_driver)['response']
             raw_posts += response['items']
     return [Post.from_raw(raw_post) for raw_post in raw_posts]
Example #22
0
    def __getChannelId(self):

        """
        Obtain channel id for channel name, if present in ``self.search_params``.
        """

        if not self.search_params.get("channelId"):
            return

        api_fixed_url = "https://www.googleapis.com/youtube/v3/channels?part=id&maxResults=1&fields=items%2Fid&"
        url = api_fixed_url + urlencode({"key": self.api_key, "forUsername": self.search_params["channelId"]})
        get = requests.get(url).json()

        try:
            self.search_params["channelId"] = get['items'][0]['id']
            return # got it

        except IndexError:
            pass # try searching now...

        api_fixed_url = "https://www.googleapis.com/youtube/v3/search?part=snippet&type=channel&fields=items%2Fid&"
        url = api_fixed_url + urlencode({"key": self.api_key, "q": self.search_params['channelId']})
        get = requests.get(url).json()

        try:
            self.search_params["channelId"] = get['items'][0]['id']['channelId']

        except IndexError:
            del self.search_params["channelId"] # channel not found
Example #23
0
def suggest(context, request):
    text = ''
    result = {
        '@id': '/suggest/?' + urlencode({'q': text}),
        '@type': ['suggest'],
        'title': 'Suggest',
        '@graph': [],
    }
    if 'q' in request.params:
        text = request.params.get('q', '')
    else:
        return []
    es = request.registry['snp_search']
    query = {
        "suggester": {
            "text": text,
            "completion": {
                "field": "name_suggest",
                "size": 10
            }
        }
    }
    try:
        results = es.suggest(index='annotations', body=query)
    except:
        return {}
    else:
        result['@id'] = '/suggest/?' + urlencode({'q': text})
        result['@graph'] = []
        for item in results['suggester'][0]['options']:
            if not any(x in item['text'] for x in ['(C. elegans)','(mus musculus)','(D. melanogaster)']):
                result['@graph'].append(item)
        return result
Example #24
0
	def twitter_request(self, path, callback=None, access_token=None,
						post_args=None, **args):
		"""Fetches the given API path, e.g., ``statuses/user_timeline/btaylor``

		The path should not include the format or API version number.
		(we automatically use JSON format and API version 1).

		If the request is a POST, ``post_args`` should be provided. Query
		string arguments should be given as keyword arguments.

		All the Twitter methods are documented at http://dev.twitter.com/

		Many methods require an OAuth access token which you can
		obtain through `~OAuthMixin.authorize_redirect` and
		`~OAuthMixin.get_authenticated_user`. The user returned through that
		process includes an 'access_token' attribute that can be used
		to make authenticated requests via this method. Example
		usage::

			class MainHandler(tornado.web.RequestHandler,
							  tornado.auth.TwitterMixin):
				@tornado.web.authenticated
				@tornado.web.asynchronous
				@tornado.gen.coroutine
				def get(self):
					new_entry = yield self.twitter_request(
						"/statuses/update",
						post_args={"status": "Testing Tornado Web Server"},
						access_token=self.current_user["access_token"])
					if not new_entry:
						# Call failed; perhaps missing permission?
						yield self.authorize_redirect()
						return
					self.finish("Posted a message!")

		"""
		if path.startswith('http:') or path.startswith('https:'):
			# Raw urls are useful for e.g. search which doesn't follow the
			# usual pattern: http://search.twitter.com/search.json
			url = path
		else:
			url = self._TWITTER_BASE_URL + path + ".json"
		# Add the OAuth resource request signature if we have credentials
		if access_token:
			all_args = {}
			all_args.update(args)
			all_args.update(post_args or {})
			method = "POST" if post_args is not None else "GET"
			oauth = self._oauth_request_parameters(
				url, access_token, all_args, method=method)
			args.update(oauth)
		if args:
			url += "?" + urllib_parse.urlencode(args)
		http = self.get_auth_http_client()
		http_callback = self.async_callback(self._on_twitter_request, callback)
		if post_args is not None:
			http.fetch(url, method="POST", body=urllib_parse.urlencode(post_args),
					   callback=http_callback)
		else:
			http.fetch(url, callback=http_callback)
Example #25
0
def asyncServe(privmsg, query, yearhint):
    if yearhint:
        params = urlencode({'title': query,
            'sort': 'num_votes,desc',
            'release_date': "%04i-06-01,%04i-06-01"%(yearhint-1, yearhint+1)})
    else:
        params = urlencode({'title': query,
            'sort': 'num_votes,desc'})
    url = "http://akas.imdb.com/search/title?" + params
    debug("int_imdb", "url", url)
    html = urllib.request.urlopen(url).read(50000).decode("iso-8859-1")
    html = html.partition('<table class="results">')[2]
    if not html:
        debug("int_imdb", "result-table not found.")
        return
    
    ttid = re.search(
        r'<a href="/title/(tt[^/]+)/">', html).group(1)
    title = htmlDecodeEntites(re.search(
        r'<a href="/title/tt[^/]+/">([^<]+)</a>', html).group(1))
    director = htmlDecodeEntites(re.search(
        r'Dir: <a[^>]+>([^<]+)</a>', html).group(1))
    #outline = re.search(
    #    r'<span class="outline">([^>]+)</span>', html)
    #if outline: outline = htmlDecodeEntites(outline.group(1))
    year = re.search(
        r'<span class="year_type">\((\d{4})', html).group(1)
    rating = re.search(
        r'itemprop="ratingValue">([0-9.]+|-)</span>', html).group(1)
    
    
    privmsg.reply_imore("%s (%s) %s [%s] http://www.imdb.com/title/%s/"%(
        title, year, director, rating, ttid))
    def test_call_app(self):
        url = reverse('two_factor:twilio_call_app', args=['123456'])
        response = self.client.get(url)
        self.assertEqual(response.content,
                         b'<?xml version="1.0" encoding="UTF-8" ?>'
                         b'<Response>'
                         b'  <Gather timeout="15" numDigits="1" finishOnKey="">'
                         b'    <Say language="en">Hi, this is testserver calling. '
                         b'Press any key to continue.</Say>'
                         b'  </Gather>'
                         b'  <Say language="en">You didn\'t press any keys. Good bye.</Say>'
                         b'</Response>')

        url = reverse('two_factor:twilio_call_app', args=['123456'])
        response = self.client.post(url)
        self.assertEqual(response.content,
                         b'<?xml version="1.0" encoding="UTF-8" ?>'
                         b'<Response>'
                         b'  <Say language="en">Your token is 1. 2. 3. 4. 5. 6. '
                         b'Repeat: 1. 2. 3. 4. 5. 6. Good bye.</Say>'
                         b'</Response>')

        # there is a en-gb voice
        response = self.client.get('%s?%s' % (url, urlencode({'locale': 'en-gb'})))
        self.assertContains(response, '<Say language="en-gb">')

        # there is no nl voice
        response = self.client.get('%s?%s' % (url, urlencode({'locale': 'nl-nl'})))
        self.assertContains(response, '<Say language="en">')
Example #27
0
def test_query_dont_unqoute_twice():
    sample_url = "http://base.place?" + urlencode({"a": "/////"})
    query = urlencode({"url": sample_url})
    full_url = "http://test_url.aha?" + query

    url = URL(full_url)
    assert url.query["url"] == sample_url
Example #28
0
	def api(self, command, args={}):
		"""
		Main Api Function
		- encodes and sends <command> with optional [args] to Poloniex api
		- raises 'ValueError' if an api key or secret is missing (and the command is 'private'), or if the <command> is not valid
		- returns decoded json api message
		"""
		if self._coaching: self.apiCoach.wait() # check in with the coach
		args['command'] = command # pass the command
		
		global PUBLIC_COMMANDS;global PRIVATE_COMMANDS
		
		if command in PRIVATE_COMMANDS: # private?
			try:
				if len(self.APIKey) < 2 or len(self.Secret) < 2:
					raise ValueError("An APIKey and Secret is needed for private api commands!")
				args['nonce'] = self.nonce
				post_data = urlencode(args)
				sign = hmac.new(self.Secret, post_data.encode('utf-8'), hashlib.sha512).hexdigest()
				headers = {'Sign': sign, 'Key': self.APIKey}
				ret = requests.post('https://poloniex.com/tradingApi', data=args, headers=headers, timeout=self.timeout)
				return json.loads(ret.text)
			except Exception as e:raise e
			finally:self.nonce+=1 # increment nonce
			
		elif command in PUBLIC_COMMANDS: # public?
			try:
				ret = requests.post('https://poloniex.com/public?' + urlencode(args), timeout=self.timeout)
				return json.loads(ret.text)
			except Exception as e:raise e
		else:raise ValueError("Invalid Command!")
Example #29
0
 def request(
         cls,
         uri,
         params={},
         client=None,
         wrapper=FreesoundObject,
         method='GET',
         data=False
         ):
     p = params if params else {}
     url = '%s?%s' % (uri, urlencode(p)) if params else uri
     d = urlencode(data) if data else None
     headers = {'Authorization': client.header}
     req = Request(url, d, headers)
     try:
         f = urlopen(req)
     except HTTPError as e:
         resp = e.read()
         if e.code >= 200 and e.code < 300:
             return resp
         else:
             raise FreesoundException(e.code, json.loads(resp))
     resp = f.read()
     f.close()
     result = None
     try:
         result = json.loads(resp)
     except:
         raise FreesoundException(0, "Couldn't parse response")
     if wrapper:
         return wrapper(result, client)
     return result
Example #30
0
 def _surlencode():
     for k,v in query.items():
         if isinstance(v, (list, tuple)):
             for vsub in v:
                 yield urlencode({ k : vsub })
         else:
             yield urlencode({ k : v })
def close(today_str, day1_str, day2_str, day3_str, day4_str, day5_str, day6_str, day7_str):
    queryParams = '?' + urlencode({
        quote_plus("authKey"): "uZ98I03VR/zZ3zAvdNIVEl8EWJy6llSwLtDAMDlOYkE=",

        quote_plus("lastModTsBgn"): day1_str,

        quote_plus("lastModTsEnd"): today_str,

        quote_plus("state"): "03",

        quote_plus("pageSize"): "10000",

        quote_plus("resultType"): "xml"
    })

    request = Request(url + queryParams)

    request.get_method = lambda: 'GET'

    response_body = urlopen(request).read()

    dict = xmltodict.parse(response_body)

    jsonString = json.dumps(dict['result']['body'], ensure_ascii=False)

    jsonObj = json.loads(jsonString)

    local_real_time_close = localWeekData['test_close']

    if jsonObj['rows']:
        cnt = 0
        for item in jsonObj['rows']['row']:
            if item['dcbYmd'] == day1_str or item['dcbYmd'] == day2_str or item['dcbYmd'] == day3_str or item['dcbYmd'] == day4_str or item['dcbYmd'] == day5_str or item['dcbYmd'] == day6_str or item['dcbYmd'] == day7_str:
                is_exist = local_real_time_close.find_one(
                    {"$and": [{"mgtNo": item['mgtNo']}, {"store_name": item['bplcNm']}]})
                if is_exist is None:
                    cnt += 1
                    city_name = ""
                    category_name = ""
                    category_kor = ""
                    city_split = ""
                    if item['rdnWhlAddr']:
                        city_split = item['rdnWhlAddr'].split(' ')[0]
                        address = item['rdnWhlAddr']
                    else:
                        city_split = item['siteWhlAddr'].split(' ')[0]
                        address = item['siteWhlAddr']

                    if city_split == "부산광역시":
                        city_name = "busan"
                    elif city_split == "충청북도":
                        city_name = "chungbuk"
                    elif city_split == "충청남도":
                        city_name = "chungnam"
                    elif city_split == "대구광역시":
                        city_name = "daegu"
                    elif city_split == "대전광역시":
                        city_name = "daejeon"
                    elif city_split == "강원도":
                        city_name = "gangwon"
                    elif city_split == "광주광역시":
                        city_name = "gwangju"
                    elif city_split == "경기도":
                        city_name = "gyeonggi"
                    elif city_split == "경상북도":
                        city_name = "gyeongbuk"
                    elif city_split == "경상남도":
                        city_name = "gyeongnam"
                    elif city_split == "인천광역시":
                        city_name = "incheon"
                    elif city_split == "제주특별자치도":
                        city_name = "jeju"
                    elif city_split == "전라북도":
                        city_name = "jeonbuk"
                    elif city_split == "전라남도":
                        city_name = "jeonnam"
                    elif city_split == "세종특별자치시":
                        city_name = "sejong"
                    elif city_split == "서울특별시":
                        city_name = "seoul"
                    elif city_split == "울산광역시":
                        city_name = "ulsan"

                    open_service_id_split = item['opnSvcId'][0:2]

                    if open_service_id_split == "01":
                        category_name = "health"
                        category_kor = "건강"
                    elif open_service_id_split == "02":
                        category_name = "animal"
                        category_kor = "동물"
                    elif open_service_id_split == "03" or open_service_id_split == "04":
                        category_name = "culture"
                        category_kor = "문화"
                    elif open_service_id_split == "05" or open_service_id_split == "06" or open_service_id_split == "08" or open_service_id_split == "10":
                        category_name = "life"
                        category_kor = "생활"
                    elif open_service_id_split == "07":
                        category_name = "food"
                        category_kor = "식품"
                    elif open_service_id_split == "09":
                        category_name = "environment"
                        category_kor = "환경"
                    elif open_service_id_split == "11":
                        category_name = "other"
                        category_kor = "기타"

                    info = {
                        "mgtNo": item['mgtNo'],
                        "data": "close",
                        "update_date": item['updateDt'],
                        "authorization_date": item['apvPermYmd'],
                        "closed_date": item['dcbYmd'],
                        "store_name": item['bplcNm'],
                        "address": address,
                        "state_code": item['trdStateGbn'],
                        "state": item['trdStateNm'],
                        "open_service_id": item['opnSvcId'],
                        "open_service": item['opnSvcNm'],
                        "detailed_classification": item['uptaeNm'],
                        "city_name": city_name,
                        "category_name": category_name,
                        "category_kor": category_kor,
                    }
                    print(cnt, "close", info)
                    # local_real_time_close = localWeekData['test_close']
                    # local_real_time_close_id = local_real_time_close.insert_one(info).inserted_id
    return cnt

url = 'http://news.donga.com/search'
param_dict = {
    'p' : '',
    'query' : KEY_WORD,
    'check_news' : '1',
    'more' : '1',
    'sorting' : '1',         # 정확도=3, 최신순=1
    'search_date' : '1',
    'v1' : '',
    'v2' : '',
    'range' : '3',
}

url_all = url + '?' + urlencode(query=param_dict)

source_code_from_url = urllib.request.urlopen(url_all)
soup = BeautifulSoup(source_code_from_url, 'lxml', from_encoding='utf-8')


# select 나 find_all 이나 결과는 똑같다. --> tag list 를 반환한다.
content_of_articles = soup.select('p.tit')
findings =  soup.find_all('p', 'tit')

search_md_string = f"## Search Result for '{KEY_WORD}'<br>\n"

for i, title in enumerate(soup.find_all('p', 'tit'),1):
    date_text = title.select('span')[-1].get_text()
    datetime_format = datetime.datetime.strptime(date_text, '%Y-%m-%d %H:%M')
    date_new = datetime.datetime.strftime(datetime_format, '%y.%m/%d(%a)')