Пример #1
0
def fetch(url, content='', method='GET', credentials=None, headers=None, multipart=False, ua='', timeout=25):
    """Does a HTTP request with method `method` to `url`.

    Returns (status, headers, content) whereas `status` is an integer status code, `headers` is a dict
    containing the headers sent by the server and `content` is the body of the http response.

    Parameters to fetch are::

    * `url` is the fully qualified request URL. It may contain query prameters.
    * `content` is the request body to be sent. It may be a dict which for all requests expect POST
      is converted to query parameters. If there are query parameters already in the `url` they are merged
      with `content`. For POST requests the data is encoded as application/x-www-form-urlencoded
      or multipart/form-data and encoded. If the parameter `multipart` is `True` or if one of the values
      in content has a `name` attribute (which is the case for file objects) multipart encoding is choosen.
    * `headers` is a dict of header values
    * `credentials` can be a user:password combination
    * `ua` should be an additional User Agent string
    * `timeout` is the maximum number of seconds the request might take. This is advisory and may not be
       enforced.
    """

    myheaders = {'Accept-Encoding': 'gzip;q=1.0, *;q=0',
                 'User-Agent': '%s/huTools.http (gzip)' % ua}
    if headers:
        myheaders.update(headers)
    if method == 'POST':
        if hasattr(content, 'items'):
            # we assume content is a dict which needs to be encoded
            # decide to use multipart/form-data encoding or application/x-www-form-urlencoded
            for val in content.values():
                if hasattr(val, 'read'):  # file() or StringIO()
                    multipart = True
            if multipart:
                datagen, mp_headers = poster_encode.multipart_encode(content)
                myheaders.update(mp_headers)
                content = "".join(datagen)
            else:
                myheaders.update({'Content-Type': 'application/x-www-form-urlencoded'})
                content = tools.urlencode(content)
    else:
        # url parmater encoding
        if hasattr(content, 'items'):
            scheme, netloc, path, params, query, fragment = urlparse.urlparse(url)
            qdict = cgi.parse_qs(query)
            # ugly Unicode issues, see http://bugs.python.org/issue1712522
            qdict.update(content)
            query = tools.urlencode(qdict)
            url = urlparse.urlunparse((scheme, netloc, path, params, query, fragment))
            content = ''
    # convert all header values to strings (what about unicode?)
    for key, val in myheaders.items():
        myheaders[key] = str(val)
    # add authentication
    if credentials and not 'Authorization' in myheaders.keys():
        authheader = "Basic %s" % credentials.encode('base64').strip()
        myheaders["Authorization"] = authheader

    return request(url, method, content, myheaders, timeout)
Пример #2
0
    def get_query(self, task):
        """Return current Task Status or redirect to the result."""
        # Statustext - Generic Variant.
        display = dict(info='Status: %s' % task.status)

        if task.status == 'done':
            # We are done!
            display['info'] = 'Fertig!'
            # Redirect to result page
            parameters = urlencode([('_longtaskjob', 'showresult'),
                                    ('_longtaskid', task.key()),
                                    ('_longtaskstartingpoint', self.request.url)]
                                    # original Parameters
                                   + [(name, self.request.get(name)) for name in self.request.arguments()
                                      if not name.startswith('_')])
            raise HTTP307_TemporaryRedirect(location=self.request.path + '?' + parameters)

        if task.status == 'ready':
            display['info'] = 'Warte auf Start.'
        if task.status == 'error':
            display['info'] = u'Fehler, wird automatisch erneut versucht.'
        if task.status == 'started':
            # We are running. Read current Status Message from memcache and prepare for display
            display['statusinfo'] = memcache.get("longtask_status_%s" % task.key())
            if not display['statusinfo']:
                display['statusinfo'] = {}
            display['info'] = u'%s<br>Läuft seit %d Sekunden.' % (display['statusinfo'].get('message'),
                                                                  time.time() - task.starttime)
            # Display Progress Bar if sufficient data is available
            # TODO: "AJAX" based progress display
            if display['statusinfo'].get('total_steps'):
                display['info'] = display['info'] + u"""<p>Fortgang:
  <progress value="%d" max="%d">%d %%</progress></p>
""" % (display['statusinfo'].get('step'),
      display['statusinfo'].get('total_steps'),
      int(display['statusinfo'].get('step') * 100.0 / display['statusinfo'].get('total_steps')))
            else:
                # indetermine progress bar
                display['info'] = display['info'] + u"<p><progress></progress></p>"

        # Generate HTML output
        html = u"""<html><head><meta http-equiv="refresh" content="3"><title>Task Status</title></head>
<body><p>%(info)s</p>
</p></body></html>""" % display
        self.response.write(html)
Пример #3
0
    def get_start(self, *args, **kwargs):
        """Prepare a task in the datasotre an fire a task queue."""

        # Start a new task
        self.login_required()
        paramters = self.prepare_task(*args, **kwargs)
        task = gaetk_LongTask(parameters_blob=pickle.dumps(paramters), path=self.request.path, status='ready')
        task.put()
        logging.info("starting %s", task)
        self.task = task
        # Start TaskQueue
        taskqueue.add(url=self.request.path, method='GET',
                  params={'_longtaskjob': 'execute', '_longtaskid': task.key()})
        self.log_progress("Starting", step=0)
        # Redirect to status page
        parameters = urlencode([('_longtaskjob', 'query'),
                                ('_longtaskid', task.key()),
                                # Original URL to restart the Task
                                ('_longtaskstartingpoint', self.request.url)]
                                # original Parameters
                               + [(name, self.request.get(name)) for name in self.request.arguments()
                                  if not name.startswith('_')])
        raise HTTP307_TemporaryRedirect(location=self.request.path + '?' + parameters)
Пример #4
0
def fetch(url,
          content='',
          method='GET',
          credentials=None,
          headers=None,
          multipart=False,
          ua='',
          timeout=25):
    """Does a HTTP request with method `method` to `url`.

    Returns (status, headers, content) whereas `status` is an integer status code, `headers` is a dict
    containing the headers sent by the server and `content` is the body of the http response.

    Parameters to fetch are::

    * `url` is the fully qualified request URL. It may contain query prameters.
    * `content` is the request body to be sent. It may be a dict which for all requests expect POST
      is converted to query parameters. If there are query parameters already in the `url` they are merged
      with `content`. For POST requests the data is encoded as application/x-www-form-urlencoded
      or multipart/form-data and encoded. If the parameter `multipart` is `True` or if one of the values
      in content has a `name` attribute (which is the case for file objects) multipart encoding is choosen.
    * `headers` is a dict of header values
    * `credentials` can be a user:password combination
    * `ua` should be an additional User Agent string
    * `timeout` is the maximum number of seconds the request might take. This is advisory and may not be
       enforced.
    """

    myheaders = {
        # Für [LH#1003] als Hotfix erstmal abgeschaltet
        # urlfetch.fetch erkennt den Response Header "content-encoding: gzip" scheinbar nicht
        #'Accept-Encoding': 'gzip;q=1.0, *;q=0',
        'User-Agent': '%s/huTools.http (gzip)' % ua
    }
    if headers:
        myheaders.update(headers)
    if method == 'POST':
        if hasattr(content, 'items'):
            # we assume content is a dict which needs to be encoded
            # decide to use multipart/form-data encoding or application/x-www-form-urlencoded
            for val in content.values():
                if hasattr(val, 'read'):  # file() or StringIO()
                    multipart = True
            if multipart:
                datagen, mp_headers = poster_encode.multipart_encode(content)
                myheaders.update(mp_headers)
                content = "".join(datagen)
            else:
                myheaders.update(
                    {'Content-Type': 'application/x-www-form-urlencoded'})
                content = tools.urlencode(content)
    else:
        # url parmater encoding
        if hasattr(content, 'items'):
            scheme, netloc, path, params, query, fragment = urlparse.urlparse(
                url)
            qdict = cgi.parse_qs(query)
            # ugly Unicode issues, see http://bugs.python.org/issue1712522
            qdict.update(content)
            query = tools.urlencode(qdict)
            url = urlparse.urlunparse(
                (scheme, netloc, path, params, query, fragment))
            content = ''
    # convert all header values to strings (what about unicode?)
    for key, val in myheaders.items():
        myheaders[key] = str(val)
    # add authentication
    if credentials and not 'Authorization' in myheaders.keys():
        authheader = "Basic %s" % credentials.encode('base64').strip()
        myheaders["Authorization"] = authheader

    return request(url, method, content, myheaders, timeout)