def fetch(url, content='', method='GET', credentials=None, headers=None, multipart=False, ua='', timeout=25):
"""Does a HTTP request with method `method` to `url`.
Returns (status, headers, content) whereas `status` is an integer status code, `headers` is a dict
containing the headers sent by the server and `content` is the body of the http response.
Parameters to fetch are::
* `url` is the fully qualified request URL. It may contain query prameters.
* `content` is the request body to be sent. It may be a dict which for all requests expect POST
is converted to query parameters. If there are query parameters already in the `url` they are merged
with `content`. For POST requests the data is encoded as application/x-www-form-urlencoded
or multipart/form-data and encoded. If the parameter `multipart` is `True` or if one of the values
in content has a `name` attribute (which is the case for file objects) multipart encoding is choosen.
* `headers` is a dict of header values
* `credentials` can be a user:password combination
* `ua` should be an additional User Agent string
* `timeout` is the maximum number of seconds the request might take. This is advisory and may not be
enforced.
"""
myheaders = {'Accept-Encoding': 'gzip;q=1.0, *;q=0',
'User-Agent': '%s/huTools.http (gzip)' % ua}
if headers:
myheaders.update(headers)
if method == 'POST':
if hasattr(content, 'items'):
# we assume content is a dict which needs to be encoded
# decide to use multipart/form-data encoding or application/x-www-form-urlencoded
for val in content.values():
if hasattr(val, 'read'): # file() or StringIO()
multipart = True
if multipart:
datagen, mp_headers = poster_encode.multipart_encode(content)
myheaders.update(mp_headers)
content = "".join(datagen)
else:
myheaders.update({'Content-Type': 'application/x-www-form-urlencoded'})
content = tools.urlencode(content)
else:
# url parmater encoding
if hasattr(content, 'items'):
scheme, netloc, path, params, query, fragment = urlparse.urlparse(url)
qdict = cgi.parse_qs(query)
# ugly Unicode issues, see http://bugs.python.org/issue1712522
qdict.update(content)
query = tools.urlencode(qdict)
url = urlparse.urlunparse((scheme, netloc, path, params, query, fragment))
content = ''
# convert all header values to strings (what about unicode?)
for key, val in myheaders.items():
myheaders[key] = str(val)
# add authentication
if credentials and not 'Authorization' in myheaders.keys():
authheader = "Basic %s" % credentials.encode('base64').strip()
myheaders["Authorization"] = authheader
return request(url, method, content, myheaders, timeout)
def get_query(self, task):
"""Return current Task Status or redirect to the result."""
# Statustext - Generic Variant.
display = dict(info='Status: %s' % task.status)
if task.status == 'done':
# We are done!
display['info'] = 'Fertig!'
# Redirect to result page
parameters = urlencode([('_longtaskjob', 'showresult'),
('_longtaskid', task.key()),
('_longtaskstartingpoint', self.request.url)]
# original Parameters
+ [(name, self.request.get(name)) for name in self.request.arguments()
if not name.startswith('_')])
raise HTTP307_TemporaryRedirect(location=self.request.path + '?' + parameters)
if task.status == 'ready':
display['info'] = 'Warte auf Start.'
if task.status == 'error':
display['info'] = u'Fehler, wird automatisch erneut versucht.'
if task.status == 'started':
# We are running. Read current Status Message from memcache and prepare for display
display['statusinfo'] = memcache.get("longtask_status_%s" % task.key())
if not display['statusinfo']:
display['statusinfo'] = {}
display['info'] = u'%s<br>Läuft seit %d Sekunden.' % (display['statusinfo'].get('message'),
time.time() - task.starttime)
# Display Progress Bar if sufficient data is available
# TODO: "AJAX" based progress display
if display['statusinfo'].get('total_steps'):
display['info'] = display['info'] + u"""<p>Fortgang:
<progress value="%d" max="%d">%d %%</progress></p>
""" % (display['statusinfo'].get('step'),
display['statusinfo'].get('total_steps'),
int(display['statusinfo'].get('step') * 100.0 / display['statusinfo'].get('total_steps')))
else:
# indetermine progress bar
display['info'] = display['info'] + u"<p><progress></progress></p>"
# Generate HTML output
html = u"""<html><head><meta http-equiv="refresh" content="3"><title>Task Status</title></head>
<body><p>%(info)s</p>
</p></body></html>""" % display
self.response.write(html)
def get_start(self, *args, **kwargs):
"""Prepare a task in the datasotre an fire a task queue."""
# Start a new task
self.login_required()
paramters = self.prepare_task(*args, **kwargs)
task = gaetk_LongTask(parameters_blob=pickle.dumps(paramters), path=self.request.path, status='ready')
task.put()
logging.info("starting %s", task)
self.task = task
# Start TaskQueue
taskqueue.add(url=self.request.path, method='GET',
params={'_longtaskjob': 'execute', '_longtaskid': task.key()})
self.log_progress("Starting", step=0)
# Redirect to status page
parameters = urlencode([('_longtaskjob', 'query'),
('_longtaskid', task.key()),
# Original URL to restart the Task
('_longtaskstartingpoint', self.request.url)]
# original Parameters
+ [(name, self.request.get(name)) for name in self.request.arguments()
if not name.startswith('_')])
raise HTTP307_TemporaryRedirect(location=self.request.path + '?' + parameters)
def fetch(url,
content='',
method='GET',
credentials=None,
headers=None,
multipart=False,
ua='',
timeout=25):
"""Does a HTTP request with method `method` to `url`.
Returns (status, headers, content) whereas `status` is an integer status code, `headers` is a dict
containing the headers sent by the server and `content` is the body of the http response.
Parameters to fetch are::
* `url` is the fully qualified request URL. It may contain query prameters.
* `content` is the request body to be sent. It may be a dict which for all requests expect POST
is converted to query parameters. If there are query parameters already in the `url` they are merged
with `content`. For POST requests the data is encoded as application/x-www-form-urlencoded
or multipart/form-data and encoded. If the parameter `multipart` is `True` or if one of the values
in content has a `name` attribute (which is the case for file objects) multipart encoding is choosen.
* `headers` is a dict of header values
* `credentials` can be a user:password combination
* `ua` should be an additional User Agent string
* `timeout` is the maximum number of seconds the request might take. This is advisory and may not be
enforced.
"""
myheaders = {
# Für [LH#1003] als Hotfix erstmal abgeschaltet
# urlfetch.fetch erkennt den Response Header "content-encoding: gzip" scheinbar nicht
#'Accept-Encoding': 'gzip;q=1.0, *;q=0',
'User-Agent': '%s/huTools.http (gzip)' % ua
}
if headers:
myheaders.update(headers)
if method == 'POST':
if hasattr(content, 'items'):
# we assume content is a dict which needs to be encoded
# decide to use multipart/form-data encoding or application/x-www-form-urlencoded
for val in content.values():
if hasattr(val, 'read'): # file() or StringIO()
multipart = True
if multipart:
datagen, mp_headers = poster_encode.multipart_encode(content)
myheaders.update(mp_headers)
content = "".join(datagen)
else:
myheaders.update(
{'Content-Type': 'application/x-www-form-urlencoded'})
content = tools.urlencode(content)
else:
# url parmater encoding
if hasattr(content, 'items'):
scheme, netloc, path, params, query, fragment = urlparse.urlparse(
url)
qdict = cgi.parse_qs(query)
# ugly Unicode issues, see http://bugs.python.org/issue1712522
qdict.update(content)
query = tools.urlencode(qdict)
url = urlparse.urlunparse(
(scheme, netloc, path, params, query, fragment))
content = ''
# convert all header values to strings (what about unicode?)
for key, val in myheaders.items():
myheaders[key] = str(val)
# add authentication
if credentials and not 'Authorization' in myheaders.keys():
authheader = "Basic %s" % credentials.encode('base64').strip()
myheaders["Authorization"] = authheader
return request(url, method, content, myheaders, timeout)
