def main(args): parsed_url = urlparse.urlparse(args.url) if not parsed_url.scheme.startswith('http'): raise Error('Invalid URI scheme (expected http or https): %s' % args.url) # Force the format specified on command-line. qdict = {} if parsed_url.query: qdict.update(urlparse.parse_qs(parsed_url.query)) f = qdict.get('format') if f: # Load the latest format specification. f = f[-1] else: # Default to JSON. f = 'json' path = parsed_url.path if qdict: path = '%s?%s' % (path, urllib.urlencode(qdict, doseq=True)) if args.attempts < 1: args.attempts = 1 retry_delay_seconds = 2 for i in xrange(args.attempts): conn = CreateHttpConn(parsed_url.netloc, path) if f == 'json': result = ReadHttpJsonResponse(conn) elif f == 'text': # Text fetching will pack the text into structured JSON. result = ReadHttpResponse(conn).read() if result: # Wrap in a structured JSON for export to recipe module. result = { 'value': result, } else: result = None else: raise ValueError('Unnknown format: %s' % (f, )) logging.info('Reading from %s (%d/%d)', conn.req_params['url'], (i + 1), args.attempts) if result is not None or (i + 1) >= args.attempts: if not args.quiet: logging.info('Read from %s (%d/%d): %s', conn.req_params['url'], (i + 1), args.attempts, result) break logging.error("Request returned empty result; sleeping %d seconds", retry_delay_seconds) time.sleep(retry_delay_seconds) retry_delay_seconds *= 2 with open(args.json_file, 'w') as json_file: json.dump(result, json_file)
def handler(conn): return ReadHttpJsonResponse(conn, **kwargs)