def getdata_pycurl(url, params, headers=None, expire=3600, post=None, error_expire=300, verbose=0, ckey=None, cert=None, doseq=True, system=None): "Fetch data via pycurl library" contact = 'data-service.' if system: contact = system + ' ' + contact if isinstance(params, dict): timer_key = '%s?%s' % (url, urllib.urlencode(params, doseq=True)) else: timer_key = '%s?%s' % (url, params) das_timer(timer_key, verbose) handler = REQUEST_HANDLER try: data, expire = handler.getdata(url, params, headers, expire, post, \ error_expire, verbose, ckey, cert, doseq) except urllib2.HTTPError as httperror: msg = 'urllib2.HTTPError, system=%s, url=%s, args=%s, headers=%s' \ % (system, url, json.dumps(params), json.dumps(headers)) data = { 'error': 'Received HTTP error from %s data-service' % contact, 'reason': msg, 'ts': time.time() } try: reason = extract_http_error(httperror.read()) data.update({'reason': reason, 'request': msg}) # TODO: err variable did not exit in this function! msg += '\n' + reason except Exception as exp: data.update({'httperror': None}) msg += '\n' + str(exp) print(dastimestamp('getdata_pycurl'), msg) data = json.dumps(data) expire = expire_timestamp(error_expire) except Exception as exp: msg = 'HTTPError, system=%s, url=%s, args=%s, headers=%s' \ % (system, url, json.dumps(params), json.dumps(headers)) print(dastimestamp('getdata_pycurl'), msg + '\n' + str(exp)) data = { 'error': 'Received generic error from %s data-service' % contact, 'reason': msg, 'ts': time.time() } data = json.dumps(data) expire = expire_timestamp(error_expire) das_timer(timer_key, verbose) return data, expire
def getdata_pycurl( url, params, headers=None, expire=3600, post=None, error_expire=300, verbose=0, ckey=None, cert=None, doseq=True, system=None, ): "Fetch data via pycurl library" contact = "data-service." if system: contact = system + " " + contact timer_key = "%s?%s" % (url, urllib.urlencode(params, doseq=True)) das_timer(timer_key, verbose) handler = REQUEST_HANDLER try: data, expire = handler.getdata(url, params, headers, expire, post, error_expire, verbose, ckey, cert, doseq) except urllib2.HTTPError as httperror: msg = "urllib2.HTTPError, system=%s, url=%s, args=%s, headers=%s" % ( system, url, json.dumps(params), json.dumps(headers), ) data = {"error": "Unable to contact %s" % contact, "reason": msg, "ts": time.time()} try: reason = extract_http_error(httperror.read()) data.update({"reason": reason, "request": msg}) msg += "\n" + err except Exception as exp: data.update({"httperror": None}) msg += "\n" + str(exp) print dastimestamp("getdata_pycurl"), msg data = json.dumps(data) expire = expire_timestamp(error_expire) except Exception as exp: msg = "HTTPError, system=%s, url=%s, args=%s, headers=%s" % ( system, url, json.dumps(params), json.dumps(headers), ) print dastimestamp("getdata_pycurl"), msg + "\n" + str(exp) data = {"error": "Unable to contact %s" % contact, "reason": msg, "ts": time.time()} data = json.dumps(data) expire = expire_timestamp(error_expire) das_timer(timer_key, verbose) return data, expire
def getdata_pycurl(url, params, headers=None, expire=3600, post=None, error_expire=300, verbose=0, ckey=None, cert=None, doseq=True, system=None): "Fetch data via pycurl library" contact = 'data-service.' if system: contact = system + ' ' + contact if isinstance(params, dict): timer_key = '%s?%s' % (url, urllib.urlencode(params, doseq=True)) else: timer_key = '%s?%s' % (url, params) das_timer(timer_key, verbose) handler = REQUEST_HANDLER try: data, expire = handler.getdata(url, params, headers, expire, post, \ error_expire, verbose, ckey, cert, doseq) except urllib2.HTTPError as httperror: msg = 'urllib2.HTTPError, system=%s, url=%s, args=%s, headers=%s' \ % (system, url, json.dumps(params), json.dumps(headers)) data = {'error': 'Received HTTP error from %s data-service' % contact, 'reason': msg, 'ts':time.time()} try: reason = extract_http_error(httperror.read()) data.update({'reason': reason, 'request': msg}) # TODO: err variable did not exit in this function! msg += '\n' + reason except Exception as exp: data.update({'httperror': None}) msg += '\n' + str(exp) print(dastimestamp('getdata_pycurl'), msg) data = json.dumps(data) expire = expire_timestamp(error_expire) except Exception as exp: msg = 'HTTPError, system=%s, url=%s, args=%s, headers=%s' \ % (system, url, json.dumps(params), json.dumps(headers)) print(dastimestamp('getdata_pycurl'), msg + '\n' + str(exp)) data = {'error': 'Received generic error from %s data-service' % contact, 'reason': msg, 'ts':time.time()} data = json.dumps(data) expire = expire_timestamp(error_expire) das_timer(timer_key, verbose) return data, expire
def getdata_urllib(url, params, headers=None, expire=3600, post=None, error_expire=300, verbose=0, ckey=None, cert=None, doseq=True, system=None, tstamp=None): """ Invoke URL call and retrieve data from data-service based on provided URL and set of parameters. Use post=True to invoke POST request. """ contact = 'data-service.' if system: contact = system + ' ' + contact timer_key = '%s?%s' % (url, urllib.urlencode(params, doseq=True)) das_timer(timer_key, verbose) encoded_data = urllib.urlencode(params, doseq=doseq) if not post: url = url + '?' + encoded_data if not headers: headers = {} if tstamp and 'If-Modified-Since' not in headers.keys(): headers['If-Modified-Since'] = http_timestamp(tstamp) if verbose: print('+++ getdata, url=%s, headers=%s' % (url, headers)) req = urllib2.Request(url) for key, val in headers.items(): req.add_header(key, val) if verbose > 1: handler = urllib2.HTTPHandler(debuglevel=1) opener = urllib2.build_opener(handler) urllib2.install_opener(opener) if ckey and cert: handler = HTTPSClientAuthHandler(ckey, cert, verbose) opener = urllib2.build_opener(handler) urllib2.install_opener(opener) try: time0 = time.time() if post: data = urllib2.urlopen(req, encoded_data) else: data = urllib2.urlopen(req) data_srv_time = time.time() - time0 info = data.info() code = data.getcode() if verbose > 1: print("+++ response code:", code) print("+++ response info\n", info) try: # get HTTP header and look for Expires e_time = expire_timestamp(\ info.__dict__['dict']['expires']) if e_time < expire_timestamp(data_srv_time): expire = max(e_time, expire_timestamp(expire)) elif e_time > time.time(): expire = e_time except Exception as _exp: pass except urllib2.HTTPError as httperror: msg = 'HTTPError, url=%s, args=%s, headers=%s' \ % (url, params, headers) data = { 'error': 'Received HTTP error from %s data-service' % contact, 'reason': msg } try: err = '%s %s' % (contact, extract_http_error(httperror.read())) data.update({'error': err}) msg += '\n' + err except Exception as exp: data.update({'httperror': None}) msg += '\n' + str(exp) print(msg) data = json.dumps(data) expire = expire_timestamp(error_expire) except Exception as exp: msg = 'HTTPError, url=%s, args=%s, headers=%s' \ % (url, params, headers) print(msg + '\n' + str(exp)) data = { 'error': 'Received generic error from %s data-service' % contact, 'reason': msg } data = json.dumps(data) expire = expire_timestamp(error_expire) das_timer(timer_key, verbose) return data, expire
def getdata_urllib(url, params, headers=None, expire=3600, post=None, error_expire=300, verbose=0, ckey=None, cert=None, doseq=True, system=None, tstamp=None): """ Invoke URL call and retrieve data from data-service based on provided URL and set of parameters. Use post=True to invoke POST request. """ contact = 'data-service.' if system: contact = system + ' ' + contact timer_key = '%s?%s' % (url, urllib.urlencode(params, doseq=True)) das_timer(timer_key, verbose) encoded_data = urllib.urlencode(params, doseq=doseq) if not post: url = url + '?' + encoded_data if not headers: headers = {} if tstamp and 'If-Modified-Since' not in headers.keys(): headers['If-Modified-Since'] = http_timestamp(tstamp) if verbose: print('+++ getdata, url=%s, headers=%s' % (url, headers)) req = urllib2.Request(url) for key, val in headers.items(): req.add_header(key, val) if verbose > 1: handler = urllib2.HTTPHandler(debuglevel=1) opener = urllib2.build_opener(handler) urllib2.install_opener(opener) if ckey and cert: handler = HTTPSClientAuthHandler(ckey, cert, verbose) opener = urllib2.build_opener(handler) urllib2.install_opener(opener) try: time0 = time.time() if post: data = urllib2.urlopen(req, encoded_data) else: data = urllib2.urlopen(req) data_srv_time = time.time()-time0 info = data.info() code = data.getcode() if verbose > 1: print("+++ response code:", code) print("+++ response info\n", info) try: # get HTTP header and look for Expires e_time = expire_timestamp(\ info.__dict__['dict']['expires']) if e_time < expire_timestamp(data_srv_time): expire = max(e_time, expire_timestamp(expire)) elif e_time > time.time(): expire = e_time except Exception as _exp: pass except urllib2.HTTPError as httperror: msg = 'HTTPError, url=%s, args=%s, headers=%s' \ % (url, params, headers) data = {'error': 'Received HTTP error from %s data-service' % contact, 'reason': msg} try: err = '%s %s' % (contact, extract_http_error(httperror.read())) data.update({'error':err}) msg += '\n' + err except Exception as exp: data.update({'httperror': None}) msg += '\n' + str(exp) print(msg) data = json.dumps(data) expire = expire_timestamp(error_expire) except Exception as exp: msg = 'HTTPError, url=%s, args=%s, headers=%s' \ % (url, params, headers) print(msg + '\n' + str(exp)) data = {'error': 'Received generic error from %s data-service' % contact, 'reason': msg} data = json.dumps(data) expire = expire_timestamp(error_expire) das_timer(timer_key, verbose) return data, expire
def getdata_urllib( url, params, headers=None, expire=3600, post=None, error_expire=300, verbose=0, ckey=None, cert=None, doseq=True, system=None, ): """ Invoke URL call and retrieve data from data-service based on provided URL and set of parameters. Use post=True to invoke POST request. """ contact = "data-service." if system: contact = system + " " + contact timer_key = "%s?%s" % (url, urllib.urlencode(params, doseq=True)) das_timer(timer_key, verbose) encoded_data = urllib.urlencode(params, doseq=doseq) if not post: url = url + "?" + encoded_data if not headers: headers = {} if verbose: print "+++ getdata, url=%s, headers=%s" % (url, headers) req = urllib2.Request(url) for key, val in headers.iteritems(): req.add_header(key, val) if verbose > 1: handler = urllib2.HTTPHandler(debuglevel=1) opener = urllib2.build_opener(handler) urllib2.install_opener(opener) if ckey and cert: handler = HTTPSClientAuthHandler(ckey, cert, verbose) opener = urllib2.build_opener(handler) urllib2.install_opener(opener) try: time0 = time.time() if post: data = urllib2.urlopen(req, encoded_data) else: data = urllib2.urlopen(req) data_srv_time = time.time() - time0 try: # get HTTP header and look for Expires e_time = expire_timestamp(data.info().__dict__["dict"]["expires"]) if e_time < expire_timestamp(data_srv_time): expire = max(e_time, expire_timestamp(expire)) elif e_time > time.time(): expire = e_time except Exception as _exp: pass except urllib2.HTTPError as httperror: msg = "HTTPError, url=%s, args=%s, headers=%s" % (url, params, headers) data = {"error": "Unable to contact %s" % contact, "reason": msg} try: err = "%s %s" % (contact, extract_http_error(httperror.read())) data.update({"error": err}) msg += "\n" + err except Exception as exp: data.update({"httperror": None}) msg += "\n" + str(exp) print msg data = json.dumps(data) expire = expire_timestamp(error_expire) except Exception as exp: msg = "HTTPError, url=%s, args=%s, headers=%s" % (url, params, headers) print msg + "\n" + str(exp) data = {"error": "Unable to contact %s" % contact, "reason": msg} data = json.dumps(data) expire = expire_timestamp(error_expire) das_timer(timer_key, verbose) return data, expire