def _request(cls, url, post_data=None, timeout=REQUEST_TIMEOUT, attempts=REQUEST_ATTEMPTS): ''' request the given url and parse it as json urllib2 raises errors on different status codes so use a try except ''' logger.info('requesting url %s with post data %s', url, post_data) post_request = (post_data is not None or 'method=post' in url) if post_request and facebook_settings.FACEBOOK_READ_ONLY: response = dict(id=123456789) return response opener = urllib2.build_opener() opener.addheaders = [('User-agent', 'Open Facebook Python')] # give it a few shots, connection is buggy at times path = url.split('?', 1)[0].rsplit('/', 1)[-1].replace('.', '_') while attempts: response_file = None encoded_params = encode_params(post_data) if post_data else None post_string = (urllib.urlencode(encoded_params) if post_data else None) try: if django_statsd: django_statsd.start('facebook.%s' % path) try: # For older python versions you could leave out the timeout # response_file = opener.open(url, post_string) response_file = opener.open(url, post_string, timeout=timeout) except (urllib2.HTTPError, ), e: # Facebook sents error codes for many of their flows # we still want the json to allow for proper handling logger.warn('FB request, error type %s, code %s', type(e), getattr(e, 'code', None)) if hasattr(e, 'code') and e.code == 500: raise urllib2.URLError('Facebook is down') if 'http error' in str(e).lower(): response_file = e else: raise response = response_file.read().decode('utf8') break except (urllib2.HTTPError, urllib2.URLError, ssl.SSLError), e: #These are often temporary errors, so we will retry before failing error_format = 'Facebook encountered a timeout or error %s' logger.warn(error_format, unicode(e)) attempts -= 1 if not attempts: #if we have no more attempts actually raise the error raise facebook_exceptions.convert_unreachable_exception(e)
def _request(cls, url, post_data=None, timeout=REQUEST_TIMEOUT, attempts=REQUEST_ATTEMPTS): ''' request the given url and parse it as json urllib2 raises errors on different status codes so use a try except ''' logger.info('requesting url %s with post data %s', url, post_data) post_request = (post_data is not None or 'method=post' in url) if post_request and facebook_settings.FACEBOOK_READ_ONLY: logger.info('running in readonly mode') response = dict(id=123456789, setting_read_only=True) return response opener = urllib2.build_opener() opener.addheaders = [('User-agent', 'Open Facebook Python')] # give it a few shots, connection is buggy at times path = url.split('?', 1)[0].rsplit('/', 1)[-1].replace('.', '_') while attempts: response_file = None encoded_params = encode_params(post_data) if post_data else None post_string = (urllib.urlencode(encoded_params) if post_data else None) try: if django_statsd: django_statsd.start('facebook.%s' % path) try: # For older python versions you could leave out the timeout # response_file = opener.open(url, post_string) response_file = opener.open(url, post_string, timeout=timeout) except (urllib2.HTTPError,), e: # Facebook sents error codes for many of their flows # we still want the json to allow for proper handling logger.warn('FB request, error type %s, code %s', type(e), getattr(e, 'code', None)) if hasattr(e, 'code') and e.code == 500: raise urllib2.URLError('Facebook is down') if 'http error' in str(e).lower(): response_file = e else: raise response = response_file.read().decode('utf8') break except (urllib2.HTTPError, urllib2.URLError, ssl.SSLError), e: #These are often temporary errors, so we will retry before failing error_format = 'Facebook encountered a timeout or error %s' logger.warn(error_format, unicode(e)) attempts -= 1 if not attempts: #if we have no more attempts actually raise the error raise facebook_exceptions.convert_unreachable_exception(e)
def _request(cls, url, post_data=None, timeout=REQUEST_TIMEOUT, attempts=REQUEST_ATTEMPTS): """ request the given url and parse it as json urllib2 raises errors on different status codes so use a try except """ logger.info("requesting url %s with post data %s", url, post_data) post_request = post_data is not None or "method=post" in url if post_request and facebook_settings.FACEBOOK_READ_ONLY: response = dict(id=123456789) return response opener = urllib2.build_opener() opener.addheaders = [("User-agent", "Open Facebook Python")] # give it a few shots, connection is buggy at times path = url.split("?", 1)[0].rsplit("/", 1)[-1].replace(".", "_") while attempts: response_file = None encoded_params = encode_params(post_data) if post_data else None post_string = urllib.urlencode(encoded_params) if post_data else None try: if django_statsd: django_statsd.start("facebook.%s" % path) try: # For older python versions you could leave out the timeout # response_file = opener.open(url, post_string) response_file = opener.open(url, post_string, timeout=timeout) except (urllib2.HTTPError,), e: # Facebook sents error codes for many of their flows # we still want the json to allow for proper handling if hasattr(e, "code") and e.code == 500: raise urllib2.URLError("Facebook is down") if "http error" in str(e).lower(): response_file = e else: raise response = response_file.read().decode("utf8") break except (urllib2.HTTPError, urllib2.URLError, ssl.SSLError), e: # These are often temporary errors, so we will retry before failing error_format = "Facebook encountered a timeout or error %s" logger.warn(error_format, unicode(e)) attempts -= 1 if not attempts: # if we have no more attempts actually raise the error raise facebook_exceptions.convert_unreachable_exception(e)
def _request(cls, url, post_data=None, timeout=REQUEST_TIMEOUT, attempts=REQUEST_ATTEMPTS): ''' request the given url and parse it as json urllib2 raises errors on different status codes so use a try except ''' # change fb__explicitly_shared to fb:explicitly_shared if post_data: post_data = dict( (k.replace('__', ':'), v) for k, v in post_data.items()) logger.info('requesting url %s with post data %s', url, post_data) post_request = (post_data is not None or 'method=post' in url) if post_request and facebook_settings.FACEBOOK_READ_ONLY: logger.info('running in readonly mode') response = dict(id=123456789, setting_read_only=True) return response # nicely identify ourselves before sending the request opener = urllib2.build_opener() opener.addheaders = [('User-agent', 'Open Facebook Python')] # get the statsd path to track response times with path = urlparse(url).path statsd_path = path.replace('.', '_') # give it a few shots, connection is buggy at times while attempts: response_file = None encoded_params = encode_params(post_data) if post_data else None post_string = (urllib.urlencode(encoded_params) if post_data else None) try: start_statsd('facebook.%s' % statsd_path) try: response_file = opener.open( url, post_string, timeout=timeout) response = response_file.read().decode('utf8') except (urllib2.HTTPError,), e: response_file = e response = response_file.read().decode('utf8') # Facebook sents error codes for many of their flows # we still want the json to allow for proper handling msg_format = 'FB request, error type %s, code %s' logger.warn(msg_format, type(e), getattr(e, 'code', None)) # detect if its a server or application error server_error = cls.is_server_error(e, response) if server_error: # trigger a retry raise urllib2.URLError( 'Facebook is down %s' % response) break except (urllib2.HTTPError, urllib2.URLError, ssl.SSLError), e: # These are often temporary errors, so we will retry before failing error_format = 'Facebook encountered a timeout or error %s' logger.warn(error_format, unicode(e)) attempts -= 1 if not attempts: # if we have no more attempts actually raise the error raise facebook_exceptions.convert_unreachable_exception(e)
def _request(cls, url, post_data=None, timeout=REQUEST_TIMEOUT, attempts=REQUEST_ATTEMPTS): # change fb__explicitly_shared to fb:explicitly_shared if post_data: post_data = dict( (k.replace('__', ':'), v) for k, v in post_data.items()) logger.info('requesting url %s with post data %s', url, post_data) post_request = (post_data is not None or 'method=post' in url) if post_request and facebook_settings.FACEBOOK_READ_ONLY: logger.info('running in readonly mode') response = dict(id=123456789, setting_read_only=True) return response # nicely identify ourselves before sending the request opener = build_opener() opener.addheaders = [('User-agent', 'Open Facebook Python')] # get the statsd path to track response times with path = urlparse(url).path statsd_path = path.replace('.', '_') # give it a few shots, connection is buggy at times timeout_mp = 0 while attempts: # gradually increase the timeout upon failure timeout_mp += 1 extended_timeout = timeout * timeout_mp response_file = None encoded_params = encode_params(post_data) if post_data else None post_string = (urlencode(encoded_params) if post_data else None) try: start_statsd('facebook.%s' % statsd_path) try: response_file = opener.open( url, post_string, timeout=extended_timeout) response = response_file.read().decode('utf8') except (HTTPError,) as e: response_file = e response = response_file.read().decode('utf8') # Facebook sents error codes for many of their flows # we still want the json to allow for proper handling msg_format = 'FB request, error type %s, code %s' logger.warn(msg_format, type(e), getattr(e, 'code', None)) # detect if its a server or application error server_error = cls.is_server_error(e, response) if server_error: # trigger a retry raise URLError( 'Facebook is down %s' % response) break except (HTTPError, URLError, ssl.SSLError) as e: # These are often temporary errors, so we will retry before # failing error_format = 'Facebook encountered a timeout (%ss) or error %s' logger.warn(error_format, extended_timeout, unicode(e)) attempts -= 1 if not attempts: # if we have no more attempts actually raise the error error_instance = facebook_exceptions.convert_unreachable_exception( e) error_msg = 'Facebook request failed after several retries, raising error %s' logger.warn(error_msg, error_instance) raise error_instance finally: if response_file: response_file.close() stop_statsd('facebook.%s' % statsd_path) # Faceboook response is either # Valid json # A string which is a querydict (a=b&c=d...etc) # A html page stating FB is having trouble (but that shouldnt reach # this part of the code) try: parsed_response = json.loads(response) logger.info('facebook send response %s' % parsed_response) except Exception as e: # using exception because we need to support multiple json libs :S parsed_response = QueryDict(response, True) logger.info('facebook send response %s' % parsed_response) if parsed_response and isinstance(parsed_response, dict): # of course we have two different syntaxes if parsed_response.get('error'): cls.raise_error(parsed_response['error']['type'], parsed_response['error']['message'], parsed_response['error'].get('code')) elif parsed_response.get('error_code'): cls.raise_error(parsed_response['error_code'], parsed_response['error_msg']) return parsed_response
def _request(cls, url, post_data=None, timeout=REQUEST_TIMEOUT, attempts=REQUEST_ATTEMPTS): # change fb__explicitly_shared to fb:explicitly_shared if post_data: post_data = dict( (k.replace('__', ':'), v) for k, v in post_data.items()) logger.info('requesting url %s with post data %s', url, post_data) post_request = (post_data is not None or 'method=post' in url) if post_request and facebook_settings.FACEBOOK_READ_ONLY: logger.info('running in readonly mode') response = dict(id=123456789, setting_read_only=True) return response # nicely identify ourselves before sending the request opener = build_opener() opener.addheaders = [('User-agent', 'Open Facebook Python')] # get the statsd path to track response times with path = urlparse(url).path statsd_path = path.replace('.', '_') # give it a few shots, connection is buggy at times timeout_mp = 0 while attempts: # gradually increase the timeout upon failure timeout_mp += 1 extended_timeout = timeout * timeout_mp response_file = None encoded_params = encode_params(post_data) if post_data else None post_string = (urlencode(encoded_params) if post_data else None) try: start_statsd('facebook.%s' % statsd_path) try: response_file = opener.open(url, post_string, timeout=extended_timeout) response = response_file.read().decode('utf8') except (HTTPError, ) as e: response_file = e response = response_file.read().decode('utf8') # Facebook sents error codes for many of their flows # we still want the json to allow for proper handling msg_format = 'FB request, error type %s, code %s' logger.warn(msg_format, type(e), getattr(e, 'code', None)) # detect if its a server or application error server_error = cls.is_server_error(e, response) if server_error: # trigger a retry raise URLError('Facebook is down %s' % response) break except (HTTPError, URLError, ssl.SSLError) as e: # These are often temporary errors, so we will retry before # failing error_format = 'Facebook encountered a timeout (%ss) or error %s' logger.warn(error_format, extended_timeout, str(e)) attempts -= 1 if not attempts: # if we have no more attempts actually raise the error error_instance = facebook_exceptions.convert_unreachable_exception( e) error_msg = 'Facebook request failed after several retries, raising error %s' logger.warn(error_msg, error_instance) raise error_instance finally: if response_file: response_file.close() stop_statsd('facebook.%s' % statsd_path) # Faceboook response is either # Valid json # A string which is a querydict (a=b&c=d...etc) # A html page stating FB is having trouble (but that shouldnt reach # this part of the code) try: parsed_response = json.loads(response) logger.info('facebook send response %s' % parsed_response) except Exception as e: # using exception because we need to support multiple json libs :S parsed_response = QueryDict(response, True) logger.info('facebook send response %s' % parsed_response) if parsed_response and isinstance(parsed_response, dict): # of course we have two different syntaxes if parsed_response.get('error'): cls.raise_error(parsed_response['error']['type'], parsed_response['error']['message'], parsed_response['error'].get('code')) elif parsed_response.get('error_code'): cls.raise_error(parsed_response['error_code'], parsed_response['error_msg']) return parsed_response
def _request(cls, url, post_data=None, timeout=REQUEST_TIMEOUT, attempts=REQUEST_ATTEMPTS): # change fb__explicitly_shared to fb:explicitly_shared if post_data: post_data = dict( (k.replace('__', ':'), v) for k, v in post_data.items()) logger.info('requesting url %s with post data %s', url, post_data) post_request = (post_data is not None or 'method=post' in url) if post_request and facebook_settings.FACEBOOK_READ_ONLY: logger.info('running in readonly mode') response = dict(id=123456789, setting_read_only=True) return response # nicely identify ourselves before sending the request opener = urllib2.build_opener() opener.addheaders = [('User-agent', 'Open Facebook Python')] # get the statsd path to track response times with path = urlparse(url).path statsd_path = path.replace('.', '_') # give it a few shots, connection is buggy at times timeout_mp = 0 while attempts: # gradually increase the timeout upon failure timeout_mp += 1 extended_timeout = timeout * timeout_mp response_file = None encoded_params = encode_params(post_data) if post_data else None post_string = (urllib.urlencode(encoded_params) if post_data else None) try: start_statsd('facebook.%s' % statsd_path) try: response_file = opener.open(url, post_string, timeout=extended_timeout) response = response_file.read().decode('utf8') except (urllib2.HTTPError, ), e: response_file = e response = response_file.read().decode('utf8') # Facebook sents error codes for many of their flows # we still want the json to allow for proper handling msg_format = 'FB request, error type %s, code %s' logger.warn(msg_format, type(e), getattr(e, 'code', None)) # detect if its a server or application error server_error = cls.is_server_error(e, response) if server_error: # trigger a retry raise urllib2.URLError('Facebook is down %s' % response) break except (urllib2.HTTPError, urllib2.URLError, ssl.SSLError), e: # These are often temporary errors, so we will retry before # failing error_format = 'Facebook encountered a timeout (%ss) or error %s' logger.warn(error_format, extended_timeout, unicode(e)) attempts -= 1 if not attempts: # if we have no more attempts actually raise the error error_instance = facebook_exceptions.convert_unreachable_exception( e) error_msg = 'Facebook request failed after several retries, raising error %s' logger.warn(error_msg, error_instance) raise error_instance