def _issue_appengine_request(method, path, **kwargs): gae_url = settings.APP_ENGINE_PHOTOS_UPLOAD_BASE_PATH + path postbody = None if method == 'GET': gae_url = urlutil.appendparams(gae_url, **kwargs) else: postbody = urllib.urlencode(kwargs) gae_request = urllib2.Request(gae_url, postbody, { 'X-LK-Secret': settings.APP_ENGINE_HEADER_SECRET, }) gae_request.get_method = lambda: method try: result = urllib2.urlopen(gae_request, None, 30.0) code = 200 except urllib2.HTTPError as e: if e.code >= 500: logging.info('GAE 500 response: %s', e.code) result = e code = e.code except urllib2.URLError as e: logging.info('GAE connection problem: %s', e) return -1, None result_content = result.read() try: json_dict = json.loads(result_content) except ValueError as e: if code < 500: logging.warn('Bad JSON response from GAE: %r', result_content) json_dict = None return code, json_dict
def _make_request(method, path, token=None, params=None): headers = {} body = None if method == 'POST' and params: headers['Content-Type'] = 'application/x-www-form-urlencoded' try: body = urllib.urlencode(params) except: logging.exception('Invalid slack postbody: %r', params) return None slack_url = 'https://slack.com/api/%s' % path if token: slack_url = urlutil.appendparams(slack_url, token=token) code, headers, data = urlfetch.send_request(slack_url, method=method, body=body, headers=headers) if code != 200: logging.warn('Slack error for URL: %s -- %s %s %s', slack_url, code, headers, data) return None return data
def _lookup_fetch(remote_url): results_dict = {} retry = 0 while not (results_dict and 'results' in results_dict): if retry >= 10: return None if retry: remote_url = urlutil.appendparams(remote_url, time=time.time()) logging.info('Invalid response from lookup API: %s trying: %s', results_dict, remote_url) time.sleep(0.333 * retry) code, headers, results_dict = urlfetch.fetch(remote_url, cache_seconds=(60 * 60), should_cache_fn=lambda c, d: d and d.get('results')) if code == 403: raise RateLimitedError() retry += 1 if code != 200: return None app_infos = results_dict['results'] if not (app_infos and isinstance(app_infos, list)): return None return app_infos
def send_bundle(bundle): email_token = EmailToken(kind=EmailToken.KIND_DOWNLOAD_BUNDLE, email=bundle.user.email) email_token.save() download_url = '%sscreenshots/dashboard/%s/download/' % (settings.SITE_URL, bundle.screenshot_set.encrypted_id) download_url = urlutil.appendparams(download_url, bundle=bundle.encrypted_id, token=email_token.token) emails.send_bundle_ready_email(bundle.user, bundle.screenshot_set, download_url)
def unsubscribe_url_for_subscription(sub): base_url = '%ssales/unsubscribe/' % settings.SITE_URL token = crypto_hack.encrypt_object( { 'time': time.time(), 'sub_id': sub.encrypted_id }, settings.UNSUB_SUB_NOTIFY_SECRET) return urlutil.appendparams(base_url, token=token)
def verification_url_for_user_email(user, email): email_token = EmailToken(kind=EmailToken.KIND_VERIFY_EMAIL, email=email) email_token.save() email_verification_url = '%saccount/verify/' % settings.SITE_URL email_verification_url = urlutil.appendparams(email_verification_url, token=email_token.token) return email_verification_url
def generate_user_unsubscribe_url(user, flag_name): if flag_name and flag_name not in USER_EDITABLE_USER_FLAGS: raise ValueError('Invalid flag_name %s', flag_name) encrypted_token = crypto_hack.encrypt_object({'user_id': user.id, 'flag_name': flag_name}, settings.UNSUBSCRIBE_URL_SECRET) url = '%saccount/unsubscribe/' % settings.SITE_URL return urlutil.appendparams(url, token=encrypted_token)
def twitter_share_url(self): try: # include URL manually here return urlutil.appendparams('https://twitter.com/share', text=self.tweet_text(include_url=False), url=self.public_url, ) except UnicodeEncodeError: return None
def verification_url_for_user_email(user, email): email_token = EmailToken(kind=EmailToken.KIND_VERIFY_EMAIL, email=email) email_token.save() email_verification_url = '%saccount/verify/' % settings.SITE_URL email_verification_url = urlutil.appendparams(email_verification_url, token=email_token.token) return email_verification_url
def fetch_reviews(app, country='us', page=1): has_next_page = False fetched_reviews = None i = 0 url = REVIEWS_URL_FORMAT % { 'app_id': app.itunes_id, 'page': page, 'country': country, } while fetched_reviews is None and i < MAX_FETCH_ATTEMPTS: if i > 0: url = urlutil.appendparams(url, invalid_response_cache_bust='%0.6f' % time.time()) # Don't hammer the site when failing. time.sleep(WAIT_BETWEEN_RETRIES) i += 1 code, headers, result = urlfetch.fetch(url) if code != 200 or not result: if code == 403: raise RateLimitedError() else: continue entry = result['feed'].get('entry') if not entry: # This is either a "no reviews at all for this app on this page" case, # or a bad potentially cached result on the apple feed side. continue if not isinstance(entry, list): # This is an invalid response that sometimes happen, and it appears to be # non-recoverable. logging.info('Bad+invalid+non-recoverable response for app: %s url: %s', app.bundle_id, url) break # Successful response's first entry is a description of the app for some reason. fetched_reviews = entry[1:] last_page_link = [link['attributes']['href'] for link in result['feed']['link'] if link['attributes']['rel'] == 'last'][0] last_page_int = int(PAGE_RE.findall(last_page_link)[0]) has_next_page = last_page_int > int(page) if i > 1: if fetched_reviews is None: logging.info('FAILED to fetch reviews. %s attempts. %s page %s', i, app.bundle_id, page) else: logging.info('SUCCESS fetching reviews. %s attempts. %s page %s', i, app.bundle_id, page) if fetched_reviews: fetched_reviews = [review_from_dict(app, d, country) for d in fetched_reviews] return has_next_page, fetched_reviews
def twitter_share_url(self): try: # include URL manually here return urlutil.appendparams( 'https://twitter.com/share', text=self.tweet_text(include_url=False), url=self.public_url, ) except UnicodeEncodeError: return None
def generate_user_unsubscribe_url(user, flag_name): if flag_name and flag_name not in USER_EDITABLE_USER_FLAGS: raise ValueError('Invalid flag_name %s', flag_name) encrypted_token = crypto_hack.encrypt_object( { 'user_id': user.id, 'flag_name': flag_name }, settings.UNSUBSCRIBE_URL_SECRET) url = '%saccount/unsubscribe/' % settings.SITE_URL return urlutil.appendparams(url, token=encrypted_token)
def app_info_with_bundle_id(bundle_id, country): remote_url = urlutil.appendparams(LOOKUP_URL % country, bundleId=bundle_id) app_infos = _lookup_fetch(remote_url) if not app_infos: return None app_info = app_infos[0] if app_info.get('kind') not in ('software', 'mac-software'): return None return model_from_dict(app_info, country)
def send_bundle(bundle): email_token = EmailToken(kind=EmailToken.KIND_DOWNLOAD_BUNDLE, email=bundle.user.email) email_token.save() download_url = '%sscreenshots/dashboard/%s/download/' % ( settings.SITE_URL, bundle.screenshot_set.encrypted_id) download_url = urlutil.appendparams(download_url, bundle=bundle.encrypted_id, token=email_token.token) emails.send_bundle_ready_email(bundle.user, bundle.screenshot_set, download_url)
def request_reset_password_email(email): user = get_user_by_email(email) if not user: return False email_token = EmailToken(kind=EmailToken.KIND_RESET_PASSWORD, email=email.lower()) email_token.save() reset_password_url = '%saccount/reset/finish/' % settings.SITE_URL reset_password_url = urlutil.appendparams(reset_password_url, token=email_token.token, email=email_token.email) emails.send_reset_password_email(user, reset_password_url) return True
def related_app_infos_with_developer_id(developer_id, country): remote_url = urlutil.appendparams(LOOKUP_URL % country, id=developer_id, entity=SOFTWARE_ENTITIES) app_infos = _lookup_fetch(remote_url) if not app_infos: return [] filtered = [] app_ids = set() for app_info in app_infos: if app_info.get('kind') not in ('software', 'mac-software'): continue if app_info['trackId'] in app_ids: continue app_ids.add(app_info['trackId']) filtered.append(app_info) return [model_from_dict(app_info, country) for app_info in filtered]
def request_reset_password_email(email): user = get_user_by_email(email) if not user: return False email_token = EmailToken(kind=EmailToken.KIND_RESET_PASSWORD, email=email.lower()) email_token.save() reset_password_url = '%saccount/reset/finish/' % settings.SITE_URL reset_password_url = urlutil.appendparams(reset_password_url, token=email_token.token, email=email_token.email) emails.send_reset_password_email(user, reset_password_url) return True
def conversion_dict_for_currency(base_currency, force=False): base_currency = base_currency.lower() key = CONVERSION_DICT_KEY_FMT % base_currency conversion_dict = cache.get(key) if conversion_dict and not force: return conversion_dict # non-SSL USD-only is all we can do now. fixer_url = urlutil.appendparams( 'http://www.apilayer.net/api/live?format=1&source=USD', access_key=APILAYER_KEY) try: r = requests.get(fixer_url, timeout=5.0) except requests.exceptions.RequestException: logging.exception( 'Itunes Connect Error - could not fetch currency conversion dict (%s)', base_currency) r = None try: response_dict = r and r.json() except: response_dict = None if response_dict and 'quotes' in response_dict and len( response_dict['quotes']) > 100: # USD: '0.25' -> usd: 0.25 conversion_dict = {} for k, v in response_dict['quotes'].items(): conversion_dict[k.lower().replace('usd', '')] = float(v) else: conversion_dict = copy.copy(FALLBACK_USD_CONVERSION_DICT) if base_currency != 'usd': usd_to_base = conversion_dict[base_currency] for k, v in conversion_dict.items(): conversion_dict[k] = v / usd_to_base del conversion_dict[base_currency] conversion_dict['usd'] = 1 / usd_to_base cache.set(key, conversion_dict) return conversion_dict
def _make_request(method, path, token=None, params=None): headers = {} body = None if method == 'POST' and params: headers['Content-Type'] = 'application/x-www-form-urlencoded' try: body = urllib.urlencode(params) except: logging.exception('Invalid slack postbody: %r', params) return None slack_url = 'https://slack.com/api/%s' % path if token: slack_url = urlutil.appendparams(slack_url, token=token) code, headers, data = urlfetch.send_request(slack_url, method=method, body=body, headers=headers) if code != 200: logging.warn('Slack error for URL: %s -- %s %s %s', slack_url, code, headers, data) return None return data
def conversion_dict_for_currency(base_currency, force=False): base_currency = base_currency.lower() key = CONVERSION_DICT_KEY_FMT % base_currency conversion_dict = cache.get(key) if conversion_dict and not force: return conversion_dict # non-SSL USD-only is all we can do now. fixer_url = urlutil.appendparams('http://www.apilayer.net/api/live?format=1&source=USD', access_key=APILAYER_KEY) try: r = requests.get(fixer_url, timeout=5.0) except requests.exceptions.RequestException: logging.exception('Itunes Connect Error - could not fetch currency conversion dict (%s)', base_currency) r = None try: response_dict = r and r.json() except: response_dict = None if response_dict and 'quotes' in response_dict and len(response_dict['quotes']) > 100: # USD: '0.25' -> usd: 0.25 conversion_dict = {} for k, v in response_dict['quotes'].items(): conversion_dict[k.lower().replace('usd', '')] = float(v) else: conversion_dict = copy.copy(FALLBACK_USD_CONVERSION_DICT) if base_currency != 'usd': usd_to_base = conversion_dict[base_currency] for k, v in conversion_dict.items(): conversion_dict[k] = v / usd_to_base del conversion_dict[base_currency] conversion_dict['usd'] = 1 / usd_to_base cache.set(key, conversion_dict) return conversion_dict
def unsubscribe_url_for_subscription(sub): base_url = '%sreviews/unsubscribe/' % settings.SITE_URL token = crypto_hack.encrypt_object( {'time': time.time(), 'sub_id': sub.encrypted_id,}, settings.UNSUB_SUB_NOTIFY_SECRET) return urlutil.appendparams(base_url, token=token)
def author_search_url(self): return urlutil.appendparams( 'https://www.google.com/search', q='"%s" twitter OR facebook OR linkedin OR apple OR email' % self.author_title)
def twitter_share_url(self): try: return urlutil.appendparams('https://twitter.com/share', text=self.tweet_text(), url=self.public_url) except UnicodeEncodeError: return None
def fetch_reviews(app, country='us', page=1): has_next_page = False fetched_reviews = None i = 0 url = REVIEWS_URL_FORMAT % { 'app_id': app.itunes_id, 'page': page, 'country': country, } while fetched_reviews is None and i < MAX_FETCH_ATTEMPTS: if i > 0: url = urlutil.appendparams(url, invalid_response_cache_bust='%0.6f' % time.time()) # Don't hammer the site when failing. time.sleep(WAIT_BETWEEN_RETRIES) i += 1 code, headers, result = urlfetch.fetch(url) if code != 200 or not result: if code == 403: raise RateLimitedError() else: continue entry = result['feed'].get('entry') if not entry: # This is either a "no reviews at all for this app on this page" case, # or a bad potentially cached result on the apple feed side. continue if not isinstance(entry, list): # This is an invalid response that sometimes happen, and it appears to be # non-recoverable. logging.info( 'Bad+invalid+non-recoverable response for app: %s url: %s', app.bundle_id, url) break # Successful response's first entry is a description of the app for some reason. fetched_reviews = entry[1:] last_page_link = [ link['attributes']['href'] for link in result['feed']['link'] if link['attributes']['rel'] == 'last' ][0] last_page_int = int(PAGE_RE.findall(last_page_link)[0]) has_next_page = last_page_int > int(page) if i > 1: if fetched_reviews is None: logging.info('FAILED to fetch reviews. %s attempts. %s page %s', i, app.bundle_id, page) else: logging.info('SUCCESS fetching reviews. %s attempts. %s page %s', i, app.bundle_id, page) if fetched_reviews: fetched_reviews = [ review_from_dict(app, d, country) for d in fetched_reviews ] return has_next_page, fetched_reviews
def author_search_url(self): return urlutil.appendparams('https://www.google.com/search', q='"%s" twitter OR facebook OR linkedin OR apple OR email' % self.author_title)