def _issue_appengine_request(method, path, **kwargs):
gae_url = settings.APP_ENGINE_PHOTOS_UPLOAD_BASE_PATH + path
postbody = None
if method == 'GET':
gae_url = urlutil.appendparams(gae_url, **kwargs)
else:
postbody = urllib.urlencode(kwargs)
gae_request = urllib2.Request(gae_url, postbody, {
'X-LK-Secret': settings.APP_ENGINE_HEADER_SECRET,
})
gae_request.get_method = lambda: method
try:
result = urllib2.urlopen(gae_request, None, 30.0)
code = 200
except urllib2.HTTPError as e:
if e.code >= 500:
logging.info('GAE 500 response: %s', e.code)
result = e
code = e.code
except urllib2.URLError as e:
logging.info('GAE connection problem: %s', e)
return -1, None
result_content = result.read()
try:
json_dict = json.loads(result_content)
except ValueError as e:
if code < 500:
logging.warn('Bad JSON response from GAE: %r', result_content)
json_dict = None
return code, json_dict
def _make_request(method, path, token=None, params=None):
headers = {}
body = None
if method == 'POST' and params:
headers['Content-Type'] = 'application/x-www-form-urlencoded'
try:
body = urllib.urlencode(params)
except:
logging.exception('Invalid slack postbody: %r', params)
return None
slack_url = 'https://slack.com/api/%s' % path
if token:
slack_url = urlutil.appendparams(slack_url, token=token)
code, headers, data = urlfetch.send_request(slack_url,
method=method,
body=body,
headers=headers)
if code != 200:
logging.warn('Slack error for URL: %s -- %s %s %s', slack_url, code,
headers, data)
return None
return data
def _lookup_fetch(remote_url):
results_dict = {}
retry = 0
while not (results_dict and 'results' in results_dict):
if retry >= 10:
return None
if retry:
remote_url = urlutil.appendparams(remote_url, time=time.time())
logging.info('Invalid response from lookup API: %s trying: %s', results_dict, remote_url)
time.sleep(0.333 * retry)
code, headers, results_dict = urlfetch.fetch(remote_url, cache_seconds=(60 * 60),
should_cache_fn=lambda c, d: d and d.get('results'))
if code == 403:
raise RateLimitedError()
retry += 1
if code != 200:
return None
app_infos = results_dict['results']
if not (app_infos and isinstance(app_infos, list)):
return None
return app_infos
def send_bundle(bundle): email_token = EmailToken(kind=EmailToken.KIND_DOWNLOAD_BUNDLE, email=bundle.user.email) email_token.save() download_url = '%sscreenshots/dashboard/%s/download/' % (settings.SITE_URL, bundle.screenshot_set.encrypted_id) download_url = urlutil.appendparams(download_url, bundle=bundle.encrypted_id, token=email_token.token) emails.send_bundle_ready_email(bundle.user, bundle.screenshot_set, download_url)
def unsubscribe_url_for_subscription(sub):
base_url = '%ssales/unsubscribe/' % settings.SITE_URL
token = crypto_hack.encrypt_object(
{
'time': time.time(),
'sub_id': sub.encrypted_id
}, settings.UNSUB_SUB_NOTIFY_SECRET)
return urlutil.appendparams(base_url, token=token)
def verification_url_for_user_email(user, email):
email_token = EmailToken(kind=EmailToken.KIND_VERIFY_EMAIL, email=email)
email_token.save()
email_verification_url = '%saccount/verify/' % settings.SITE_URL
email_verification_url = urlutil.appendparams(email_verification_url,
token=email_token.token)
return email_verification_url
def generate_user_unsubscribe_url(user, flag_name):
if flag_name and flag_name not in USER_EDITABLE_USER_FLAGS:
raise ValueError('Invalid flag_name %s', flag_name)
encrypted_token = crypto_hack.encrypt_object({'user_id': user.id, 'flag_name': flag_name},
settings.UNSUBSCRIBE_URL_SECRET)
url = '%saccount/unsubscribe/' % settings.SITE_URL
return urlutil.appendparams(url, token=encrypted_token)
def twitter_share_url(self):
try:
# include URL manually here
return urlutil.appendparams('https://twitter.com/share',
text=self.tweet_text(include_url=False),
url=self.public_url,
)
except UnicodeEncodeError:
return None
def verification_url_for_user_email(user, email):
email_token = EmailToken(kind=EmailToken.KIND_VERIFY_EMAIL, email=email)
email_token.save()
email_verification_url = '%saccount/verify/' % settings.SITE_URL
email_verification_url = urlutil.appendparams(email_verification_url,
token=email_token.token)
return email_verification_url
def fetch_reviews(app, country='us', page=1):
has_next_page = False
fetched_reviews = None
i = 0
url = REVIEWS_URL_FORMAT % {
'app_id': app.itunes_id,
'page': page,
'country': country,
}
while fetched_reviews is None and i < MAX_FETCH_ATTEMPTS:
if i > 0:
url = urlutil.appendparams(url, invalid_response_cache_bust='%0.6f' % time.time())
# Don't hammer the site when failing.
time.sleep(WAIT_BETWEEN_RETRIES)
i += 1
code, headers, result = urlfetch.fetch(url)
if code != 200 or not result:
if code == 403:
raise RateLimitedError()
else:
continue
entry = result['feed'].get('entry')
if not entry:
# This is either a "no reviews at all for this app on this page" case,
# or a bad potentially cached result on the apple feed side.
continue
if not isinstance(entry, list):
# This is an invalid response that sometimes happen, and it appears to be
# non-recoverable.
logging.info('Bad+invalid+non-recoverable response for app: %s url: %s', app.bundle_id, url)
break
# Successful response's first entry is a description of the app for some reason.
fetched_reviews = entry[1:]
last_page_link = [link['attributes']['href']
for link in result['feed']['link']
if link['attributes']['rel'] == 'last'][0]
last_page_int = int(PAGE_RE.findall(last_page_link)[0])
has_next_page = last_page_int > int(page)
if i > 1:
if fetched_reviews is None:
logging.info('FAILED to fetch reviews. %s attempts. %s page %s', i, app.bundle_id, page)
else:
logging.info('SUCCESS fetching reviews. %s attempts. %s page %s', i, app.bundle_id, page)
if fetched_reviews:
fetched_reviews = [review_from_dict(app, d, country) for d in fetched_reviews]
return has_next_page, fetched_reviews
def twitter_share_url(self):
try:
# include URL manually here
return urlutil.appendparams(
'https://twitter.com/share',
text=self.tweet_text(include_url=False),
url=self.public_url,
)
except UnicodeEncodeError:
return None
def generate_user_unsubscribe_url(user, flag_name):
if flag_name and flag_name not in USER_EDITABLE_USER_FLAGS:
raise ValueError('Invalid flag_name %s', flag_name)
encrypted_token = crypto_hack.encrypt_object(
{
'user_id': user.id,
'flag_name': flag_name
}, settings.UNSUBSCRIBE_URL_SECRET)
url = '%saccount/unsubscribe/' % settings.SITE_URL
return urlutil.appendparams(url, token=encrypted_token)
def app_info_with_bundle_id(bundle_id, country):
remote_url = urlutil.appendparams(LOOKUP_URL % country, bundleId=bundle_id)
app_infos = _lookup_fetch(remote_url)
if not app_infos:
return None
app_info = app_infos[0]
if app_info.get('kind') not in ('software', 'mac-software'):
return None
return model_from_dict(app_info, country)
def send_bundle(bundle):
email_token = EmailToken(kind=EmailToken.KIND_DOWNLOAD_BUNDLE,
email=bundle.user.email)
email_token.save()
download_url = '%sscreenshots/dashboard/%s/download/' % (
settings.SITE_URL, bundle.screenshot_set.encrypted_id)
download_url = urlutil.appendparams(download_url,
bundle=bundle.encrypted_id,
token=email_token.token)
emails.send_bundle_ready_email(bundle.user, bundle.screenshot_set,
download_url)
def request_reset_password_email(email):
user = get_user_by_email(email)
if not user:
return False
email_token = EmailToken(kind=EmailToken.KIND_RESET_PASSWORD, email=email.lower())
email_token.save()
reset_password_url = '%saccount/reset/finish/' % settings.SITE_URL
reset_password_url = urlutil.appendparams(reset_password_url,
token=email_token.token, email=email_token.email)
emails.send_reset_password_email(user, reset_password_url)
return True
def related_app_infos_with_developer_id(developer_id, country):
remote_url = urlutil.appendparams(LOOKUP_URL % country, id=developer_id, entity=SOFTWARE_ENTITIES)
app_infos = _lookup_fetch(remote_url)
if not app_infos:
return []
filtered = []
app_ids = set()
for app_info in app_infos:
if app_info.get('kind') not in ('software', 'mac-software'):
continue
if app_info['trackId'] in app_ids:
continue
app_ids.add(app_info['trackId'])
filtered.append(app_info)
return [model_from_dict(app_info, country) for app_info in filtered]
def request_reset_password_email(email):
user = get_user_by_email(email)
if not user:
return False
email_token = EmailToken(kind=EmailToken.KIND_RESET_PASSWORD,
email=email.lower())
email_token.save()
reset_password_url = '%saccount/reset/finish/' % settings.SITE_URL
reset_password_url = urlutil.appendparams(reset_password_url,
token=email_token.token,
email=email_token.email)
emails.send_reset_password_email(user, reset_password_url)
return True
def conversion_dict_for_currency(base_currency, force=False):
base_currency = base_currency.lower()
key = CONVERSION_DICT_KEY_FMT % base_currency
conversion_dict = cache.get(key)
if conversion_dict and not force:
return conversion_dict
# non-SSL USD-only is all we can do now.
fixer_url = urlutil.appendparams(
'http://www.apilayer.net/api/live?format=1&source=USD',
access_key=APILAYER_KEY)
try:
r = requests.get(fixer_url, timeout=5.0)
except requests.exceptions.RequestException:
logging.exception(
'Itunes Connect Error - could not fetch currency conversion dict (%s)',
base_currency)
r = None
try:
response_dict = r and r.json()
except:
response_dict = None
if response_dict and 'quotes' in response_dict and len(
response_dict['quotes']) > 100:
# USD: '0.25' -> usd: 0.25
conversion_dict = {}
for k, v in response_dict['quotes'].items():
conversion_dict[k.lower().replace('usd', '')] = float(v)
else:
conversion_dict = copy.copy(FALLBACK_USD_CONVERSION_DICT)
if base_currency != 'usd':
usd_to_base = conversion_dict[base_currency]
for k, v in conversion_dict.items():
conversion_dict[k] = v / usd_to_base
del conversion_dict[base_currency]
conversion_dict['usd'] = 1 / usd_to_base
cache.set(key, conversion_dict)
return conversion_dict
def _make_request(method, path, token=None, params=None):
headers = {}
body = None
if method == 'POST' and params:
headers['Content-Type'] = 'application/x-www-form-urlencoded'
try:
body = urllib.urlencode(params)
except:
logging.exception('Invalid slack postbody: %r', params)
return None
slack_url = 'https://slack.com/api/%s' % path
if token:
slack_url = urlutil.appendparams(slack_url, token=token)
code, headers, data = urlfetch.send_request(slack_url, method=method, body=body, headers=headers)
if code != 200:
logging.warn('Slack error for URL: %s -- %s %s %s', slack_url, code, headers, data)
return None
return data
def conversion_dict_for_currency(base_currency, force=False):
base_currency = base_currency.lower()
key = CONVERSION_DICT_KEY_FMT % base_currency
conversion_dict = cache.get(key)
if conversion_dict and not force:
return conversion_dict
# non-SSL USD-only is all we can do now.
fixer_url = urlutil.appendparams('http://www.apilayer.net/api/live?format=1&source=USD',
access_key=APILAYER_KEY)
try:
r = requests.get(fixer_url, timeout=5.0)
except requests.exceptions.RequestException:
logging.exception('Itunes Connect Error - could not fetch currency conversion dict (%s)', base_currency)
r = None
try:
response_dict = r and r.json()
except:
response_dict = None
if response_dict and 'quotes' in response_dict and len(response_dict['quotes']) > 100:
# USD: '0.25' -> usd: 0.25
conversion_dict = {}
for k, v in response_dict['quotes'].items():
conversion_dict[k.lower().replace('usd', '')] = float(v)
else:
conversion_dict = copy.copy(FALLBACK_USD_CONVERSION_DICT)
if base_currency != 'usd':
usd_to_base = conversion_dict[base_currency]
for k, v in conversion_dict.items():
conversion_dict[k] = v / usd_to_base
del conversion_dict[base_currency]
conversion_dict['usd'] = 1 / usd_to_base
cache.set(key, conversion_dict)
return conversion_dict
def unsubscribe_url_for_subscription(sub):
base_url = '%sreviews/unsubscribe/' % settings.SITE_URL
token = crypto_hack.encrypt_object(
{'time': time.time(), 'sub_id': sub.encrypted_id,},
settings.UNSUB_SUB_NOTIFY_SECRET)
return urlutil.appendparams(base_url, token=token)
def author_search_url(self):
return urlutil.appendparams(
'https://www.google.com/search',
q='"%s" twitter OR facebook OR linkedin OR apple OR email' %
self.author_title)
def twitter_share_url(self):
try:
return urlutil.appendparams('https://twitter.com/share', text=self.tweet_text(), url=self.public_url)
except UnicodeEncodeError:
return None
def fetch_reviews(app, country='us', page=1):
has_next_page = False
fetched_reviews = None
i = 0
url = REVIEWS_URL_FORMAT % {
'app_id': app.itunes_id,
'page': page,
'country': country,
}
while fetched_reviews is None and i < MAX_FETCH_ATTEMPTS:
if i > 0:
url = urlutil.appendparams(url,
invalid_response_cache_bust='%0.6f' %
time.time())
# Don't hammer the site when failing.
time.sleep(WAIT_BETWEEN_RETRIES)
i += 1
code, headers, result = urlfetch.fetch(url)
if code != 200 or not result:
if code == 403:
raise RateLimitedError()
else:
continue
entry = result['feed'].get('entry')
if not entry:
# This is either a "no reviews at all for this app on this page" case,
# or a bad potentially cached result on the apple feed side.
continue
if not isinstance(entry, list):
# This is an invalid response that sometimes happen, and it appears to be
# non-recoverable.
logging.info(
'Bad+invalid+non-recoverable response for app: %s url: %s',
app.bundle_id, url)
break
# Successful response's first entry is a description of the app for some reason.
fetched_reviews = entry[1:]
last_page_link = [
link['attributes']['href'] for link in result['feed']['link']
if link['attributes']['rel'] == 'last'
][0]
last_page_int = int(PAGE_RE.findall(last_page_link)[0])
has_next_page = last_page_int > int(page)
if i > 1:
if fetched_reviews is None:
logging.info('FAILED to fetch reviews. %s attempts. %s page %s',
i, app.bundle_id, page)
else:
logging.info('SUCCESS fetching reviews. %s attempts. %s page %s',
i, app.bundle_id, page)
if fetched_reviews:
fetched_reviews = [
review_from_dict(app, d, country) for d in fetched_reviews
]
return has_next_page, fetched_reviews
def author_search_url(self):
return urlutil.appendparams('https://www.google.com/search',
q='"%s" twitter OR facebook OR linkedin OR apple OR email' % self.author_title)