def parseRequest(schema, request, **kwargs):
data = { }
### Parse Request
try:
if request.method == 'GET':
rawData = request.GET
elif request.method == 'POST':
rawData = request.POST
else:
raise
# Build the dict because django sucks
for k, v in rawData.iteritems():
if v == '':
v = None
data[k] = v
data.pop('oauth_token', None)
data.pop('client_id', None)
data.pop('client_secret', None)
logData = data.copy()
obfuscate = kwargs.pop('obfuscate', [])
obfuscate.append('password')
for item in obfuscate:
if item in logData:
logData[item] = '*****'
logs.form(logData)
if schema is None:
if len(data) > 0:
raise
return
schema.dataImport(data)
schema.validate()
logs.debug("Parsed request data")
return schema
except (KeyError, AttributeError, ValueError) as e:
msg = u"Invalid form (%s): %s vs %s" % (e, pformat(data), schema)
logs.warning(msg)
logs.warning(utils.getFormattedException())
raise StampedHTTPError(400, "invalid_request")
except Exception as e:
msg = u"Invalid form (%s): %s vs %s" % (e, pformat(data), schema)
logs.warning(msg)
logs.warning(utils.getFormattedException())
raise e
def handleStampedExceptions(e, handlers=None):
if isinstance(e, StampedHTTPError):
exceptions = [(StampedHTTPError, e.code, e.kind, e.msg)]
elif handlers is not None:
exceptions = handlers + defaultExceptions
else:
exceptions = defaultExceptions
for (exception, code, kind, msg) in exceptions:
if isinstance(e, exception):
logs.warning("%s Error (%s): %s" % (code, kind, msg))
logs.warning(utils.getFormattedException())
logs.error(code)
kind = kind
if kind is None:
kind = 'stamped_error'
message = msg
if message is None and e.msg is not None:
message = e.msg
error = {}
error['error'] = kind
if message is not None:
error['message'] = unicode(message)
return transformOutput(error, status=code)
else:
error = {
'error' : 'stamped_error',
'message' : "An error occurred. Please try again later.",
}
logs.warning("500 Error: %s" % e)
logs.warning(utils.getFormattedException())
logs.error(500)
# Email dev if a 500 occurs
if libs.ec2_utils.is_ec2():
try:
email = {}
email['from'] = 'Stamped <*****@*****.**>'
email['to'] = '*****@*****.**'
email['subject'] = '%s - 500 Error - %s' % (stampedAPI.node_name, datetime.utcnow().isoformat())
email['body'] = logs.getHtmlFormattedLog()
utils.sendEmail(email, format='html')
except Exception as e:
logs.warning('UNABLE TO SEND EMAIL: %s')
return transformOutput(error, status=500)
def run(self, options):
logs.begin(
saveLog=stampedAPI._logsDB.saveLog,
saveStat=stampedAPI._statsDB.addStat,
nodeName=stampedAPI.node_name
)
logs.async_request('alerts')
lock = os.path.join(base, 'alerts.lock')
if os.path.exists(lock):
logs.warning('Locked - aborting')
return
try:
open(lock, 'w').close()
self.buildAlerts(limit=options.limit, noop=options.noop)
self.buildInvitations(limit=options.limit, noop=options.noop)
if len(self._emailQueue) > 0:
self.sendEmails(noop=options.noop)
if len(self._pushQueue) > 0:
self.sendPush(noop=options.noop)
self.cleanupPush(noop=options.noop)
except Exception as e:
logs.warning('Exception: %s' % e)
logs.warning(utils.getFormattedException())
logs.error(500)
finally:
os.remove(lock)
try:
logs.save()
except Exception:
print '\n\n\nWARNING: UNABLE TO SAVE LOGS\n\n\n'
def sendEmails(self, noop=False):
logs.info("Submitting emails to %s users" % len(self._emailQueue))
# Apply rate limit
limit = 8
ses = boto.connect_ses(keys.aws.AWS_ACCESS_KEY_ID, keys.aws.AWS_SECRET_KEY)
for emailAddress, emailQueue in self._emailQueue.iteritems():
if IS_PROD or emailAddress in self._adminEmails:
count = 0
emailQueue.reverse()
for email in emailQueue:
count += 1
if count > limit:
logs.debug("Limit exceeded for email '%s'" % emailAddress)
break
try:
logs.debug("Send email: %s" % (email))
if not noop:
ses.send_email(email.sender, email.title, email.body, emailAddress, format='html')
except Exception as e:
logs.warning("Email failed: %s" % email)
logs.warning(utils.getFormattedException())
logs.info("Success!")
def _wrapper(request, *args, **kwargs):
import servers.web2.error.views as web_error
try:
logs.begin(saveLog=stampedAPIProxy.api._logsDB.saveLog,
saveStat=stampedAPIProxy.api._statsDB.addStat,
requestData=request,
nodeName=stampedAPIProxy.api.node_name)
logs.info("%s %s" % (request.method, request.path))
subkwargs = kwargs
if schema is not None:
parse_kwargs = parse_request_kwargs or {}
django_kwargs = {}
if parse_django_kwargs:
django_kwargs = kwargs or {}
subkwargs = {}
result = parse_request(request, schema(), django_kwargs, overflow=ignore_extra_params, **parse_kwargs)
subkwargs['schema'] = result
response = fn(request, *args, **subkwargs)
logs.info("End request: Success")
if no_cache:
expires = (dt.datetime.utcnow() - dt.timedelta(minutes=10)).ctime()
cache_control = 'no-cache'
elif utils.is_ec2():
expires = (dt.datetime.utcnow() + dt.timedelta(minutes=60)).ctime()
cache_control = 'max-age=600'
else:
# disable caching for local development / debugging
expires = (dt.datetime.utcnow() - dt.timedelta(minutes=10)).ctime()
cache_control = 'max-age=0'
response['Expires'] = expires
response['Cache-Control'] = cache_control
return response
except urllib2.HTTPError, e:
logs.warning("%s Error: %s" % (e.code, e))
logs.warning(utils.getFormattedException())
if e.code == 404:
return web_error.error_404(request)
elif e.code >= 500:
return web_error.error_500(request)
raise # invoke django's default 500 handler
def _transform_stamps(self, stamps):
if stamps is None:
stamps = []
ret = []
for stamp in stamps:
try:
ret.append(HTTPStamp().importStamp(stamp).dataExport())
except Exception:
logs.warn(utils.getFormattedException())
return ret
def parse_request(request, schema, django_kwargs, overflow, **kwargs):
data = { }
try:
if request.method == 'GET':
rawData = request.GET
elif request.method == 'POST':
rawData = request.POST
else:
raise "invalid HTTP method '%s'" % request.method
# Build the dict because django sucks
for k, v in rawData.iteritems():
data[k] = v
for k, v in django_kwargs.iteritems():
if k in data:
msg = "duplicate django kwarg '%s' found in request %s data" % (k, request.method)
raise msg
data[k] = v
logs.info("REQUEST: %s" % pformat(data))
data.pop('oauth_token', None)
data.pop('client_id', None)
data.pop('client_secret', None)
logData = data.copy()
obfuscate = kwargs.pop('obfuscate', [])
obfuscate.append('password')
for item in obfuscate:
if item in logData:
logData[item] = '*****'
logs.form(logData)
if schema is None:
if len(data) > 0:
raise
return
schema.dataImport(data, overflow=overflow)
return schema
except Exception as e:
msg = "Invalid form (%s): %s vs %s" % (e, pformat(data), schema)
logs.warning(msg)
logs.warning(utils.getFormattedException())
raise StampedHTTPError("invalid_form", 400, msg)
def render(self, context):
try:
if self._context_variable is None:
context_dict = self._simplify_context(context)
else:
context_dict = self._context_variable.resolve(context)
result = self._library.render(self._name, context_dict)
if len(result.strip()) == 0:
logs.warn("%s.render warning empty result (%s)" % (self, self._name))
return result
except Exception, e:
logs.warn("%s.render error (%s): %s" % (self, self._name, e))
logs.warn(utils.getFormattedException())
return ''
def sendPush(self, noop=False):
logs.info("Submitting push notifications to %s devices" % len(self._pushQueue))
# Apply rate limit
limit = 3
for deviceId, pushQueue in self._pushQueue.iteritems():
if IS_PROD or deviceId in self._adminTokens:
count = 0
apns_wrapper = APNSNotificationWrapper(self._apnsCert, sandbox=self._sandbox)
pushQueue.reverse()
for notification in pushQueue:
count += 1
if count > limit:
logs.debug("Limit exceeded for device '%s'" % deviceId)
break
try:
# Build APNS notification
logs.debug("Push activityId '%s' to device '%s'" % (notification.activityId, deviceId))
apnsNotification = APNSNotification()
apnsNotification.token(binascii.unhexlify(deviceId))
if notification.message is not None:
msg = notification.message.encode('ascii', 'ignore')
apnsNotification.alert(msg)
if notification.badge is not None:
apnsNotification.badge(notification.badge)
# Add notification to wrapper
apns_wrapper.append(apnsNotification)
except Exception as e:
logs.warning("Push failed: '%s'" % notification)
logs.warning(utils.getFormattedException())
if apns_wrapper.count() > 0:
if not noop:
apns_wrapper.notify()
logs.info("Success!")
def process_user(self, user, categories=None):
assert user is not None
if categories is None:
categories = self._categories
retries = 0
while retries < 3:
try:
for category in categories:
ts = {
'user_id' : user.user_id,
'scope' : 'user'
}
if category != 'default':
if category == 'app':
ts['subcategory'] = 'app'
else:
ts['category'] = category
ts['limit'] = 100
collage = self._collages[category]
stamp_slice = HTTPTimeSlice().dataImport(ts).exportTimeSlice()
stamps = self.api.getStampCollection(stamp_slice)
entities = map(lambda s: s.entity, stamps)
entities = utils.shuffle(entities)[:30]
logs.info("creating collage for user '%s' w/ category '%s' and %d entities" % (user.screen_name, category, len(entities)))
images = collage.generate_from_user(user, entities)
for image in images:
filename = "collage-%s-%s-%sx%s.jpg" % (user.screen_name, category, image.size[0], image.size[1])
self.save_image(image, filename)
break
except Exception, e:
logs.warn("unexpected error processing user %s: %s" % (str(user), e))
logs.warn(utils.getFormattedException())
retries += 1
time.sleep(2 ** retries)
def _buildMap(template, stamps):
try:
result = []
for stamp in stamps:
try:
if "deleted" not in stamp:
result.append(HTTPStamp().importSchema(stamp).dataExport())
except:
logs.warn(utils.getFormattedException())
result = json.dumps(result, sort_keys=True)
response = render_to_response(template, {"stamps": result})
response["Expires"] = (datetime.utcnow() + timedelta(minutes=10)).ctime()
response["Cache-Control"] = "max-age=600"
return response
except Exception as e:
logs.warning("Error: %s" % e)
raise Http404
def stamped_view(schema=None,
ignore_extra_params=False,
parse_request_kwargs=None,
parse_django_kwargs=True,
no_cache=False):
def decorator(fn):
# NOTE (travis): if you hit this assertion, you're likely using the
# handleHTTPRequest decorator incorrectly.
assert callable(fn)
@wraps(fn)
def _wrapper(request, *args, **kwargs):
import servers.web2.error.views as web_error
try:
logs.begin(saveLog=stampedAPIProxy.api._logsDB.saveLog,
saveStat=stampedAPIProxy.api._statsDB.addStat,
requestData=request,
nodeName=stampedAPIProxy.api.node_name)
logs.info("%s %s" % (request.method, request.path))
subkwargs = kwargs
if schema is not None:
parse_kwargs = parse_request_kwargs or {}
django_kwargs = {}
if parse_django_kwargs:
django_kwargs = kwargs or {}
subkwargs = {}
result = parse_request(request, schema(), django_kwargs, overflow=ignore_extra_params, **parse_kwargs)
subkwargs['schema'] = result
response = fn(request, *args, **subkwargs)
logs.info("End request: Success")
if no_cache:
expires = (dt.datetime.utcnow() - dt.timedelta(minutes=10)).ctime()
cache_control = 'no-cache'
elif utils.is_ec2():
expires = (dt.datetime.utcnow() + dt.timedelta(minutes=60)).ctime()
cache_control = 'max-age=600'
else:
# disable caching for local development / debugging
expires = (dt.datetime.utcnow() - dt.timedelta(minutes=10)).ctime()
cache_control = 'max-age=0'
response['Expires'] = expires
response['Cache-Control'] = cache_control
return response
except urllib2.HTTPError, e:
logs.warning("%s Error: %s" % (e.code, e))
logs.warning(utils.getFormattedException())
if e.code == 404:
return web_error.error_404(request)
elif e.code >= 500:
return web_error.error_500(request)
raise # invoke django's default 500 handler
#response = HttpResponse("%s" % e, status=e.code)
#logs.error(response.status_code)
#return response
except StampedHTTPError as e:
logs.warning("%s Error: %s" % (e.code, e.msg))
logs.warning(utils.getFormattedException())
raise # invoke django's default 500 handler
#response = HttpResponse(e.msg, status=e.code)
#logs.error(response.status_code)
#return response
except StampedAuthError as e:
logs.warning("401 Error: %s" % (e.msg))
logs.warning(utils.getFormattedException())
response = HttpResponse(e.msg, status=401)
logs.auth(e.msg)
return response
raise # invoke django's default 500 handler
#response = HttpResponse(e.msg, status=e.code)
#logs.error(response.status_code)
#return response
except StampedAuthError as e:
logs.warning("401 Error: %s" % (e.msg))
logs.warning(utils.getFormattedException())
response = HttpResponse(e.msg, status=401)
logs.auth(e.msg)
return response
except StampedInputError as e:
logs.warning("400 Error: %s" % (e.msg))
logs.warning(utils.getFormattedException())
response = HttpResponse("invalid_request", status=400)
logs.error(response.status_code)
return response
except StampedIllegalActionError as e:
logs.warning("403 Error: %s" % (e.msg))
logs.warning(utils.getFormattedException())
response = HttpResponse("illegal_action", status=403)
logs.error(response.status_code)
return response
except StampedPermissionsError as e:
logs.warning("403 Error: %s" % (e.msg))