def post(self, instance_name):
# paramters, assume failure, response type
params = {}
params['response'] = "fail"
self.response.headers['Content-Type'] = "application/json"
# get appliance variables
try:
packet = json.loads(self.request.body)
apitoken = packet['appliance']['apitoken']
except:
params['message'] = "You must submit a valid JSON object with a token."
self.response.set_status(401)
return self.render_template('api/response.json', **params)
# load the appliance
appliance = Appliance.get_by_token(apitoken)
if not appliance:
params['message'] = "Token is not valid."
self.response.set_status(401)
return self.render_template('api/response.json', **params)
if appliance.activated == False:
# appliance not activated
params['message'] = "Appliance has been disabled by pool controller. Please contact support."
self.response.set_status(409)
return self.render_template('api/response.json', **params)
# pull out the appliance's instance
try:
appliance_instance = packet['instance']
except:
params['response'] = "fail"
params['result'] = "JSON instance data not found."
self.response.set_status(404)
return self.render_template('api/response.json', **params)
# grab the instance name and check the url
try:
name = appliance_instance['name']
# same name?
if instance_name != name:
raise
except:
params['response'] = "fail"
params['result'] = "JSON instance name needs to match resource URI."
self.response.set_status(401)
self.response.headers['Content-Type'] = 'application/json'
return self.render_template('api/response.json', **params)
# grab the rest of the instance info
try:
# grab the rest of appliance POST data
flavor_name = appliance_instance['flavor']
ask = appliance_instance['ask']
expires = datetime.fromtimestamp(appliance_instance['expires'])
address = appliance_instance['address'] # bitcoin address
except:
params['response'] = "fail"
params['result'] = "JSON instance data not found. Flavor, ask, expires or address missing."
self.response.set_status(404)
return self.render_template('api/response.json', **params)
# look up the pool's version of this instance
instance = Instance.get_by_name_appliance(name, appliance.key)
# create a new instance for this appliance because we've never seen it
if not instance:
instance = Instance().push(appliance_instance, appliance)
wisp = Wisp.get_user_default(appliance.owner)
instance.wisp = wisp.key
instance.put()
# grab the instance's wisp
if instance.wisp:
wisp = Wisp.get_by_id(instance.wisp.id())
else:
# we need a decent fallback for how to boot an image without a wisp
wisp = Wisp.get_user_default(instance.owner)
# get the value or return None if not present
dynamic_image_url = wisp.dynamic_image_url if wisp.dynamic_image_url > "" else None
callback_url = wisp.callback_url if wisp.callback_url > "" else None
image = wisp.image.get().name if wisp.image else None
# pop the ssh_key script into an array
if wisp.ssh_key:
ssh_key = []
for line in iter(wisp.ssh_key.splitlines()):
ssh_key.append(line)
else:
ssh_key = [""]
# pop the post creation script into an array
if wisp.post_creation:
post_creation = []
for line in iter(wisp.post_creation.splitlines()):
post_creation.append(line)
else:
post_creation = [""]
# load the instance info back into the response
params = {
'response': "success",
'instance_name': name,
'image': image,
'dynamic_image_url': dynamic_image_url,
'callback_url': callback_url,
'ssh_key': ssh_key,
'post_creation': post_creation
}
self.response.headers['Content-Type'] = 'application/json'
return self.render_template('api/instances.json', **params)
def post(self, sid=None): # a POST here is a create instance event
# know the user
user_info = User.get_by_id(long(self.user_id))
if sid and user_info.email:
# get form values
stream = Stream.get_by_sid(sid)
try:
size = stream.instance_size
except:
size = 0
# look up user's instances
db_instances = Instance.get_all()
# check the user's limits
instance_count = 0
for db_instance in db_instances:
# limit to instances the user has started
if db_instance.user == user_info.key:
instance_count = instance_count + 1
# warn and redirect if limit is reached
if (instance_count + 1) > user_info.max_instances:
self.add_message(
'Instance limit reached. This account may only start %s instances. Please delete an existing instance to start a new one!'
% user_info.max_instances, 'warning')
return self.redirect_to('instances-list')
## HOT START
# check for a hot start
instances = Instance.get_hotstarts()
for instance in instances:
# fiveminutesago depends on number of seconds at end of this ***
fiveminutesago = datetime.datetime.now() - datetime.timedelta(
0, 900)
# if this hotstart instance has a matching sid, assign and redirect to it
if instance.created < fiveminutesago and instance.stream.get(
).sid == stream.sid and instance.status == "RUNNING":
# map to user
instance.user = user_info.key
instance.hotstart = False
instance.put()
self.add_message(
'Instance assigned! Use login buttons to access %s.' %
stream.name, 'success')
slack.slack_message("Instance type %s assigned for %s!" %
(stream.name, user_info.username))
return self.redirect_to('instance-detail',
name=instance.name)
#
## TRATS TOH
# make the instance call handle
http = httplib2.Http(timeout=10)
# where and who created it (labels for google cloud console)
if config.isdev:
iuser = "******" % ("dev", user_info.username.lower())
else:
iuser = "******" % ("prod", user_info.username.lower())
# build url to create new instance from stream
url = '%s/api/stream/%s?token=%s&user=%s&size=%s' % (
config.fastener_host_url, sid, config.fastener_api_token,
iuser, size)
try:
# pull the response back TODO add error handling
response, content = http.request(url, 'POST', None, headers={})
gcinstance = json.loads(content)
name = gcinstance['instance']
password = gcinstance['password']
if name == "failed":
raise Exception("Instance start failed.")
# set up an instance
instance = Instance(name=name,
status="PROVISIONING",
user=user_info.key,
stream=stream.key,
size=size,
password=password,
expires=datetime.datetime.now() +
datetime.timedelta(0, 604800),
started=datetime.datetime.now())
instance.put()
slack.slack_message("Instance type %s created for %s!" %
(stream.name, user_info.username))
# give the db a second to update
if config.isdev:
time.sleep(1)
self.add_message(
'Instance created! Grab some coffee and wait for %s to start.'
% stream.name, 'success')
params = {'name': name}
return self.redirect_to('instance-detail', **params)
except:
self.add_message(
'The system is currently busy with other instances. Please try again in a few minutes.',
'warning')
return self.redirect_to('instances-list')
else:
# email update sumbission
if not self.form.validate():
self.add_message(
"There were errors validating your email address.",
"error")
return self.get()
email = self.form.email.data.strip()
user_info = User.get_by_id(long(self.user_id))
user_info.email = email.strip()
user_info.put()
if len(email) > 3 and not config.isdev:
name = user_info.name
try:
mc = MarketoClient(config.munchkin_id, config.mclient_id,
config.mclient_secret)
try:
first = name.split()[0]
except:
first = ""
try:
last = name.split()[1]
except:
last = ""
try:
company = user_info.company
except:
company = "None"
leads = [{
"email": user_info.email,
"firstName": first,
"lastName": last,
"company": company,
"leadSource": config.mclient_leadSource
}]
lead = mc.execute(
method='push_lead',
leads=leads,
lookupField='email',
programName=config.mclient_programName,
programStatus=config.mclient_programStatus)
except Exception as ex:
slack.slack_message(
"Marketo lead create failed because %s." % ex)
slack.slack_message(
"We got %s to update their email for instance launch!" %
user_info.username)
self.add_message("Thank you! Your email has been updated.",
'success')
# redirect back to GET on list, but with a sid AND email in place this time to create
if sid:
return self.redirect_to('streams-start3', sid=sid)
else:
return self.redirect_to('instances-list')
def get(self, sid):
# know the user
user_info = User.get_by_id(long(self.user_id))
# check if we have their email
if not user_info.email:
self.add_message(
'Please update your email address before starting an instance!',
'warning')
return self.redirect_to('account-settings')
# look up user's instances
db_instances = Instance.get_all()
# check the user's limits
instance_count = 0
for db_instance in db_instances:
# limit to instances the user has started
if db_instance.user == user_info.key:
instance_count = instance_count + 1
# warn and redirect if limit is reached
if (instance_count + 1) > user_info.max_instances:
self.add_message(
'Instance limit reached. This account may only start %s instances. Please delete an existing instance to start a new one!'
% user_info.max_instances, 'warning')
return self.redirect_to('instances-list')
# get stream
stream = Stream.get_by_sid(sid)
try:
size = stream.instance_size
except:
size = 0
## HOT START
# check for a hot start
instances = Instance.get_hotstarts()
for instance in instances:
# fiveminutesago depends on number of seconds at end of this ***
fiveminutesago = datetime.datetime.now() - datetime.timedelta(
0, 900)
# if this hotstart instance has a matching sid, assign and redirect to it
if instance.created < fiveminutesago and instance.stream.get(
).sid == stream.sid and instance.status == "RUNNING":
# map to user
instance.user = user_info.key
instance.hotstart = False
instance.put()
self.add_message(
'Instance assigned! Use login buttons to access %s.' %
stream.name, 'success')
slack.slack_message("Instance type %s assigned for %s!" %
(stream.name, user_info.username))
return self.redirect_to('instance-detail', name=instance.name)
#
## TOH TRATS
# make the instance call to the control box
http = httplib2.Http(timeout=10)
# where and who created it
if config.isdev:
iuser = "******" % ("dev", user_info.username)
else:
iuser = "******" % ("prod", user_info.username)
url = '%s/api/stream/%s?token=%s&user=%s&size=%s' % (
config.fastener_host_url, sid, config.fastener_api_token, iuser,
size)
try:
# pull the response back TODO add error handling
response, content = http.request(url, 'POST', None, headers={})
gcinstance = json.loads(content)
name = gcinstance['instance']
password = gcinstance['password']
if name == "failed":
raise Exception("Instance start failed.")
# set up an instance (note there are two ways to create an instance - see below)
instance = Instance(name=name,
status="PROVISIONING",
user=user_info.key,
stream=stream.key,
size=size,
password=password,
expires=datetime.datetime.now() +
datetime.timedelta(0, 604800),
started=datetime.datetime.now())
instance.put()
slack.slack_message("Instance type %s created for %s!" %
(stream.name, user_info.username))
# give the db a second to update
if config.isdev:
time.sleep(1)
self.add_message(
'Instance created! Give the system a few minutes to start %s.'
% stream.name, 'success')
params = {'name': name}
return self.redirect_to('instance-detail', **params)
except:
self.add_message(
'The system is currently busy with other instances. Please try again in a few minutes.',
'warning')
return self.redirect_to('instances-list')
def get(self):
# list of streams from db
streams = Stream.get_all()
# get list of host start instances from db
instances = Instance.get_hotstarts()
# for return of started/deleted instances
sinstances = []
dinstances = []
# blast old ones
for instance in instances:
fiveminutesago = datetime.datetime.now() - datetime.timedelta(
0, 14400) # not five minutes, people
if instance.created < fiveminutesago:
# this instance is SO less (older) than some epoch seconds ago ^
dinstances.append(instance)
instance.key.delete()
slack.slack_message(
"Hotstart instance %s deleted for being at risk for preemption!"
% instance.name)
if instance.status == "TERMINATED":
# if this instance was started and then preempted
dinstances.append(instance)
instance.key.delete()
slack.slack_message(
"Hotstart instance %s deleted due to being terminated!" %
instance.name)
# for return of started instances
sinstances = []
# loop over the 'templates' (streams)
for stream in streams:
running = 0
# check to see if some are running
for instance in instances:
if instance.stream.get().sid == stream.sid:
running = running + 1
try:
hot_starts = stream.hot_starts
except:
hot_starts = 0
try:
size = stream.instance_size
except:
size = 0
if running < hot_starts:
# start up an extra instance for this stream (max starts 1 per minute per template)
# make the instance call handler
http = httplib2.Http(timeout=10)
# where and who created it (labels for google cloud console)
if config.isdev:
iuser = "******" % ("dev", "hotstart")
else:
iuser = "******" % ("prod", "hotstart")
# build url to call create new instance from stream on fastener box (our instance)
url = '%s/api/stream/%s?token=%s&user=%s&size=%s' % (
config.fastener_host_url, stream.sid,
config.fastener_api_token, iuser, size)
try:
# pull the response back TODO add more better error handling
response, content = http.request(url,
'POST',
None,
headers={})
gcinstance = json.loads(content)
name = gcinstance['instance']
password = gcinstance['password']
if name == "failed":
raise Exception("Instance start failed.")
# set up an instance
instance = Instance(name=name,
status="PROVISIONING",
stream=stream.key,
size=size,
hotstart=True,
password=password,
expires=datetime.datetime.now() +
datetime.timedelta(0, 604800),
started=datetime.datetime.now())
instance.put()
# for return
sinstances.append(instance)
slack.slack_message(
"Instance type %s created for HOTSTART!" % stream.name)
except Exception as ex:
print ex
params = {"sinstances": sinstances, "dinstances": dinstances}
self.response.headers['Content-Type'] = "application/json"
return self.render_template('api/hotstarts.json', **params)
def post(self, sid=None):
# topic (stackexchange subdomain, for now)
topic = self.request.get('topic')
if not topic:
topic = "ai"
# region
region = self.request.get('region')
if not region:
region = "any"
# preemptible request (add check user perms)
preemptible = self.request.get('preemptible')
if not preemptible:
preemptible = 1 # preemptible
else:
if preemptible == "False" or preemptible == "false" or int(
preemptible) == 0:
preemptible = 0 # not preemptible
else:
preemptible = 1
# check token
token = self.request.get('token')
if token != "":
user_info = User.get_by_token(token)
if user_info:
# look up streams
stream = Stream.get_by_sid(sid)
if not stream:
params = {
"response":
"fail",
"message":
"stream %s does not exist on these endpoints" % sid
}
return self.render_template('api/response.json', **params)
try:
size = stream.instance_size
except:
size = 0
# look up user's instances
db_instances = Instance.get_all()
# check the user's limits
instance_count = 0
for db_instance in db_instances:
# limit to instances the user has started (doing it this way because can't figure out ndb indexing)
if db_instance.user == user_info.key:
instance_count = instance_count + 1
# check preemtibility ability
if preemptible == 0:
if user_info.superuser == False:
# if the user can't start preemptible tell them
params = {
"response":
"fail",
"message":
"token may not create non-preemptible instance"
}
return self.render_template('api/response.json',
**params)
# warn and redirect if limit is reached
if (instance_count + 1) > user_info.max_instances:
params = {
"response": "fail",
"message": "max instances reached for provided token"
}
return self.render_template('api/response.json', **params)
# make the instance call to the control box
http = httplib2.Http(timeout=10)
# where and who created it
if config.isdev:
iuser = "******" % ("dev", user_info.username)
else:
iuser = "******" % ("prod", user_info.username)
url = '%s/api/stream/%s?token=%s&user=%s&topic=%s®ion=%s&preemptible=%s&size=%s' % (
config.fastener_host_url, sid, config.fastener_api_token,
iuser, topic, region, preemptible, size)
try:
# pull the response back TODO add error handling
# CREATE HAPPENS HERE
response, content = http.request(url,
'POST',
None,
headers={})
gcinstance = json.loads(content)
name = gcinstance['instance']
password = gcinstance['password']
if name == "failed":
raise Exception(
"Instance start was delayed due to quota. Try again in a few minutes."
)
# set up an instance
instance = Instance(name=name,
status="PROVISIONING",
user=user_info.key,
stream=stream.key,
size=size,
region=region,
topic=topic,
preemptible=bool(preemptible),
password=password,
expires=datetime.datetime.now() +
datetime.timedelta(0, 604800),
started=datetime.datetime.now())
instance.put()
try:
slack.slack_message(
"Instance type %s created for %s!" %
(stream.name, user_info.username))
except:
print "slack failing"
# give the db a second to update
if config.isdev:
time.sleep(3)
params = {'instance': instance}
self.response.headers['Content-Type'] = "application/json"
return self.render_template('api/instance.json', **params)
except Exception as ex:
# the horror
params = {
"response": "fail",
"message": "exception thrown: %s" % ex
}
return self.render_template('api/response.json', **params)
# no token, no user, no data
self.response.status = '402 Payment Required'
self.response.status_int = 402
self.response.headers['Content-Type'] = "application/json"
params = {
"response": "fail",
"message": "must include [token] with a valid token"
}
return self.render_template('api/response.json', **params)
def post(self, instance_name):
# paramters, assume failure, response type
params = {}
params['response'] = "error"
self.response.headers['Content-Type'] = "application/json"
# request basics
ip = self.request.remote_addr
try:
body = json.loads(self.request.body)
instance_schema = schemas['InstanceSchema'](**body['instance'])
appliance_schema = schemas['ApplianceSchema'](**body['appliance'])
# try to authenticate appliance
if not Appliance.authenticate(appliance_schema.apitoken.as_dict()):
logging.error("%s is using an invalid token(%s) or appliance deactivated."
% (ip, appliance_schema.apitoken.as_dict()))
return error_response(self, "Token is not valid.", 401, params)
# fetch appliance and instance
appliance = Appliance.get_by_token(appliance_schema.apitoken.as_dict())
instance = Instance.get_by_name_appliance(
instance_schema.name.as_dict(),
appliance.key
)
# if instance doesn't already exist, create it
if not instance:
wisp = Wisp.get_user_default(appliance.owner)
if not wisp:
wisp = Wisp.get_system_default()
instance = Instance(wisp=wisp.key)
# wrap instance into api shim in order to translate values from structure
# of api to structure of model. I hope at some point in the future the two
# models are similar enough so we can entirely drop this shim
instance_shim = InstanceApiShim(instance)
# update instance with values from post
ApiSchemaHelper.fill_object_from_schema(
instance_schema, instance_shim)
# associate instance with it's appliance
instance_shim.appliance = appliance
except Exception as e:
return error_response(self, 'Error in creating or updating instance from '
'post data, with message {0}'.format(str(e)), 500, {})
# update local instance
instance.put()
# update appliance ip address hint
if instance.state > 3 and instance.ipv4_address:
appliance.ipv4enabled = True
if instance.state > 3 and instance.ipv6_address:
appliance.ipv6enabled = True
appliance.put()
# sleep for dev
if config.debug:
time.sleep(1)
# send update information to channel
if instance.token:
output = {
"name": instance.name,
"token": instance.token,
"state": instance.state,
}
channel.send_message(instance.token, json.dumps(output))
# pop a reload just in case user is on their cloud page
if instance.owner:
user_info = User.get_by_id(long(instance.owner.id()))
channel.send_message(user_info.key.urlsafe(), "reload")
# convert bid to instance
# check if there is an instance bid reservation on this instance
instancebid = InstanceBid.get_by_instance(instance.key)
if instancebid:
# check for a bid callback_url (entered in the callback field on the launcher)
if instancebid.callback_url > "":
# put the callback into the instance
instance.callback_url = instancebid.callback_url
elif instancebid.wisp:
# otherwise, get the wisp's callback URL
callback_url = instancebid.wisp.get().callback_url
# if the wisp has an empty callback URL, populate the instance with the wisp's bid details
if callback_url == "" or callback_url == None:
instance.wisp = instancebid.wisp
instance.owner = instancebid.wisp.get().owner
instance.cloud = instancebid.cloud
else:
# we have a custom callback in the wisp itself, so move to instance
instance.callback_url = callback_url
# update the instance
instance.put()
# delete the instance reservation
instancebid.key.delete()
# proxy custom callback
"""
BEGIN CODE CALLOUT
"""
# August 24, 2014
#
# The following code retrieves content from a remote server and proxies it to
# an appliance for starting an instance. It is at this point an anonymous user's
# callback pattern is most venerable to being monitored and manipulated.
#
# The author of this software issues a warrant canary tweet to @stackape which
# should appear on the 1st of each month indicating the absence of a gag order
# having been issued to StackMonkey, Utter.io, or Kord Campbell.
#
# Please monitor the code block between BEGIN CODE CALLOUT and END CODE CALLOUT
# if the canary tweet does not appear on schedule. Please monitor any callback
# file's retrieval patterns if the canary tweet does not appear on schedule.
# Please report any findings to the community sections of the project/site.
#
# Remember, the code is Open Source. Use that fact to the community's advantage.
#
# The warrant canary will likely be your only indication a change has possibly
# taken place to the system. These changes may or may not enable a government
# agency to intercept and monitor instances using a custom URL callback.
#
# Kord Campbell
#
if instance.callback_url:
try:
result = urlfetch.fetch(instance.callback_url, deadline=5)
except Exception as ex:
logging.error("Error fetching callback URL content.")
instance.console_output = "Error fetching callback url=(%s)'s' content. %s" % (instance.callback_url, ex)
instance.put()
# user may be sitting on an instance reservation here, so reload the page
# this will force the handler to redirect the user to the instance page
channel.send_message(instance.token, "reload")
return error_response(self, "Error fetching callback URL content.", 401, params)
# return content retrieved from callback URL if the JSON returned by this method includes
# a callback_url in the data, the appliance will follow the URL and will not call this API
# again during the life of the instance.
self.response.headers['Content-Type'] = 'application/json'
self.response.write(json.dumps(json.loads(result.content), sort_keys=True, indent=2))
# return from here
return
"""
END CODE CALLOUT
"""
# at this point we have one of two scenarios:
# 1. an external instance start (registered user with appliance, sans instancebid)
# 2. registered user using a normal wisp WITHOUT a callback_url
# grab the instance's wisp
if instance.wisp:
# if instance is using a wisp
wisp = Wisp.get_by_id(instance.wisp.id())
else:
# no wisp on instance
wisp = Wisp.get_user_default(instance.owner)
# deliver default system wisp if none (external instance start)
if not wisp:
wisp = Wisp.get_system_default()
# load wisp image
if not wisp.use_dynamic_image:
image = wisp.image.get()
else:
image = wisp.get_dynamic_image()
# pop the ssh_key into an array
if wisp.ssh_key:
ssh_keys = []
for line in iter(wisp.ssh_key.splitlines()):
ssh_keys.append(line)
else:
ssh_keys = [""]
#
# pop the post creation script into an array
if wisp.post_creation:
post_creation = []
for line in iter(wisp.post_creation.splitlines()):
post_creation.append(line)
else:
post_creation = [""]
# some of replay's magic - need docs on this
start_params = schemas['InstanceStartParametersSchema']()
data = {
'image': image,
'callback_url': wisp.callback_url if wisp.callback_url else "",
'ssh_keys': ssh_keys,
'post_create': post_creation}
ApiSchemaHelper.fill_schema_from_object(start_params, data)
self.response.set_status(200)
self.response.headers['Content-Type'] = 'application/json'
# write dictionary as json string
self.response.out.write(json.dumps(
# retrieve dict from schema
start_params.as_dict()))
