def delete(self, instance_id=None): # lookup user's auth info user_info = User.get_by_id(long(self.user_id)) # delete instance instance = Instance.get_by_id(long(instance_id)) slack.slack_message("Instance %s deleted for %s!" % (instance.name, user_info.username)) instance.key.delete() # hangout for a second if config.isdev: time.sleep(1) params = { "response": "success", "message": "instance %s deleted" % instance.name } return self.render_template('api/response.json', **params)
def get(self, npid=None): # get our request code back from the social login handler above code = self.request.get('code') # fire up the github auth object scope = 'user:email' github_helper = github.GithubAuth(scope) # retrieve the access token using the code and auth try: access_token = github_helper.get_access_token(code) user_data = github.get_user_info(access_token) except: message = 'Error while tokening with Github.' self.add_message(message, 'error') return self.redirect_to('index') # see if user is in database uid = str(user_data['id']) # github id user_info = User.get_by_uid(uid) # less than ideal way to handle excessive 2FA requests #if not user_info.activated: # self.add_message("This account has been deactivated due to excessive 2FA requests. Please contact us to resolve.", "error") # return self.redirect_to('about') # never seen them, so create user if not user_info: name = user_data['name'] username = user_data['login'] email = user_data['email'] location = user_data['location'] company = user_data['company'] # create entry in db user_info = User(last_login=datetime.now(), uid=str(uid), username=username, name=name, company=company, location=location, email=email, activated=True) # try to create unique username while True: user_info.unique_properties = ['username'] uniques = ['User.username:%s' % user_info.username] success, existing = Unique.create_multi(uniques) # if we already have that username, create a new one and try again if existing: user_info.username = "******" % (username, random.randrange(100)) else: break # write out the user user_info.put() # wait a few seconds for database server to update if config.isdev: time.sleep(1) # seriously? # send to marketo if we have email # if len(email) > 3: try: email_test = len(email) except Exception as ex: slack.slack_message( "New user's email appears to be empty: %s." % ex) email_test = 0 if email_test > 3 and not config.isdev: try: mc = MarketoClient(config.munchkin_id, config.mclient_id, config.mclient_secret) try: first = name.split()[0] except: first = "" try: last = name.split()[1] except: last = "" leads = [{ "email": email, "firstName": first, "lastName": last, "company": company, "leadSource": config.mclient_leadSource }] lead = mc.execute( method='push_lead', leads=leads, lookupField='email', programName=config.mclient_programName, programStatus=config.mclient_programStatus) except Exception as ex: slack.slack_message( "Marketo lead create failed because %s." % ex) # slack the new user signup slack.slack_message("New user signed up: %s|%s|%s|%s|%s" % (name, username, email, location, company)) # check out 2FA status now_minus_age = datetime.now() + timedelta(0, -config.session_age) # load the next destination, if any if npid: np_info = NextPages.get_by_npid(npid) next_page = np_info.url print next_page else: next_page = "" # check if 2FA is on if user_info.tfenabled and (user_info.last_login < now_minus_age): return self.redirect_to('login-tfa', next=next_page, uid=user_info.uid) else: # two factor is disabled, or already complete user_info.last_login = datetime.now() user_info.put() # log the user in self.auth.set_session(self.auth.store.user_to_dict(user_info), remember=True) # log visit log_message = "user logged in" log = LogVisit(user=user_info.key, message=log_message, uastring=self.request.user_agent, ip=self.request.remote_addr) log.put() message = "You have successfully logged in!" self.add_message(message, 'success') # remove the next page if np_info: np_info.key.delete() # get the destination URL from the next cookie if next_page > "": return self.redirect(str(next_page)) else: return self.redirect_to('account-dashboard') try: pass except Exception as ex: message = "User login went wrong: %s" % ex self.add_message(message, 'error') return self.redirect_to('index')
def get(self, name, command): # lookup user's auth info user_info = User.get_by_id(long(self.user_id)) # get instance instance = Instance.get_by_name(name) if not instance: params = {} return self.redirect_to('instances-list', **params) slack.slack_message( "request for an instance we can't find - SPAMSPAMSPAM") else: # check user owns it or user is admin if user_info.admin != True and long(instance.user.id()) != long( self.user_id): params = { "response": "failure", "message": "instance %s not modifiable by calling user" % name } self.response.set_status(500) slack.slack_message("%s doesn't own %s" % (user_info.username, name)) else: # start the instance if command == "start" and instance.status != "RUNNING": slack.slack_message("firing up %s" % instance.name) try: instance.started = datetime.datetime.now() instance.tender_action = "START" instance.put() params = { "response": "success", "message": "Instance %s marked to be started." % instance.name } slack.slack_message( "updated db for %s with %s" % (instance.name, instance.tender_action)) except Exception as ex: params = {"response": "failure", "message": "%s" % ex} self.response.headers['Content-Type'] = "application/json" return self.render_template('api/response.json', **params) # add ssh_key to instance elif command == "addkey": # make the instance call to the control box http = httplib2.Http(timeout=10) url = '%s/api/instance/%s/addkey?token=%s&ssh_key=%s&username=%s' % ( config.fastener_host_url, name, config.fastener_api_token, urllib.quote(user_info.ssh_key), user_info.username) try: # pull the response back TODO add error handling response, content = http.request(url, 'GET', None, headers={}) # delete if google returns pending if json.loads(content)['status'] == "SUCCESS": params = { "response": "success", "message": "instance %s updated with key" % name } else: params = { "response": "failure", "message": "instance %s operation failure" % name } response.set_status(500) except: params = { "response": "failure", "message": "instance %s failure" % name } self.response.headers['Content-Type'] = "application/json" return self.render_template('api/response.json', **params) # just the status elif command == "status": params = {"instance": instance} self.response.headers['Content-Type'] = "application/json" return self.render_template('api/instance.json', **params) # delete the instance - C'est la vie elif command == "delete": instance.key.delete() # let the tender script delete it params = { "response": "success", "message": "Instance marked to be deleted." } self.response.headers['Content-Type'] = "application/json" return self.render_template('api/response.json', **params) # rename it elif command == "rename": renamed = self.request.get('renamed') instance.renamed = renamed instance.put() params = {"instance": instance} self.response.headers['Content-Type'] = "application/json" return self.render_template('api/instance.json', **params) else: params = { "response": "failure", "message": "bad command, skippy" } self.response.set_status(500) self.response.headers['Content-Type'] = "application/json" return self.render_template('api/response.json', **params)
def post(self, sid=None): # a POST here is a create instance event # know the user user_info = User.get_by_id(long(self.user_id)) if sid and user_info.email: # get form values stream = Stream.get_by_sid(sid) try: size = stream.instance_size except: size = 0 # look up user's instances db_instances = Instance.get_all() # check the user's limits instance_count = 0 for db_instance in db_instances: # limit to instances the user has started if db_instance.user == user_info.key: instance_count = instance_count + 1 # warn and redirect if limit is reached if (instance_count + 1) > user_info.max_instances: self.add_message( 'Instance limit reached. This account may only start %s instances. Please delete an existing instance to start a new one!' % user_info.max_instances, 'warning') return self.redirect_to('instances-list') ## HOT START # check for a hot start instances = Instance.get_hotstarts() for instance in instances: # fiveminutesago depends on number of seconds at end of this *** fiveminutesago = datetime.datetime.now() - datetime.timedelta( 0, 900) # if this hotstart instance has a matching sid, assign and redirect to it if instance.created < fiveminutesago and instance.stream.get( ).sid == stream.sid and instance.status == "RUNNING": # map to user instance.user = user_info.key instance.hotstart = False instance.put() self.add_message( 'Instance assigned! Use login buttons to access %s.' % stream.name, 'success') slack.slack_message("Instance type %s assigned for %s!" % (stream.name, user_info.username)) return self.redirect_to('instance-detail', name=instance.name) # ## TRATS TOH # make the instance call handle http = httplib2.Http(timeout=10) # where and who created it (labels for google cloud console) if config.isdev: iuser = "******" % ("dev", user_info.username.lower()) else: iuser = "******" % ("prod", user_info.username.lower()) # build url to create new instance from stream url = '%s/api/stream/%s?token=%s&user=%s&size=%s' % ( config.fastener_host_url, sid, config.fastener_api_token, iuser, size) try: # pull the response back TODO add error handling response, content = http.request(url, 'POST', None, headers={}) gcinstance = json.loads(content) name = gcinstance['instance'] password = gcinstance['password'] if name == "failed": raise Exception("Instance start failed.") # set up an instance instance = Instance(name=name, status="PROVISIONING", user=user_info.key, stream=stream.key, size=size, password=password, expires=datetime.datetime.now() + datetime.timedelta(0, 604800), started=datetime.datetime.now()) instance.put() slack.slack_message("Instance type %s created for %s!" % (stream.name, user_info.username)) # give the db a second to update if config.isdev: time.sleep(1) self.add_message( 'Instance created! Grab some coffee and wait for %s to start.' % stream.name, 'success') params = {'name': name} return self.redirect_to('instance-detail', **params) except: self.add_message( 'The system is currently busy with other instances. Please try again in a few minutes.', 'warning') return self.redirect_to('instances-list') else: # email update sumbission if not self.form.validate(): self.add_message( "There were errors validating your email address.", "error") return self.get() email = self.form.email.data.strip() user_info = User.get_by_id(long(self.user_id)) user_info.email = email.strip() user_info.put() if len(email) > 3 and not config.isdev: name = user_info.name try: mc = MarketoClient(config.munchkin_id, config.mclient_id, config.mclient_secret) try: first = name.split()[0] except: first = "" try: last = name.split()[1] except: last = "" try: company = user_info.company except: company = "None" leads = [{ "email": user_info.email, "firstName": first, "lastName": last, "company": company, "leadSource": config.mclient_leadSource }] lead = mc.execute( method='push_lead', leads=leads, lookupField='email', programName=config.mclient_programName, programStatus=config.mclient_programStatus) except Exception as ex: slack.slack_message( "Marketo lead create failed because %s." % ex) slack.slack_message( "We got %s to update their email for instance launch!" % user_info.username) self.add_message("Thank you! Your email has been updated.", 'success') # redirect back to GET on list, but with a sid AND email in place this time to create if sid: return self.redirect_to('streams-start3', sid=sid) else: return self.redirect_to('instances-list')
def get(self, sid): # know the user user_info = User.get_by_id(long(self.user_id)) # check if we have their email if not user_info.email: self.add_message( 'Please update your email address before starting an instance!', 'warning') return self.redirect_to('account-settings') # look up user's instances db_instances = Instance.get_all() # check the user's limits instance_count = 0 for db_instance in db_instances: # limit to instances the user has started if db_instance.user == user_info.key: instance_count = instance_count + 1 # warn and redirect if limit is reached if (instance_count + 1) > user_info.max_instances: self.add_message( 'Instance limit reached. This account may only start %s instances. Please delete an existing instance to start a new one!' % user_info.max_instances, 'warning') return self.redirect_to('instances-list') # get stream stream = Stream.get_by_sid(sid) try: size = stream.instance_size except: size = 0 ## HOT START # check for a hot start instances = Instance.get_hotstarts() for instance in instances: # fiveminutesago depends on number of seconds at end of this *** fiveminutesago = datetime.datetime.now() - datetime.timedelta( 0, 900) # if this hotstart instance has a matching sid, assign and redirect to it if instance.created < fiveminutesago and instance.stream.get( ).sid == stream.sid and instance.status == "RUNNING": # map to user instance.user = user_info.key instance.hotstart = False instance.put() self.add_message( 'Instance assigned! Use login buttons to access %s.' % stream.name, 'success') slack.slack_message("Instance type %s assigned for %s!" % (stream.name, user_info.username)) return self.redirect_to('instance-detail', name=instance.name) # ## TOH TRATS # make the instance call to the control box http = httplib2.Http(timeout=10) # where and who created it if config.isdev: iuser = "******" % ("dev", user_info.username) else: iuser = "******" % ("prod", user_info.username) url = '%s/api/stream/%s?token=%s&user=%s&size=%s' % ( config.fastener_host_url, sid, config.fastener_api_token, iuser, size) try: # pull the response back TODO add error handling response, content = http.request(url, 'POST', None, headers={}) gcinstance = json.loads(content) name = gcinstance['instance'] password = gcinstance['password'] if name == "failed": raise Exception("Instance start failed.") # set up an instance (note there are two ways to create an instance - see below) instance = Instance(name=name, status="PROVISIONING", user=user_info.key, stream=stream.key, size=size, password=password, expires=datetime.datetime.now() + datetime.timedelta(0, 604800), started=datetime.datetime.now()) instance.put() slack.slack_message("Instance type %s created for %s!" % (stream.name, user_info.username)) # give the db a second to update if config.isdev: time.sleep(1) self.add_message( 'Instance created! Give the system a few minutes to start %s.' % stream.name, 'success') params = {'name': name} return self.redirect_to('instance-detail', **params) except: self.add_message( 'The system is currently busy with other instances. Please try again in a few minutes.', 'warning') return self.redirect_to('instances-list')
def get(self): # list of streams from db streams = Stream.get_all() # get list of host start instances from db instances = Instance.get_hotstarts() # for return of started/deleted instances sinstances = [] dinstances = [] # blast old ones for instance in instances: fiveminutesago = datetime.datetime.now() - datetime.timedelta( 0, 14400) # not five minutes, people if instance.created < fiveminutesago: # this instance is SO less (older) than some epoch seconds ago ^ dinstances.append(instance) instance.key.delete() slack.slack_message( "Hotstart instance %s deleted for being at risk for preemption!" % instance.name) if instance.status == "TERMINATED": # if this instance was started and then preempted dinstances.append(instance) instance.key.delete() slack.slack_message( "Hotstart instance %s deleted due to being terminated!" % instance.name) # for return of started instances sinstances = [] # loop over the 'templates' (streams) for stream in streams: running = 0 # check to see if some are running for instance in instances: if instance.stream.get().sid == stream.sid: running = running + 1 try: hot_starts = stream.hot_starts except: hot_starts = 0 try: size = stream.instance_size except: size = 0 if running < hot_starts: # start up an extra instance for this stream (max starts 1 per minute per template) # make the instance call handler http = httplib2.Http(timeout=10) # where and who created it (labels for google cloud console) if config.isdev: iuser = "******" % ("dev", "hotstart") else: iuser = "******" % ("prod", "hotstart") # build url to call create new instance from stream on fastener box (our instance) url = '%s/api/stream/%s?token=%s&user=%s&size=%s' % ( config.fastener_host_url, stream.sid, config.fastener_api_token, iuser, size) try: # pull the response back TODO add more better error handling response, content = http.request(url, 'POST', None, headers={}) gcinstance = json.loads(content) name = gcinstance['instance'] password = gcinstance['password'] if name == "failed": raise Exception("Instance start failed.") # set up an instance instance = Instance(name=name, status="PROVISIONING", stream=stream.key, size=size, hotstart=True, password=password, expires=datetime.datetime.now() + datetime.timedelta(0, 604800), started=datetime.datetime.now()) instance.put() # for return sinstances.append(instance) slack.slack_message( "Instance type %s created for HOTSTART!" % stream.name) except Exception as ex: print ex params = {"sinstances": sinstances, "dinstances": dinstances} self.response.headers['Content-Type'] = "application/json" return self.render_template('api/hotstarts.json', **params)
def get(self): try: # grab a list of instances from the Fastener API http = httplib2.Http(timeout=30) url = '%s/api/instance/list?token=%s' % (config.fastener_host_url, config.fastener_api_token) response, content = http.request(url, 'GET') # list of instances from Google Cloud (see ./fastener/sample-output.json) gcinstances = json.loads(content) if len(gcinstances) > 0: message = "ok" else: message = "no instances were returned from fastener API" except Exception as ex: gcinstances = [] slack.slack_message( "Tender::Exception with list query to fastener box.") message = "failed to contact fastener API" # list of instances from db instances = Instance.get_all() # bail if we didn't get any instances from Google if len(gcinstances) == 0: params = { "message": message, "gc_count": len(gcinstances), "db_count": len(instances) } slack.slack_message("Tender::No instances from Google?") self.response.headers['Content-Type'] = "application/json" return self.render_template('api/tender.json', **params) ###### # loop through list of instances in local or production DB for instance in instances: name = instance.name found = False # loop through the instances we got from google for gcinstance in gcinstances: # check if the names match if name == gcinstance['name']: # got a match found = True try: # grab the IP address and status instance.ip = gcinstance['networkInterfaces'][0][ 'accessConfigs'][0]['natIP'] except: # got limited or no data about instance instance.ip = "None" # leave commented out slack.slack_message("Tender::%s has status %s" % (instance.name, instance.status)) # if not BUILDING then update status right from google if instance.status not in ("BUILDING"): instance.status = gcinstance['status'] # at this point if the instance.status is NOT BUILDING, then we grab # whatever google says the instance is doing. if it's RUNNING, then # we do an instance link test to ensure the fusion service is running # if Fusion does not respond, we set it to CONFIGURING # BUILDING state is kept until an update of BUILDING state is done # see APIs # are we running? if instance.status == "RUNNING": # check if the box is running fusion admin yet (CONFIGURING if NOT) try: # fast fail connection for checking if fusion is up http_test = httplib2.Http(timeout=2) test_url = 'http://%s:8764' % instance.ip response, content = http_test.request( test_url, 'GET') test_status = response['status'] except: slack.slack_message( "Tender::%s FAILED Fusion port test." % (instance.name)) test_status = "404" # set admin_link if box is running and test comes back 200 if test_status == "200": instance.admin_link = test_url # build app link and update, if it exists try: if instance.stream.get().app_stub: app_stub = instance.stream.get().app_stub instance.app_link = "http://%s%s" % ( instance.ip, app_stub) else: instance.app_link = None except: instance.app_link = None else: # show the box is in CONFIGURING instance.status = "CONFIGURING" instance.admin_link = None instance.app_link = None instance.put() else: # NOT RUNNING STATUS (FROM GOOGLE) OR BUILDING (FROM DEPLOY SCRIPT) # should we start it? if instance.tender_action == "START": # try to start it http = httplib2.Http(timeout=10) url = '%s/api/instance/%s/start?token=%s' % ( config.fastener_host_url, name, config.fastener_api_token) try: # pull the response back TODO add error handling response, content = http.request(url, 'GET', None, headers={}) # update if google returns pending if json.loads(content)[ 'status'] == "PENDING" or json.loads( content)['status'] == "DONE": params = { "response": "success", "message": "instance %s started" % name } instance.status = "PROVISIONING" instance.tender_action = "NONE" instance.started = datetime.datetime.now() instance.put() slack.slack_message( "Tender::%s wanted and got a START" % (instance.name)) else: slack.slack_message( "Tender::%s wanted START but google reported %s" % (instance.name, json.loads(content)['status'])) except Exception as ex: slack.slack_message( "Tender::Exception %s with %s" % (ex, instance.name)) else: # slack.slack_message("Tender::%s not requesting any actions." % instance.name) pass # instance has been terminated if instance.status == "TERMINATED": # set start time to far in the past instance.started = instance.created - datetime.timedelta( 0, 604800) # make sure we write all changes out instance.put() break # no need to keep looking else: # instance doesn't match the one we're working on pass if not found: # box wasn't found on GCP (via fastener LIST call) slack.slack_message( "Instance %s noted not being on GCP - looking" % name) http = httplib2.Http(timeout=10) url = '%s/api/instance/%s/status?token=%s' % ( config.fastener_host_url, name, config.fastener_api_token) # pull the response back response, content = http.request(url, 'GET', None, headers={}) result = json.loads(content) if not result: # we could not verify box was or wasn't running (fastener might not be running) slack.slack_message("Can't tell what is going on.") pass else: try: if result['error'] == "NOTFOUND": slack.slack_message( "Deleting instance %s from DB for not being on GCP." % name) instance.key.delete() except: # no error # why are we here, we got a response this box is running pass else: # no instances in db pass # cleanup stray and EOL instances for gcinstance in gcinstances: instance = Instance.get_by_name(gcinstance['name']) if instance: name = instance.name # if instance is expired, end it if instance.expires < datetime.datetime.now(): # make the instance call to the control box try: http = httplib2.Http(timeout=10) url = '%s/api/instance/%s/delete?token=%s' % ( config.fastener_host_url, name, config.fastener_api_token) # pull the response back response, content = http.request(url, 'GET', None, headers={}) if content['status'] == "PENDING": instance.key.delete() slack.slack_message( "DELETING instance %s's from GCP because expired." % name) except: slack.slack_message( "ERROR: failed deleting instance %s's from GCP because expired." % name) else: # instance wasn't found in db name = gcinstance['name'] # make sure we don't delete non-demo or prod instances if 'button' in name and config.isdev == False: # i.e. put 'button' in an instance name & this will delete the instance slack.slack_message( "Not found in DB. Will try to delete instance %s's from Google Cloud." % name) # make the instance call to the control box # THIS IS THE DANGEROUS BITS try: http = httplib2.Http(timeout=10) url = '%s/api/instance/%s/delete?token=%s' % ( config.fastener_host_url, name, config.fastener_api_token) # pull the response back response, content = http.request(url, 'GET', None, headers={}) if content['status'] == "PENDING": slack.slack_message( "DELETING instance %s's from Google Cloud." % name) else: slack.slack_message( "ERROR: funky content returned while deleting instance %s's from Google Cloud." % name) except: slack.slack_message( "ERROR: failed deleting instance %s from Google Cloud." % name) else: # no instances from cloud - this should never run pass params = { "message": message, "gc_count": len(gcinstances), "db_count": len(instances) } self.response.headers['Content-Type'] = "application/json" return self.render_template('api/tender.json', **params)
def post(self, sid=None): # topic (stackexchange subdomain, for now) topic = self.request.get('topic') if not topic: topic = "ai" # region region = self.request.get('region') if not region: region = "any" # preemptible request (add check user perms) preemptible = self.request.get('preemptible') if not preemptible: preemptible = 1 # preemptible else: if preemptible == "False" or preemptible == "false" or int( preemptible) == 0: preemptible = 0 # not preemptible else: preemptible = 1 # check token token = self.request.get('token') if token != "": user_info = User.get_by_token(token) if user_info: # look up streams stream = Stream.get_by_sid(sid) if not stream: params = { "response": "fail", "message": "stream %s does not exist on these endpoints" % sid } return self.render_template('api/response.json', **params) try: size = stream.instance_size except: size = 0 # look up user's instances db_instances = Instance.get_all() # check the user's limits instance_count = 0 for db_instance in db_instances: # limit to instances the user has started (doing it this way because can't figure out ndb indexing) if db_instance.user == user_info.key: instance_count = instance_count + 1 # check preemtibility ability if preemptible == 0: if user_info.superuser == False: # if the user can't start preemptible tell them params = { "response": "fail", "message": "token may not create non-preemptible instance" } return self.render_template('api/response.json', **params) # warn and redirect if limit is reached if (instance_count + 1) > user_info.max_instances: params = { "response": "fail", "message": "max instances reached for provided token" } return self.render_template('api/response.json', **params) # make the instance call to the control box http = httplib2.Http(timeout=10) # where and who created it if config.isdev: iuser = "******" % ("dev", user_info.username) else: iuser = "******" % ("prod", user_info.username) url = '%s/api/stream/%s?token=%s&user=%s&topic=%s®ion=%s&preemptible=%s&size=%s' % ( config.fastener_host_url, sid, config.fastener_api_token, iuser, topic, region, preemptible, size) try: # pull the response back TODO add error handling # CREATE HAPPENS HERE response, content = http.request(url, 'POST', None, headers={}) gcinstance = json.loads(content) name = gcinstance['instance'] password = gcinstance['password'] if name == "failed": raise Exception( "Instance start was delayed due to quota. Try again in a few minutes." ) # set up an instance instance = Instance(name=name, status="PROVISIONING", user=user_info.key, stream=stream.key, size=size, region=region, topic=topic, preemptible=bool(preemptible), password=password, expires=datetime.datetime.now() + datetime.timedelta(0, 604800), started=datetime.datetime.now()) instance.put() try: slack.slack_message( "Instance type %s created for %s!" % (stream.name, user_info.username)) except: print "slack failing" # give the db a second to update if config.isdev: time.sleep(3) params = {'instance': instance} self.response.headers['Content-Type'] = "application/json" return self.render_template('api/instance.json', **params) except Exception as ex: # the horror params = { "response": "fail", "message": "exception thrown: %s" % ex } return self.render_template('api/response.json', **params) # no token, no user, no data self.response.status = '402 Payment Required' self.response.status_int = 402 self.response.headers['Content-Type'] = "application/json" params = { "response": "fail", "message": "must include [token] with a valid token" } return self.render_template('api/response.json', **params)