def console_output(self): from webapp.libs.openstack import instance_console response = instance_console(self) if response['response'] == "error": raise Exception("Failed to get console_output.") if response['response'] == "success": lines = response['result']['console'].split('\n') return lines return []
def housekeeping(self): from webapp.libs.openstack import instance_info from webapp.libs.openstack import instance_decommission from webapp.libs.openstack import instance_console # build the response response = {"response": "success", "result": {"message": "", "server": {}}} # get instance (server) info cluster_response = instance_info(self) server = cluster_response['result']['server'] # we all have limited time in this reality epoch_time = int(time.time()) # set start state start_state = self.state # this is complicated...because we aren't EC with OpenStack...or I'm crazy if cluster_response['response'] == "success": # openstack responded it found this instance if server.status == "ACTIVE": # openstack says the server is running if self.expires < epoch_time: # suspend the instance for non-payment try: self.suspend() response['result']['message'] = "Instance %s suspended." % self.name except Exception as e: response['response'] = 'error' response['result']['message'] = \ 'Instance {instance} suspending failed: "{error}".'.format( instance=self.name, error=str(e)) return response self.state = 5 elif self.expires > epoch_time: # openstack says we're running, and we're paid if self.state == 5 or self.state == 6: # we move the instance to starting mode response['result']['message'] = "Instance %s is starting." % self.name self.state = 3 elif server.status == "SUSPENDED" or server.status == "SHUTOFF": # openstack says this instance is suspended if self.expires > epoch_time: # should be running because not expired try: self.resume() response['result']['message'] = "Instance %s resumed." % self.name except Exception as e: response['response'] = 'error' response['result']['message'] = \ 'Instance {instance} resume failed: "{error}".'.format( instance=self.name, error=str(e)) return response self.state = 3 # mark as starting if self.expires + app.config['POOL_DECOMMISSION_TIME'] < epoch_time: # should be destroyed (suspended for +2 hours without pay) response['result']['message'] = "Instance %s decommissioned." % self.name self.state = 7 else: # openstack indicates another state besides SUSPENDED or ACTIVE if self.expires > epoch_time: # we should be running, but in a weird state - destroy then restart response = instance_decommission(self) response['result']['message'] = "Instance %s restarted." % self.name self.state = 2 # set as paid and ready to start app.logger.error("OpenStack says instance=(%s) isn't in the correct state. Setting to restart." % self.name) else: # expired but in a weird state - destroy response = instance_decommission(self) response['result']['message'] = "Instance %s decommissioned." % self.name self.state = 7 else: # openstack can't find this instance if self.expires > epoch_time: if self.state == 2: # check error rate if self.message_count > 10: # we're failing to start the instance, so decomission response['result']['message'] = "Instance %s decommissioned." % self.name self.state = 7 app.logger.error("Exceeded error rate on callbacks for instance=(%s). Decomissioning." % self.name) else: # set instance to restart - not expired, should be running response['response'] = "error" # technically, someone is probably f*****g with things response['result']['message'] = "Setting instance %s to restart." % self.name self.state = 2 # will be started shortly after this by start app.logger.error("OpenStack doesn't know about instance=(%s). Setting to restart." % self.name) else: # no reason to be running response['response'] = "error" response['result']['message'] = "Instance %s decommissioned." % self.name self.state = 7 # will be deleted shortly after this by trashman # get instance console output cluster_response = instance_console(self) if 'console' in response['result']: self.console = response['result']['console'] # save updated properties self.save() # make a call to the callback url to report instance details if either the # state has changed or the last state change is less than 900 secs ago if self.state != start_state or self.updated >= int(time.time()) - 900: appliance = Appliance().get() callback_url = self.callback_url pool_response = pool_instances( url=callback_url, instance=self, appliance=appliance) return response
def nudge(self): from webapp.libs.openstack import try_associate_floating_ip from webapp.libs.openstack import instance_info from webapp.libs.openstack import instance_console from webapp.libs.openstack import instance_decommission # get instance console output response = instance_console(self) if 'console' in response['result']: self.console = response['result']['console'] self.update() # get instance (server) info response = instance_info(self) # set start state start_state = self.state # set instance meta data if response['response'] == "success": server = response['result']['server'] # if the state is ACTIVE, we set to be running state==4 if server.status == "ACTIVE": # set network info self.state = 4 # try to get a floating ip for the new server float_response = try_associate_floating_ip(server) # check if call got a floating IP if float_response['response'] == "success": # get instance info again to pick up new IP response = instance_info(self) # load the response into the server object if response['response'] == "success": server = response['result']['server'] else: # log 'errors' in floating assignment app.logger.info(float_response['result']['message']) # extract IP addresses using IPy # in some circumstances this will squash multiple same/same address types # we only extract and store one each of private ipv4, public ipv4, and public ipv6 for key in server.networks.keys(): # any network names for address in server.networks[key]: # loop through each address for each network # private IPv4 if IP(address).iptype() == "PRIVATE" and IP(address).version() == 4: self.privateipv4 = address # public IPv4 elif IP(address).iptype() == "PUBLIC" and IP(address).version() == 4: self.publicipv4 = address # public IPv6 elif IP(address).iptype() == "ALLOCATED ARIN" and IP(address).version() == 6: self.publicipv6 = address # update the instance self.update() # ERROR status from openstack elif server.status == "ERROR": # instance failed to start, so delete and reset to paid response = instance_decommission(self) self.state = 2 # will be started again shortly self.update() response['response'] = "error" response['result']['message'] = "OpenStack errored on instance start." app.logger.error("OpenStack error on starting instance=(%s). Setting to restart." % self.name) # SPAWNING status from openstack else: # we all have limited time in this reality epoch_time = int(time.time()) # wait_timer is 5 minutes after the last update wait_timer = self.updated + 300 # test to see if we are 'hung' on SPAWNING for more than wait_timer if epoch_time > wait_timer: # we're now past when the instance needed to move to RUNNING response = instance_decommission(self) response['response'] = "error" response['result']['message'] = "Setting instance %s to restart." % self.name self.state = 2 # will be started shortly after this by start self.updated = epoch_time # given we 'timed' out, give the instance more time self.update() """ of anyplace, this is where you *might* want to add some time to the instance because a time based payment has been made on it. however, this could lead to a situation where an instance gets stuck in a circular state of erroring, getting more time, erroring again, rinse and repeat. instead of embracing this eventuality, we choose to short the customer her measly few cents instead, and let it serve as a as an excuse to add 'karma hits' on bad starts from providers as a feature later """ app.logger.error("OpenStack hung starting instance=(%s). Setting to restart." % self.name) else: # this is a 'soft' fail response['response'] = "error" response['result']['message'] = "Still starting instance=(%s)." % self.name # OpenStack reports instance NOT FOUND else: # we all have limited time in this reality epoch_time = int(time.time()) # we first check if we're outright expired (shouldn't happen) if self.expires < epoch_time: # no reason to be running as we're expired self.state = 7 # will be deleted shortly after this by trashman self.update() response['response'] = "error" response['result']['message'] = "Instance %s decommissioned." % self.name app.logger.error("OpenStack couldn't find expired instance=(%s). Decomissioning." % self.name) else: # we didn't find the instance in openstack, yet we should be running self.state = 2 # set to be started again self.update() response['response'] = "error" response['result']['message'] = "OpenStack couldn't find instance. Restarting." app.logger.error("OpenStack couldn't find instance=(%s). Setting to restart." % self.name) # make a call to the callback url to report instance details on state change if self.state != start_state: appliance = Appliance().get() callback_url = self.callback_url pool_response = pool_instances( url=callback_url, instance=self, appliance=appliance) return response
def coinop(self, amount): # build response response = {"response": "success", "result": {"message": "", "instance": {}}} # calculate the purchased seconds based on payment we received ask = float(self.flavor.ask)/1000000 # BTC per hour try: purchased_seconds = (amount/ask)*3600 # amount in BTC/ask in BTC * seconds in hour except: purchased_seconds = 0 # handle local appliance start if amount == 0: purchased_seconds = 15*60 # give 15 minutes to instance for free # current UTC time in seconds since epoch epoch_time = int(time.time()) # if we're not running (state==1 or 10), set the run state to light (to be started) # if we're suspended (state==5), set the run state to relight (to be unsuspended) # cron jobs will take care of the rest of the job of starting/unsuspending # NOTE: We're getting paid pennies for doing nothing until cronjob runs! if self.state == 1 or self.state == 10: self.state = 2 self.expires = epoch_time + purchased_seconds # starting from now self.updated = epoch_time elif self.state == 5: self.state = 6 self.expires = epoch_time + purchased_seconds # starting from now self.updated = epoch_time else: # states 0, 2, 3, 4, 6, 7 self.expires = self.expires + purchased_seconds # starting from expire time self.updated = epoch_time # get instance console output - only run if we've got an osid # basically this only runs when we get a repayment if self.osid: from webapp.libs.openstack import instance_console response = instance_console(self) if 'console' in response['result']: self.console = response['result']['console'] # update the instance self.update() # make a call to the callback url to report instance details callback_url = self.callback_url appliance = Appliance().get() pool_response = pool_instances( url=callback_url, instance=self, appliance=appliance) if pool_response['response'] == "success": # overload response response['result']['message'] = "Added %s seconds to %s's expire time." % (purchased_seconds, self.name) response['result']['instance'] = row2dict(self) else: # note the error in the instance object self.message_count = self.message_count + 1 self.message = pool_response['result']['message'] self.update() # load response and log response = pool_response app.logger.error("Error sending instance=(%s) data to pool." % self.name) return response
def housekeeping(self): from webapp.libs.openstack import instance_info from webapp.libs.openstack import instance_decommission from webapp.libs.openstack import instance_console # build the response response = { "response": "success", "result": { "message": "", "server": {} } } # get instance (server) info cluster_response = instance_info(self) server = cluster_response['result']['server'] # we all have limited time in this reality epoch_time = int(time.time()) # set start state start_state = self.state # this is complicated...because we aren't EC with OpenStack...or I'm crazy if cluster_response['response'] == "success": # openstack responded it found this instance if server.status == "ACTIVE": # openstack says the server is running if self.expires < epoch_time: # suspend the instance for non-payment try: self.suspend() response['result'][ 'message'] = "Instance %s suspended." % self.name except Exception as e: response['response'] = 'error' response['result']['message'] = \ 'Instance {instance} suspending failed: "{error}".'.format( instance=self.name, error=str(e)) return response self.state = 5 elif self.expires > epoch_time: # openstack says we're running, and we're paid if self.state == 5 or self.state == 6: # we move the instance to starting mode response['result'][ 'message'] = "Instance %s is starting." % self.name self.state = 3 elif server.status == "SUSPENDED" or server.status == "SHUTOFF": # openstack says this instance is suspended if self.expires > epoch_time: # should be running because not expired try: self.resume() response['result'][ 'message'] = "Instance %s resumed." % self.name except Exception as e: response['response'] = 'error' response['result']['message'] = \ 'Instance {instance} resume failed: "{error}".'.format( instance=self.name, error=str(e)) return response self.state = 3 # mark as starting if self.expires + app.config[ 'POOL_DECOMMISSION_TIME'] < epoch_time: # should be destroyed (suspended for +2 hours without pay) response['result'][ 'message'] = "Instance %s decommissioned." % self.name self.state = 7 else: # openstack indicates another state besides SUSPENDED or ACTIVE if self.expires > epoch_time: # we should be running, but in a weird state - destroy then restart response = instance_decommission(self) response['result'][ 'message'] = "Instance %s restarted." % self.name self.state = 2 # set as paid and ready to start app.logger.error( "OpenStack says instance=(%s) isn't in the correct state. Setting to restart." % self.name) else: # expired but in a weird state - destroy response = instance_decommission(self) response['result'][ 'message'] = "Instance %s decommissioned." % self.name self.state = 7 else: # openstack can't find this instance if self.expires > epoch_time: if self.state == 2: # check error rate if self.message_count > 10: # we're failing to start the instance, so decomission response['result'][ 'message'] = "Instance %s decommissioned." % self.name self.state = 7 app.logger.error( "Exceeded error rate on callbacks for instance=(%s). Decomissioning." % self.name) else: # set instance to restart - not expired, should be running response[ 'response'] = "error" # technically, someone is probably f*****g with things response['result'][ 'message'] = "Setting instance %s to restart." % self.name self.state = 2 # will be started shortly after this by start app.logger.error( "OpenStack doesn't know about instance=(%s). Setting to restart." % self.name) else: # no reason to be running response['response'] = "error" response['result'][ 'message'] = "Instance %s decommissioned." % self.name self.state = 7 # will be deleted shortly after this by trashman # get instance console output cluster_response = instance_console(self) if 'console' in response['result']: self.console = response['result']['console'] # save updated properties self.save() # make a call to the callback url to report instance details if either the # state has changed or the last state change is less than 900 secs ago if self.state != start_state or self.updated >= int(time.time()) - 900: appliance = Appliance().get() callback_url = self.callback_url pool_response = pool_instances(url=callback_url, instance=self, appliance=appliance) return response
def nudge(self): from webapp.libs.openstack import try_associate_floating_ip from webapp.libs.openstack import instance_info from webapp.libs.openstack import instance_console from webapp.libs.openstack import instance_decommission # get instance console output response = instance_console(self) if 'console' in response['result']: self.console = response['result']['console'] self.update() # get instance (server) info response = instance_info(self) # set start state start_state = self.state # set instance meta data if response['response'] == "success": server = response['result']['server'] # if the state is ACTIVE, we set to be running state==4 if server.status == "ACTIVE": # set network info self.state = 4 # try to get a floating ip for the new server float_response = try_associate_floating_ip(server) # check if call got a floating IP if float_response['response'] == "success": # get instance info again to pick up new IP response = instance_info(self) # load the response into the server object if response['response'] == "success": server = response['result']['server'] else: # log 'errors' in floating assignment app.logger.info(float_response['result']['message']) # extract IP addresses using IPy # in some circumstances this will squash multiple same/same address types # we only extract and store one each of private ipv4, public ipv4, and public ipv6 for key in server.networks.keys(): # any network names for address in server.networks[ key]: # loop through each address for each network # private IPv4 if IP(address).iptype() == "PRIVATE" and IP( address).version() == 4: self.privateipv4 = address # public IPv4 elif IP(address).iptype() == "PUBLIC" and IP( address).version() == 4: self.publicipv4 = address # public IPv6 elif IP(address).iptype() == "ALLOCATED ARIN" and IP( address).version() == 6: self.publicipv6 = address # update the instance self.update() # ERROR status from openstack elif server.status == "ERROR": # instance failed to start, so delete and reset to paid response = instance_decommission(self) self.state = 2 # will be started again shortly self.update() response['response'] = "error" response['result'][ 'message'] = "OpenStack errored on instance start." app.logger.error( "OpenStack error on starting instance=(%s). Setting to restart." % self.name) # SPAWNING status from openstack else: # we all have limited time in this reality epoch_time = int(time.time()) # wait_timer is 5 minutes after the last update wait_timer = self.updated + 300 # test to see if we are 'hung' on SPAWNING for more than wait_timer if epoch_time > wait_timer: # we're now past when the instance needed to move to RUNNING response = instance_decommission(self) response['response'] = "error" response['result'][ 'message'] = "Setting instance %s to restart." % self.name self.state = 2 # will be started shortly after this by start self.updated = epoch_time # given we 'timed' out, give the instance more time self.update() """ of anyplace, this is where you *might* want to add some time to the instance because a time based payment has been made on it. however, this could lead to a situation where an instance gets stuck in a circular state of erroring, getting more time, erroring again, rinse and repeat. instead of embracing this eventuality, we choose to short the customer her measly few cents instead, and let it serve as a as an excuse to add 'karma hits' on bad starts from providers as a feature later """ app.logger.error( "OpenStack hung starting instance=(%s). Setting to restart." % self.name) else: # this is a 'soft' fail response['response'] = "error" response['result'][ 'message'] = "Still starting instance=(%s)." % self.name # OpenStack reports instance NOT FOUND else: # we all have limited time in this reality epoch_time = int(time.time()) # we first check if we're outright expired (shouldn't happen) if self.expires < epoch_time: # no reason to be running as we're expired self.state = 7 # will be deleted shortly after this by trashman self.update() response['response'] = "error" response['result'][ 'message'] = "Instance %s decommissioned." % self.name app.logger.error( "OpenStack couldn't find expired instance=(%s). Decomissioning." % self.name) else: # we didn't find the instance in openstack, yet we should be running self.state = 2 # set to be started again self.update() response['response'] = "error" response['result'][ 'message'] = "OpenStack couldn't find instance. Restarting." app.logger.error( "OpenStack couldn't find instance=(%s). Setting to restart." % self.name) # make a call to the callback url to report instance details on state change if self.state != start_state: appliance = Appliance().get() callback_url = self.callback_url pool_response = pool_instances(url=callback_url, instance=self, appliance=appliance) return response
def coinop(self, amount): # build response response = { "response": "success", "result": { "message": "", "instance": {} } } # calculate the purchased seconds based on payment we received ask = float(self.flavor.ask) / 1000000 # BTC per hour try: purchased_seconds = ( amount / ask) * 3600 # amount in BTC/ask in BTC * seconds in hour except: purchased_seconds = 0 # handle local appliance start if amount == 0: purchased_seconds = 15 * 60 # give 15 minutes to instance for free # current UTC time in seconds since epoch epoch_time = int(time.time()) # if we're not running (state==1 or 10), set the run state to light (to be started) # if we're suspended (state==5), set the run state to relight (to be unsuspended) # cron jobs will take care of the rest of the job of starting/unsuspending # NOTE: We're getting paid pennies for doing nothing until cronjob runs! if self.state == 1 or self.state == 10: self.state = 2 self.expires = epoch_time + purchased_seconds # starting from now self.updated = epoch_time elif self.state == 5: self.state = 6 self.expires = epoch_time + purchased_seconds # starting from now self.updated = epoch_time else: # states 0, 2, 3, 4, 6, 7 self.expires = self.expires + purchased_seconds # starting from expire time self.updated = epoch_time # get instance console output - only run if we've got an osid # basically this only runs when we get a repayment if self.osid: from webapp.libs.openstack import instance_console response = instance_console(self) if 'console' in response['result']: self.console = response['result']['console'] # update the instance self.update() # make a call to the callback url to report instance details callback_url = self.callback_url appliance = Appliance().get() pool_response = pool_instances(url=callback_url, instance=self, appliance=appliance) if pool_response['response'] == "success": # overload response response['result'][ 'message'] = "Added %s seconds to %s's expire time." % ( purchased_seconds, self.name) response['result']['instance'] = row2dict(self) else: # note the error in the instance object self.message_count = self.message_count + 1 self.message = pool_response['result']['message'] self.update() # load response and log response = pool_response app.logger.error("Error sending instance=(%s) data to pool." % self.name) return response