def start(self):
""" Starts web2py server """
#password = self.password.get()
password = '******'
if not password:
self.error('no password, no web admin interface')
#ip = self.selected_ip.get()
ip = "127.0.0.1"
if not is_valid_ip_address(ip):
return self.error('invalid host ip address')
try:
#port = int(self.port_number.get())
port = 8000
except:
return self.error('invalid port number')
# Check for non default value for ssl inputs
if (len(self.options.ssl_certificate) > 0 or
len(self.options.ssl_private_key) > 0):
proto = 'https'
else:
proto = 'http'
self.url = get_url(ip, proto=proto, port=port)
self.connect_pages()
#self.button_start.configure(state='disabled')
try:
options = self.options
req_queue_size = options.request_queue_size
self.server = main.HttpServer(
ip,
port,
password,
pid_filename=options.pid_filename,
log_filename=options.log_filename,
profiler_dir=options.profiler_dir,
ssl_certificate=options.ssl_certificate,
ssl_private_key=options.ssl_private_key,
ssl_ca_certificate=options.ssl_ca_certificate,
min_threads=options.minthreads,
max_threads=options.maxthreads,
server_name=options.server_name,
request_queue_size=req_queue_size,
timeout=options.timeout,
shutdown_timeout=options.shutdown_timeout,
path=options.folder,
interfaces=options.interfaces)
thread.start_new_thread(self.server.start, ())
except Exception, e:
#self.button_start.configure(state='normal')
return self.error(str(e))
def test_is_valid_ip_address(self):
# IPv4
# False
# self.assertEqual(is_valid_ip_address('127.0'), False) # Fail with AppVeyor?? should pass
self.assertEqual(is_valid_ip_address('unknown'), False)
self.assertEqual(is_valid_ip_address(''), False)
# True
self.assertEqual(is_valid_ip_address('127.0.0.1'), True)
self.assertEqual(is_valid_ip_address('localhost'), True)
self.assertEqual(is_valid_ip_address('::1'), True)
# IPv6
# True
# Compressed
self.assertEqual(is_valid_ip_address('::ffff:7f00:1'), True) # IPv6 127.0.0.1 compressed
self.assertEqual(is_valid_ip_address('2001:660::1'), True)
# Expanded
self.assertEqual(is_valid_ip_address('0:0:0:0:0:ffff:7f00:1'), True) # IPv6 127.0.0.1 expanded
self.assertEqual(is_valid_ip_address('2607:fa48:6d50:69f1:21f:3cff:fe9d:9be3'), True) # Any address
def get_client(env):
"""
guess the client address from the environment variables
first tries 'http_x_forwarded_for', secondly 'remote_addr'
if all fails, assume '127.0.0.1' or '::1' (running locally)
"""
eget = env.get
g = regex_client.search(eget('http_x_forwarded_for', ''))
client = (g.group() or '').split(',')[0] if g else None
if client in (None, '', 'unknown'):
g = regex_client.search(eget('remote_addr', ''))
if g:
client = g.group()
elif env.http_host.startswith('['): # IPv6
client = '::1'
else:
client = '127.0.0.1' # IPv4
if not is_valid_ip_address(client):
raise HTTP(400, "Bad Request (request.client=%s)" % client)
return client
def get_client(env):
"""
Guesses the client address from the environment variables
First tries 'http_x_forwarded_for', secondly 'remote_addr'
if all fails, assume '127.0.0.1' or '::1' (running locally)
"""
eget = env.get
g = regex_client.search(eget('http_x_forwarded_for', ''))
client = (g.group() or '').split(',')[0] if g else None
if client in (None, '', 'unknown'):
g = regex_client.search(eget('remote_addr', ''))
if g:
client = g.group()
elif env.http_host.startswith('['): # IPv6
client = '::1'
else:
client = '127.0.0.1' # IPv4
if not is_valid_ip_address(client):
raise HTTP(400, "Bad Request (request.client=%s)" % client)
return client
def get_client(env):
"""
Guesses the client address from the environment variables
First tries 'http_x_forwarded_for', secondly 'remote_addr'
if all fails, assume '127.0.0.1' or '::1' (running locally)
"""
eget = env.get
m = REGEX_CLIENT.search(eget('http_x_forwarded_for', ''))
client = m and m.group()
if client in (None, '', 'unknown'):
m = REGEX_CLIENT.search(eget('remote_addr', ''))
if m:
client = m.group()
elif env.http_host.startswith('['): # IPv6
client = '::1'
else:
client = '127.0.0.1' # IPv4
if not is_valid_ip_address(client):
raise HTTP(400, "Bad Request (request.client=%s)" % client)
return client
def start(self):
""" Starts web2py server """
password = self.password.get()
if not password:
self.error('no password, no web admin interface')
ip = self.selected_ip.get()
if not is_valid_ip_address(ip):
return self.error('invalid host ip address')
try:
port = int(self.port_number.get())
except:
return self.error('invalid port number')
if self.options.server_key and self.options.server_cert:
proto = 'https'
else:
proto = 'http'
self.url = get_url(ip, proto=proto, port=port)
self.connect_pages()
self.button_start.configure(state='disabled')
try:
options = self.options
req_queue_size = options.request_queue_size
self.server = main.HttpServer(
ip,
port,
password,
pid_filename=options.pid_filename,
log_filename=options.log_filename,
profiler_dir=options.profiler_dir,
ssl_certificate=options.server_cert,
ssl_private_key=options.server_key,
ssl_ca_certificate=options.ca_cert,
min_threads=options.min_threads,
max_threads=options.max_threads,
server_name=options.server_name,
request_queue_size=req_queue_size,
timeout=options.timeout,
shutdown_timeout=options.shutdown_timeout,
path=options.folder,
interfaces=options.interfaces)
thread.start_new_thread(self.server.start, ())
except Exception as e:
self.button_start.configure(state='normal')
return self.error(str(e))
if not self.server_ready():
self.button_start.configure(state='normal')
return
self.button_stop.configure(state='normal')
if not options.taskbar:
thread.start_new_thread(
start_browser, (get_url(ip, proto=proto, port=port), True))
self.password.configure(state='readonly')
[ip.configure(state='disabled') for ip in self.ips.values()]
self.port_number.configure(state='readonly')
if self.tb:
self.tb.SetServerRunning()
def start(self):
""" Starts web2py server """
password = self.password.get()
if not password:
self.error('no password, no web admin interface')
ip = self.selected_ip.get()
if not is_valid_ip_address(ip):
return self.error('invalid host ip address')
try:
port = int(self.port_number.get())
except:
return self.error('invalid port number')
# Check for non default value for ssl inputs
if (len(self.options.ssl_certificate) > 0 or
len(self.options.ssl_private_key) > 0):
proto = 'https'
else:
proto = 'http'
self.url = get_url(ip, proto=proto, port=port)
self.connect_pages()
self.button_start.configure(state='disabled')
try:
options = self.options
req_queue_size = options.request_queue_size
self.server = main.HttpServer(
ip,
port,
password,
pid_filename=options.pid_filename,
log_filename=options.log_filename,
profiler_dir=options.profiler_dir,
ssl_certificate=options.ssl_certificate,
ssl_private_key=options.ssl_private_key,
ssl_ca_certificate=options.ssl_ca_certificate,
min_threads=options.minthreads,
max_threads=options.maxthreads,
server_name=options.server_name,
request_queue_size=req_queue_size,
timeout=options.timeout,
shutdown_timeout=options.shutdown_timeout,
path=options.folder,
interfaces=options.interfaces)
thread.start_new_thread(self.server.start, ())
except Exception as e:
self.button_start.configure(state='normal')
return self.error(str(e))
if not self.server_ready():
self.button_start.configure(state='normal')
return
self.button_stop.configure(state='normal')
if not options.taskbar:
thread.start_new_thread(
start_browser, (get_url(ip, proto=proto, port=port), True))
self.password.configure(state='readonly')
[ip.configure(state='disabled') for ip in self.ips.values()]
self.port_number.configure(state='readonly')
if self.tb:
self.tb.SetServerRunning()
def ip_addr(v):
if not is_valid_ip_address(v):
raise argparse.ArgumentTypeError("bad IP address %s" % v)
return v
def start(self):
""" Starts web2py server """
password = self.password.get()
if not password:
self.error('no password, no web admin interface')
ip = self.selected_ip.get()
if not is_valid_ip_address(ip):
return self.error('invalid host ip address')
try:
port = int(self.port_number.get())
except ValueError:
return self.error('invalid port number')
if self.options.server_key and self.options.server_cert:
proto = 'https'
else:
proto = 'http'
self.url = get_url(ip, proto=proto, port=port)
self.connect_pages()
self.update_schedulers()
# softcron is stopped with HttpServer, thus if starting again
# need to reset newcron._stopping to re-enable cron
if self.options.soft_cron:
newcron.reset()
# FIXME: if the HttpServer is stopped, then started again,
# does not start because of following error:
# WARNING:Rocket.Errors.Port8000:Listener started when not ready.
self.button_start.configure(state='disabled')
try:
options = self.options
req_queue_size = options.request_queue_size
self.server = main.HttpServer(
ip,
port,
password,
pid_filename=options.pid_filename,
log_filename=options.log_filename,
profiler_dir=options.profiler_dir,
ssl_certificate=options.server_cert,
ssl_private_key=options.server_key,
ssl_ca_certificate=options.ca_cert,
min_threads=options.min_threads,
max_threads=options.max_threads,
server_name=options.server_name,
request_queue_size=req_queue_size,
timeout=options.timeout,
shutdown_timeout=options.shutdown_timeout,
path=options.folder,
interfaces=options.interfaces)
threading.Thread(target=self.server.start).start()
except Exception as e:
self.button_start.configure(state='normal')
return self.error(str(e))
if not self.server_ready():
self.button_start.configure(state='normal')
return
self.button_stop.configure(state='normal')
if not options.taskbar:
cpt = threading.Thread(target=start_browser,
args=(get_url(ip, proto=proto, port=port), True))
cpt.setDaemon(True)
cpt.start()
self.password.configure(state='readonly')
for ip in self.ips.values():
ip.configure(state='disabled')
self.port_number.configure(state='readonly')
if self.tb:
self.tb.SetServerRunning()
def ip_addr(v):
if not is_valid_ip_address(v):
raise argparse.ArgumentTypeError("bad IP address %s" % v)
return v
def serverTask(accountId, appId, devPhase, region, kvCheck, pbPath=None):
# TODO: it would be nice to store these in a nice protected dict
# and then write out the key to disk only for running the playbook...
# We could put all of them in and identify a default key. Then if we
# have an entry for the ssh key identified by the host, use that....
sshKeyFilePath = "/home/ec2-user/.ssh/msca-devops.pem"
# Directory to write out inventories and playbooks...
runtimeDir = "/data/parapet/"
grid = {}
grid['validHosts'] = {}
logger.debug("Task UUID: {0}".format(W2P_TASK.uuid))
logger.debug("Account ID: {0}".format(accountId))
if int(accountId) == 1:
logger.debug("Setting account Query to all accounts")
accountQuery = db.hostInfo.accountNumber > 1
else:
accountQuery = db.hostInfo.accountNumber == accountId
logger.debug("Application: '{0}'".format(appId))
if appId == "All Applications":
appQuery = db.hostInfo.app.like('%')
else:
appQuery = db.hostInfo.app == appId
logger.debug("DevPhase: {0}".format(devPhase))
if len(devPhase) > 1:
devQuery = db.hostInfo.devPhase == devPhase
else:
logger.debug("Setting devPhase to %")
devQuery = db.hostInfo.devPhase.like('%')
logger.debug("Region: {0}".format(region))
if len(region) > 1:
regionQuery = db.hostInfo.region == region
else:
logger.debug("Setting region to %")
regionQuery = db.hostInfo.region.like('%')
logger.debug("hostFilter: {0}".format(kvCheck))
hostFilter = json.loads(kvCheck)
try:
hostFilter['awsInfo']['ec2_state'] = 'running'
except KeyError:
hostFilter['awsInfo'] = {}
hostFilter['awsInfo']['ec2_state'] = 'running'
logger.debug("HF: {0}".format(hostFilter))
# Get the hosts that match the base query
dbQuery = ((accountQuery)&(appQuery)&(devQuery)&(regionQuery))
s = db(dbQuery)
rows = s.select()
# Iterate through the core hosts and apply the hostFilter
for row in rows:
# Get the host data from the notes field
hostNotes = json.loads(row['notes'])
# Verify that all of the things in the hostFilter are true
for key in hostFilter.keys():
if hostNotes.has_key(key):
for check in hostFilter[key].keys():
try:
if hostFilter[key][check] == hostNotes[key][check]:
if grid['validHosts'].has_key(row['instance_id']) is False:
# Passes the test, set the AWS instanceID to the databaseID
grid['validHosts'][row['instance_id']] = row['id']
# If this host has already failed a prior test, don't add it now
elif grid['validHosts'][row['instance_id']] is None:
pass
else:
# Host fails the test, set it to None (clean it up later)
grid['validHosts'][row['instance_id']] = None
except KeyError:
# If the host doesn't have a matching key, then it doesn't match the filter
grid['validHosts'][row['instance_id']] = None
# Get rid of the hosts that don't match the hostFilter
for key in grid['validHosts'].keys():
if grid['validHosts'][key] is None:
del grid['validHosts'][key]
logger.debug("HostIDs: {0}".format(grid['validHosts'].values()))
logger.debug("This search found {0} hosts".format(len(grid['validHosts'])))
# Download and parse playbook file here... write it out as:
# runtimeDir/Task_UUID.yml
# use serializers.loads_yaml()
if pbPath:
pbData = serializers.loads_yaml(urllib2.urlopen(pbPath).read())
hostGroup = pbData[0]['hosts']
fileutils.write_file(os.path.join(runtimeDir,W2P_TASK.uuid+".yml"),serializers.yaml(pbData))
# Generate inventory file
# runtimeDir/Task_UUID.inv
# Need to parse out teh playbook file first to determine what group we should put the hosts in.
invHosts = "[{0}]\n".format(hostGroup)
for row in db(db.hostInfo.id.belongs(grid['validHosts'].values())).select():
hostNotes = serializers.loads_json(row.notes)
if utils.is_valid_ip_address(hostNotes['awsInfo']['ec2_ip_address']):
sshKeyFilePath = os.path.join(runtimeDir,W2P_TASK.uuid+hostNotes['awsInfo']['ec2_ip_address']+".key")
try:
thisHostKey = serverTask_config.keyData[hostNotes['awsInfo']['ec2_key_name']]
sshKeyFilePath = os.path.join(runtimeDir,W2P_TASK.uuid+hostNotes['awsInfo']['ec2_ip_address']+".key")
except KeyError:
logger.debug("Unable to find a key named {0} using default".format(hostNotes['awsInfo']['ec2_key_name']))
thisHostKey = serverTask_config.keyData['default']
fileutils.write_file(sshKeyFilePath,thisHostKey)
os.chmod(sshKeyFilePath,0600)
thisHostString = "{0} ansible_ssh_host={1} ansible_ssh_private_key_file={2} ansible_ssh_user=ec2-user\n".format(row.instance_id,hostNotes['awsInfo']['ec2_ip_address'],sshKeyFilePath)
invHosts = "{0} {1}".format(invHosts,thisHostString)
else:
logger.warn("{0} is not a valid IP address".format(hostNotes['awsInfo']['ec2_ip_address']))
fileutils.write_file(os.path.join(runtimeDir,W2P_TASK.uuid+".inv"),invHosts)
# Run the task
cmdLine = "/usr/bin/ansible-playbook -e 'parapetServer=true' -vvv -i {invFile} {pbFile}".format(invFile=os.path.join(runtimeDir,W2P_TASK.uuid+".inv"),pbFile=os.path.join(runtimeDir,W2P_TASK.uuid+".yml"))
logger.debug("{0}".format(cmdLine))
#ansibleOut = subprocess.Popen(cmdLine, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
ansible = subprocess.Popen(cmdLine,shell=True,cwd=runtimeDir,stdout=subprocess.PIPE)
while ansible.poll() is None:
# Only 1 newline??
print ansible.stdout.readline(),
#logger.info(ansibleOut)
#print ansibleOut
try:
keyFiles = fileutils.listdir(runtimeDir,expression="^"+W2P_TASK.uuid+".*\.key$", drop=False)
for keyFile in keyFiles:
logger.debug("Removing: {0}".format(keyFile))
fileutils.recursive_unlink(keyFile)
except:
logger.error("Unable to remove key files")
return 0
