def logs(self, location, other_args=None): """ 'logs' provides a cleaner experience for users than the appscale-gather-logs command, by using the configuration options present in the AppScalefile found in the current working directory. Args: location: The path on the local filesystem where logs should be copied to. other_args: A list of other args from sys.argv. Raises: AppScalefileException: If there is no AppScalefile in the current working directory. """ contents = self.read_appscalefile() contents_as_yaml = yaml.safe_load(contents) # construct the appscale-gather-logs command command = [] if 'keyname' in contents_as_yaml: command.append("--keyname") command.append(contents_as_yaml["keyname"]) command.append("--location") command.append(location) if other_args: command += other_args # and exec it options = ParseArgs(command, "appscale-gather-logs").args AppScaleTools.gather_logs(options)
def logs(self, location, other_args=None): """ 'logs' provides a cleaner experience for users than the appscale-gather-logs command, by using the configuration options present in the AppScalefile found in the current working directory. Args: location: The path on the local filesystem where logs should be copied to. other_args: A list of other args from sys.argv. Raises: AppScalefileException: If there is no AppScalefile in the current working directory. """ contents = self.read_appscalefile() contents_as_yaml = yaml.safe_load(contents) # construct the appscale-gather-logs command command = [] if 'keyname' in contents_as_yaml: command.append("--keyname") command.append(contents_as_yaml["keyname"]) command.append("--location") command.append(location) if other_args: command += other_args # and exec it options = ParseArgs(command, "appscale-gather-logs").args AppScaleTools.gather_logs(options)
def test_appscale_in_two_node_virt_deployment(self): # pretend that the place we're going to put logs into doesn't exist flexmock(os.path) os.path.should_call('exists') # set the fall-through os.path.should_receive('exists').with_args('/tmp/foobaz').and_return( False) # and mock out the mkdir operation flexmock(os) os.should_receive('mkdir').with_args('/tmp/foobaz').and_return() # next, mock out finding the login ip address os.path.should_receive('exists').with_args( LocalState.get_locations_json_location( self.keyname)).and_return(True) fake_nodes_json = flexmock(name="fake_nodes_json") nodes_info = { "node_info": [ { "public_ip": "public1", "private_ip": "private1", "jobs": [ "load_balancer", "taskqueue_master", "zookeeper", "db_master", "taskqueue", "shadow", "login" ] }, { "public_ip": "public2", "private_ip": "private2", "jobs": ["memcache", "appengine", "zookeeper"] }, { "public_ip": "public3", "private_ip": "private3", "jobs": ["memcache", "appengine"] }, ] } fake_nodes_json.should_receive('read').and_return( json.dumps(nodes_info)) builtins = flexmock(sys.modules['__builtin__']) builtins.should_call('open') builtins.should_receive('open').with_args( LocalState.get_locations_json_location(self.keyname), 'r') \ .and_return(fake_nodes_json) # mock out writing the secret key to ~/.appscale, as well as reading it # later secret_key_location = LocalState.get_secret_key_location(self.keyname) fake_secret = flexmock(name="fake_secret") fake_secret.should_receive('read').and_return('the secret') builtins.should_receive('open').with_args(secret_key_location, 'r') \ .and_return(fake_secret) # and slip in a fake appcontroller to report on the two IP addrs fake_appcontroller = flexmock(name='fake_appcontroller') fake_appcontroller.should_receive('get_all_public_ips').with_args( 'the secret').and_return( json.dumps(['public1', 'public2', 'public3'])) fake_appcontroller.should_receive('get_role_info').with_args( 'the secret').and_return(json.dumps(nodes_info['node_info'])) flexmock(SOAPpy) SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \ .and_return(fake_appcontroller) # fake the creation of the log directories locally flexmock(utils) utils.should_receive('mkdir').with_args( '/tmp/foobaz/symlinks/private-ips') utils.should_receive('mkdir').with_args('/tmp/foobaz/public1') utils.should_receive('mkdir').with_args( '/tmp/foobaz/public1/cassandra') utils.should_receive('mkdir').with_args('/tmp/foobaz/public1/rabbitmq') utils.should_receive('mkdir').with_args('/tmp/foobaz/public2') utils.should_receive('mkdir').with_args( '/tmp/foobaz/public2/cassandra') utils.should_receive('mkdir').with_args('/tmp/foobaz/public2/rabbitmq') utils.should_receive('mkdir').with_args('/tmp/foobaz/public3') utils.should_receive('mkdir').with_args( '/tmp/foobaz/public3/cassandra') utils.should_receive('mkdir').with_args('/tmp/foobaz/public3/rabbitmq') utils.should_receive('mkdir').with_args( '/tmp/foobaz/symlinks/load_balancer') utils.should_receive('mkdir').with_args( '/tmp/foobaz/symlinks/taskqueue_master') utils.should_receive('mkdir').with_args( '/tmp/foobaz/symlinks/zookeeper') utils.should_receive('mkdir').with_args( '/tmp/foobaz/symlinks/db_master') utils.should_receive('mkdir').with_args( '/tmp/foobaz/symlinks/taskqueue') utils.should_receive('mkdir').with_args('/tmp/foobaz/symlinks/shadow') utils.should_receive('mkdir').with_args('/tmp/foobaz/symlinks/login') utils.should_receive('mkdir').with_args( '/tmp/foobaz/symlinks/memcache') utils.should_receive('mkdir').with_args( '/tmp/foobaz/symlinks/appengine') # fake creation of symlink to for friendly navigation links_mapping = { '../../public1': [ '/tmp/foobaz/symlinks/private-ips/private1', '/tmp/foobaz/symlinks/load_balancer/public1', '/tmp/foobaz/symlinks/taskqueue_master/public1', '/tmp/foobaz/symlinks/zookeeper/public1', '/tmp/foobaz/symlinks/db_master/public1', '/tmp/foobaz/symlinks/taskqueue/public1', '/tmp/foobaz/symlinks/shadow/public1', '/tmp/foobaz/symlinks/login/public1', ], '../../public2': [ '/tmp/foobaz/symlinks/private-ips/private2', '/tmp/foobaz/symlinks/zookeeper/public2', '/tmp/foobaz/symlinks/appengine/public2', '/tmp/foobaz/symlinks/memcache/public2', ], '../../public3': [ '/tmp/foobaz/symlinks/private-ips/private3', '/tmp/foobaz/symlinks/appengine/public3', '/tmp/foobaz/symlinks/memcache/public3', ] } for original_dir, expected_links in links_mapping.iteritems(): for expected_link in expected_links: os.should_receive('symlink').with_args(original_dir, expected_link) # finally, fake the copying of the log files flexmock(subprocess) subprocess.should_receive('Popen').with_args(re.compile('/var/log/appscale'), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \ .and_return(self.success) subprocess.should_receive('Popen').with_args(re.compile('/var/log/kern.log*'), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \ .and_return(self.success) subprocess.should_receive('Popen').with_args(re.compile('/var/log/monit*'), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \ .and_return(self.success) subprocess.should_receive('Popen').with_args( re.compile('/var/log/haproxy*'), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT).and_return(self.success) subprocess.should_receive('Popen').with_args(re.compile('/var/log/nginx'), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \ .and_return(self.success) subprocess.should_receive('Popen').with_args( re.compile('/var/log/rabbitmq'), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT).and_return(self.success) subprocess.should_receive('Popen').with_args(re.compile('/var/log/syslog*'), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \ .and_return(self.success) subprocess.should_receive('Popen').with_args(re.compile('/var/log/zookeeper'), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \ .and_return(self.success) subprocess.should_receive('Popen').with_args( re.compile('/opt/cassandra/cassandra/logs'), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT).and_return(self.success) argv = ["--keyname", self.keyname, "--location", "/tmp/foobaz"] options = ParseArgs(argv, self.function).args AppScaleTools.gather_logs(options)
def test_appscale_in_two_node_virt_deployment(self): # pretend that the place we're going to put logs into doesn't exist flexmock(os.path) os.path.should_call('exists') # set the fall-through os.path.should_receive('exists').with_args('/tmp/foobaz').and_return( False) # and mock out the mkdir operation flexmock(os) os.should_receive('mkdir').with_args('/tmp/foobaz').and_return() # next, mock out finding the login ip address os.path.should_receive('exists').with_args( LocalState.get_locations_json_location( self.keyname)).and_return(True) fake_nodes_json = flexmock(name="fake_nodes_json") fake_nodes_json.should_receive('read').and_return( json.dumps({ "node_info": [{ "public_ip": "public1", "private_ip": "private1", "jobs": ["shadow", "login"] }] })) builtins = flexmock(sys.modules['__builtin__']) builtins.should_call('open') builtins.should_receive('open').with_args( LocalState.get_locations_json_location(self.keyname), 'r') \ .and_return(fake_nodes_json) # mock out writing the secret key to ~/.appscale, as well as reading it # later secret_key_location = LocalState.get_secret_key_location(self.keyname) fake_secret = flexmock(name="fake_secret") fake_secret.should_receive('read').and_return('the secret') builtins.should_receive('open').with_args(secret_key_location, 'r') \ .and_return(fake_secret) # and slip in a fake appcontroller to report on the two IP addrs fake_appcontroller = flexmock(name='fake_appcontroller') fake_appcontroller.should_receive('get_all_public_ips').with_args( 'the secret').and_return(json.dumps(['public1', 'public2'])) flexmock(SOAPpy) SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \ .and_return(fake_appcontroller) # fake the creation of the log directories locally os.should_receive('mkdir').with_args( '/tmp/foobaz/public1').and_return() os.should_receive('mkdir').with_args('/tmp/foobaz/public1/cassandra') os.should_receive('mkdir').with_args( '/tmp/foobaz/public2').and_return() os.should_receive('mkdir').with_args('/tmp/foobaz/public2/cassandra') # finally, fake the copying of the log files flexmock(subprocess) subprocess.should_receive('Popen').with_args(re.compile('/var/log/appscale'), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \ .and_return(self.success) subprocess.should_receive('Popen').with_args(re.compile('/var/log/kern.log*'), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \ .and_return(self.success) subprocess.should_receive('Popen').with_args(re.compile('/var/log/monit*'), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \ .and_return(self.success) subprocess.should_receive('Popen').with_args(re.compile('/var/log/nginx'), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \ .and_return(self.success) subprocess.should_receive('Popen').with_args(re.compile('/var/log/syslog*'), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \ .and_return(self.success) subprocess.should_receive('Popen').with_args(re.compile('/var/log/zookeeper'), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \ .and_return(self.success) subprocess.should_receive('Popen').with_args( re.compile('/opt/cassandra/cassandra/logs'), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT).and_return(self.success) argv = ["--keyname", self.keyname, "--location", "/tmp/foobaz"] options = ParseArgs(argv, self.function).args AppScaleTools.gather_logs(options)
def test_appscale_in_two_node_virt_deployment(self): # pretend that the place we're going to put logs into doesn't exist flexmock(os.path) os.path.should_call("exists") # set the fall-through os.path.should_receive("exists").with_args("/tmp/foobaz").and_return(False) # and mock out the mkdir operation flexmock(os) os.should_receive("mkdir").with_args("/tmp/foobaz").and_return() # next, mock out finding the login ip address os.path.should_receive("exists").with_args(LocalState.get_locations_json_location(self.keyname)).and_return( True ) fake_nodes_json = flexmock(name="fake_nodes_json") fake_nodes_json.should_receive("read").and_return( json.dumps({"node_info": [{"public_ip": "public1", "private_ip": "private1", "jobs": ["shadow", "login"]}]}) ) builtins = flexmock(sys.modules["__builtin__"]) builtins.should_call("open") builtins.should_receive("open").with_args(LocalState.get_locations_json_location(self.keyname), "r").and_return( fake_nodes_json ) # mock out writing the secret key to ~/.appscale, as well as reading it # later secret_key_location = LocalState.get_secret_key_location(self.keyname) fake_secret = flexmock(name="fake_secret") fake_secret.should_receive("read").and_return("the secret") builtins.should_receive("open").with_args(secret_key_location, "r").and_return(fake_secret) # and slip in a fake appcontroller to report on the two IP addrs fake_appcontroller = flexmock(name="fake_appcontroller") fake_appcontroller.should_receive("get_all_public_ips").with_args("the secret").and_return( json.dumps(["public1", "public2"]) ) flexmock(SOAPpy) SOAPpy.should_receive("SOAPProxy").with_args("https://public1:17443").and_return(fake_appcontroller) # fake the creation of the log directories locally os.should_receive("mkdir").with_args("/tmp/foobaz/public1").and_return() os.should_receive("mkdir").with_args("/tmp/foobaz/public1/cassandra") os.should_receive("mkdir").with_args("/tmp/foobaz/public2").and_return() os.should_receive("mkdir").with_args("/tmp/foobaz/public2/cassandra") # finally, fake the copying of the log files flexmock(subprocess) subprocess.should_receive("Popen").with_args( re.compile("/var/log/appscale"), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT ).and_return(self.success) subprocess.should_receive("Popen").with_args( re.compile("/var/log/kern.log*"), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT ).and_return(self.success) subprocess.should_receive("Popen").with_args( re.compile("/var/log/monit*"), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT ).and_return(self.success) subprocess.should_receive("Popen").with_args( re.compile("/var/log/nginx"), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT ).and_return(self.success) subprocess.should_receive("Popen").with_args( re.compile("/var/log/syslog*"), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT ).and_return(self.success) subprocess.should_receive("Popen").with_args( re.compile("/var/log/zookeeper"), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT ).and_return(self.success) subprocess.should_receive("Popen").with_args( re.compile("/opt/cassandra/cassandra/logs"), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT, ).and_return(self.success) argv = ["--keyname", self.keyname, "--location", "/tmp/foobaz"] options = ParseArgs(argv, self.function).args AppScaleTools.gather_logs(options)
def test_appscale_in_two_node_virt_deployment(self): # pretend that the place we're going to put logs into doesn't exist flexmock(os.path) os.path.should_call('exists') # set the fall-through os.path.should_receive('exists').with_args('/tmp/foobaz').and_return(False) # and mock out the mkdir operation flexmock(os) os.should_receive('mkdir').with_args('/tmp/foobaz').and_return() # next, mock out finding the login ip address os.path.should_receive('exists').with_args( LocalState.get_locations_json_location(self.keyname)).and_return(True) fake_nodes_json = flexmock(name="fake_nodes_json") nodes_info = { "node_info": [ { "public_ip": "public1", "private_ip": "private1", "roles": ["load_balancer", "taskqueue_master", "zookeeper", "db_master", "taskqueue", "shadow"] }, { "public_ip": "public2", "private_ip": "private2", "roles": ["memcache", "appengine", "zookeeper"] }, { "public_ip": "public3", "private_ip": "private3", "roles": ["memcache", "appengine"] }, ] } fake_nodes_json.should_receive('read').and_return(json.dumps(nodes_info)) builtins = flexmock(sys.modules['__builtin__']) builtins.should_call('open') builtins.should_receive('open').with_args( LocalState.get_locations_json_location(self.keyname), 'r') \ .and_return(fake_nodes_json) # mock out writing the secret key to ~/.appscale, as well as reading it # later secret_key_location = LocalState.get_secret_key_location(self.keyname) fake_secret = flexmock(name="fake_secret") fake_secret.should_receive('read').and_return('the secret') builtins.should_receive('open').with_args(secret_key_location, 'r') \ .and_return(fake_secret) # and slip in a fake appcontroller to report on the two IP addrs fake_appcontroller = flexmock(name='fake_appcontroller') fake_appcontroller.should_receive('get_all_public_ips').with_args( 'the secret').and_return(json.dumps(['public1', 'public2', 'public3'])) fake_appcontroller.should_receive('get_role_info').with_args( 'the secret').and_return(json.dumps(nodes_info['node_info'])) flexmock(SOAPpy) SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \ .and_return(fake_appcontroller) # fake the creation of the log directories locally flexmock(utils) utils.should_receive('mkdir').with_args('/tmp/foobaz/symlinks/private-ips') utils.should_receive('mkdir').with_args('/tmp/foobaz/public1') utils.should_receive('mkdir').with_args('/tmp/foobaz/public1/cassandra') utils.should_receive('mkdir').with_args('/tmp/foobaz/public1/rabbitmq') utils.should_receive('mkdir').with_args('/tmp/foobaz/public2') utils.should_receive('mkdir').with_args('/tmp/foobaz/public2/cassandra') utils.should_receive('mkdir').with_args('/tmp/foobaz/public2/rabbitmq') utils.should_receive('mkdir').with_args('/tmp/foobaz/public3') utils.should_receive('mkdir').with_args('/tmp/foobaz/public3/cassandra') utils.should_receive('mkdir').with_args('/tmp/foobaz/public3/rabbitmq') utils.should_receive('mkdir').with_args('/tmp/foobaz/symlinks/load_balancer') utils.should_receive('mkdir').with_args( '/tmp/foobaz/symlinks/taskqueue_master') utils.should_receive('mkdir').with_args('/tmp/foobaz/symlinks/zookeeper') utils.should_receive('mkdir').with_args('/tmp/foobaz/symlinks/db_master') utils.should_receive('mkdir').with_args('/tmp/foobaz/symlinks/taskqueue') utils.should_receive('mkdir').with_args('/tmp/foobaz/symlinks/shadow') utils.should_receive('mkdir').with_args('/tmp/foobaz/symlinks/memcache') utils.should_receive('mkdir').with_args('/tmp/foobaz/symlinks/appengine') # fake creation of symlink to for friendly navigation links_mapping = { '../../public1': [ '/tmp/foobaz/symlinks/private-ips/private1', '/tmp/foobaz/symlinks/load_balancer/public1', '/tmp/foobaz/symlinks/taskqueue_master/public1', '/tmp/foobaz/symlinks/zookeeper/public1', '/tmp/foobaz/symlinks/db_master/public1', '/tmp/foobaz/symlinks/taskqueue/public1', '/tmp/foobaz/symlinks/shadow/public1', ], '../../public2': [ '/tmp/foobaz/symlinks/private-ips/private2', '/tmp/foobaz/symlinks/zookeeper/public2', '/tmp/foobaz/symlinks/appengine/public2', '/tmp/foobaz/symlinks/memcache/public2', ], '../../public3': [ '/tmp/foobaz/symlinks/private-ips/private3', '/tmp/foobaz/symlinks/appengine/public3', '/tmp/foobaz/symlinks/memcache/public3', ] } for original_dir, expected_links in links_mapping.iteritems(): for expected_link in expected_links: os.should_receive('symlink').with_args(original_dir, expected_link) # finally, fake the copying of the log files flexmock(subprocess) subprocess.should_receive('Popen').with_args(re.compile('/var/log/appscale'), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \ .and_return(self.success) subprocess.should_receive('Popen').with_args(re.compile('/var/log/kern.log*'), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \ .and_return(self.success) subprocess.should_receive('Popen').with_args(re.compile('/var/log/monit*'), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \ .and_return(self.success) subprocess.should_receive('Popen').with_args( re.compile('/var/log/haproxy*'), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT).and_return(self.success) subprocess.should_receive('Popen').with_args(re.compile('/var/log/nginx'), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \ .and_return(self.success) subprocess.should_receive('Popen').with_args( re.compile('/var/log/rabbitmq'), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT).and_return(self.success) subprocess.should_receive('Popen').with_args(re.compile('/var/log/syslog*'), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \ .and_return(self.success) subprocess.should_receive('Popen').with_args(re.compile('/var/log/zookeeper'), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \ .and_return(self.success) subprocess.should_receive('Popen').with_args( re.compile('/opt/cassandra/cassandra/logs'), shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT ).and_return(self.success) argv = [ "--keyname", self.keyname, "--location", "/tmp/foobaz" ] options = ParseArgs(argv, self.function).args AppScaleTools.gather_logs(options)