def get_application_info(cls, owner, app_language, app_dir):
name = AppEngineHelper.get_app_id_from_app_config(app_dir)
version = AppEngineHelper.get_app_version_from_app_config(app_dir)
app = Application(name, version, owner)
dependencies_path = app_dir + os.sep + 'dependencies.yaml'
if app_language == 'java':
dependencies_path = app_dir + os.sep + 'war' + os.sep + 'WEB-INF' + os.sep + 'dependencies.yaml'
api_specs_dir = app_dir + os.sep + 'war' + os.sep + 'WEB-INF' + os.sep + 'specs'
if os.path.exists(api_specs_dir):
for f in os.listdir(api_specs_dir):
if f.endswith('.json'):
api = API(api_specs_dir + os.sep + f)
AppScaleLogger.log('Detected API: {0}-v{1}'.format(api.name, api.version))
app.api_list.append(api)
if os.path.exists(dependencies_path):
dependencies_file = open(dependencies_path, 'r')
dependencies = yaml.load(dependencies_file)
dependencies_file.close()
if dependencies:
EagerHelper.validate_dependencies(dependencies)
app.dependencies = dependencies['dependencies']
return app
def test_upload_java_app_with_no_appid(self):
# add in mocks so that there is an appengine-web.xml, but with no appid set
flexmock(os.path)
os.path.should_call('exists')
os.path.should_receive('exists').with_args(
AppEngineHelper.get_app_yaml_location(
self.app_dir)).and_return(False)
appengine_web_xml_location = AppEngineHelper.get_appengine_web_xml_location(
self.app_dir)
os.path.should_receive('exists').with_args(
AppEngineHelper.get_appengine_web_xml_location(
self.app_dir)).and_return(True)
flexmock(AppEngineHelper).should_receive(
'get_app_id_from_app_config').and_return('app_id')
flexmock(AppEngineHelper).should_receive(
'get_app_runtime_from_app_config').and_return('runtime')
flexmock(LocalState).should_receive('get_secret_key').and_return()
# mock out reading the app.yaml file
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open') # set the fall-through
fake_appengine_web_xml = flexmock(name="fake_appengine_web_xml")
fake_appengine_web_xml.should_receive('read').and_return(
"<baz></baz>\n" + "<application></application>")
builtins.should_receive('open').with_args(appengine_web_xml_location, 'r') \
.and_return(fake_appengine_web_xml)
argv = ["--keyname", self.keyname, "--file", self.app_dir]
options = ParseArgs(argv, self.function).args
self.assertRaises(AppEngineConfigException, AppScaleTools.upload_app,
options)
def test_upload_java_app_with_no_appid(self):
# if the user gives us an app with a reserved appid, we should abort
# add in mocks so that there is an appengine-web.xml, but with no appid set
flexmock(os.path)
os.path.should_call('exists')
os.path.should_receive('exists').with_args(
AppEngineHelper.get_app_yaml_location(self.app_dir)).and_return(False)
appengine_web_xml_location = AppEngineHelper.get_appengine_web_xml_location(
self.app_dir)
os.path.should_receive('exists').with_args(
AppEngineHelper.get_appengine_web_xml_location(self.app_dir)).and_return(True)
# mock out reading the app.yaml file
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open') # set the fall-through
fake_appengine_web_xml = flexmock(name="fake_appengine_web_xml")
fake_appengine_web_xml.should_receive('read').and_return("<baz></baz>\n" +
"<application></application>")
builtins.should_receive('open').with_args(appengine_web_xml_location, 'r') \
.and_return(fake_appengine_web_xml)
argv = [
"--keyname", self.keyname,
"--file", self.app_dir
]
options = ParseArgs(argv, self.function).args
self.assertRaises(AppEngineConfigException, AppScaleTools.upload_app, options)
def test_upload_java_app_with_no_appid(self):
# add in mocks so that there is an appengine-web.xml, but with no appid set
flexmock(os.path)
os.path.should_call('exists')
os.path.should_receive('exists').with_args(
AppEngineHelper.get_app_yaml_location(self.app_dir)).and_return(False)
appengine_web_xml_location = AppEngineHelper.get_appengine_web_xml_location(
self.app_dir)
os.path.should_receive('exists').with_args(
AppEngineHelper.get_appengine_web_xml_location(self.app_dir)).and_return(True)
flexmock(AppEngineHelper).should_receive('get_app_id_from_app_config').and_return('app_id')
flexmock(AppEngineHelper).should_receive('get_app_runtime_from_app_config').and_return('runtime')
flexmock(LocalState).should_receive('get_secret_key').and_return()
flexmock(AppControllerClient).should_receive('get_uaserver_host').and_return('1.2.3.4')
# mock out reading the app.yaml file
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open') # set the fall-through
fake_appengine_web_xml = flexmock(name="fake_appengine_web_xml")
fake_appengine_web_xml.should_receive('read').and_return("<baz></baz>\n" +
"<application></application>")
builtins.should_receive('open').with_args(appengine_web_xml_location, 'r') \
.and_return(fake_appengine_web_xml)
argv = [
"--keyname", self.keyname,
"--file", self.app_dir
]
options = ParseArgs(argv, self.function).args
self.assertRaises(AppEngineConfigException, AppScaleTools.upload_app, options)
def copy_app_to_host(cls, app_location, keyname, is_verbose):
"""Copies the given application to a machine running the Login service
within an AppScale deployment.
Args:
app_location: The location on the local filesystem where the application
can be found.
keyname: The name of the SSH keypair that uniquely identifies this
AppScale deployment.
is_verbose: A bool that indicates if we should print the commands we exec
to copy the app to the remote host to stdout.
Returns:
A str corresponding to the location on the remote filesystem where the
application was copied to.
"""
app_id = AppEngineHelper.get_app_id_from_app_config(app_location)
AppScaleLogger.log("Tarring application")
rand = str(uuid.uuid4()).replace('-', '')[:8]
local_tarred_app = "{0}/appscale-app-{1}-{2}.tar.gz".format(
tempfile.gettempdir(), app_id, rand)
LocalState.shell(
"cd '{0}' && COPYFILE_DISABLE=1 tar -czhf {1} --exclude='*.pyc' *".
format(app_location, local_tarred_app), is_verbose)
AppScaleLogger.log("Copying over application")
remote_app_tar = "{0}/{1}.tar.gz".format(cls.REMOTE_APP_DIR, app_id)
cls.scp(LocalState.get_login_host(keyname), keyname, local_tarred_app,
remote_app_tar, is_verbose)
os.remove(local_tarred_app)
return remote_app_tar
def test_java_good_sdk_version(self):
target_jar = AppEngineHelper.JAVA_SDK_JAR_PREFIX + '-' \
+ AppEngineHelper.SUPPORTED_SDK_VERSION + '.jar'
good_jars = ['test.jar', target_jar]
flexmock(os)
os.should_receive('listdir').with_args('/war/WEB-INF/lib').and_return(good_jars)
self.assertEquals(False, AppEngineHelper.is_sdk_mismatch(''))
def copy_app_to_host(cls, app_location, keyname, is_verbose):
"""Copies the given application to a machine running the Login service within
an AppScale deployment.
Args:
app_location: The location on the local filesystem where the application
can be found.
keyname: The name of the SSH keypair that uniquely identifies this
AppScale deployment.
is_verbose: A bool that indicates if we should print the commands we exec
to copy the app to the remote host to stdout.
Returns:
A str corresponding to the location on the remote filesystem where the
application was copied to.
"""
AppScaleLogger.log("Creating remote directory to copy app into")
app_id = AppEngineHelper.get_app_id_from_app_config(app_location)
remote_app_dir = "/var/apps/{0}/app".format(app_id)
cls.ssh(LocalState.get_login_host(keyname), keyname,
'mkdir -p {0}'.format(remote_app_dir), is_verbose)
AppScaleLogger.log("Tarring application")
rand = str(uuid.uuid4()).replace('-', '')[:8]
local_tarred_app = "/tmp/appscale-app-{0}-{1}.tar.gz".format(app_id, rand)
LocalState.shell("cd {0} && tar -czf {1} *".format(app_location,
local_tarred_app), is_verbose)
AppScaleLogger.log("Copying over application")
remote_app_tar = "{0}/{1}.tar.gz".format(remote_app_dir, app_id)
cls.scp(LocalState.get_login_host(keyname), keyname, local_tarred_app,
remote_app_tar, is_verbose)
os.remove(local_tarred_app)
return remote_app_tar
def test_upload_app_with_non_alpha_appid(self):
# add in mocks so that there is an app.yaml, but with no appid set
flexmock(os.path)
os.path.should_call('exists')
app_yaml_location = AppEngineHelper.get_app_yaml_location(self.app_dir)
os.path.should_receive('exists').with_args(app_yaml_location) \
.and_return(True)
# mock out reading the app.yaml file
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open') # set the fall-through
fake_app_yaml = flexmock(name="fake_app_yaml")
fake_app_yaml.should_receive('read').and_return(
yaml.dump({
'application': 'baz*',
'runtime': 'python'
}))
builtins.should_receive('open').with_args(app_yaml_location, 'r') \
.and_return(fake_app_yaml)
argv = ["--keyname", self.keyname, "--file", self.app_dir]
options = ParseArgs(argv, self.function).args
self.assertRaises(AppEngineConfigException, AppScaleTools.upload_app,
options)
def test_upload_app_with_reserved_app_id(self):
# users can't choose reserved appids for their own applications
# add in mocks so that there is an app.yaml, but with no appid set
flexmock(os.path)
os.path.should_call('exists')
app_yaml_location = AppEngineHelper.get_app_yaml_location(self.app_dir)
os.path.should_receive('exists').with_args(app_yaml_location) \
.and_return(True)
# mock out reading the app.yaml file
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open') # set the fall-through
fake_app_yaml = flexmock(name="fake_app_yaml")
fake_app_yaml.should_receive('read').and_return(yaml.dump({
'application' : 'none', # a reserved appid
'runtime' : 'python'
}))
builtins.should_receive('open').with_args(app_yaml_location, 'r') \
.and_return(fake_app_yaml)
argv = [
"--keyname", self.keyname,
"--file", self.app_dir
]
options = ParseArgs(argv, self.function).args
self.assertRaises(AppEngineConfigException, AppScaleTools.upload_app, options)
def test_upload_app_with_non_alpha_appid(self):
# we only support four runtimes - abort if the user gives us an unsupported
# one
# add in mocks so that there is an app.yaml, but with no appid set
flexmock(os.path)
os.path.should_call('exists')
app_yaml_location = AppEngineHelper.get_app_yaml_location(self.app_dir)
os.path.should_receive('exists').with_args(app_yaml_location) \
.and_return(True)
# mock out reading the app.yaml file
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open') # set the fall-through
fake_app_yaml = flexmock(name="fake_app_yaml")
fake_app_yaml.should_receive('read').and_return(yaml.dump({
'application' : 'baz*',
'runtime' : 'python'
}))
builtins.should_receive('open').with_args(app_yaml_location, 'r') \
.and_return(fake_app_yaml)
argv = [
"--keyname", self.keyname,
"--file", self.app_dir
]
options = ParseArgs(argv, self.function).args
self.assertRaises(AppEngineConfigException, AppScaleTools.upload_app, options)
def copy_app_to_host(cls, app_location, keyname, is_verbose):
"""Copies the given application to a machine running the Login service
within an AppScale deployment.
Args:
app_location: The location on the local filesystem where the application
can be found.
keyname: The name of the SSH keypair that uniquely identifies this
AppScale deployment.
is_verbose: A bool that indicates if we should print the commands we exec
to copy the app to the remote host to stdout.
Returns:
A str corresponding to the location on the remote filesystem where the
application was copied to.
"""
app_id = AppEngineHelper.get_app_id_from_app_config(app_location)
AppScaleLogger.log("Tarring application")
rand = str(uuid.uuid4()).replace('-', '')[:8]
local_tarred_app = "{0}/appscale-app-{1}-{2}.tar.gz".format(tempfile.gettempdir(),
app_id, rand)
LocalState.shell("cd '{0}' && COPYFILE_DISABLE=1 tar -czhf {1} --exclude='*.pyc' *".format(
app_location, local_tarred_app), is_verbose)
AppScaleLogger.log("Copying over application")
remote_app_tar = "{0}/{1}.tar.gz".format(cls.REMOTE_APP_DIR, app_id)
cls.scp(LocalState.get_login_host(keyname), keyname, local_tarred_app,
remote_app_tar, is_verbose)
os.remove(local_tarred_app)
return remote_app_tar
def test_upload_app_with_no_app_yaml_or_appengine_web_xml(self):
# all app engine apps must have a config file - abort if we can't find one
# add in mocks so that the config files aren't found
flexmock(os.path)
os.path.should_call('exists')
os.path.should_receive('exists').with_args(
AppEngineHelper.get_app_yaml_location(
self.app_dir)).and_return(False)
os.path.should_receive('exists').with_args(
AppEngineHelper.get_appengine_web_xml_location(self.app_dir)) \
.and_return(False)
argv = ["--keyname", self.keyname, "--file", self.app_dir]
options = ParseArgs(argv, self.function).args
self.assertRaises(AppEngineConfigException, AppScaleTools.upload_app,
options)
def test_java_good_sdk_version(self):
target_jar = AppEngineHelper.JAVA_SDK_JAR_PREFIX + '-' \
+ AppEngineHelper.SUPPORTED_SDK_VERSION + '.jar'
good_jars = ['test.jar', target_jar]
flexmock(os)
os.should_receive('listdir').with_args('/war/WEB-INF/lib').and_return(
good_jars)
self.assertEquals(False, AppEngineHelper.is_sdk_mismatch(''))
def test_upload_app_with_no_app_yaml_or_appengine_web_xml(self):
# all app engine apps must have a config file - abort if we can't find one
# add in mocks so that the config files aren't found
flexmock(os.path)
os.path.should_call('exists')
os.path.should_receive('exists').with_args(
AppEngineHelper.get_app_yaml_location(self.app_dir)).and_return(False)
os.path.should_receive('exists').with_args(
AppEngineHelper.get_appengine_web_xml_location(self.app_dir)) \
.and_return(False)
argv = [
"--keyname", self.keyname,
"--file", self.app_dir
]
options = ParseArgs(argv, self.function).args
self.assertRaises(AppEngineConfigException, AppScaleTools.upload_app, options)
def test_java_good_sdk_version(self):
target_jar = AppEngineHelper.JAVA_SDK_JAR_PREFIX + '-' \
+ AppEngineHelper.SUPPORTED_SDK_VERSION + '.jar'
good_jars = ['test.jar', target_jar]
aeh = flexmock(AppEngineHelper)
aeh.should_receive('get_appengine_lib_locations').and_return(['blah'])
flexmock(os)
os.should_receive('listdir').and_return(good_jars)
self.assertEquals(False, AppEngineHelper.is_sdk_mismatch(''))
def test_java_good_sdk_version(self):
target_jar = AppEngineHelper.JAVA_SDK_JAR_PREFIX + '-' \
+ AppEngineHelper.SUPPORTED_SDK_VERSION + '.jar'
good_jars = ['test.jar', target_jar]
aeh = flexmock(AppEngineHelper)
aeh.should_receive('get_appengine_lib_locations').and_return(['blah'])
flexmock(os)
os.should_receive('listdir').and_return(good_jars)
self.assertEquals(False, AppEngineHelper.is_sdk_mismatch(''))
def test_upload_app_when_app_admin_not_this_user(self):
# we don't let you upload an app if the appid is registered to someone else,
# so abort
# add in mocks so that there is an app.yaml, but with no appid set
flexmock(os.path)
os.path.should_call('exists')
app_yaml_location = AppEngineHelper.get_app_yaml_location(self.app_dir)
os.path.should_receive('exists').with_args(app_yaml_location) \
.and_return(True)
# mock out reading the app.yaml file
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open') # set the fall-through
fake_app_yaml = flexmock(name="fake_app_yaml")
fake_app_yaml.should_receive('read').and_return(yaml.dump({
'application' : 'baz',
'runtime' : 'python'
}))
builtins.should_receive('open').with_args(app_yaml_location, 'r') \
.and_return(fake_app_yaml)
# mock out the SOAP call to the AppController and assume it succeeded
fake_appcontroller = flexmock(name='fake_appcontroller')
fake_appcontroller.should_receive('status').with_args('the secret') \
.and_return('Database is at public1')
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://*****:*****@a.com', 'the secret').and_return('false')
fake_userappserver.should_receive('commit_new_user').with_args(
'*****@*****.**', str, 'xmpp_user', 'the secret').and_return('true')
fake_userappserver.should_receive('commit_new_user').with_args(
'a@public1', str, 'xmpp_user', 'the secret').and_return('true')
fake_userappserver.should_receive('get_app_data').with_args(
'baz', 'the secret').and_return('\n\nnum_ports:0\n\napp_owner:[email protected]')
SOAPpy.should_receive('SOAPProxy').with_args('https://*****:*****@a.com")
flexmock(getpass)
getpass.should_receive('getpass').and_return('aaaaaa')
argv = [
"--keyname", self.keyname,
"--file", self.app_dir
]
options = ParseArgs(argv, self.function).args
self.assertRaises(AppScaleException, AppScaleTools.upload_app, options)
def upload_app(cls, options):
"""Uploads the given App Engine application into AppScale.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
Returns:
A tuple containing the host and port where the application is serving
traffic from.
"""
start_time = time.time()
if cls.TAR_GZ_REGEX.search(options.file):
file_location = LocalState.extract_tgz_app_to_dir(options.file,
options.verbose)
created_dir = True
elif cls.ZIP_REGEX.search(options.file):
file_location = LocalState.extract_zip_app_to_dir(options.file,
options.verbose)
created_dir = True
elif os.path.isdir(options.file):
file_location = options.file
created_dir = False
else:
raise AppEngineConfigException('{0} is not a tar.gz file, a zip file, ' \
'or a directory. Please try uploading either a tar.gz file, a zip ' \
'file, or a directory.'.format(options.file))
try:
app_id = AppEngineHelper.get_app_id_from_app_config(file_location)
except AppEngineConfigException:
# Java App Engine users may have specified their war directory. In that
# case, just move up one level, back to the app's directory.
file_location = file_location + os.sep + ".."
app_id = AppEngineHelper.get_app_id_from_app_config(file_location)
app_language = AppEngineHelper.get_app_runtime_from_app_config(
file_location)
AppEngineHelper.validate_app_id(app_id)
if app_language == 'java':
if AppEngineHelper.is_sdk_mismatch(file_location):
AppScaleLogger.warn('AppScale did not find the correct SDK jar ' +
'versions in your app. The current supported ' +
'SDK version is ' + AppEngineHelper.SUPPORTED_SDK_VERSION + '.')
acc = AppControllerClient(LocalState.get_login_host(options.keyname),
LocalState.get_secret_key(options.keyname))
userappserver_host = acc.get_uaserver_host(options.verbose)
userappclient = UserAppClient(userappserver_host, LocalState.get_secret_key(
options.keyname))
if options.test:
username = LocalState.DEFAULT_USER
elif options.email:
username = options.email
else:
username = LocalState.get_username_from_stdin(is_admin=False)
if not userappclient.does_user_exist(username):
password = LocalState.get_password_from_stdin()
RemoteHelper.create_user_accounts(username, password, userappserver_host,
options.keyname, clear_datastore=False)
app_exists = userappclient.does_app_exist(app_id)
app_admin = userappclient.get_app_admin(app_id)
if app_admin is not None and username != app_admin:
raise AppScaleException("The given user doesn't own this application" + \
", so they can't upload an app with that application ID. Please " + \
"change the application ID and try again.")
eager_app = None
eager_enabled = not options.disable_eager
t1 = t2 = t3 = t4 = 0
if eager_enabled:
t1 = time.time()
eager_app = EagerHelper.get_application_info(username, app_language, file_location)
valid = EagerHelper.perform_eager_validation(eager_app, options.keyname)
if valid:
AppScaleLogger.success('EAGER validation was successful. Continuing with the deployment.')
else:
AppScaleLogger.warn('EAGER validation failed. Aborting app deployment!')
end_time = time.time()
AppScaleLogger.log("Time elapsed: {0} ms".format((end_time - start_time) * 1000))
AppScaleLogger.log("Time spent on EAGER: {0} ms".format((end_time - t1) * 1000))
return
t2 = time.time()
if app_exists:
AppScaleLogger.log("Uploading new version of app {0}".format(app_id))
else:
AppScaleLogger.log("Uploading initial version of app {0}".format(app_id))
userappclient.reserve_app_id(username, app_id, app_language)
remote_file_path = RemoteHelper.copy_app_to_host(file_location,
options.keyname, options.verbose)
acc.done_uploading(app_id, remote_file_path)
acc.update([app_id])
# now that we've told the AppController to start our app, find out what port
# the app is running on and wait for it to start serving
AppScaleLogger.log("Please wait for your app to start serving.")
if app_exists:
time.sleep(20) # give the AppController time to restart the app
serving_host, serving_port = userappclient.get_serving_info(app_id,
options.keyname)
RemoteHelper.sleep_until_port_is_open(serving_host, serving_port,
options.verbose)
app_url = "http://{0}:{1}".format(serving_host, serving_port)
if eager_enabled and eager_app.api_list:
t3 = time.time()
EagerHelper.publish_api_list(eager_app.api_list, app_url, options.keyname)
t4 = time.time()
AppScaleLogger.success("Your app can be reached at the following URL: " +
app_url)
if created_dir:
shutil.rmtree(file_location)
end_time = time.time()
AppScaleLogger.log("Time elapsed: {0} ms".format((end_time - start_time) * 1000))
AppScaleLogger.log("Time spent on EAGER: {0} ms".format((t2 - t1 + t4 - t3) * 1000))
return (serving_host, serving_port)
def test_upload_app_when_app_exists_on_virt_cluster(self):
# we do let you upload an app if it's already running
# add in mocks so that there is an app.yaml with an appid set
flexmock(os.path)
os.path.should_call('exists')
app_yaml_location = AppEngineHelper.get_app_yaml_location(self.app_dir)
os.path.should_receive('exists').with_args(app_yaml_location) \
.and_return(True)
# mock out reading the app.yaml file
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open') # set the fall-through
fake_app_yaml = flexmock(name="fake_app_yaml")
fake_app_yaml.should_receive('read').and_return(yaml.dump({
'application' : 'baz',
'runtime' : 'python27'
}))
builtins.should_receive('open').with_args(app_yaml_location, 'r') \
.and_return(fake_app_yaml)
# mock out the SOAP call to the AppController and assume it succeeded
fake_appcontroller = flexmock(name='fake_appcontroller')
fake_appcontroller.should_receive('status').with_args('the secret') \
.and_return('Database is at public1')
fake_appcontroller.should_receive('done_uploading').with_args(
'baz', '/opt/appscale/apps/baz.tar.gz', 'the secret').and_return('OK')
fake_appcontroller.should_receive('update').with_args(
['baz'], 'the secret').and_return('OK')
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://*****:*****@a.com', 'the secret').and_return('false')
fake_userappserver.should_receive('does_user_exist').with_args(
'a@public1', 'the secret').and_return('false')
fake_userappserver.should_receive('commit_new_user').with_args(
'*****@*****.**', str, 'xmpp_user', 'the secret').and_return('true')
fake_userappserver.should_receive('commit_new_user').with_args(
'a@public1', str, 'xmpp_user', 'the secret').and_return('true')
fake_userappserver.should_receive('get_app_data').with_args(
'baz', 'the secret').and_return(app_data)
SOAPpy.should_receive('SOAPProxy').with_args('https://*****:*****@a.com")
flexmock(getpass)
getpass.should_receive('getpass').and_return('aaaaaa')
# mock out making the remote app directory
flexmock(subprocess)
subprocess.should_receive('Popen').with_args(re.compile('mkdir -p'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.success)
# and mock out tarring and copying the app
subprocess.should_receive('Popen').with_args(re.compile('tar -czhf'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.success)
subprocess.should_receive('Popen').with_args(re.compile(
'/tmp/appscale-app-baz.tar.gz'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.success)
# as well as removing the tar'ed app once we're done copying it
flexmock(os)
os.should_receive('remove').with_args('/tmp/appscale-app-baz-1234.tar.gz') \
.and_return()
# and slap in a mock that says the app comes up after waiting for it
# three times
fake_socket = flexmock(name='fake_socket')
fake_socket.should_receive('connect').with_args(('public1',
8080)).and_raise(Exception).and_raise(Exception) \
.and_return(None)
flexmock(socket)
socket.should_receive('socket').and_return(fake_socket)
argv = [
"--keyname", self.keyname,
"--file", self.app_dir
]
options = ParseArgs(argv, self.function).args
(host, port) = AppScaleTools.upload_app(options)
self.assertEquals('public1', host)
self.assertEquals(8080, port)
def upload_app(cls, options):
"""Uploads the given App Engine application into AppScale.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
Returns:
A tuple containing the host and port where the application is serving
traffic from.
"""
if cls.TAR_GZ_REGEX.search(options.file):
file_location = LocalState.extract_tgz_app_to_dir(options.file,
options.verbose)
created_dir = True
elif cls.ZIP_REGEX.search(options.file):
file_location = LocalState.extract_zip_app_to_dir(options.file,
options.verbose)
created_dir = True
elif os.path.isdir(options.file):
file_location = options.file
created_dir = False
else:
raise AppEngineConfigException('{0} is not a tar.gz file, a zip file, ' \
'or a directory. Please try uploading either a tar.gz file, a zip ' \
'file, or a directory.'.format(options.file))
try:
app_id = AppEngineHelper.get_app_id_from_app_config(file_location)
except AppEngineConfigException as config_error:
AppScaleLogger.log(config_error)
if 'yaml' in str(config_error):
raise config_error
# Java App Engine users may have specified their war directory. In that
# case, just move up one level, back to the app's directory.
file_location = file_location + os.sep + ".."
app_id = AppEngineHelper.get_app_id_from_app_config(file_location)
app_language = AppEngineHelper.get_app_runtime_from_app_config(
file_location)
AppEngineHelper.validate_app_id(app_id)
if app_language == 'java':
if AppEngineHelper.is_sdk_mismatch(file_location):
AppScaleLogger.warn('AppScale did not find the correct SDK jar ' +
'versions in your app. The current supported ' +
'SDK version is ' + AppEngineHelper.SUPPORTED_SDK_VERSION + '.')
login_host = LocalState.get_login_host(options.keyname)
secret_key = LocalState.get_secret_key(options.keyname)
acc = AppControllerClient(login_host, secret_key)
if options.test:
username = LocalState.DEFAULT_USER
elif options.email:
username = options.email
else:
username = LocalState.get_username_from_stdin(is_admin=False)
if not acc.does_user_exist(username):
password = LocalState.get_password_from_stdin()
RemoteHelper.create_user_accounts(username, password,
login_host, options.keyname, clear_datastore=False)
app_exists = acc.does_app_exist(app_id)
app_admin = acc.get_app_admin(app_id)
if app_admin is not None and username != app_admin:
raise AppScaleException("The given user doesn't own this application" + \
", so they can't upload an app with that application ID. Please " + \
"change the application ID and try again.")
if app_exists:
AppScaleLogger.log("Uploading new version of app {0}".format(app_id))
else:
AppScaleLogger.log("Uploading initial version of app {0}".format(app_id))
acc.reserve_app_id(username, app_id, app_language)
# Ignore all .pyc files while tarring.
if app_language == 'python27':
AppScaleLogger.log("Ignoring .pyc files")
remote_file_path = RemoteHelper.copy_app_to_host(file_location,
options.keyname, options.verbose)
acc.done_uploading(app_id, remote_file_path)
acc.update([app_id])
# now that we've told the AppController to start our app, find out what port
# the app is running on and wait for it to start serving
AppScaleLogger.log("Please wait for your app to start serving.")
if app_exists:
time.sleep(20) # give the AppController time to restart the app
# Makes a call to the AppController to get all the stats and looks
# through them for the http port the app can be reached on.
sleep_time = 2 * cls.SLEEP_TIME
current_app = None
for i in range(cls.MAX_RETRIES):
try:
result = acc.get_all_stats()
json_result = json.loads(result)
apps_result = json_result['apps']
current_app = apps_result[app_id]
http_port = current_app['http']
break
except ValueError:
pass
except KeyError:
pass
AppScaleLogger.verbose("Waiting {0} second(s) for a port to be assigned to {1}".\
format(sleep_time, app_id), options.verbose)
time.sleep(sleep_time)
if not current_app:
raise AppScaleException("Unable to get the serving port for the application.")
RemoteHelper.sleep_until_port_is_open(login_host, http_port, options.verbose)
AppScaleLogger.success("Your app can be reached at the following URL: " +
"http://{0}:{1}".format(login_host, http_port))
if created_dir:
shutil.rmtree(file_location)
return (login_host, http_port)
def test_upload_app_when_app_exists_on_virt_cluster(self):
# we do let you upload an app if it's already running
# add in mocks so that there is an app.yaml with an appid set
flexmock(os.path)
os.path.should_call('exists')
app_yaml_location = AppEngineHelper.get_app_yaml_location(self.app_dir)
os.path.should_receive('exists').with_args(app_yaml_location) \
.and_return(True)
# mock out reading the app.yaml file
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open') # set the fall-through
fake_app_yaml = flexmock(name="fake_app_yaml")
fake_app_yaml.should_receive('read').and_return(
yaml.dump({
'application': 'baz',
'runtime': 'python27'
}))
builtins.should_receive('open').with_args(app_yaml_location, 'r') \
.and_return(fake_app_yaml)
# mock out the SOAP call to the AppController and assume it succeeded
fake_appcontroller = flexmock(name='fake_appcontroller')
fake_appcontroller.should_receive('status').with_args('the secret') \
.and_return('Database is at public1')
fake_appcontroller.should_receive('done_uploading').with_args(
'baz', '/opt/appscale/apps/baz.tar.gz',
'the secret').and_return('OK')
fake_appcontroller.should_receive('update').with_args(
['baz'], 'the secret').and_return('OK')
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://*****:*****@a.com', 'the secret').and_return('false')
fake_userappserver.should_receive('does_user_exist').with_args(
'a@public1', 'the secret').and_return('false')
fake_userappserver.should_receive('commit_new_user').with_args(
'*****@*****.**', str, 'xmpp_user', 'the secret').and_return('true')
fake_userappserver.should_receive('commit_new_user').with_args(
'a@public1', str, 'xmpp_user', 'the secret').and_return('true')
fake_userappserver.should_receive('get_app_data').with_args(
'baz', 'the secret').and_return(app_data)
SOAPpy.should_receive('SOAPProxy').with_args('https://*****:*****@a.com")
flexmock(getpass)
getpass.should_receive('getpass').and_return('aaaaaa')
# mock out making the remote app directory
flexmock(subprocess)
subprocess.should_receive('Popen').with_args(re.compile('mkdir -p'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.success)
# and mock out tarring and copying the app
subprocess.should_receive('Popen').with_args(re.compile('tar -czhf'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.success)
subprocess.should_receive('Popen').with_args(re.compile(
'/tmp/appscale-app-baz.tar.gz'),
shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
.and_return(self.success)
# as well as removing the tar'ed app once we're done copying it
flexmock(os)
os.should_receive('remove').with_args('/tmp/appscale-app-baz-1234.tar.gz') \
.and_return()
# and slap in a mock that says the app comes up after waiting for it
# three times
fake_socket = flexmock(name='fake_socket')
fake_socket.should_receive('connect').with_args(('public1',
8080)).and_raise(Exception).and_raise(Exception) \
.and_return(None)
flexmock(socket)
socket.should_receive('socket').and_return(fake_socket)
argv = ["--keyname", self.keyname, "--file", self.app_dir]
options = ParseArgs(argv, self.function).args
(host, port) = AppScaleTools.upload_app(options)
self.assertEquals('public1', host)
self.assertEquals(8080, port)
def upload_app(cls, options):
"""Uploads the given App Engine application into AppScale.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
Returns:
A tuple containing the host and port where the application is serving
traffic from.
"""
if cls.TAR_GZ_REGEX.search(options.file):
file_location = LocalState.extract_tgz_app_to_dir(
options.file, options.verbose)
created_dir = True
elif cls.ZIP_REGEX.search(options.file):
file_location = LocalState.extract_zip_app_to_dir(
options.file, options.verbose)
created_dir = True
elif os.path.isdir(options.file):
file_location = options.file
created_dir = False
else:
raise AppEngineConfigException('{0} is not a tar.gz file, a zip file, ' \
'or a directory. Please try uploading either a tar.gz file, a zip ' \
'file, or a directory.'.format(options.file))
try:
app_id = AppEngineHelper.get_app_id_from_app_config(file_location)
except AppEngineConfigException:
# Java App Engine users may have specified their war directory. In that
# case, just move up one level, back to the app's directory.
file_location = file_location + os.sep + ".."
app_id = AppEngineHelper.get_app_id_from_app_config(file_location)
app_language = AppEngineHelper.get_app_runtime_from_app_config(
file_location)
AppEngineHelper.validate_app_id(app_id)
if app_language == 'java':
if AppEngineHelper.is_sdk_mismatch(file_location):
AppScaleLogger.warn(
'AppScale did not find the correct SDK jar ' +
'versions in your app. The current supported ' +
'SDK version is ' + AppEngineHelper.SUPPORTED_SDK_VERSION +
'.')
acc = AppControllerClient(LocalState.get_login_host(options.keyname),
LocalState.get_secret_key(options.keyname))
userappserver_host = acc.get_uaserver_host(options.verbose)
userappclient = UserAppClient(
userappserver_host, LocalState.get_secret_key(options.keyname))
if options.test:
username = LocalState.DEFAULT_USER
elif options.email:
username = options.email
else:
username = LocalState.get_username_from_stdin(is_admin=False)
if not userappclient.does_user_exist(username):
password = LocalState.get_password_from_stdin()
RemoteHelper.create_user_accounts(username,
password,
userappserver_host,
options.keyname,
clear_datastore=False)
app_exists = userappclient.does_app_exist(app_id)
app_admin = userappclient.get_app_admin(app_id)
if app_admin is not None and username != app_admin:
raise AppScaleException("The given user doesn't own this application" + \
", so they can't upload an app with that application ID. Please " + \
"change the application ID and try again.")
if app_exists:
AppScaleLogger.log(
"Uploading new version of app {0}".format(app_id))
else:
AppScaleLogger.log(
"Uploading initial version of app {0}".format(app_id))
userappclient.reserve_app_id(username, app_id, app_language)
remote_file_path = RemoteHelper.copy_app_to_host(
file_location, options.keyname, options.verbose)
acc.done_uploading(app_id, remote_file_path)
acc.update([app_id])
# now that we've told the AppController to start our app, find out what port
# the app is running on and wait for it to start serving
AppScaleLogger.log("Please wait for your app to start serving.")
if app_exists:
time.sleep(20) # give the AppController time to restart the app
serving_host, serving_port = userappclient.get_serving_info(
app_id, options.keyname)
RemoteHelper.sleep_until_port_is_open(serving_host, serving_port,
options.verbose)
AppScaleLogger.success(
"Your app can be reached at the following URL: " +
"http://{0}:{1}".format(serving_host, serving_port))
if created_dir:
shutil.rmtree(file_location)
return (serving_host, serving_port)
def upload_app(cls, options):
"""Uploads the given App Engine application into AppScale.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
Returns:
A tuple containing the host and port where the application is serving
traffic from.
"""
if cls.TAR_GZ_REGEX.search(options.file):
file_location = LocalState.extract_app_to_dir(
options.file, options.verbose)
created_dir = True
else:
file_location = options.file
created_dir = False
app_id = AppEngineHelper.get_app_id_from_app_config(file_location)
app_language = AppEngineHelper.get_app_runtime_from_app_config(
file_location)
AppEngineHelper.validate_app_id(app_id)
acc = AppControllerClient(LocalState.get_login_host(options.keyname),
LocalState.get_secret_key(options.keyname))
userappserver_host = acc.get_uaserver_host(options.verbose)
userappclient = UserAppClient(
userappserver_host, LocalState.get_secret_key(options.keyname))
if options.test:
username = LocalState.DEFAULT_USER
elif options.email:
username = options.email
else:
username = LocalState.get_username_from_stdin(is_admin=False)
if not userappclient.does_user_exist(username):
password = LocalState.get_password_from_stdin()
RemoteHelper.create_user_accounts(username, password,
userappserver_host,
options.keyname)
app_exists = userappclient.does_app_exist(app_id)
app_admin = userappclient.get_app_admin(app_id)
if app_admin is not None and username != app_admin:
raise AppScaleException("The given user doesn't own this application" + \
", so they can't upload an app with that application ID. Please " + \
"change the application ID and try again.")
if app_exists:
AppScaleLogger.log(
"Uploading new version of app {0}".format(app_id))
else:
AppScaleLogger.log(
"Uploading initial version of app {0}".format(app_id))
userappclient.reserve_app_id(username, app_id, app_language)
remote_file_path = RemoteHelper.copy_app_to_host(
file_location, options.keyname, options.verbose)
acc.done_uploading(app_id, remote_file_path)
acc.update([app_id])
# now that we've told the AppController to start our app, find out what port
# the app is running on and wait for it to start serving
AppScaleLogger.log("Please wait for your app to start serving.")
if app_exists:
time.sleep(20) # give the AppController time to restart the app
serving_host, serving_port = userappclient.get_serving_info(
app_id, options.keyname)
RemoteHelper.sleep_until_port_is_open(serving_host, serving_port,
options.verbose)
AppScaleLogger.success(
"Your app can be reached at the following URL: " +
"http://{0}:{1}".format(serving_host, serving_port))
if created_dir:
shutil.rmtree(file_location)
return (serving_host, serving_port)
def test_upload_app_when_app_admin_not_this_user(self):
# we don't let you upload an app if the appid is registered to someone else,
# so abort
# add in mocks so that there is an app.yaml, but with no appid set
flexmock(os.path)
os.path.should_call('exists')
app_yaml_location = AppEngineHelper.get_app_yaml_location(self.app_dir)
os.path.should_receive('exists').with_args(app_yaml_location) \
.and_return(True)
# mock out reading the app.yaml file
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open') # set the fall-through
fake_app_yaml = flexmock(name="fake_app_yaml")
fake_app_yaml.should_receive('read').and_return(
yaml.dump({
'application': 'baz',
'runtime': 'python27'
}))
builtins.should_receive('open').with_args(app_yaml_location, 'r') \
.and_return(fake_app_yaml)
# mock out the SOAP call to the AppController and assume it succeeded
fake_appcontroller = flexmock(name='fake_appcontroller')
fake_appcontroller.should_receive('status').with_args('the secret') \
.and_return('Database is at public1')
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://*****:*****@a.com', 'the secret').and_return('false')
fake_userappserver.should_receive('does_user_exist').with_args(
'a@public1', 'the secret').and_return('false')
fake_userappserver.should_receive('commit_new_user').with_args(
'*****@*****.**', str, 'xmpp_user', 'the secret').and_return('true')
fake_userappserver.should_receive('commit_new_user').with_args(
'a@public1', str, 'xmpp_user', 'the secret').and_return('true')
fake_userappserver.should_receive('get_app_data').with_args(
'baz',
'the secret').and_return('\n\nnum_ports:0\n\napp_owner:[email protected]')
SOAPpy.should_receive('SOAPProxy').with_args('https://*****:*****@a.com")
flexmock(getpass)
getpass.should_receive('getpass').and_return('aaaaaa')
argv = ["--keyname", self.keyname, "--file", self.app_dir]
options = ParseArgs(argv, self.function).args
self.assertRaises(AppScaleException, AppScaleTools.upload_app, options)
def test_java_bad_sdk_version(self):
bad_jars = ['test.jar', 'appengine-api-1.0-sdk-1.7.3.jar']
flexmock(os)
os.should_receive('listdir').and_return(bad_jars)
self.assertEquals(True, AppEngineHelper.is_sdk_mismatch(''))
def test_upload_php_app_successfully(self):
app_dir = '/tmp/appscale-app-1234'
# add in mocks so that the gzip'ed file gets extracted to /tmp
# as well as for removing it later
flexmock(os)
os.should_receive('mkdir').with_args(app_dir) \
.and_return(True)
flexmock(shutil)
shutil.should_receive('rmtree').with_args(app_dir).and_return()
local_state = flexmock(LocalState)
local_state.should_receive('shell')\
.with_args(re.compile('tar zxvf'),False)\
.and_return()
# add in mocks so that there is an app.yaml, but with no appid set
flexmock(os.path)
os.path.should_call('exists')
app_yaml_location = AppEngineHelper.get_app_yaml_location(app_dir)
os.path.should_receive('exists').with_args(app_yaml_location) \
.and_return(True)
# mock out reading the app.yaml file
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open') # set the fall-through
fake_app_yaml = flexmock(name="fake_app_yaml")
fake_app_yaml.should_receive('read').and_return(yaml.dump({
'application' : 'baz',
'runtime' : 'php'
}))
builtins.should_receive('open').with_args(app_yaml_location, 'r') \
.and_return(fake_app_yaml)
# Mock out service host and port
app_data = {'owner' : '*****@*****.**',
'hosts' : {'192.168.1.1' : { 'http' : 8080, 'https' : 4380 }}}
app_stats_data = {'apps': {'baz': {'http': 8080, 'language': 'python27',
'total_reqs': 'no_change', 'appservers': 1, 'https': 4380, 'reqs_enqueued': None}}}
# mock out the SOAP call to the AppController and assume it succeeded
fake_appcontroller = flexmock(name='fake_appcontroller')
fake_appcontroller.should_receive('status').with_args('the secret') \
.and_return('Database is at public1')
fake_appcontroller.should_receive('done_uploading').with_args('baz',
'/opt/appscale/apps/baz.tar.gz', 'the secret').and_return()
fake_appcontroller.should_receive('update').with_args(['baz'],
'the secret').and_return()
fake_appcontroller.should_receive('is_app_running').with_args('baz',
'the secret').and_return(False).and_return(True)
fake_appcontroller.should_receive('does_user_exist').with_args(
'*****@*****.**', 'the secret').and_return('true')
fake_appcontroller.should_receive('does_user_exist').with_args(
'a@public1', 'the secret').and_return('true')
fake_appcontroller.should_receive('does_app_exist').with_args(
'baz', 'the secret').and_return(json.dumps(app_data))
fake_appcontroller.should_receive('get_app_data').with_args(
'baz', 'the secret').and_return(json.dumps(app_data))
fake_appcontroller.should_receive('get_all_stats').with_args(
'the secret').and_return(json.dumps(app_stats_data))
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://*****:*****@a.com")
flexmock(getpass)
getpass.should_receive('getpass').and_return('aaaaaa')
# mock out making the remote app directory
local_state.should_receive('shell') \
.with_args(re.compile('^ssh'), False, 5, stdin=re.compile('^mkdir -p')) \
.and_return()
# and mock out tarring and copying the app
local_state.should_receive('shell') \
.with_args(re.compile('tar -czf'), False) \
.and_return()
local_state.should_receive('shell') \
.with_args(re.compile('/tmp/appscale-app-baz.tar.gz'), False, 5) \
.and_return()
# as well as removing the tar'ed app once we're done copying it
flexmock(os)
os.should_receive('remove').with_args('/tmp/appscale-app-baz-1234.tar.gz') \
.and_return()
os.should_receive('listdir').and_return(['app.yaml','index.py'])
# and slap in a mock that says the app comes up after waiting for it
# three times
fake_socket = flexmock(name='fake_socket')
fake_socket.should_receive('connect').with_args(('public1',
8080)).and_raise(Exception).and_raise(Exception) \
.and_return(None)
flexmock(socket)
socket.should_receive('socket').and_return(fake_socket)
argv = [
"--keyname", self.keyname,
"--file", self.app_dir + ".tar.gz"
]
options = ParseArgs(argv, self.function).args
(host, port) = AppScaleTools.upload_app(options)
self.assertEquals('public1', host)
self.assertEquals(8080, port)
def upload_app(cls, options):
"""Uploads the given App Engine application into AppScale.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
"""
if cls.TAR_GZ_REGEX.search(options.file):
file_location = LocalState.extract_app_to_dir(options.file,
options.verbose)
created_dir = True
else:
file_location = options.file
created_dir = False
app_id = AppEngineHelper.get_app_id_from_app_config(file_location)
app_language = AppEngineHelper.get_app_runtime_from_app_config(
file_location)
AppEngineHelper.validate_app_id(app_id)
acc = AppControllerClient(LocalState.get_login_host(options.keyname),
LocalState.get_secret_key(options.keyname))
userappserver_host = acc.get_uaserver_host(options.verbose)
userappclient = UserAppClient(userappserver_host, LocalState.get_secret_key(
options.keyname))
if options.test:
username = LocalState.DEFAULT_USER
elif options.email:
username = options.email
else:
username = LocalState.get_username_from_stdin(is_admin=False)
if not userappclient.does_user_exist(username):
password = LocalState.get_password_from_stdin()
RemoteHelper.create_user_accounts(username, password, userappserver_host,
options.keyname)
if userappclient.does_app_exist(app_id):
raise AppScaleException("The given application is already running in " + \
"AppScale. Please choose a different application ID or use " + \
"appscale-remove-app to take down the existing application.")
app_admin = userappclient.get_app_admin(app_id)
if app_admin is not None and username != app_admin:
raise AppScaleException("The given user doesn't own this application" + \
", so they can't upload an app with that application ID. Please " + \
"change the application ID and try again.")
AppScaleLogger.log("Uploading {0}".format(app_id))
userappclient.reserve_app_id(username, app_id, app_language)
remote_file_path = RemoteHelper.copy_app_to_host(file_location,
options.keyname, options.verbose)
acc.done_uploading(app_id, remote_file_path)
acc.update([app_id])
# now that we've told the AppController to start our app, find out what port
# the app is running on and wait for it to start serving
AppScaleLogger.log("Please wait for your app to start serving.")
serving_host, serving_port = userappclient.get_serving_info(app_id,
options.keyname)
RemoteHelper.sleep_until_port_is_open(serving_host, serving_port,
options.verbose)
AppScaleLogger.success("Your app can be reached at the following URL: " +
"http://{0}:{1}".format(serving_host, serving_port))
if created_dir:
shutil.rmtree(file_location)
def test_upload_tar_gz_app_successfully(self):
# mock out generating a random app dir, for later mocks
flexmock(uuid)
uuid.should_receive('uuid4').and_return('12345678')
app_dir = '/tmp/appscale-app-12345678'
# add in mocks so that the gzip'ed file gets extracted to /tmp
# as well as for removing it later
flexmock(os)
os.should_receive('mkdir').with_args(app_dir) \
.and_return(True)
flexmock(shutil)
shutil.should_receive('rmtree').with_args(app_dir).and_return()
local_state = flexmock(LocalState)
local_state.should_receive('shell')\
.with_args(re.compile('tar zxvf'),False)\
.and_return()
# add in mocks so that there is an app.yaml, but with no appid set
flexmock(os.path)
os.path.should_call('exists')
app_yaml_location = AppEngineHelper.get_app_yaml_location(app_dir)
os.path.should_receive('exists').with_args(app_yaml_location) \
.and_return(True)
# mock out reading the app.yaml file
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open') # set the fall-through
fake_app_yaml = flexmock(name="fake_app_yaml")
fake_app_yaml.should_receive('read').and_return(yaml.dump({
'application' : 'baz',
'runtime' : 'python'
}))
builtins.should_receive('open').with_args(app_yaml_location, 'r') \
.and_return(fake_app_yaml)
# mock out the SOAP call to the AppController and assume it succeeded
fake_appcontroller = flexmock(name='fake_appcontroller')
fake_appcontroller.should_receive('status').with_args('the secret') \
.and_return('Database is at public1')
fake_appcontroller.should_receive('done_uploading').with_args('baz',
'/var/apps/baz/app/baz.tar.gz', 'the secret').and_return()
fake_appcontroller.should_receive('update').with_args(['baz'],
'the secret').and_return()
fake_appcontroller.should_receive('is_app_running').with_args('baz',
'the secret').and_return(False).and_return(True)
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://*****:*****@a.com', 'the secret').and_return('false')
fake_userappserver.should_receive('commit_new_user').with_args(
'*****@*****.**', str, 'xmpp_user', 'the secret').and_return('true')
fake_userappserver.should_receive('commit_new_user').with_args(
'a@public1', str, 'xmpp_user', 'the secret').and_return('true')
fake_userappserver.should_receive('get_app_data').with_args(
'baz', 'the secret').and_return('\n\nnum_ports:0\n') \
.and_return(app_data).and_return(app_data).and_return(app_data)
fake_userappserver.should_receive('commit_new_app').with_args(
'baz', '*****@*****.**', 'python', 'the secret').and_return('true')
SOAPpy.should_receive('SOAPProxy').with_args('https://*****:*****@a.com")
flexmock(getpass)
getpass.should_receive('getpass').and_return('aaaaaa')
# mock out making the remote app directory
local_state.should_receive('shell')\
.with_args(re.compile('^ssh'),False,5,stdin=re.compile('^mkdir -p'))\
.and_return()
# and mock out tarring and copying the app
local_state.should_receive('shell')\
.with_args(re.compile('tar -czf'),False)\
.and_return()
local_state.should_receive('shell')\
.with_args(re.compile('/tmp/appscale-app-baz.tar.gz'),False,5)\
.and_return()
# as well as removing the tar'ed app once we're done copying it
flexmock(os)
os.should_receive('remove').with_args('/tmp/appscale-app-baz.tar.gz') \
.and_return()
# and slap in a mock that says the app comes up after waiting for it
# three times
fake_socket = flexmock(name='fake_socket')
fake_socket.should_receive('connect').with_args(('public1',
8080)).and_raise(Exception).and_raise(Exception) \
.and_return(None)
flexmock(socket)
socket.should_receive('socket').and_return(fake_socket)
argv = [
"--keyname", self.keyname,
"--file", self.app_dir + ".tar.gz"
]
options = ParseArgs(argv, self.function).args
AppScaleTools.upload_app(options)
def test_java_bad_sdk_version(self):
bad_jars = ['test.jar', 'appengine-api-1.0-sdk-1.7.3.jar']
flexmock(os)
os.should_receive('listdir').with_args('/war/WEB-INF/lib').and_return(bad_jars)
self.assertEquals(True, AppEngineHelper.is_sdk_mismatch(''))
def test_upload_php_app_successfully(self):
app_dir = '/tmp/appscale-app-1234'
# add in mocks so that the gzip'ed file gets extracted to /tmp
# as well as for removing it later
flexmock(os)
os.should_receive('mkdir').with_args(app_dir) \
.and_return(True)
flexmock(shutil)
shutil.should_receive('rmtree').with_args(app_dir).and_return()
local_state = flexmock(LocalState)
local_state.should_receive('shell')\
.with_args(re.compile('tar zxvf'),False)\
.and_return()
# add in mocks so that there is an app.yaml, but with no appid set
flexmock(os.path)
os.path.should_call('exists')
app_yaml_location = AppEngineHelper.get_app_yaml_location(app_dir)
os.path.should_receive('exists').with_args(app_yaml_location) \
.and_return(True)
# mock out reading the app.yaml file
builtins = flexmock(sys.modules['__builtin__'])
builtins.should_call('open') # set the fall-through
fake_app_yaml = flexmock(name="fake_app_yaml")
fake_app_yaml.should_receive('read').and_return(
yaml.dump({
'application': 'baz',
'runtime': 'php'
}))
builtins.should_receive('open').with_args(app_yaml_location, 'r') \
.and_return(fake_app_yaml)
# mock out the SOAP call to the AppController and assume it succeeded
fake_appcontroller = flexmock(name='fake_appcontroller')
fake_appcontroller.should_receive('status').with_args('the secret') \
.and_return('Database is at public1')
fake_appcontroller.should_receive('done_uploading').with_args(
'baz', '/opt/appscale/apps/baz.tar.gz', 'the secret').and_return()
fake_appcontroller.should_receive('update').with_args(
['baz'], 'the secret').and_return()
fake_appcontroller.should_receive('is_app_running').with_args(
'baz', 'the secret').and_return(False).and_return(True)
flexmock(SOAPpy)
SOAPpy.should_receive('SOAPProxy').with_args('https://*****:*****@a.com', 'the secret').and_return('false')
fake_userappserver.should_receive('does_user_exist').with_args(
'a@public1', 'the secret').and_return('false')
fake_userappserver.should_receive('commit_new_user').with_args(
'*****@*****.**', str, 'xmpp_user', 'the secret').and_return('true')
fake_userappserver.should_receive('commit_new_user').with_args(
'a@public1', str, 'xmpp_user', 'the secret').and_return('true')
fake_userappserver.should_receive('get_app_data').with_args(
'baz', 'the secret').and_return('\n\nnum_ports:0\n') \
.and_return(app_data).and_return(app_data).and_return(app_data)
fake_userappserver.should_receive('commit_new_app').with_args(
'baz', '*****@*****.**', 'php', 'the secret').and_return('true')
SOAPpy.should_receive('SOAPProxy').with_args('https://*****:*****@a.com")
flexmock(getpass)
getpass.should_receive('getpass').and_return('aaaaaa')
# mock out making the remote app directory
local_state.should_receive('shell') \
.with_args(re.compile('^ssh'), False, 5, stdin=re.compile('^mkdir -p')) \
.and_return()
# and mock out tarring and copying the app
local_state.should_receive('shell') \
.with_args(re.compile('tar -czf'), False) \
.and_return()
local_state.should_receive('shell') \
.with_args(re.compile('/tmp/appscale-app-baz.tar.gz'), False, 5) \
.and_return()
# as well as removing the tar'ed app once we're done copying it
flexmock(os)
os.should_receive('remove').with_args('/tmp/appscale-app-baz-1234.tar.gz') \
.and_return()
os.should_receive('listdir').and_return(['app.yaml', 'index.py'])
# and slap in a mock that says the app comes up after waiting for it
# three times
fake_socket = flexmock(name='fake_socket')
fake_socket.should_receive('connect').with_args(('public1',
8080)).and_raise(Exception).and_raise(Exception) \
.and_return(None)
flexmock(socket)
socket.should_receive('socket').and_return(fake_socket)
argv = ["--keyname", self.keyname, "--file", self.app_dir + ".tar.gz"]
options = ParseArgs(argv, self.function).args
(host, port) = AppScaleTools.upload_app(options)
self.assertEquals('public1', host)
self.assertEquals(8080, port)
def upload_app(cls, options):
"""Uploads the given App Engine application into AppScale.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
Returns:
A tuple containing the host and port where the application is serving
traffic from.
"""
if cls.TAR_GZ_REGEX.search(options.file):
file_location = LocalState.extract_tgz_app_to_dir(
options.file, options.verbose)
created_dir = True
elif cls.ZIP_REGEX.search(options.file):
file_location = LocalState.extract_zip_app_to_dir(
options.file, options.verbose)
created_dir = True
elif os.path.isdir(options.file):
file_location = options.file
created_dir = False
else:
raise AppEngineConfigException('{0} is not a tar.gz file, a zip file, ' \
'or a directory. Please try uploading either a tar.gz file, a zip ' \
'file, or a directory.'.format(options.file))
try:
app_id = AppEngineHelper.get_app_id_from_app_config(file_location)
except AppEngineConfigException as config_error:
AppScaleLogger.log(config_error)
if 'yaml' in str(config_error):
raise config_error
# Java App Engine users may have specified their war directory. In that
# case, just move up one level, back to the app's directory.
file_location = file_location + os.sep + ".."
app_id = AppEngineHelper.get_app_id_from_app_config(file_location)
app_language = AppEngineHelper.get_app_runtime_from_app_config(
file_location)
AppEngineHelper.validate_app_id(app_id)
if app_language == 'java':
if AppEngineHelper.is_sdk_mismatch(file_location):
AppScaleLogger.warn(
'AppScale did not find the correct SDK jar ' +
'versions in your app. The current supported ' +
'SDK version is ' + AppEngineHelper.SUPPORTED_SDK_VERSION +
'.')
login_host = LocalState.get_login_host(options.keyname)
secret_key = LocalState.get_secret_key(options.keyname)
acc = AppControllerClient(login_host, secret_key)
if options.test:
username = LocalState.DEFAULT_USER
elif options.email:
username = options.email
else:
username = LocalState.get_username_from_stdin(is_admin=False)
if not acc.does_user_exist(username):
password = LocalState.get_password_from_stdin()
RemoteHelper.create_user_accounts(username,
password,
login_host,
options.keyname,
clear_datastore=False)
app_exists = acc.does_app_exist(app_id)
app_admin = acc.get_app_admin(app_id)
if app_admin is not None and username != app_admin:
raise AppScaleException("The given user doesn't own this application" + \
", so they can't upload an app with that application ID. Please " + \
"change the application ID and try again.")
if app_exists:
AppScaleLogger.log(
"Uploading new version of app {0}".format(app_id))
else:
AppScaleLogger.log(
"Uploading initial version of app {0}".format(app_id))
acc.reserve_app_id(username, app_id, app_language)
# Ignore all .pyc files while tarring.
if app_language == 'python27':
AppScaleLogger.log("Ignoring .pyc files")
remote_file_path = RemoteHelper.copy_app_to_host(
file_location, options.keyname, options.verbose)
acc.done_uploading(app_id, remote_file_path)
acc.update([app_id])
# now that we've told the AppController to start our app, find out what port
# the app is running on and wait for it to start serving
AppScaleLogger.log("Please wait for your app to start serving.")
if app_exists:
time.sleep(20) # give the AppController time to restart the app
# Makes a call to the AppController to get all the stats and looks
# through them for the http port the app can be reached on.
sleep_time = 2 * cls.SLEEP_TIME
current_app = None
for i in range(cls.MAX_RETRIES):
try:
result = acc.get_all_stats()
json_result = json.loads(result)
apps_result = json_result['apps']
current_app = apps_result[app_id]
http_port = current_app['http']
break
except ValueError:
pass
except KeyError:
pass
AppScaleLogger.verbose("Waiting {0} second(s) for a port to be assigned to {1}".\
format(sleep_time, app_id), options.verbose)
time.sleep(sleep_time)
if not current_app:
raise AppScaleException(
"Unable to get the serving port for the application.")
RemoteHelper.sleep_until_port_is_open(login_host, http_port,
options.verbose)
AppScaleLogger.success(
"Your app can be reached at the following URL: " +
"http://{0}:{1}".format(login_host, http_port))
if created_dir:
shutil.rmtree(file_location)
return (login_host, http_port)
def upload_app(cls, options):
"""Uploads the given App Engine application into AppScale.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
Returns:
A tuple containing the host and port where the application is serving
traffic from.
"""
if cls.TAR_GZ_REGEX.search(options.file):
file_location = LocalState.extract_tgz_app_to_dir(options.file, options.verbose)
created_dir = True
elif cls.ZIP_REGEX.search(options.file):
file_location = LocalState.extract_zip_app_to_dir(options.file, options.verbose)
created_dir = True
elif os.path.isdir(options.file):
file_location = options.file
created_dir = False
else:
raise AppEngineConfigException(
"{0} is not a tar.gz file, a zip file, "
"or a directory. Please try uploading either a tar.gz file, a zip "
"file, or a directory.".format(options.file)
)
try:
app_id = AppEngineHelper.get_app_id_from_app_config(file_location)
except AppEngineConfigException:
# Java App Engine users may have specified their war directory. In that
# case, just move up one level, back to the app's directory.
file_location = file_location + os.sep + ".."
app_id = AppEngineHelper.get_app_id_from_app_config(file_location)
app_language = AppEngineHelper.get_app_runtime_from_app_config(file_location)
AppEngineHelper.validate_app_id(app_id)
if app_language == "java":
if AppEngineHelper.is_sdk_mismatch(file_location):
AppScaleLogger.warn(
"AppScale did not find the correct SDK jar "
+ "versions in your app. The current supported "
+ "SDK version is "
+ AppEngineHelper.SUPPORTED_SDK_VERSION
+ "."
)
acc = AppControllerClient(
LocalState.get_login_host(options.keyname), LocalState.get_secret_key(options.keyname)
)
userappclient = UserAppClient(
LocalState.get_login_host(options.keyname), LocalState.get_secret_key(options.keyname)
)
if options.test:
username = LocalState.DEFAULT_USER
elif options.email:
username = options.email
else:
username = LocalState.get_username_from_stdin(is_admin=False)
if not userappclient.does_user_exist(username):
password = LocalState.get_password_from_stdin()
RemoteHelper.create_user_accounts(
username, password, LocalState.get_login_host(options.keyname), options.keyname, clear_datastore=False
)
app_exists = userappclient.does_app_exist(app_id)
app_admin = userappclient.get_app_admin(app_id)
if app_admin is not None and username != app_admin:
raise AppScaleException(
"The given user doesn't own this application"
+ ", so they can't upload an app with that application ID. Please "
+ "change the application ID and try again."
)
if app_exists:
AppScaleLogger.log("Uploading new version of app {0}".format(app_id))
else:
AppScaleLogger.log("Uploading initial version of app {0}".format(app_id))
userappclient.reserve_app_id(username, app_id, app_language)
# Ignore all .pyc files while tarring.
if app_language == "python27":
AppScaleLogger.log("Ignoring .pyc files")
remote_file_path = RemoteHelper.copy_app_to_host(file_location, options.keyname, options.verbose)
acc.done_uploading(app_id, remote_file_path)
acc.update([app_id])
# now that we've told the AppController to start our app, find out what port
# the app is running on and wait for it to start serving
AppScaleLogger.log("Please wait for your app to start serving.")
if app_exists:
time.sleep(20) # give the AppController time to restart the app
serving_host, serving_port = userappclient.get_serving_info(app_id, options.keyname)
RemoteHelper.sleep_until_port_is_open(serving_host, serving_port, options.verbose)
AppScaleLogger.success(
"Your app can be reached at the following URL: " + "http://{0}:{1}".format(serving_host, serving_port)
)
if created_dir:
shutil.rmtree(file_location)
return (serving_host, serving_port)
