def from_contents(contents, file_name):
""" Constructs a Version from the contents of a config file.
Args:
contents: A string containing the entire configuration contents.
file_name: A string specifying the type of config file
(app.yaml or appengine-web.xml).
Returns:
A Version object.
Raise:
AppengineConfigException if the configuration contents are invalid.
"""
if file_name == 'app.yaml':
try:
app_yaml = yaml.safe_load(contents)
except yaml.YAMLError as error:
raise AppEngineConfigException(
'Invalid app.yaml: {}'.format(error))
return Version.from_yaml(app_yaml)
else:
try:
appengine_web_xml = ElementTree.fromstring(contents)
except ElementTree.ParseError as error:
raise AppEngineConfigException(
'Invalid appengine-web.xml: {}'.format(error))
return Version.from_xml(appengine_web_xml)
def from_zip(zip_location):
""" Constructs a Version from a zip file.
Args:
zip_location: A string specifying a location to a zip file.
Returns:
A Version object.
Raises:
AppengineConfigException if the config is invalid or cannot be extracted.
"""
with zipfile.ZipFile(zip_location) as zip_file:
file_name = 'app.yaml'
name_list = zip_file.namelist()
config_location = shortest_path_from_list(file_name, name_list)
if config_location is None:
file_name = 'appengine-web.xml'
config_location = shortest_path_from_list(file_name, name_list)
if config_location is None:
raise AppEngineConfigException(
'Unable to find app.yaml or appengine-web.xml')
with zip_file.open(config_location) as config_file:
contents = config_file.read()
return Version.from_contents(contents, file_name)
def from_tar_gz(tar_location):
""" Constructs a Version from a gzipped tarball.
Args:
tar_location: A string specifying a location to a gzipped tarball.
Returns:
A Version object.
Raises:
AppengineConfigException if the config is invalid or cannot be extracted.
"""
with tarfile.open(tar_location, 'r:gz') as tar:
file_name = 'app.yaml'
name_list = [member.name for member in tar.getmembers()]
config_location = shortest_path_from_list(file_name, name_list)
if config_location is None:
file_name = 'appengine-web.xml'
config_location = shortest_path_from_list(file_name, name_list)
if config_location is None:
raise AppEngineConfigException(
'Unable to find app.yaml or appengine-web.xml')
config_file = tar.extractfile(config_location)
try:
contents = config_file.read()
finally:
config_file.close()
return Version.from_contents(contents, file_name)
def validate_app_id(cls, app_id):
"""Checks the given app_id to make sure that it represents an app_id that
we can run within AppScale.
Args:
app_id: A str that represents the application ID.
Raises:
AppEngineConfigException: If the given application ID is a reserved
app_id, or does not represent an acceptable app_id.
"""
if app_id in cls.DISALLOWED_APP_IDS:
raise AppEngineConfigException(
"{0} is a reserved appid".format(app_id))
if not cls.APP_ID_REGEX.match(app_id):
raise AppEngineConfigException("Invalid application ID. You can only" + \
" use alphanumeric characters and/or '-'.")
def validate_app_id(cls, app_id):
"""Checks the given app_id to make sure that it represents an app_id that
we can run within AppScale.
Args:
app_id: A str that represents the application ID.
Raises:
AppEngineConfigException: If the given application ID is a reserved
app_id, or does not represent an acceptable app_id.
"""
if app_id in cls.DISALLOWED_APP_IDS:
raise AppEngineConfigException(
"{0} is a reserved appid".format(app_id))
if not cls.APP_ID_REGEX.match(app_id):
raise AppEngineConfigException("Invalid application ID." + \
" It must be 6 to 30 lowercase letters, digits, " + \
"or hyphens. It must start with a letter.")
def from_yaml(app_yaml):
""" Constructs a Version from a parsed app.yaml.
Args:
app_yaml: A dictionary generated by a parsed app.yaml.
Returns:
A Version object.
Raises:
AppengineConfigException if app_yaml is invalid.
"""
try:
runtime = app_yaml['runtime']
except KeyError:
raise AppEngineConfigException('Missing app.yaml element: runtime')
version = Version(runtime)
version.configuration_type = 'app.yaml'
version.project_id = app_yaml.get('application')
if 'service' in app_yaml and 'module' in app_yaml:
raise AppEngineConfigException(
'Invalid app.yaml: If "service" is defined, "module" cannot be '
'defined.')
version.service_id = (app_yaml.get('service') or app_yaml.get('module')
or DEFAULT_SERVICE)
version.env_variables = app_yaml.get('env_variables', {})
version.inbound_services = app_yaml.get('inbound_services', [])
if version.runtime in ('python27', 'java'):
try:
version.threadsafe = app_yaml['threadsafe']
except KeyError:
raise AppEngineConfigException(
'Invalid app.yaml: {} applications require the "threadsafe" '
'element'.format(version.runtime))
if not isinstance(version.threadsafe, bool):
raise AppEngineConfigException(
'Invalid app.yaml: "threadsafe" must be a boolean')
return version
def from_yaml_file(yaml_location):
""" Constructs a Version from an app.yaml file.
Args:
yaml_location: A string specifying the location to an app.yaml.
Returns:
A Version object.
Raises:
AppengineConfigException if the app.yaml is invalid or missing.
"""
try:
with open(yaml_location) as yaml_file:
try:
app_yaml = yaml.safe_load(yaml_file)
except yaml.YAMLError as error:
raise AppEngineConfigException(
'Invalid app.yaml: {}'.format(error))
return Version.from_yaml(app_yaml)
except IOError:
raise AppEngineConfigException(
'Unable to read {}'.format(yaml_location))
def from_xml_file(xml_location):
""" Constructs a Version from an appengine-web.xml file.
Args:
xml_location: A string specifying the location to an appengine-web.xml.
Returns:
A Version object.
Raises:
AppengineConfigException if the appengine-web.xml is invalid or missing.
"""
try:
tree = ElementTree.parse(xml_location)
except (IOError, ElementTree.ParseError) as error:
raise AppEngineConfigException(
'Invalid appengine-web.xml: {}'.format(error))
return Version.from_xml(tree.getroot())
def from_directory(source_location):
""" Constructs a Version from a directory.
Args:
source_location: A string specifying a path to the source directory.
Returns:
A Version object.
Raises:
AppengineConfigException if the app's config file is invalid or missing.
"""
config_location = shortest_directory_path('app.yaml', source_location)
if config_location is not None:
return Version.from_yaml_file(config_location)
config_location = shortest_directory_path('appengine-web.xml',
source_location)
if config_location is not None:
return Version.from_xml_file(config_location)
raise AppEngineConfigException(
'Unable to find app.yaml or appengine-web.xml')
def from_yaml(cls, yaml_entry):
""" Creates a Handler from a parsed section from app.yaml.
Args:
yaml_entry: A dictionary generated by a parsed handler section.
Returns:
A Handler object.
"""
try:
url = yaml_entry['url']
except KeyError:
raise AppEngineConfigException(
'Missing url from handler: {}'.format(yaml_entry))
handler = Handler(url)
for field in yaml_entry:
if field not in cls.FIELD_RULES:
raise AppEngineConfigException(
'Unrecognized handler field: {}'.format(field))
for field, rule in cls.FIELD_RULES.items():
value = yaml_entry.get(field)
if value is not None:
if not rule(value):
raise AppEngineConfigException(
'Invalid {} value: {}'.format(field, value))
setattr(handler, field, value)
if handler.script is not None and handler.static_defined:
raise AppEngineConfigException(
'Handler cannot contain both script and static elements')
if handler.script is None and not handler.static_defined:
raise AppEngineConfigException(
'Handler must contain either script or static element')
if handler.static_defined:
if handler.static_dir is not None and handler.static_files is not None:
raise AppEngineConfigException(
'Handler cannot contain both static_dir and static_files')
return handler
def upload_app(cls, options):
"""Uploads the given App Engine application into AppScale.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
Returns:
A tuple containing the host and port where the application is serving
traffic from.
"""
custom_service_yaml = None
if cls.TAR_GZ_REGEX.search(options.file):
file_location = LocalState.extract_tgz_app_to_dir(options.file)
created_dir = True
version = Version.from_tar_gz(options.file)
elif cls.ZIP_REGEX.search(options.file):
file_location = LocalState.extract_zip_app_to_dir(options.file)
created_dir = True
version = Version.from_zip(options.file)
elif os.path.isdir(options.file):
file_location = options.file
created_dir = False
version = Version.from_directory(options.file)
elif options.file.endswith('.yaml'):
file_location = os.path.dirname(options.file)
created_dir = False
version = Version.from_yaml_file(options.file)
custom_service_yaml = options.file
else:
raise AppEngineConfigException('{0} is not a tar.gz file, a zip file, ' \
'or a directory. Please try uploading either a tar.gz file, a zip ' \
'file, or a directory.'.format(options.file))
if options.project:
if version.runtime == 'java':
raise BadConfigurationException("AppScale doesn't support --project for"
"Java yet. Please specify the application id in appengine-web.xml.")
version.project_id = options.project
if version.project_id is None:
if version.config_type == 'app.yaml':
message = 'Specify --project or define "application" in your app.yaml'
else:
message = 'Define "application" in your appengine-web.xml'
raise AppEngineConfigException(message)
# Let users know that versions are not supported yet.
AppEngineHelper.warn_if_version_defined(version, options.test)
AppEngineHelper.validate_app_id(version.project_id)
extras = {}
if version.runtime == 'go':
extras = LocalState.get_extra_go_dependencies(options.file, options.test)
if (version.runtime == 'java'
and AppEngineHelper.is_sdk_mismatch(file_location)):
AppScaleLogger.warn(
'AppScale did not find the correct SDK jar versions in your app. The '
'current supported SDK version is '
'{}.'.format(AppEngineHelper.SUPPORTED_SDK_VERSION))
head_node_public_ip = LocalState.get_host_with_role(
options.keyname, 'shadow')
secret_key = LocalState.get_secret_key(options.keyname)
admin_client = AdminClient(head_node_public_ip, secret_key)
remote_file_path = RemoteHelper.copy_app_to_host(
file_location, version.project_id, options.keyname,
extras, custom_service_yaml)
AppScaleLogger.log(
'Deploying service {} for {}'.format(version.service_id,
version.project_id))
operation_id = admin_client.create_version(version, remote_file_path)
# now that we've told the AppController to start our app, find out what port
# the app is running on and wait for it to start serving
AppScaleLogger.log("Please wait for your app to start serving.")
deadline = time.time() + cls.MAX_OPERATION_TIME
while True:
if time.time() > deadline:
raise AppScaleException('The deployment operation took too long.')
operation = admin_client.get_operation(version.project_id, operation_id)
if not operation['done']:
time.sleep(1)
continue
if 'error' in operation:
raise AppScaleException(operation['error']['message'])
version_url = operation['response']['versionUrl']
break
AppScaleLogger.success(
'Your app can be reached at the following URL: {}'.format(version_url))
if created_dir:
shutil.rmtree(file_location)
match = re.match('http://(.+):(\d+)', version_url)
login_host = match.group(1)
http_port = int(match.group(2))
return login_host, http_port
def from_xml(root):
""" Constructs a Version from a parsed appengine-web.xml.
Args:
root: An XML Element object representing the document root.
Returns:
A Version object.
"""
runtime_element = root.find(''.join([XML_NAMESPACE, 'runtime']))
runtime = 'java7'
if runtime_element is not None:
runtime = runtime_element.text
version = Version(runtime)
version.configuration_type = 'appengine-web.xml'
application_element = root.find(''.join([XML_NAMESPACE,
'application']))
if application_element is not None:
version.project_id = application_element.text
service_element = root.find(''.join([XML_NAMESPACE, 'service']))
module_element = root.find(''.join([XML_NAMESPACE, 'module']))
if service_element is not None and module_element is not None:
raise AppEngineConfigException(
'Invalid appengine-web.xml: If "service" is defined, "module" cannot '
'be defined.')
if module_element is not None:
version.service_id = module_element.text
if service_element is not None:
version.service_id = service_element.text
if not version.service_id:
version.service_id = DEFAULT_SERVICE
env_vars_element = root.find(''.join([XML_NAMESPACE, 'env-variables']))
if env_vars_element is not None:
version.env_variables = {
var.attrib['name']: var.attrib['value']
for var in env_vars_element
}
inbound_services = root.find(''.join(
[XML_NAMESPACE, 'inbound-services']))
if inbound_services is not None:
version.inbound_services = [
service.text for service in inbound_services
]
threadsafe_element = root.find(''.join([XML_NAMESPACE, 'threadsafe']))
if threadsafe_element is None:
raise AppEngineConfigException(
'Invalid appengine-web.xml: missing "threadsafe" element')
if threadsafe_element.text.lower() not in ('true', 'false'):
raise AppEngineConfigException(
'Invalid appengine-web.xml: Invalid "threadsafe" value. '
'It must be either "true" or "false".')
version.threadsafe = threadsafe_element.text.lower() == 'true'
return version
def from_yaml(app_yaml):
""" Constructs a Version from a parsed app.yaml.
Args:
app_yaml: A dictionary generated by a parsed app.yaml.
Returns:
A Version object.
Raises:
AppengineConfigException if app_yaml is invalid.
"""
try:
runtime = app_yaml['runtime']
except KeyError:
raise AppEngineConfigException('Missing app.yaml element: runtime')
try:
handlers = app_yaml['handlers']
except KeyError:
raise AppEngineConfigException(
'Missing app.yaml element: handlers')
version = Version(runtime, 'app.yaml')
version.project_id = app_yaml.get('application')
version.handlers = [Handler.from_yaml(handler) for handler in handlers]
if 'service' in app_yaml and 'module' in app_yaml:
raise AppEngineConfigException(
'Invalid app.yaml: If "service" is defined, "module" cannot be '
'defined.')
version.service_id = (app_yaml.get('service') or app_yaml.get('module')
or DEFAULT_SERVICE)
version.env_variables = app_yaml.get('env_variables', {})
version.inbound_services = app_yaml.get('inbound_services', [])
automatic_scaling = app_yaml.get('automatic_scaling', None)
manual_scaling = app_yaml.get('manual_scaling', None)
if automatic_scaling and manual_scaling:
raise AppEngineConfigException(
'Invalid app.yaml: If "automatic_scaling" is defined, "manual_scaling" '
'cannot be defined.')
elif manual_scaling:
try:
version.manual_scaling = {
'instances': int(manual_scaling['instances'])
}
except StandardError:
raise AppEngineConfigException(
'Invalid app.yaml: manual_scaling invalid.')
elif automatic_scaling:
version.automatic_scaling = defaultdict(dict)
try:
min_instances = automatic_scaling.get('min_instances')
if min_instances is not None:
version.automatic_scaling['standardSchedulerSettings'][
'minInstances'] = (int(min_instances))
max_instances = automatic_scaling.get('max_instances')
if max_instances is not None:
version.automatic_scaling['standardSchedulerSettings'][
'maxInstances'] = (int(max_instances))
min_idle = automatic_scaling.get('min_idle_instances')
if min_idle is not None:
version.automatic_scaling['minIdleInstances'] = int(
min_idle)
max_idle = automatic_scaling.get('max_idle_instances')
if max_idle is not None:
version.automatic_scaling['maxIdleInstances'] = int(
max_idle)
except ValueError:
raise AppEngineConfigException(
'Invalid app.yaml: value for '
'automatic scaling option is not integer.')
if version.runtime in ('python27', 'java'):
try:
version.threadsafe = app_yaml['threadsafe']
except KeyError:
raise AppEngineConfigException(
'Invalid app.yaml: {} applications require the "threadsafe" '
'element'.format(version.runtime))
if not isinstance(version.threadsafe, bool):
raise AppEngineConfigException(
'Invalid app.yaml: "threadsafe" must be a boolean')
return version
def from_xml(root):
""" Constructs a Version from a parsed appengine-web.xml.
Args:
root: An XML Element object representing the document root.
Returns:
A Version object.
"""
qname = lambda name: ''.join([XML_NAMESPACE, name])
runtime_element = root.find(qname('runtime'))
runtime = 'java7'
if runtime_element is not None:
runtime = runtime_element.text
version = Version(runtime, 'appengine-web.xml')
application_element = root.find(qname('application'))
if application_element is not None:
version.project_id = application_element.text
service_element = root.find(qname('service'))
module_element = root.find(qname('module'))
if service_element is not None and module_element is not None:
raise AppEngineConfigException(
'Invalid appengine-web.xml: If "service" is defined, "module" cannot '
'be defined.')
if module_element is not None:
version.service_id = module_element.text
if service_element is not None:
version.service_id = service_element.text
if not version.service_id:
version.service_id = DEFAULT_SERVICE
env_vars_element = root.find(qname('env-variables'))
if env_vars_element is not None:
version.env_variables = {
var.attrib['name']: var.attrib['value']
for var in env_vars_element
}
inbound_services = root.find(qname('inbound-services'))
if inbound_services is not None:
version.inbound_services = [
service.text for service in inbound_services
]
automatic_scaling = root.find(qname('automatic-scaling'))
manual_scaling = root.find(qname('manual-scaling'))
if automatic_scaling is not None and manual_scaling is not None:
raise AppEngineConfigException(
'Invalid appengine-web.xml: If "automatic-scaling" is defined, '
'"manual-scaling" cannot be defined.')
elif manual_scaling is not None:
try:
version.manual_scaling = {
'instances':
int(manual_scaling.findtext(qname('instances')))
}
except StandardError:
raise AppEngineConfigException(
'Invalid app.yaml: manual_scaling invalid.')
elif automatic_scaling is not None:
version.automatic_scaling = defaultdict(dict)
try:
min_instances = automatic_scaling.findtext(
qname('min-instances'))
if min_instances is not None:
version.automatic_scaling['standardSchedulerSettings'][
'minInstances'] = (int(min_instances))
max_instances = automatic_scaling.findtext(
qname('max-instances'))
if max_instances is not None:
version.automatic_scaling['standardSchedulerSettings'][
'maxInstances'] = (int(max_instances))
min_idle = automatic_scaling.findtext(
qname('min-idle-instances'))
if min_idle is not None:
version.automatic_scaling['minIdleInstances'] = int(
min_idle)
max_idle = automatic_scaling.findtext(
qname('max-idle-instances'))
if max_idle is not None:
version.automatic_scaling['maxIdleInstances'] = int(
max_idle)
except ValueError:
raise AppEngineConfigException(
'Invalid appengine-web.xml: value for '
'automatic scaling option is not integer.')
threadsafe_element = root.find(qname('threadsafe'))
if threadsafe_element is None:
raise AppEngineConfigException(
'Invalid appengine-web.xml: missing "threadsafe" element')
if threadsafe_element.text.lower() not in ('true', 'false'):
raise AppEngineConfigException(
'Invalid appengine-web.xml: Invalid "threadsafe" value. '
'It must be either "true" or "false".')
version.threadsafe = threadsafe_element.text.lower() == 'true'
return version