def main():
""" This main function allows you to run the groomer manually. """
zk_connection_locations = appscale_info.get_zk_locations_string()
zookeeper = zk.ZKTransaction(host=zk_connection_locations, start_gc=False)
db_info = appscale_info.get_db_info()
table = db_info[':table']
master = appscale_info.get_db_master_ip()
datastore_path = "{0}:8888".format(master)
ds_groomer = DatastoreGroomer(zookeeper, table, datastore_path)
logging.debug("Trying to get groomer lock.")
if ds_groomer.get_groomer_lock():
logging.info("Got the groomer lock.")
try:
ds_groomer.run_groomer()
except Exception as exception:
logging.exception('Encountered exception {} while running the groomer.'
.format(str(exception)))
try:
ds_groomer.zoo_keeper.release_lock_with_path(zk.DS_GROOM_LOCK_PATH)
except zk.ZKTransactionException, zk_exception:
logging.error("Unable to release zk lock {0}.".\
format(str(zk_exception)))
except zk.ZKInternalException, zk_exception:
logging.error("Unable to release zk lock {0}.".\
format(str(zk_exception)))
def wait_for_quorum(keyname, db_nodes, replication):
""" Waits until enough Cassandra nodes are up for a quorum.
Args:
keyname: A string containing the deployment's keyname.
db_nodes: An integer specifying the total number of DB nodes.
replication: An integer specifying the keyspace replication factor.
"""
command = cassandra_interface.NODE_TOOL + " " + 'status'
key_file = '{}/{}.key'.format(utils.KEY_DIRECTORY, keyname)
ssh_cmd = ['ssh', '-i', key_file, appscale_info.get_db_master_ip(), command]
# Determine the number of nodes needed for a quorum.
if db_nodes < 1 or replication < 1:
raise dbconstants.AppScaleDBError('At least 1 database machine is needed.')
if replication > db_nodes:
raise dbconstants.AppScaleDBError(
'The replication factor cannot exceed the number of database machines.')
can_fail = math.ceil(replication/2.0 - 1)
needed = int(db_nodes - can_fail)
while True:
output = subprocess.check_output(ssh_cmd)
nodes_ready = len(
[line for line in output.splitlines() if line.startswith('UN')])
logging.info('{} nodes are up. {} are needed.'.format(nodes_ready, needed))
if nodes_ready >= needed:
break
time.sleep(1)
def main():
""" This main function allows you to run the groomer manually. """
zk_connection_locations = appscale_info.get_zk_locations_string()
zookeeper = zk.ZKTransaction(host=zk_connection_locations)
db_info = appscale_info.get_db_info()
table = db_info[':table']
master = appscale_info.get_db_master_ip()
datastore_path = "{0}:8888".format(master)
ds_groomer = DatastoreGroomer(zookeeper, table, datastore_path)
logging.debug("Trying to get groomer lock.")
if ds_groomer.get_groomer_lock():
logging.info("Got the groomer lock.")
try:
ds_groomer.run_groomer()
except Exception as exception:
logging.exception('Encountered exception {} while running the groomer.'
.format(str(exception)))
try:
ds_groomer.zoo_keeper.release_lock_with_path(zk.DS_GROOM_LOCK_PATH)
except zk.ZKTransactionException, zk_exception:
logging.error("Unable to release zk lock {0}.".\
format(str(zk_exception)))
except zk.ZKInternalException, zk_exception:
logging.error("Unable to release zk lock {0}.".\
format(str(zk_exception)))
def wait_for_quorum(keyname, db_nodes, replication):
""" Waits until enough Cassandra nodes are up for a quorum.
Args:
keyname: A string containing the deployment's keyname.
db_nodes: An integer specifying the total number of DB nodes.
replication: An integer specifying the keyspace replication factor.
"""
command = cassandra_interface.NODE_TOOL + " " + 'status'
key_file = '{}/{}.key'.format(utils.KEY_DIRECTORY, keyname)
ssh_cmd = [
'ssh', '-i', key_file,
appscale_info.get_db_master_ip(), command
]
# Determine the number of nodes needed for a quorum.
if db_nodes < 1 or replication < 1:
raise dbconstants.AppScaleDBError(
'At least 1 database machine is needed.')
if replication > db_nodes:
raise dbconstants.AppScaleDBError(
'The replication factor cannot exceed the number of database machines.'
)
can_fail = math.ceil(replication / 2.0 - 1)
needed = int(db_nodes - can_fail)
while True:
output = subprocess.check_output(ssh_cmd)
nodes_ready = len(
[line for line in output.splitlines() if line.startswith('UN')])
logging.info('{} nodes are up. {} are needed.'.format(
nodes_ready, needed))
if nodes_ready >= needed:
break
time.sleep(1)
def get_soap_accessor():
""" Returns the SOAP server accessor to deal with application and users.
Returns:
A soap server accessor.
"""
db_ip = appscale_info.get_db_master_ip()
bindport = constants.UA_SERVER_PORT
return SOAPpy.SOAPProxy("https://{0}:{1}".format(db_ip, bindport))
def get_soap_accessor():
""" Returns the SOAP server accessor to deal with application and users.
Returns:
A soap server accessor.
"""
db_ip = appscale_info.get_db_master_ip()
bindport = constants.UA_SERVER_PORT
return SOAPpy.SOAPProxy("https://{0}:{1}".format(db_ip, bindport))
def main():
""" This main function allows you to run the groomer manually. """
zk_connection_locations = appscale_info.get_zk_locations_string()
zookeeper = zk.ZKTransaction(host=zk_connection_locations)
db_info = appscale_info.get_db_info()
table = db_info[':table']
master = appscale_info.get_db_master_ip()
datastore_path = "{0}:8888".format(master)
ds_groomer = DatastoreGroomer(zookeeper, table, datastore_path)
try:
ds_groomer.run_groomer()
finally:
zookeeper.close()
def main():
""" This main function allows you to run the groomer manually. """
zk_connection_locations = appscale_info.get_zk_locations_string()
zookeeper = zk.ZKTransaction(host=zk_connection_locations)
db_info = appscale_info.get_db_info()
table = db_info[':table']
master = appscale_info.get_db_master_ip()
datastore_path = "{0}:8888".format(master)
ds_groomer = DatastoreGroomer(zookeeper, table, datastore_path)
try:
ds_groomer.run_groomer()
finally:
zookeeper.close()
def __init__(self):
""" DistributedTaskQueue Constructor. """
file_io.set_logging_format()
file_io.mkdir(self.LOG_DIR)
file_io.mkdir(TaskQueueConfig.CELERY_WORKER_DIR)
file_io.mkdir(TaskQueueConfig.CELERY_CONFIG_DIR)
setup_env()
master_db_ip = appscale_info.get_db_master_ip()
connection_str = master_db_ip + ":" + str(constants.DB_SERVER_PORT)
ds_distrib = datastore_distributed.DatastoreDistributed(
constants.DASHBOARD_APP_ID, connection_str, False, False)
apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', ds_distrib)
os.environ['APPLICATION_ID'] = constants.DASHBOARD_APP_ID
def deploy_apps(app_paths):
""" Deploys all apps that reside in /opt/appscale/apps.
Args:
app_paths: A list of the full paths of the apps to be deployed.
Returns:
True on success, False otherwise.
"""
uaserver = SOAPpy.SOAPProxy('https://{0}:{1}'.format(
appscale_info.get_db_master_ip(), UA_SERVER_PORT))
acc = AppControllerClient(appscale_info.get_login_ip(),
appscale_info.get_secret())
# Wait for Cassandra to come up after a restore.
time.sleep(15)
for app_path in app_paths:
# Extract app ID.
app_id = app_path[app_path.rfind('/') + 1:app_path.find('.')]
if not app_id:
logging.error(
"Malformed source code archive. Cannot complete "
"application recovery for '{}'. Aborting...".format(app_path))
return False
# Retrieve app admin via uaserver.
app_data = uaserver.get_app_data(app_id, appscale_info.get_secret())
app_admin_re = re.search("\napp_owner:(.+)\n", app_data)
if app_admin_re:
app_admin = app_admin_re.group(1)
else:
logging.error(
"Missing application data. Cannot complete application "
"recovery for '{}'. Aborting...".format(app_id))
return False
file_suffix = re.search("\.(.*)\Z", app_path).group(1)
logging.warning(
"Restoring app '{}', from '{}', with owner '{}'.".format(
app_id, app_path, app_admin))
acc.upload_app(app_path, file_suffix, app_admin)
return True
def __init__(self):
""" DistributedTaskQueue Constructor. """
file_io.set_logging_format()
file_io.mkdir(self.LOG_DIR)
file_io.mkdir(TaskQueueConfig.CELERY_WORKER_DIR)
file_io.mkdir(TaskQueueConfig.CELERY_CONFIG_DIR)
setup_env()
# Cache all queue information in memory.
self.__queue_info_cache = {}
master_db_ip = appscale_info.get_db_master_ip()
connection_str = master_db_ip + ":" + str(constants.DB_SERVER_PORT)
ds_distrib = datastore_distributed.DatastoreDistributed(
constants.DASHBOARD_APP_ID, connection_str, require_indexes=False)
apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', ds_distrib)
os.environ['APPLICATION_ID'] = constants.DASHBOARD_APP_ID
def deploy_apps(app_paths):
""" Deploys all apps that reside in /opt/appscale/apps.
Args:
app_paths: A list of the full paths of the apps to be deployed.
Returns:
True on success, False otherwise.
"""
uaserver = SOAPpy.SOAPProxy('https://{0}:{1}'.format(
appscale_info.get_db_master_ip(), UA_SERVER_PORT))
acc = AppControllerClient(appscale_info.get_login_ip(),
appscale_info.get_secret())
# Wait for Cassandra to come up after a restore.
time.sleep(15)
for app_path in app_paths:
# Extract app ID.
app_id = app_path[app_path.rfind('/')+1:app_path.find('.')]
if not app_id:
logging.error("Malformed source code archive. Cannot complete "
"application recovery for '{}'. Aborting...".format(app_path))
return False
# Retrieve app admin via uaserver.
app_data = uaserver.get_app_data(app_id, appscale_info.get_secret())
app_admin_re = re.search("\napp_owner:(.+)\n", app_data)
if app_admin_re:
app_admin = app_admin_re.group(1)
else:
logging.error("Missing application data. Cannot complete application "
"recovery for '{}'. Aborting...".format(app_id))
return False
file_suffix = re.search("\.(.*)\Z", app_path).group(1)
logging.warning("Restoring app '{}', from '{}', with owner '{}'.".
format(app_id, app_path, app_admin))
acc.upload_app(app_path, file_suffix, app_admin)
return True
def get_node_info():
""" Creates a list of JSON objects that contain node information and are
needed to perform a backup/restore task on the current AppScale deployment.
"""
# TODO
# Add logic for choosing minimal set of nodes that need to perform a task.
# e.g. Only the node that owns the entire keyspace.
nodes = [{
NodeInfoTags.HOST:
get_br_service_url(appscale_info.get_db_master_ip()),
NodeInfoTags.ROLE:
'db_master',
NodeInfoTags.INDEX:
None
}]
index = 0
for node in appscale_info.get_db_slave_ips():
host = get_br_service_url(node)
# Make sure we don't send the same request on DB roles that reside on the
# same node.
if host not in nodes[0].values():
nodes.append({
NodeInfoTags.HOST: host,
NodeInfoTags.ROLE: 'db_slave',
NodeInfoTags.INDEX: index
})
index += 1
index = 0
for node in appscale_info.get_zk_node_ips():
nodes.append({
NodeInfoTags.HOST: get_br_service_url(node),
NodeInfoTags.ROLE: 'zk',
NodeInfoTags.INDEX: index
})
index += 1
return nodes
def deploy_sensor_app():
""" Uploads the sensor app for registered deployments. """
deployment_id = helper.get_deployment_id()
#If deployment is not registered, then do nothing.
if not deployment_id:
return
uaserver = SOAPpy.SOAPProxy('https://{0}:{1}'.format(
appscale_info.get_db_master_ip(), hermes_constants.UA_SERVER_PORT))
# If the appscalesensor app is already running, then do nothing.
is_app_enabled = uaserver.is_app_enabled(hermes_constants.APPSCALE_SENSOR,
appscale_info.get_secret())
if is_app_enabled == "true":
return
pwd = appscale_utils.encrypt_password(
hermes_constants.USER_EMAIL,
appscale_utils.random_password_generator())
if create_appscale_user(pwd, uaserver) and create_xmpp_user(pwd, uaserver):
logging.debug("Created new user and now tarring app to be deployed.")
file_path = os.path.join(os.path.dirname(__file__), '../Apps/sensor')
app_dir_location = os.path.join(hermes_constants.APP_DIR_LOCATION,
hermes_constants.APPSCALE_SENSOR)
archive = tarfile.open(app_dir_location, "w|gz")
archive.add(file_path, arcname=hermes_constants.APPSCALE_SENSOR)
archive.close()
try:
logging.info(
"Deploying the sensor app for registered deployments.")
acc = appscale_info.get_appcontroller_client()
acc.upload_app(app_dir_location, hermes_constants.FILE_SUFFIX,
hermes_constants.USER_EMAIL)
except AppControllerException:
logging.exception("AppControllerException while trying to deploy "
"appscalesensor app.")
else:
logging.error("Error while creating or accessing the user to deploy "
"appscalesensor app.")
def __init__(self):
""" DistributedTaskQueue Constructor. """
file_io.set_logging_format()
file_io.mkdir(self.LOG_DIR)
file_io.mkdir(TaskQueueConfig.CELERY_WORKER_DIR)
file_io.mkdir(TaskQueueConfig.CELERY_CONFIG_DIR)
setup_env()
# Cache all queue information in memory.
self.__queue_info_cache = {}
master_db_ip = appscale_info.get_db_master_ip()
connection_str = master_db_ip + ":" + str(constants.DB_SERVER_PORT)
ds_distrib = datastore_distributed.DatastoreDistributed(
constants.DASHBOARD_APP_ID, connection_str, require_indexes=False)
apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', ds_distrib)
os.environ['APPLICATION_ID'] = constants.DASHBOARD_APP_ID
# Flag to see if code needs to be reloaded.
self.__force_reload = False
def deploy_sensor_app():
""" Uploads the sensor app for registered deployments. """
deployment_id = helper.get_deployment_id()
#If deployment is not registered, then do nothing.
if not deployment_id:
return
uaserver = SOAPpy.SOAPProxy('https://{0}:{1}'.format(
appscale_info.get_db_master_ip(), hermes_constants.UA_SERVER_PORT))
# If the appscalesensor app is already running, then do nothing.
is_app_enabled = uaserver.is_app_enabled(hermes_constants.APPSCALE_SENSOR,
appscale_info.get_secret())
if is_app_enabled == "true":
return
pwd = appscale_utils.encrypt_password(hermes_constants.USER_EMAIL,
appscale_utils.random_password_generator())
if create_appscale_user(pwd, uaserver) and create_xmpp_user(pwd, uaserver):
logging.debug("Created new user and now tarring app to be deployed.")
file_path = os.path.join(os.path.dirname(__file__), '../Apps/sensor')
app_dir_location = os.path.join(hermes_constants.APP_DIR_LOCATION,
hermes_constants.APPSCALE_SENSOR)
archive = tarfile.open(app_dir_location, "w|gz")
archive.add(file_path, arcname= hermes_constants.APPSCALE_SENSOR)
archive.close()
try:
logging.info("Deploying the sensor app for registered deployments.")
acc = appscale_info.get_appcontroller_client()
acc.upload_app(app_dir_location, hermes_constants.FILE_SUFFIX,
hermes_constants.USER_EMAIL)
except AppControllerException:
logging.exception("AppControllerException while trying to deploy "
"appscalesensor app.")
else:
logging.error("Error while creating or accessing the user to deploy "
"appscalesensor app.")
def get_node_info():
""" Creates a list of JSON objects that contain node information and are
needed to perform a backup/restore task on the current AppScale deployment.
"""
# TODO
# Add logic for choosing minimal set of nodes that need to perform a task.
# e.g. Only the node that owns the entire keyspace.
nodes = [{
NodeInfoTags.HOST: get_br_service_url(appscale_info.get_db_master_ip()),
NodeInfoTags.ROLE: 'db_master',
NodeInfoTags.INDEX: None
}]
index = 0
for node in appscale_info.get_db_slave_ips():
host = get_br_service_url(node)
# Make sure we don't send the same request on DB roles that reside on the
# same node.
if host not in nodes[0].values():
nodes.append({
NodeInfoTags.HOST: host,
NodeInfoTags.ROLE: 'db_slave',
NodeInfoTags.INDEX: index
})
index += 1
index = 0
for node in appscale_info.get_zk_node_ips():
nodes.append({
NodeInfoTags.HOST: get_br_service_url(node),
NodeInfoTags.ROLE: 'zk',
NodeInfoTags.INDEX: index
})
index += 1
return nodes
import constants
from google.appengine.runtime import apiproxy_errors
from google.appengine.api import apiproxy_stub_map
from google.appengine.api import datastore_errors
from google.appengine.api import datastore_distributed
from google.appengine.api import datastore
from google.appengine.ext import db
sys.path.append(TaskQueueConfig.CELERY_CONFIG_DIR)
sys.path.append(TaskQueueConfig.CELERY_WORKER_DIR)
app_id = "APP_ID"
config = TaskQueueConfig(TaskQueueConfig.RABBITMQ, app_id)
module_name = TaskQueueConfig.get_celery_worker_module_name(app_id)
celery = Celery(module_name, broker=config.get_broker_string(), backend="amqp")
celery.config_from_object("CELERY_CONFIGURATION")
logger = get_task_logger(__name__)
master_db_ip = appscale_info.get_db_master_ip()
connection_str = master_db_ip + ":" + str(constants.DB_SERVER_PORT)
ds_distrib = datastore_distributed.DatastoreDistributed("appscaledashboard", connection_str, require_indexes=False)
apiproxy_stub_map.apiproxy.RegisterStub("datastore_v3", ds_distrib)
os.environ["APPLICATION_ID"] = "appscaledashboard"
# This template header and tasks can be found in appscale/AppTaskQueue/templates
from google.appengine.runtime import apiproxy_errors
from google.appengine.api import apiproxy_stub_map
from google.appengine.api import datastore_errors
from google.appengine.api import datastore_distributed
from google.appengine.api import datastore
from google.appengine.ext import db
sys.path.append(TaskQueueConfig.CELERY_CONFIG_DIR)
sys.path.append(TaskQueueConfig.CELERY_WORKER_DIR)
app_id = 'APP_ID'
config = TaskQueueConfig(TaskQueueConfig.RABBITMQ, app_id)
module_name = TaskQueueConfig.get_celery_worker_module_name(app_id)
celery = Celery(module_name, broker=config.get_broker_string(), backend='amqp')
celery.config_from_object('CELERY_CONFIGURATION')
logger = get_task_logger(__name__)
master_db_ip = appscale_info.get_db_master_ip()
connection_str = master_db_ip + ":" + str(constants.DB_SERVER_PORT)
ds_distrib = datastore_distributed.DatastoreDistributed("appscaledashboard",
connection_str,
require_indexes=False)
apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', ds_distrib)
os.environ['APPLICATION_ID'] = "appscaledashboard"
# This template header and tasks can be found in appscale/AppTaskQueue/templates