def main(): """ This main function allows you to run the backup manually. """ parser = init_parser() args = parser.parse_args() # Set up logging. level = logging.INFO if args.debug: level = logging.DEBUG logging.basicConfig(format='%(asctime)s %(levelname)s %(filename)s:' \ '%(lineno)s %(message)s ', level=level) logging.info("Logging started") message = "Backing up " if args.source_code: message += "source and " message += "data for: {0}".format(args.app_id) logging.info(message) zk_connection_locations = appscale_info.get_zk_locations_string() zookeeper = zk.ZKTransaction(host=zk_connection_locations) db_info = appscale_info.get_db_info() table = db_info[':table'] skip_list = args.skip if not skip_list: skip_list = [] logging.info("Will skip the following kinds: {0}".format(sorted(skip_list))) ds_backup = DatastoreBackup(args.app_id, zookeeper, table, source_code=args.source_code, skip_list=sorted(skip_list)) try: ds_backup.run() finally: zookeeper.close()
def main(): global datastore_path global deployment_config logging.basicConfig(format=LOG_FORMAT, level=logging.INFO) parser = argparse.ArgumentParser() parser.add_argument('-p', '--port', type=int, default=DEFAULT_PORT, required=True, help="The blobstore server's port") parser.add_argument('-d', '--datastore-path', required=True, help='The location of the datastore server') args = parser.parse_args() datastore_path = args.datastore_path deployment_config = DeploymentConfig(appscale_info.get_zk_locations_string()) setup_env() http_server = tornado.httpserver.HTTPServer( Application(), max_buffer_size=MAX_REQUEST_BUFF_SIZE) http_server.listen(args.port) acc = appscale_info.get_appcontroller_client() acc.add_routing_for_blob_server() logging.info('Starting BlobServer on {}'.format(args.port)) tornado.ioloop.IOLoop.instance().start()
def main(): """ This main function allows you to run the groomer manually. """ zk_connection_locations = appscale_info.get_zk_locations_string() zookeeper = zk.ZKTransaction(host=zk_connection_locations) db_info = appscale_info.get_db_info() table = db_info[':table'] master = appscale_info.get_db_master_ip() datastore_path = "{0}:8888".format(master) ds_groomer = DatastoreGroomer(zookeeper, table, datastore_path) logging.debug("Trying to get groomer lock.") if ds_groomer.get_groomer_lock(): logging.info("Got the groomer lock.") try: ds_groomer.run_groomer() except Exception as exception: logging.exception('Encountered exception {} while running the groomer.' .format(str(exception))) try: ds_groomer.zoo_keeper.release_lock_with_path(zk.DS_GROOM_LOCK_PATH) except zk.ZKTransactionException, zk_exception: logging.error("Unable to release zk lock {0}.".\ format(str(zk_exception))) except zk.ZKInternalException, zk_exception: logging.error("Unable to release zk lock {0}.".\ format(str(zk_exception)))
def main(): """ This main function allows you to run the groomer manually. """ zk_connection_locations = appscale_info.get_zk_locations_string() zookeeper = zk.ZKTransaction(host=zk_connection_locations, start_gc=False) db_info = appscale_info.get_db_info() table = db_info[':table'] master = appscale_info.get_db_master_ip() datastore_path = "{0}:8888".format(master) ds_groomer = DatastoreGroomer(zookeeper, table, datastore_path) logging.debug("Trying to get groomer lock.") if ds_groomer.get_groomer_lock(): logging.info("Got the groomer lock.") try: ds_groomer.run_groomer() except Exception as exception: logging.exception('Encountered exception {} while running the groomer.' .format(str(exception))) try: ds_groomer.zoo_keeper.release_lock_with_path(zk.DS_GROOM_LOCK_PATH) except zk.ZKTransactionException, zk_exception: logging.error("Unable to release zk lock {0}.".\ format(str(zk_exception))) except zk.ZKInternalException, zk_exception: logging.error("Unable to release zk lock {0}.".\ format(str(zk_exception)))
def main(): """ This main function allows you to run the restore manually. """ # Parse CLI arguments. parser = init_parser() args = parser.parse_args() # Set up logging. level = logging.INFO if args.debug: level = logging.DEBUG logging.basicConfig(format='%(asctime)s %(levelname)s %(filename)s:' \ '%(lineno)s %(message)s ', level=level) logging.info("Logging started") logging.info(args) # Verify app is deployed. if not app_is_deployed(args.app_id): return # Verify backup dir exists. if not backup_dir_exists(args.backup_dir): return if args.clear_datastore: message = "Deleting \"{0}\" data...".\ format(args.app_id, args.backup_dir) logging.info(message) try: tables_to_clear = { APP_ENTITY_TABLE: APP_ENTITY_SCHEMA, ASC_PROPERTY_TABLE: PROPERTY_SCHEMA, DSC_PROPERTY_TABLE: PROPERTY_SCHEMA, COMPOSITE_TABLE: COMPOSITE_SCHEMA, APP_KIND_TABLE: APP_KIND_SCHEMA, TRANSACTIONS_TABLE: TRANSACTIONS_SCHEMA } for table, schema in tables_to_clear.items(): fetch_and_delete_entities('cassandra', table, schema, args.app_id, False) except Exception as exception: logging.error("Unhandled exception while deleting \"{0}\" data: {1} " \ "Exiting...".format(args.app_id, exception.message)) return # Initialize connection to Zookeeper and database related variables. zk_connection_locations = appscale_info.get_zk_locations_string() zookeeper = zk.ZKTransaction(host=zk_connection_locations) db_info = appscale_info.get_db_info() table = db_info[':table'] # Start restore process. ds_restore = DatastoreRestore(args.app_id.strip('/'), args.backup_dir, zookeeper, table) try: ds_restore.run() finally: zookeeper.close()
def main(): """ Updates a composite index after prompting the user. """ try: opts, remainder = getopt.getopt(sys.argv[1:], 't:a:', ['type=', 'app_id=']) except getopt.GetoptError: usage() sys.exit(1) db_type = None app_id = None for opt, arg in opts: if opt in ('-t', '--type'): db_type = arg elif opt in ('-a', '--app_id'): app_id = arg if not db_type or not app_id: usage() sys.exit(1) datastore_batch = appscale_datastore_batch.DatastoreFactory.\ getDatastore(db_type) zookeeper_locations = appscale_info.get_zk_locations_string() zookeeper = zk.ZKTransaction(host=zookeeper_locations) datastore_access = DatastoreDistributed(datastore_batch, zookeeper=zookeeper) pb_indices = datastore_access.datastore_batch.get_indices(app_id) indices = [datastore_pb.CompositeIndex(index) for index in pb_indices] if len(indices) == 0: print('No composite indices found for app {}'.format(app_id)) zookeeper.close() sys.exit(1) selection = -1 selection_range = range(1, len(indices) + 1) while selection not in selection_range: for number, index in enumerate(indices, start=1): pretty_index = prettify_index(index.definition()) print('{}) {}'.format(number, pretty_index)) try: selection = int( raw_input( 'Select the index you want to update. (1-{}) '.format( len(indices)))) except KeyboardInterrupt: zookeeper.close() sys.exit() selected_index = indices[selection - 1] datastore_access.update_composite_index(app_id, selected_index) zookeeper.close()
def main(): """ Updates a composite index after prompting the user. """ try: opts, remainder = getopt.getopt( sys.argv[1:], 't:a:', ['type=', 'app_id=']) except getopt.GetoptError: usage() sys.exit(1) db_type = None app_id = None for opt, arg in opts: if opt in ('-t', '--type'): db_type = arg elif opt in ('-a', '--app_id'): app_id = arg if not db_type or not app_id: usage() sys.exit(1) datastore_batch = appscale_datastore_batch.DatastoreFactory.\ getDatastore(db_type) zookeeper_locations = appscale_info.get_zk_locations_string() zookeeper = zk.ZKTransaction(host=zookeeper_locations) datastore_access = DatastoreDistributed(datastore_batch, zookeeper=zookeeper) pb_indices = datastore_access.datastore_batch.get_indices(app_id) indices = [datastore_pb.CompositeIndex(index) for index in pb_indices] if len(indices) == 0: print('No composite indices found for app {}'.format(app_id)) zookeeper.close() sys.exit(1) selection = -1 selection_range = range(1, len(indices) + 1) while selection not in selection_range: for number, index in enumerate(indices, start=1): pretty_index = prettify_index(index.definition()) print('{}) {}'.format(number, pretty_index)) try: selection = int(raw_input('Select the index you want to update. (1-{}) ' .format(len(indices)))) except KeyboardInterrupt: zookeeper.close() sys.exit() selected_index = indices[selection - 1] datastore_access.update_composite_index(app_id, selected_index) zookeeper.close()
def main(): """ This main function allows you to run the groomer manually. """ zk_connection_locations = appscale_info.get_zk_locations_string() zookeeper = zk.ZKTransaction(host=zk_connection_locations) datastore_path = "localhost:8888" db_info = appscale_info.get_db_info() table = db_info[':table'] ds_groomer = DatastoreGroomer(zookeeper, table, datastore_path) try: ds_groomer.run_groomer() finally: zookeeper.close()
def main(): logging.basicConfig(level=logging.INFO, format=LOG_FORMAT) logger = logging.getLogger(__name__) zookeeper_locations = appscale_info.get_zk_locations_string() gc_zookeeper = zk.ZKTransaction(host=zookeeper_locations, start_gc=False) logger.info("Using ZK locations {0}".format(zookeeper_locations)) ds_groomer = groomer.DatastoreGroomer(gc_zookeeper, "cassandra", LOCAL_DATASTORE) try: ds_groomer.start() except Exception, exception: logger.warning("An exception slipped through:") logger.exception(exception) logger.warning("Exiting service.")
def main(): """ This main function allows you to run the groomer manually. """ zk_connection_locations = appscale_info.get_zk_locations_string() zookeeper = zk.ZKTransaction(host=zk_connection_locations) db_info = appscale_info.get_db_info() table = db_info[':table'] master = appscale_info.get_db_master_ip() datastore_path = "{0}:8888".format(master) ds_groomer = groomer.DatastoreGroomer(zookeeper, table, datastore_path) try: ds_groomer.remove_old_logs(None) finally: zookeeper.close()
def main(): """ Starts a web service for handing datastore requests. """ global datastore_access zookeeper_locations = appscale_info.get_zk_locations_string() parser = argparse.ArgumentParser() parser.add_argument('-t', '--type', choices=dbconstants.VALID_DATASTORES, default=dbconstants.VALID_DATASTORES[0], help='Database type') parser.add_argument('-p', '--port', type=int, default=dbconstants.DEFAULT_PORT, help='Datastore server port') parser.add_argument('-v', '--verbose', action='store_true', help='Output debug-level logging') args = parser.parse_args() if args.verbose: logger.setLevel(logging.DEBUG) datastore_batch = DatastoreFactory.getDatastore( args.type, log_level=logger.getEffectiveLevel()) zookeeper = zktransaction.ZKTransaction( host=zookeeper_locations, start_gc=True, db_access=datastore_batch, log_level=logger.getEffectiveLevel()) datastore_access = DatastoreDistributed( datastore_batch, zookeeper=zookeeper, log_level=logger.getEffectiveLevel()) server = tornado.httpserver.HTTPServer(pb_application) server.listen(args.port) tornado.ioloop.IOLoop.current().start()
def main(app_id, db_type): """ Updates a composite index after prompting the user. Args: app_id: A string containing the application ID. db_type: A string specifying which database backend to use. """ datastore_batch = appscale_datastore_batch.DatastoreFactory.\ getDatastore(db_type) zookeeper_locations = appscale_info.get_zk_locations_string() zookeeper = zk.ZKTransaction(host=zookeeper_locations) datastore_access = DatastoreDistributed(datastore_batch, zookeeper=zookeeper) pb_indices = datastore_access.datastore_batch.get_indices(app_id) indices = [datastore_pb.CompositeIndex(index) for index in pb_indices] if len(indices) == 0: print('No composite indices found for app {}'.format(app_id)) zookeeper.close() sys.exit(1) selection = -1 selection_range = range(1, len(indices) + 1) while selection not in selection_range: for number, index in enumerate(indices, start=1): pretty_index = prettify_index(index.definition()) print('{}) {}'.format(number, pretty_index)) try: selection = int( raw_input( 'Select the index you want to update. (1-{}) '.format( len(indices)))) except KeyboardInterrupt: zookeeper.close() sys.exit() selected_index = indices[selection - 1] datastore_access.update_composite_index(app_id, selected_index) zookeeper.close()
def main(app_id, db_type): """ Updates a composite index after prompting the user. Args: app_id: A string containing the application ID. db_type: A string specifying which database backend to use. """ datastore_batch = appscale_datastore_batch.DatastoreFactory.\ getDatastore(db_type) zookeeper_locations = appscale_info.get_zk_locations_string() zookeeper = zk.ZKTransaction(host=zookeeper_locations) datastore_access = DatastoreDistributed(datastore_batch, zookeeper=zookeeper) pb_indices = datastore_access.datastore_batch.get_indices(app_id) indices = [datastore_pb.CompositeIndex(index) for index in pb_indices] if len(indices) == 0: print('No composite indices found for app {}'.format(app_id)) zookeeper.close() sys.exit(1) selection = -1 selection_range = range(1, len(indices) + 1) while selection not in selection_range: for number, index in enumerate(indices, start=1): pretty_index = prettify_index(index.definition()) print('{}) {}'.format(number, pretty_index)) try: selection = int(raw_input('Select the index you want to update. (1-{}) ' .format(len(indices)))) except KeyboardInterrupt: zookeeper.close() sys.exit() selected_index = indices[selection - 1] datastore_access.update_composite_index(app_id, selected_index) zookeeper.close()
if not backup_dir_exists(args.backup_dir): return if args.clear_datastore: message = "Deleting \"{0}\" data...".\ format(args.app_id, args.backup_dir) logging.info(message) try: delete_all_records.main('cassandra', args.app_id, True) except Exception, exception: logging.error("Unhandled exception while deleting \"{0}\" data: {1} " \ "Exiting...".format(args.app_id, exception.message)) return # Initialize connection to Zookeeper and database related variables. zk_connection_locations = appscale_info.get_zk_locations_string() zookeeper = zk.ZKTransaction(host=zk_connection_locations) db_info = appscale_info.get_db_info() table = db_info[':table'] # Start restore process. ds_restore = DatastoreRestore(args.app_id.strip('/'), args.backup_dir, zookeeper, table) try: ds_restore.run() finally: zookeeper.close() if __name__ == "__main__": main()
for ii in REQUIRED_CONFIG_FIELDS: try: if config[ii]: pass except KeyError: logging.error("Unable to find " + str(ii) + " in configuration") return False return True ################################ # MAIN ################################ if __name__ == "__main__": file_io.set_logging_format() deployment_config = DeploymentConfig(appscale_info.get_zk_locations_string()) INTERNAL_IP = appscale_info.get_private_ip() SERVER = SOAPpy.SOAPServer((INTERNAL_IP, constants.APP_MANAGER_PORT)) SERVER.registerFunction(start_app) SERVER.registerFunction(stop_app) SERVER.registerFunction(stop_app_instance) SERVER.registerFunction(restart_app_instances_for_app) while 1: try: SERVER.serve_forever() except SSL.SSLError: pass
return else: self.finish(UPLOAD_ERROR + "</br>" + str(e.hdrs) + "</br>" + str(e)) return if __name__ == "__main__": logging.basicConfig(format=LOG_FORMAT, level=logging.INFO) parser = argparse.ArgumentParser() parser.add_argument('-p', '--port', type=int, default=DEFAULT_PORT, required=True, help="The blobstore server's port") parser.add_argument('-d', '--datastore-path', required=True, help='The location of the datastore server') args = parser.parse_args() datastore_path = args.datastore_path deployment_config = DeploymentConfig(appscale_info.get_zk_locations_string()) setup_env() http_server = tornado.httpserver.HTTPServer( Application(), max_buffer_size=MAX_REQUEST_BUFF_SIZE) http_server.listen(args.port) acc = appscale_info.get_appcontroller_client() acc.add_routing_for_blob_server() logging.info('Starting BlobServer on {}'.format(args.port)) tornado.ioloop.IOLoop.instance().start()
""" Provides a service which periodically runs the groomer. """ import logging import os import sys import groomer from zkappscale import zktransaction as zk sys.path.append(os.path.join(os.path.dirname(__file__), "../lib/")) import appscale_info # Location to find the datastore service. LOCAL_DATASTORE = "localhost:8888" if __name__ == "__main__": logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) zookeeper_locations = appscale_info.get_zk_locations_string() gc_zookeeper = zk.ZKTransaction(host=zookeeper_locations) logger.info("Using ZK locations {0}".format(zookeeper_locations)) ds_groomer = groomer.DatastoreGroomer(gc_zookeeper, "cassandra", LOCAL_DATASTORE) logger.info("Starting service.") try: ds_groomer.start() except Exception, exception: logger.warning("An exception slipped through:") logger.exception(exception) logger.warning("Exiting service.")
for ii in REQUIRED_CONFIG_FIELDS: try: if config[ii]: pass except KeyError: logging.error("Unable to find " + str(ii) + " in configuration") return False return True ################################ # MAIN ################################ if __name__ == "__main__": file_io.set_logging_format() deployment_config = DeploymentConfig( appscale_info.get_zk_locations_string()) INTERNAL_IP = appscale_info.get_private_ip() SERVER = SOAPpy.SOAPServer((INTERNAL_IP, constants.APP_MANAGER_PORT)) SERVER.registerFunction(start_app) SERVER.registerFunction(stop_app) SERVER.registerFunction(stop_app_instance) while 1: try: SERVER.serve_forever() except SSL.SSLError: pass
def main(): """ Starts a web service for handing datastore requests. """ logging.basicConfig(format=LOG_FORMAT, level=logging.INFO) global logger logger = logging.getLogger(__name__) global datastore_access zookeeper_locations = appscale_info.get_zk_locations_string() db_info = appscale_info.get_db_info() db_type = db_info[':table'] port = dbconstants.DEFAULT_SSL_PORT is_encrypted = True verbose = False argv = sys.argv[1:] try: opts, args = getopt.getopt( argv, "t:p:n:v:", ["type=", "port", "no_encryption", "verbose"]) except getopt.GetoptError: usage() sys.exit(1) for opt, arg in opts: if opt in ("-t", "--type"): db_type = arg print "Datastore type: ", db_type elif opt in ("-p", "--port"): port = int(arg) elif opt in ("-n", "--no_encryption"): is_encrypted = False elif opt in ("-v", "--verbose"): verbose = True if verbose: logger.setLevel(logging.DEBUG) if db_type not in dbconstants.VALID_DATASTORES: print "This datastore is not supported for this version of the AppScale\ datastore API:" + db_type sys.exit(1) datastore_batch = DatastoreFactory.getDatastore( db_type, log_level=logger.getEffectiveLevel()) zookeeper = zktransaction.ZKTransaction( host=zookeeper_locations, start_gc=True, db_access=datastore_batch, log_level=logger.getEffectiveLevel()) datastore_access = DatastoreDistributed( datastore_batch, zookeeper=zookeeper, log_level=logger.getEffectiveLevel()) if port == dbconstants.DEFAULT_SSL_PORT and not is_encrypted: port = dbconstants.DEFAULT_PORT server = tornado.httpserver.HTTPServer(pb_application) server.listen(port) while 1: try: # Start Server # tornado.ioloop.IOLoop.instance().start() except SSL.SSLError: # This happens when connections timeout, there is a just a bad # SSL connection such as it does not use SSL when expected. pass except KeyboardInterrupt: print "Server interrupted by user, terminating..." zookeeper.close() sys.exit(1)
def main(): """ Starts a web service for handing datastore requests. """ logging.basicConfig(format=LOG_FORMAT, level=logging.INFO) global logger logger = logging.getLogger(__name__) global datastore_access zookeeper_locations = appscale_info.get_zk_locations_string() db_info = appscale_info.get_db_info() db_type = db_info[":table"] port = dbconstants.DEFAULT_SSL_PORT is_encrypted = True verbose = False argv = sys.argv[1:] try: opts, args = getopt.getopt(argv, "t:p:n:v:", ["type=", "port", "no_encryption", "verbose"]) except getopt.GetoptError: usage() sys.exit(1) for opt, arg in opts: if opt in ("-t", "--type"): db_type = arg print "Datastore type: ", db_type elif opt in ("-p", "--port"): port = int(arg) elif opt in ("-n", "--no_encryption"): is_encrypted = False elif opt in ("-v", "--verbose"): verbose = True if verbose: logger.setLevel(logging.DEBUG) if db_type not in dbconstants.VALID_DATASTORES: print "This datastore is not supported for this version of the AppScale\ datastore API:" + db_type sys.exit(1) datastore_batch = DatastoreFactory.getDatastore(db_type, log_level=logger.getEffectiveLevel()) zookeeper = zktransaction.ZKTransaction( host=zookeeper_locations, start_gc=True, db_access=datastore_batch, log_level=logger.getEffectiveLevel() ) datastore_access = DatastoreDistributed(datastore_batch, zookeeper=zookeeper, log_level=logger.getEffectiveLevel()) if port == dbconstants.DEFAULT_SSL_PORT and not is_encrypted: port = dbconstants.DEFAULT_PORT server = tornado.httpserver.HTTPServer(pb_application) server.listen(port) while 1: try: # Start Server # tornado.ioloop.IOLoop.instance().start() except SSL.SSLError: # This happens when connections timeout, there is a just a bad # SSL connection such as it does not use SSL when expected. pass except KeyboardInterrupt: print "Server interrupted by user, terminating..." zookeeper.close() sys.exit(1)