def create_sandbox(db=None): """'Copy' files from the current project directory to a temp directory. This function creates a sandbox directory, and symlinks the entire 'current' directory tree (the one in which this file is found) to the sandbox directory, excepting directories that hold the database. This makes sure that work on the database won't change your current environment. Arguments: db: if not None, this file is copied into the sandbox datastore/ directory, and is used as the starting db for the test datastore. It should be specified relative to the app-root of the current directory tree (the directory where app.yaml lives). Note that since this file is copied, there are no changes made to the db file you specify. If None, an empty db file is used. Returns: The root directory of the copy. This directory will have app.yaml in it, but will live in /tmp or some such. """ # Find the 'root' directory of the project the tests are being # run in. ka_root = rootdir.project_rootdir() # Create a 'sandbox' directory that symlinks to ka_root, # except for the 'datastore' directory (we don't want to mess # with your actual datastore for these tests!) tmpdir = tempfile.mkdtemp() for f in os.listdir(ka_root): if 'datastore' not in f: os.symlink(os.path.join(ka_root, f), os.path.join(tmpdir, f)) os.mkdir(os.path.join(tmpdir, 'datastore')) if db: shutil.copy(os.path.join(ka_root, db), os.path.join(tmpdir, 'datastore', 'test.sqlite')) return tmpdir
def setUp(self, db_consistency_probability=0, use_test_db=False, test_db_filename='testutil/test_db.sqlite', queue_yaml_dir='.', app_id='dev~khan-academy'): """Initialize a testbed for appengine, and also initialize the cache. This sets up the backend state (databases, queues, etc) to a known, pure state before each test. Arguments: db_consistency_probability: a number between 0 and 1 indicating the percent of times db writes are immediately visible. If set to 1, then the database seems consistent. If set to 0, then writes are never visible until a commit-causing command is run: get()/put()/delete()/ancestor queries. 0 is the default, and does the best testing that the code does not make assumptions about immediate consistency. See https://developers.google.com/appengine/docs/python/datastore/overview#Datastore_Writes_and_Data_Visibility for details on GAE's consistency policies with the High Replication Datastore (HRD). use_test_db: if True, then initialize the datastore with the contents of testutil/test_db.sqlite, rather than being empty. This routine makes a copy of the db file (in /tmp) so changes from one test won't affect another. test_db_filename: the file to use with use_test_db, relative to the project root (that is, the directory with app.yaml in it). It is ignored if use_test_db is False. It is unusual to want to change this value from the default, but it can be done if you have another test-db in a non-standard location. queue_yaml_dir: the directory where queue.yaml lives, relative to the project root. If set, we will initialize the taskqueue stub. This is needed if you wish to run mapreduces in your test. This will almost always be '.'. app_id: what we should pretend our app-id is. The default matches the app-id used to make test_db.sqlite, so database lookups on that file will succeed. """ self.testbed = testbed.Testbed() # This lets us use testutil's test_db.sqlite if we want to. self.testbed.setup_env(app_id=app_id) self.testbed.activate() # Create a consistency policy that will simulate the High # Replication consistency model. self.policy = datastore_stub_util.PseudoRandomHRConsistencyPolicy( probability=db_consistency_probability) if use_test_db: root = rootdir.project_rootdir() (fd, self.datastore_filename) = tempfile.mkstemp(prefix='test_db_', suffix='.sqlite') _copy_to_fd(open(os.path.join(root, test_db_filename)), fd) os.close(fd) else: self.datastore_filename = None self.testbed.init_datastore_v3_stub( consistency_policy=self.policy, datastore_file=self.datastore_filename, use_sqlite=(self.datastore_filename is not None)) self.testbed.init_user_stub() self.testbed.init_memcache_stub() if queue_yaml_dir: root = rootdir.project_rootdir() self.testbed.init_taskqueue_stub( root_path=os.path.join(root, queue_yaml_dir), auto_task_running=True) instance_cache.flush()
def start_dev_appserver(db=None, persist_db_changes=False): """Start up a dev-appserver instance on an unused port, return its url. This function creates a sandbox directory, and symlinks the entire 'current' directory tree (the one in which this file is found) to the sandbox directory, excepting directories that hold the database. This makes sure that work on the dev_appserver won't change your current environment. It starts looking on port 9000 for a free port, and will check 10000 ports, so it should be able to start up no matter what. This function sets the module-global variables appserver_url, tmpdir, and pid. Applications are free to examine these. They are None if a dev_appserver instance isn't currently running. Arguments: db: if not None, this file will be used as the starting db for the datastore. It should be specified relative to the app-root of the current directory tree (the directory where app.yaml lives). If None, an empty db file is used. persist_db_changes: if db is not None and persist_db_changes is True, the database is symlinked into the sandbox instead of copied. This will result in changes to the database mutating the specified db file. Ignored if db is None. """ global appserver_url, tmpdir, pid ka_root = rootdir.project_rootdir() tmpdir = create_sandbox(db) sandbox_db_path = os.path.join(tmpdir, 'datastore', 'test.sqlite') if persist_db_changes: # Symlink the db rather than copying it, so that changes get made to # the "master" copy. os.unlink(sandbox_db_path) os.symlink(os.path.join(ka_root, db), sandbox_db_path) # Find an unused port to run the appserver on. There's a small # race condition here, but we can hope for the best. Too bad # dev_appserver doesn't allow input to be port=0! for port in xrange(9000, 19000): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) try: sock.connect(('', port)) del sock # reclaim the socket except socket.error: # means nothing is running on that socket! dev_appserver_port = port break else: # for/else: if we got here, we never found a good port raise IOError('Could not find an unused port in range 9000-19000') # Start dev_appserver args = ['dev_appserver.py', '-p%s' % dev_appserver_port, '--use_sqlite', '--high_replication', '--address=0.0.0.0', '--skip_sdk_update_check', ('--datastore_path=%s' % sandbox_db_path), ('--blobstore_path=%s' % os.path.join(tmpdir, 'datastore/blobs')), tmpdir] # Its output is noisy, but useful, so store it in tmpdir. Third # arg to open() uses line-buffering so the output is available. dev_appserver_file = dev_appserver_logfile_name() dev_appserver_output = open(dev_appserver_file, 'w', 1) print 'NOTE: Starting dev_appserver.py; output in %s' % dev_appserver_file # Run the tests with USE_SCREEN to spawn the API server in a screen # to make it interactive (useful for pdb) # # e.g. # USE_SCREEN=1 python tools/runtests.py --max-size=large api/labs # # This works especially well if you're already in a screen session, since # it will just open a new screen window in your pre-existing sesssion if os.environ.get('USE_SCREEN'): args = ['/usr/bin/screen'] + args pid = subprocess.Popen(args, stdout=dev_appserver_output, stderr=subprocess.STDOUT).pid # Wait for the server to start up time.sleep(1) # it *definitely* takes at least a second connect_seconds = 60 # wait for 60 seconds, until we give up for _ in xrange(connect_seconds * 5): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) try: sock.connect(('', dev_appserver_port)) break except socket.error: del sock # reclaim the socket time.sleep(0.2) else: # for/else: we get here if we never hit the 'break' above raise IOError('Unable to connect to localhost:%s even after %s seconds' % (dev_appserver_port, connect_seconds)) # Set the useful variables for subclasses to use global appserver_url appserver_url = 'http://localhost:%d' % dev_appserver_port return appserver_url