def create_sandbox(db=None):
"""'Copy' files from the current project directory to a temp directory.
This function creates a sandbox directory, and symlinks the entire
'current' directory tree (the one in which this file is found) to
the sandbox directory, excepting directories that hold the
database. This makes sure that work on the database won't change
your current environment.
Arguments:
db: if not None, this file is copied into the sandbox
datastore/ directory, and is used as the starting db for the
test datastore. It should be specified relative to the
app-root of the current directory tree (the directory where
app.yaml lives). Note that since this file is copied, there
are no changes made to the db file you specify. If None, an
empty db file is used.
Returns:
The root directory of the copy. This directory will have
app.yaml in it, but will live in /tmp or some such.
"""
# Find the 'root' directory of the project the tests are being
# run in.
ka_root = rootdir.project_rootdir()
# Create a 'sandbox' directory that symlinks to ka_root,
# except for the 'datastore' directory (we don't want to mess
# with your actual datastore for these tests!)
tmpdir = tempfile.mkdtemp()
for f in os.listdir(ka_root):
if 'datastore' not in f:
os.symlink(os.path.join(ka_root, f),
os.path.join(tmpdir, f))
os.mkdir(os.path.join(tmpdir, 'datastore'))
if db:
shutil.copy(os.path.join(ka_root, db),
os.path.join(tmpdir, 'datastore', 'test.sqlite'))
return tmpdir
def setUp(self,
db_consistency_probability=0,
use_test_db=False,
test_db_filename='testutil/test_db.sqlite',
queue_yaml_dir='.',
app_id='dev~khan-academy'):
"""Initialize a testbed for appengine, and also initialize the cache.
This sets up the backend state (databases, queues, etc) to a known,
pure state before each test.
Arguments:
db_consistency_probability: a number between 0 and 1
indicating the percent of times db writes are
immediately visible. If set to 1, then the database
seems consistent. If set to 0, then writes are never
visible until a commit-causing command is run:
get()/put()/delete()/ancestor queries. 0 is the
default, and does the best testing that the code does
not make assumptions about immediate consistency. See
https://developers.google.com/appengine/docs/python/datastore/overview#Datastore_Writes_and_Data_Visibility
for details on GAE's consistency policies with the High
Replication Datastore (HRD).
use_test_db: if True, then initialize the datastore with the
contents of testutil/test_db.sqlite, rather than being
empty. This routine makes a copy of the db file (in /tmp)
so changes from one test won't affect another.
test_db_filename: the file to use with use_test_db, relative
to the project root (that is, the directory with
app.yaml in it). It is ignored if use_test_db is False.
It is unusual to want to change this value from the
default, but it can be done if you have another test-db
in a non-standard location.
queue_yaml_dir: the directory where queue.yaml lives, relative
to the project root. If set, we will initialize the
taskqueue stub. This is needed if you wish to run
mapreduces in your test. This will almost always be '.'.
app_id: what we should pretend our app-id is. The default
matches the app-id used to make test_db.sqlite, so
database lookups on that file will succeed.
"""
self.testbed = testbed.Testbed()
# This lets us use testutil's test_db.sqlite if we want to.
self.testbed.setup_env(app_id=app_id)
self.testbed.activate()
# Create a consistency policy that will simulate the High
# Replication consistency model.
self.policy = datastore_stub_util.PseudoRandomHRConsistencyPolicy(
probability=db_consistency_probability)
if use_test_db:
root = rootdir.project_rootdir()
(fd, self.datastore_filename) = tempfile.mkstemp(prefix='test_db_',
suffix='.sqlite')
_copy_to_fd(open(os.path.join(root, test_db_filename)), fd)
os.close(fd)
else:
self.datastore_filename = None
self.testbed.init_datastore_v3_stub(
consistency_policy=self.policy,
datastore_file=self.datastore_filename,
use_sqlite=(self.datastore_filename is not None))
self.testbed.init_user_stub()
self.testbed.init_memcache_stub()
if queue_yaml_dir:
root = rootdir.project_rootdir()
self.testbed.init_taskqueue_stub(
root_path=os.path.join(root, queue_yaml_dir),
auto_task_running=True)
instance_cache.flush()
def start_dev_appserver(db=None, persist_db_changes=False):
"""Start up a dev-appserver instance on an unused port, return its url.
This function creates a sandbox directory, and symlinks the entire
'current' directory tree (the one in which this file is found) to
the sandbox directory, excepting directories that hold the
database. This makes sure that work on the dev_appserver won't
change your current environment.
It starts looking on port 9000 for a free port, and will check
10000 ports, so it should be able to start up no matter what.
This function sets the module-global variables appserver_url,
tmpdir, and pid. Applications are free to examine these. They
are None if a dev_appserver instance isn't currently running.
Arguments:
db: if not None, this file will be used as the starting db for the
datastore. It should be specified relative to the app-root of the
current directory tree (the directory where app.yaml lives). If None,
an empty db file is used.
persist_db_changes: if db is not None and persist_db_changes is True,
the database is symlinked into the sandbox instead of copied. This
will result in changes to the database mutating the specified db file.
Ignored if db is None.
"""
global appserver_url, tmpdir, pid
ka_root = rootdir.project_rootdir()
tmpdir = create_sandbox(db)
sandbox_db_path = os.path.join(tmpdir, 'datastore', 'test.sqlite')
if persist_db_changes:
# Symlink the db rather than copying it, so that changes get made to
# the "master" copy.
os.unlink(sandbox_db_path)
os.symlink(os.path.join(ka_root, db), sandbox_db_path)
# Find an unused port to run the appserver on. There's a small
# race condition here, but we can hope for the best. Too bad
# dev_appserver doesn't allow input to be port=0!
for port in xrange(9000, 19000):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
try:
sock.connect(('', port))
del sock # reclaim the socket
except socket.error: # means nothing is running on that socket!
dev_appserver_port = port
break
else: # for/else: if we got here, we never found a good port
raise IOError('Could not find an unused port in range 9000-19000')
# Start dev_appserver
args = ['dev_appserver.py',
'-p%s' % dev_appserver_port,
'--use_sqlite',
'--high_replication',
'--address=0.0.0.0',
'--skip_sdk_update_check',
('--datastore_path=%s' % sandbox_db_path),
('--blobstore_path=%s'
% os.path.join(tmpdir, 'datastore/blobs')),
tmpdir]
# Its output is noisy, but useful, so store it in tmpdir. Third
# arg to open() uses line-buffering so the output is available.
dev_appserver_file = dev_appserver_logfile_name()
dev_appserver_output = open(dev_appserver_file, 'w', 1)
print 'NOTE: Starting dev_appserver.py; output in %s' % dev_appserver_file
# Run the tests with USE_SCREEN to spawn the API server in a screen
# to make it interactive (useful for pdb)
#
# e.g.
# USE_SCREEN=1 python tools/runtests.py --max-size=large api/labs
#
# This works especially well if you're already in a screen session, since
# it will just open a new screen window in your pre-existing sesssion
if os.environ.get('USE_SCREEN'):
args = ['/usr/bin/screen'] + args
pid = subprocess.Popen(args,
stdout=dev_appserver_output,
stderr=subprocess.STDOUT).pid
# Wait for the server to start up
time.sleep(1) # it *definitely* takes at least a second
connect_seconds = 60 # wait for 60 seconds, until we give up
for _ in xrange(connect_seconds * 5):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
try:
sock.connect(('', dev_appserver_port))
break
except socket.error:
del sock # reclaim the socket
time.sleep(0.2)
else: # for/else: we get here if we never hit the 'break' above
raise IOError('Unable to connect to localhost:%s even after %s seconds'
% (dev_appserver_port, connect_seconds))
# Set the useful variables for subclasses to use
global appserver_url
appserver_url = 'http://localhost:%d' % dev_appserver_port
return appserver_url
