def SetupDataStore():
global PORT
if PORT:
return
PORT = [portpicker.PickUnusedPort(), portpicker.PickUnusedPort()]
_SetConfig()
_StartServers()
try:
data_store.DB = http_data_store.HTTPDataStore()
except http_data_store.HTTPDataStoreError:
data_store.DB = None
_CloseServer()
def setUpClass(cls):
super(GRRHTTPServerTest, cls).setUpClass()
cls.config_overrider = test_lib.ConfigOverrider({
"Rekall.profile_server":
rekall_test_lib.TestRekallRepositoryProfileServer.__name__,
})
cls.config_overrider.Start()
# Bring up a local server for testing.
port = portpicker.PickUnusedPort()
ip = utils.ResolveHostnameToIP("localhost", port)
cls.httpd = frontend.GRRHTTPServer((ip, port),
frontend.GRRHTTPServerHandler)
if ipaddr.IPAddress(ip).version == 6:
cls.address_family = socket.AF_INET6
cls.base_url = "http://[%s]:%d/" % (ip, port)
else:
cls.address_family = socket.AF_INET
cls.base_url = "http://%s:%d/" % (ip, port)
cls.httpd_thread = threading.Thread(target=cls.httpd.serve_forever)
cls.httpd_thread.daemon = True
cls.httpd_thread.start()
def setUpClass(cls):
"""Bring up the HTTP server locally for tests."""
cls._session = rekall_session.InteractiveSession(logging_level=10)
port = portpicker.PickUnusedPort()
cls.tempdir = tempfile.mkdtemp()
cls.config = agent.Configuration.from_keywords(
session=cls._session,
server=http_server.HTTPServerPolicy.from_keywords(
session=cls._session,
base_url="http://127.0.0.1:%s/" % port,
root_directory=cls.tempdir,
bind_port=port,
private_key=crypto.RSAPrivateKey(
session=cls._session).generate_key(),
))
cls._session.SetParameter("agent_config_obj", cls.config)
cls.httpd = http_server.RekallHTTPServer(
("127.0.0.1", port),
http_server.RekallHTTPServerHandler,
session=cls._session)
cls.httpd_thread = threading.Thread(target=cls.httpd.serve_forever)
cls.httpd_thread.daemon = True
cls.httpd_thread.start()
cls.base_url = "http://%s/%s" % cls.httpd.server_address
def setUpClass(cls):
super(GRRHTTPServerTest, cls).setUpClass()
cls.config_overrider = test_lib.ConfigOverrider({
"Rekall.profile_server":
rekall_test_lib.TestRekallRepositoryProfileServer.__name__,
"FileUploadFileStore.root_dir":
test_lib.TempDirPath()
})
cls.config_overrider.Start()
# Frontend must be initialized to register all the stats counters.
frontend_lib.FrontendInit().RunOnce()
# Bring up a local server for testing.
port = portpicker.PickUnusedPort()
ip = utils.ResolveHostnameToIP("localhost", port)
cls.httpd = frontend.GRRHTTPServer((ip, port),
frontend.GRRHTTPServerHandler)
if ipaddr.IPAddress(ip).version == 6:
cls.address_family = socket.AF_INET6
cls.base_url = "http://[%s]:%d/" % (ip, port)
else:
cls.address_family = socket.AF_INET
cls.base_url = "http://%s:%d/" % (ip, port)
cls.httpd_thread = threading.Thread(target=cls.httpd.serve_forever)
cls.httpd_thread.daemon = True
cls.httpd_thread.start()
def setUp(self):
# Select a free port
port = portpicker.PickUnusedPort()
HTTPApiEndToEndTestProgram.server_port = port
logging.info("Picked free AdminUI port %d.", port)
self.trd = runtests.DjangoThread(port)
self.trd.StartAndWaitUntilServing()
def __init__(self,
emulator_cmd=None,
deadline=10,
start_options=(),
silent=False):
"""Constructs a DatastoreEmulator.
Clients should use DatastoreEmulatorFactory to construct DatastoreEmulator
instances.
Args:
emulator_cmd: A string representing the path to an executable script that
invokes emulator binary.
deadline: A integer representing number of seconds to wait for the
datastore to start.
start_options: A list of additional command-line options to pass to the
emulator 'start' command.
silent: A bool indicates if emulator runs in silent mode.
Raises:
IOError: if the emulator failed to start within the deadline
"""
self._emulator_cmd = emulator_cmd
self._http = httplib2.Http()
self.__running = False
self._silent = silent
self._redirected_output = open(os.devnull, 'wb') if self._silent else None
# Start the emulator and wait for it to start responding to requests.
cmd = [self._emulator_cmd, 'start'] + _DEFAULT_EMULATOR_OPTIONS
if start_options:
cmd.extend(start_options)
port = ParsePortFromOption(start_options or [])
if not port:
port = portpicker.PickUnusedPort()
cmd.append('--port=%d' % port)
self._host = 'http://localhost:%d' % port
cmd.append(tempfile.mkdtemp())
# On windows, cloud_datastore_emulator.bat always prompts up
# 'Terminate batch job (Y/N)'. Passing nul to this .bat avoids self.Stop()
# hang at this prompt up.
if sys.platform.startswith('win'):
cmd.append('<nul')
popen_kwargs = {}
if self._silent:
popen_kwargs.update(
stdout=self._redirected_output, stderr=self._redirected_output)
self.emulator_proc = subprocess.Popen(cmd, **popen_kwargs)
if not self._WaitForStartup(deadline):
raise IOError('emulator did not respond within %ds' % deadline)
self.__datastore = None
self.__running = True
def setUp(self):
super(SeleniumTestProgram, self).setUp()
# Select a free port
port = portpicker.PickUnusedPort()
logging.info("Picked free AdminUI port %d.", port)
# Start up a server in another thread
self.trd = runtests.DjangoThread(port)
self.trd.StartAndWaitUntilServing()
self.SetupSelenium(port)
def setUp(self):
# Select a free port
HTTPApiEndToEndTestProgram.server_port = (portpicker.PickUnusedPort())
config_lib.CONFIG.Set("AdminUI.port",
HTTPApiEndToEndTestProgram.server_port)
logging.info("Picked free AdminUI port %d.",
HTTPApiEndToEndTestProgram.server_port)
self.trd = runtests.DjangoThread()
self.trd.StartAndWaitUntilServing()
def setUp(self):
super(HTTPApiEndToEndTestProgram, self).setUp()
# Set up HTTP server
port = portpicker.PickUnusedPort()
HTTPApiEndToEndTestProgram.server_port = port
logging.info("Picked free AdminUI port for HTTP %d.", port)
self.trd = wsgiapp_testlib.ServerThread(port)
self.trd.StartAndWaitUntilServing()
def setUp(self):
# Select a free port
port = portpicker.PickUnusedPort()
config_lib.CONFIG.Set("AdminUI.port", port)
logging.info("Picked free AdminUI port %d.", port)
# Start up a server in another thread
self.trd = runtests.DjangoThread()
self.trd.StartAndWaitUntilServing()
self.SetupSelenium()
def setUp(self):
super(ApiSSLE2ETest, self).setUp()
key = rdf_crypto.RSAPrivateKey.GenerateKey()
key_path = os.path.join(self.temp_dir, "key.pem")
with open(key_path, "wb") as f:
f.write(key.AsPEM())
subject = issuer = x509.Name([
x509.NameAttribute(oid.NameOID.COMMON_NAME, u"localhost"),
])
cert = x509.CertificateBuilder().subject_name(subject).issuer_name(
issuer).public_key(
key.GetPublicKey().GetRawPublicKey()).serial_number(
x509.random_serial_number()).not_valid_before(
datetime.datetime.utcnow()).not_valid_after(
datetime.datetime.utcnow() +
datetime.timedelta(days=1)).add_extension(
x509.SubjectAlternativeName(
[x509.DNSName(u"localhost")]),
critical=False,
).sign(key.GetRawPrivateKey(), hashes.SHA256(),
backends.default_backend())
cert_path = os.path.join(self.temp_dir, "certificate.pem")
with open(cert_path, "wb") as f:
f.write(cert.public_bytes(serialization.Encoding.PEM))
self.config_overrider = test_lib.ConfigOverrider({
"AdminUI.enable_ssl":
True,
"AdminUI.ssl_key_file":
key_path,
"AdminUI.ssl_cert_file":
cert_path,
})
self.config_overrider.Start()
self.prev_environ = dict(os.environ)
os.environ["REQUESTS_CA_BUNDLE"] = cert_path
self.port = portpicker.PickUnusedPort()
self.thread = wsgiapp_testlib.ServerThread(self.port)
self.thread.StartAndWaitUntilServing()
api_auth_manager.APIACLInit.InitApiAuthManager()
self.token.username = "******"
webauth.WEBAUTH_MANAGER.SetUserName(self.token.username)
self.endpoint = "https://localhost:%s" % self.port
self.api = grr_api.InitHttp(api_endpoint=self.endpoint)
def __init__(self, gcd_sh, working_directory, project_id, deadline,
start_options, quiet=False):
"""Constructs a local datastore.
Clients should use LocalCloudDatastoreFactory to construct
LocalCloudDatastore instances.
Args:
gcd_sh: path to gcd.sh
working_directory: directory file where temporary files will be stored
project_id: project ID
deadline: number of seconds to wait for the datastore to start
start_options: a list of additional command-line options to pass to the
gcd.sh start command
quiet: if true, silences the gcd tool output
Raises:
IOError: if the datastore failed to start within the deadline
"""
self._project_id = project_id
self._gcd_sh = gcd_sh
self._http = httplib2.Http()
self.__running = False
self._tmp_dir = tempfile.mkdtemp(dir=working_directory)
self._project_directory = os.path.join(self._tmp_dir, self._project_id)
p = subprocess.Popen([gcd_sh,
'create',
'--project_id=%s' % self._project_id,
self._project_directory])
if p.wait() != 0:
raise IOError('could not create project in directory: %s'
% self._project_directory)
# Start GCD and wait for it to start responding to requests.
port = portpicker.PickUnusedPort()
self._host = 'http://localhost:%d' % port
cmd = [self._gcd_sh, 'start', '--port=%d' % port]
cmd.extend(_DEFAULT_GCD_OPTIONS)
if start_options:
cmd.extend(start_options)
cmd.append(self._project_directory)
out = open(os.devnull, 'wb') if quiet else None
subprocess.Popen(cmd, stdout=out, stderr=out)
if not self._WaitForStartup(deadline):
raise IOError('datastore did not respond within %ds' % deadline)
endpoint = '%s/datastore/v1beta3/projects/%s' % (self._host,
self._project_id)
self.__datastore = connection.Datastore(project_endpoint=endpoint)
self.__running = True
def setUpClass(cls):
super(ApiE2ETest, cls).setUpClass()
with ApiE2ETest._api_set_up_lock:
if not ApiE2ETest._api_set_up_done:
# Set up HTTP server
port = portpicker.PickUnusedPort()
ApiE2ETest.server_port = port
logging.info("Picked free AdminUI port for HTTP %d.", port)
ApiE2ETest.trd = wsgiapp_testlib.ServerThread(port)
ApiE2ETest.trd.StartAndWaitUntilServing()
ApiE2ETest._api_set_up_done = True
def SetupDataStore(self):
global PORT
if PORT:
return
PORT = portpicker.PickUnusedPort()
self._SetConfig(self.temp_dir)
_StartServer(self.temp_dir)
try:
data_store.DB = http_data_store.HTTPDataStore()
except http_data_store.HTTPDataStoreError as e:
data_store.DB = None
_CloseServer()
self.fail("Error: %s" % str(e))
def setUp(self):
super(ApiSslProxyTest, self).setUp()
attempts_count = 0
self.proxy_server = None
while self.proxy_server is None:
try:
self.proxy_port = portpicker.PickUnusedPort()
self.proxy_server = TCPServerV6(("::", self.proxy_port), Proxy)
except socket.error:
attempts_count += 1
if attempts_count == 10:
self.fail("Can't initialize proxy server.")
threading.Thread(target=self.proxy_server.serve_forever).start()
def setUpClass(cls):
super(GRRSeleniumTest, cls).setUpClass()
with GRRSeleniumTest._selenium_set_up_lock:
if not GRRSeleniumTest._selenium_set_up_done:
port = portpicker.PickUnusedPort()
logging.info("Picked free AdminUI port %d.", port)
# Start up a server in another thread
GRRSeleniumTest._server_trd = wsgiapp_testlib.ServerThread(port)
GRRSeleniumTest._server_trd.StartAndWaitUntilServing()
GRRSeleniumTest._SetUpSelenium(port)
GRRSeleniumTest._selenium_set_up_done = True
def setUpClass(cls):
super(GRRHTTPServerTest, cls).setUpClass()
# Frontend must be initialized to register all the stats counters.
front_end.FrontendInit().RunOnce()
# Bring up a local server for testing.
cls.httpd = http_server.GRRHTTPServer(
("127.0.0.1", portpicker.PickUnusedPort()),
http_server.GRRHTTPServerHandler)
cls.httpd_thread = threading.Thread(target=cls.httpd.serve_forever)
cls.httpd_thread.daemon = True
cls.httpd_thread.start()
cls.base_url = "http://%s:%s/" % cls.httpd.server_address
def setUpClass(cls): # pylint: disable=invalid-name
if cls.api_version not in [1, 2]:
raise ValueError("api_version may be 1 or 2 only")
if not HttpApiRegressionTestMixinBase.endpoint:
port = portpicker.PickUnusedPort()
logging.info("Picked free AdminUI port %d.", port)
# Force creation of new APIAuthorizationManager.
api_auth_manager.APIACLInit.InitApiAuthManager()
trd = wsgiapp_testlib.ServerThread(port)
trd.StartAndWaitUntilServing()
cls.endpoint = "http://localhost:%d" % port
def __init__(self, emulator_cmd, working_directory, project_id, deadline,
start_options):
"""Constructs a DatastoreEmulator.
Clients should use DatastoreEmulatorFactory to construct DatastoreEmulator
instances.
Args:
emulator_cmd: path to cloud_datastore_emulator
working_directory: directory file where temporary files will be stored
project_id: project ID
deadline: number of seconds to wait for the datastore to start
start_options: a list of additional command-line options to pass to the
emulator 'start' command
Raises:
IOError: if the emulator failed to start within the deadline
"""
self._project_id = project_id
self._emulator_cmd = emulator_cmd
self._http = httplib2.Http()
self.__running = False
self._tmp_dir = tempfile.mkdtemp(dir=working_directory)
self._project_directory = os.path.join(self._tmp_dir, self._project_id)
p = subprocess.Popen([
emulator_cmd, 'create',
'--project_id=%s' % self._project_id, self._project_directory
])
if p.wait() != 0:
raise IOError('could not create project in directory: %s' %
self._project_directory)
# Start the emulator and wait for it to start responding to requests.
port = portpicker.PickUnusedPort()
self._host = 'http://localhost:%d' % port
cmd = [self._emulator_cmd, 'start', '--port=%d' % port]
cmd.extend(_DEFAULT_EMULATOR_OPTIONS)
if start_options:
cmd.extend(start_options)
cmd.append(self._project_directory)
subprocess.Popen(cmd)
if not self._WaitForStartup(deadline):
raise IOError('emulator did not respond within %ds' % deadline)
endpoint = '%s/v1/projects/%s' % (self._host, self._project_id)
self.__datastore = connection.Datastore(project_endpoint=endpoint)
self.__running = True
def InitDatastore(self):
global PORT
if not PORT:
PORT = portpicker.PickUnusedPort()
_SetConfig(self.temp_dir)
if not STARTED_SERVER:
_StartServer(self.temp_dir)
else:
# Change location of the database.
HTTP_DB.ChangeLocation(self.temp_dir)
try:
data_store.DB = http_data_store.HTTPDataStore()
except http_data_store.HTTPDataStoreError as e:
data_store.DB = None
_CloseServer()
self.fail("Error: %s" % str(e))
self.old_security_manager = None
def test_basic(self):
"""Basic functionality test of START_PROCESS_REVERSE flavor."""
portpicker.PickUnusedPort().AndReturn(2345)
# As the lock is mocked out, this provides a mox expectation.
with self.proxy._process_lock:
safe_subprocess.start_process_file(
args=['/runtime'],
input_string=self.runtime_config.SerializeToString(),
env={'foo': 'bar',
'PORT': '2345'},
cwd=self.tmpdir,
stderr=subprocess.PIPE).AndReturn(self.process)
self.proxy._stderr_tee = FakeTee('')
self.mox.ReplayAll()
self.proxy.start()
self.assertEquals(2345, self.proxy._proxy._port)
self.mox.VerifyAll()
def setUpClass(cls): # pylint: disable=invalid-name
if cls.api_version not in [1, 2]:
raise ValueError("api_version may be 1 or 2 only")
if not HttpApiRegressionTestMixinBase._connector:
port = portpicker.PickUnusedPort()
logging.info("Picked free AdminUI port %d.", port)
testing_startup.TestInit()
# Force creation of new APIAuthorizationManager.
api_auth_manager.APIACLInit.InitApiAuthManager()
trd = django_lib.DjangoThread(port)
trd.StartAndWaitUntilServing()
endpoint = ("http://localhost:%s" % port)
HttpApiRegressionTestMixinBase._connector = http_connector.HttpConnector(
api_endpoint=endpoint)
def get_available_host_port():
"""Gets a host port number available for adb forward.
Returns:
An integer representing a port number on the host available for adb
forward.
Raises:
Error: when no port is found after MAX_PORT_ALLOCATION_RETRY times.
"""
for _ in range(MAX_PORT_ALLOCATION_RETRY):
port = portpicker.PickUnusedPort()
# Make sure adb is not using this port so we don't accidentally
# interrupt ongoing runs by trying to bind to the port.
if port not in adb.list_occupied_adb_ports():
return port
raise Error('Failed to find available port after {} retries'.format(
MAX_PORT_ALLOCATION_RETRY))
def GetConnector(api_version):
if api_version not in [1, 2]:
raise ValueError("api_version may be 1 or 2 only")
with _HTTP_ENDPOINTS_LOCK:
if api_version not in _HTTP_ENDPOINTS:
port = portpicker.PickUnusedPort()
logging.info("Picked free AdminUI port %d.", port)
# Force creation of new APIAuthorizationManager.
api_auth_manager.APIACLInit.InitApiAuthManager()
trd = wsgiapp_testlib.ServerThread(port)
trd.StartAndWaitUntilServing()
_HTTP_ENDPOINTS[api_version] = "http://localhost:%d" % port
return http_connector.HttpConnector(
api_endpoint=_HTTP_ENDPOINTS[api_version])
def setUpClass(cls):
super(GRRHTTPServerTest, cls).setUpClass()
# Frontend must be initialized to register all the stats counters.
front_end.FrontendInit().RunOnce()
# Bring up a local server for testing.
port = portpicker.PickUnusedPort()
ip = utils.ResolveHostnameToIP("localhost", port)
cls.httpd = frontend.GRRHTTPServer((ip, port),
frontend.GRRHTTPServerHandler)
if ipaddr.IPAddress(ip).version == 6:
cls.address_family = socket.AF_INET6
cls.base_url = "http://[%s]:%d/" % (ip, port)
else:
cls.address_family = socket.AF_INET4
cls.base_url = "http://%s:%d/" % (ip, port)
cls.httpd_thread = threading.Thread(target=cls.httpd.serve_forever)
cls.httpd_thread.daemon = True
cls.httpd_thread.start()
def start(self):
"""Starts the runtime process and waits until it is ready to serve."""
runtime_config = self._runtime_config_getter()
# TODO: Use a different process group to isolate the child process
# from signals sent to the parent. Only available in subprocess in
# Python 2.7.
assert self._start_process_flavor in self._VALID_START_PROCESS_FLAVORS
host = 'localhost'
if self._start_process_flavor == START_PROCESS:
serialized_config = base64.b64encode(runtime_config.SerializeToString())
with self._process_lock:
assert not self._process, 'start() can only be called once'
self._process = safe_subprocess.start_process(
self._args,
serialized_config,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=self._env,
cwd=self._module_configuration.application_root)
port = self._process.stdout.readline()
if '\t' in port: # Split out the host if present.
host, port = port.split('\t', 1)
elif self._start_process_flavor == START_PROCESS_FILE:
serialized_config = runtime_config.SerializeToString()
with self._process_lock:
assert not self._process, 'start() can only be called once'
self._process = safe_subprocess.start_process_file(
args=self._args,
input_string=serialized_config,
env=self._env,
cwd=self._module_configuration.application_root,
stderr=subprocess.PIPE)
port = self._read_start_process_file()
_remove_retry_sharing_violation(self._process.child_out.name)
elif self._start_process_flavor == START_PROCESS_REVERSE:
serialized_config = runtime_config.SerializeToString()
with self._process_lock:
assert not self._process, 'start() can only be called once'
port = portpicker.PickUnusedPort()
self._env['PORT'] = str(port)
# If any of the strings in args contain {port}, replace that substring
# with the selected port. This allows a user-specified runtime to
# pass the port along to the subprocess as a command-line argument.
args = [arg.replace('{port}', str(port)) for arg in self._args]
self._process = safe_subprocess.start_process_file(
args=args,
input_string=serialized_config,
env=self._env,
cwd=self._module_configuration.application_root,
stderr=subprocess.PIPE)
elif self._start_process_flavor == START_PROCESS_REVERSE_NO_FILE:
serialized_config = runtime_config.SerializeToString()
with self._process_lock:
assert not self._process, 'start() can only be called once'
port = portpicker.PickUnusedPort()
if self._extra_args_getter:
self._args.append(self._extra_args_getter(port))
# If any of the strings in _args contain {port}, {api_host}, {api_port},
# replace that substring with the selected port. This allows
# a user-specified runtime to pass the port along to the subprocess
# as a command-line argument.
args = [arg.replace('{port}', str(port))
.replace('{api_port}', str(runtime_config.api_port))
.replace('{api_host}', runtime_config.api_host)
for arg in self._args]
self._process = safe_subprocess.start_process(
args=args,
input_string=serialized_config,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=self._env,
cwd=self._module_configuration.application_root)
# _stderr_tee may be pre-set by unit tests.
if self._stderr_tee is None:
self._stderr_tee = tee.Tee(self._process.stderr, sys.stderr)
self._stderr_tee.start()
error = None
try:
port = int(port)
except ValueError:
error = 'bad runtime process port [%r]' % port
logging.error(error)
finally:
self._proxy = http_proxy.HttpProxy(
host=host, port=port,
instance_died_unexpectedly=self._instance_died_unexpectedly,
instance_logs_getter=self._get_instance_logs,
error_handler_file=application_configuration.get_app_error_file(
self._module_configuration),
prior_error=error)
self._proxy.wait_for_connection()
def create_api_server(request_info, storage_path, options, app_id, app_root):
"""Creates an API server.
Args:
request_info: An apiproxy_stub.RequestInfo instance used by the stubs to
lookup information about the request associated with an API call.
storage_path: A string directory for storing API stub data.
options: An instance of argparse.Namespace containing command line flags.
app_id: String representing an application ID, used for configuring paths
and string constants in API stubs.
app_root: The path to the directory containing the user's
application e.g. "/home/joe/myapp", used for locating application yaml
files, eg index.yaml for the datastore stub.
Returns:
An instance of APIServer.
"""
emulator_launching_thread = None
if options.support_datastore_emulator and not os.environ.get(
'DATASTORE_EMULATOR_HOST'):
gcd_emulator_port = portpicker.PickUnusedPort()
emulator_launching_thread = threading.Thread(
target=GCD_EMULATOR_MANAGER.launch,
args=[
gcd_emulator_port, options.dev_appserver_log_level != 'debug',
os.path.join(app_root, 'index.yaml'), options.require_indexes
])
emulator_launching_thread.start()
os.environ[
'DATASTORE_EMULATOR_HOST'] = 'localhost:%d' % gcd_emulator_port
datastore_path = options.datastore_path or os.path.join(
storage_path, 'datastore.db')
logs_path = options.logs_path or os.path.join(storage_path, 'logs.db')
search_index_path = options.search_indexes_path or os.path.join(
storage_path, 'search_indexes')
blobstore_path = options.blobstore_path or os.path.join(
storage_path, 'blobs')
if options.clear_datastore:
_clear_datastore_storage(datastore_path)
if options.clear_search_indexes:
_clear_search_indexes_storage(search_index_path)
if options.auto_id_policy == datastore_stub_util.SEQUENTIAL:
logging.warn(
"--auto_id_policy='sequential' is deprecated. This option "
"will be removed in a future release.")
application_address = '%s' % options.host
if options.port and options.port != 80:
application_address += ':' + str(options.port)
user_login_url = '/%s?%s=%%s' % (login.LOGIN_URL_RELATIVE,
login.CONTINUE_PARAM)
user_logout_url = '/%s?%s=%%s' % (login.LOGOUT_URL_RELATIVE,
login.CONTINUE_PARAM)
if options.datastore_consistency_policy == 'time':
consistency = datastore_stub_util.TimeBasedHRConsistencyPolicy()
elif options.datastore_consistency_policy == 'random':
consistency = datastore_stub_util.PseudoRandomHRConsistencyPolicy()
elif options.datastore_consistency_policy == 'consistent':
consistency = datastore_stub_util.PseudoRandomHRConsistencyPolicy(1.0)
else:
assert 0, ('unknown consistency policy: %r' %
options.datastore_consistency_policy)
# Check if local datastore data should be converted.
# Using GCD Emulator this could convert python file stub or sqlite stub data
# to Emulator data format; Without GCD Emulator this converts python file stub
# to sqlite stub data.
if os.path.exists(datastore_path):
data_type = datastore_converter.get_data_type(datastore_path)
if options.support_datastore_emulator:
if data_type in [
datastore_converter.StubTypes.PYTHON_FILE_STUB,
datastore_converter.StubTypes.PYTHON_SQLITE_STUB
]:
if emulator_launching_thread:
emulator_launching_thread.join()
gcd_emulator_host = os.environ.get('DATASTORE_EMULATOR_HOST')
datastore_converter.convert_python_data_to_emulator(
app_id, data_type, datastore_path, gcd_emulator_host)
else:
if data_type != datastore_converter.StubTypes.PYTHON_SQLITE_STUB:
datastore_converter.convert_datastore_file_stub_data_to_sqlite(
app_id, datastore_path)
stub_util.setup_stubs(
request_data=request_info,
app_id=app_id,
application_root=app_root,
# The "trusted" flag is only relevant for Google administrative
# applications.
trusted=getattr(options, 'trusted', False),
appidentity_email_address=options.appidentity_email_address,
appidentity_private_key_path=os.path.abspath(
options.appidentity_private_key_path)
if options.appidentity_private_key_path else None,
blobstore_path=blobstore_path,
datastore_path=datastore_path,
datastore_consistency=consistency,
datastore_require_indexes=options.require_indexes,
datastore_auto_id_policy=options.auto_id_policy,
images_host_prefix='http://%s' % application_address,
logs_path=logs_path,
mail_smtp_host=options.smtp_host,
mail_smtp_port=options.smtp_port,
mail_smtp_user=options.smtp_user,
mail_smtp_password=options.smtp_password,
mail_enable_sendmail=options.enable_sendmail,
mail_show_mail_body=options.show_mail_body,
mail_allow_tls=options.smtp_allow_tls,
search_index_path=search_index_path,
taskqueue_auto_run_tasks=options.enable_task_running,
taskqueue_default_http_server=application_address,
user_login_url=user_login_url,
user_logout_url=user_logout_url,
default_gcs_bucket_name=options.default_gcs_bucket_name,
appidentity_oauth_url=options.appidentity_oauth_url,
datastore_grpc_stub_class=(datastore_grpc_stub.DatastoreGrpcStub
if options.support_datastore_emulator else
None))
if emulator_launching_thread:
emulator_launching_thread.join()
return APIServer(
options.api_host, options.api_port, app_id,
options.api_server_supports_grpc or options.support_datastore_emulator,
options.grpc_api_port, options.enable_host_checking)
def setUp(self):
super(ApiSslProxyTest, self).setUp()
self.proxy_port = portpicker.PickUnusedPort()
self.proxy_server = TCPServerV6(("::", self.proxy_port), Proxy)
threading.Thread(target=self.proxy_server.serve_forever).start()
def create_api_server(request_info, storage_path, options, app_id, app_root):
"""Creates an API server.
Args:
request_info: An apiproxy_stub.RequestInfo instance used by the stubs to
lookup information about the request associated with an API call.
storage_path: A string directory for storing API stub data.
options: An instance of argparse.Namespace containing command line flags.
app_id: String representing an application ID, used for configuring paths
and string constants in API stubs.
app_root: The path to the directory containing the user's
application e.g. "/home/joe/myapp", used for locating application yaml
files, eg index.yaml for the datastore stub.
Returns:
An instance of APIServer.
Raises:
DatastoreFileError: Cloud Datastore emulator is used while stub_type is
datastore_converter.StubTypes.PYTHON_FILE_STUB.
"""
datastore_path = get_datastore_path(storage_path, options.datastore_path)
logs_path = options.logs_path or os.path.join(storage_path, 'logs.db')
search_index_path = options.search_indexes_path or os.path.join(
storage_path, 'search_indexes')
blobstore_path = options.blobstore_path or os.path.join(
storage_path, 'blobs')
if options.clear_datastore:
_clear_datastore_storage(datastore_path)
if options.clear_search_indexes:
_clear_search_indexes_storage(search_index_path)
if options.auto_id_policy == datastore_stub_util.SEQUENTIAL:
logging.warn(
"--auto_id_policy='sequential' is deprecated. This option "
"will be removed in a future release.")
application_address = '%s' % options.host
if options.port and options.port != 80:
application_address += ':' + str(options.port)
user_login_url = '/%s?%s=%%s' % (login.LOGIN_URL_RELATIVE,
login.CONTINUE_PARAM)
user_logout_url = '/%s?%s=%%s' % (login.LOGOUT_URL_RELATIVE,
login.CONTINUE_PARAM)
if options.datastore_consistency_policy == 'time':
consistency = datastore_stub_util.TimeBasedHRConsistencyPolicy()
elif options.datastore_consistency_policy == 'random':
consistency = datastore_stub_util.PseudoRandomHRConsistencyPolicy()
elif options.datastore_consistency_policy == 'consistent':
consistency = datastore_stub_util.PseudoRandomHRConsistencyPolicy(1.0)
else:
assert 0, ('unknown consistency policy: %r' %
options.datastore_consistency_policy)
stub_type = (datastore_converter.get_stub_type(datastore_path)
if os.path.exists(datastore_path) else None)
gcd_emulator_launching_thread = None
if options.support_datastore_emulator:
if stub_type == datastore_converter.StubTypes.PYTHON_FILE_STUB:
raise DatastoreFileError(
'The datastore file %s cannot be recognized by dev_appserver. Please '
'restart dev_appserver with --clear_datastore=1' %
datastore_path)
env_emulator_host = os.environ.get('DATASTORE_EMULATOR_HOST')
if env_emulator_host: # emulator already running, reuse it.
logging.warning(
'Detected environment variable DATASTORE_EMULATOR_HOST=%s, '
'dev_appserver will speak to the Cloud Datastore emulator running on '
'this address. The datastore_path %s will be neglected.\nIf you '
'want datastore to store on %s, remove DATASTORE_EMULATOR_HOST '
'from environment variables and restart dev_appserver',
env_emulator_host, datastore_path, datastore_path)
else:
gcd_emulator_launching_thread = _launch_gcd_emulator(
app_id=app_id,
emulator_port=(options.datastore_emulator_port
if options.datastore_emulator_port else
portpicker.PickUnusedPort()),
silent=options.dev_appserver_log_level != 'debug',
index_file=os.path.join(app_root, 'index.yaml'),
require_indexes=options.require_indexes,
datastore_path=datastore_path,
stub_type=stub_type,
cmd=options.datastore_emulator_cmd,
is_test=options.datastore_emulator_is_test_mode)
else:
# Use SQLite stub.
# For historic reason we are still supporting conversion from file stub to
# SQLite stub data. But this conversion will go away.
if stub_type == datastore_converter.StubTypes.PYTHON_FILE_STUB:
datastore_converter.convert_datastore_file_stub_data_to_sqlite(
app_id, datastore_path)
stub_util.setup_stubs(
request_data=request_info,
app_id=app_id,
application_root=app_root,
# The "trusted" flag is only relevant for Google administrative
# applications.
trusted=getattr(options, 'trusted', False),
appidentity_email_address=options.appidentity_email_address,
appidentity_private_key_path=os.path.abspath(
options.appidentity_private_key_path)
if options.appidentity_private_key_path else None,
blobstore_path=blobstore_path,
datastore_path=datastore_path,
datastore_consistency=consistency,
datastore_require_indexes=options.require_indexes,
datastore_auto_id_policy=options.auto_id_policy,
images_host_prefix='http://%s' % application_address,
logs_path=logs_path,
mail_smtp_host=options.smtp_host,
mail_smtp_port=options.smtp_port,
mail_smtp_user=options.smtp_user,
mail_smtp_password=options.smtp_password,
mail_enable_sendmail=options.enable_sendmail,
mail_show_mail_body=options.show_mail_body,
mail_allow_tls=options.smtp_allow_tls,
search_index_path=search_index_path,
taskqueue_auto_run_tasks=options.enable_task_running,
taskqueue_default_http_server=application_address,
user_login_url=user_login_url,
user_logout_url=user_logout_url,
default_gcs_bucket_name=options.default_gcs_bucket_name,
appidentity_oauth_url=options.appidentity_oauth_url,
datastore_grpc_stub_class=(datastore_grpc_stub.DatastoreGrpcStub
if options.support_datastore_emulator else
None))
return APIServer(
options.api_host, options.api_port, app_id,
options.api_server_supports_grpc or options.support_datastore_emulator,
options.grpc_api_port, options.enable_host_checking,
gcd_emulator_launching_thread)
def setUp(cls):
cls.port = portpicker.PickUnusedPort()
