def start_emulators(persist_data, emulators=None, storage_dir=None, task_target_port=None, autodetect_task_port=True):
# This prevents restarting of the emulators when Django code reload
# kicks in
if os.environ.get(DJANGO_AUTORELOAD_ENV) == 'true':
return
emulators = emulators or _ALL_EMULATORS
storage_dir = storage_dir or os.path.join(get_application_root(), ".storage")
if "datastore" in emulators:
os.environ["DATASTORE_EMULATOR_HOST"] = "127.0.0.1:%s" % DATASTORE_PORT
os.environ["DATASTORE_PROJECT_ID"] = "example"
# Start the cloud datastore emulator
command = "gcloud beta emulators datastore start --consistency=1.0 --quiet --project=example"
command += " --host-port=127.0.0.1:%s" % DATASTORE_PORT
if not persist_data:
command += " --no-store-on-disk"
_ACTIVE_EMULATORS["datastore"] = _launch_process(command)
_wait_for_datastore(DATASTORE_PORT)
if "tasks" in emulators:
from djangae.tasks import cloud_tasks_parent_path
default_queue = "%s/queues/default" % cloud_tasks_parent_path()
if task_target_port is None:
if sys.argv[1] == "runserver" and autodetect_task_port:
from django.core.management.commands.runserver import Command as RunserverCommand
parser = RunserverCommand().create_parser('django', 'runserver')
args = parser.parse_args(sys.argv[2:])
if args.addrport:
task_target_port = args.addrport.split(":")[-1]
else:
task_target_port = _DJANGO_DEFAULT_PORT
else:
task_target_port = _DJANGO_DEFAULT_PORT
os.environ["TASKS_EMULATOR_HOST"] = "127.0.0.1:%s" % TASKS_PORT
_ACTIVE_EMULATORS["tasks"] = _launch_process(
"gcloud-tasks-emulator start -q --port=%s --target-port=%s --default-queue=%s" % (
TASKS_PORT, task_target_port, default_queue
)
)
_wait_for_tasks(TASKS_PORT)
if "storage" in emulators:
os.environ["STORAGE_EMULATOR_HOST"] = "http://127.0.0.1:%s" % STORAGE_PORT
command = "gcloud-storage-emulator start -q --port=%s" % STORAGE_PORT
if not persist_data:
command += " --no-store-on-disk"
_ACTIVE_EMULATORS["storage"] = _launch_process(command)
_wait_for_storage(STORAGE_PORT)
def start_emulators(
persist_data: bool,
project_id: str = DEFAULT_PROJECT_ID,
emulators: Sequence[str] = _ALL_EMULATORS,
datastore_port: int = DATASTORE_PORT,
datastore_dir: Optional[str] = None,
tasks_port: int = TASKS_PORT,
task_target_port: Optional[int] = None,
autodetect_task_port: bool = True,
storage_port: int = STORAGE_PORT,
storage_dir: Optional[str] = None,
):
# This prevents restarting of the emulators when Django code reload kicks in
if os.environ.get(DJANGO_AUTORELOAD_ENV) == 'true':
return
storage_dir = storage_dir or os.path.join(get_application_root(), ".storage")
enable_test_environment_variables()
if "datastore" in emulators:
if not port_is_open(SERVICE_HOST, datastore_port):
# If start_emulators is call explicitly passing the Datastore Emulator port
# and the port is not available raise and Runtime exception.
if datastore_port != DATASTORE_PORT:
raise RuntimeError(f"Unable to start Cloud Datastore Emulator at port {datastore_port}.")
else:
datastore_port = get_next_available_port(SERVICE_HOST, datastore_port)
os.environ["DATASTORE_EMULATOR_HOST"] = f"{SERVICE_HOST}:{datastore_port}"
os.environ["DATASTORE_PROJECT_ID"] = project_id
# Start the cloud datastore emulator
command = f"gcloud beta emulators datastore start --user-output-enabled=false --consistency=1.0 --quiet --project={project_id}" # noqa
command += f" --host-port={SERVICE_HOST}:{datastore_port}"
if datastore_dir:
command += " --data-dir=%s" % datastore_dir
if not persist_data:
command += " --no-store-on-disk"
_ACTIVE_EMULATORS["datastore"] = _launch_process(command)
_wait_for_datastore(datastore_port)
if "tasks" in emulators:
# If start_emulators is call explicitly passing the Cloud Task emulator port
# and the port is not available raise and Runtime exception.
if not port_is_open(SERVICE_HOST, tasks_port):
if tasks_port != TASKS_PORT:
raise RuntimeError(f"Unable to start Cloud Tasks Emulator at port {tasks_port}.")
else:
tasks_port = get_next_available_port(SERVICE_HOST, tasks_port)
from djangae.tasks import cloud_tasks_location, cloud_tasks_parent_path, cloud_tasks_project
default_queue = "%s/queues/default" % cloud_tasks_parent_path()
if task_target_port is None:
if sys.argv[1] == "runserver" and autodetect_task_port:
from django.core.management.commands.runserver import Command as RunserverCommand
parser = RunserverCommand().create_parser('django', 'runserver')
args = parser.parse_args(sys.argv[2:])
if args.addrport:
task_target_port = args.addrport.split(":")[-1]
else:
task_target_port = _DJANGO_DEFAULT_PORT
else:
task_target_port = _DJANGO_DEFAULT_PORT
command = "gcloud-tasks-emulator start -q --port=%s --target-port=%s --default-queue=%s" % (
tasks_port, task_target_port, default_queue
)
# If the project contains a queue.yaml, pass it to the Tasks Emulator so that those queues
# can be created (needs version >= 0.4.0)
queue_yaml = os.path.join(get_application_root(), "queue.yaml")
if os.path.exists(queue_yaml):
command += " --queue-yaml=%s --queue-yaml-project=%s --queue-yaml-location=%s" % (
queue_yaml, cloud_tasks_project(), cloud_tasks_location()
)
os.environ["TASKS_EMULATOR_HOST"] = f"{SERVICE_HOST}:{tasks_port}"
_ACTIVE_EMULATORS["tasks"] = _launch_process(command)
_wait_for_tasks(tasks_port)
if "storage" in emulators:
# If start_emulators is call explicitly passing the Cloud Storage emulator port
# and the port is not available raise and Runtime exception.
if not port_is_open(SERVICE_HOST, storage_port):
if storage_port != STORAGE_PORT:
raise RuntimeError(f"Unable to start Cloud Storage Emulator at port {storage_port}.")
else:
storage_port = get_next_available_port(SERVICE_HOST, storage_port)
os.environ["STORAGE_EMULATOR_HOST"] = f"http://{SERVICE_HOST}:{storage_port}"
command = "gcloud-storage-emulator start -q --port=%s --default-bucket=%s" % (
storage_port, DEFAULT_BUCKET)
if not persist_data:
command += " --no-store-on-disk"
_ACTIVE_EMULATORS["storage"] = _launch_process(command)
_wait_for_storage(storage_port)
def start_emulators(
persist_data: bool,
project_id: str = DEFAULT_PROJECT_ID,
emulators: Sequence[str] = _ALL_EMULATORS,
datastore_port: int = DATASTORE_PORT,
datastore_dir: Optional[str] = None,
tasks_port: int = TASKS_PORT,
task_target_port: Optional[int] = None,
autodetect_task_port: bool = True,
storage_port: int = STORAGE_PORT,
storage_dir: Optional[str] = None,
):
# This prevents restarting of the emulators when Django code reload kicks in
if os.environ.get(DJANGO_AUTORELOAD_ENV) == 'true':
return
storage_dir = storage_dir or os.path.join(get_application_root(), ".storage")
enable_test_environment_variables()
if "datastore" in emulators:
os.environ["DATASTORE_EMULATOR_HOST"] = "127.0.0.1:%s" % datastore_port
os.environ["DATASTORE_PROJECT_ID"] = project_id
# Start the cloud datastore emulator
command = "gcloud beta emulators datastore start --consistency=1.0 --quiet --project=example"
command += " --host-port=127.0.0.1:%s" % datastore_port
if datastore_dir:
command += " --data-dir=%s" % datastore_dir
if not persist_data:
command += " --no-store-on-disk"
_ACTIVE_EMULATORS["datastore"] = _launch_process(command)
_wait_for_datastore(datastore_port)
if "tasks" in emulators:
from djangae.tasks import cloud_tasks_parent_path, cloud_tasks_project, cloud_tasks_location
default_queue = "%s/queues/default" % cloud_tasks_parent_path()
if task_target_port is None:
if sys.argv[1] == "runserver" and autodetect_task_port:
from django.core.management.commands.runserver import Command as RunserverCommand
parser = RunserverCommand().create_parser('django', 'runserver')
args = parser.parse_args(sys.argv[2:])
if args.addrport:
task_target_port = args.addrport.split(":")[-1]
else:
task_target_port = _DJANGO_DEFAULT_PORT
else:
task_target_port = _DJANGO_DEFAULT_PORT
command = "gcloud-tasks-emulator start -q --port=%s --target-port=%s --default-queue=%s" % (
tasks_port, task_target_port, default_queue
)
# If the project contains a queue.yaml, pass it to the Tasks Emulator so that those queues
# can be created (needs version >= 0.4.0)
queue_yaml = os.path.join(get_application_root(), "queue.yaml")
if os.path.exists(queue_yaml):
command += " --queue-yaml=%s --queue-yaml-project=%s --queue-yaml-location=%s" % (
queue_yaml, cloud_tasks_project(), cloud_tasks_location()
)
os.environ["TASKS_EMULATOR_HOST"] = "127.0.0.1:%s" % tasks_port
_ACTIVE_EMULATORS["tasks"] = _launch_process(command)
_wait_for_tasks(tasks_port)
if "storage" in emulators:
os.environ["STORAGE_EMULATOR_HOST"] = "http://127.0.0.1:%s" % storage_port
command = "gcloud-storage-emulator start -q --port=%s --default-bucket=%s" % (
storage_port, DEFAULT_BUCKET)
if not persist_data:
command += " --no-store-on-disk"
_ACTIVE_EMULATORS["storage"] = _launch_process(command)
_wait_for_storage(storage_port)
def start_emulators(
persist_data: bool,
project_id: str = DEFAULT_PROJECT_ID,
emulators: Sequence[str] = _ALL_EMULATORS,
datastore_port: int = DEFAULT_DATASTORE_PORT,
datastore_dir: Optional[str] = None,
tasks_port: int = DEFAULT_TASKS_PORT,
task_target_port: Optional[int] = None,
task_queue_yaml: Optional[str] = None,
autodetect_task_port: bool = True,
storage_port: int = DEFAULT_STORAGE_PORT,
storage_dir: Optional[str] = None,
):
# This prevents restarting of the emulators when Django code reload kicks in
if os.environ.get(DJANGO_AUTORELOAD_ENV) == 'true':
return
# If storage_dir and datastore_dir are specified, we just
# use them verbatim, otherwise we do some guesswork
if not (storage_dir and datastore_dir):
# sys.path[0] is nearly always the parent path of the
# executed script (e.g. manage.py)
base_path = sys.path[0]
# Fall-back to the application root with a warning
if not base_path:
logging.warn(
"Unable to determine script path, using "
"application root for storage directories"
)
base_path = get_application_root()
storage_dir = storage_dir or os.path.join(base_path, ".clouddata", "storage")
datastore_dir = datastore_dir or os.path.join(base_path, ".clouddata", "datastore")
os.makedirs(storage_dir, exist_ok=True)
os.makedirs(datastore_dir, exist_ok=True)
enable_test_environment_variables()
if "datastore" in emulators:
if not port_is_open(SERVICE_HOST, datastore_port):
# If start_emulators is call explicitly passing the Datastore Emulator port
# and the port is not available raise and Runtime exception.
if datastore_port != DEFAULT_DATASTORE_PORT:
raise RuntimeError(f"Unable to start Cloud Datastore Emulator at port {datastore_port}.")
else:
datastore_port = get_next_available_port(SERVICE_HOST, datastore_port)
os.environ["DATASTORE_EMULATOR_HOST"] = f"{SERVICE_HOST}:{datastore_port}"
os.environ["DATASTORE_PROJECT_ID"] = project_id
# Start the cloud datastore emulator
command = f"gcloud beta emulators datastore start --user-output-enabled=false --consistency=1.0 --quiet --project={project_id}" # noqa
command += f" --host-port={SERVICE_HOST}:{datastore_port}"
if datastore_dir:
command += " --data-dir=%s" % datastore_dir
if not persist_data:
command += " --no-store-on-disk"
_ACTIVE_EMULATORS["datastore"] = _launch_process(command)
_wait_for_datastore(datastore_port)
if "tasks" in emulators:
# If start_emulators is call explicitly passing the Cloud Task emulator port
# and the port is not available raise and Runtime exception.
if not port_is_open(SERVICE_HOST, tasks_port):
if tasks_port != DEFAULT_TASKS_PORT:
raise RuntimeError(f"Unable to start Cloud Tasks Emulator at port {tasks_port}.")
else:
tasks_port = get_next_available_port(SERVICE_HOST, tasks_port)
from djangae.tasks import cloud_tasks_location, cloud_tasks_parent_path, cloud_tasks_project
default_queue = "%s/queues/default" % cloud_tasks_parent_path()
if task_target_port is None:
if sys.argv[1] == "runserver" and autodetect_task_port:
from django.core.management.commands.runserver import Command as RunserverCommand
parser = RunserverCommand().create_parser('django', 'runserver')
args = parser.parse_args(sys.argv[2:])
if args.addrport:
task_target_port = args.addrport.split(":")[-1]
else:
task_target_port = _DJANGO_DEFAULT_PORT
else:
task_target_port = _DJANGO_DEFAULT_PORT
command = "gcloud-tasks-emulator start -q --port=%s --target-port=%s --default-queue=%s" % (
tasks_port, task_target_port, default_queue
)
# If the project contains a queue.yaml, pass it to the Tasks Emulator so that those queues
# can be created (needs version >= 0.4.0)
if task_queue_yaml and os.path.exists(task_queue_yaml):
command += " --queue-yaml=%s --queue-yaml-project=%s --queue-yaml-location=%s" % (
task_queue_yaml, cloud_tasks_project(), cloud_tasks_location()
)
elif task_queue_yaml:
logger.warn("task_queue_yaml was passed, but the file does not exist. Ignoring.")
os.environ["TASKS_EMULATOR_HOST"] = f"{SERVICE_HOST}:{tasks_port}"
_ACTIVE_EMULATORS["tasks"] = _launch_process(command)
_wait_for_tasks(tasks_port)
if "storage" in emulators:
# If start_emulators is call explicitly passing the Cloud Storage emulator port
# and the port is not available raise and Runtime exception.
if not port_is_open(SERVICE_HOST, storage_port):
if storage_port != DEFAULT_STORAGE_PORT:
raise RuntimeError(f"Unable to start Cloud Storage Emulator at port {storage_port}.")
else:
storage_port = get_next_available_port(SERVICE_HOST, storage_port)
os.environ["STORAGE_EMULATOR_HOST"] = f"http://{SERVICE_HOST}:{storage_port}"
command = "gcloud-storage-emulator start -q --port=%s --default-bucket=%s" % (
storage_port, DEFAULT_BUCKET)
if storage_dir:
command += " --data-dir=%s" % storage_dir
if not persist_data:
command += " --no-store-on-disk"
_ACTIVE_EMULATORS["storage"] = _launch_process(command)
_wait_for_storage(storage_port)
