def upload_logs(stop_event, raise_errors=False): """Upload logs to telemetry server every hour, or when log directory size is > 10MB""" # We add a sanity check to ensure that no logs are uploaded in our # buildkite/azure testing pipelines. The check is present at upload to # allow for testing of logs being correctly written. if is_running_in_test(): return try: last_run = datetime.datetime.now() - datetime.timedelta(minutes=120) dagster_log_dir = get_dir_from_dagster_home("logs") dagster_log_queue_dir = get_dir_from_dagster_home(".logs_queue") in_progress = False while not stop_event.is_set(): log_size = 0 if os.path.isdir(dagster_log_dir): log_size = sum( os.path.getsize(os.path.join(dagster_log_dir, f)) for f in os.listdir(dagster_log_dir) if os.path.isfile(os.path.join(dagster_log_dir, f)) ) log_queue_size = 0 if os.path.isdir(dagster_log_queue_dir): log_queue_size = sum( os.path.getsize(os.path.join(dagster_log_queue_dir, f)) for f in os.listdir(dagster_log_queue_dir) if os.path.isfile(os.path.join(dagster_log_queue_dir, f)) ) if log_size == 0 and log_queue_size == 0: return if not in_progress and ( datetime.datetime.now() - last_run > datetime.timedelta(minutes=60) or log_size >= MAX_BYTES or log_queue_size >= MAX_BYTES ): in_progress = True # Prevent concurrent _upload_logs invocations last_run = datetime.datetime.now() dagster_log_dir = get_dir_from_dagster_home("logs") dagster_log_queue_dir = get_dir_from_dagster_home(".logs_queue") _upload_logs( dagster_log_dir, log_size, dagster_log_queue_dir, raise_errors=raise_errors ) in_progress = False stop_event.wait(600) # Sleep for 10 minutes except Exception: # pylint: disable=broad-except if raise_errors: raise
def test_dagster_telemetry_disabled(caplog): with seven.TemporaryDirectory() as temp_dir: with environ({'DAGSTER_HOME': temp_dir}): with open(os.path.join(temp_dir, 'dagster.yaml'), 'w') as fd: yaml.dump({'telemetry': { 'enabled': False }}, fd, default_flow_style=False) DagsterInstance.local_temp(temp_dir) runner = CliRunner(env={'DAGSTER_HOME': temp_dir}) with pushd(path_to_tutorial_file('')): pipeline_name = 'hello_cereal_pipeline' result = runner.invoke( pipeline_execute_command, [ '-f', path_to_tutorial_file('hello_cereal.py'), '-n', pipeline_name ], ) assert not os.path.exists( os.path.join(get_dir_from_dagster_home('logs'), 'event.log')) assert len(caplog.records) == 0 assert result.exit_code == 0
def test_dagster_telemetry_disabled(caplog): with seven.TemporaryDirectory() as temp_dir: with environ({"DAGSTER_HOME": temp_dir}): with open(os.path.join(temp_dir, "dagster.yaml"), "w") as fd: yaml.dump({"telemetry": { "enabled": False }}, fd, default_flow_style=False) DagsterInstance.local_temp(temp_dir) runner = CliRunner(env={"DAGSTER_HOME": temp_dir}) with pushd(path_to_file("")): pipeline_name = "foo_pipeline" result = runner.invoke( pipeline_execute_command, [ "-f", path_to_file("test_cli_commands.py"), "-a", pipeline_name, ], ) assert not os.path.exists( os.path.join(get_dir_from_dagster_home("logs"), "event.log")) assert len(caplog.records) == 0 assert result.exit_code == 0
def test_launch_successful_execution_telemetry(schedule_origin_context): with _default_instance(): with schedule_origin_context("simple_schedule") as schedule_origin: sync_launch_scheduled_execution(schedule_origin) event_log_path = "{logs_dir}/event.log".format( logs_dir=get_dir_from_dagster_home("logs") ) with open(event_log_path, "r") as f: event_log = f.readlines() assert len(event_log) == 2 message_start = json.loads(event_log[0]) message_end = json.loads(event_log[1]) assert message_start.get("action") == "_launch_scheduled_executions_started" assert message_end.get("action") == "_launch_scheduled_executions_ended"
def test_dagster_telemetry_disabled(caplog): with instance_for_test(overrides={"telemetry": {"enabled": False}}): runner = CliRunner() with pushd(path_to_file("")): pipeline_name = "foo_pipeline" result = runner.invoke( pipeline_execute_command, [ "-f", path_to_file("test_cli_commands.py"), "-a", pipeline_name, ], ) assert not os.path.exists(os.path.join(get_dir_from_dagster_home("logs"), "event.log")) assert len(caplog.records) == 0 assert result.exit_code == 0