def upload_logs(stop_event, raise_errors=False):
"""Upload logs to telemetry server every hour, or when log directory size is > 10MB"""
# We add a sanity check to ensure that no logs are uploaded in our
# buildkite/azure testing pipelines. The check is present at upload to
# allow for testing of logs being correctly written.
if is_running_in_test():
return
try:
last_run = datetime.datetime.now() - datetime.timedelta(minutes=120)
dagster_log_dir = get_dir_from_dagster_home("logs")
dagster_log_queue_dir = get_dir_from_dagster_home(".logs_queue")
in_progress = False
while not stop_event.is_set():
log_size = 0
if os.path.isdir(dagster_log_dir):
log_size = sum(
os.path.getsize(os.path.join(dagster_log_dir, f))
for f in os.listdir(dagster_log_dir)
if os.path.isfile(os.path.join(dagster_log_dir, f))
)
log_queue_size = 0
if os.path.isdir(dagster_log_queue_dir):
log_queue_size = sum(
os.path.getsize(os.path.join(dagster_log_queue_dir, f))
for f in os.listdir(dagster_log_queue_dir)
if os.path.isfile(os.path.join(dagster_log_queue_dir, f))
)
if log_size == 0 and log_queue_size == 0:
return
if not in_progress and (
datetime.datetime.now() - last_run > datetime.timedelta(minutes=60)
or log_size >= MAX_BYTES
or log_queue_size >= MAX_BYTES
):
in_progress = True # Prevent concurrent _upload_logs invocations
last_run = datetime.datetime.now()
dagster_log_dir = get_dir_from_dagster_home("logs")
dagster_log_queue_dir = get_dir_from_dagster_home(".logs_queue")
_upload_logs(
dagster_log_dir, log_size, dagster_log_queue_dir, raise_errors=raise_errors
)
in_progress = False
stop_event.wait(600) # Sleep for 10 minutes
except Exception: # pylint: disable=broad-except
if raise_errors:
raise
def test_dagster_telemetry_disabled(caplog):
with seven.TemporaryDirectory() as temp_dir:
with environ({'DAGSTER_HOME': temp_dir}):
with open(os.path.join(temp_dir, 'dagster.yaml'), 'w') as fd:
yaml.dump({'telemetry': {
'enabled': False
}},
fd,
default_flow_style=False)
DagsterInstance.local_temp(temp_dir)
runner = CliRunner(env={'DAGSTER_HOME': temp_dir})
with pushd(path_to_tutorial_file('')):
pipeline_name = 'hello_cereal_pipeline'
result = runner.invoke(
pipeline_execute_command,
[
'-f',
path_to_tutorial_file('hello_cereal.py'), '-n',
pipeline_name
],
)
assert not os.path.exists(
os.path.join(get_dir_from_dagster_home('logs'), 'event.log'))
assert len(caplog.records) == 0
assert result.exit_code == 0
def test_dagster_telemetry_disabled(caplog):
with seven.TemporaryDirectory() as temp_dir:
with environ({"DAGSTER_HOME": temp_dir}):
with open(os.path.join(temp_dir, "dagster.yaml"), "w") as fd:
yaml.dump({"telemetry": {
"enabled": False
}},
fd,
default_flow_style=False)
DagsterInstance.local_temp(temp_dir)
runner = CliRunner(env={"DAGSTER_HOME": temp_dir})
with pushd(path_to_file("")):
pipeline_name = "foo_pipeline"
result = runner.invoke(
pipeline_execute_command,
[
"-f",
path_to_file("test_cli_commands.py"),
"-a",
pipeline_name,
],
)
assert not os.path.exists(
os.path.join(get_dir_from_dagster_home("logs"), "event.log"))
assert len(caplog.records) == 0
assert result.exit_code == 0
def test_launch_successful_execution_telemetry(schedule_origin_context):
with _default_instance():
with schedule_origin_context("simple_schedule") as schedule_origin:
sync_launch_scheduled_execution(schedule_origin)
event_log_path = "{logs_dir}/event.log".format(
logs_dir=get_dir_from_dagster_home("logs")
)
with open(event_log_path, "r") as f:
event_log = f.readlines()
assert len(event_log) == 2
message_start = json.loads(event_log[0])
message_end = json.loads(event_log[1])
assert message_start.get("action") == "_launch_scheduled_executions_started"
assert message_end.get("action") == "_launch_scheduled_executions_ended"
def test_dagster_telemetry_disabled(caplog):
with instance_for_test(overrides={"telemetry": {"enabled": False}}):
runner = CliRunner()
with pushd(path_to_file("")):
pipeline_name = "foo_pipeline"
result = runner.invoke(
pipeline_execute_command,
[
"-f",
path_to_file("test_cli_commands.py"),
"-a",
pipeline_name,
],
)
assert not os.path.exists(os.path.join(get_dir_from_dagster_home("logs"), "event.log"))
assert len(caplog.records) == 0
assert result.exit_code == 0
