def _load_activity_record(self, info):
"""Private method to load an activity record if it has not been previously loaded and set"""
if not self._activity_record:
# Load record from database
if not self.commit:
raise ValueError(
"Must set `commit` on object creation to resolve detail record"
)
# Load store instance
lb = self._get_loader(info).load(
f"{get_logged_in_username()}&{self.owner}&{self.name}").get()
store = ActivityStore(lb)
# Retrieve record
self._activity_record = store.get_activity_record(self.commit)
# Set class properties
self.linked_commit = self._activity_record.linked_commit
self.message = self._activity_record.message
self.type = ActivityRecordTypeEnum.get(
self._activity_record.type.value).value
self.show = self._activity_record.show
self.tags = self._activity_record.tags
self.timestamp = self._activity_record.timestamp
self.importance = self._activity_record.importance
self.username = self._activity_record.username
self.email = self._activity_record.email
def test_multiple_cells(self, redis_client, mock_labbook, mock_kernel):
"""Test processing notebook activity"""
dummy_file = os.path.join(mock_labbook[2].root_dir, 'code', 'Test.ipynb')
with open(dummy_file, 'wt') as tf:
tf.write("Dummy file")
monitor_key = "dev_env_monitor:{}:{}:{}:{}:activity_monitor:{}".format('test',
'test',
'labbook1',
'jupyterlab-ubuntu1604',
uuid.uuid4())
monitor = JupyterLabNotebookMonitor("test", "test", mock_labbook[2].name,
monitor_key, config_file=mock_labbook[0])
# Setup monitoring metadata
metadata = {"kernel_id": "XXXX",
"kernel_name": 'python',
"kernel_type": 'notebook',
"path": 'code/Test.ipynb'}
# Perform an action
mock_kernel[0].execute("print('Hello, World')")
mock_kernel[0].execute("print('Cell number 2!')")
# Check lab book repo state
status = mock_labbook[2].git.status()
assert len(status["untracked"]) == 1
assert status["untracked"][0] == 'code/Test.ipynb'
# Process messages
msg1a = mock_kernel[0].get_iopub_msg()
msg2a = mock_kernel[0].get_iopub_msg()
msg3a = mock_kernel[0].get_iopub_msg()
msg4a = mock_kernel[0].get_iopub_msg()
msg1b = mock_kernel[0].get_iopub_msg()
msg2b = mock_kernel[0].get_iopub_msg()
msg3b = mock_kernel[0].get_iopub_msg()
msg4b = mock_kernel[0].get_iopub_msg()
# Process first state change message
assert monitor.kernel_status == 'idle'
assert monitor.can_store_activity_record is False
monitor.handle_message(msg1a)
assert monitor.kernel_status == 'busy'
# Process input message
monitor.handle_message(msg2a)
assert len(monitor.current_cell.code) > 0
assert len(monitor.cell_data) == 0
assert monitor.can_store_activity_record is False
# Process output message
monitor.handle_message(msg3a)
assert len(monitor.current_cell.result) > 0
assert len(monitor.cell_data) == 0
assert monitor.can_store_activity_record is False
# Process final state change message
monitor.handle_message(msg4a)
assert monitor.kernel_status == 'idle'
assert monitor.can_store_activity_record is True
assert len(monitor.cell_data) == 1
# Process first state change message
assert monitor.kernel_status == 'idle'
monitor.handle_message(msg1b)
assert monitor.can_store_activity_record is False
assert monitor.kernel_status == 'busy'
# Process input message
monitor.handle_message(msg2b)
assert len(monitor.current_cell.code) > 0
assert len(monitor.cell_data) == 1
assert monitor.can_store_activity_record is False
# Process output message
monitor.handle_message(msg3b)
assert len(monitor.current_cell.result) > 0
assert len(monitor.cell_data) == 1
assert monitor.can_store_activity_record is False
# Process final state change message
monitor.handle_message(msg4b)
assert monitor.kernel_status == 'idle'
assert monitor.can_store_activity_record is True
assert len(monitor.cell_data) == 2
# Store the record manually for this test
monitor.store_record(metadata)
assert monitor.can_store_activity_record is False
assert len(monitor.cell_data) == 0
# Check lab book repo state
status = mock_labbook[2].git.status()
assert len(status["untracked"]) == 0
assert len(status["staged"]) == 0
assert len(status["unstaged"]) == 0
# Check activity entry
log = mock_labbook[2].git.log()
assert len(log) == 4
assert 'code/Test.ipynb' in log[0]['message']
a_store = ActivityStore(mock_labbook[2])
record = a_store.get_activity_record(log[0]['commit'])
assert record.type == ActivityType.CODE
assert record.show is True
assert record.importance == 0
assert not record.tags
assert record.message == 'Executed cell in notebook code/Test.ipynb'
assert len(record._detail_objects) == 5
assert record._detail_objects[0][0] is True
assert record._detail_objects[0][1] == ActivityDetailType.RESULT.value
assert record._detail_objects[0][2] == 155
assert record._detail_objects[1][0] is True
assert record._detail_objects[1][1] == ActivityDetailType.RESULT.value
assert record._detail_objects[1][2] == 154
assert record._detail_objects[2][0] is False
assert record._detail_objects[2][1] == ActivityDetailType.CODE.value
assert record._detail_objects[2][2] == 255
assert record._detail_objects[3][0] is False
assert record._detail_objects[3][1] == ActivityDetailType.CODE_EXECUTED.value
assert record._detail_objects[3][2] == 255
assert record._detail_objects[4][0] is False
assert record._detail_objects[4][1] == ActivityDetailType.CODE_EXECUTED.value
assert record._detail_objects[4][2] == 254
def test_add_many_files(self, redis_client, mock_labbook, mock_kernel):
"""Test processing notebook activity when lots of output files have been created"""
for file_number in range(0, 260):
with open(os.path.join(mock_labbook[2].root_dir, 'output', f"{file_number}.dat"), 'wt') as tf:
tf.write("blah")
monitor_key = "dev_env_monitor:{}:{}:{}:{}:activity_monitor:{}".format('test',
'test',
'labbook1',
'jupyterlab-ubuntu1604',
uuid.uuid4())
monitor = JupyterLabNotebookMonitor("test", "test", mock_labbook[2].name,
monitor_key, config_file=mock_labbook[0])
# Setup monitoring metadata
metadata = {"kernel_id": "XXXX",
"kernel_name": 'python',
"kernel_type": 'notebook',
"path": 'code/Test.ipynb'}
# Perform an action
mock_kernel[0].execute("print('Generated 260 output files')")
# Check lab book repo state
status = mock_labbook[2].git.status()
assert len(status["untracked"]) == 260
# Process messages
msg1 = mock_kernel[0].get_iopub_msg()
msg2 = mock_kernel[0].get_iopub_msg()
msg3 = mock_kernel[0].get_iopub_msg()
msg4 = mock_kernel[0].get_iopub_msg()
# Process first state change message
assert monitor.kernel_status == 'idle'
assert monitor.can_store_activity_record is False
monitor.handle_message(msg1)
assert monitor.kernel_status == 'busy'
# Process input message
monitor.handle_message(msg2)
assert len(monitor.current_cell.code) > 0
assert len(monitor.cell_data) == 0
assert monitor.can_store_activity_record is False
# Process output message
monitor.handle_message(msg3)
assert len(monitor.current_cell.result) > 0
assert len(monitor.cell_data) == 0
assert monitor.can_store_activity_record is False
# Process final state change message
monitor.handle_message(msg4)
assert monitor.kernel_status == 'idle'
assert monitor.can_store_activity_record is True
assert len(monitor.cell_data) == 1
# Store the record manually for this test
monitor.store_record(metadata)
assert monitor.can_store_activity_record is False
assert len(monitor.cell_data) == 0
# Check lab book repo state
status = mock_labbook[2].git.status()
assert len(status["untracked"]) == 0
assert len(status["staged"]) == 0
assert len(status["unstaged"]) == 0
# Check activity entry
log = mock_labbook[2].git.log()
assert len(log) == 4
assert 'code/Test.ipynb' in log[0]['message']
a_store = ActivityStore(mock_labbook[2])
record = a_store.get_activity_record(log[0]['commit'])
assert record.type == ActivityType.CODE
assert record.show is True
assert record.importance == 0
assert not record.tags
assert record.message == 'Executed cell in notebook code/Test.ipynb'
assert len(record._detail_objects) == 256
assert record.num_detail_objects == 256
assert record._detail_objects[0][0] is True
assert record._detail_objects[0][1] == ActivityDetailType.NOTE.value
assert record._detail_objects[0][2] == 0
assert record._detail_objects[1][0] is True
assert record._detail_objects[1][1] == ActivityDetailType.RESULT.value
assert record._detail_objects[1][2] == 155
assert record._detail_objects[2][0] is False
assert record._detail_objects[2][1] == ActivityDetailType.CODE_EXECUTED.value
assert record._detail_objects[2][2] == 255
assert record._detail_objects[3][0] is False
assert record._detail_objects[3][1] == ActivityDetailType.OUTPUT_DATA.value
assert record._detail_objects[3][2] == 255
assert record._detail_objects[4][0] is False
assert record._detail_objects[4][1] == ActivityDetailType.OUTPUT_DATA.value
assert record._detail_objects[4][2] == 254
assert record._detail_objects[48][0] is False
assert record._detail_objects[48][1] == ActivityDetailType.OUTPUT_DATA.value
assert record._detail_objects[48][2] == 210
assert record._detail_objects[254][0] is False
assert record._detail_objects[254][1] == ActivityDetailType.OUTPUT_DATA.value
assert record._detail_objects[254][2] == 4
assert record._detail_objects[255][0] is False
assert record._detail_objects[255][1] == ActivityDetailType.OUTPUT_DATA.value
assert record._detail_objects[255][2] == 3
def test_gtm_comments(self, redis_client, mock_labbook, mock_kernel):
"""Test processing notebook activity that's modified by gtm: comments
Testing here is lighter than above - we only test aspects relevant to
showing / hiding/ ignoring"""
monitor_key = "dev_env_monitor:{}:{}:{}:{}:activity_monitor:{}".format('test',
'test',
'labbook1',
'jupyterlab-ubuntu1604',
uuid.uuid4())
monitor = JupyterLabNotebookMonitor("test", "test", mock_labbook[2].name,
monitor_key, config_file=mock_labbook[0])
# Setup monitoring metadata
metadata = {"kernel_id": "XXXX",
"kernel_name": 'python',
"kernel_type": 'notebook',
"path": 'code/Test.ipynb'}
# We'll use this repeatedly below
a_store = ActivityStore(mock_labbook[2])
# First, we check both cases for the default (auto)
# Perform an action that generates output (default shown)
mock_kernel[0].execute("# gtm:auto\nprint('Hello!')")
# Process messages
for _ in range(4):
monitor.handle_message(mock_kernel[0].get_iopub_msg())
# This is the one redundant assertion with other tests in this file (and it's repeated in each execution below),
# but it's fairly central to what we're doing!
assert monitor.can_store_activity_record is True
# Store the record manually for this test
monitor.store_record(metadata)
# Check activity entry
log = mock_labbook[2].git.log()
record = a_store.get_activity_record(log[0]['commit'])
assert record.show is True
# Perform an action that doesn't generate output (default hidden)
mock_kernel[0].execute("# gtm:auto\na=1")
# Process messages
for _ in range(3):
monitor.handle_message(mock_kernel[0].get_iopub_msg())
assert monitor.can_store_activity_record is True
# Store the record manually for this test
monitor.store_record(metadata)
# Check activity entry
log = mock_labbook[2].git.log()
record = a_store.get_activity_record(log[0]['commit'])
assert record.show is False
# Perform an action that generates output (default shown)
# Hide comment should override
mock_kernel[0].execute("# gtm:hide\nprint('Hello!')")
# Process messages
for _ in range(4):
monitor.handle_message(mock_kernel[0].get_iopub_msg())
assert monitor.can_store_activity_record is True
# Store the record manually for this test
monitor.store_record(metadata)
# Check activity entry
log = mock_labbook[2].git.log()
record = a_store.get_activity_record(log[0]['commit'])
assert record.show is False
# Perform an action that doesn't generate output (default hidden)
# Show comment should override
mock_kernel[0].execute("# gtm:show\na=1")
# Process messages
for _ in range(3):
monitor.handle_message(mock_kernel[0].get_iopub_msg())
assert monitor.can_store_activity_record is True
# Store the record manually for this test
monitor.store_record(metadata)
# Check activity entry
log = mock_labbook[2].git.log()
record = a_store.get_activity_record(log[0]['commit'])
assert record.show is True
# Finally, we test the ignore feature
# Here, the default shouldn't matter - we shouldn't end up with a new record
# Show comment should override
mock_kernel[0].execute("# gtm:ignore\na=1")
# Process messages
for _ in range(3):
monitor.handle_message(mock_kernel[0].get_iopub_msg())
assert monitor.can_store_activity_record is True
# Store the record manually for this test
monitor.store_record(metadata)
# Check activity entry
log = mock_labbook[2].git.log()
record = a_store.get_activity_record(log[0]['commit'])
# assert there is "nothing" in the activity record (we still generated one)
assert record.num_detail_objects == 0
def test_no_show(self, redis_client, mock_labbook, mock_kernel):
"""Test processing notebook activity that doesn't have any important detail items"""
monitor_key = "dev_env_monitor:{}:{}:{}:{}:activity_monitor:{}".format('test',
'test',
'labbook1',
'jupyterlab-ubuntu1604',
uuid.uuid4())
monitor = JupyterLabNotebookMonitor("test", "test", mock_labbook[2].name,
monitor_key, config_file=mock_labbook[0])
# Setup monitoring metadata
metadata = {"kernel_id": "XXXX",
"kernel_name": 'python',
"kernel_type": 'notebook',
"path": 'code/Test.ipynb'}
# Perform an action
mock_kernel[0].execute("a=1")
# Check lab book repo state
status = mock_labbook[2].git.status()
assert len(status["untracked"]) == 0
# Process messages
msg1 = mock_kernel[0].get_iopub_msg()
msg2 = mock_kernel[0].get_iopub_msg()
msg3 = mock_kernel[0].get_iopub_msg()
# Process first state change message
assert monitor.kernel_status == 'idle'
assert monitor.can_store_activity_record is False
monitor.handle_message(msg1)
assert monitor.kernel_status == 'busy'
# Process input message
monitor.handle_message(msg2)
assert len(monitor.current_cell.code) > 0
assert len(monitor.cell_data) == 0
assert monitor.can_store_activity_record is False
# Process output message
monitor.handle_message(msg3)
assert monitor.kernel_status == 'idle'
assert monitor.can_store_activity_record is True
assert len(monitor.cell_data) == 1
# Store the record manually for this test
monitor.store_record(metadata)
assert monitor.can_store_activity_record is False
assert len(monitor.cell_data) == 0
# Check activity entry
log = mock_labbook[2].git.log()
assert len(log) == 4
assert 'code/Test.ipynb' in log[0]['message']
a_store = ActivityStore(mock_labbook[2])
record = a_store.get_activity_record(log[0]['commit'])
assert record.type == ActivityType.CODE
assert record.show is False
assert record.importance == 0
assert not record.tags
assert record.message == 'Executed cell in notebook code/Test.ipynb'
assert len(record._detail_objects) == 1
assert record._detail_objects[0][0] is False
assert record._detail_objects[0][1] == ActivityDetailType.CODE_EXECUTED.value
assert record._detail_objects[0][2] == 255
def test_start_modify(self, redis_client, mock_labbook, mock_kernel):
"""Test processing notebook activity and have it modify an existing file & create some files"""
dummy_file = os.path.join(mock_labbook[2].root_dir, 'code', 'Test.ipynb')
dummy_output = os.path.join(mock_labbook[2].root_dir, 'output', 'result.bin')
with open(dummy_file, 'wt') as tf:
tf.write("Dummy file")
monitor_key = "dev_env_monitor:{}:{}:{}:{}:activity_monitor:{}".format('test',
'test',
'labbook1',
'jupyterlab-ubuntu1604',
uuid.uuid4())
monitor = JupyterLabNotebookMonitor("test", "test", mock_labbook[2].name,
monitor_key, config_file=mock_labbook[0])
# Setup monitoring metadata
metadata = {"kernel_id": "XXXX",
"kernel_name": 'python',
"kernel_type": 'notebook',
"path": 'code/Test.ipynb'}
# Perform an action
mock_kernel[0].execute("print('Hello, World')")
# Check lab book repo state
status = mock_labbook[2].git.status()
assert len(status["untracked"]) == 1
assert status["untracked"][0] == 'code/Test.ipynb'
# Process messages
msg1 = mock_kernel[0].get_iopub_msg()
msg2 = mock_kernel[0].get_iopub_msg()
msg3 = mock_kernel[0].get_iopub_msg()
msg4 = mock_kernel[0].get_iopub_msg()
# Process first state change message
assert monitor.kernel_status == 'idle'
assert monitor.can_store_activity_record is False
monitor.handle_message(msg1)
assert monitor.kernel_status == 'busy'
# Process input message
monitor.handle_message(msg2)
assert len(monitor.current_cell.code) > 0
assert len(monitor.cell_data) == 0
assert monitor.can_store_activity_record is False
# Process output message
monitor.handle_message(msg3)
assert len(monitor.current_cell.result) > 0
assert len(monitor.cell_data) == 0
assert monitor.can_store_activity_record is False
# Check lab book repo state
status = mock_labbook[2].git.status()
assert len(status["untracked"]) == 1
assert status["untracked"][0] == 'code/Test.ipynb'
# Process final state change message
monitor.handle_message(msg4)
assert monitor.kernel_status == 'idle'
assert monitor.can_store_activity_record is True
assert len(monitor.cell_data) == 1
# Store the record manually for this test
monitor.store_record(metadata)
assert monitor.can_store_activity_record is False
assert len(monitor.cell_data) == 0
# Check lab book repo state
status = mock_labbook[2].git.status()
assert len(status["untracked"]) == 0
assert len(status["staged"]) == 0
assert len(status["unstaged"]) == 0
# Check activity entry
log = mock_labbook[2].git.log()
assert len(log) == 4
assert 'code/Test.ipynb' in log[0]['message']
# Mock Performing an action AGAIN, faking editing the file and generating some output files
mock_kernel[0].execute("a=100\nprint('Hello, World 2')")
with open(dummy_file, 'wt') as tf:
tf.write("change the fake notebook")
with open(dummy_output, 'wt') as tf:
tf.write("some result data")
# Process messages
msg1 = mock_kernel[0].get_iopub_msg()
msg2 = mock_kernel[0].get_iopub_msg()
msg3 = mock_kernel[0].get_iopub_msg()
msg4 = mock_kernel[0].get_iopub_msg()
# Process first state change message
assert monitor.kernel_status == 'idle'
assert monitor.can_store_activity_record is False
monitor.handle_message(msg1)
assert monitor.kernel_status == 'busy'
# Process input message
monitor.handle_message(msg2)
assert len(monitor.current_cell.code) > 0
assert len(monitor.cell_data) == 0
assert monitor.can_store_activity_record is False
# Process output message
monitor.handle_message(msg3)
assert len(monitor.current_cell.result) > 0
assert len(monitor.cell_data) == 0
assert monitor.can_store_activity_record is False
# Check lab book repo state
status = mock_labbook[2].git.status()
assert len(status["staged"]) == 0
assert len(status["untracked"]) == 1
assert len(status["unstaged"]) == 1
assert status["unstaged"][0][0] == 'code/Test.ipynb'
assert status["unstaged"][0][1] == 'modified'
# Process final state change message
monitor.handle_message(msg4)
assert monitor.kernel_status == 'idle'
assert monitor.can_store_activity_record is True
assert len(monitor.cell_data) == 1
# Store the record manually for this test
monitor.store_record(metadata)
assert monitor.can_store_activity_record is False
assert len(monitor.cell_data) == 0
# Check lab book repo state
status = mock_labbook[2].git.status()
assert len(status["untracked"]) == 0
assert len(status["staged"]) == 0
assert len(status["unstaged"]) == 0
# Check activity entry
log = mock_labbook[2].git.log()
assert len(log) == 6
assert 'code/Test.ipynb' in log[0]['message']
a_store = ActivityStore(mock_labbook[2])
record = a_store.get_activity_record(log[0]['commit'])
assert record.type == ActivityType.CODE
assert record.show is True
assert record.importance == 0
assert not record.tags
assert record.message == 'Executed cell in notebook code/Test.ipynb'
assert len(record._detail_objects) == 4
assert record._detail_objects[0][0] is True
assert record._detail_objects[0][1] == ActivityDetailType.RESULT.value
assert record._detail_objects[0][2] == 155
assert record._detail_objects[1][0] is False
assert record._detail_objects[1][1] == ActivityDetailType.CODE.value
assert record._detail_objects[1][2] == 255
assert record._detail_objects[2][0] is False
assert record._detail_objects[2][1] == ActivityDetailType.CODE_EXECUTED.value
assert record._detail_objects[2][2] == 255
assert record._detail_objects[3][0] is False
assert record._detail_objects[3][1] == ActivityDetailType.OUTPUT_DATA.value
assert record._detail_objects[3][2] == 255
detail = a_store.get_detail_record(record._detail_objects[3][3].key)
assert len(detail.data) == 1
assert detail.data['text/markdown'] == 'Created new Output Data file `output/result.bin`'
detail = a_store.get_detail_record(record._detail_objects[1][3].key)
assert len(detail.data) == 1
assert detail.data['text/markdown'] == 'Modified Code file `code/Test.ipynb`'
