def test_multiplecells(self, redis_client, mock_labbook):
"""Make sure that RStudio detects and splits cells"""
server_monitor = RStudioServerMonitor("test",
"test",
mock_labbook[2].name,
"foo:activity_monitor:73467b78",
config_file=mock_labbook[0])
mitmlog = open(
f"{os.path.dirname(os.path.realpath(__file__))}/73467b78.rserver.dump",
"rb")
# Read activity and return an aggregated activity record
server_monitor.process_activity(mitmlog)
# call processor
server_monitor.store_record()
a_store = ActivityStore(mock_labbook[2])
ars = a_store.get_activity_records()
# details object [x][3] gets the x^th object
cell_1 = a_store.get_detail_record(
ars[0]._detail_objects[2][3].key).data
cell_2 = a_store.get_detail_record(
ars[0]._detail_objects[3][3].key).data
# if the cells were divided, there will be two records
assert (cell_1['text/plain'][55:58] == 'pop')
assert (cell_2['text/plain'][200:204] == 'stan')
def test_code_and_image(self, redis_client, mock_labbook):
"""Test reading a log and storing a record"""
# create a server monitor
server_monitor = RStudioServerMonitor("test",
"test",
mock_labbook[2].name,
"foo:activity_monitor:52f5a3a9",
config_file=mock_labbook[0])
mitmlog = open(
f"{os.path.dirname(os.path.realpath(__file__))}/52f5a3a9.rserver.dump",
"rb")
# Read activity and return an aggregated activity record
server_monitor.process_activity(mitmlog)
# call processor
server_monitor.store_record()
a_store = ActivityStore(mock_labbook[2])
ars = a_store.get_activity_records()
# details object [x][3] gets the x^th object
code_dict = a_store.get_detail_record(
ars[0]._detail_objects[1][3].key).data
# check the code results
assert (code_dict['text/markdown'][101:109] == 'y("knitr')
# check part of an image
imgdata = a_store.get_detail_record(
ars[1]._detail_objects[1][3].key).data['image/png'][0:20]
assert (imgdata == '/9j/4AAQSkZJRgABAQAA')
def _load_detail_record(self, info):
"""Private method to load a detail record if it has not been previously loaded and set"""
if not self._detail_record:
# Load record from database
if not self.key:
raise ValueError(
"Must set `key` on object creation to resolve detail record"
)
# Load store instance
lb = self._get_loader(info).load(
f"{get_logged_in_username()}&{self.owner}&{self.name}").get()
store = ActivityStore(lb)
# Retrieve record
self._detail_record: ActivityDetailRecord = store.get_detail_record(
self.key)
# Set class properties
self.type = ActivityDetailTypeEnum.get(
self._detail_record.type.value).value
self.show = self._detail_record.show
self.tags = self._detail_record.tags
self.importance = self._detail_record.importance
self.action = ActivityActionTypeEnum.get(
self._detail_record.action.value).value
def test_start_modify(self, redis_client, mock_labbook, mock_kernel):
"""Test processing notebook activity and have it modify an existing file & create some files"""
dummy_file = os.path.join(mock_labbook[2].root_dir, 'code', 'Test.ipynb')
dummy_output = os.path.join(mock_labbook[2].root_dir, 'output', 'result.bin')
with open(dummy_file, 'wt') as tf:
tf.write("Dummy file")
monitor_key = "dev_env_monitor:{}:{}:{}:{}:activity_monitor:{}".format('test',
'test',
'labbook1',
'jupyterlab-ubuntu1604',
uuid.uuid4())
monitor = JupyterLabNotebookMonitor("test", "test", mock_labbook[2].name,
monitor_key, config_file=mock_labbook[0])
# Setup monitoring metadata
metadata = {"kernel_id": "XXXX",
"kernel_name": 'python',
"kernel_type": 'notebook',
"path": 'code/Test.ipynb'}
# Perform an action
mock_kernel[0].execute("print('Hello, World')")
# Check lab book repo state
status = mock_labbook[2].git.status()
assert len(status["untracked"]) == 1
assert status["untracked"][0] == 'code/Test.ipynb'
# Process messages
msg1 = mock_kernel[0].get_iopub_msg()
msg2 = mock_kernel[0].get_iopub_msg()
msg3 = mock_kernel[0].get_iopub_msg()
msg4 = mock_kernel[0].get_iopub_msg()
# Process first state change message
assert monitor.kernel_status == 'idle'
assert monitor.can_store_activity_record is False
monitor.handle_message(msg1)
assert monitor.kernel_status == 'busy'
# Process input message
monitor.handle_message(msg2)
assert len(monitor.current_cell.code) > 0
assert len(monitor.cell_data) == 0
assert monitor.can_store_activity_record is False
# Process output message
monitor.handle_message(msg3)
assert len(monitor.current_cell.result) > 0
assert len(monitor.cell_data) == 0
assert monitor.can_store_activity_record is False
# Check lab book repo state
status = mock_labbook[2].git.status()
assert len(status["untracked"]) == 1
assert status["untracked"][0] == 'code/Test.ipynb'
# Process final state change message
monitor.handle_message(msg4)
assert monitor.kernel_status == 'idle'
assert monitor.can_store_activity_record is True
assert len(monitor.cell_data) == 1
# Store the record manually for this test
monitor.store_record(metadata)
assert monitor.can_store_activity_record is False
assert len(monitor.cell_data) == 0
# Check lab book repo state
status = mock_labbook[2].git.status()
assert len(status["untracked"]) == 0
assert len(status["staged"]) == 0
assert len(status["unstaged"]) == 0
# Check activity entry
log = mock_labbook[2].git.log()
assert len(log) == 4
assert 'code/Test.ipynb' in log[0]['message']
# Mock Performing an action AGAIN, faking editing the file and generating some output files
mock_kernel[0].execute("a=100\nprint('Hello, World 2')")
with open(dummy_file, 'wt') as tf:
tf.write("change the fake notebook")
with open(dummy_output, 'wt') as tf:
tf.write("some result data")
# Process messages
msg1 = mock_kernel[0].get_iopub_msg()
msg2 = mock_kernel[0].get_iopub_msg()
msg3 = mock_kernel[0].get_iopub_msg()
msg4 = mock_kernel[0].get_iopub_msg()
# Process first state change message
assert monitor.kernel_status == 'idle'
assert monitor.can_store_activity_record is False
monitor.handle_message(msg1)
assert monitor.kernel_status == 'busy'
# Process input message
monitor.handle_message(msg2)
assert len(monitor.current_cell.code) > 0
assert len(monitor.cell_data) == 0
assert monitor.can_store_activity_record is False
# Process output message
monitor.handle_message(msg3)
assert len(monitor.current_cell.result) > 0
assert len(monitor.cell_data) == 0
assert monitor.can_store_activity_record is False
# Check lab book repo state
status = mock_labbook[2].git.status()
assert len(status["staged"]) == 0
assert len(status["untracked"]) == 1
assert len(status["unstaged"]) == 1
assert status["unstaged"][0][0] == 'code/Test.ipynb'
assert status["unstaged"][0][1] == 'modified'
# Process final state change message
monitor.handle_message(msg4)
assert monitor.kernel_status == 'idle'
assert monitor.can_store_activity_record is True
assert len(monitor.cell_data) == 1
# Store the record manually for this test
monitor.store_record(metadata)
assert monitor.can_store_activity_record is False
assert len(monitor.cell_data) == 0
# Check lab book repo state
status = mock_labbook[2].git.status()
assert len(status["untracked"]) == 0
assert len(status["staged"]) == 0
assert len(status["unstaged"]) == 0
# Check activity entry
log = mock_labbook[2].git.log()
assert len(log) == 6
assert 'code/Test.ipynb' in log[0]['message']
a_store = ActivityStore(mock_labbook[2])
record = a_store.get_activity_record(log[0]['commit'])
assert record.type == ActivityType.CODE
assert record.show is True
assert record.importance == 0
assert not record.tags
assert record.message == 'Executed cell in notebook code/Test.ipynb'
assert len(record._detail_objects) == 4
assert record._detail_objects[0][0] is True
assert record._detail_objects[0][1] == ActivityDetailType.RESULT.value
assert record._detail_objects[0][2] == 155
assert record._detail_objects[1][0] is False
assert record._detail_objects[1][1] == ActivityDetailType.CODE.value
assert record._detail_objects[1][2] == 255
assert record._detail_objects[2][0] is False
assert record._detail_objects[2][1] == ActivityDetailType.CODE_EXECUTED.value
assert record._detail_objects[2][2] == 255
assert record._detail_objects[3][0] is False
assert record._detail_objects[3][1] == ActivityDetailType.OUTPUT_DATA.value
assert record._detail_objects[3][2] == 255
detail = a_store.get_detail_record(record._detail_objects[3][3].key)
assert len(detail.data) == 1
assert detail.data['text/markdown'] == 'Created new Output Data file `output/result.bin`'
detail = a_store.get_detail_record(record._detail_objects[1][3].key)
assert len(detail.data) == 1
assert detail.data['text/markdown'] == 'Modified Code file `code/Test.ipynb`'
