def _get_record_set(notes_directory,
file_path,
line_number,
parse=True,
parse_format='json',
include_config=False):
'''Get the full contents of a record set
If `parse` is set to False then the record set
contents are returned as a string
If `parse` is set to True then the record set
contents are loaded and returned in the specified
format. Valid formats are `json` and `csv`
'''
# Validate inputs
if parse and parse_format not in ['json', 'csv']:
raise ValueError(f'Unknown parse format {parse_format}')
# Get full path if only a relative path is supplied
if notes_directory not in file_path:
file_path = get_full_path(notes_directory, file_path)
record_set_lines = []
is_content = False
with open(file_path, 'r') as f:
file_data = f.read()
for idx, line in enumerate(file_data.split('\n')):
if idx + 1 < line_number:
continue
elif idx + 1 == line_number and line != '```rec-data':
raise ValueError(f'Found unexpected line "{line}"')
elif idx + 1 == line_number and line == '```rec-data':
is_content = True
elif is_content and line != '```':
record_set_lines.append(line)
continue
elif is_content and line == '```':
is_content = False
break
else:
raise ValueError('Found a bug in record set parsing logic!')
record_set_raw = '\n'.join(record_set_lines)
if parse:
record_set = load_from_string(record_set_raw)
if parse_format == 'json':
if include_config:
output = {
'records': list(record_set.all()),
'config': record_set.get_config(),
'fields': record_set.get_fields()
}
return output
output = record_set.get_json()
return output
elif parse_format == 'csv':
output = record_set.get_csv()
return output
else:
return record_set_raw
def test_loading_record_set(self):
'''Load a record set to test exports of in later tests
'''
with open('rec_data/export_test.rec', 'r') as f:
record_data = f.read()
record_set = load_from_string(record_data)
assert len(record_set.records) == 2
def test_valid_config_parsing(self):
with open('rec_data/valid_config.rec', 'r') as f:
valid_config_data = f.read()
split_config = re.split(r'#.*?\n', valid_config_data)
split_config = [config for config in split_config if config.strip()]
for valid_config in split_config:
record_set = load_from_string(valid_config)
assert record_set.config.keys()
def test_csv_export(self):
'''Test exporting a record set to CSV format
'''
with open('rec_data/export_test.rec', 'r') as f:
record_data = f.read()
record_set = load_from_string(record_data)
assert record_set.get_csv() == 'Id,A,B\r\n'\
'0,test,test\r\n'\
'1,test,test\r\n'
def test_invalid_config_parsing(self):
with open('rec_data/invalid_config.rec', 'r') as f:
invalid_config_data = f.read()
split_config = re.split(r'#.*?\n', invalid_config_data)
split_config = [config for config in split_config if config.strip()]
for invalid_config in split_config:
with pytest.raises(ValueError) as e:
_ = load_from_string(invalid_config)
assert str(e.value)
def test_valid_record_parsing(self):
with open('rec_data/valid_records.rec', 'r') as f:
valid_record_data = f.read()
valid_record_sets = re.split(r'#.*?\n', valid_record_data)
valid_record_sets = [
record_set for record_set in valid_record_sets
if record_set.strip()
]
for valid_record_set in valid_record_sets:
loaded_record_set = load_from_string(valid_record_set)
assert loaded_record_set.records
def test_rec_export(self):
'''Test exporting a record set to rec format
'''
# Test exporting with config included
# Test exporting with config excluded
with open('rec_data/export_test.rec', 'r') as f:
record_data = f.read()
record_set = load_from_string(record_data)
assert record_set.get_rec() == 'Id: 0\nA: test\nB: test\n\n'\
'Id: 1\nA: test\nB: test'
def test_invalid_record_parsing(self):
with open('rec_data/invalid_records.rec', 'r') as f:
invalid_record_data = f.read()
invalid_record_sets = re.split(r'#.*?\n', invalid_record_data)
invalid_record_sets = [
record_set for record_set in invalid_record_sets
if record_set.strip()
]
for invalid_record_set in invalid_record_sets:
with pytest.raises(ValueError) as e:
_ = load_from_string(invalid_record_set)
assert str(e.value)
def test_load_csv_with_pk(self):
'''Test importing data from a CSV file into a
record set with a primary key field set
'''
with open('rec_data/base_config_with_pk.rec', 'r') as f:
base_record_set_raw = f.read()
record_set = load_from_string(base_record_set_raw)
with open('rec_data/import_data.csv', 'r') as f:
csv_data = f.read()
record_set.insert_csv(csv_data)
assert len(record_set.records) == 5
def test_load_json_no_pk(self):
'''Test importing data from a JSON file into a
record set with no primary key field set
'''
with open('rec_data/base_config_no_pk.rec', 'r') as f:
base_record_set_raw = f.read()
record_set = load_from_string(base_record_set_raw)
with open('rec_data/import_data.json', 'r') as f:
csv_data = f.read()
record_set.insert_json(csv_data)
assert len(record_set.records) == 6
def test_json_export(self):
'''Test exporting a record set to JSON format
'''
with open('rec_data/export_test.rec', 'r') as f:
record_data = f.read()
record_set = load_from_string(record_data)
exported_record_set = [{
"Id": [0],
"A": ["test"],
"B": ["test"]
}, {
"Id": [1],
"A": ["test"],
"B": ["test"]
}]
self.assertCountEqual(json.loads(record_set.get_json()),
exported_record_set)
def get_rendered_markdown(markdown_content, note_path):
'''Pre-render all non-standard notes file
elements into HTML
markdown_content: Raw unprocessed note content
note_path: Relative path within the notes directory
of the markdown file being rendered
'''
html_content_lines = []
toc_content_lines = []
markdown_content_lines = markdown_content.split('\n')
is_fenced_code_block = False
is_diagram_block = False
is_rec_data_block = False
is_equation_block = False
rec_data_lines = []
for idx, markdown_line in enumerate(markdown_content_lines):
line_number = idx + 1
# Add whitespace at the beginning of the span line to
# match the content line indent level
span_whitespace = ''
leading_whitespace_match = leading_whitespace_regex.match(
markdown_line)
if leading_whitespace_match:
span_whitespace = leading_whitespace_match.group(0)
line_span = f'{span_whitespace}<span id="line-number-{line_number}"></span>'
# Grab everything for record sets
if markdown_line.strip()[:3] != '```' and is_rec_data_block:
rec_data_lines.append(markdown_line)
html_content_lines.append(line_span)
continue
# Special handling for diagram blocks
if is_diagram_block and '```' not in markdown_line:
if not markdown_line.strip():
# Skip empty lines
continue
else:
# Get rid of any indentation so it doesn't trip up the
# Markdown parser into creating new paragraphs which
# mess up mermaid
html_content_lines.append(markdown_line.strip())
continue
# Handle empty or pseudo-empty lines
if not markdown_line.strip():
html_content_lines.append(markdown_line)
continue
# Handle edges of fenced code blocks
if markdown_line.strip()[:3] == '```':
# Special handling for diagram blocks
if markdown_line.strip()[:10] == '```mermaid':
is_diagram_block = True
html_content_lines.append(line_span)
html_content_lines.append('<div class="mermaid">')
continue
elif is_diagram_block:
is_diagram_block = False
html_content_lines.append('</div>')
continue
# Special handling for record sets
if markdown_line.strip()[:11] == '```rec-data':
is_rec_data_block = True
continue
elif is_rec_data_block:
is_rec_data_block = False
record_set = load_from_string('\n'.join(rec_data_lines))
record_set_data = json.dumps(list(record_set.all()))
column_config = [{
'title': field,
'data': field,
'defaultContent': ''
} for field in record_set.get_fields()]
record_set_name = record_set.get_config().get('rec',
{}).get('name')
if record_set_name:
html_content_lines.append(f'##### Record Set: '
f'{record_set_name}')
record_set_html = f'<div><div class="record-set-data">'\
f'{record_set_data}'\
f'</div><table class="table table-striped '\
f'table-bordered record-set-table" '\
f'style="width:100%" data-rec=\'\' '\
f'data-cols='\
f'\'{json.dumps(column_config)}\'>'\
f'</table></div>'
html_content_lines.append(record_set_html)
html_content_lines.append(line_span)
rec_data_lines = []
continue
is_fenced_code_block = not is_fenced_code_block
html_content_lines.append(markdown_line)
continue
# Handle contents of fenced code blocks
if is_fenced_code_block:
html_content_lines.append(markdown_line)
continue
# Process internal links
markdown_line = internal_link_regex.sub(
lambda match: replace_link_path(match, note_path), markdown_line)
# Process internal images
markdown_line = image_regex.sub(
lambda match: rewrite_image_path(match, note_path), markdown_line)
if gps_regex.search(markdown_line):
markdown_line = gps_regex.sub(
'<location lat="\\g<2>" lon="\\g<4>">'
'<span class="location-name"><i class="bi-geo-fill"></i>\\g<6></span>'
'(<span class="location-coordinates">\\g<2>, \\g<4></span>)'
'</location>', markdown_line)
# Process All to-dos
if len(markdown_line) >= 4:
if markdown_line.strip()[:4] in ['[ ] ', '[X] ', '[S] '] or \
markdown_line.strip()[:3] == '[] ':
html_content_lines.append(line_span)
todo_element = get_todo_element(markdown_line)
html_content_lines.append(todo_element)
continue
# Process Questions & Answers
if len(markdown_line) >= 2:
if markdown_line.strip()[:2] in ['? ', '@ ']:
html_content_lines.append(line_span)
question_element = get_question_element(markdown_line)
html_content_lines.append(question_element)
continue
# Process Definitions
definition_match = definition_regex.match(markdown_line)
if definition_match:
html_content_lines.append(line_span)
definition_element = get_definition_element(
definition_match, markdown_line)
html_content_lines.append(definition_element)
continue
# Process Headings for Navigation
if markdown_line[0] == '#':
split_heading = markdown_line.split(' ', 1)
heading_level = len(split_heading[0])
element_id = split_heading[1].replace(' ', '-')
heading_div = f'<span id="{element_id}"></span>'
toc_markdown_line = f'{" " * (heading_level - 1)}- '\
f'[{split_heading[1]}](#{element_id})'
html_content_lines.append(line_span)
html_content_lines.append(heading_div)
html_content_lines.append(markdown_line)
toc_content_lines.append(toc_markdown_line)
continue
# Special handling for markdown tables
if markdown_line.lstrip()[0] == '|':
# Adding span tags into a table will break it, so we exclude them
html_content_lines.append(markdown_line)
continue
# Handle edges of equation blocks
if not is_equation_block and markdown_line.strip().startswith('$$') \
and not markdown_line.strip().endswith('$$'):
is_equation_block = True
html_content_lines.append(markdown_line)
continue
elif is_equation_block and not markdown_line.strip().startswith('$$') \
and markdown_line.strip().endswith('$$'):
is_equation_block = False
html_content_lines.append(markdown_line)
continue
elif is_equation_block and not markdown_line.strip().endswith('$$'):
html_content_lines.append(markdown_line)
continue
# Catch-all for everything else
html_content_lines.append(line_span)
html_content_lines.append(markdown_line)
html_content = '\n'.join(html_content_lines)
toc_content = '\n'.join(toc_content_lines)
return html_content, toc_content