def handle(self, *args, **options):
fn = options['data.yaml']
basedir = os.path.dirname(fn)
# Pre-process command-line arguments.
self.load_startapps_catalog(options)
# Open the end-user data file in a manner that we can.
# update it. If --init is used, then instantiate a
# new file for a new app.
default = None
if options["init"]:
# If --init is given, allow creating a new file.
default = {}
with rtyaml.edit(fn, default=default) as data:
if not isinstance(data, dict):
raise ValueError("File does not contain a YAML mapping.")
# If --init is given, it holds a path to an app.
# Construct a new YAML file with an empty organization
# block and an app pointing to the given app.
if options["init"]:
if data:
raise ValueError(
"--init cannot be used if file is not empty.")
data["organization"] = {"name": None}
data["app"] = os.path.relpath(
options["init"], basedir
) # compute path relative to the output file, not the current working directory
# Run the file.
self.AssembleApp(data, basedir, options)
def handle(self, *args, **options):
if not options["appsource"]:
# Show valid appsources.
print("Specify one of the following app sources plus an app name:")
for appsrc in AppSource.objects.all():
if appsrc.spec["type"] == "local" and appsrc.spec.get("path"):
print(appsrc.slug, "(path: " + appsrc.spec["path"] + ")")
print("")
print("Or create a new AppSource using '--path path/to/apps appsource'.")
elif options["path"]:
if options["appname"]:
print("You cannot use --path together with an appname.")
return
# Make directory if it doesn't exist.
os.makedirs(options["path"], exist_ok=True)
# Create a new AppSource.
appsrc = AppSource.objects.create(
slug=options["appsource"],
spec={ "type": "local", "path": options["path"] }
)
print("Created new AppSource", appsrc, "using local path", options["path"])
elif not options["appname"]:
print("You must specify an app name, which should be a valid directory name (i.e. no spaces), or or --path to create a new AppSource.")
else:
# Ok do the real work.
appsrc = AppSource.objects.get(slug=options["appsource"])
# Do we have a path?
if not appsrc.spec.get("path"):
print("AppSource does not have a local path specified!")
return
# What's the path to the app?
path = os.path.join(appsrc.spec["path"], options["appname"])
# Does this app already exist?
if os.path.exists(path):
print("An app with that name already exists.")
return
# Copy stub files.
guidedmodules_path = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
shutil.copytree(os.path.join(guidedmodules_path, "stubs", "q-files","stub_app"), path, copy_function=shutil.copy)
# Edit the app title.
with rtyaml.edit(os.path.join(path, "app.yaml")) as app:
app['title'] = options["appname"]
# Which AppSource is used?
print("Created new app in AppSource", appsrc, "at", path)
def handle(self, *args, **options):
if not options["appsource"]:
# Show valid appsources.
print("Specify one of the following app sources plus an app name:")
for appsrc in AppSource.objects.all():
if appsrc.spec["type"] == "local" and appsrc.spec.get("path"):
print(appsrc.slug, "(path: " + appsrc.spec["path"] + ")")
print("")
print(
"Or create a new AppSource using '--path path/to/apps appsource'."
)
elif options["path"]:
if options["appname"]:
print("You cannot use --path together with an appname.")
return
# Make directory if it doesn't exist.
os.makedirs(options["path"], exist_ok=True)
# Create a new AppSource.
appsrc = AppSource.objects.create(slug=options["appsource"],
spec={
"type": "local",
"path": options["path"]
})
print("Created new AppSource", appsrc, "using local path",
options["path"])
elif not options["appname"]:
print(
"You must specify an app name, which should be a valid directory name (i.e. no spaces), or or --path to create a new AppSource."
)
else:
# Ok do the real work.
appsrc = AppSource.objects.get(slug=options["appsource"])
# Do we have a path?
if not appsrc.spec.get("path"):
print("AppSource does not have a local path specified!")
return
# What's the path to the app?
path = os.path.join(appsrc.spec["path"], options["appname"])
# Does this app already exist?
if os.path.exists(path):
print("An app with that name already exists.")
return
# Copy stub files.
guidedmodules_path = os.path.dirname(
os.path.dirname(os.path.dirname(__file__)))
shutil.copytree(os.path.join(guidedmodules_path, "stub_app"),
path,
copy_function=shutil.copy)
# Edit the app title.
with rtyaml.edit(os.path.join(path, "app.yaml")) as app:
app['title'] = options["appname"]
# Create a unique icon for the app and delete the existing app icon
# svg file that we know is in the stub.
from mondrianish import generate_image
colors = ("#FFF8F0", "#FCAA67", "#7DB7C0", "#932b25", "#498B57")
with open(os.path.join(path, "assets", "app.png"), "wb") as f:
generate_image("png", (128, 128), 3, colors, f)
os.unlink(os.path.join(path, "assets", "app.svg"))
# Which AppSource is used?
print("Created new app in AppSource", appsrc, "at", path)
def get_file_content(component_fn, controls_fn):
controls_fn = os.path.join(os.path.dirname(component_fn), controls_fn)
with rtyaml.edit(controls_fn) as controls:
return controls.get("satisfies", [])
# The files in this example use some non-conformant changes to the
# OpenControl file formats. This script undoes those changes.
import glob
import os.path
import rtyaml
# Component files can list other files that hold control narratives.
# Put them back into the main component file.
def get_file_content(component_fn, controls_fn):
controls_fn = os.path.join(os.path.dirname(component_fn), controls_fn)
with rtyaml.edit(controls_fn) as controls:
return controls.get("satisfies", [])
for fn in glob.glob("components/*/component.yaml"):
with rtyaml.edit(fn) as component:
if "satisfies" in component:
satisfies = []
for item in component['satisfies']:
satisfies.extend(get_file_content(fn, item))
component['satisfies'] = satisfies
def loadsend_file(self, filename):
# Verify file can be opened.
if not os.path.exists(filename): raise FileNotFoundError(filename)
file_templ_vars = dict(
file_name=os.path.abspath(filename),
file_dir=os.path.abspath(os.path.dirname(filename)),
file_uuid1=uuid.uuid1(),
work_dir=self.opts.workdir,
)
log.info('Processing "%s"...', filename)
with rtyaml.edit(filename, default={}) as data:
query_results, defaults = self.query_notes(self.opts.query, data)
if query_results.empty:
log.warning("Query returned no results.")
else:
if self.opts.question:
log.info("")
log.info("Running in question mode.")
log.info("Query results:\n %s", str(query_results))
log.info("")
log.info("Query result details below.")
log.info("")
else:
if self.opts.annotations:
log.info("")
log.info("Running in annotations mode.")
log.info("")
log.debug("Query results:\n %s", str(query_results))
annotations_field = defaults['annotationsField']
for i in query_results.index:
rtnote = query_results.rtnote[i]
note_id = str(rtnote['id'])
skip = query_results.skip[i]
deck = query_results.deckName[i]
model = query_results.modelName[i]
use_md = query_results.useMarkdown[i]
md_sty = query_results.markdownStyle[i]
md_lineno = query_results.markdownLineNums[i]
md_tablen = query_results.markdownTabLength[i]
md_mathext = query_results.useMarkdownMathExt[i]
tags = sorted(query_results.tags[i].replace(',', '\n').split())
fields = query_results.fields[i]
media = query_results.media[i]
description = "{}:{}".format(note_id, fields)
if self.opts.question:
log.info("*** ID: {}".format(note_id))
log.info("--- Should skip: {}".format(skip))
log.info("--- Note tags:")
log.info(rtyaml.dump(tags))
log.info("--- Note annotations:")
log.info(rtyaml.dump(query_results.annotations[i]))
log.info("--- Note fields:")
log.info(rtyaml.dump(fields))
log.info("--- Raw fields:")
log.info(fields)
log.info("")
continue
if skip:
log.info("Skipping note with ID: {}".format(note_id))
continue
log.info("Processing note with ID: {}".format(note_id))
log.debug("Note fields: {}".format(fields))
# Check for note with given ID.
# Get info for existing note.
creating_new_note = True
note_info = self.anki.notesInfo([note_id])
if note_info.get("error", None) or not note_info['result'][0]:
if self.opts.annotations:
log.info(
"Can't find note with ID %s; skipping annotations for this note.",
note_id)
continue
else:
log.info(
"Can't find note with ID %s; a new note will be created.",
note_id)
else:
creating_new_note = False
note_templ_vars = dict(file_templ_vars)
note_templ_vars['note_id'] = note_id
note_uuid1 = uuid.uuid1()
note_templ_vars['note_uuid1'] = note_uuid1
file_uuid1 = note_templ_vars['file_uuid1']
if creating_new_note:
# No provided ID; assume new note should be created.
log.debug("Creating new note...")
temporary_fields = {
k: self.format_text(str(v), False, md_sty, md_lineno,
md_tablen, md_mathext,
**note_templ_vars)
for (k, v) in fields.items()
}
# Create, obtaining returned ID
anki_result = self.anki.addNote(deck,
model,
temporary_fields,
tags=tags)
if anki_result.get("error", None):
log.warning("Can't create note: %s", description)
else:
# Add ID to note_node
note_id = anki_result['result']
note_templ_vars['note_id'] = note_id
prev_id, rtnote['id'] = rtnote['id'], note_id
log.info("ID %s replaced with %s.", prev_id, note_id)
log.debug("Updating note...")
# Assume provided ID is valid for existing note to be updated.
# Convert each field from Markdown (if `use_md` is True).
# Special handling for the annotations field. If we can find this note
# in Anki's flashcard deck, then we'll grab any annotations the user has
# made for that note, and store them in the YAML file. If we are instead
# creating a new note, we'll transfer any annotations from the YAML file
# to the new note.
#
# Because of this special handling, we remove annotations from the
# shallow copy of the note. We'll instead use the round-trip version.
if self.opts.annotations:
pass
else:
if annotations_field in fields:
del fields[annotations_field]
converted_fields = {
k: self.format_text(str(v), use_md, md_sty, md_lineno,
md_tablen, md_mathext, field_no,
**note_templ_vars)
for (field_no, (k, v)) in enumerate(fields.items())
}
for media_item in media:
item_path = media_item['path']
item_path = string.Template(item_path).safe_substitute(
note_templ_vars)
item_name = media_item.get('name',
os.path.basename(item_path))
item_name = string.Template(item_name).safe_substitute(
note_templ_vars)
log.info("Considering sending media item...")
log.info(" local path: {}".format(item_path))
log.info(" remote name: {}".format(item_name))
anki_result = self.anki.statMediaFile(item_name)
must_send_new_media_item = False
item_data = None
if anki_result.get("error", None):
log.info(
"Can't get remote media file status (probably missing)..."
)
must_send_new_media_item = True
else:
if not anki_result['result']:
log.info(
"... Media item is not present on remote..."
)
must_send_new_media_item = True
else:
log.info(
"... Media item is already present on remote..."
)
log.info("... Reading local data...")
item_data = open(item_path, 'rb').read()
item_adler32 = zlib.adler32(item_data)
remote_adler32 = anki_result['result'][
'adler32']
log.info(" Remote checksum: {}".format(
remote_adler32))
log.info(" Local checksum: {}".format(
item_adler32))
if remote_adler32 == item_adler32:
log.info(
"... Remote checksum matches that of local version..."
)
else:
log.info(
"... Remote checksum is not the same as local..."
)
must_send_new_media_item = True
if must_send_new_media_item:
if item_data is None:
log.info("... Reading local data...")
item_data = open(item_path, 'rb').read()
log.info("... Encoding {} bytes of local data...".
format(len(item_data)))
item_base64 = base64.b64encode(item_data).decode(
"utf-8")
log.info(
"... Sending {} bytes of encoded data to remote..."
.format(len(item_base64)))
anki_result = self.anki.storeMediaFile(
item_name, item_base64)
if anki_result.get("error", None):
log.warning("Can't store media file: %s",
item_name)
log.info("... Done with media item.")
# If we found this note in Anki's flashcard deck, then we'll grab any
# annotations the user has made for that note, and store them in the
# YAML file. If we are instead creating a new note, we'll transfer any
# annotations from the YAML file to the new note.
if creating_new_note:
log.debug("Transferring annotations to new note...")
# Transfer annotations from YAML file to new note.
annotations = rtnote['fields'].get(annotations_field, "")
else:
log.debug("Transferring annotations from existing note...")
upstream_fields = note_info['result'][0]['fields']
annotations = upstream_fields.get(annotations_field,
dict(value=''))['value']
# Transfer annotations from existing note to YAML file.
rtnote['fields'][annotations_field] = annotations
if not self.opts.annotations:
converted_fields[annotations_field] = annotations
# Update converted note fields...
result = self.anki.updateNoteFields(
note_id, converted_fields)
if result.get("error", None):
log.warning("Can't update note: %s", description)
continue
# Update note tags...
## First get existing note tags.
note_info = self.anki.notesInfo([note_id])
if note_info.get("error", None):
log.warning("Can't get tags for note: %s", description)
continue
current_tags = sorted(note_info['result'][0]['tags'])
if current_tags != tags:
rt_non_annot_tags = set(
filter(lambda s: not s.startswith('ann:'),
rtnote.get('tags', list())))
non_annot_tags = set(
filter(lambda s: not s.startswith('ann:'), tags))
cur_non_annot_tags = set(
filter(lambda s: not s.startswith('ann:'),
current_tags))
cur_annot_tags = set(
filter(lambda s: s.startswith('ann:'), current_tags))
tags = sorted(list(non_annot_tags.union(cur_annot_tags)))
rt_tags = sorted(
list(rt_non_annot_tags.union(cur_annot_tags)))
rtnote['tags'] = rt_tags
## Remove existing note tags.
log.info("Removing tags %s...", cur_non_annot_tags)
result = self.anki.removeTags([note_id],
" ".join(cur_non_annot_tags))
if result.get("error", None):
log.warning("Can't remove tags for note: %s",
description)
## Add new note tags.
log.info("Replacing with tags %s...", tags)
result = self.anki.addTags([note_id], " ".join(tags))
if result.get("error", None):
log.warning("Can't add tags for note: %s", description)
note_info = self.anki.notesInfo([note_id])