Exemple #1
0
    def write(self):

        dockerfile = os.path.join(self.image_dir, 'Dockerfile')
        os.makedirs(self.image_dir, exist_ok=True)
        if os.path.exists(dockerfile):
            os.remove(dockerfile)
        with open(dockerfile, 'x'):
            pass
        snippets = []

        snippets.extend(self.snippets)
        snippets.append(
            Snippet(os.path.join(docker_path, 'snippets/workdir.in')))
        snippets.append(
            Snippet(os.path.join(docker_path, 'snippets/entrypoint.in')))

        for snippet in snippets:
            with open(dockerfile, 'a') as f:
                try:
                    f.write(snippet.substitute(self.substitutions))
                except Exception as e:
                    raise RuntimeError(
                        'Error creating Dockerfile for {}: {}'.format(
                            self.name, e))
        for file in self.files:
            shutil.copy(file, self.image_dir)
Exemple #2
0
def mixGPAll(snippets, params):
    logger.mixer("Starting mixGPAll mix method")
    combined = Snippet("combined")

    # copy a few things, assume first snippet has the defaults
    combined.optSteps = snippets[0].optSteps
    combined.learningRate = snippets[0].learningRate
    combined.lossTolerance = snippets[0].lossTolerance

    logger.mixer("Adding training data")
    # add training data
    for snippet in snippets:
        for tdata in snippet.data:
            combined.addData(tdata)

    # train the new snippet
    logger.mixer("Training new snippet")
    combined.train()

    # sample the stuff
    logger.mixer("Sampling new snippet")
    sampler = Rejection(combined, params["x0"], n=20)
    sampler.start()

    # this is threaded so we'll join then pull the results out
    sampler.join()
    return sampler.results
Exemple #3
0
def parse_wrap(nd):
    """This parses one section with class="wrap" (it builds a package) """
    # Title has package name and version
    title = ''
    a = nd.find('.//div/div/div/h2/a')
    if a is None or a.tail is None:
        return

    # Parse the section title
    title = a.tail.strip()
    title = re.sub('\s+', ' ', title)

    # Patch
    title = title.replace('xml::parser', 'xml-parser')

    # Extract build instructions
    snippets = []
    divs = nd.findall('./div')
    with_pkg = False
    sbu = sz = None
    for div in divs:
        # A section may have different (sub-)classes
        if 'class' in div.attrib:
            kl = div.attrib['class']
            if kl == 'package':
                sbu, sz = parse_package(div)
                # print(f'{title}, {sbu} SBU, {sz} GB')
                with_pkg = True
                continue
            # sect2 is used for the 8.4 grub section
            if kl not in ['installation', 'configuration', 'sect2']:
                continue
            for pre in div.findall('.//pre'):
                s = ''
                if pre.text:
                    t = pre.text.strip()
                    s += t
                if 'class' in pre.attrib:
                    kl = pre.attrib['class']
                    # Note: <pre class="root" has only been found in sect1 sections
                    if kl in ['userinput', 'root']:
                        # This is a code snippet
                        kbds = pre.findall('.//kbd[@class="command"]')
                        s += get_kbds_script(kbds)
                        if len(s) > 0:
                            snippets.append(Snippet('userinput', s))
                    elif kl == 'screen':
                        # This is some program output
                        s += get_em_code(pre)
                        if len(s) > 0:
                            snippets.append(Snippet('screen', s))

    # Only create section object if there are instructions
    if len(snippets) > 0:
        return Section(title, snippets, with_pkg=with_pkg, sbu=sbu, sz=sz)
Exemple #4
0
    def on_cell_edited(self, cell, path, new_text):
        if new_text != '':
            piter = self.model.get_iter(path)
            node = self.model.get_value(piter, self.SNIPPET_COLUMN)

            if node:
                if node == self.snippet.data:
                    s = self.snippet
                else:
                    s = Snippet(node)

                s['description'] = new_text
                self.snippet_changed(piter)
                self.select_iter(piter)
            else:
                # This is the `Add a new snippet...` item
                # We create a new snippet
                snippet = self.new_snippet({'description': new_text})

                if snippet:
                    self.model.set_value(piter, self.SNIPPET_COLUMN,
                                         snippet.data)
                    self.snippet_changed(piter)
                    self.snippet = snippet
                    self.selection_changed()
Exemple #5
0
    def new_snippet(self, properties=None):
        if not self.language_path:
            return None

        snippet = Library().new_snippet(self.get_language(self.language_path),
                                        properties)

        return Snippet(snippet)
Exemple #6
0
    def snippet_changed(self, piter=None):
        if piter:
            node = self.model.get_value(piter, self.SNIPPET_COLUMN)
            s = Snippet(node)
        else:
            s = self.snippet
            piter = self.find_iter(self.model.get_iter(self.language_path), s)

        if piter:
            nm = s.display()

            self.model.set_value(piter, self.NAME_COLUMN, nm)
            self.model.set_value(piter, self.SORT_COLUMN, nm)
            self.update_toolbar_buttons()
            self.entry_tab_trigger_update_valid()

        return piter
Exemple #7
0
        def snippet_changed(self, piter = None):
                if piter:
                        node = self.model.get_value(piter, self.SNIPPET_COLUMN)
                        s = Snippet(node)
                else:
                        s = self.snippet
                        piter = self.find_iter(self.model.get_iter(self.language_path), s)

                if piter:
                        nm = s.display()

                        self.model.set_value(piter, self.NAME_COLUMN, nm)
                        self.model.set_value(piter, self.SORT_COLUMN, nm)
                        self.update_toolbar_buttons()
                        self.entry_tab_trigger_update_valid()

                return piter
    def parse_snippets(self, path: str) -> Type[HeaderFile]:
        result = HeaderFile(path)
        snippet = None
        group = []
        for line_num, line in enumerate(readlines(path), 1):
            if line.startswith(BEGIN_TAG):
                dprint(f'Process {line}')
                key = line[len(BEGIN_TAG):].strip().strip('`')
                if snippet is not None:
                    raise Exception(f'Unexpected open section '
                                    f'({key}) {path}:{line_num} {line}')
                snippet = Snippet(key,
                                  enabled=key.startswith(ALWAYS_INCLUDE_TAG))
            elif line.startswith(END_TAG):
                dprint(f'Process {line}')
                key = line[len(END_TAG):].strip().strip('`')
                if snippet is None or snippet.tag != key:
                    raise Exception(f'Unexpected close section '
                                    f'({key}) {path}:{line_num} {line}')
                if key in self.snippets:
                    raise Exception(f'Duplicated key found '
                                    f'({key}) {path}:{line_num}')
                snippet.body = '\n'.join(group)
                self.snippets[snippet.tag] = snippet
                result.add_snippet(snippet)
                snippet = None
                group = []
            elif line.startswith(REQUIRED_TAG) and snippet is not None:
                for dependency in filter(
                        None,
                        map(str.strip, line[len(REQUIRED_TAG):].split(','))):
                    dprint(f'{snippet.tag} depends on `{dependency}`')
                    if dependency not in self.snippets:
                        raise Exception(f'Unknown dependency {dependency} '
                                        f'for snippet {snippet.tag}'
                                        f'({key}) {path}:{line_num}')
                    snippet.add_dependency(self.snippets[dependency])
            elif snippet is not None:
                group.append(line)

        if snippet is not None:
            raise Exception(f'Missed last closing tag {path} ({snippet.tag})')

        return result
Exemple #9
0
 def addSnippet(self, name, paramInfo=None):
     if name not in self.snippets:
         self.snippets[name] = Snippet(name, paramInfo)
         return DSStatus(code=0, message="Added Snippet: {0}".format(name))
     else:
         return DSStatus(
             code=-1,
             message="Failed to add Snippet with name {0}. Snippet already exists.".format(
                 name
             ),
         )
Exemple #10
0
 def __init__(self, name, snippets, files, substitutions):
     self.name = name
     self.snippets = [
         Snippet(os.path.join(docker_path, 'snippets', snippet + '.in'))
         for snippet in snippets
     ]
     self.substitutions = substitutions
     self.files = files
     self.status = None
     self.image_dir = os.path.join(images_dir, self.name)
     self.image_name = 'gulinux/planetaryimager_build:{}'.format(self.name)
Exemple #11
0
def recite(button):
    # TODO: randomly select unique Snippet
    # TODO: show reference
    # TODO: if practice, show contents
    global g_verse
    g_verse = Snippet.random()

    verse.set_text(g_verse.contents)
    reference.set_text(g_verse.reference)

    user_input.set_edit_text(u"")
Exemple #12
0
    def from_json_decoded(cls, obj):
        """Return an LFSBook object from a json-decoded object."""
        d = {}
        # We iterate on the members actually present, ignoring absent ones.
        for k, v in obj.items():
            d[k] = v

        # Properties with non-json-serializable values
        if 'snippets' in obj:
            d['snippets'] = [
                Snippet.from_json_decoded(x) for x in obj['snippets']
            ]

        return cls(**d)
Exemple #13
0
def parse_sect1(nd):
    # Title has package name and version
    title = ''
    a = nd.find('.//div/div/div/h2/a')
    if a is None or a.tail is None:
        return

    # Parse the section title
    title = a.tail.strip()
    title = re.sub('\s+', ' ', title)

    # Extract build instructions
    snippets = []
    for pre in nd.findall('.//pre'):
        s = ''
        if pre.text:
            t = pre.text.strip()
            s += t
        if 'class' in pre.attrib:
            kl = pre.attrib['class']
            # Note: <pre class="root" has only been found in sect1 sections
            if kl in ['userinput', 'root']:
                # This is a code snippet
                kbds = pre.findall('.//kbd[@class="command"]')
                s += get_kbds_script(kbds)
                if len(s) > 0:
                    snippets.append(Snippet('userinput', s))
            elif kl == 'screen':
                # This is some program output
                s += get_em_code(pre)
                if len(s) > 0:
                    snippets.append(Snippet('screen', s))

    # Only create section object if there are instructions
    if len(snippets) > 0:
        return Section(title, snippets, with_pkg=False)
Exemple #14
0
class Proposal(GObject.Object, GtkSource.CompletionProposal):
        __gtype_name__ = "GeditSnippetsProposal"

        def __init__(self, snippet):
                GObject.Object.__init__(self)
                self._snippet = Snippet(snippet)

        def snippet(self):
                return self._snippet.data

        # Interface implementation
        def do_get_markup(self):
                return self._snippet.display()

        def do_get_info(self):
                return self._snippet.data['text']
Exemple #15
0
    def on_tree_view_selection_changed(self, selection):
        parent, piter, node = self.selected_snippet()

        if self.snippet:
            self.on_entry_tab_trigger_focus_out(self['entry_tab_trigger'],
                                                None)
            self.on_source_view_snippet_focus_out(self['source_view_snippet'],
                                                  None)
            self.on_entry_drop_targets_focus_out(
                self['combo_drop_targets'].get_child(), None)

        self.update_language_path()

        if node:
            self.snippet = Snippet(node)
        else:
            self.snippet = None

        self.selection_changed()
Exemple #16
0
    def find_snippet(filename: str = Constants.PYTHON_SNIPPETS_FILE) -> List[Snippet]:
        with open(filename) as snippets_file:
            snippets_db = []
            snippet = Snippet()
            for line in snippets_file.readlines():
                if "snippet " in line:
                    if (snippet.name and snippet.prefix) is not None:
                        snippets_db.append(copy.deepcopy(snippet))
                        snippet.clean()

                    snippet.name = line.replace("snippet", "").lstrip().rstrip()
                    snippet.prefix = snippet.name.split()[0]
                    continue
                if line.find("#") == 0:
                    continue

                clean_line = line.replace("\t", "", 1).replace("\n", "")
                if line:
                    snippet.body.append(clean_line)
            return snippets_db
Exemple #17
0
    def __init__(self,
                 snippet_limit: int = 3,
                 take_words: int = 3,
                 results_limit: int = 10,
                 data_dir: str = 'pages') -> None:
        """
        This class is used for basic, no SQL searching.

        :param take_words: How many words to display is snippets before/after the found word.
        :param results_limit: How many rows to show when displaying top results with snippets.
        :param data_dir: Directory in which the HTML sites are saved relative to the current directory.
        """
        # nltk.download('punkt')
        # nltk.download('stopwords')
        self.time_needed_to_search = None
        self.take_words = take_words
        self.results_limit = results_limit
        self.data_dir = os.path.join(os.getcwd(), data_dir)
        self.path = data_dir
        self.snippet_limit = snippet_limit
        self.snip = Snippet(self.take_words, self.results_limit)

        self.stop_words_slovene = set(stopwords.words("Slovene")).union({
            "ter", "nov", "novo", "nova", "zato", "še", "zaradi", "a", "ali",
            "april", "avgust", "b", "bi", "bil", "bila", "bile", "bili",
            "bilo", "biti", "blizu", "bo", "bodo", "bojo", "bolj", "bom",
            "bomo", "boste", "bova", "boš", "brez", "c", "cel", "cela", "celi",
            "celo", "d", "da", "daleč", "dan", "danes", "datum", "december",
            "deset", "deseta", "deseti", "deseto", "devet", "deveta", "deveti",
            "deveto", "do", "dober", "dobra", "dobri", "dobro", "dokler",
            "dol", "dolg", "dolga", "dolgi", "dovolj", "drug", "druga",
            "drugi", "drugo", "dva", "dve", "e", "eden", "en", "ena", "ene",
            "eni", "enkrat", "eno", "etc.", "f", "februar", "g", "g.", "ga",
            "ga.", "gor", "gospa", "gospod", "h", "halo", "i", "idr.", "ii",
            "iii", "in", "iv", "ix", "iz", "j", "januar", "jaz", "je", "ji",
            "jih", "jim", "jo", "julij", "junij", "jutri", "k", "kadarkoli",
            "kaj", "kajti", "kako", "kakor", "kamor", "kamorkoli", "kar",
            "karkoli", "katerikoli", "kdaj", "kdo", "kdorkoli", "ker", "ki",
            "kje", "kjer", "kjerkoli", "ko", "koder", "koderkoli", "koga",
            "komu", "kot", "kratek", "kratka", "kratke", "kratki", "l",
            "lahka", "lahke", "lahki", "lahko", "le", "lep", "lepa", "lepe",
            "lepi", "lepo", "leto", "m", "maj", "majhen", "majhna", "majhni",
            "malce", "malo", "manj", "marec", "me", "med", "medtem", "mene",
            "mesec", "mi", "midva", "midve", "mnogo", "moj", "moja", "moje",
            "mora", "morajo", "moram", "moramo", "morate", "moraš", "morem",
            "mu", "n", "na", "nad", "naj", "najina", "najino", "najmanj",
            "naju", "največ", "nam", "narobe", "nas", "nato", "nazaj", "naš",
            "naša", "naše", "ne", "nedavno", "nedelja", "nek", "neka", "nekaj",
            "nekatere", "nekateri", "nekatero", "nekdo", "neke", "nekega",
            "neki", "nekje", "neko", "nekoga", "nekoč", "ni", "nikamor",
            "nikdar", "nikjer", "nikoli", "nič", "nje", "njega", "njegov",
            "njegova", "njegovo", "njej", "njemu", "njen", "njena", "njeno",
            "nji", "njih", "njihov", "njihova", "njihovo", "njiju", "njim",
            "njo", "njun", "njuna", "njuno", "no", "nocoj", "november", "npr.",
            "o", "ob", "oba", "obe", "oboje", "od", "odprt", "odprta",
            "odprti", "okoli", "oktober", "on", "onadva", "one", "oni",
            "onidve", "osem", "osma", "osmi", "osmo", "oz.", "p", "pa", "pet",
            "peta", "petek", "peti", "peto", "po", "pod", "pogosto", "poleg",
            "poln", "polna", "polni", "polno", "ponavadi", "ponedeljek",
            "ponovno", "potem", "povsod", "pozdravljen", "pozdravljeni",
            "prav", "prava", "prave", "pravi", "pravo", "prazen", "prazna",
            "prazno", "prbl.", "precej", "pred", "prej", "preko", "pri",
            "pribl.", "približno", "primer", "pripravljen", "pripravljena",
            "pripravljeni", "proti", "prva", "prvi", "prvo", "r", "ravno",
            "redko", "res", "reč", "s", "saj", "sam", "sama", "same", "sami",
            "samo", "se", "sebe", "sebi", "sedaj", "sedem", "sedma", "sedmi",
            "sedmo", "sem", "september", "seveda", "si", "sicer", "skoraj",
            "skozi", "slab", "smo", "so", "sobota", "spet", "sreda", "srednja",
            "srednji", "sta", "ste", "stran", "stvar", "sva", "t", "ta", "tak",
            "taka", "take", "taki", "tako", "takoj", "tam", "te", "tebe",
            "tebi", "tega", "težak", "težka", "težki", "težko", "ti", "tista",
            "tiste", "tisti", "tisto", "tj.", "tja", "to", "toda", "torek",
            "tretja", "tretje", "tretji", "tri", "tu", "tudi", "tukaj", "tvoj",
            "tvoja", "tvoje", "u", "v", "vaju", "vam", "vas", "vaš", "vaša",
            "vaše", "ve", "vedno", "velik", "velika", "veliki", "veliko",
            "vendar", "ves", "več", "vi", "vidva", "vii", "viii", "visok",
            "visoka", "visoke", "visoki", "vsa", "vsaj", "vsak", "vsaka",
            "vsakdo", "vsake", "vsaki", "vsakomur", "vse", "vsega", "vsi",
            "vso", "včasih", "včeraj", "x", "z", "za", "zadaj", "zadnji",
            "zakaj", "zaprta", "zaprti", "zaprto", "zdaj", "zelo", "zunaj",
            "č", "če", "često", "četrta", "četrtek", "četrti", "četrto", "čez",
            "čigav", "š", "šest", "šesta", "šesti", "šesto", "štiri", "ž",
            "že", "svoj", "jesti", "imeti", "\u0161e", "iti", "kak", "www",
            "km", "eur", "pač", "del", "kljub", "šele", "prek", "preko",
            "znova", "morda", "kateri", "katero", "katera", "ampak", "lahek",
            "lahka", "lahko", "morati", "torej", "gl", "xsd", "ipd", "om",
            "gt", "lt", "d.o.o"
        })
Exemple #18
0
class BasicSearch:
    def __init__(self,
                 snippet_limit: int = 3,
                 take_words: int = 3,
                 results_limit: int = 10,
                 data_dir: str = 'pages') -> None:
        """
        This class is used for basic, no SQL searching.

        :param take_words: How many words to display is snippets before/after the found word.
        :param results_limit: How many rows to show when displaying top results with snippets.
        :param data_dir: Directory in which the HTML sites are saved relative to the current directory.
        """
        # nltk.download('punkt')
        # nltk.download('stopwords')
        self.time_needed_to_search = None
        self.take_words = take_words
        self.results_limit = results_limit
        self.data_dir = os.path.join(os.getcwd(), data_dir)
        self.path = data_dir
        self.snippet_limit = snippet_limit
        self.snip = Snippet(self.take_words, self.results_limit)

        self.stop_words_slovene = set(stopwords.words("Slovene")).union({
            "ter", "nov", "novo", "nova", "zato", "še", "zaradi", "a", "ali",
            "april", "avgust", "b", "bi", "bil", "bila", "bile", "bili",
            "bilo", "biti", "blizu", "bo", "bodo", "bojo", "bolj", "bom",
            "bomo", "boste", "bova", "boš", "brez", "c", "cel", "cela", "celi",
            "celo", "d", "da", "daleč", "dan", "danes", "datum", "december",
            "deset", "deseta", "deseti", "deseto", "devet", "deveta", "deveti",
            "deveto", "do", "dober", "dobra", "dobri", "dobro", "dokler",
            "dol", "dolg", "dolga", "dolgi", "dovolj", "drug", "druga",
            "drugi", "drugo", "dva", "dve", "e", "eden", "en", "ena", "ene",
            "eni", "enkrat", "eno", "etc.", "f", "februar", "g", "g.", "ga",
            "ga.", "gor", "gospa", "gospod", "h", "halo", "i", "idr.", "ii",
            "iii", "in", "iv", "ix", "iz", "j", "januar", "jaz", "je", "ji",
            "jih", "jim", "jo", "julij", "junij", "jutri", "k", "kadarkoli",
            "kaj", "kajti", "kako", "kakor", "kamor", "kamorkoli", "kar",
            "karkoli", "katerikoli", "kdaj", "kdo", "kdorkoli", "ker", "ki",
            "kje", "kjer", "kjerkoli", "ko", "koder", "koderkoli", "koga",
            "komu", "kot", "kratek", "kratka", "kratke", "kratki", "l",
            "lahka", "lahke", "lahki", "lahko", "le", "lep", "lepa", "lepe",
            "lepi", "lepo", "leto", "m", "maj", "majhen", "majhna", "majhni",
            "malce", "malo", "manj", "marec", "me", "med", "medtem", "mene",
            "mesec", "mi", "midva", "midve", "mnogo", "moj", "moja", "moje",
            "mora", "morajo", "moram", "moramo", "morate", "moraš", "morem",
            "mu", "n", "na", "nad", "naj", "najina", "najino", "najmanj",
            "naju", "največ", "nam", "narobe", "nas", "nato", "nazaj", "naš",
            "naša", "naše", "ne", "nedavno", "nedelja", "nek", "neka", "nekaj",
            "nekatere", "nekateri", "nekatero", "nekdo", "neke", "nekega",
            "neki", "nekje", "neko", "nekoga", "nekoč", "ni", "nikamor",
            "nikdar", "nikjer", "nikoli", "nič", "nje", "njega", "njegov",
            "njegova", "njegovo", "njej", "njemu", "njen", "njena", "njeno",
            "nji", "njih", "njihov", "njihova", "njihovo", "njiju", "njim",
            "njo", "njun", "njuna", "njuno", "no", "nocoj", "november", "npr.",
            "o", "ob", "oba", "obe", "oboje", "od", "odprt", "odprta",
            "odprti", "okoli", "oktober", "on", "onadva", "one", "oni",
            "onidve", "osem", "osma", "osmi", "osmo", "oz.", "p", "pa", "pet",
            "peta", "petek", "peti", "peto", "po", "pod", "pogosto", "poleg",
            "poln", "polna", "polni", "polno", "ponavadi", "ponedeljek",
            "ponovno", "potem", "povsod", "pozdravljen", "pozdravljeni",
            "prav", "prava", "prave", "pravi", "pravo", "prazen", "prazna",
            "prazno", "prbl.", "precej", "pred", "prej", "preko", "pri",
            "pribl.", "približno", "primer", "pripravljen", "pripravljena",
            "pripravljeni", "proti", "prva", "prvi", "prvo", "r", "ravno",
            "redko", "res", "reč", "s", "saj", "sam", "sama", "same", "sami",
            "samo", "se", "sebe", "sebi", "sedaj", "sedem", "sedma", "sedmi",
            "sedmo", "sem", "september", "seveda", "si", "sicer", "skoraj",
            "skozi", "slab", "smo", "so", "sobota", "spet", "sreda", "srednja",
            "srednji", "sta", "ste", "stran", "stvar", "sva", "t", "ta", "tak",
            "taka", "take", "taki", "tako", "takoj", "tam", "te", "tebe",
            "tebi", "tega", "težak", "težka", "težki", "težko", "ti", "tista",
            "tiste", "tisti", "tisto", "tj.", "tja", "to", "toda", "torek",
            "tretja", "tretje", "tretji", "tri", "tu", "tudi", "tukaj", "tvoj",
            "tvoja", "tvoje", "u", "v", "vaju", "vam", "vas", "vaš", "vaša",
            "vaše", "ve", "vedno", "velik", "velika", "veliki", "veliko",
            "vendar", "ves", "več", "vi", "vidva", "vii", "viii", "visok",
            "visoka", "visoke", "visoki", "vsa", "vsaj", "vsak", "vsaka",
            "vsakdo", "vsake", "vsaki", "vsakomur", "vse", "vsega", "vsi",
            "vso", "včasih", "včeraj", "x", "z", "za", "zadaj", "zadnji",
            "zakaj", "zaprta", "zaprti", "zaprto", "zdaj", "zelo", "zunaj",
            "č", "če", "često", "četrta", "četrtek", "četrti", "četrto", "čez",
            "čigav", "š", "šest", "šesta", "šesti", "šesto", "štiri", "ž",
            "že", "svoj", "jesti", "imeti", "\u0161e", "iti", "kak", "www",
            "km", "eur", "pač", "del", "kljub", "šele", "prek", "preko",
            "znova", "morda", "kateri", "katero", "katera", "ampak", "lahek",
            "lahka", "lahko", "morati", "torej", "gl", "xsd", "ipd", "om",
            "gt", "lt", "d.o.o"
        })

    def print_results(self, query: list, results: list) -> None:
        """
        Prints the results of the query to the std output

        :param query: The user search query in list format.
        :param results: A list of lists containing the results in format [frequencies, document, snippets].
        The list should be ordered in descending order.
        """
        print(f'Results for query: "{" ".join(query)}"')
        print('{} results found in {:.0f}s'.format(len(results),
                                                   self.time_needed_to_search))
        print("{:<12} {:<40} {}".format('Frequencies', 'Document', 'Snippets'))
        print("{} {} {}".format('-' * 12, '-' * 40, '-' * 80))
        for i in range(min(self.results_limit, len(results))):
            print("{:<12} {:<40} {}".format(
                results[i][0], results[i][1], '... ' +
                ' ... '.join(results[i][2][:self.snippet_limit]) + ' ...'))

    def get_files(self) -> list:
        """
        Gets a list of paths to all HTML files which are contained in the self.path folder.

        :return: List of all HTML files.
        """
        files_path = [
            os.path.join(dp, f) for dp, dn, filenames in os.walk(self.path)
            for f in filenames if os.path.splitext(f)[1].lower() == '.html'
        ]
        return files_path

    def get_document(self, page_name: str) -> object:
        """
        Gets and opens the provided document.

        :param page_name: The HTML document that should be opened for reading.
        :return: The opened document.
        """
        map_name = '.'.join(page_name.split('.')[:3])
        return open(os.path.join(self.data_dir,
                                 "{}/{}".format(map_name, page_name)),
                    'r',
                    encoding='utf-8')

    def parse_file(self, file: str, search_words: list) -> tuple:
        """
        Searches the HTML document for the words in the query and returns the frequency and snippets.

        :param file: The HTML file to search.
        :param search_words: The user search query.
        :return: A touple with frequencies and result snippets.
        """
        # open the file
        f = codecs.open(file, 'r', 'utf-8')
        soup = BeautifulSoup(f.read(), features="html.parser")
        for script in soup(["script", "style"]):
            script.decompose()

        # tokenize and clean file
        strips = list(soup.stripped_strings)
        document = ' '.join(strips)
        tokens = word_tokenize(document,
                               language='Slovene',
                               preserve_line=False)
        indexes = list()
        for i in range(len(tokens)):
            token = tokens[i].lower().replace("'",
                                              "").replace("'", '').replace(
                                                  '`', '').replace('·', '')
            # skip if token doesn't contain letters
            if not re.search('[a-žA-ž]', token):
                continue
            # skip tokens of length 1, if they're not a word or number
            if len(token) == 1 and not re.match("^[A-Ža-ž0-9]*$", token):
                continue
            # if token ends with a special character, remove it
            if len(token) >= 2 and not re.match("^[A-Ža-ž0-9]*$", token[-1]):
                token = token[:-1]
            # if token starts with a special character, remove it
            if len(token) >= 2 and not re.match("^[A-Ža-ž0-9]*$", token[0]):
                token = token[1:]
            # if token is a stop-word, continue
            if token not in self.stop_words_slovene:
                # if we're searching for this token, add the index to the list
                if token in search_words:
                    indexes.append(i)
        try:
            # snippet = self.get_snip(tokens, indexes)
            snippet = self.snip.get_snip(tokens, indexes)
        except AssertionError:
            snippet = []
        return len(indexes), snippet

    def search(self, words: list) -> None:
        """
        Performs the search in all files for the user query.

        :param words: The user search query.
        """
        results = []
        # start the timer and do the search, after that calculate the time needed for search
        start_time = time.time()
        files = self.get_files()
        for file in files:
            f, s = self.parse_file(file, words)
            if f > 0:
                results.append([f, ntpath.split(file)[1], s])
        end_time = time.time()
        self.time_needed_to_search = end_time - start_time
        results.sort(key=lambda x: x[0], reverse=True)
        self.print_results(words, results)
Exemple #19
0
def musicThumbnailing(x,
                      Fs,
                      shortTermSize=1.0,
                      shortTermStep=0.5,
                      thumbnailSize=10.0,
                      Limit1=0,
                      Limit2=1):
    # self-similarity matrix
    x = audioBasicIO.stereo2mono(x)
    stFeatures = aF.stFeatureExtraction(x, Fs, Fs * shortTermSize,
                                        Fs * shortTermStep)
    S = aS.selfSimilarityMatrix(stFeatures)

    # moving filter:
    M = int(round(thumbnailSize / shortTermStep))
    B = np.eye(M, M)
    S = scipy.signal.convolve2d(S, B, 'valid')

    MIN = np.min(S)
    # post-processing (remove main diagonal elements)
    for i in range(S.shape[0]):
        for j in range(S.shape[1]):
            if abs(i - j) < 5.0 / shortTermStep or i > j:
                S[i, j] = MIN

    # find max position:
    S[0:int(Limit1 * S.shape[0]), :] = MIN
    S[:, 0:int(Limit1 * S.shape[0])] = MIN
    S[int(Limit2 * S.shape[0])::, :] = MIN
    S[:, int(Limit2 * S.shape[0])::] = MIN

    matches = []
    maxMax = maxVal = np.max(S)
    i1 = i2 = j1 = j2 = 0
    Sbak = np.copy(S)

    while maxVal > maxMax / 3 * 2 > MIN:  # currently arbitrary cutoff
        [I, J] = np.unravel_index(S.argmax(), S.shape)

        # expand:
        i1 = I
        i2 = I
        j1 = J
        j2 = J

        while i2 - i1 < M:
            if i1 <= 0 or j1 <= 0 or i2 >= S.shape[0] - 2 or j2 >= S.shape[
                    1] - 2:
                break
            if S[i1 - 1, j1 - 1] > S[i2 + 1, j2 + 1]:
                i1 -= 1
                j1 -= 1
            else:
                i2 += 1
                j2 += 1
            S[i1, j1] = S[i2, j2] = MIN

        # only add to potential matches if we have enough overlap or new record
        if i2 - i1 >= M:
            matches.append(
                Snippet(maxVal, shortTermStep * i1, shortTermStep * i2,
                        shortTermStep * j1, shortTermStep * j2))

        S[I, J] = MIN
        maxVal = np.max(S)

    return (matches, Sbak)
Exemple #20
0
def fetch_snippets(num_snippets, start_time, end_time, extra_tags):
    """
    Fetches snippets from StackOverflow by looking at 'python' questions
    in the time-period specified, and retrieving `<pre><code>` code blocks
    from these questions (and their answers, where applicable)

    Note that currently the number of snippets returned is not the same as
    the number requested: it depends on the number of retrieved questions with
    a code block, as well as the number with an accepted answer with a code block
    """
    filters = load_filter_file()

    snippets = []
    answer_ids = []

    questions_retrieved = 0
    page_num = 1
    while questions_retrieved < num_snippets:
        questions, has_more = fetch_recent_questions(
            num_questions=(num_snippets - questions_retrieved),
            from_time=start_time,
            to_time=end_time,
            tags=extra_tags,
            page_num=page_num,
            filter_name=filters.Questions)
        current_time = datetime.utcnow()
        for q in questions:
            snippets += [
                Snippet(snippet_id=q['question_id'],
                        code=block,
                        url=q['link'],
                        author='stack-overflow',
                        retrieved_at=current_time,
                        additional_url=None)
                for block in get_snippets(q['body'])
                if check_source_and_warn(block, q['link'])
            ]
            answer_ids.append(q.get('accepted_answer_id', None))

        questions_retrieved += len(questions)
        page_num += 1
        if not has_more:
            warn('No more questions to fetch: Terminating')
            break

    # Filter out `None` accepted_answer_id's
    answer_ids = [str(ans_id) for ans_id in answer_ids if ans_id]

    log('Retrieving {} accepted answers for analysis...'.format(
        len(answer_ids)))
    for i in range(0, len(answer_ids), ANSWER_BATCH_SIZE):
        batch = answer_ids[i:i + ANSWER_BATCH_SIZE]
        answers = fetch_answers(batch, filters.Answers)
        for a in answers:
            snippets += [
                Snippet(snippet_id=a['answer_id'],
                        code=block,
                        url=a['link'],
                        author='stack-overflow',
                        retrieved_at=current_time,
                        additional_url=None)
                for block in get_snippets(a['body'])
                if check_source_and_warn(block, q['link'])
            ]

    success('Retrieved {} snippets'.format(len(snippets)))
    return snippets
Exemple #21
0
        def apply_snippet(self, snippet, start = None, end = None):
                if not snippet.valid:
                        return False

                buf = self.view.get_buffer()
                s = Snippet(snippet)

                if not start:
                        start = buf.get_iter_at_mark(buf.get_insert())

                if not end:
                        end = buf.get_iter_at_mark(buf.get_selection_bound())

                if start.equal(end) and self.uses_current_word(s):
                        # There is no tab trigger and no selection and the snippet uses
                        # the current word. Set start and end to the word boundary so that
                        # it will be removed
                        start, end = buffer_word_boundary(buf)
                elif start.equal(end) and self.uses_current_line(s):
                        # There is no tab trigger and no selection and the snippet uses
                        # the current line. Set start and end to the line boundary so that
                        # it will be removed
                        start, end = buffer_line_boundary(buf)

                # Set environmental variables
                self.update_environment()

                # You know, we could be in an end placeholder
                (current, next) = self.next_placeholder()
                if current and current.__class__ == PlaceholderEnd:
                        self.goto_placeholder(current, None)

                buf.begin_user_action()

                # Remove the tag, selection or current word
                buf.delete(start, end)

                # Insert the snippet
                holders = len(self.placeholders)

                if len(self.active_snippets) == 0:
                        self.first_snippet_inserted()

                sn = s.insert_into(self, start)
                self.active_snippets.append(sn)

                # Put cursor at first tab placeholder
                keys = filter(lambda x: x > 0, sn.placeholders.keys())

                if len(keys) == 0:
                        if 0 in sn.placeholders:
                                self.goto_placeholder(self.active_placeholder, sn.placeholders[0])
                        else:
                                buf.place_cursor(sn.begin_iter())
                else:
                        self.goto_placeholder(self.active_placeholder, sn.placeholders[keys[0]])

                if sn in self.active_snippets:
                        # Check if we can get end_iter in view without moving the
                        # current cursor position out of view
                        cur = buf.get_iter_at_mark(buf.get_insert())
                        last = sn.end_iter()

                        curloc = self.view.get_iter_location(cur)
                        lastloc = self.view.get_iter_location(last)

                        if (lastloc.y + lastloc.height) - curloc.y <= \
                           self.view.get_visible_rect().height:
                                self.view.scroll_mark_onscreen(sn.end_mark)

                buf.end_user_action()
                self.view.grab_focus()

                return True
Exemple #22
0
    def apply_snippet(self, snippet, start=None, end=None, environ={}):
        if not snippet.valid:
            return False

        # Set environmental variables
        env = self.get_environment()

        if environ:
            for k in environ['utf8']:
                env['utf8'][k] = environ['utf8'][k]

            for k in environ['noenc']:
                env['noenc'][k] = environ['noenc'][k]

        buf = self.view.get_buffer()
        s = Snippet(snippet, env)

        if not start:
            start = buf.get_iter_at_mark(buf.get_insert())

        if not end:
            end = buf.get_iter_at_mark(buf.get_selection_bound())

        if start.equal(end) and self.uses_current_word(s):
            # There is no tab trigger and no selection and the snippet uses
            # the current word. Set start and end to the word boundary so that
            # it will be removed
            start, end = buffer_word_boundary(buf)
        elif start.equal(end) and self.uses_current_line(s):
            # There is no tab trigger and no selection and the snippet uses
            # the current line. Set start and end to the line boundary so that
            # it will be removed
            start, end = buffer_line_boundary(buf)

        # You know, we could be in an end placeholder
        (current, next) = self.next_placeholder()
        if current and current.__class__ == PlaceholderEnd:
            self.goto_placeholder(current, None)

        buf.begin_user_action()

        # Remove the tag, selection or current word
        buf.delete(start, end)

        # Insert the snippet
        holders = len(self.placeholders)

        if len(self.active_snippets) == 0:
            self.first_snippet_inserted()

        sn = s.insert_into(self, start)
        self.active_snippets.append(sn)

        # Put cursor at first tab placeholder
        keys = filter(lambda x: x > 0, sn.placeholders.keys())

        if len(keys) == 0:
            if 0 in sn.placeholders:
                self.goto_placeholder(self.active_placeholder,
                                      sn.placeholders[0])
            else:
                buf.place_cursor(sn.begin_iter())
        else:
            self.goto_placeholder(self.active_placeholder,
                                  sn.placeholders[keys[0]])

        if sn in self.active_snippets:
            # Check if we can get end_iter in view without moving the
            # current cursor position out of view
            cur = buf.get_iter_at_mark(buf.get_insert())
            last = sn.end_iter()

            curloc = self.view.get_iter_location(cur)
            lastloc = self.view.get_iter_location(last)

            if (lastloc.y + lastloc.height) - curloc.y <= \
               self.view.get_visible_rect().height:
                self.view.scroll_mark_onscreen(sn.end_mark)

        buf.end_user_action()
        self.view.grab_focus()

        return True
Exemple #23
0
class SQLiteSearch:
    def __init__(self,
                 snippet_limit=3,
                 take_words=3,
                 results_limit=10,
                 db_name='inverted-index.db',
                 data_dir='pages'):
        self.time_needed_to_search = None
        self.db_name = db_name
        self.take_words = take_words
        self.snippet_limit = snippet_limit
        self.results_limit = results_limit
        self.data_dir = os.path.join(os.getcwd(), data_dir)
        self.all_results = None
        self.snip = Snippet(self.take_words, self.results_limit)

        self.stop_words_slovene = set(
            nltk.corpus.stopwords.words("Slovene")
        ).union({
            "ter",
            "nov", "novo", "nova", "zato", "še", "zaradi", "a", "ali", "april",
            "avgust", "b", "bi", "bil", "bila", "bile", "bili", "bilo", "biti",
            "blizu", "bo", "bodo", "bojo", "bolj", "bom", "bomo",
            "boste", "bova", "boš", "brez", "c", "cel", "cela", "celi", "celo",
            "d", "da", "daleč", "dan", "danes", "datum", "december", "deset",
            "deseta", "deseti", "deseto", "devet", "deveta", "deveti",
            "deveto", "do", "dober", "dobra", "dobri", "dobro", "dokler",
            "dol", "dolg", "dolga", "dolgi", "dovolj", "drug", "druga",
            "drugi", "drugo", "dva", "dve", "e", "eden", "en", "ena", "ene",
            "eni", "enkrat", "eno", "etc.", "f", "februar", "g", "g.", "ga",
            "ga.", "gor", "gospa", "gospod", "h", "halo", "i", "idr.", "ii",
            "iii", "in", "iv", "ix", "iz", "j", "januar", "jaz", "je", "ji",
            "jih", "jim", "jo", "julij", "junij", "jutri", "k", "kadarkoli",
            "kaj", "kajti", "kako", "kakor", "kamor", "kamorkoli", "kar",
            "karkoli", "katerikoli", "kdaj", "kdo", "kdorkoli", "ker", "ki",
            "kje", "kjer", "kjerkoli", "ko", "koder",
            "koderkoli", "koga", "komu", "kot", "kratek", "kratka", "kratke",
            "kratki", "l", "lahka", "lahke", "lahki", "lahko", "le",
            "lep", "lepa", "lepe", "lepi", "lepo", "leto", "m", "maj",
            "majhen", "majhna", "majhni", "malce", "malo", "manj", "marec",
            "me", "med", "medtem", "mene", "mesec", "mi", "midva", "midve",
            "mnogo", "moj", "moja", "moje", "mora", "morajo", "moram",
            "moramo", "morate", "moraš", "morem", "mu", "n", "na", "nad",
            "naj", "najina", "najino", "najmanj", "naju", "največ", "nam",
            "narobe", "nas", "nato", "nazaj", "naš", "naša", "naše", "ne",
            "nedavno", "nedelja", "nek", "neka", "nekaj", "nekatere",
            "nekateri", "nekatero", "nekdo", "neke", "nekega", "neki",
            "nekje", "neko", "nekoga", "nekoč", "ni", "nikamor", "nikdar",
            "nikjer", "nikoli", "nič", "nje", "njega", "njegov", "njegova",
            "njegovo", "njej", "njemu", "njen", "njena", "njeno",
            "nji", "njih", "njihov", "njihova", "njihovo", "njiju", "njim",
            "njo", "njun", "njuna", "njuno", "no", "nocoj", "november", "npr.",
            "o", "ob", "oba", "obe", "oboje", "od", "odprt", "odprta",
            "odprti", "okoli", "oktober", "on", "onadva", "one", "oni",
            "onidve", "osem", "osma", "osmi", "osmo", "oz.", "p", "pa", "pet",
            "peta", "petek", "peti", "peto", "po", "pod", "pogosto", "poleg",
            "poln", "polna", "polni", "polno", "ponavadi", "ponedeljek",
            "ponovno", "potem", "povsod", "pozdravljen", "pozdravljeni",
            "prav", "prava", "prave", "pravi", "pravo", "prazen", "prazna",
            "prazno", "prbl.", "precej", "pred", "prej", "preko", "pri",
            "pribl.", "približno", "primer", "pripravljen", "pripravljena",
            "pripravljeni", "proti", "prva", "prvi", "prvo", "r", "ravno",
            "redko", "res", "reč", "s", "saj", "sam", "sama", "same", "sami",
            "samo", "se", "sebe", "sebi", "sedaj", "sedem", "sedma", "sedmi",
            "sedmo", "sem", "september", "seveda", "si", "sicer", "skoraj",
            "skozi", "slab", "smo", "so", "sobota", "spet", "sreda", "srednja",
            "srednji", "sta", "ste", "stran", "stvar", "sva", "t", "ta", "tak",
            "taka", "take", "taki", "tako", "takoj", "tam", "te", "tebe",
            "tebi", "tega", "težak", "težka", "težki", "težko", "ti", "tista",
            "tiste", "tisti", "tisto", "tj.", "tja", "to", "toda", "torek",
            "tretja", "tretje", "tretji", "tri", "tu", "tudi", "tukaj", "tvoj",
            "tvoja", "tvoje", "u", "v", "vaju", "vam", "vas", "vaš", "vaša",
            "vaše", "ve", "vedno", "velik", "velika", "veliki", "veliko",
            "vendar", "ves", "več", "vi", "vidva", "vii", "viii", "visok",
            "visoka", "visoke", "visoki", "vsa", "vsaj", "vsak", "vsaka",
            "vsakdo", "vsake", "vsaki", "vsakomur", "vse", "vsega", "vsi",
            "vso", "včasih", "včeraj", "x", "z", "za", "zadaj", "zadnji",
            "zakaj", "zaprta", "zaprti", "zaprto", "zdaj", "zelo", "zunaj",
            "č", "če", "često", "četrta", "četrtek", "četrti", "četrto", "čez",
            "čigav", "š", "šest", "šesta", "šesti", "šesto", "štiri", "ž",
            "že", "svoj", "jesti", "imeti", "\u0161e", "iti", "kak", "www",
            "km", "eur", "pač", "del", "kljub", "šele", "prek", "preko",
            "znova", "morda", "kateri", "katero", "katera", "ampak", "lahek",
            "lahka", "lahko", "morati", "torej", "gl", "xsd", "ipd", "om",
            "gt", "lt", "d.o.o"
        })

        self.conn = sqlite3.connect(self.db_name)

    def build_query(self, words):
        # -3 because we want to remove the last or and space (so 3 chars) from the list
        rm = len('or ') * -1
        if not isinstance(words, list):
            words = [words]
        whr = ' '.join(['word = \'{}\' or'.format(w) for w in words])[:rm]
        qry = "select documentName as file, SUM(frequency) 'freq', GROUP_CONCAT(indexes) as idx " \
              "from posting " \
              "where {} " \
              "group by documentName " \
              "order by freq desc ".format(whr)
        return qry

    @staticmethod
    def process_results(results):
        if len(results):
            return [[
                result[0], result[1], [int(i) for i in result[2].split(',')]
            ] for result in results]

    def print_results(self, query, frequencies, pages, snippets):
        print(f'Results for query: "{" ".join(query)}"')
        print('{} results found in {:.0f}ms'.format(
            self.all_results, self.time_needed_to_search))
        print("{:<12} {:<40} {}".format('Frequencies', 'Document', 'Snippets'))
        print("{} {} {}".format('-' * 12, '-' * 40, '-' * 80))
        for i in range(min(self.results_limit, len(pages))):
            print("{:<12} {:<40} {}".format(
                frequencies[i], pages[i], '... ' +
                ' ... '.join(snippets[i][:self.snippet_limit]) + ' ...'))

    def search_db(self, words):
        cursor = self.conn.cursor()
        query = self.build_query(words)
        cursor.execute(query)
        results = cursor.fetchall()
        return self.process_results(results)

    def get_document(self, page_name):
        map_name = '.'.join(page_name.split('.')[:3])
        return open(os.path.join(self.data_dir,
                                 "{}/{}".format(map_name, page_name)),
                    'r',
                    encoding='utf-8')

    def search(self, words):
        # start the timer and do the search, after that calculate the time needed for search
        start_time = time.time()
        results = self.search_db(words)
        self.all_results = len(results)
        end_time = time.time()
        self.time_needed_to_search = (end_time - start_time) * 1000

        pages, frequencies, snippets = [], [], []
        ctr = 0
        for page_file, frequency, indexes in results:
            if ctr > self.results_limit:
                break
            ctr += 1
            pages.append(page_file)
            frequencies.append(frequency)
            document = self.get_document(page_file)
            soup = BeautifulSoup(document, features="html.parser")
            for script in soup(["script", "style"]):
                script.decompose()

            strips = list(soup.stripped_strings)
            text = ' '.join(strips)
            tokens = nltk.word_tokenize(text,
                                        language='Slovene',
                                        preserve_line=False)
            word_tokens = []
            for i in range(len(tokens)):
                w = tokens[i].lower().replace("'",
                                              "").replace("'", '').replace(
                                                  '`', '').replace('·', '')
                if not re.search('[a-žA-ž]', w):
                    continue
                if len(w) == 1 and not re.match("^[A-Ža-ž0-9]*$", w):
                    continue
                if len(w) >= 2 and not re.match("^[A-Ža-ž0-9]*$", w[-1]):
                    w = w[:-1]
                if len(w) >= 2 and not re.match("^[A-Ža-ž0-9]*$", w[0]):
                    w = w[1:]
                if w not in self.stop_words_slovene:
                    word_tokens.append(w)
            # snippets.append(self.get_snip(tokens, indexes))
            snippets.append(self.snip.get_snip(tokens, indexes))
        self.print_results(words, frequencies, pages, snippets)
Exemple #24
0
import urwid
from difflib import SequenceMatcher

from snippet import Snippet

g_verse = None  # placeholder for currently selected verse
g_verse = Snippet.random()


def do_diff(text, n_text):
    seqm = SequenceMatcher(None, text, n_text)
    output_orig = []
    output_new = []
    for opcode, a0, a1, b0, b1 in seqm.get_opcodes():
        orig_seq = seqm.a[a0:a1]
        new_seq = seqm.b[b0:b1]
        if opcode == 'equal':
            output_orig.append(orig_seq)
            output_new.append(orig_seq)
        elif opcode == 'insert':
            output_new.append(('extra_text', new_seq))
        elif opcode == 'delete':
            output_orig.append(('missing_text', orig_seq))
        elif opcode == 'replace':
            output_new.append(('wrong_text', new_seq))
            output_orig.append(('wrong_text', orig_seq))
        else:
            raise ('Error')
    return output_orig, output_new, seqm.quick_ratio()

#!/usr/bin/python3

from snippet import Snippet
from utils import *

text = """

#!/usr/bin python3

print('hello world!')

"""

with open('example.html') as fp:
    text = fp.read().strip()
text = text[:int(len(text) / 32)]

snippet = Snippet(text)
png_path = 'hello.png'

screen_width_px, screen_height_px = 1920, 1080
font_size_px = 16
snippet.to_image(png_path, font_size=font_size_px)

size = [1920, 1080]
scale_image(png_path, 'scaled.png', size)
Exemple #26
0
 def __init__(self, snippet):
         GObject.Object.__init__(self)
         self._snippet = Snippet(snippet)
Exemple #27
0
def split_snippets(s: Snippet, l: int):
    length = s.length
    n = max(length // l, 1)  # at least 1
    return s.split(n, l)
Exemple #28
0
    def apply_directive(self,
                        directive,
                        bk,
                        code=None,
                        s=None,
                        repl_records=None):
        """Apply directives.

        The value in directive[2] depends on the command:
        'replace': the argument is the string to use as a replacement
        'push': the argument is the filepath to save into
"""
        snip_id = directive[0]
        cmd = directive[1]

        if cmd == 'push':
            # Perform replacements on the directive argument if necessary
            arg = directive[2]
            if repl_records is not None:
                for rec in repl_records:
                    # rec[0] = snippet id, rec[1] = array of replacement values
                    if rec[0] == snip_id:
                        arg = replace_values(arg, rec[1])
                        break
            # Save the current (code, filepath) on the stack
            bk.code_stack.append((code + s, arg))
            code = s = ''

        elif cmd == 'pop':
            # Finalize code being generated
            code += s
            # Pop up one stack level and resume that level's code generation
            prev_code, filepath = bk.code_stack.pop()
            print(f'{self.id}: apply_directive: writing "{filepath}"')

            # Write out to file
            chrooted = self.id.startswith('6.') or self.id.startswith('9.')
            with open(filepath, 'w') as f:
                f.write(
                    file_header(filepath, bk, chrooted=chrooted) + code +
                    file_footer(filepath))

            # Reset code generation for this stack level
            code = prev_code
            s = ''

        elif cmd == 'add':
            arg = directive[2]
            t = Snippet('userinput', arg)

            # Replace placeholders with actual values
            if repl_records is not None:
                for rec in repl_records:
                    # rec[0] = snippet id, rec[1] = array of replacement values
                    if rec[0] == snip_id:
                        t.text = t.replace_values(rec[1])
                        break
            s += t.generate(f'{self.id}_{snip_id} [add]')
        else:
            print(f'Sect: {self.id}, snippet directive: {snip_id}' +
                  f', unknown command "{cmd}"')

        return code, s