예제 #1
0
    def write(self):

        dockerfile = os.path.join(self.image_dir, 'Dockerfile')
        os.makedirs(self.image_dir, exist_ok=True)
        if os.path.exists(dockerfile):
            os.remove(dockerfile)
        with open(dockerfile, 'x'):
            pass
        snippets = []

        snippets.extend(self.snippets)
        snippets.append(
            Snippet(os.path.join(docker_path, 'snippets/workdir.in')))
        snippets.append(
            Snippet(os.path.join(docker_path, 'snippets/entrypoint.in')))

        for snippet in snippets:
            with open(dockerfile, 'a') as f:
                try:
                    f.write(snippet.substitute(self.substitutions))
                except Exception as e:
                    raise RuntimeError(
                        'Error creating Dockerfile for {}: {}'.format(
                            self.name, e))
        for file in self.files:
            shutil.copy(file, self.image_dir)
예제 #2
0
파일: parse.py 프로젝트: joaomdsc/lfs
def parse_wrap(nd):
    """This parses one section with class="wrap" (it builds a package) """
    # Title has package name and version
    title = ''
    a = nd.find('.//div/div/div/h2/a')
    if a is None or a.tail is None:
        return

    # Parse the section title
    title = a.tail.strip()
    title = re.sub('\s+', ' ', title)

    # Patch
    title = title.replace('xml::parser', 'xml-parser')

    # Extract build instructions
    snippets = []
    divs = nd.findall('./div')
    with_pkg = False
    sbu = sz = None
    for div in divs:
        # A section may have different (sub-)classes
        if 'class' in div.attrib:
            kl = div.attrib['class']
            if kl == 'package':
                sbu, sz = parse_package(div)
                # print(f'{title}, {sbu} SBU, {sz} GB')
                with_pkg = True
                continue
            # sect2 is used for the 8.4 grub section
            if kl not in ['installation', 'configuration', 'sect2']:
                continue
            for pre in div.findall('.//pre'):
                s = ''
                if pre.text:
                    t = pre.text.strip()
                    s += t
                if 'class' in pre.attrib:
                    kl = pre.attrib['class']
                    # Note: <pre class="root" has only been found in sect1 sections
                    if kl in ['userinput', 'root']:
                        # This is a code snippet
                        kbds = pre.findall('.//kbd[@class="command"]')
                        s += get_kbds_script(kbds)
                        if len(s) > 0:
                            snippets.append(Snippet('userinput', s))
                    elif kl == 'screen':
                        # This is some program output
                        s += get_em_code(pre)
                        if len(s) > 0:
                            snippets.append(Snippet('screen', s))

    # Only create section object if there are instructions
    if len(snippets) > 0:
        return Section(title, snippets, with_pkg=with_pkg, sbu=sbu, sz=sz)
예제 #3
0
파일: manager.py 프로젝트: onia/pygi
    def on_cell_edited(self, cell, path, new_text):
        if new_text != '':
            piter = self.model.get_iter(path)
            node = self.model.get_value(piter, self.SNIPPET_COLUMN)

            if node:
                if node == self.snippet.data:
                    s = self.snippet
                else:
                    s = Snippet(node)

                s['description'] = new_text
                self.snippet_changed(piter)
                self.select_iter(piter)
            else:
                # This is the `Add a new snippet...` item
                # We create a new snippet
                snippet = self.new_snippet({'description': new_text})

                if snippet:
                    self.model.set_value(piter, self.SNIPPET_COLUMN,
                                         snippet.data)
                    self.snippet_changed(piter)
                    self.snippet = snippet
                    self.selection_changed()
예제 #4
0
def mixGPAll(snippets, params):
    logger.mixer("Starting mixGPAll mix method")
    combined = Snippet("combined")

    # copy a few things, assume first snippet has the defaults
    combined.optSteps = snippets[0].optSteps
    combined.learningRate = snippets[0].learningRate
    combined.lossTolerance = snippets[0].lossTolerance

    logger.mixer("Adding training data")
    # add training data
    for snippet in snippets:
        for tdata in snippet.data:
            combined.addData(tdata)

    # train the new snippet
    logger.mixer("Training new snippet")
    combined.train()

    # sample the stuff
    logger.mixer("Sampling new snippet")
    sampler = Rejection(combined, params["x0"], n=20)
    sampler.start()

    # this is threaded so we'll join then pull the results out
    sampler.join()
    return sampler.results
예제 #5
0
파일: manager.py 프로젝트: onia/pygi
    def new_snippet(self, properties=None):
        if not self.language_path:
            return None

        snippet = Library().new_snippet(self.get_language(self.language_path),
                                        properties)

        return Snippet(snippet)
예제 #6
0
 def __init__(self, name, snippets, files, substitutions):
     self.name = name
     self.snippets = [
         Snippet(os.path.join(docker_path, 'snippets', snippet + '.in'))
         for snippet in snippets
     ]
     self.substitutions = substitutions
     self.files = files
     self.status = None
     self.image_dir = os.path.join(images_dir, self.name)
     self.image_name = 'gulinux/planetaryimager_build:{}'.format(self.name)
예제 #7
0
 def addSnippet(self, name, paramInfo=None):
     if name not in self.snippets:
         self.snippets[name] = Snippet(name, paramInfo)
         return DSStatus(code=0, message="Added Snippet: {0}".format(name))
     else:
         return DSStatus(
             code=-1,
             message="Failed to add Snippet with name {0}. Snippet already exists.".format(
                 name
             ),
         )
예제 #8
0
파일: parse.py 프로젝트: joaomdsc/lfs
def parse_sect1(nd):
    # Title has package name and version
    title = ''
    a = nd.find('.//div/div/div/h2/a')
    if a is None or a.tail is None:
        return

    # Parse the section title
    title = a.tail.strip()
    title = re.sub('\s+', ' ', title)

    # Extract build instructions
    snippets = []
    for pre in nd.findall('.//pre'):
        s = ''
        if pre.text:
            t = pre.text.strip()
            s += t
        if 'class' in pre.attrib:
            kl = pre.attrib['class']
            # Note: <pre class="root" has only been found in sect1 sections
            if kl in ['userinput', 'root']:
                # This is a code snippet
                kbds = pre.findall('.//kbd[@class="command"]')
                s += get_kbds_script(kbds)
                if len(s) > 0:
                    snippets.append(Snippet('userinput', s))
            elif kl == 'screen':
                # This is some program output
                s += get_em_code(pre)
                if len(s) > 0:
                    snippets.append(Snippet('screen', s))

    # Only create section object if there are instructions
    if len(snippets) > 0:
        return Section(title, snippets, with_pkg=False)
예제 #9
0
파일: manager.py 프로젝트: onia/pygi
    def snippet_changed(self, piter=None):
        if piter:
            node = self.model.get_value(piter, self.SNIPPET_COLUMN)
            s = Snippet(node)
        else:
            s = self.snippet
            piter = self.find_iter(self.model.get_iter(self.language_path), s)

        if piter:
            nm = s.display()

            self.model.set_value(piter, self.NAME_COLUMN, nm)
            self.model.set_value(piter, self.SORT_COLUMN, nm)
            self.update_toolbar_buttons()
            self.entry_tab_trigger_update_valid()

        return piter
예제 #10
0
    def parse_snippets(self, path: str) -> Type[HeaderFile]:
        result = HeaderFile(path)
        snippet = None
        group = []
        for line_num, line in enumerate(readlines(path), 1):
            if line.startswith(BEGIN_TAG):
                dprint(f'Process {line}')
                key = line[len(BEGIN_TAG):].strip().strip('`')
                if snippet is not None:
                    raise Exception(f'Unexpected open section '
                                    f'({key}) {path}:{line_num} {line}')
                snippet = Snippet(key,
                                  enabled=key.startswith(ALWAYS_INCLUDE_TAG))
            elif line.startswith(END_TAG):
                dprint(f'Process {line}')
                key = line[len(END_TAG):].strip().strip('`')
                if snippet is None or snippet.tag != key:
                    raise Exception(f'Unexpected close section '
                                    f'({key}) {path}:{line_num} {line}')
                if key in self.snippets:
                    raise Exception(f'Duplicated key found '
                                    f'({key}) {path}:{line_num}')
                snippet.body = '\n'.join(group)
                self.snippets[snippet.tag] = snippet
                result.add_snippet(snippet)
                snippet = None
                group = []
            elif line.startswith(REQUIRED_TAG) and snippet is not None:
                for dependency in filter(
                        None,
                        map(str.strip, line[len(REQUIRED_TAG):].split(','))):
                    dprint(f'{snippet.tag} depends on `{dependency}`')
                    if dependency not in self.snippets:
                        raise Exception(f'Unknown dependency {dependency} '
                                        f'for snippet {snippet.tag}'
                                        f'({key}) {path}:{line_num}')
                    snippet.add_dependency(self.snippets[dependency])
            elif snippet is not None:
                group.append(line)

        if snippet is not None:
            raise Exception(f'Missed last closing tag {path} ({snippet.tag})')

        return result
예제 #11
0
파일: manager.py 프로젝트: onia/pygi
    def on_tree_view_selection_changed(self, selection):
        parent, piter, node = self.selected_snippet()

        if self.snippet:
            self.on_entry_tab_trigger_focus_out(self['entry_tab_trigger'],
                                                None)
            self.on_source_view_snippet_focus_out(self['source_view_snippet'],
                                                  None)
            self.on_entry_drop_targets_focus_out(
                self['combo_drop_targets'].get_child(), None)

        self.update_language_path()

        if node:
            self.snippet = Snippet(node)
        else:
            self.snippet = None

        self.selection_changed()
예제 #12
0
    def find_snippet(filename: str = Constants.PYTHON_SNIPPETS_FILE) -> List[Snippet]:
        with open(filename) as snippets_file:
            snippets_db = []
            snippet = Snippet()
            for line in snippets_file.readlines():
                if "snippet " in line:
                    if (snippet.name and snippet.prefix) is not None:
                        snippets_db.append(copy.deepcopy(snippet))
                        snippet.clean()

                    snippet.name = line.replace("snippet", "").lstrip().rstrip()
                    snippet.prefix = snippet.name.split()[0]
                    continue
                if line.find("#") == 0:
                    continue

                clean_line = line.replace("\t", "", 1).replace("\n", "")
                if line:
                    snippet.body.append(clean_line)
            return snippets_db
#!/usr/bin/python3

from snippet import Snippet
from utils import *

text = """

#!/usr/bin python3

print('hello world!')

"""

with open('example.html') as fp:
    text = fp.read().strip()
text = text[:int(len(text) / 32)]

snippet = Snippet(text)
png_path = 'hello.png'

screen_width_px, screen_height_px = 1920, 1080
font_size_px = 16
snippet.to_image(png_path, font_size=font_size_px)

size = [1920, 1080]
scale_image(png_path, 'scaled.png', size)
예제 #14
0
    def __init__(self,
                 snippet_limit: int = 3,
                 take_words: int = 3,
                 results_limit: int = 10,
                 data_dir: str = 'pages') -> None:
        """
        This class is used for basic, no SQL searching.

        :param take_words: How many words to display is snippets before/after the found word.
        :param results_limit: How many rows to show when displaying top results with snippets.
        :param data_dir: Directory in which the HTML sites are saved relative to the current directory.
        """
        # nltk.download('punkt')
        # nltk.download('stopwords')
        self.time_needed_to_search = None
        self.take_words = take_words
        self.results_limit = results_limit
        self.data_dir = os.path.join(os.getcwd(), data_dir)
        self.path = data_dir
        self.snippet_limit = snippet_limit
        self.snip = Snippet(self.take_words, self.results_limit)

        self.stop_words_slovene = set(stopwords.words("Slovene")).union({
            "ter", "nov", "novo", "nova", "zato", "še", "zaradi", "a", "ali",
            "april", "avgust", "b", "bi", "bil", "bila", "bile", "bili",
            "bilo", "biti", "blizu", "bo", "bodo", "bojo", "bolj", "bom",
            "bomo", "boste", "bova", "boš", "brez", "c", "cel", "cela", "celi",
            "celo", "d", "da", "daleč", "dan", "danes", "datum", "december",
            "deset", "deseta", "deseti", "deseto", "devet", "deveta", "deveti",
            "deveto", "do", "dober", "dobra", "dobri", "dobro", "dokler",
            "dol", "dolg", "dolga", "dolgi", "dovolj", "drug", "druga",
            "drugi", "drugo", "dva", "dve", "e", "eden", "en", "ena", "ene",
            "eni", "enkrat", "eno", "etc.", "f", "februar", "g", "g.", "ga",
            "ga.", "gor", "gospa", "gospod", "h", "halo", "i", "idr.", "ii",
            "iii", "in", "iv", "ix", "iz", "j", "januar", "jaz", "je", "ji",
            "jih", "jim", "jo", "julij", "junij", "jutri", "k", "kadarkoli",
            "kaj", "kajti", "kako", "kakor", "kamor", "kamorkoli", "kar",
            "karkoli", "katerikoli", "kdaj", "kdo", "kdorkoli", "ker", "ki",
            "kje", "kjer", "kjerkoli", "ko", "koder", "koderkoli", "koga",
            "komu", "kot", "kratek", "kratka", "kratke", "kratki", "l",
            "lahka", "lahke", "lahki", "lahko", "le", "lep", "lepa", "lepe",
            "lepi", "lepo", "leto", "m", "maj", "majhen", "majhna", "majhni",
            "malce", "malo", "manj", "marec", "me", "med", "medtem", "mene",
            "mesec", "mi", "midva", "midve", "mnogo", "moj", "moja", "moje",
            "mora", "morajo", "moram", "moramo", "morate", "moraš", "morem",
            "mu", "n", "na", "nad", "naj", "najina", "najino", "najmanj",
            "naju", "največ", "nam", "narobe", "nas", "nato", "nazaj", "naš",
            "naša", "naše", "ne", "nedavno", "nedelja", "nek", "neka", "nekaj",
            "nekatere", "nekateri", "nekatero", "nekdo", "neke", "nekega",
            "neki", "nekje", "neko", "nekoga", "nekoč", "ni", "nikamor",
            "nikdar", "nikjer", "nikoli", "nič", "nje", "njega", "njegov",
            "njegova", "njegovo", "njej", "njemu", "njen", "njena", "njeno",
            "nji", "njih", "njihov", "njihova", "njihovo", "njiju", "njim",
            "njo", "njun", "njuna", "njuno", "no", "nocoj", "november", "npr.",
            "o", "ob", "oba", "obe", "oboje", "od", "odprt", "odprta",
            "odprti", "okoli", "oktober", "on", "onadva", "one", "oni",
            "onidve", "osem", "osma", "osmi", "osmo", "oz.", "p", "pa", "pet",
            "peta", "petek", "peti", "peto", "po", "pod", "pogosto", "poleg",
            "poln", "polna", "polni", "polno", "ponavadi", "ponedeljek",
            "ponovno", "potem", "povsod", "pozdravljen", "pozdravljeni",
            "prav", "prava", "prave", "pravi", "pravo", "prazen", "prazna",
            "prazno", "prbl.", "precej", "pred", "prej", "preko", "pri",
            "pribl.", "približno", "primer", "pripravljen", "pripravljena",
            "pripravljeni", "proti", "prva", "prvi", "prvo", "r", "ravno",
            "redko", "res", "reč", "s", "saj", "sam", "sama", "same", "sami",
            "samo", "se", "sebe", "sebi", "sedaj", "sedem", "sedma", "sedmi",
            "sedmo", "sem", "september", "seveda", "si", "sicer", "skoraj",
            "skozi", "slab", "smo", "so", "sobota", "spet", "sreda", "srednja",
            "srednji", "sta", "ste", "stran", "stvar", "sva", "t", "ta", "tak",
            "taka", "take", "taki", "tako", "takoj", "tam", "te", "tebe",
            "tebi", "tega", "težak", "težka", "težki", "težko", "ti", "tista",
            "tiste", "tisti", "tisto", "tj.", "tja", "to", "toda", "torek",
            "tretja", "tretje", "tretji", "tri", "tu", "tudi", "tukaj", "tvoj",
            "tvoja", "tvoje", "u", "v", "vaju", "vam", "vas", "vaš", "vaša",
            "vaše", "ve", "vedno", "velik", "velika", "veliki", "veliko",
            "vendar", "ves", "več", "vi", "vidva", "vii", "viii", "visok",
            "visoka", "visoke", "visoki", "vsa", "vsaj", "vsak", "vsaka",
            "vsakdo", "vsake", "vsaki", "vsakomur", "vse", "vsega", "vsi",
            "vso", "včasih", "včeraj", "x", "z", "za", "zadaj", "zadnji",
            "zakaj", "zaprta", "zaprti", "zaprto", "zdaj", "zelo", "zunaj",
            "č", "če", "često", "četrta", "četrtek", "četrti", "četrto", "čez",
            "čigav", "š", "šest", "šesta", "šesti", "šesto", "štiri", "ž",
            "že", "svoj", "jesti", "imeti", "\u0161e", "iti", "kak", "www",
            "km", "eur", "pač", "del", "kljub", "šele", "prek", "preko",
            "znova", "morda", "kateri", "katero", "katera", "ampak", "lahek",
            "lahka", "lahko", "morati", "torej", "gl", "xsd", "ipd", "om",
            "gt", "lt", "d.o.o"
        })
예제 #15
0
def musicThumbnailing(x,
                      Fs,
                      shortTermSize=1.0,
                      shortTermStep=0.5,
                      thumbnailSize=10.0,
                      Limit1=0,
                      Limit2=1):
    # self-similarity matrix
    x = audioBasicIO.stereo2mono(x)
    stFeatures = aF.stFeatureExtraction(x, Fs, Fs * shortTermSize,
                                        Fs * shortTermStep)
    S = aS.selfSimilarityMatrix(stFeatures)

    # moving filter:
    M = int(round(thumbnailSize / shortTermStep))
    B = np.eye(M, M)
    S = scipy.signal.convolve2d(S, B, 'valid')

    MIN = np.min(S)
    # post-processing (remove main diagonal elements)
    for i in range(S.shape[0]):
        for j in range(S.shape[1]):
            if abs(i - j) < 5.0 / shortTermStep or i > j:
                S[i, j] = MIN

    # find max position:
    S[0:int(Limit1 * S.shape[0]), :] = MIN
    S[:, 0:int(Limit1 * S.shape[0])] = MIN
    S[int(Limit2 * S.shape[0])::, :] = MIN
    S[:, int(Limit2 * S.shape[0])::] = MIN

    matches = []
    maxMax = maxVal = np.max(S)
    i1 = i2 = j1 = j2 = 0
    Sbak = np.copy(S)

    while maxVal > maxMax / 3 * 2 > MIN:  # currently arbitrary cutoff
        [I, J] = np.unravel_index(S.argmax(), S.shape)

        # expand:
        i1 = I
        i2 = I
        j1 = J
        j2 = J

        while i2 - i1 < M:
            if i1 <= 0 or j1 <= 0 or i2 >= S.shape[0] - 2 or j2 >= S.shape[
                    1] - 2:
                break
            if S[i1 - 1, j1 - 1] > S[i2 + 1, j2 + 1]:
                i1 -= 1
                j1 -= 1
            else:
                i2 += 1
                j2 += 1
            S[i1, j1] = S[i2, j2] = MIN

        # only add to potential matches if we have enough overlap or new record
        if i2 - i1 >= M:
            matches.append(
                Snippet(maxVal, shortTermStep * i1, shortTermStep * i2,
                        shortTermStep * j1, shortTermStep * j2))

        S[I, J] = MIN
        maxVal = np.max(S)

    return (matches, Sbak)
예제 #16
0
파일: section.py 프로젝트: joaomdsc/lfs
    def apply_directive(self,
                        directive,
                        bk,
                        code=None,
                        s=None,
                        repl_records=None):
        """Apply directives.

        The value in directive[2] depends on the command:
        'replace': the argument is the string to use as a replacement
        'push': the argument is the filepath to save into
"""
        snip_id = directive[0]
        cmd = directive[1]

        if cmd == 'push':
            # Perform replacements on the directive argument if necessary
            arg = directive[2]
            if repl_records is not None:
                for rec in repl_records:
                    # rec[0] = snippet id, rec[1] = array of replacement values
                    if rec[0] == snip_id:
                        arg = replace_values(arg, rec[1])
                        break
            # Save the current (code, filepath) on the stack
            bk.code_stack.append((code + s, arg))
            code = s = ''

        elif cmd == 'pop':
            # Finalize code being generated
            code += s
            # Pop up one stack level and resume that level's code generation
            prev_code, filepath = bk.code_stack.pop()
            print(f'{self.id}: apply_directive: writing "{filepath}"')

            # Write out to file
            chrooted = self.id.startswith('6.') or self.id.startswith('9.')
            with open(filepath, 'w') as f:
                f.write(
                    file_header(filepath, bk, chrooted=chrooted) + code +
                    file_footer(filepath))

            # Reset code generation for this stack level
            code = prev_code
            s = ''

        elif cmd == 'add':
            arg = directive[2]
            t = Snippet('userinput', arg)

            # Replace placeholders with actual values
            if repl_records is not None:
                for rec in repl_records:
                    # rec[0] = snippet id, rec[1] = array of replacement values
                    if rec[0] == snip_id:
                        t.text = t.replace_values(rec[1])
                        break
            s += t.generate(f'{self.id}_{snip_id} [add]')
        else:
            print(f'Sect: {self.id}, snippet directive: {snip_id}' +
                  f', unknown command "{cmd}"')

        return code, s
예제 #17
0
파일: completion.py 프로젝트: onia/pygi
 def __init__(self, snippet):
         GObject.Object.__init__(self)
         self._snippet = Snippet(snippet)
예제 #18
0
    def apply_snippet(self, snippet, start=None, end=None, environ={}):
        if not snippet.valid:
            return False

        # Set environmental variables
        env = self.get_environment()

        if environ:
            for k in environ['utf8']:
                env['utf8'][k] = environ['utf8'][k]

            for k in environ['noenc']:
                env['noenc'][k] = environ['noenc'][k]

        buf = self.view.get_buffer()
        s = Snippet(snippet, env)

        if not start:
            start = buf.get_iter_at_mark(buf.get_insert())

        if not end:
            end = buf.get_iter_at_mark(buf.get_selection_bound())

        if start.equal(end) and self.uses_current_word(s):
            # There is no tab trigger and no selection and the snippet uses
            # the current word. Set start and end to the word boundary so that
            # it will be removed
            start, end = buffer_word_boundary(buf)
        elif start.equal(end) and self.uses_current_line(s):
            # There is no tab trigger and no selection and the snippet uses
            # the current line. Set start and end to the line boundary so that
            # it will be removed
            start, end = buffer_line_boundary(buf)

        # You know, we could be in an end placeholder
        (current, next) = self.next_placeholder()
        if current and current.__class__ == PlaceholderEnd:
            self.goto_placeholder(current, None)

        buf.begin_user_action()

        # Remove the tag, selection or current word
        buf.delete(start, end)

        # Insert the snippet
        holders = len(self.placeholders)

        if len(self.active_snippets) == 0:
            self.first_snippet_inserted()

        sn = s.insert_into(self, start)
        self.active_snippets.append(sn)

        # Put cursor at first tab placeholder
        keys = filter(lambda x: x > 0, sn.placeholders.keys())

        if len(keys) == 0:
            if 0 in sn.placeholders:
                self.goto_placeholder(self.active_placeholder,
                                      sn.placeholders[0])
            else:
                buf.place_cursor(sn.begin_iter())
        else:
            self.goto_placeholder(self.active_placeholder,
                                  sn.placeholders[keys[0]])

        if sn in self.active_snippets:
            # Check if we can get end_iter in view without moving the
            # current cursor position out of view
            cur = buf.get_iter_at_mark(buf.get_insert())
            last = sn.end_iter()

            curloc = self.view.get_iter_location(cur)
            lastloc = self.view.get_iter_location(last)

            if (lastloc.y + lastloc.height) - curloc.y <= \
               self.view.get_visible_rect().height:
                self.view.scroll_mark_onscreen(sn.end_mark)

        buf.end_user_action()
        self.view.grab_focus()

        return True
예제 #19
0
def fetch_snippets(num_snippets, start_time, end_time, extra_tags):
    """
    Fetches snippets from StackOverflow by looking at 'python' questions
    in the time-period specified, and retrieving `<pre><code>` code blocks
    from these questions (and their answers, where applicable)

    Note that currently the number of snippets returned is not the same as
    the number requested: it depends on the number of retrieved questions with
    a code block, as well as the number with an accepted answer with a code block
    """
    filters = load_filter_file()

    snippets = []
    answer_ids = []

    questions_retrieved = 0
    page_num = 1
    while questions_retrieved < num_snippets:
        questions, has_more = fetch_recent_questions(
            num_questions=(num_snippets - questions_retrieved),
            from_time=start_time,
            to_time=end_time,
            tags=extra_tags,
            page_num=page_num,
            filter_name=filters.Questions)
        current_time = datetime.utcnow()
        for q in questions:
            snippets += [
                Snippet(snippet_id=q['question_id'],
                        code=block,
                        url=q['link'],
                        author='stack-overflow',
                        retrieved_at=current_time,
                        additional_url=None)
                for block in get_snippets(q['body'])
                if check_source_and_warn(block, q['link'])
            ]
            answer_ids.append(q.get('accepted_answer_id', None))

        questions_retrieved += len(questions)
        page_num += 1
        if not has_more:
            warn('No more questions to fetch: Terminating')
            break

    # Filter out `None` accepted_answer_id's
    answer_ids = [str(ans_id) for ans_id in answer_ids if ans_id]

    log('Retrieving {} accepted answers for analysis...'.format(
        len(answer_ids)))
    for i in range(0, len(answer_ids), ANSWER_BATCH_SIZE):
        batch = answer_ids[i:i + ANSWER_BATCH_SIZE]
        answers = fetch_answers(batch, filters.Answers)
        for a in answers:
            snippets += [
                Snippet(snippet_id=a['answer_id'],
                        code=block,
                        url=a['link'],
                        author='stack-overflow',
                        retrieved_at=current_time,
                        additional_url=None)
                for block in get_snippets(a['body'])
                if check_source_and_warn(block, q['link'])
            ]

    success('Retrieved {} snippets'.format(len(snippets)))
    return snippets