예제 #1
0
def create_yaml(api_key, talks_dir, speakers_dir):
    if not exists(talks_dir):
        makedirs(talks_dir)
    if not exists(speakers_dir):
        makedirs(speakers_dir)

    for proposal_state in PROPOSAL_STATES:
        r = get(
            'https://www.papercall.io/api/v1/submissions?_token={0}&state={1}&per_page=1000'
            .format(
                api_key,
                proposal_state,
            ))

        speakers = {}

        for proposal in r.json():
            talk_title_slug = slugify(proposal['talk']['title'])

            post = frontmatter.loads(proposal['talk']['description'])
            post['type'] = 'talk'
            post['title'] = proposal['talk']['title']
            post['level'] = proposal['talk']['audience_level']
            post['abstract'] = proposal['talk']['abstract']
            post['speakers'] = []

            speaker_name = proposal['profile']['name']
            if '/' in speaker_name:
                speaker_name = speaker_name.split('/')
            elif ' and ' in speaker_name:
                speaker_name = speaker_name.split(' and ')
            elif ',' in speaker_name and speaker_name[-5:] != ', MBA':
                speaker_name = speaker_name.split(',')
            else:
                speaker_name = [speaker_name]

            for name in map(str.strip, speaker_name):
                speaker_slug = slugify(name)

                if speaker_slug not in speakers:
                    speakers[speaker_slug] = frontmatter.loads(
                        proposal['profile']['bio'])
                    speakers[speaker_slug]['name'] = name
                    speakers[speaker_slug]['talks'] = []

                post['speakers'].append(name)
                speakers[speaker_slug]['talks'].append(post['title'])

            talk_filename = '{}/{}.md'.format(talks_dir, talk_title_slug)
            with open(talk_filename, 'wb') as file_to_write:
                frontmatter.dump(post, file_to_write)

            print('saved {!r}'.format(talk_filename))

        for speaker_slug, speaker in speakers.items():
            speaker_filename = '{}/{}.md'.format(speakers_dir, speaker_slug)
            with open(speaker_filename, 'wb') as file_to_write:
                frontmatter.dump(speaker, file_to_write)

            print('saved {!r}'.format(speaker_filename))
예제 #2
0
def sync(
    all_sheets: bool = False,
    output_folder: str = "_players",
    sheet_app_id: str = typer.Option(envvar="GOOGLE_SHEET_APP_ID", default=""),
    sheet_name: str = typer.Option(envvar="GOOGLE_SHEET_NAME",
                                   default="Sheet1"),
):
    typer.secho("sync-players", fg="yellow")
    try:
        sa = SpreadsheetApp(from_env=True)
    except AttributeError:
        print_expected_env_variables()
        raise typer.Exit()

    try:
        spreadsheet = sa.open_by_id(sheet_app_id)
    except Exception:
        typer.echo(
            f"We can't find that 'sheet_app_id'.\n"
            f"Please double check that 'GOOGLE_SHEET_APP_ID' is set. (Currently set to: '{sheet_app_id}')"
        )
        raise typer.Exit()

    if all_sheets:
        sheets = spreadsheet.get_sheets()
    else:
        try:
            sheets = [spreadsheet.get_sheet_by_name(sheet_name)]
        except Exception:
            typer.echo(
                f"We can't find that 'sheet_name' aka the tab.\n"
                f"Please double check that 'SHEET_NAME' is set. (Currently set to: '{sheet_name}')"
            )
            raise typer.Exit()

    for sheet in sheets:
        data_range = sheet.get_data_range()

        table = Table(data_range, backgrounds=True)

        metadata = {}
        for item in table:
            for key in item.header:
                value = item.get_field_value(key)
                metadata[key] = value

            player = Player(**metadata)

            if not Path(output_folder).exists():
                Path(output_folder).mkdir()

            player_filename = Path(output_folder, f"{player.slug}.md")
            if player_filename.exists():
                post = frontmatter.loads(player_filename.read_text())
            else:
                post = frontmatter.loads("")

            post.metadata.update(player.dict(by_alias=True))

            player_filename.write_text(frontmatter.dumps(post))
예제 #3
0
def accomplishment(date: str = "now", overwrite: bool = False):
    parsed_date = maya.when(date, timezone="US/Central").datetime(naive=True)

    day_of_month = parsed_date.day
    week_number = (day_of_month - 1) // 7 + 1

    output_filename = f"{parsed_date.year}-{parsed_date.month:02}-week{week_number}.md"
    output_filename = Path(
        str(OUTPUT_FOLDER.joinpath(output_filename)).format(
            year=parsed_date.year))

    typer.echo(str(output_filename))

    if not output_filename.parent.exists():
        output_filename.parent.mkdir()

    if TASKS_DATA.exists():
        data = yaml.load(TASKS_DATA.read_text(), Loader=yaml.FullLoader)
    else:
        data = dict()

    context_data = data.copy()
    context_data["date"] = parsed_date

    # Handle Accomplishments
    post = frontmatter.loads(ACCOMPLISHMENT_TEMPLATE.read_text())

    t = jinja2.Template(post.content)
    contents = t.render(context_data)

    post["date"] = parsed_date
    post.content = contents

    if not output_filename.exists() or overwrite:
        output_filename.write_text(frontmatter.dumps(post))

    # Handle README
    output_filename = Path("README.md")

    post = frontmatter.loads(README_TEMPLATE.read_text())

    t = jinja2.Template(post.content)
    contents = t.render(context_data)

    post["date"] = parsed_date
    post.content = contents

    # output_filename.write_text(frontmatter.dumps(post))
    output_filename.write_text(post.content)
예제 #4
0
    def generate(self, presite):
        parsed = frontmatter.loads(self.content)

        self.config.update(parsed)

        if self.is_markdown:
            parsed.content = markdown.markdown(parsed.content)

        template = Template(parsed.content)

        result = template.render(self.config)

        combined_config = self.config
        combined_config['content'] = result

        templatetxt = '{{content}}'

        if 'template' in combined_config:
            with open(presite.selected_template_folder() +
                      combined_config['template'] + '.html') as f:
                templatetxt = f.read()

        template = Template(templatetxt)
        result = template.render(combined_config)

        currentdir = './'

        for dir in self.dest_path_elements[:-1]:
            currentdir = currentdir + dir + '/'
            if not os.path.exists(currentdir):
                os.mkdir(currentdir)

        with open(self.dest_path, 'w+') as f:
            f.write(result)
            print('wrote ' + self.dest_path)
예제 #5
0
def add(post_path):
    """Add shorturl to a post."""
    filename = basename(post_path)
    m = env.re_file_foramt.match(filename)

    date_obj = datetime.strptime(
        '{0}-{1}-{2}'.format(m.group('year'), m.group('month'),
                             m.group('day')), '%Y-%m-%d')

    post_url = env.tpl_post_url.format(date=date_obj.strftime('%Y/%j'),
                                       slug=m.group('slug'))

    with codecs.open(join(env.root_dir, post_path), 'r', 'utf-8') as fobj:
        post = frontmatter.loads(fobj.read())

    if post.get('shorturl'):
        abort('Post already has a short url: ' '{shorturl}.'.format(**post))

    meta = post.to_dict()
    content = meta['content']
    del (meta['content'])

    meta['shorturl'] = shorturl(post_url)

    with codecs.open(join(env.root_dir, post_path), 'w', 'utf-8') as fobj:
        new_post = frontmatter.Post(content, **meta)
        frontmatter.dump(new_post, fobj)

    puts('Added the short url: {shorturl}.'.format(**meta))
예제 #6
0
    def _getData(self):
        result = urlfetch.fetch(BASE_URI + self.key)

        if result.status_code == 200:
            return frontmatter.loads(result.content)
        else:  # TODO handle other cases
            pass
예제 #7
0
def parse_to_recipe(content: str) -> Recipe:
    """Parse a Markdown formatted string to a Recipe object

    Args:
        content (str): A Markdown formatted string

    Returns:
        Recipe: A Recipe object representing the given content
    """

    recipe_metadata = frontmatter.loads(content)

    document = marko.parse(content)
    recipe_name = get_recipe_name(document)
    quote = get_quote(document)
    ingredients = get_ingredients(document)
    instructions = get_instructions(document)

    return Recipe(
        name=recipe_metadata["name"],
        residence=recipe_metadata["residence"],
        category=RecipeCategory(recipe_metadata["category"].lower()),
        recipe_name=recipe_name,
        quote=quote,
        ingredients=ingredients,
        instructions=instructions,
    )
예제 #8
0
def load_logos(args):

    retVal = dict()

    found_files = pathlib.Path(args.logodir).glob('**/index.md')
    for fn in found_files:
        f = open(str(fn))
        fmstr = f.read()
        f.close()

        fm = frontmatter.loads(fmstr)

        website = fm["website"]
        if website.startswith("http:"):
            website = website[:4] + "s" + website[4:]
        if website.startswith("https://www."):
            website = "https://" + website[12:]

        if "guide" in fm.metadata:
            guide = fm["guide"]
        else:
            guide = ""

        data = {
            "logohandle": fm["logohandle"],
            "website": website,
            "guide": guide
        }

        retVal[fm["logohandle"]] = data

    if args.verbose:
        sys.stdout.write("INFO: %d logos loaded\n" % len(retVal))

    return retVal
예제 #9
0
def sync_file_to_block(filename, block, links: dict = {}):
    logger.info(f"Syncing {filename} to block {block.id}")

    with open(filename) as markdown_fd:
        contents = markdown_fd.read()

    post = frontmatter.loads(contents)

    def resolve_link(target):
        try:
            parsed = urlparse(target)

            if parsed.scheme:
                return target
        except:
            pass

        target_path = os.path.realpath(
            os.path.join(os.path.dirname(filename), target))

        block = links.get(target_path)

        if not block:
            return target

        return block.get_browseable_url()

    markdown_blocks = convert(str(post), link_resolver=resolve_link)

    sync_markdown_blocks_to_block(markdown_blocks, block)
예제 #10
0
def parse_matter(string):
    post = frontmatter.loads(string)
    content = None
    backmatter = None

    if post.content.strip().endswith('+++'):
        content = post.content.rsplit("+++", 2)
        backmatter = content[1]
        content = content[0]
    else:
        content = post.content
        backmatter = None

    # Split new comment from backmatter
    comment = None
    if backmatter and '::: Add Comment' in backmatter:
        comment = backmatter.split('::: Add Comment')
        comment = comment[-1].strip().splitlines()[1:]
        comment = "\n".join(comment).strip()

    if not comment:
        comment = []

    return {
        'frontmatter': post.metadata,
        'content': content,
        'backmatter': backmatter,
        'comment': comment
    }
예제 #11
0
    def build(cls, path):
        with open(path, "r", encoding="utf-8") as input_file:
            fm = frontmatter.loads(input_file.read())
        slug = PathFromFile.build(path).slug
        is_blog_entry = cls.__is_blog_entry(fm.metadata)

        return cls(slug, fm.metadata, is_blog_entry)
예제 #12
0
    def test_custom_handler(self):
        "allow caller to specify a custom delimiter/handler"

        # not including this in the regular test directory
        # because it would/should be invalid per the defaults
        custom = textwrap.dedent("""
        ...
        dummy frontmatter
        ...
        dummy content
        """)

        # and a custom handler that really doesn't do anything
        class DummyHandler(object):
            def load(self, fm):
                return {'value': fm}

            def split(self, text):
                return "dummy frontmatter", "dummy content"

        # but we tell frontmatter that it is the appropriate handler
        # for the '...' delimiter
        # frontmatter.handlers['...'] = DummyHandler()
        post = frontmatter.loads(custom, handler=DummyHandler())

        self.assertEqual(post['value'], 'dummy frontmatter')
예제 #13
0
    def getPage(self, pageName, isMobile):
        if self.data.platform == "PYINSTALLER":
            lhome = os.path.join(self.data.platformHome)
        else:
            lhome = "."
        filename = lhome + pageName
        if filename.find("http") != -1:
            print("external link")
            return None
        with open(filename) as f:
            page = frontmatter.loads(f.read())
        pageContent = page.content
        #filterString = '([^\!]|^)\[(.+)\]\((.+)\)'
        filterString = '(?:[^\!]|^)\[([^\[\]]+)\]\((?!http)([^()]+)\)'
        filteredPage = re.sub(
            filterString,
            r"""<a href='#' onclick='requestPage("\2");'>\1</a>""",
            pageContent)
        filteredPage = filteredPage.replace("{{ site.baseurl }}{% link ", "")
        filteredPage = filteredPage.replace(".md %}", ".md")
        filteredPage = markdown.markdown(
            filteredPage, extensions=["markdown.extensions.tables"])
        filteredPage = filteredPage.replace("Â", "")
        filteredPage = filteredPage.replace("{: .label .label-blue }", "")
        filteredPage = filteredPage.replace("<a href=\"http",
                                            "<a target='_blank' href=\"http")
        filteredPage = filteredPage.replace("<table>", "<table class='table'>")

        if isMobile:
            # make all images 100%
            filteredPage = filteredPage.replace("img alt=",
                                                "img width='100%' alt=")

        return filteredPage
    def add_slideshare_embed_urls(self, blog_posts):
        count = 0
        # Loop through all the markdown posts in the directory
        for post in blog_posts:
            front_matter = frontmatter.loads(open(post, "r").read())
            # Get Slideshare URL from each blog post and add the embed url
            try:
                slideshare_url = front_matter["slideshare_presentation_url"]
                if slideshare_url == "None":
                    pass
                else:
                    #resource_url =  front_matter["link"]
                    embed_url = get_slideshare_embed_url(slideshare_url)
                    if front_matter["slideshare_embed_url"] == embed_url:
                        pass
                    else:
                        front_matter["slideshare_embed_url"] = embed_url
                        with open(post, "w") as changed_file:
                            changed_file.writelines(
                                frontmatter.dumps(front_matter))
                            # print("{0} changed for {1}".format(embed_url, post))
                        count += 1
            except Exception as e:
                print(e)
                pass

        print("{0} posts changed!".format(count))

        return True
예제 #15
0
 def getYAMLProjects(self):
     """ Get projects from projects.yml and appends to self.projects"""
     with open("projects.yml", 'r') as stream:
         try:
             yaml_data = yaml.safe_load(stream)
             for each in yaml_data:
                 md_file = frontmatter.loads(open("template.md","r").read())
                 # Add front matter attributes
                 md_file['title'] = each["name"]
                 md_file['image'] = "/assets/images/projects/" + each["image"]
                 # Prepend the output folder and the new_post_name
                 file_name = each["name"].replace(" ", "-").lower() +  ".md"
                 if each["url"]:
                     md_file["url"] = each["url"]
                     # response = requests.get(each["url"])
                     # soup = BeautifulSoup(response.text, features="html.parser")
                     # metas = soup.find_all('meta')
                     # description = [ meta.attrs['content'] for meta in metas if 'name' in meta.attrs and meta.attrs['name'] == 'description' ]
                     # if len(description) > 0:
                     #     md_file["description"] = '"{0}"'.format(description[0])
                 output_object = "projects/" + file_name
                 # Create the new post object and write the front matter to post.
                 with open(output_object,"w") as new_post_file:
                     new_post_file.writelines(frontmatter.dumps(md_file))
                     print("Jekyll project created for {0} at {1}".format(each["name"], output_object))
         except yaml.YAMLError as exc:
             print(exc)
예제 #16
0
    def from_md(cls, md_content: str):
        """
        Class method to generate new dataobj from a well formatted markdown string

        Call like this:

        ```python
        Dataobj.from_md(content)

        ```
        """
        data = frontmatter.loads(md_content)
        dataobj = {}
        dataobj["content"] = data.content
        for pair in ["tags", "id", "title", "path"]:
            try:
                dataobj[pair] = data[pair]
            except KeyError:
                # files sometimes get moved temporarily by applications while you edit
                # this can create bugs where the data is not loaded correctly
                # this handles that scenario as validation will simply fail and the event will
                # be ignored
                break

        dataobj["type"] = "processed-dataobj"
        return cls(**dataobj)
예제 #17
0
def process(logohandle, logodir):

    srcfile = os.path.join(logodir, "index.md")

    if not os.path.isfile(srcfile):
        print("WARNING: no index file for '%s' (%s)" % (logohandle, srcfile))
        return

    print("INFO: processing %s" % (logohandle))

    fmsrc = frontmatter.load(srcfile)
    if "images" not in fmsrc.metadata:
        print("WARNING: no images, skipping")
        return

    fmdst = frontmatter.loads("---\nlayout: amp\nnoindex: true\n---")
    fmdst[
        "redirect_to"] = "https://www.vectorlogo.zone/logos/%s/index.html" % logohandle

    dstfile = os.path.join(logodir, "index.amp.html")

    f = open(dstfile, 'w')
    f.write(frontmatter.dumps(fmdst))
    f.write('\n')
    f.close()
예제 #18
0
def test_md_get_node_for_key(md_path: pathlib.Path) -> None:
    """Test node fetching."""
    contents = frontmatter.loads(
        md_path.open('r', encoding=const.FILE_ENCODING).read())
    markdown_wo_header = contents.content
    lines = markdown_wo_header.split('\n')

    tree: MarkdownNode = MarkdownNode.build_tree_from_markdown(lines)
    assert tree.get_node_for_key('header1') is None
    assert tree.get_node_for_key('nonexisting header') is None
    node: MarkdownNode = tree.get_node_for_key('# 2. MD Header 2 Blockquotes')
    assert node is not None
    # Assert returned node has content
    assert node.key == '# 2. MD Header 2 Blockquotes'
    assert len(node.content.blockquotes) == 2
    assert node.content.text[2] == 'some text after blockquote'
    # Assert unstrict and strict matching return same notes
    deep_node_unstrict = tree.get_node_for_key('5.2.2.1',
                                               strict_matching=False)
    deep_node_strict = tree.get_node_for_key(
        '#### 5.2.2.1 A even deeper section here 2')
    assert deep_node_strict == deep_node_unstrict
    # Assert substrings are matched
    node = tree.get_node_for_key('Header 4 Tricky', strict_matching=False)
    assert node is not None
    # Assert first match returned if strict matching is off
    node = tree.get_node_for_key('5.1.1', strict_matching=False)
    assert node.key == '### 5.1.1 A deeper section 1'
예제 #19
0
    def update_existing_posts(self, sessions, users):
        """Take the latest export of sessions and users and update any information that has changed"""
        if self._post_location:
            count = 0
            blog_posts = self.get_blog_posts(self._post_location)
            # Loop through all the markdown posts in the directory
            for post in blog_posts:
                changed = False
                front_matter = frontmatter.loads(open(post,"r").read())
                for session in sessions:
                    if session['session_id'] == front_matter['session_id']:
                        # Gather speaker information
                        emails = session["speakers"].split(",")
                        names = []
                        for speaker_email in emails:
                            for attendee in users:
                                if attendee["speaker_email"] == speaker_email:
                                    name = attendee["first_name"] + " " + attendee["second_name"]
                                    bio_formatted =  f'"{attendee["bio"]}"'
                                    speaker_dict = {
                                            "name":name,
                                            "biography": bio_formatted,
                                            "job-title": attendee["job-title"],
                                            "company": attendee["company"],
                                            "speaker-image": attendee["image-name"]
                                            }
                                    names.append(speaker_dict)
                        # Check if there are changes to speakers
                        if front_matter['speakers'] != names:
                            front_matter['speakers'] = names
                            changed = True

                        # Check if session tracks have changed.
                        tracks = session["tracks"].replace(";",", ")
                        if front_matter["session_track"] != tracks:
                            front_matter["session_track"] = tracks
                            changed = True
                        
                        # Check if title has changed
                        title = re.sub('[^A-Za-z0-9-!: ()]+', '', session["title"])
                        if front_matter['title'] != title:
                            front_matter['title'] = title
                            changed = True
                        
                        # Check if post content has changed
                        content = session['blurb']
                        if front_matter.content != content:
                            front_matter.content = content
                            changed = True
                        
                        if changed:
                            # Write the changed frontmatter to the file.
                            with open(post,"w") as changed_file:
                                changed_file.writelines(frontmatter.dumps(front_matter))
                                print("{0} post updated!".format(session['session_id']))
                                count += 1
            print("{0} posts updated!".format(count))
                        
        else:
            return False
예제 #20
0
def _build_content_pages(items: List[ContentItem]) -> Iterator[Page]:
    for item in items:
        post = fm.loads(item.content)
        content = post.content
        html = resources.markdown.reset().convert(content)
        permalink = _build_permalink(item.location)
        image, image_thumbnail = _process_image(post)
        frontmatter = Frontmatter(
            title=post["title"],
            description=post.get("description"),
            category=post.get("category"),
            date=post.get("date"),
            image=image,
            image_thumbnail=image_thumbnail,
            image_caption=post.get("image_caption"),
            tags=post.get("tags", []),
        )
        meta = _build_meta(permalink, frontmatter)

        yield Page(
            html=html,
            content=content,
            permalink=permalink,
            frontmatter=frontmatter,
            meta=meta,
        )
예제 #21
0
def parse_markdown(parser, template, filepath, metadata):
    parser.reset()
    metadata = deepcopy(metadata)

    # Try to extract frontmatter metadata
    with open(filepath, encoding="utf-8") as markdown_file:
        file_content = markdown_file.read()

        try:
            file_parts = frontmatter.loads(file_content)
            metadata.update(file_parts.metadata)
            metadata["content"] = parser.convert(file_parts.content)
        except (ScannerError, ParserError):
            """
            If there's a parsererror, it's because frontmatter had to parse
            the entire file (not finding frontmatter at the top)
            and encountered an unexpected format somewhere in it.
            This means the file has no frontmatter, so we can simply continue.
            """

            metadata["content"] = parser.convert(file_content)
        except ParseError:
            """
            If there is a parse error in a file, it is useful to know
            which file it is
            """

            print("Error parsing file: {}".format(filepath))
            raise

    # Now add on any multimarkdown-format metadata
    if hasattr(parser, "Meta"):
        # Restructure markdown parser metadata to the same format as we expect
        markdown_meta = parser.Meta

        for name, value in markdown_meta.items():
            if type(value) == list and len(value) == 1:
                markdown_meta[name] = value[0]

        metadata.update(markdown_meta)

    toc_soup = BeautifulSoup(parser.toc, "html.parser")

    nav_item_strings = []

    # Only get <h2> items, to avoid getting crazy
    for item in toc_soup.select(".toc > ul > li > ul > li"):
        for child in item("ul"):
            child.extract()

        item["class"] = "p-toc__item"

        for anchor in item("a"):
            anchor["class"] = "p-toc__link"

        nav_item_strings.append(str(item))

    metadata["toc_items"] = "\n".join(nav_item_strings)

    return template.render(metadata)
예제 #22
0
    def test_custom_handler(self):
        "allow caller to specify a custom delimiter/handler"

        # not including this in the regular test directory
        # because it would/should be invalid per the defaults
        custom = textwrap.dedent("""
        ...
        dummy frontmatter
        ...
        dummy content
        """)

        # and a custom handler that really doesn't do anything
        class DummyHandler(object):
            def load(self, fm, Loader=None):
                return {'value': fm}

            def split(self, text):
                return "dummy frontmatter", "dummy content"

        # but we tell frontmatter that it is the appropriate handler
        # for the '...' delimiter
        # frontmatter.handlers['...'] = DummyHandler()
        post = frontmatter.loads(custom, handler=DummyHandler())

        self.assertEqual(post['value'], 'dummy frontmatter')
예제 #23
0
파일: migrate.py 프로젝트: johnsonhit/blog
def _migrate(post):
    print('migrate post {}'.format(post))
    # 处理首行不含 --- 的文档
    with open(post) as f:
        contents = f.read()
        if not contents.startswith('---'):
            contents = '---\n' + contents

    post_yaml = frontmatter.loads(contents)
    _date = post_yaml['date']
    if not isinstance(_date, str):
        _date = _date.strftime('%Y-%m-%d %H:%M:%S')
    try:
        _date_dt = datetime.strptime(_date, '%Y-%m-%d %H:%M:%S')
    except ValueError as e:
        _date_dt = datetime.strptime(_date, '%Y-%m-%d %H:%M')
    except Exception as e:
        print(e)
        raise

    metadata = {}
    metadata['created'] = _date_dt.strftime('%Y-%m-%d_%H-%M-%S')

    title = post_yaml['title']
    metadata['title'] = title
    # 正文首行补充一级标题
    content = '# ' + title + '\n\n' + post_yaml.content
    return (metadata, content)
예제 #24
0
    def _extract_metadata(self, text):
        # fast test
        if not text.startswith("---"):
            return text

        content = frontmatter.loads(text)
        self.metadata = content.metadata
        return content.content
예제 #25
0
 def __init__(self, content):
     parsed = frontmatter.loads(text_or_list(content))
     self.content = parsed.content
     for k in ('args', 'function', 'returns'):
         try:
             setattr(self, k, parsed[k])
         except KeyError:
             setattr(self, k, None)
예제 #26
0
파일: tags.py 프로젝트: Kronifer/bot
 def __init__(self, content_path: Path):
     post = frontmatter.loads(content_path.read_text("utf8"))
     self.file_path = content_path
     self.content = post.content
     self.metadata = post.metadata
     self._restricted_to: set[int] = set(
         self.metadata.get("restricted_to", ()))
     self._cooldowns: dict[discord.TextChannel, float] = {}
예제 #27
0
    def _extract_metadata(self, text):
        # fast test
        if not text.startswith("---"):
            return text

        content = frontmatter.loads(text)
        self.metadata = content.metadata
        return content.content
예제 #28
0
def jekyll_format(s: str, p: Parser) -> Entry:
    post = frontmatter.loads(s)
    entry = Entry(title=post["title"],
                  text=post.content,
                  updated_at=parse(str(post["updated"])),
                  meta=post.metadata,
                  p=p)
    return entry
예제 #29
0
def main(input, output, extra_context):
    chunk = input.read()
    post = frontmatter.loads(chunk)

    if extra_context:
        post.metadata.update(extra_context)

    frontmatter.dump(post, output)
예제 #30
0
def test_tree_text_equal_to_md(md_path: pathlib.Path) -> None:
    """Test tree construction."""
    contents = frontmatter.loads(
        md_path.open('r', encoding=const.FILE_ENCODING).read())
    markdown_wo_header = contents.content
    lines = markdown_wo_header.split('\n')

    tree: MarkdownNode = MarkdownNode.build_tree_from_markdown(lines)
    assert markdown_wo_header == tree.content.raw_text
예제 #31
0
def get_entries(drive_service, file_list):
    entries = []

    raw_files = get_raw_files(drive_service, file_list)

    for raw_file in raw_files:
        entries.append(frontmatter.loads(raw_file))

    return entries
예제 #32
0
파일: sync.py 프로젝트: adamfast/lfk.im
def sync_cuisines():
    click.echo("sync-cuisines")

    aliases = load_aliases()
    cuisine_aliases = aliases["cuisines"]

    data = []
    places = Path("_places").glob("*.md")
    for place in places:
        post = frontmatter.loads(place.read_text())
        cuisines = post["cuisines"]
        if cuisines and len(cuisines):
            data += cuisines

    if not Path("_cuisines").exists():
        Path("_cuisines").mkdir()

    for cuisine in CUISINE_INITIAL:
        cuisine_slug = slugify(cuisine)
        if not Path("_cuisines").joinpath(f"{cuisine_slug}.md").exists():
            post = frontmatter.loads("")
            post["active"] = True
            post["name"] = cuisine
            post["sitemap"] = True
            post["slug"] = cuisine_slug

            Path("_cuisines").joinpath(f"{cuisine_slug}.md").write_text(
                frontmatter.dumps(post))

    data = set(data)

    for cuisine in data:
        cuisine_slug = slugify(cuisine)
        if not any(
            [alias for alias in cuisine_aliases if cuisine in alias["name"]]):
            if not Path("_cuisines").joinpath(f"{cuisine_slug}.md").exists():
                post = frontmatter.loads("")
                post["active"] = False
                post["name"] = cuisine
                post["sitemap"] = False
                post["slug"] = cuisine_slug

                Path("_cuisines").joinpath(f"{cuisine_slug}.md").write_text(
                    frontmatter.dumps(post))
def _getMarkdownFrontMatter(text):
    """
    Args:
        text: string of markdown

    Return:
        frontmatter in json format
    """
    data = frontmatter.loads(text)
    return data
예제 #34
0
def before_change_note(mapper, connection, target):
    title = None

    data = frontmatter.loads(target.text)

    if isinstance(data.get('title'), str) and len(data.get('title')) > 0:
        title = data.get('title')

    if title and not target.is_date:
        target.name = title
def _getMarkdownFrontMatter(text):
    """
    Args:
        text: string of markdown

    Return:
        frontmatter in json format
    """
    data = frontmatter.loads(text)
    return data
예제 #36
0
    def sanity_check(self, filename, handler_type):
        "Ensure we can load -> dump -> load"
        post = frontmatter.load(filename)

        self.assertIsInstance(post.handler, handler_type)

        # dump and reload
        repost = frontmatter.loads(frontmatter.dumps(post))

        self.assertEqual(post.metadata, repost.metadata)
        self.assertEqual(post.content, repost.content)
        self.assertEqual(post.handler, repost.handler)
예제 #37
0
def parse_frontmatter(markdown_content):
    metadata = {}
    try:
        file_parts = frontmatter.loads(markdown_content)
        metadata = file_parts.metadata
    except (ScannerError, ParserError):
        """
        If there's a parsererror, it's because frontmatter had to parse
        the entire file (not finding frontmatter at the top)
        and encountered an unexpected format somewhere in it.
        This means the file has no frontmatter, so we can simply continue.
        """
        pass
    return metadata
예제 #38
0
def tags(root='.'):
    """Build and return tag -> [files] dict."""
    tags_dict = dict()
    for dirpath, filename, file, date in traverse_posts(root):
        # extract tags from frontmatter
        with open('{}/{}'.format(dirpath, filename), 'r') as f:
            fm = frontmatter.loads(f.read())
            for tag in fm['tags']:
                if tag in tags_dict:
                    tags_dict[tag].append(file)
                else:
                    tags_dict[tag] = [file]

    return OrderedDict(reversed(sorted(tags_dict.items(), key=lambda x: len(x[1]))))
예제 #39
0
파일: api.py 프로젝트: adugan/site
def guides():
  guides = []
  files = []
  for root, dirs, filenames in os.walk("getting_started/data"):
    for f in filenames:
      files.append(f)

  files = sorted(files, reverse=True)

  for f in files:
    slug = f.replace(".md", "")
    markdown = frontmatter.loads(open(os.path.join(root, f),'r').read())
    guides.append({"content": markdown.content, "metadata": markdown.metadata, "slug": slug})

  return jsonify({"guides": guides})
예제 #40
0
def create_block(message, block_path):
    block = frontmatter.loads('')
    last_block_hash = get_last_block_hash(block_path)
    if last_block_hash:
        block['prev'] = last_block_hash
    block['created_at'] = datetime.datetime.now().isoformat()
    block['nonce'] = 0
    block.content = message
    while True:
        block['nonce'] += 1
        block_hash = hashlib.sha256(
            frontmatter.dumps(block).encode()
        ).hexdigest()
        if block_hash[:2] == '00':
            break
    return block, block_hash
예제 #41
0
def categories(root='.'):
    """Build and return cat -> [files] dict."""
    cats_dict = dict()
    for dirpath, filename, file, date in traverse_posts(root):
        # extract title and categories from frontmatter
        with open('{}/{}'.format(dirpath, filename), 'r') as f:
            fm = frontmatter.loads(f.read())
            title, cat = fm['title'], fm['categories'].replace(' ', '/')
            file = {'date': date, 'file': file, 'title': title}
            for base_cat in parse_base_cats(cat):
                if not base_cat in cats_dict:
                    cats_dict[base_cat] = []
            if cat in cats_dict:
                cats_dict[cat].append(file)
            else:
                cats_dict[cat] = [file]

    return OrderedDict(sorted(cats_dict.items(), key=lambda x: x[0]))
예제 #42
0
def readfile(filename, section=None):
    """Opens a filename and returns either yaml or content"""
    try:
        with open(filename, 'rb') as f:
            post = frontmatter.loads(f.read())
            if section == 'content':
                # TODO:
                # Check case of \r\n for Windows
                # WARNING: Removes trailing newlines
                return post.content.splitlines()
            elif section == 'metadata':
                # WARNING: Implicitly converts dates to datetime
                return post.metadata

    except (LookupError, SyntaxError, UnicodeError, scanner.ScannerError):
        # Return Error; require utf-8
        # Should this sys.exit(1) here?
        print(f"{filename} caused the Blueberry script to crash.")
        sys.exit(1)
예제 #43
0
    def test_external(self):
        filename = self.data['filename']
        content = self.data['content']
        metadata = self.data['metadata']
        content_stripped = content.strip()

        post = frontmatter.load(filename)

        self.assertEqual(post.content, content_stripped)
        for k, v in metadata.items():
            self.assertEqual(post[k], v)

        # dumps and then loads to ensure round trip conversions.
        posttext = frontmatter.dumps(post, handler=self.handler)
        post_2 = frontmatter.loads(posttext)

        for k in post.metadata:
            self.assertEqual(post.metadata[k], post_2.metadata[k])

        self.assertEqual(post.content, post_2.content)
def run(username, save_folder, prefix=None, data_file_name="gist_data.json"):
	gist_all = get_all_gists_for_user(username)
	gist_filenames = get_all_gist_filenames(gist_all)
	add_json_filenames_to_list(gist_filenames)
	delete_non_matching_files_from_folder(gist_filenames, save_folder)

	for gist in gist_all:
		content_data = get_content_data_for_gist(gist, prefix)
		if not content_data:
			continue
		file_path = os.path.join(save_folder, content_data["name"])
		file_data_path = os.path.join(save_folder, content_data["name"].replace('.md', '.json'))
		markdown_content = download_markdown_gist(file_path, content_data["data"])
		if markdown_content:
			gist_frontmatter = frontmatter.loads(markdown_content)
			content = gist_frontmatter.content
			metadata = gist_frontmatter.metadata
			content_data["content"] = content
			content_data["metadata"] = metadata
			with open(file_data_path, 'w') as target_file:
				json.dump(content_data, target_file, indent=True)
예제 #45
0
def find_old_guides(count=20):
    """Print a list of the 20 oldest guides in the library.

    Results are sorted by modification date (in front matter)
    and deprecated guides are ignored.

    Command line arguments:
    count: number of guides to list (default: 20)
    """
    old_guides = []
    guides_scanned = 0
    rootdir = 'docs'
    for filename in glob.glob('docs/**/*.md',recursive=True):
        guides_scanned += 1
        with open(filename, 'r') as f:
            yaml = frontmatter.loads(f.read())
            if 'modified' in yaml and 'deprecated' not in yaml:
                record = make_record(parsed)
                record['path'] = filename
                old_guides.append(record)
    print(str(guides_scanned) + " guides scanned.")
    old_guides.sort(key=itemgetter('updated'))
    oldest_guides = old_guides[0:count]
    print(tabulate(oldest_guides))        
예제 #46
0
 def __init__(self, bookmark):
     self.bookmark = bookmark
     self.frontmatter = frontmatter.loads(bookmark.extended)
예제 #47
0
def test_underscore_alias():
    with open('ci/data/underscore-alias.md','r') as f:
        yaml = frontmatter.loads(f.read()).metadata

    assert valid_alias(yaml)[1] == "applications/containers/this_is_an_alias/ should use dashes instead of underscores."
예제 #48
0
def test_uppercase_alias():
    with open('ci/data/SHOUTING_BAD_FILE.md','r') as f:
        yaml = frontmatter.loads(f.read()).metadata

    assert valid_alias(yaml)[1] == "applications/containers/this-is-an-ALIAS/ should be lowercase."
예제 #49
0
import datetime

for p in pathlib.Path('import_versions/exitwp/_posts').glob('*.markdown'):
    fm = frontmatter.load(str(p))
    tags = fm['tags']
    cats = fm['categories']

    tags_str = "tags:\n" + '\n'.join("- {}".format(tag) for tag in tags)
    cats_str = "categories:\n" + '\n'.join("- {}".format(cat) for cat in cats)

    try:
        dest_post = '_posts/{}.md'.format(p.stem)
        with open(dest_post) as f:
            content = f.read()
    except FileNotFoundError:
        p_split = p.stem.split('-', 3)
        date_prefix = [int(x) for x in p_split[:3]]
        orig_date = datetime.date(*date_prefix)
        new_date = orig_date - datetime.timedelta(1)
        dest_post = '_posts/{:%Y-%m-%d-}{}.md'.format(new_date, p_split[-1])
        with open(dest_post) as f:
            content = f.read()

    nfm = frontmatter.loads(content)
    if not any((nfm.get('tags'), nfm.get('categories'))):
        loc = content.find('\n---\n')
        new_content = "{}\n{}\n{}{}".format(content[:loc], tags_str, cats_str,
                                            content[loc:])
        with open(dest_post, 'w') as f:
            f.write(new_content)
예제 #50
0
def test_valid_alias():
    with open('ci/data/goodfile.md', 'r') as f:
        yaml = frontmatter.loads(f.read()).metadata

    assert valid_alias(yaml) == None
예제 #51
0
                    nargs='?', default=sys.stdin,
                    help='Markdown file to preprocess')
parser.add_argument('output', type=argparse.FileType('w'),
                    nargs='?', default=sys.stdout,
                    help='the name of the output file (defaults to stdout)')
args = parser.parse_args()

# Save arguments
input_file = args.input_file
output = args.output

# Bump up all headings line by line
def increase_heading_level(line):
    return(re.sub("^(#+)", "\\1#", line, flags=re.MULTILINE))


# Parse the final string for YAML frontmatter
parsed_yaml = frontmatter.loads(args.input_file.read())

# If there's a title key, extract it, bump up the headings, and add the title
# to the beginning of the file
if 'title' in parsed_yaml.keys():
    chapter_title = parsed_yaml['title']
    final_result = '# ' + chapter_title + '\n\n' + increase_heading_level(parsed_yaml.content)
else:
    final_result = parsed_yaml.content

# All done
with output as f:
    f.write(final_result)
예제 #52
0
 def test_with_crlf_string(self):
     import codecs
     markdown_bytes = b'---\r\ntitle: "my title"\r\ncontent_type: "post"\r\npublished: no\r\n---\r\n\r\nwrite your content in markdown here'
     loaded = frontmatter.loads(codecs.decode(markdown_bytes, 'utf-8'))
     self.assertEqual(loaded['title'], 'my title')
예제 #53
0
def test_required_yaml():
    with open('ci/data/goodfile.md','r') as f:
        yaml = frontmatter.loads(f.read()).metadata
    assert require_yaml(yaml) == None
예제 #54
0
 def test_micropost_extracts_tags(self):
     filename = self.micropostCmd.run(post='this is a post #giggles #stuff', network=None)
     post = frontmatter.loads(self.site.load_file(filename))
     self.assertEquals('giggles,stuff', post.metadata['tags'])
예제 #55
0
#args = parser.parse_args()

con = sqlite3.connect("_build/out/dv.db")
con.row_factory = sqlite3.Row
cur = con.cursor()

cur.execute("select * from meta_article");

r = cur.fetchone()
while r:
    md = pypandoc.convert(r['article'], 'markdown_strict', format='html')
    title_md = pypandoc.convert(r['title'], 'markdown_strict', format='html', extra_args=["--columns", "9999"])

    md = fix(articlelink(md))
    title_md = fix(title_md, title=True)
    
    post = frontmatter.loads("")
    post.content= md
    post['title'] = title_md
    post['date'] = r['date']
    post['author'] = r['author']
    post['id'] = r['id']
    
    f = "_meta/" + r['date'] + "-" + yamltitle(title_md) + ".md"
    out = codecs.open(f, 'w', 'utf-8')
    frontmatter.dump(post, out, Dumper=frontmatter.yaml.Dumper, allow_unicode=True)

    r = cur.fetchone()


예제 #56
0
def test_extra_yaml_header():
    with open('ci/data/extra_yaml_header.md','r') as f:
        yaml = frontmatter.loads(f.read()).metadata
    assert only_allowed_yaml(yaml)[1] == "Non-allowed metadata: fake_header"
예제 #57
0
def test_missing_yaml_header():
    with open('ci/data/missing_yaml_header.md','r') as f:
        yaml = frontmatter.loads(f.read()).metadata
    assert require_yaml(yaml)[1] == "Missing required metadata: published"
예제 #58
0
def main():

    # input is teams csv datafile from TBA
    # -> https://github.com/the-blue-alliance/the-blue-alliance-data
    csv_fname = abspath(sys.argv[1])
    max_team = int(sys.argv[2])
    mode = sys.argv[3]
    
    if mode not in ['new', 'update']:
        print("Error: invalid mode")
        return

    os.chdir(abspath(join(dirname(__file__), '..')))
    cwd = os.getcwd()
    
    for row in read_team_csv(csv_fname):
        # this changes on occasion...
        number, name, sponsors, l1, l2, l3, website, rookie_year, \
            facebook, twitter, youtube, github, instagram, periscope = row
        
        name = name
        rookie_year = rookie_year
        
        if rookie_year:
            rookie_year = int(rookie_year)
        
        number = number[3:]
        if int(number) > max_team:
            continue
        
        d1 = '%04d' % (int(int(number)/1000)*1000,)
        d2 = '%03d' % (int(int(number)/100)*100,)
        
        f = join(cwd, 'frc%s' % d1, '_frc', d2, '%s.md' % number)
        
        if mode == 'new' and exists(f):
            continue
        
        if 'firstinspires' in website:
            website = ''
        
        if l3:
            location = '%s, %s, %s' % (l1, l2, l3)
        elif l2:
            location = '%s, %s' % (l1, l2)
        else:
            location = l1
            
        sponsors = [s.strip() for s in sponsors.split('/')]
        if sponsors == ['']:
            sponsors = None
        else:
            if '&' in sponsors[-1]:
                sN = sponsors[-1].split('&')
                del sponsors[-1]
                sponsors += [s.strip() for s in sN]
        
        if mode == 'update':
            try:
                fm = frontmatter.load(f)
            except:
                print("Error at %s" % f)
                raise
                
            reformatted = str(frontmatter.dumps(fm))
            
            if 'team' not in fm.metadata:
                raise Exception("Error in %s" % f)
                
            team = fm.metadata['team']
            if 'links' not in fm.metadata['team']:
                links = OrderedDict()
            else:
                links = fm.metadata['team']['links']
        else:
            data = OrderedDict()
            team = OrderedDict()
            links = OrderedDict()
            
            data['title'] = 'FRC Team %s' % number
            data['team'] = team
            
            team['type'] = 'FRC'
            team['number'] = int(number)
        
        add_maybe(team, 'name', name)
        add_maybe(team, 'rookie_year', rookie_year)
        add_maybe(team, 'location', location)
        
        if sponsors and mode != 'update':
            team['sponsors'] = sponsors
        
        if 'Github' in links:
            links['GitHub'] = links['Github']
            del links['Github']
        
        add_maybe_web(links, 'Website', website)
        add_maybe_web(links, 'Facebook', facebook)
        add_maybe_web(links, 'Twitter', twitter)
        add_maybe_web(links, 'YouTube', youtube)
        add_maybe_web(links, 'GitHub', github)
        add_maybe_web(links, 'Instagram', instagram)
        add_maybe_web(links, 'Periscope', periscope)
        
        if mode == 'update':
            
            if links:
                fm.metadata['team']['links'] = links
            
            if fm.content.strip() == 'No content has been added for this team':
                fm.content = '{% include remove_this_line_and_add_a_paragraph %}'
            
            page = str(frontmatter.dumps(fm))
            if reformatted == page:
                # don't make gratuitious changes
                continue
        
        elif mode == 'new':
            
            if links:
                team['links'] = links
        
            page = '---\n%s\n---\n\n{%% include remove_this_line_and_add_a_paragraph %%}\n' % (
                yaml.safe_dump(data)
            )
            
            # roundtrip through frontmatter to get the formatting consistent
            page = frontmatter.dumps(frontmatter.loads(page))
            
        if not exists(dirname(f)):
            os.makedirs(dirname(f))
            
        with open(f, 'w') as fp:
            fp.write(page)
예제 #59
0
def test_alias_trailing_slash():
    with open('ci/data/trailing-slash-alias.md','r') as f:
        yaml = frontmatter.loads(f.read()).metadata

    assert valid_alias(yaml)[1] == "applications/containers/this-is-an-alias should end with a slash (/)."