def parse_article(self, url): html = requests.get(url=url, headers=self.headers).text html = bs(html, features="lxml") html = html.find("div", class_="content") md = Tomd(str(html)).markdown md = md.replace("\n", "") return md
def parse_news(self, response): log.msg("Start to parse news " + response.url, level=log.INFO) item = SpiderNewsAllItem() day = title = _type = keywords = url = article = '' url = response.url day = response.meta['day'] title = response.meta['title'] _type = response.meta['_type'] response = response.body soup = BeautifulSoup(response) # try: # items_keywords = soup.find(class_='ar_keywords').find_all('a') # for i in range(0, len(items_keywords)): # keywords += items_keywords[i].text.strip() + ' ' # except: # log.msg("News " + title + " dont has keywords!", level=log.INFO) try: ################################## # 分情况获取储存新闻内容的标签 # "码云推荐",获取项目简介(通常是 README.md 文档内容) if re.search("translate",url): article = soup.find_all("div",class_ = "translate-content") markdown = "".join(str(article)) markdown = Tomd(markdown).markdown article = [tag.text.strip() for tag in article] article = ''.join(article) else: if re.match("https://gitee.com",url): article = soup.find("div",class_="file_content markdown-body")# CSS选择器:#git-readme > div > div.file_content.markdown-body # "码云周刊" elif re.match("https://blog.gitee.com",url): article = soup.find("div",class_="entry-content") elif re.search("translate",url): article = soup.find_all("div",class_ = "translate-content") # 其他常见页面 elif soup.find("div",class_= ["content","box-aw main"]): article = soup.find("div",class_= ["content","box-aw main"]) else: article = soup.find("section",class_= ["wrap cke_editable cke_editable_themed cke_contents_ltr cke_show_borders clearfix"]) if article and not article.find("div",class_="ad-wrap")==None: article.find("div",class_="ad-wrap").extract() markdown = markdown = Tomd(str(article)).markdown article = article.text.strip() #提取标签文本 except: log.msg("News " + title + " dont has article!", level=log.INFO) item['title'] = title item['day'] = day item['_type'] = _type item['url'] = url item['keywords'] = keywords item['article'] = article item['site'] = '开源中国' item['markdown'] = markdown return item
def Combinations(n=1, k=1, tablePrinted=False): var = [1, 0] VARS = {} numOfVars = n numOfPlaces = k condition = str(numOfPlaces) + "==(0" for f in range(numOfVars): VARS["_V" + str(f + 1)] = var condition += "+" + "_V" + str(f + 1) condition += ")" print(""" Number of symbols to allocate:\t\t""" + str(numOfVars) + """ Number of locations to be place in:\t""" + str(numOfPlaces) + """ ValidationCondition:\t""" + condition + """ """) RES = [{ "name": "_ValidCombination", "value": condition }] Evaluation = chipEvaluator(VARS, RES) from tomd import Tomd Table = Tomd(Evaluation).markdown.replace(cSpace, " ") ResTable = Table.split("\n") num = len(ResTable[3].split()) - 2 tot = 0 elements = ResTable[1].replace("|", "").split()[:-1] elements = [elem[1:] for elem in elements] print("Elements:\t", listToSet(elements)) print("\nCombinations:\n{") for f in ResTable: row = [n for n in f.split()] if (len(row) > num and str(row[num]) == "1"): o = f.replace("|", "").split() print("\t{", end=" ") for g in range(len(o) - 1): if (TF(o[g]) == 1): print(elements[g], end=" ") print("}") tot += 1 print(("\t" * numOfPlaces) + "}\n\nTotal of valid results: ", tot, "\n") if (tablePrinted): print("\n", Table) return (0)
def __str__(self): ret = self.text ret = ret.replace('_', '') keymap = { r'\le': r"<=", r'\cdot': r".", r'\ge': r">=", r'\lt': r"<", r'$': r"_", r'\dots': r"..." } for key in keymap: value = keymap[key] ret = ret.replace(key, value) # temprorily convert to html mdProcessor = markdown.Markdown() myHtmlFragment = str(mdProcessor.convert(ret)) myHtmlFragment = myHtmlFragment.replace('<code>', r"```") myHtmlFragment = myHtmlFragment.replace('</code>', r"```") ret = Tomd(myHtmlFragment).markdown # calling like this (all CLI options supported, check def main ret = mdv.main(ret) return ret
def post(update, context): if update.channel_post: if update.channel_post.chat.username == "rememberbox": text = update.channel_post.text entities = update.channel_post.parse_entities() first, *others = text.splitlines() length = len(first) + 2 if first.startswith('👉'): path = first[2:] first = path.split('/')[-1] # remove trigger emoji filename = path.lower().replace(' ', '-') + ".md" rest = parse_bbcode("\n".join(others), length, entities, urled=True) html = "<p>" + unescape(render_html(rest)) + "</p>" context.bot.send_message(chat_id=691609650, text=html) rest = Tomd(html).markdown[length + 1:].replace('<br />', '\n') content = "# " + first + "\n\n" + rest try: commit = repo.create_file( filename, "automated post from telegram channel", content, branch="master") sha = getattr(commit['commit'], 'sha') url = "https://github.com/Lulzx/til/commit/" + sha context.bot.send_message( chat_id=691609650, text="new addition in TIL repository: {}".format(url)) except: pass
def render(self): endnotes = '' if self.refs: self.content = self.refs.process(html.unescape(self.content)) endnotes = '\n' + self.refs.endnotes return Tomd(self.content).markdown + endnotes
def main(args): html = 0 var = [1, 0] VARS = { "_Armed": var, "_Door": var, "_Glass": var, "_Motion": var } RES = [{ "name": "_Alarm", "title": "Armed ∧ ( Door ∨ Glass ∨ Motion )", "value": "AND(_Armed, OR(_Door,_Glass,_Motion ))" }] Evaluation = chipEvaluator(VARS, RES) if (not html): try: from tomd import Tomd print(Tomd(Evaluation).markdown.replace(cSpace, " ")) except: print( "MarkDown format requires Tomd library installed.\nPlease install it doing * pip install tomd *\n" ) exit(1) else: print(Evaluation) return (0)
def main(args): html=0; var=[1,0]; VARS={ "_P" : var, "_Q" : var }; RES=[ { "name" : "(P ∧ Q)", "value" : "AND(_P,_Q)" }, { "name" : "(P ⊽ Q)", "value" : "NOR(_P,_Q)" }, { "name" : "_OUT_MINE=(P ∧ Q) ⊽ ( P ⊽ Q )", "value" : "NOR(AND(_P,_Q),NOR(_P,_Q))" }, { "name" : "(P⊼P)", "value" : "NAND(_P,_P)" }, { "name" : "((P⊼P)⊼Q)", "value" : "NAND(NAND(_P,_P),_Q)" }, { "name" : "(Q⊼Q)", "value" : "NAND(_Q,_Q)" }, { "name" : "((Q⊼Q)⊼P)", "value" : "NAND(NAND(_Q,_Q),_P)" }, { "name" : "_OUT_OPT=((P⊼P)⊼Q)⊼((Q⊼Q)⊼P)", "value" : "NAND(NAND(NAND(_P,_P),_Q),NAND(NAND(_Q,_Q),_P))" }, { "name" : "_XOR=(P⊻Q)", "value" : "XOR(_P,_Q)" } ]; Evaluation=chipEvaluator(VARS,RES); if(not html): try: from tomd import Tomd; print(Tomd(Evaluation).markdown.replace(cSpace," ")); except: print("MarkDown format requires Tomd library installed.\nPlease install it doing * pip install tomd *\n"); exit(1); else: print(Evaluation); return(0);
def render(self): endnotes = '' if self.refs: self.content = self.refs.process(self.content) endnotes = '\n' + self.refs.endnotes return self.title + '\n' + self.url + '\n' + Tomd( self.content).markdown + endnotes
def html2md(url): # 获取html内容 html_data = requests.get(url).text text = Tomd(html_data).markdown # print(text) with open("html.md", "w", encoding='utf-8') as text_file: text_file.write(text)
def DrupalScrap(): links = buildlist() for item in links: get_site = req.Request(item) with req.urlopen(get_site) as response: site_content = response.read() soup = BeautifulSoup(site_content, 'html.parser') # Position on the article text box for img in soup.find_all('img'): img.decompose() header_box = soup.find('h1', attrs={'class': 'page-title'}) article_box = soup.find('div', attrs={'class': 'node__content'}) article_markdown = Tomd(str(article_box)).markdown page_title = Tomd(str(header_box)).markdown # permalink_nospace = header_box.replace(" ", "_") # permalink = permalink_nospace.lower() article = (page_title, article_markdown) return article
def html2mdtest(): html_data = """<pre>ls -a 列出目录所有文件,包含以.开始的隐藏文件 ls -A 列出除.及..的其它文件 ls -r 反序排列 ls -t 以文件修改时间排序 ls -S 以文件大小排序 ls -h 以易读大小显示 ls -l 除了文件名之外,还将文件的权限、所有者、文件大小等信息详细列出来</pre>""" text = Tomd(html_data).markdown print(text)
def _scrap(self, year, day, day_dir): """Scrap data from AoC website.""" url = self._aoc_uri.format(year=year, day=day) response = self.request(url, cookies={"session": self._token}) if not response: print(f"Exercise for {day}/12/{year} is not available") return html = response.text self._year_titles.append(self._pattern_title.search(html).groups()[0]) # Why not group(0) ? Because it returns matching text with pattern, # not just the matching text itself begin = end = 0 if day == 1: # we get the introduction text # No BS4 ? Not for this trivial parsing begin = html.find('</h2>') + len('</h2>') end_str = self._pattern_gl.search(html).group(0) end = html.find(end_str) + len(end_str) self._year_readme_text = Tomd(html[begin:end]).markdown # Get the problem wording begin = end if end else html.find('</h2>') + len('</h2>') end = html.find('</article>') + len('</article>') problem_text = Tomd(html[begin:end]).markdown with open(f"{day_dir}/README.md", 'w') as readme: readme.write(f"# {self._year_titles[-1]}") readme.write(problem_text) # Get the input response = self.request(f"{url}/input", cookies={"session": self._token}) inpiout = response.text with open(f"{day_dir}/input", 'w') as input_file: input_file.write(inpiout)
def htmlToMd(htmlFile, mdFile): print(htmlFile) with open(htmlFile, 'r', encoding="utf-8") as f: contents = f.read() f.close() #print(contents) md = Tomd(contents).markdown #print(md) with open(mdFile, 'w', encoding="utf-8") as f: f.write(md) f.close()
def html2md(data, feed_name=None, rooturl=None): # data = Tomd(data).markdown # data = u'''<!DOCTYPE html><html lang="zh-CN"><head><meta charset="utf-8"></head><body>%s </body></html>''' % data soup = BeautifulSoup(data, 'lxml') #,'html.parser',from_encoding='utf-8' head = str(soup.head) content = soup.find(id='content') if content == None: content = soup.body clears = [ 'h4', ] for c in clears: All = content.findAll(c) for a in All: try: a.find('a').decompose() except Exception as e: pass try: a['class'].clear() except Exception as e: pass dels = ['comments', 'nav-single'] for tag in dels: ts = content.findAll(id=tag) for t in ts: if t != None: t.decompose() imgs = content.findAll('img') for img in imgs: try: parsed = urllib.parse.urlparse(img['src']) if parsed.scheme == '': img['src'] = rooturl + img['src'] except: pass filename = './%s.md' % feed_name data = u'''<!DOCTYPE html><html lang="zh-CN">%s<body>%s </body></html>''' % ( head, str(content)) data = Tomd(data).markdown with open(filename, 'w') as file_writer: file_writer.write(data) file_writer.close() return data
def getAllBody(self,url): soup = BeautifulSoup(fetchHtml(url),"html5lib") body_class = soup.find_all(class_="article-body") body = str(body_class[0]) #print(str(body).replace('\r\n','--')) with open("3.body","w",encoding="utf-8") as f: f.write(body.replace("\n","--")) with open("3.md","w",encoding="utf-8") as f: md = Tomd(body).markdown f.write(md)
def post_entry(self, destination, entry): keys = (self.default_settings if destination["keys"] == "default" else destination["keys"]) # find any images from within the summary and add them to the embed soup = BeautifulSoup(entry["summary"], "html.parser") imgs = soup.find_all("img") image = None for img in imgs: if not image: image = img["src"] img.decompose() entry["summary"] = re.sub(r"<br ?/?>", "\n", str(soup)) entry["summary"] = re.sub(r"<!--.*-->", "", entry["summary"]) # remove html formatting from the description print(entry["summary"]) summary = html.unescape( Tomd("<p>" + entry["summary"].replace("%22", '"').replace("%3E", ">") + "</p>").markdown.strip()) entry["summary"] = summary if summary != "" else entry["summary"] entry["description"] = entry["summary"] # alias # turns a key format into value string def format_key(t): v = [] # for each word in the value of the key if value exists for k in keys[t].split() if t in keys and keys[t] else []: if k[0] == "$": v.append(entry[k[1:]] if k[1:] in entry else None) else: v.append(k) # if we're missing any of the values, display nothing return " ".join(v) if not None in v else None return self.bot.msg( destination["target"], format_key("message") + " ", embed=self.bot.server.embed( title=format_key("title"), desc=format_key("desc")[:2047], author_name=format_key("author_name"), author_link=format_key("author_link"), author_icon=format_key("author_icon"), footer=format_key("footer"), url=format_key("url"), color=format_key("color"), image=image, ), )
def build_file(name): path = "posts/" + name + "/index.html" f = codecs.open(path, 'r') html = f.read() soup = BeautifulSoup(html, features="html5lib") # Save title md = "---" md += "\ntitle: " + soup.find("h1").get_text().strip() md += "\ndate: " + soup.find("time").get_text().replace("-", "/") md += "\ntags: " + soup.find("a", "tag-link").get_text() md += "\n---\n" content = soup.find(itemprop="articleBody") for child in content.children: if str(child)[:4] == '<img': end_point = child['src'].rfind('/') + 1 file_name = child['src'][end_point:] text = "{% asset_img " + file_name + " %}" md += "\n" + text + "\n" elif str(child)[:2] == '<h': num_pounds = "#" * int(str(child)[2]) md += "\n" + num_pounds + " " + child.get_text() + "\n" elif str(child)[:3] == '<if': md += "\n" + str(child) + "\n" elif str(child)[:24] == '<figure class="highlight': code_sample = str(child) code_type = code_sample[25:code_sample.find('"', 24)] temp_md = Tomd(str(child)).markdown temp_md = temp_md[temp_md.find('<td class="code"'):] temp_md = BeautifulSoup(temp_md, features="html5lib").find("pre") pre_md = str(temp_md) pre_md = pre_md[5:-6] temp_md = "\n``` " temp_md += code_type + '\n' for i, char in enumerate(pre_md): if pre_md[i:i + 5] == '<br/>': temp_md += '\n' temp_md += char else: temp_md += char temp_md += '```' md += temp_md.replace('<br/>', '') else: md += Tomd(str(child)).markdown with open('posts/' + name + '.md', 'w') as file: file.write(md)
def html_to_markdown_proc_2(html_code_str, asset_folder_path): tmpHtmlCode = html_code_str target_html_file_path = str( PurePath(asset_folder_path).joinpath(TEMP_HTML_FILE_NAME)) target_markdown_file_path = str( PurePath(asset_folder_path).joinpath(TEMP_MD_FILE_NAME)) with open(target_html_file_path, 'w', encoding='utf-8') as html_file: html_file.write(tmpHtmlCode) html_file.close() markdown_str = Tomd(html_code_str).markdown return markdown_str
def main(lang='ja'): alphabet = str(Path('.').absolute().name) contest_id = str(Path('.').absolute().parent.name) client = AtCoderClient() client.login() ic(client.check_logging_in()) contest = Contest(ic(contest_id)) ic(contest.get_url()) problem_list = client.download_problem_list(ic(contest)) problem = problem_list[['A', 'B', 'C', 'D', 'E', 'F'].index(ic(alphabet))] html_doc = client.download_problem_content(problem).original_html soup = BeautifulSoup(html_doc, 'html.parser') title = soup.find(class_="h2").get_text() task_statement = soup.find(id="task-statement") if lang == 'ja': task_statement = task_statement.find(class_='lang-ja') def sanitize_html_for_ipynb(html_doc): replace_dict = { '<var>': '$', '</var>': '$', '<pre>': '<pre><code>', '</pre>': '</code></pre>' } for old_word, new_word in replace_dict.items(): html_doc = html_doc.replace(old_word, new_word) return ic(html_doc) title = str(sanitize_html_for_ipynb(str(title))) title = title.lstrip().split('\n')[0] task_statement = Tomd(sanitize_html_for_ipynb(str(task_statement))) with open('problem.md', 'w+') as f: f.write(f"## {ic(title)}\n") f.write('---\n') f.write(task_statement.markdown)
def getDetailFromLocal(html): soup = BeautifulSoup(html, "html.parser") content = soup.select_one("#content_views") # 删除注释 for useless_tag in content(text=lambda text: isinstance(text, Comment)): useless_tag.extract() # 删除无用标签 tags = ["svg", "ul", ".hljs-button.signin"] delete_ele(content, tags) # 删除标签属性 attrs = ["class", "name", "id", "onclick", "style", "data-token", "rel"] delete_ele_attr(content, attrs) # 删除空白标签 eles_except = ["img", "br", "hr"] delete_blank_ele(content, eles_except) # 转换为markdown md = Tomd(str(content)).markdown return md
def parse_news(self, response): log.msg("Start to parse news " + response.url, level=log.INFO) item = SpiderNewsAllItem() day = title = _type = keywords = url = article = '' url = response.url day = response.meta['day'] title = response.meta['title'] _type = response.meta['_type'] response = response.body soup = BeautifulSoup(response) # try: # items_keywords = soup.find(class_='ar_keywords').find_all('a') # for i in range(0, len(items_keywords)): # keywords += items_keywords[i].text.strip() + ' ' # except: # log.msg("News " + title + " dont has keywords!", level=log.INFO) try: content_paragraph = soup.find("div", class_="text_info") article = [] for tag in content_paragraph.find( "div", class_="clear").previous_siblings: article.insert(0, tag) markdown = Tomd(''.join( str(article))).markdown.decode("unicode-escape") article = BeautifulSoup(''.join([str(tag) for tag in article ])).get_text().strip() except: log.msg("News " + title + " dont has article!", level=log.INFO) item['title'] = title item['day'] = day item['_type'] = _type item['url'] = url item['keywords'] = keywords item['article'] = article item['site'] = 'InfoQ' item['markdown'] = markdown return item
def _get_book_info_from_bs(book_bs): bookInfo = BookInfo() content_bs = book_bs.find('section', {'class': 'content'}) if not content_bs: return None title_bs = content_bs.find('h3', {'class': 'product-title'}) if title_bs: bookInfo.title = title_bs.get_text() if not bookInfo.title: return None details_bs = content_bs.find('div', {'class': 'details'}) if details_bs: details_list_bs = details_bs.find('ul', {'class': 'list-unstyled'}) if details_list_bs: bookInfo.details = details_list_bs.get_text().strip('\n') body_bs_list = content_bs.find_all('div', {'class': 'body'}) if len(body_bs_list) == 6: description_bs = body_bs_list[3] bookInfo.description = Tomd(str(description_bs)).markdown.strip() download_bs = content_bs.find('span', {'class': 'tn-download'}) if download_bs: bookInfo.tn_url.append(download_bs.get('tn-url')) return bookInfo
def getArticle(url): text = url2str(url) hasUrl = len(osCmd('cat history.log | grep ' + url)) if hasUrl == 0: os.system('echo ' + url + ' >> history.log') try: # 获取文章内容 doc = pq(text) articleTitle = doc('.mainnews_l h1').text() articleDetails = doc('.mainnews_l .newsdetails') articleDate = getDate(doc('.mainnews_l .time').text()) articleTime = str(int(time.time())) mdArticle = Tomd(str(articleDetails)\ .replace(' ','')\ .replace('\t','')\ ).markdown html = markdown.markdown(mdArticle) articleKey = getKeywords(articleTitle) if len(html) > 100: # 提交文章到系统 curlCommand = "curl -s 'http://www.56.com/admin/index.php?controller=module&project=article&operate=save' -H 'Cookie: PHPSESSID=hm8kb0pg9l2cc9vgrve7als2r1' --data 'title={title}&classid=5&tags={key}&hits=0&info=&content={content}&time={time}&status=1&html=&template=%2Farticle_content.html&date={date}' --compressed".format( title=articleTitle, content=html, time=articleTime, date=articleDate, key=articleKey) res = json.loads(osCmd(curlCommand)[0])['status'] if res == 'y': print('文章添加成功' + url) else: print('文章添加失败' + url) else: print('文章内容过短') except: print('文章抓取内容出错: ' + url) else: print('重复文章: ' + url)
def blog(addr): r = requests.get(blog_domain+addr, headers=headers) r.encoding = 'utf-8' soup = BeautifulSoup(r.text, 'html.parser') blog_title = soup.title.string file_name = blog_title.replace(' ', '-') file_name = file_name.replace(' ', '-') file_name = file_name.replace('.', '-') file_name = file_name.replace('/', '-') file_name = file_name.replace('+', '-') file_name = file_name.replace('_', '-') file_name = file_name.replace('(', '') file_name = file_name.replace(')', '') print (addr) # hexo new os.system('hexo new "{}"'.format(file_name)) time.sleep(0.5); blog_title_arr = blog_title.split('·') blog_header = ''' --- title: {} date: 2018-09-28 15:47:45 categories: [alidb-monthly, {}, {}] tags: [{}, {}] --- '''.format(blog_title, blog_title_arr[0].strip(), blog_title_arr[1].strip(), blog_title_arr[0].strip(), blog_title_arr[1].strip()) blog_footer = ''' ## 郑重声明 > 文章来自淘宝技术博客 [数据库内核月报](http://mysql.taobao.org/monthly/2017/04/01/) > > 本人为了学习方便而分类整理 ''' # print (soup.select('.post')[0].prettify()) # print (blog_header + Tomd(str(soup.select('.post')[0])).markdown) write_file(file_name + '.md', blog_header + Tomd(str(soup.select('.post')[0])).markdown + blog_footer)
def get_question_detail(self, titleSlug): url = "https://leetcode-cn.com/graphql" headers = { "x-csrftoken": x_csrftoken, "referer": "https://leetcode-cn.com", "cookie": cookie, "Content-Type": "application/json" } payload = { "operationName": "getQuestionDetail", "variables": {"titleSlug": titleSlug}, "query": query } try: r = requests.post(url, data=json.dumps( payload), headers=headers, timeout=5) r = json.loads(r.text)['data']['question']['content'] md = Tomd(r).markdown return md except Exception as e: logging.error(e) time.sleep(1) return self.get_question_detail(titleSlug)
def getOneArticle(article_url): ''' article_url:blog的url 提取数据,保存为md文件 ''' try: r = requests.get(article_url)#发送请求 r.raise_for_status()#获取网页状态 #初始化生成一个XPath解析对象 selectors = Selector(r.text) #使用XPath选取每一篇blog的标题、内容,返回列表 title = selectors.xpath('//h1[@class="title-article"]/text()').get() content = selectors.xpath('//article').get() #内容转换为md格式 text = Tomd(content).markdown #使用空字符串替换不要的内容 text = re.sub('<a.*?a>', '', text) print(title) #保存文件 with open(title+'.md', 'w', encoding='utf-8') as f: f.write('#' + title) f.write(text) except: return "获取网页异常"
<b>bold</b> <i>italic</i> <b><i>bold italic</i></b> <em>em</em> <strong>strong</strong> </p> <hr/> <table> <thead> <tr class="1"> <th>th1</th> <th>th2</th> </tr> </thead> <tbody> <tr> <td>td</td> <td>td</td> </tr> <tr> <td>td</td> <td>td</td> </tr></tbody></table> """ print(Tomd(string).markdown) print(tomd.convert(string))
<tr> <td>erf(x)</td> <td>Returns the error function at x</td> </tr> <tr> <td>erfc(x)</td> <td>Returns the complementary error function at x</td> </tr> <tr> <td>gamma(x)</td> <td>Returns the Gamma function at x</td> </tr> <tr> <td>lgamma(x)</td> <td>Returns the natural logarithm of the absolute value of the Gamma function at x</td> </tr> <tr> <td>pi</td> <td>Mathematical constant, the ratio of circumference of a circle to it's diameter (3.14159...)</td> </tr> <tr> <td>e</td> <td>mathematical constant e (2.71828...)</td> </tr> </tbody> </table> """ print(Tomd(html).markdown)
def get_single_page(inputurl, en): page = req.get(inputurl, headers=head).text bs_ = bs(page, "lxml") img_url = bs_.find(class_="entry-thumbnail") if img_url.img: img_url = img_url.img['src'] for i in range(5): try: img = req.get(img_url, headers=head).content break except: print('Get Image Time Out') time.sleep(10) img_path = img_folder + img_url.split('/')[-1] with open('app/' + img_path, "wb") as f: f.write(img) else: img_path = "" title = bs_.find(class_="entry-title").string date = parse(bs_.find(class_="entry-date")['datetime']) body = str(bs_.find(class_="entry-content")) md = Tomd(body).markdown category = bs_.find(class_="entry-categories").a.string print(category) if not Category.query.filter_by(name=category).first(): c = Category(name=category) db.session.add(c) db.session.commit() c = Category.query.filter_by(name=category).first() print(c) else: c = Category.query.filter_by(name=category).first() u = User.query.get(1) print(f'is_en value is: {en}') if Post.query.filter_by(title=title).first(): p = Post.query.filter_by(title=title).first() p.title = title p.body = md p.timestamp = date p.cover = img_path p.category = c p.is_en = en else: p = Post(title=title, body=md, timestamp=date, author=u, cover=img_path, category=c, is_en=en) db.session.add(p) db.session.commit() return bs_