def sync(obj: Base, *args, **kwargs):
# define number of posts to check
# no_of_posts = int(kwargs.get("no_of_posts", 10))
soup = Internet.get_soup_phjs(obj.sync_type.base_url)
all_cards = soup.find('div', {'id':'project-grid'})
posts = all_cards.find_all('div', {'class':['card']})[::-1]
for post in posts:
pid = post.find('div', {'class':'simplefavorite-button has-count'}).get('data-postid').strip()
data = {
'name': post.find("h4", {"class":"card-title"}).text.strip(),
'url':post.find('a').get('href'),
'download_url': Bfy.get_download_link(obj, pid)
}
data["text"] = (
f"Name: {data.get('name')}\n"
f"URL: {data.get('url')}\n"
f"Link: {data.get('download_url')}\n"
)
obj.add_text_task(
unique_key=pid,
name=data['name'],
url=data['url'],
data=data
)
def sync(obj: Base, *args, **kwargs):
soup = Internet.get_soup_phjs(obj.sync_type.base_url)
banner_div = soup.find('div', {'class': 'card-body m-0 p-0 pt-3'})
a = banner_div.find('a')
link = urllib.parse.urljoin(obj.sync_type.base_url,
a.get('href').strip())
name = a.text.strip()
obj.add_text_task(unique_key=link,
name=name,
url=link,
data=dict(title=name, text=link))
# find remaining div
divs = soup.find_all('div', {'class': "card-body m-0 p-0 mt-2"})[::-1]
for div in divs:
a = div.find('a')
link = urllib.parse.urljoin(obj.sync_type.base_url,
a.get('href').strip())
name = a.text.strip()
obj.add_text_task(unique_key=link,
name=name,
url=link,
data=dict(title=name, text=link))
def sync(obj: Base, *args, **kwargs):
url = "https://deepstash.com/trending"
soup = Internet.get_soup_phjs(url)
links = soup.find_all('a', {'class': 'css-rgh4li'})[::-1]
print(len(links))
links = links[::-1][:-1]
print(len(links))
for a in links:
print(a.text.strip(), "\n")
def sync(obj: Base, *args, **kwargs):
tag = kwargs.get("tag", "programming")
url = obj.sync_type.base_url.format(tag=tag)
post_url = obj.sync_type.extras.get("post_url")
soup = Internet.get_soup_phjs(url)
posts = soup.find_all('div', {'class': 'title-wrapper'})[::-1]
for post in posts:
a = post.find('a')
link = urllib.parse.urljoin(post_url, a.get('href').strip())
obj.add_text_task(unique_key=link,
name=a.text.strip(),
url=link,
data=dict(title=a.text.strip(), text=link))
def sync(obj: Base, *args, **kwargs):
soup = Internet.get_soup_phjs(obj.sync_type.base_url)
links = soup.find_all('a', {'class': 'o-eZTujG o-fyWCgU'})
for a in links[::-1]:
link = a.get('href')
url = urllib.parse.urljoin(obj.sync_type.base_url, link)
name = a.text.strip()
obj.add_text_task(unique_key=url,
name=name,
url=url,
data=dict(text=url))
def sync(obj: Base, *args, **kwargs):
url = obj.sync_type.base_url
soup = Internet.get_soup_phjs(url)
divs = soup.find_all('div',
{'class': 'entry-grid-content hgrid-span-12'})
for div in divs[::-1]:
h2 = div.find("h2", {"class": "entry-title"})
a = h2.find('a')
url = a.get('href')
name = a.text.strip()
desc_div = div.find("div", {"class": "entry-summary"})
if desc_div:
desc = desc_div.text.strip()
obj.add_text_task(unique_key=url,
name=name,
url=url,
data=dict(text=url, desc=desc))
def sync(obj: Base, *args, **kwargs):
category = kwargs.get("category", "Startup")
url = obj.sync_type.base_url.format(category=category)
soup = Internet.get_soup_phjs(url)
lis = soup.find_all('li', {'class': 'sc-hMFtBS gpleaq'})[::-1]
# li are returend in double
for li in lis:
a = li.find('a')
div = li.find('div', {'class': 'sc-gqPbQI iIXuvz'})
title_a = div.find("a")
name = title_a.text.strip()
if name.strip() == "":
continue
link = urllib.parse.urljoin(obj.sync_type.base_url,
a.get('href').strip())
obj.add_text_task(unique_key=link,
name=name,
url=link,
data=dict(text=link))