def sync(obj: Base, *args, **kwargs): # define number of posts to check # no_of_posts = int(kwargs.get("no_of_posts", 10)) soup = Internet.get_soup_phjs(obj.sync_type.base_url) all_cards = soup.find('div', {'id':'project-grid'}) posts = all_cards.find_all('div', {'class':['card']})[::-1] for post in posts: pid = post.find('div', {'class':'simplefavorite-button has-count'}).get('data-postid').strip() data = { 'name': post.find("h4", {"class":"card-title"}).text.strip(), 'url':post.find('a').get('href'), 'download_url': Bfy.get_download_link(obj, pid) } data["text"] = ( f"Name: {data.get('name')}\n" f"URL: {data.get('url')}\n" f"Link: {data.get('download_url')}\n" ) obj.add_text_task( unique_key=pid, name=data['name'], url=data['url'], data=data )
def sync(obj: Base, *args, **kwargs): soup = Internet.get_soup_phjs(obj.sync_type.base_url) banner_div = soup.find('div', {'class': 'card-body m-0 p-0 pt-3'}) a = banner_div.find('a') link = urllib.parse.urljoin(obj.sync_type.base_url, a.get('href').strip()) name = a.text.strip() obj.add_text_task(unique_key=link, name=name, url=link, data=dict(title=name, text=link)) # find remaining div divs = soup.find_all('div', {'class': "card-body m-0 p-0 mt-2"})[::-1] for div in divs: a = div.find('a') link = urllib.parse.urljoin(obj.sync_type.base_url, a.get('href').strip()) name = a.text.strip() obj.add_text_task(unique_key=link, name=name, url=link, data=dict(title=name, text=link))
def sync(obj: Base, *args, **kwargs): url = "https://deepstash.com/trending" soup = Internet.get_soup_phjs(url) links = soup.find_all('a', {'class': 'css-rgh4li'})[::-1] print(len(links)) links = links[::-1][:-1] print(len(links)) for a in links: print(a.text.strip(), "\n")
def sync(obj: Base, *args, **kwargs): tag = kwargs.get("tag", "programming") url = obj.sync_type.base_url.format(tag=tag) post_url = obj.sync_type.extras.get("post_url") soup = Internet.get_soup_phjs(url) posts = soup.find_all('div', {'class': 'title-wrapper'})[::-1] for post in posts: a = post.find('a') link = urllib.parse.urljoin(post_url, a.get('href').strip()) obj.add_text_task(unique_key=link, name=a.text.strip(), url=link, data=dict(title=a.text.strip(), text=link))
def sync(obj: Base, *args, **kwargs): soup = Internet.get_soup_phjs(obj.sync_type.base_url) links = soup.find_all('a', {'class': 'o-eZTujG o-fyWCgU'}) for a in links[::-1]: link = a.get('href') url = urllib.parse.urljoin(obj.sync_type.base_url, link) name = a.text.strip() obj.add_text_task(unique_key=url, name=name, url=url, data=dict(text=url))
def sync(obj: Base, *args, **kwargs): url = obj.sync_type.base_url soup = Internet.get_soup_phjs(url) divs = soup.find_all('div', {'class': 'entry-grid-content hgrid-span-12'}) for div in divs[::-1]: h2 = div.find("h2", {"class": "entry-title"}) a = h2.find('a') url = a.get('href') name = a.text.strip() desc_div = div.find("div", {"class": "entry-summary"}) if desc_div: desc = desc_div.text.strip() obj.add_text_task(unique_key=url, name=name, url=url, data=dict(text=url, desc=desc))
def sync(obj: Base, *args, **kwargs): category = kwargs.get("category", "Startup") url = obj.sync_type.base_url.format(category=category) soup = Internet.get_soup_phjs(url) lis = soup.find_all('li', {'class': 'sc-hMFtBS gpleaq'})[::-1] # li are returend in double for li in lis: a = li.find('a') div = li.find('div', {'class': 'sc-gqPbQI iIXuvz'}) title_a = div.find("a") name = title_a.text.strip() if name.strip() == "": continue link = urllib.parse.urljoin(obj.sync_type.base_url, a.get('href').strip()) obj.add_text_task(unique_key=link, name=name, url=link, data=dict(text=link))