async def get_archived_posts(self,
links: Optional[list] = None,
limit=10,
offset=0,
refresh=True) -> list:
api_type = "archived_posts"
if not refresh:
result = handle_refresh(self, api_type)
if result:
return result
if links is None:
links = []
api_count = self.archivedPostsCount
if api_count and not links:
link = endpoint_links(identifier=self.id,
global_limit=limit,
global_offset=offset).archived_posts
ceil = math.ceil(api_count / limit)
numbers = list(range(ceil))
for num in numbers:
num = num * limit
link = link.replace(f"limit={limit}", f"limit={limit}")
new_link = link.replace("offset=0", f"offset={num}")
links.append(new_link)
results = await api_helper.scrape_endpoint_links(
links, self.session_manager, api_type)
final_results = [create_post(x, self) for x in results if x]
self.temp_scraped.Archived.Posts = final_results
return final_results
async def get_posts(self,
links: Optional[list] = None,
limit: int = 10,
offset: int = 0,
refresh=True) -> Optional[list[create_post]]:
api_type = "posts"
if not refresh:
result = handle_refresh(self, api_type)
if result:
return result
if links is None:
links = []
temp_results: list[Any] = []
while True:
link = endpoint_links(identifier=self.id,
global_offset=offset).post_api
response = await self.session_manager.json_request(link)
data = response["response"]
temp_posts = data["posts"]
if not temp_posts:
break
offset = temp_posts[-1]["id"]
temp_results.append(data)
results: dict[Any, Any] = merge({},
*temp_results,
strategy=Strategy.ADDITIVE)
final_results = [
create_post(x, self, results) for x in results["posts"]
]
self.temp_scraped.Posts = final_results
return final_results
async def get_highlights(self,
identifier="",
refresh=True,
limit=100,
offset=0,
hightlight_id="") -> list:
api_type = "highlights"
if not refresh:
result = handle_refresh(self, api_type)
if result:
return result
if not identifier:
identifier = self.id
if not hightlight_id:
link = endpoint_links(identifier=identifier,
global_limit=limit,
global_offset=offset).list_highlights
results = await self.session_manager.json_request(link)
results = await remove_errors(results)
results = [create_highlight(x) for x in results]
else:
link = endpoint_links(identifier=hightlight_id,
global_limit=limit,
global_offset=offset).highlight
results = await self.session_manager.json_request(link)
results = [create_story(x) for x in results["stories"]]
return results
async def get_chats(
self,
links: Optional[list] = None,
limit=100,
offset=0,
refresh=True,
inside_loop=False,
) -> list:
api_type = "chats"
if not self.active:
return []
if not refresh:
result = handle_refresh(self, api_type)
if result:
return result
if links is None:
links = []
api_count = self.chatMessagesCount
if api_count and not links:
link = endpoint_links(identifier=self.id,
global_limit=limit,
global_offset=offset).list_chats
ceil = math.ceil(api_count / limit)
numbers = list(range(ceil))
for num in numbers:
num = num * limit
link = link.replace(f"limit={limit}", f"limit={limit}")
new_link = link.replace("offset=0", f"offset={num}")
links.append(new_link)
multiplier = getattr(self.session_manager.pool, "_processes")
if links:
link = links[-1]
else:
link = endpoint_links(identifier=self.id,
global_limit=limit,
global_offset=offset).list_chats
links2 = api_helper.calculate_the_unpredictable(
link, limit, multiplier)
if not inside_loop:
links += links2
else:
links = links2
results = await self.session_manager.async_requests(links)
has_more = results[-1]["hasMore"]
final_results = [x["list"] for x in results]
final_results = list(chain.from_iterable(final_results))
if has_more:
results2 = await self.get_chats(links=[links[-1]],
limit=limit,
offset=limit + offset,
inside_loop=True)
final_results.extend(results2)
final_results.sort(key=lambda x: x["withUser"]["id"], reverse=True)
self.chats = final_results
return final_results
async def get_lists(self, refresh=True, limit=100, offset=0):
api_type = "lists"
if not self.active:
return
if not refresh:
subscriptions = handle_refresh(self, api_type)
return subscriptions
link = endpoint_links(global_limit=limit, global_offset=offset).lists
results = await self.session_manager.json_request(link)
self.lists = results
return results
async def get_messages(
self,
links: Optional[list] = None,
limit=10,
offset=0,
refresh=True,
inside_loop=False,
) -> list:
api_type = "messages"
if not self.subscriber or self.is_me():
return []
if not refresh:
result = handle_refresh(self, api_type)
if result:
return result
if links is None:
links = []
multiplier = getattr(self.session_manager.pool, "_processes")
if links:
link = links[-1]
else:
link = endpoint_links(identifier=self.id,
global_limit=limit,
global_offset=offset).message_api
links.append(link)
links2 = api_helper.calculate_the_unpredictable(
link, limit, multiplier)
if not inside_loop:
links += links2
else:
links = links2
results = await self.session_manager.async_requests(links)
results = await remove_errors(results)
results = [x for x in results if x]
has_more = results[-1]["hasMore"] if results else False
final_results = [x["list"] for x in results if "list" in x]
final_results = list(chain.from_iterable(final_results))
if has_more:
results2 = await self.get_messages(links=[links[-1]],
limit=limit,
offset=limit + offset,
inside_loop=True)
final_results.extend(results2)
print
if not inside_loop:
final_results = [
create_message.create_message(x, self) for x in final_results
if x
]
else:
final_results.sort(key=lambda x: x["fromUser"]["id"], reverse=True)
self.temp_scraped.Messages = final_results
return final_results
async def get_archived_stories(self, refresh=True, limit=100, offset=0):
api_type = "archived_stories"
if not refresh:
result = handle_refresh(self, api_type)
if result:
return result
link = endpoint_links(global_limit=limit,
global_offset=offset).archived_stories
results = await self.session_manager.json_request(link)
results = await remove_errors(results)
results = [create_story(x) for x in results]
return results
async def get_stories(self, refresh=True, limit=100, offset=0) -> list:
api_type = "stories"
if not refresh:
result = handle_refresh(self, api_type)
if result:
return result
if not self.hasStories:
return []
link = [
endpoint_links(identifier=self.id,
global_limit=limit,
global_offset=offset).stories_api
]
results = await api_helper.scrape_endpoint_links(
link, self.session_manager, api_type)
results = [create_story(x) for x in results]
self.temp_scraped.Stories = results
return results
async def get_paid_content(
self,
check: bool = False,
refresh: bool = True,
limit: int = 99,
offset: int = 0,
inside_loop: bool = False,
) -> list[Union[create_message, create_post]]:
return []
api_type = "paid_content"
if not self.active:
return []
if not refresh:
result = handle_refresh(self, api_type)
if result:
return result
link = endpoint_links(global_limit=limit,
global_offset=offset).paid_api
final_results = await self.session_manager.json_request(link)
if not isinstance(final_results, error_details):
if len(final_results) >= limit and not check:
results2 = self.get_paid_content(limit=limit,
offset=limit + offset,
inside_loop=True)
final_results.extend(results2)
if not inside_loop:
temp = []
for final_result in final_results:
content = None
if final_result["responseType"] == "message":
user = create_user(final_result["fromUser"], self)
content = create_message(final_result, user)
print
elif final_result["responseType"] == "post":
user = create_user(final_result["author"], self)
content = create_post(final_result, user)
if content:
temp.append(content)
final_results = temp
self.paid_content = final_results
return final_results
async def get_mass_messages(self,
resume=None,
refresh=True,
limit=10,
offset=0) -> list:
api_type = "mass_messages"
if not self.active:
return []
if not refresh:
result = handle_refresh(self, api_type)
if result:
return result
link = endpoint_links(global_limit=limit,
global_offset=offset).mass_messages_api
results = await self.session_manager.json_request(link)
items = results.get("list", [])
if not items:
return items
if resume:
for item in items:
if any(x["id"] == item["id"] for x in resume):
resume.sort(key=lambda x: x["id"], reverse=True)
self.mass_messages = resume
return resume
else:
resume.append(item)
if results["hasMore"]:
results2 = self.get_mass_messages(resume=resume,
limit=limit,
offset=limit + offset)
items.extend(results2)
if resume:
items = resume
items.sort(key=lambda x: x["id"], reverse=True)
self.mass_messages = items
return items