def _limited_call(self, func, *args, **kwargs):
"""Rate limit calls to a function.
"""
# Check seconds that have passed
now = datetime.datetime.now()
diff = (now - self._rate_limit_start).total_seconds()
if diff >= 60:
# If greater than a minute, reset the rate limit
self._rate_limit_count = 0
self._rate_limit_start = now
else:
# Check if the per-minute limit has been exceeded
if self._rate_limit_count >= constants.FA_PAGE_REQUESTS_PER_MINUTE:
# Wait until next minute, then reset the count/time
wait_time = 60 - diff
logger.debug("Hit rate limit, waiting %d seconds" % wait_time)
time.sleep(wait_time)
self._rate_limit_count = 0
self._rate_limit_start = datetime.datetime.now()
self._rate_limit_count += 1
return func(*args, **kwargs)
def _limited_call(self, func, *args, **kwargs):
"""Rate limit calls to a function.
"""
# Check seconds that have passed
now = datetime.datetime.now()
diff = (now - self._rate_limit_start).total_seconds()
if diff >= 60:
# If greater than a minute, reset the rate limit
self._rate_limit_count = 0
self._rate_limit_start = now
else:
# Check if the per-minute limit has been exceeded
if self._rate_limit_count >= constants.FA_PAGE_REQUESTS_PER_MINUTE:
# Wait until next minute, then reset the count/time
wait_time = 60 - diff
logger.debug("Hit rate limit, waiting %d seconds" % wait_time)
time.sleep(wait_time)
self._rate_limit_count = 0
self._rate_limit_start = datetime.datetime.now()
self._rate_limit_count += 1
return func(*args, **kwargs)
def _load_folders(self):
logger.debug("Loading folders")
self._root_folders = []
url = constants.WZL_ROOT + "/api/users/%s/view" % self.username
res = self._requests.get(url)
folders = res.json()["folders"]
for folder_struct in folders:
folder = self._folders.get(folder_struct["folder_id"])
if folder is None:
folder = Folder()
folder._session = self
folder.id = folder_struct["folder_id"]
self._folders[folder.id] = folder
folder.title = folder_struct["title"]
folder.children = []
self._root_folders.append(folder)
if "subfolders" in folder_struct:
for subfolder_struct in folder_struct["subfolders"]:
subfolder = self._folders.get(subfolder_struct["folder_id"])
if subfolder is None:
subfolder = Folder()
subfolder._session = self
subfolder.id = subfolder_struct["folder_id"]
self._folders[subfolder.id] = subfolder
subfolder.title = subfolder_struct["title"]
subfolder.children = []
folder.children.append(subfolder)
def _scan_folder(self, folder):
logger.debug("Scanning folder %r" % folder)
url = constants.FA_ROOT + "/gallery/%s/folder/%d/-/%%d/" % (
self.username, folder.id)
submissions = self._scan_submission_page(url)
folder.submissions = []
for sub in submissions:
folder.submissions.append(sub)
def _scan_folder(self, folder):
logger.debug("Scanning folder %r" % folder)
url = constants.FA_ROOT + "/gallery/%s/folder/%d/-/%%d/" % (
self.username, folder.id)
submissions = self._scan_submission_page(url)
folder.submissions = []
for sub in submissions:
folder.submissions.append(sub)
def _scan_gallery(self, folder_id=None):
next_id = None
url = constants.WZL_ROOT + "/api/users/%s/gallery" % self.username
submissions = []
logger.debug("Scanning gallery folder %r" % folder_id)
while True:
params = {}
if next_id is not None:
params["nextid"] = next_id
if folder_id is not None:
params["folderid"] = folder_id
res = self._requests.get(url, params=params)
data = res.json()
next_id = data["nextid"]
for sub_struct in data["submissions"]:
sub = self._load_submission_from_struct(sub_struct)
submissions.append(sub)
if next_id is None:
break
logger.debug("Found %d submissions" % len(data["submissions"]))
if folder_id is None:
self._gallery_submissions = submissions
return submissions
def _scan_gallery(self, folder_id=None):
next_id = None
url = constants.WZL_ROOT + "/api/users/%s/gallery" % self.username
submissions = []
logger.debug("Scanning gallery folder %r" % folder_id)
while True:
params = {}
if next_id is not None:
params["nextid"] = next_id
if folder_id is not None:
params["folderid"] = folder_id
res = self._requests.get(url, params=params)
data = res.json()
next_id = data["nextid"]
for sub_struct in data["submissions"]:
sub = self._load_submission_from_struct(sub_struct)
submissions.append(sub)
if next_id is None:
break
logger.debug("Found %d submissions" % len(data["submissions"]))
if folder_id is None:
self._gallery_submissions = submissions
return submissions
def _load_folders(self):
logger.debug("Loading folders")
self._root_folders = []
url = constants.WZL_ROOT + "/api/users/%s/view" % self.username
res = self._requests.get(url)
folders = res.json()["folders"]
for folder_struct in folders:
folder = self._folders.get(folder_struct["folder_id"])
if folder is None:
folder = Folder()
folder._session = self
folder.id = folder_struct["folder_id"]
self._folders[folder.id] = folder
folder.title = folder_struct["title"]
folder.children = []
self._root_folders.append(folder)
if "subfolders" in folder_struct:
for subfolder_struct in folder_struct["subfolders"]:
subfolder = self._folders.get(
subfolder_struct["folder_id"])
if subfolder is None:
subfolder = Folder()
subfolder._session = self
subfolder.id = subfolder_struct["folder_id"]
self._folders[subfolder.id] = subfolder
subfolder.title = subfolder_struct["title"]
subfolder.children = []
folder.children.append(subfolder)
def _scan_scraps(self):
logger.debug("Scanning scraps")
url = constants.FA_ROOT + "/scraps/%s/%%d/" % self.username
submissions = self._scan_submission_page(url)
return submissions
def _scan_gallery(self):
logger.debug("Scanning gallery")
url = constants.FA_ROOT + "/gallery/%s/%%d/" % self.username
submissions = self._scan_submission_page(url)
return submissions
def _scan_submission_page(self, url_format):
"""Return submissions found in pages of a base url.
Args:
url_format (str): URL, with a %d that holds the page id
Returns:
A list of submission objects.
"""
submissions = []
try:
page = 1
while True:
url = url_format % page
doc = self._limited_call(self._html_get, url)
logger.debug("Scanning submissions from %s" % url)
count = 0
for el in doc.cssselect(".gallery > *"):
if el.get("id") == "no-images":
continue
id_str = el.get("id")[4:]
if id_str == "":
continue
id = int(id_str)
submission = self._submissions.get(id)
if submission is None:
submission = Submission()
submission._session = self
submission.id = id
self._submissions[id] = submission
submission.title = str(
el.cssselect("span")[0].text_content())
if "r-adult" in el.classes:
submission.rating = "adult"
elif "r-mature" in el.classes:
submission.rating = "mature"
elif "r-general" in el.classes:
submission.rating = "general"
else:
raise exceptions.ScraperError()
if "t-image" in el.classes:
submission.type = "image"
elif "t-text" in el.classes:
submission.type = "text"
elif "t-audio" in el.classes:
submission.type = "audio"
elif "t-flash" in el.classes:
submission.type = "flash"
else:
raise exceptions.ScraperError()
submission.thumbnail_url = "https:" + el.cssselect(
"img")[0].get("src")
submissions.append(submission)
count += 1
if count == 0:
break
logger.debug("Found %d submissions" % count)
page += 1
except (IndexError, ValueError):
raise exceptions.ScraperError()
return submissions
def _load_folders(self):
logger.debug("Loading folders")
self._root_folders = []
url = constants.FA_ROOT + "/controls/folders/submissions/"
doc = self._limited_call(self._html_get, url)
# get groups
for group_el in doc.cssselect(".group-row"):
try:
title = str(group_el.cssselect("strong")[0].text_content())
id_match = re.search("group-([0-9]+)", group_el.get("class"))
id = int(id_match.group(1))
group = self._folders.get(id)
if group is None:
group = Folder()
group._session = self
group.id = id
self._folders[id] = group
group.title = title
group.children = []
group.submissions = []
self._root_folders.append(group)
except (IndexError, ValueError):
raise exceptions.ScraperError()
# Get folders
for folder_el in doc.cssselect(".folder-row"):
try:
title = str(
folder_el.cssselect(".folder-name strong")
[0].text_content())
id_match = re.search("folder-([0-9]+)", folder_el.get("class"))
group_match = re.search("group-([0-9]+)",
folder_el.get("class"))
id = int(id_match.group(1))
parent_id = int(group_match.group(1))
folder = self._folders.get(id)
if folder is None:
folder = Folder()
folder._session = self
folder.id = id
self._folders[id] = folder
folder.title = title
folder.children = []
parent = self._folders.get(parent_id)
if parent is None:
self._root_folders.append(folder)
else:
parent.children.append(folder)
except (IndexError, ValueError):
raise exceptions.ScraperError()
def _scan_scraps(self):
logger.debug("Scanning scraps")
url = constants.FA_ROOT + "/scraps/%s/%%d/" % self.username
submissions = self._scan_submission_page(url)
return submissions
def _scan_gallery(self):
logger.debug("Scanning gallery")
url = constants.FA_ROOT + "/gallery/%s/%%d/" % self.username
submissions = self._scan_submission_page(url)
return submissions
def _scan_submission_page(self, url_format):
"""Return submissions found in pages of a base url.
Args:
url_format (str): URL, with a %d that holds the page id
Returns:
A list of submission objects.
"""
submissions = []
try:
page = 1
while True:
url = url_format % page
doc = self._limited_call(self._html_get, url)
logger.debug("Scanning submissions from %s" % url)
count = 0
for el in doc.cssselect(".gallery > *"):
if el.get("id") == "no-images":
continue
id_str = el.get("id")[4:]
if id_str == "":
continue
id = int(id_str)
submission = self._submissions.get(id)
if submission is None:
submission = Submission()
submission._session = self
submission.id = id
self._submissions[id] = submission
submission.title = str(
el.cssselect("span")[0].text_content())
if "r-adult" in el.classes:
submission.rating = "adult"
elif "r-mature" in el.classes:
submission.rating = "mature"
elif "r-general" in el.classes:
submission.rating = "general"
else:
raise exceptions.ScraperError()
if "t-image" in el.classes:
submission.type = "image"
elif "t-text" in el.classes:
submission.type = "text"
elif "t-audio" in el.classes:
submission.type = "audio"
elif "t-flash" in el.classes:
submission.type = "flash"
else:
raise exceptions.ScraperError()
submission.thumbnail_url = "https:" + el.cssselect("img")[
0].get("src")
submissions.append(submission)
count += 1
if count == 0:
break
logger.debug("Found %d submissions" % count)
page += 1
except (IndexError, ValueError):
raise exceptions.ScraperError()
return submissions
def _load_folders(self):
logger.debug("Loading folders")
self._root_folders = []
url = constants.FA_ROOT + "/controls/folders/submissions/"
doc = self._limited_call(self._html_get, url)
# get groups
for group_el in doc.cssselect(".group-row"):
try:
title = str(group_el.cssselect("strong")[0].text_content())
id_match = re.search("group-([0-9]+)", group_el.get("class"))
id = int(id_match.group(1))
group = self._folders.get(id)
if group is None:
group = Folder()
group._session = self
group.id = id
self._folders[id] = group
group.title = title
group.children = []
group.submissions = []
self._root_folders.append(group)
except (IndexError, ValueError):
raise exceptions.ScraperError()
# Get folders
for folder_el in doc.cssselect(".folder-row"):
try:
title = str(folder_el.cssselect(".folder-name strong")[
0].text_content())
id_match = re.search("folder-([0-9]+)", folder_el.get("class"))
group_match = re.search("group-([0-9]+)",
folder_el.get("class"))
id = int(id_match.group(1))
parent_id = int(group_match.group(1))
folder = self._folders.get(id)
if folder is None:
folder = Folder()
folder._session = self
folder.id = id
self._folders[id] = folder
folder.title = title
folder.children = []
parent = self._folders.get(parent_id)
if parent is None:
self._root_folders.append(folder)
else:
parent.children.append(folder)
except (IndexError, ValueError):
raise exceptions.ScraperError()