def __init__(
self,
port,
*,
password,
authorize_by_ip,
enable_zip,
enable_directory_listings,
overall_ratelimit,
individual_ratelimit,
):
self.port = port
self.password = password
self.authorize_by_ip = authorize_by_ip
self.enable_directory_listings = enable_directory_listings
self.enable_zip = enable_zip
self.overall_ratelimit = ratelimiter.Ratelimiter(overall_ratelimit)
self.individual_ratelimit = individual_ratelimit
if authorize_by_ip:
self.accepted_ips = set()
self.accepted_tokens = None
else:
self.accepted_tokens = set()
self.accepted_ips = None
def __init__(self, total_bytes):
self.limiter = ratelimiter.Ratelimiter(allowance=8, mode='reject')
self.limiter.balance = 1
self.total_bytes = max(1, total_bytes)
self.divisor = bytestring.get_appropriate_divisor(total_bytes)
self.total_format = bytestring.bytestring(total_bytes,
force_unit=self.divisor)
self.downloaded_format = '{:>%d}' % len(self.total_format)
self.blank_char = ' '
self.solid_char = '█'
def limiter_or_none(value):
if isinstance(value, str):
value = bytestring.parsebytes(value)
if isinstance(value, ratelimiter.Ratelimiter):
limiter = value
elif value is not None:
limiter = ratelimiter.Ratelimiter(allowance=value, period=1)
else:
limiter = None
return limiter
def _initialize_ratelimiter():
global ratelimit
if ratelimit is not None:
return
log.debug('Initializing pushshift ratelimiter.')
url = 'https://api.pushshift.io/meta'
response = session.get(url)
response.raise_for_status()
response = response.json()
limit = response['server_ratelimit_per_minute']
log.debug('Pushshift ratelimit is %d requests per minute.', limit)
ratelimit = ratelimiter.Ratelimiter(allowance=limit, period=60)
'edited': False,
'link_flair_css_class': None,
'link_flair_text': None,
'score': 0,
'selftext': '',
}
contact_info_message = '''
Please add a CONTACT_INFO string variable to your bot.py file.
This will be added to your pushshift useragent.
'''.strip()
if not getattr(common.bot, 'CONTACT_INFO', ''):
raise ValueError(contact_info_message)
useragent = USERAGENT.format(version=common.VERSION, contact=common.bot.CONTACT_INFO)
ratelimit = ratelimiter.Ratelimiter(allowance=60, period=60)
session = requests.session()
session.headers.update({'User-Agent': useragent})
class DummyObject:
'''
These classes are used to convert the JSON data we get from pushshift into
objects so that the rest of timesearch can operate transparently.
This requires a bit of whack-a-mole including:
- Fleshing out the attributes which PS did not include because they were
null (we use FALLBACK_ATTRIBUTES to replace them).
- Providing the convenience methods and @properties that PRAW provides.
- Mimicking the rich attributes like author and subreddit.
'''
def __init__(self, **attributes):
import mimetypes
import os
import urllib.parse
import pathlib
import random
import socketserver
import sys
import types
# pip install voussoirkit
from voussoirkit import bytestring
from voussoirkit import pathclass
from voussoirkit import ratelimiter
FILE_READ_CHUNK = bytestring.MIBIBYTE
RATELIMITER = ratelimiter.Ratelimiter(16 * bytestring.MIBIBYTE)
# The paths which the user may access.
# Attempting to access anything outside will 403.
# These are convered to Path objects after that class definition.
OKAY_PATHS = set(['files', 'favicon.ico'])
OPENDIR_TEMPLATE = '''
<html>
<body>
<meta charset="UTF-8">
<style type="text/css">Body {{font-family:Consolas}}</style>
<table style="width: 95%">
{entries}
</table>
SQL_COLUMNS = sqlhelpers.extract_table_column_map(DB_INIT)
sql = sqlite3.connect('sticks.db')
sql.executescript(DB_INIT)
USERAGENT = '''
Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)
Chrome/79.0.3945.130 Safari/537.36
'''.replace('\n', ' ').strip()
HEADERS = {'User-Agent': USERAGENT}
session = requests.Session()
session.headers.update(HEADERS)
DOWNLOAD_RATELIMITER = ratelimiter.Ratelimiter(allowance=1, period=5)
WINRAR = winwhich.which('winrar')
def get_now():
return datetime.datetime.now(datetime.timezone.utc).timestamp()
def id_from_direct_url(direct_url):
id = direct_url.split('/direct/')[-1]
id = id.split('/')[0].split('?')[0]
return id
# DB FUNCTIONS
def __init__(self, total_bytes):
self.total_bytes = max(1, total_bytes)
self.limiter = ratelimiter.Ratelimiter(allowance=8, mode='reject')
self.limiter.balance = 1
self.total_bytes_string = '{:,}'.format(self.total_bytes)
self.bytes_downloaded_string = '{:%d,}' % len(self.total_bytes_string)
def prepare_plan(
url,
localname,
auth=None,
bytespersecond=None,
callback_progress=None,
do_head=True,
headers=None,
overwrite=False,
raise_for_undersized=True,
timeout=TIMEOUT,
):
# Chapter 1: File existence
headers = headers or {}
user_provided_range = 'range' in headers
real_localname = localname
temp_localname = localname + TEMP_EXTENSION
real_exists = os.path.exists(real_localname)
if real_exists and overwrite is False and not user_provided_range:
print('File exists and overwrite is off. Nothing to do.')
return None
temp_exists = os.path.exists(temp_localname)
real_localsize = int(real_exists and os.path.getsize(real_localname))
temp_localsize = int(temp_exists and os.path.getsize(temp_localname))
# Chapter 2: Ratelimiting
if bytespersecond is None:
limiter = None
elif isinstance(bytespersecond, ratelimiter.Ratelimiter):
limiter = bytespersecond
else:
limiter = ratelimiter.Ratelimiter(allowance=bytespersecond)
# Chapter 3: Extracting range
if user_provided_range:
user_range_min = int(headers['range'].split('bytes=')[1].split('-')[0])
user_range_max = headers['range'].split('-')[1]
if user_range_max != '':
user_range_max = int(user_range_max)
else:
user_range_min = None
user_range_max = None
# Chapter 4: Server range support
# Always include a range on the first request to figure out whether the
# server supports it. Use 0- to get correct remote_total_bytes
temp_headers = headers
temp_headers.update({'range': 'bytes=0-'})
if do_head:
# I'm using a GET instead of an actual HEAD here because some servers respond
# differently, even though they're not supposed to.
head = request('get',
url,
stream=True,
headers=temp_headers,
auth=auth)
remote_total_bytes = int(head.headers.get('content-length', 0))
server_respects_range = (head.status_code == 206
and 'content-range' in head.headers)
head.connection.close()
else:
remote_total_bytes = None
server_respects_range = False
if user_provided_range and not server_respects_range:
if not do_head:
raise Exception(
'Cannot determine range support without the head request')
else:
raise Exception('Server did not respect your range header')
# Chapter 5: Plan definitions
plan_base = {
'url': url,
'auth': auth,
'callback_progress': callback_progress,
'limiter': limiter,
'headers': headers,
'real_localname': real_localname,
'raise_for_undersized': raise_for_undersized,
'remote_total_bytes': remote_total_bytes,
'timeout': timeout,
}
plan_fulldownload = dict(
plan_base,
download_into=temp_localname,
header_range_min=None,
header_range_max=None,
plan_type='fulldownload',
seek_to=0,
)
plan_resume = dict(
plan_base,
download_into=temp_localname,
header_range_min=temp_localsize,
header_range_max='',
plan_type='resume',
seek_to=temp_localsize,
)
plan_partial = dict(
plan_base,
download_into=real_localname,
header_range_min=user_range_min,
header_range_max=user_range_max,
plan_type='partial',
seek_to=user_range_min,
)
# Chapter 6: Redeem your meal vouchers here
if real_exists:
if overwrite:
os.remove(real_localname)
if user_provided_range:
return plan_partial
return plan_fulldownload
elif temp_exists and temp_localsize > 0:
if overwrite:
return plan_fulldownload
if user_provided_range:
return plan_partial
if server_respects_range:
print('Resume from byte %d' % plan_resume['seek_to'])
return plan_resume
else:
if user_provided_range:
return plan_partial
return plan_fulldownload
raise Exception('No plan was chosen?')
warnings.simplefilter('ignore')
HEADERS = {
'User-Agent':
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.152 Safari/537.36'
}
FILENAME_BADCHARS = '*?"<>|\r\n'
last_request = 0
CHUNKSIZE = 4 * bytestring.KIBIBYTE
TIMEOUT = 60
TEMP_EXTENSION = '.downloadytemp'
PRINT_LIMITER = ratelimiter.Ratelimiter(allowance=5, mode='reject')
class NotEnoughBytes(Exception):
pass
def download_file(
url,
localname=None,
auth=None,
bytespersecond=None,
callback_progress=None,
do_head=True,
headers=None,
overwrite=False,
def __init__(self, request, client_info, server, individual_ratelimit):
self.individual_ratelimit = ratelimiter.Ratelimiter(individual_ratelimit)
super().__init__(request, client_info, server)