def __init__(self, key):
self.youtube = apiclient.discovery.build(
cache_discovery=False,
developerKey=key,
serviceName='youtube',
version='v3',
)
self.log = vlogging.getLogger(__name__)
def __init__(
self,
youtube,
create=True,
data_directory=None,
log_level=vlogging.NOTSET,
skip_version_check=False,
):
super().__init__()
self.youtube = youtube
# DATA DIR PREP
if data_directory is None:
data_directory = constants.DEFAULT_DATADIR
self.data_directory = pathclass.Path(data_directory)
if self.data_directory.exists and not self.data_directory.is_dir:
raise exceptions.BadDataDirectory(
self.data_directory.absolute_path)
# LOGGING
self.log = vlogging.getLogger(
f'{__name__}:{self.data_directory.absolute_path}')
self.log.setLevel(log_level)
self.youtube.log.setLevel(log_level)
# DATABASE
self.database_filepath = self.data_directory.with_child(
constants.DEFAULT_DBNAME)
existing_database = self.database_filepath.exists
if not existing_database and not create:
msg = f'"{self.database_filepath.absolute_path}" does not exist and create is off.'
raise FileNotFoundError(msg)
self.data_directory.makedirs(exist_ok=True)
self.sql = sqlite3.connect(self.database_filepath.absolute_path)
if existing_database:
if not skip_version_check:
self._check_version()
self._load_pragmas()
else:
self._first_time_setup()
# CONFIG
self.config_filepath = self.data_directory.with_child(
constants.DEFAULT_CONFIGNAME)
self.load_config()
self.caches = {
'channel': cacheclass.Cache(maxlen=20_000),
'video': cacheclass.Cache(maxlen=50_000),
}
import argparse
import sys
from voussoirkit import pipeable
from voussoirkit import vlogging
log = vlogging.getLogger(__name__, 'printstdout')
def printstdout_argparse(args):
for text in args.texts:
text = pipeable.input(text)
for line in text:
pipeable.stdout(line)
def main(argv):
argv = vlogging.main_level_by_argv(argv)
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('texts', nargs='+')
parser.set_defaults(func=printstdout_argparse)
args = parser.parse_args(argv)
return args.func(args)
if __name__ == '__main__':
raise SystemExit(main(sys.argv[1:]))
import argparse
import logging
import requests
import sqlite3
import sys
import time
from voussoirkit import backoff
from voussoirkit import betterhelp
from voussoirkit import ratelimiter
from voussoirkit import sqlhelpers
from voussoirkit import threadpool
from voussoirkit import vlogging
log = vlogging.getLogger('hnarchive')
VERSION = 1
HEADERS = {
'User-Agent': f'voussoir/hnarchive v{VERSION}.',
}
DB_INIT = '''
PRAGMA user_version = 1;
CREATE TABLE IF NOT EXISTS items(
id INT PRIMARY KEY NOT NULL,
deleted INT,
type TEXT,
author TEXT,
time INT,
text TEXT,
import shutil
import subprocess
import sys
import time
import traceback
from voussoirkit import betterhelp
from voussoirkit import bytestring
from voussoirkit import operatornotify
from voussoirkit import pathclass
from voussoirkit import subproctools
from voussoirkit import vlogging
from voussoirkit import winglob
from voussoirkit import winwhich
log = vlogging.getLogger(__name__, 'rarpar')
WINRAR = winwhich.which('winrar')
PAR2 = winwhich.which('phpar2')
RESERVE_SPACE_ON_DRIVE = 5 * bytestring.GIBIBYTE
COMPRESSION_STORE = 0
COMPRESSION_MAX = 5
class RarParException(Exception):
pass
class RarExists(RarParException):
A string like "50g" or "100 gb"
drive:
Filepath to the drive you want to check. Defaults to cwd drive.
'''
import argparse
import os
import shutil
import sys
from voussoirkit import betterhelp
from voussoirkit import bytestring
from voussoirkit import dotdict
from voussoirkit import vlogging
log = vlogging.getLogger(__name__, 'reserve_disk_space')
class NotEnoughSpace(Exception):
def __init__(self, free, reserve, drive):
self.free = free
self.reserve = reserve
self.drive = drive
def reserve_disk_space(reserve, drive):
drive = drive.replace('\\', os.sep).replace('/', os.sep)
drive = os.path.abspath(drive)
drive = os.path.splitdrive(drive)[0]
log.debug('Checking drive %s', drive)
See voussoirkit.operatornotify.py for details.
'''
import argparse
import bs4
import requests
import sys
import time
from voussoirkit import backoff
from voussoirkit import betterhelp
from voussoirkit import downloady
from voussoirkit import operatornotify
from voussoirkit import pathclass
from voussoirkit import vlogging
log = vlogging.getLogger(__name__, 'fdroidapk')
session = requests.Session()
def get_apk_url(package_name):
url = f'https://f-droid.org/en/packages/{package_name}'
log.debug('Downloading page %s', url)
response = session.get(url, timeout=30)
response.raise_for_status()
soup = bs4.BeautifulSoup(response.text, 'html.parser')
li = soup.find('li', {'class': 'package-version'})
aa = li.find_all('a')
aa = [a for a in aa if a.get('href', '').endswith('.apk')]
apk_url = aa[0]['href']
return apk_url
import os import praw3 as praw import requests import sqlite3 import string import sys import time import traceback from voussoirkit import operatornotify from voussoirkit import httperrors httperrors.monkeypatch_requests() from voussoirkit import sqlhelpers from voussoirkit import vlogging log = vlogging.getLogger(__name__, 'newsletterly') #################################################################################################### # CONFIGURATION #################################################################################### # https://www.reddit.com/comments/3cm1p8/how_to_make_your_bot_use_oauth2/ import newsletterly_credentials # Username is not used for login purposes, just message text SELF_USERNAME = newsletterly_credentials.USERNAME USERAGENT = newsletterly_credentials.USERAGENT APP_ID = newsletterly_credentials.APP_ID APP_SECRET = newsletterly_credentials.APP_SECRET APP_URI = newsletterly_credentials.APP_URI APP_REFRESH = newsletterly_credentials.APP_REFRESH
import PIL.Image
import argparse
import sys
from voussoirkit import pipeable
from voussoirkit import sentinel
from voussoirkit import vlogging
from voussoirkit import winglob
log = vlogging.getLogger(__name__, 'stitch')
VERTICAL = sentinel.Sentinel('vertical')
HORIZONTAL = sentinel.Sentinel('horizontal')
def stitch_argparse(args):
patterns = pipeable.input_many(args.image_files,
skip_blank=True,
strip=True)
files = [file for pattern in patterns for file in winglob.glob(pattern)]
images = [PIL.Image.open(file) for file in files]
if args.vertical:
direction = VERTICAL
else:
direction = HORIZONTAL
gapcount = len(images) - 1
if direction is HORIZONTAL:
width = sum(i.size[0] for i in images) + (gapcount * args.gap)
height = max(i.size[1] for i in images)
import argparse
import datetime
import gzip
import PIL.Image
import random
import requests
import sqlite3
import sys
import time
from voussoirkit import threadpool
from voussoirkit import vlogging
log = vlogging.getLogger(__name__, 'pixelcanvasdl')
vlogging.getLogger('urllib3.connectionpool').setLevel(vlogging.CRITICAL)
WHITE = (255, 255, 255)
LIGHTGRAY = (228, 228, 228)
DARKGRAY = (136, 136, 136)
BLACK = (34, 34, 34)
PINK = (255, 167, 209)
RED = (229, 0, 0)
ORANGE = (229, 149, 0)
BROWN = (160, 106, 66)
YELLOW = (229, 217, 0)
LIGHTGREEN = (148, 224, 68)
DARKGREEN = (2, 190, 1)
LIGHTBLUE = (0, 211, 221)
MEDIUMBLUE = (0, 131, 199)
DARKBLUE = (0, 0, 234)
LIGHTPURPLE = (207, 110, 228)
import sys
import time
from voussoirkit import backoff
from voussoirkit import betterhelp
from voussoirkit import httperrors
from voussoirkit import mutables
from voussoirkit import operatornotify
from voussoirkit import pathclass
from voussoirkit import ratelimiter
from voussoirkit import sqlhelpers
from voussoirkit import threadpool
from voussoirkit import treeclass
from voussoirkit import vlogging
log = vlogging.getLogger(__name__, 'hnarchive')
VERSION = '1.0.0'
HEADERS = {
'User-Agent': f'voussoir/hnarchive v{VERSION}.',
}
session = requests.Session()
session.headers.update(HEADERS)
DB_INIT = '''
BEGIN;
PRAGMA user_version = 1;
CREATE TABLE IF NOT EXISTS items(
id INT PRIMARY KEY NOT NULL,
import argparse
import collections
import os
import shutil
import sys
from voussoirkit import passwordy
from voussoirkit import pathclass
from voussoirkit import pipeable
from voussoirkit import spinal
from voussoirkit import vlogging
from voussoirkit import winglob
log = vlogging.getLogger(__name__, 'sole_subdir_lift')
def sole_lift_argparse(args):
starting = pathclass.Path(args.starting)
queue = collections.deque()
queue.extend(spinal.walk(starting, yield_files=False, yield_directories=True))
while len(queue) > 0:
directory = queue.popleft()
if not directory.exists:
log.debug('%s no longer exists.', directory)
continue
if directory not in starting:
log.debug('%s is outside of starting.', directory)
continue
children = directory.listdir()
import argparse
import codecs
import os
import pyperclip
import re
import sys
from voussoirkit import interactive
from voussoirkit import pathclass
from voussoirkit import pipeable
from voussoirkit import spinal
from voussoirkit import vlogging
from voussoirkit import winglob
log = vlogging.getLogger(__name__, 'contentreplace')
def contentreplace(file,
replace_from,
replace_to,
autoyes=False,
do_regex=False):
file = pathclass.Path(file)
with file.open('r', encoding='utf-8') as f:
content = f.read()
if do_regex:
occurances = len(re.findall(replace_from, content, flags=re.MULTILINE))
else:
occurances = content.count(replace_from)
import argparse
import os
import re
import sys
from voussoirkit import interactive
from voussoirkit import pathclass
from voussoirkit import pipeable
from voussoirkit import vlogging
from voussoirkit import winglob
log = vlogging.getLogger(__name__, 'adbinstall')
def natural_sorter(x):
'''
Used for sorting files in 'natural' order instead of lexicographic order,
so that you get 1 2 3 4 5 6 7 8 9 10 11 12 13 ...
instead of 1 10 11 12 13 2 3 4 5 ...
Thank you Mark Byers
http://stackoverflow.com/a/11150413
'''
convert = lambda text: int(text) if text.isdigit() else text.lower()
alphanum_key = lambda key: [convert(c) for c in re.split('([0-9]+)', key)]
return alphanum_key(x)
def adbinstall_argparse(args):
patterns = pipeable.input_many(args.apks, skip_blank=True, strip=True)
apks = [file for pattern in patterns for file in winglob.glob(pattern)]
installs = []
import argparse
import flask; from flask import request
import os
import pyperclip
import sys
from voussoirkit import vlogging
log = vlogging.getLogger(__name__, 'heresmyclipboard')
site = flask.Flask(__name__)
TEMPLATE = '''
<html>
<body>
<pre>
{{clip}}
</pre>
</body>
</html>
'''
@site.route('/')
def root():
clip = pyperclip.paste()
return flask.render_template_string(TEMPLATE, clip=clip)
def heresmyclipboard_argparse(args):
log.info(f'Starting server on port {args.port}, pid={os.getpid()}')
site.run(host='0.0.0.0', port=args.port)
import argparse
import re
import subprocess
import sys
from voussoirkit import pathclass
from voussoirkit import winglob
from voussoirkit import winwhich
from voussoirkit import vlogging
log = vlogging.getLogger(__name__, 'ffstreams')
AUDIO_EXTENSIONS = {
'aac': 'm4a',
'ac3': 'ac3',
'flac': 'flac',
'mp3': 'mp3',
'opus': 'opus',
'vorbis': 'ogg',
'*': 'mka',
}
SUBTITLE_EXTENSIONS = {
'ass': 'ass',
'subrip': 'srt',
'*': 'mks',
}
FFMPEG = winwhich.which('ffmpeg')
def make_maps(input_file, prefix, search_pattern, extension_map, moveto=None):
import bs4
import requests
from voussoirkit import vlogging
from . import exceptions
log = vlogging.getLogger(__name__)
session = requests.Session()
def _get_user_videos(channel_id):
log.info(f'Fetching RSS for {channel_id}.')
url = f'https://www.youtube.com/feeds/videos.xml?channel_id={channel_id}'
response = session.get(url)
response.raise_for_status()
soup = bs4.BeautifulSoup(response.text, 'lxml')
# find_all does not work on namespaced tags unless you add a limit paramter.
video_ids = [v.text for v in soup.find_all('yt:videoid', limit=9999)]
log.loud('RSS got %s.', video_ids)
return video_ids
def get_user_videos(channel_id):
try:
return _get_user_videos(channel_id)
except Exception:
raise exceptions.RSSAssistFailed(f'Failed to fetch RSS videos.') from exc
def get_user_videos_since(channel_id, video_id):
video_ids = get_user_videos(channel_id)
try:
> bitwise_or file1 file2 --output file3
'''
import argparse
import os
import sys
from voussoirkit import betterhelp
from voussoirkit import interactive
from voussoirkit import operatornotify
from voussoirkit import pathclass
from voussoirkit import pipeable
from voussoirkit import vlogging
from voussoirkit import winglob
log = vlogging.getLogger(__name__, 'bitwise_or')
CHUNK_SIZE = 2**20
def bitwise_or_argparse(args):
patterns = pipeable.input_many(args.files, skip_blank=True, strip=True)
files = [file for pattern in patterns for file in winglob.glob(pattern)]
files = [pathclass.Path(file) for file in files]
if len(files) < 2:
log.fatal('Need at least two input files.')
return 1
handles = [file.open('rb') for file in files]
import subprocess
import sys
import time
import tkinter
import traceback
import types
from voussoirkit import betterhelp
from voussoirkit import mutables
from voussoirkit import operatornotify
from voussoirkit import pathclass
from voussoirkit import pipeable
from voussoirkit import sqlhelpers
from voussoirkit import vlogging
log = vlogging.getLogger(__name__, 'sb')
USERAGENT = '''
/u/GoldenSights SubredditBirthdays data collection:
Gathering the creation dates of subreddits for visualization.
More at https://github.com/voussoir/reddit/tree/master/SubredditBirthdays
'''.replace('\n', ' ').strip()
LOWERBOUND_STR = '2qh0j'
LOWERBOUND_INT = 4594339
FORMAT_MEMBER = '{idstr:>5s}, {human}, {nsfw}, {name:<25s} {subscribers:>10,}'
FORMAT_MESSAGE_NEW = 'New: {idstr:>5s} : {human} : {nsfw} : {name} : {subscribers}'
FORMAT_MESSAGE_UPDATE = 'Upd: {idstr:>5s} : {human} : {nsfw} : {name} : {subscribers} ({subscriber_diff})'
RANKS_UP_TO = 20000
import argparse
import sys
from voussoirkit import pipeable
from voussoirkit import vlogging
log = vlogging.getLogger(__name__, 'printstderr')
def printstderr_argparse(args):
for text in args.texts:
text = pipeable.input(text)
for line in text:
pipeable.stderr(line)
def main(argv):
argv = vlogging.main_level_by_argv(argv)
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('texts', nargs='+')
parser.set_defaults(func=printstderr_argparse)
args = parser.parse_args(argv)
return args.func(args)
if __name__ == '__main__':
raise SystemExit(main(sys.argv[1:]))
