def download(url):
from main import session_folder
with lock:
filename = cmd(['youtube-dl', '--no-playlist', '-o', OUTPUT_TEMPLATE, '--get-filename', url]).strip()
filename, ext = os.path.splitext(filename)
filename = re.sub('[^A-Za-z0-9.-_]', '_', filename)
filepath = '{}.%(ext)s'.format(filename)
output_path = os.path.join(session_folder, filepath)
cmd(['youtube-dl', '--no-playlist', '-o', output_path, '-x', '--audio-format=mp3', url])
return filename
def convert(source, dest, start, end, title, artist, category, fade_in,
fade_out):
def format_filter(name, **kwargs):
return '{}={}'.format(
name, ':'.join('{}={}'.format(k, v) for k, v in kwargs.items()))
# end is actually "end not including fade out"
if end and fade_out:
end += fade_out
cut_args = []
if start:
cut_args += ['-ss', start]
if end:
cut_args += ['-t', end - start]
map_args = ['-map', '0:a']
filters = []
if fade_in:
filters.append(
format_filter('afade', type='in', start_time=0, duration=fade_in))
if fade_out:
# we need to know duration, hopefully we can work it out from inputs
# if we can't, we fall back to an ffprobe call
if not end:
end = get_audio_length(source)
if not start:
start = 0
duration = end - start
filters.append(
format_filter('afade',
type='out',
start_time=duration - fade_out,
duration=fade_out))
filter_args = ['-filter', ','.join(filters)] if filters else []
metadata = dict(title=title, artist=artist, genre=category)
metadata_args = sum((['-metadata', '{}={}'.format(k, v)]
for k, v in metadata.items() if v), [])
output_args = ['-strict', '-2'
] + map_args + filter_args + metadata_args + [dest]
input_args = cut_args + ['-i', source]
cmd(['ffmpeg', '-y'] + input_args + output_args)
def main(*youtube_dl_args, **kwargs):
log, conf, hook, lock, creds, filename_template = [
kwargs[k]
for k in ('log', 'conf', 'hook', 'lock', 'creds', 'filename_template')
]
conf, hook, lock, creds = [
os.path.expanduser(s) for s in (conf, hook, lock, creds)
]
logging.basicConfig(
level=log.upper(),
format='%(levelname)s:%(asctime)s:%(process)d:%(name)s:%(message)s')
logging.info(
"Executing with youtube-dl args: {!r}".format(youtube_dl_args))
with FLock(lock):
logging.info("Acquired lock {!r}".format(lock))
channels = parse_conf(conf)
# Do least recently checked first, in case we hit our api quota before finishing.
channels.sort(key=lambda (u, p, t): 0 if t is None else t)
logging.info("Got config for {} channels".format(len(channels)))
with open(creds) as f:
creds = json.load(f)
client = GoogleAPIClient(
base_url='https://www.googleapis.com/youtube/v3', **creds)
for url, original_path, timestamp in channels:
path = os.path.abspath(original_path)
if path != original_path:
logging.warning("Corrected relative path {!r} -> {!r}".format(
original_path, path))
start_time = time.time()
logging.info("Checking for new videos from {!r}".format(url))
new_files = check_url(client, url, path, youtube_dl_args,
filename_template)
update_conf(conf, {(url, original_path): start_time})
logging.info("Got {} new files".format(len(new_files)))
if new_files:
if os.access(hook, os.X_OK):
logging.info("Calling hook {!r}".format(hook))
cmd([hook], stdin='\n'.join(new_files) + '\n')
else:
logging.info(
"Hook {!r} does not exist or is not executable".format(
hook))
logging.info("Ran for entry {}".format(url))
logging.info("Ran successfully")
def get_audio_length(filename): args = [ 'ffprobe', '-select_streams', 'a:0', '-show_entries', 'format=duration', '-of', 'default=noprint_wrappers=1:nokey=1', filename, ] output = cmd(args) return int(output)
def update(self, msg, target, *ref):
"""Update a locally managed library to a given git ref
Fetches latest upstream, updates to given ref and installs.
If ref is blank, uses currently checked out branch.
Give target "list" to list known targets.
"""
ref = ' '.join(ref)
# input sanitation: ref may contain leading - and be parsed as a cli option
if ref.startswith('-'):
self.reply(msg, "Refusing to checkout ref with leading dash - consider refs/heads/... form?")
return
if target == 'list':
self.reply(msg, "Update targets: {}".format(", ".join(self.config.targets)))
return
if target not in self.config.targets:
self.reply(msg, "No such update target: {}".format(target))
return
target_info = self.config.targets[target]
if isinstance(target_info, basestring):
target_dir = target_info
target_cmd = [self.config.pip_cmd, 'install', '-U', '--no-deps', '{}/']
else:
target_dir, target_cmd = target_info
if target_cmd:
target_cmd = [arg.replace('{}', target_dir) for arg in target_cmd]
self.logger.info("Updating {} with ref {}".format(target, ref))
self.logger.debug("Target {} has dir {!r}, cmd {!r}".format(target, target_dir, target_cmd))
try:
if ref:
self.git(target_dir, 'checkout', ref, '--')
self.git(target_dir, 'pull', '--ff-only')
if target_cmd:
cmd(target_cmd)
except FailedProcessError as e:
self.logger.warning("Failed to update {}".format(target), exc_info=True)
self.reply(msg, "Update {} failed: {}".format(target, e.summary()))
else:
self.reply(msg, "Updated {}".format(target))
def convert(source, dest, start, end, title, artist, category, fade_in, fade_out):
def format_filter(name, **kwargs):
return '{}={}'.format(name, ':'.join('{}={}'.format(k, v) for k, v in kwargs.items()))
# end is actually "end not including fade out"
if end and fade_out:
end += fade_out
cut_args = []
if start:
cut_args += ['-ss', start]
if end:
cut_args += ['-t', end - start]
map_args = ['-map', '0:a']
filters = []
if fade_in:
filters.append(format_filter('afade', type='in', start_time=0, duration=fade_in))
if fade_out:
# we need to know duration, hopefully we can work it out from inputs
# if we can't, we fall back to an ffprobe call
if not end:
end = get_audio_length(source)
if not start:
start = 0
duration = end - start
filters.append(format_filter('afade', type='out', start_time=duration-fade_out, duration=fade_out))
filter_args = ['-filter', ','.join(filters)] if filters else []
metadata = dict(title=title, artist=artist, genre=category)
metadata_args = sum((
['-metadata', '{}={}'.format(k, v)]
for k, v in metadata.items() if v
), [])
output_args = ['-strict', '-2'] + map_args + filter_args + metadata_args + [dest]
input_args = cut_args + ['-i', source]
cmd(['ffmpeg', '-y'] + input_args + output_args)
def get_audio_length(filename):
args = [
'ffprobe',
'-select_streams',
'a:0',
'-show_entries',
'format=duration',
'-of',
'default=noprint_wrappers=1:nokey=1',
filename,
]
output = cmd(args)
return int(output)
def check_url(youtube_client, url, path, youtube_dl_args, filename_template):
"""For given playlist, user or channel url,
checks for videos that don't already exist, and if so downloads them to
given path. Adds any given youtube-dl args as extra args. Returns a list of new files."""
parsed = urlparse.urlparse(url)
query = urlparse.parse_qs(parsed.query)
if parsed.path.startswith('/playlist'):
playlist = query['list']
logging.info("Interpreted url {} as playlist {}".format(url, playlist))
else:
if parsed.path.startswith('/user'):
user = parsed.path.split('/')[2]
params = {'forUsername': user}
elif parsed.path.startswith('/channel'):
channel = parsed.path.split('/')[2]
params = {'id': channel}
else:
raise ValueError("Unrecognised url {!r}".format(url))
result = youtube_client.request('GET',
'channels',
part='contentDetails',
**params)
if not result['items']:
raise ValueError("No channel found for {}".format(params))
if len(result['items']) > 1:
raise ValueError("Multiple channels found for {}: {}".format(
params, result['items']))
item = result['items'][0]
playlist = item['contentDetails']['relatedPlaylists']['uploads']
logging.info(
"Interpreted url {} as {}, got uploads playlist {}".format(
url, params, playlist))
logging.info("Looking for new videos in playlist {}".format(playlist))
# download list of items in playlist
items = []
token = None
while True:
result = youtube_client.request('GET',
'playlistItems',
playlistId=playlist,
part='snippet',
**({
'pageToken': token
} if token is not None else {}))
items += result['items']
if 'nextPageToken' not in result:
break
token = result['nextPageToken']
logging.info("Found {} videos in playlist".format(len(items)))
items = [item['snippet']['resourceId']['videoId'] for item in items]
try:
os.makedirs(path)
except OSError as e:
if e.errno != errno.EEXIST:
raise
exists = os.listdir(path)
to_download = []
for id in items:
# ignore ones that already exist
matches = [name for name in exists if "-{}.".format(id) in name]
if matches:
logging.debug("Ignoring video {}: already exists as {}".format(
id, matches))
continue
to_download.append(id)
if not to_download:
return []
logging.info("Downloading videos: {}".format(to_download))
# In order to get a list of downloaded files, we resort to a hack:
# we download to a temp dir first, then rename.
# This also protects us from partial downloads.
tempdir = tempfile.mkdtemp(prefix='youtube-dl-channel-bot-',
suffix='.tmp.d',
dir=path)
try:
output_template = '{}/{}'.format(tempdir, filename_template)
# Unfortunately, youtube-dl will exit 1 if there are any copyright-blocked videos,
# even with --ignore-errors. We allow 1 as a success exit code.
cmd(
['youtube-dl', '--ignore-errors'] + list(youtube_dl_args) +
['-o', output_template, '--'] + to_download,
stdout=sys.stdout,
success=[0, 1],
)
ret = []
for name in os.listdir(tempdir):
new_path = os.path.join(path, name)
logging.info("Saving new file {!r}".format(new_path))
os.rename(os.path.join(tempdir, name), new_path)
ret.append(new_path)
return ret
finally:
# attempt to clean up tempdir as much as we can
for name in os.listdir(tempdir):
try:
os.remove(os.path.join(tempdir, name))
except EnvironmentError:
pass
try:
os.rmdir(tempdir)
except EnvironmentError:
pass
def youtube_dl(link, filebase):
cmd(['youtube-dl', link, '-o', '{}.%(ext)s'.format(filebase)])
filename, = glob('{}.*'.format(filebase))
return filename
def upload(source, name):
_, ext = os.path.splitext(source)
name = '{}.{}'.format(name, ext.lstrip('.'))
cmd(['scp', source, 'tyranicmoron:public_html/rdp/{}'.format(name)])
return 'http://tyranicmoron.uk/~ekimekim/rdp/{}'.format(name)
def install_package(self, package): cmd(['pacman', '-Sy', '--noconfirm', package])
def index_packages(cls):
data = cmd(['pacman', '-Ql'])
for line in data.strip().split('\n'):
package, filepath = line.split(' ', 1)
cls.set_package(filepath, package)
def git(self, target_dir, subcmd, *args):
self.logger.debug("Running git {} in {} with args {}".format(subcmd, target_dir, args))
return cmd(['git', '-C', target_dir, subcmd] + list(args))
def youtube_dl(link, filebase):
cmd(['youtube-dl', link, '-o', '{}.%(ext)s'.format(filebase)])
filename, = glob('{}.*'.format(filebase))
return filename
def upload(source, name):
_, ext = os.path.splitext(source)
name = '{}.{}'.format(name, ext.lstrip('.'))
cmd(['scp', source, 'tyranicmoron:public_html/rdp/{}'.format(name)])
return 'http://tyranicmoron.uk/~ekimekim/rdp/{}'.format(name)
def check_package(self, package): cmd(['pacman', '-Qq', package])
def cmd(*args, **kwargs): print 'running', ' '.join(map(str, args)) return easycmd.cmd(args, **kwargs)
def process_file(top_level_dir, include_dir, tests_dir, extra_link_dirs,
objs_dir, filename):
name, _ = os.path.splitext(filename)
filepath = os.path.join(tests_dir, filename)
config = dict(Memory=Memory, Test=Test, random=random.Random(name))
execfile(filepath, config) # loads config as defined globals
if 'file' not in config:
raise ValueError("You must specify a target file, or None")
include_file = config['file']
link_files = config.get('files')
target = config.get('target')
extra_asm = config.get('asm', '')
mems = {
label: value.contents
for label, value in config.items()
if isinstance(value, Memory) and not label.startswith('_')
}
tests = {
testname: test
for testname, test in config.items() if isinstance(test, Test)
}
if target is None and any(test.target is None for test in tests.values()):
raise ValueError(
"You must specify a target function, either at top-level or for every test case"
)
if include_file is None:
include_asm = ''
else:
include_path = os.path.join(top_level_dir,
'{}.asm'.format(include_file))
with open(include_path) as f:
include_asm = f.read()
if link_files is None:
asm_files = os.listdir(top_level_dir)
for extra_link_dir in extra_link_dirs:
asm_files += [
os.path.join(extra_link_dir, filename)
for filename in os.listdir(extra_link_dir)
]
link_files = [
os.path.splitext(asm_file)[0] for asm_file in asm_files
if asm_file.endswith('.asm') and asm_file != 'header.asm'
]
if include_file in link_files:
link_files.remove(include_file)
link_files = [
os.path.join(objs_dir, link_file) for link_file in link_files
]
link_paths = [
os.path.join(top_level_dir, '{}.o'.format(link_file))
for link_file in link_files
]
gendir = os.path.join(tests_dir, name)
if not os.path.exists(gendir):
os.mkdir(gendir)
for i, (testname, test) in enumerate(
sorted(tests.items(), key=lambda (n, t): t.order)):
testname = '{i:0{w}d}_{t}'.format(i=i,
t=testname,
w=len(str(len(tests) - 1)))
asm = test.gen_asm(include_asm, target, extra_asm, mems)
path = os.path.join(gendir, testname)
asm_path = '{}.asm'.format(path)
obj_path = '{}.o'.format(path)
sym_path = '{}.sym'.format(path)
rom_path = '{}.gb'.format(path)
with open(asm_path, 'w') as f:
f.write(asm)
# We pad wth 0x40 = ld b, b = BGB breakpoint
cmd([
'rgbasm', '-DDEBUG', '-i', include_dir, '-v', '-o', obj_path,
asm_path
])
cmd([
'rgblink', '-n', sym_path, '-o', rom_path, '-p', '0x40', obj_path
] + link_paths)
cmd(['rgbfix', '-v', '-p', '0x40', rom_path])
