def download(url): from main import session_folder with lock: filename = cmd(['youtube-dl', '--no-playlist', '-o', OUTPUT_TEMPLATE, '--get-filename', url]).strip() filename, ext = os.path.splitext(filename) filename = re.sub('[^A-Za-z0-9.-_]', '_', filename) filepath = '{}.%(ext)s'.format(filename) output_path = os.path.join(session_folder, filepath) cmd(['youtube-dl', '--no-playlist', '-o', output_path, '-x', '--audio-format=mp3', url]) return filename
def convert(source, dest, start, end, title, artist, category, fade_in, fade_out): def format_filter(name, **kwargs): return '{}={}'.format( name, ':'.join('{}={}'.format(k, v) for k, v in kwargs.items())) # end is actually "end not including fade out" if end and fade_out: end += fade_out cut_args = [] if start: cut_args += ['-ss', start] if end: cut_args += ['-t', end - start] map_args = ['-map', '0:a'] filters = [] if fade_in: filters.append( format_filter('afade', type='in', start_time=0, duration=fade_in)) if fade_out: # we need to know duration, hopefully we can work it out from inputs # if we can't, we fall back to an ffprobe call if not end: end = get_audio_length(source) if not start: start = 0 duration = end - start filters.append( format_filter('afade', type='out', start_time=duration - fade_out, duration=fade_out)) filter_args = ['-filter', ','.join(filters)] if filters else [] metadata = dict(title=title, artist=artist, genre=category) metadata_args = sum((['-metadata', '{}={}'.format(k, v)] for k, v in metadata.items() if v), []) output_args = ['-strict', '-2' ] + map_args + filter_args + metadata_args + [dest] input_args = cut_args + ['-i', source] cmd(['ffmpeg', '-y'] + input_args + output_args)
def main(*youtube_dl_args, **kwargs): log, conf, hook, lock, creds, filename_template = [ kwargs[k] for k in ('log', 'conf', 'hook', 'lock', 'creds', 'filename_template') ] conf, hook, lock, creds = [ os.path.expanduser(s) for s in (conf, hook, lock, creds) ] logging.basicConfig( level=log.upper(), format='%(levelname)s:%(asctime)s:%(process)d:%(name)s:%(message)s') logging.info( "Executing with youtube-dl args: {!r}".format(youtube_dl_args)) with FLock(lock): logging.info("Acquired lock {!r}".format(lock)) channels = parse_conf(conf) # Do least recently checked first, in case we hit our api quota before finishing. channels.sort(key=lambda (u, p, t): 0 if t is None else t) logging.info("Got config for {} channels".format(len(channels))) with open(creds) as f: creds = json.load(f) client = GoogleAPIClient( base_url='https://www.googleapis.com/youtube/v3', **creds) for url, original_path, timestamp in channels: path = os.path.abspath(original_path) if path != original_path: logging.warning("Corrected relative path {!r} -> {!r}".format( original_path, path)) start_time = time.time() logging.info("Checking for new videos from {!r}".format(url)) new_files = check_url(client, url, path, youtube_dl_args, filename_template) update_conf(conf, {(url, original_path): start_time}) logging.info("Got {} new files".format(len(new_files))) if new_files: if os.access(hook, os.X_OK): logging.info("Calling hook {!r}".format(hook)) cmd([hook], stdin='\n'.join(new_files) + '\n') else: logging.info( "Hook {!r} does not exist or is not executable".format( hook)) logging.info("Ran for entry {}".format(url)) logging.info("Ran successfully")
def get_audio_length(filename): args = [ 'ffprobe', '-select_streams', 'a:0', '-show_entries', 'format=duration', '-of', 'default=noprint_wrappers=1:nokey=1', filename, ] output = cmd(args) return int(output)
def update(self, msg, target, *ref): """Update a locally managed library to a given git ref Fetches latest upstream, updates to given ref and installs. If ref is blank, uses currently checked out branch. Give target "list" to list known targets. """ ref = ' '.join(ref) # input sanitation: ref may contain leading - and be parsed as a cli option if ref.startswith('-'): self.reply(msg, "Refusing to checkout ref with leading dash - consider refs/heads/... form?") return if target == 'list': self.reply(msg, "Update targets: {}".format(", ".join(self.config.targets))) return if target not in self.config.targets: self.reply(msg, "No such update target: {}".format(target)) return target_info = self.config.targets[target] if isinstance(target_info, basestring): target_dir = target_info target_cmd = [self.config.pip_cmd, 'install', '-U', '--no-deps', '{}/'] else: target_dir, target_cmd = target_info if target_cmd: target_cmd = [arg.replace('{}', target_dir) for arg in target_cmd] self.logger.info("Updating {} with ref {}".format(target, ref)) self.logger.debug("Target {} has dir {!r}, cmd {!r}".format(target, target_dir, target_cmd)) try: if ref: self.git(target_dir, 'checkout', ref, '--') self.git(target_dir, 'pull', '--ff-only') if target_cmd: cmd(target_cmd) except FailedProcessError as e: self.logger.warning("Failed to update {}".format(target), exc_info=True) self.reply(msg, "Update {} failed: {}".format(target, e.summary())) else: self.reply(msg, "Updated {}".format(target))
def convert(source, dest, start, end, title, artist, category, fade_in, fade_out): def format_filter(name, **kwargs): return '{}={}'.format(name, ':'.join('{}={}'.format(k, v) for k, v in kwargs.items())) # end is actually "end not including fade out" if end and fade_out: end += fade_out cut_args = [] if start: cut_args += ['-ss', start] if end: cut_args += ['-t', end - start] map_args = ['-map', '0:a'] filters = [] if fade_in: filters.append(format_filter('afade', type='in', start_time=0, duration=fade_in)) if fade_out: # we need to know duration, hopefully we can work it out from inputs # if we can't, we fall back to an ffprobe call if not end: end = get_audio_length(source) if not start: start = 0 duration = end - start filters.append(format_filter('afade', type='out', start_time=duration-fade_out, duration=fade_out)) filter_args = ['-filter', ','.join(filters)] if filters else [] metadata = dict(title=title, artist=artist, genre=category) metadata_args = sum(( ['-metadata', '{}={}'.format(k, v)] for k, v in metadata.items() if v ), []) output_args = ['-strict', '-2'] + map_args + filter_args + metadata_args + [dest] input_args = cut_args + ['-i', source] cmd(['ffmpeg', '-y'] + input_args + output_args)
def check_url(youtube_client, url, path, youtube_dl_args, filename_template): """For given playlist, user or channel url, checks for videos that don't already exist, and if so downloads them to given path. Adds any given youtube-dl args as extra args. Returns a list of new files.""" parsed = urlparse.urlparse(url) query = urlparse.parse_qs(parsed.query) if parsed.path.startswith('/playlist'): playlist = query['list'] logging.info("Interpreted url {} as playlist {}".format(url, playlist)) else: if parsed.path.startswith('/user'): user = parsed.path.split('/')[2] params = {'forUsername': user} elif parsed.path.startswith('/channel'): channel = parsed.path.split('/')[2] params = {'id': channel} else: raise ValueError("Unrecognised url {!r}".format(url)) result = youtube_client.request('GET', 'channels', part='contentDetails', **params) if not result['items']: raise ValueError("No channel found for {}".format(params)) if len(result['items']) > 1: raise ValueError("Multiple channels found for {}: {}".format( params, result['items'])) item = result['items'][0] playlist = item['contentDetails']['relatedPlaylists']['uploads'] logging.info( "Interpreted url {} as {}, got uploads playlist {}".format( url, params, playlist)) logging.info("Looking for new videos in playlist {}".format(playlist)) # download list of items in playlist items = [] token = None while True: result = youtube_client.request('GET', 'playlistItems', playlistId=playlist, part='snippet', **({ 'pageToken': token } if token is not None else {})) items += result['items'] if 'nextPageToken' not in result: break token = result['nextPageToken'] logging.info("Found {} videos in playlist".format(len(items))) items = [item['snippet']['resourceId']['videoId'] for item in items] try: os.makedirs(path) except OSError as e: if e.errno != errno.EEXIST: raise exists = os.listdir(path) to_download = [] for id in items: # ignore ones that already exist matches = [name for name in exists if "-{}.".format(id) in name] if matches: logging.debug("Ignoring video {}: already exists as {}".format( id, matches)) continue to_download.append(id) if not to_download: return [] logging.info("Downloading videos: {}".format(to_download)) # In order to get a list of downloaded files, we resort to a hack: # we download to a temp dir first, then rename. # This also protects us from partial downloads. tempdir = tempfile.mkdtemp(prefix='youtube-dl-channel-bot-', suffix='.tmp.d', dir=path) try: output_template = '{}/{}'.format(tempdir, filename_template) # Unfortunately, youtube-dl will exit 1 if there are any copyright-blocked videos, # even with --ignore-errors. We allow 1 as a success exit code. cmd( ['youtube-dl', '--ignore-errors'] + list(youtube_dl_args) + ['-o', output_template, '--'] + to_download, stdout=sys.stdout, success=[0, 1], ) ret = [] for name in os.listdir(tempdir): new_path = os.path.join(path, name) logging.info("Saving new file {!r}".format(new_path)) os.rename(os.path.join(tempdir, name), new_path) ret.append(new_path) return ret finally: # attempt to clean up tempdir as much as we can for name in os.listdir(tempdir): try: os.remove(os.path.join(tempdir, name)) except EnvironmentError: pass try: os.rmdir(tempdir) except EnvironmentError: pass
def youtube_dl(link, filebase): cmd(['youtube-dl', link, '-o', '{}.%(ext)s'.format(filebase)]) filename, = glob('{}.*'.format(filebase)) return filename
def upload(source, name): _, ext = os.path.splitext(source) name = '{}.{}'.format(name, ext.lstrip('.')) cmd(['scp', source, 'tyranicmoron:public_html/rdp/{}'.format(name)]) return 'http://tyranicmoron.uk/~ekimekim/rdp/{}'.format(name)
def install_package(self, package): cmd(['pacman', '-Sy', '--noconfirm', package])
def index_packages(cls): data = cmd(['pacman', '-Ql']) for line in data.strip().split('\n'): package, filepath = line.split(' ', 1) cls.set_package(filepath, package)
def git(self, target_dir, subcmd, *args): self.logger.debug("Running git {} in {} with args {}".format(subcmd, target_dir, args)) return cmd(['git', '-C', target_dir, subcmd] + list(args))
def check_package(self, package): cmd(['pacman', '-Qq', package])
def cmd(*args, **kwargs): print 'running', ' '.join(map(str, args)) return easycmd.cmd(args, **kwargs)
def process_file(top_level_dir, include_dir, tests_dir, extra_link_dirs, objs_dir, filename): name, _ = os.path.splitext(filename) filepath = os.path.join(tests_dir, filename) config = dict(Memory=Memory, Test=Test, random=random.Random(name)) execfile(filepath, config) # loads config as defined globals if 'file' not in config: raise ValueError("You must specify a target file, or None") include_file = config['file'] link_files = config.get('files') target = config.get('target') extra_asm = config.get('asm', '') mems = { label: value.contents for label, value in config.items() if isinstance(value, Memory) and not label.startswith('_') } tests = { testname: test for testname, test in config.items() if isinstance(test, Test) } if target is None and any(test.target is None for test in tests.values()): raise ValueError( "You must specify a target function, either at top-level or for every test case" ) if include_file is None: include_asm = '' else: include_path = os.path.join(top_level_dir, '{}.asm'.format(include_file)) with open(include_path) as f: include_asm = f.read() if link_files is None: asm_files = os.listdir(top_level_dir) for extra_link_dir in extra_link_dirs: asm_files += [ os.path.join(extra_link_dir, filename) for filename in os.listdir(extra_link_dir) ] link_files = [ os.path.splitext(asm_file)[0] for asm_file in asm_files if asm_file.endswith('.asm') and asm_file != 'header.asm' ] if include_file in link_files: link_files.remove(include_file) link_files = [ os.path.join(objs_dir, link_file) for link_file in link_files ] link_paths = [ os.path.join(top_level_dir, '{}.o'.format(link_file)) for link_file in link_files ] gendir = os.path.join(tests_dir, name) if not os.path.exists(gendir): os.mkdir(gendir) for i, (testname, test) in enumerate( sorted(tests.items(), key=lambda (n, t): t.order)): testname = '{i:0{w}d}_{t}'.format(i=i, t=testname, w=len(str(len(tests) - 1))) asm = test.gen_asm(include_asm, target, extra_asm, mems) path = os.path.join(gendir, testname) asm_path = '{}.asm'.format(path) obj_path = '{}.o'.format(path) sym_path = '{}.sym'.format(path) rom_path = '{}.gb'.format(path) with open(asm_path, 'w') as f: f.write(asm) # We pad wth 0x40 = ld b, b = BGB breakpoint cmd([ 'rgbasm', '-DDEBUG', '-i', include_dir, '-v', '-o', obj_path, asm_path ]) cmd([ 'rgblink', '-n', sym_path, '-o', rom_path, '-p', '0x40', obj_path ] + link_paths) cmd(['rgbfix', '-v', '-p', '0x40', rom_path])