def wav_wav(orig, dest, **_kwargs):
"""Combine the fnames into output"""
# options = kwargs.get("tree").cmd_options.get("options", [])
# first demux it to 16 bit 48khz
dest_list = []
for index, orig_elem in enumerate(tools.get_iter(orig)):
tmp_dest = os.path.join(
os.path.dirname(dest), "{0}_{1}".format(index,
os.path.basename(dest)))
cmd = "ffmpeg -i {orig} -acodec pcm_s16le -ar 48000 {dest}".format(
dest=tmp_dest, orig=orig_elem)
logger.debug(cmd)
try:
subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as error:
logger.error(error)
logger.error(tools.to_unicode(error.output))
continue
dest_list.append(tmp_dest)
if len(dest_list) > 1:
cmd = "sox {orig} {dest}".format(orig=" ".join(orig), dest=dest)
logger.debug(cmd)
try:
subprocess.check_call(cmd, shell=True)
except subprocess.CalledProcessError as error:
logger.error(error)
logger.error(tools.to_unicode(error.output))
else:
os.rename(dest_list[0], dest)
return dest
def get_streams(orig):
"""Get audio stream type"""
streams = {}
cmd = "ffprobe '{0}'".format(orig)
cache = tools.Cache()
try:
output = subprocess.check_output(
cmd, shell=True,
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as error:
logger.error(error)
logger.error(tools.to_unicode(error.output))
return None
output = tools.to_unicode(output)
for line in output.split("\n"):
if cache(re.search(r".*Stream #0.(\d+).*: (.*?): (.+?)[, ].*", line)):
stream, stype, encoding = cache.output.groups()
if stype != "Audio":
continue
if encoding not in ENCODINGS:
logger.error("Unknown encoding: {0}".format(encoding))
else:
streams[int(stream)] = encoding
return streams
def latex(orig, dest_tex, latex_cmd, settings, run_bibtex=True):
"""Run (and rerun) latex command"""
cmd = ("{latex} -halt-on-error -interaction=nonstopmode" +
" -output-directory={dirname} {tex}").format(
latex=latex_cmd,
tex=dest_tex,
dirname=os.path.dirname(dest_tex))
# -aux-directory=/tmp/thesis
try:
output = subprocess.check_output(cmd,
cwd=os.path.dirname(
os.path.realpath(orig)),
shell=True)
except subprocess.CalledProcessError as error:
output = error.output
output = tools.to_unicode(output)
messages, extra, errors = debug_output(output, settings, dest_tex)
if "fname" in extra and not errors:
if run_bibtex and "bibtex" in extra:
bibtex(extra["bibtex"], orig, dest_tex)
return latex(orig, dest_tex, latex_cmd, settings, run_bibtex=False)
if "rerun" in extra:
return latex(orig, dest_tex, latex_cmd, settings, run_bibtex)
for fname, page, debug, message in messages:
if errors and debug != "error":
continue
getattr(logger, debug)("{fname}, page={page}\n{msg}\n".format(
fname=fname,
page=page,
msg=re.sub("\n\n\n+", "\n\n", message).strip()))
return (extra["fname"] if not errors and "fname" in extra else None)
def video_audio(orig, dest, *_args, **_kwargs):
"""Extract audio from video file"""
streams = get_streams(orig)
if dest is None:
return [(str(stream), None, {ENCODINGS[encoding][0]})
for stream, encoding in streams.items()]
dirname = os.path.dirname(dest)
stream = int(os.path.basename(dest))
_mimetype, ext = ENCODINGS[streams[stream]]
newname = os.path.join(
dirname,
os.path.splitext(os.path.basename(orig))[0] + ext)
tools.create_dir(newname, remove=True)
cmd = ("ffmpeg -i '{orig}' -vcodec copy -acodec copy -map 0:{stream}" +
" '{newname}'").format(
orig=orig, newname=newname, stream=stream)
# mplayer -dumpaudio -dumpfile $output $input
# yes n | avidemux2_cli $input --save-raw-audio $output
logger.debug(cmd)
try:
subprocess.check_output(
cmd, shell=True,
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as error:
logger.error(error)
logger.error(tools.to_unicode(error.output))
return None
# dest = os.path.join(
# dirname,
# os.path.splitext(os.path.basename(orig))[0] +
# mime.get_extension({encoding}))
# if newname != dest:
# os.rename(newname, dest)
return newname
def zpsql_psql(orig, dest, **_kwargs):
"""Convert condensed psql to txt psql"""
try:
subprocess.check_call(
"pg_restore {0} > {1}".format(orig, dest),
shell=True)
except subprocess.CalledProcessError as error:
logger.error(error)
logger.error(tools.to_unicode(error.output))
return None
return dest
def pdf_pdf(orig, dest, **kwargs):
"""Combine the fnames into output"""
if isinstance(orig, list):
pdfmarks = os.path.join(os.path.dirname(dest), "pdfmarks")
with io.open(pdfmarks, "w") as fobj:
fobj.write("""[ /Title (Merged)
/DOCINFO pdfmark
[/_objdef {pl} /type /dict /OBJ pdfmark
[{pl} <</Nums [\n""")
page = 1
for orig_elem in tools.get_iter(orig):
pdfobj = PyPDF2.PdfFileReader(orig_elem)
filebase = os.path.splitext(os.path.basename(orig_elem))[0]
for subpage in range(pdfobj.getNumPages()):
fobj.write("{page:d} << /P ({name} {subpage}) >>\n".format(
page=page,
name=filebase,
subpage=(" [{0}]".format(subpage +
1) if subpage > 0 else "")))
page += 1
fobj.write("{0:d} << >>\n".format(page))
fobj.write("""]>> /PUT pdfmark
[{Catalog} <</PageLabels {pl}>> /PUT pdfmark\n""")
else:
pdfmarks = ""
options = kwargs.get("tree").cmd_options["options"]
# -dDOPDFMARKS
import shlex
cmd = (
"gs -q -dNOPAUSE -dBATCH -sDEVICE=pdfwrite {bw} {size}" +
" -sOUTPUTFILE={output} {orig} {pdfmarks}").format(
output=shlex.quote(dest),
orig=" ".join(
[shlex.quote(fname) for fname in tools.get_iter(orig)]),
pdfmarks=pdfmarks,
bw=("-sColorConversionStrategy=Gray" +
" -dProcessColorModel=/DeviceGray" if "bw" in options else ""),
size=(
"-dPDFSETTINGS=/screen" if "q=screen" in options else
"-dPDFSETTINGS=/ebook" if "q=ebook" in options else
"-dPDFSETTINGS=/prepress" if "q=prepress" in options else ""))
logger.info(cmd)
try:
subprocess.check_call(cmd, shell=True)
except subprocess.CalledProcessError as error:
logger.error(error)
logger.error(tools.to_unicode(error.output))
return None
return dest
def ly_pdf(orig, dest, *_args, **_kwargs):
"""Convert lilypond to pdf"""
cmd = "lilypond --ps --output={destbase} {orig}".format(
orig=orig,
destbase=os.path.splitext(dest)[0])
try:
logger.debug("Running: {0}".format(cmd))
output = subprocess.check_output(
cmd, stderr=subprocess.STDOUT, shell=True)
output = tools.to_unicode(output)
except subprocess.CalledProcessError as error:
logger.error(error)
logger.error(tools.to_unicode(error.output))
return None
cache = tools.Cache()
if cache(re.search("Layout output to `(.*)'", output, re.MULTILINE)):
return os.path.join(os.path.dirname(dest), cache.output.group(1))
else:
logger.error("ERROR: {0}".format(output))
return None
def unzip_fast(orig, dest, *_args, **kwargs):
"""Unzip immediately"""
overwrite = kwargs.get("tree").cmd_options.get("overwrite", False)
cmd = "unzip {options} {orig} -d {dest}".format(
orig=orig,
dest=dest,
options="-o" if overwrite else "")
try:
logger.debug("Call: {0}".format(cmd))
subprocess.check_call(cmd, shell=True, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as error:
logger.error(error)
logger.error(tools.to_unicode(error.output))
return False
return True
def ftype_mp4(fname, file_options):
"""File type for mp4 files
is it video or audio"""
cache = tools.Cache()
cmd = "ffprobe {0}".format(pipes.quote(fname))
try:
output = subprocess.check_output(cmd,
shell=True,
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as error:
logger.error(error)
logger.error(tools.to_unicode(error))
return
output = tools.to_unicode(output)
streams = set()
video = False
for line in output.split("\n"):
if cache(re.search(r".*Stream #0.\d+.*: (.*?): ([a-zA-Z0-9]*).*",
line)):
stype, encoding = cache.output.groups()
if stype == "Video":
video = True
break
if encoding not in ENCODINGS:
logger.error("Unknown encoding: {0}".format(encoding))
streams.add(encoding)
else:
streams.add(ENCODINGS[encoding])
if video and not streams:
return {file_options["mime_type"]}
else:
if video:
print("TODO: has video, but also audio")
return streams
def bibtex(fnames, orig, dest_tex):
"""Run bibtex over files with missing references"""
for fname in fnames:
auxname = os.path.join(os.path.dirname(dest_tex),
os.path.splitext(fname)[0] + ".aux")
auxdir = os.path.dirname(fname)
cmd = "openout_any=a TEXMFOUTPUT={0} bibtex {1}".format(
os.path.normpath(os.path.join(os.path.dirname(dest_tex), auxdir)),
auxname)
try:
output = subprocess.check_output(cmd,
cwd=os.path.dirname(orig),
shell=True)
except subprocess.CalledProcessError as error:
output = error.output
output = tools.to_unicode(output)
if "error messages" in output:
print(output)
