def mcdplib_run_make(mcdplib):
makefile = os.path.join(mcdplib, 'Makefile')
assert os.path.exists(makefile)
cwd = mcdplib
cmd = [
'make',
'clean',
'all',
]
# do not use too many resources
circle = 'CIRCLECI' in os.environ
parallel = not circle
if parallel:
cmd.append('-j')
from system_cmd.meat import system_cmd_result
logger.debug('$ cd %s' % cwd)
env = os.environ.copy()
if all_disabled():
env['DISABLE_CONTRACTS'] = '1'
msg = ('Disabling contracts in environment by adding '
'DISABLE_CONTRACTS=%r.' % env['DISABLE_CONTRACTS'])
logger.debug(msg)
system_cmd_result(cwd,
cmd,
display_stdout=True,
display_stderr=True,
raise_on_error=True,
env=env)
def png_from_pdf(pdf_data, density):
""" Converts a pdf to png with the given density """
d = mkdtemp()
try:
tmpfile = os.path.join(d, 'file.pdf')
with open(tmpfile, 'wb') as f:
f.write(pdf_data)
out = os.path.join(d, 'file.png')
cmd = [
'convert',
'-density', str(density),
tmpfile,
'-background', 'white',
'-alpha','remove',
'-alpha','off',
]
shave = True
if shave:
# warnings.warn('Using shave to fix some bug in imagemagic')
cmd += ['-shave', '1']
cmd += ['-strip']
cmd += [out]
try:
res = system_cmd_result(cwd='.', cmd=cmd,
display_stdout=False,
display_stderr=False,
raise_on_error=True)
if not os.path.exists(out):
msg = "ImageMagick did not fail, but it didn't write the image it promised."
msg += "\n"+indent(" ".join(cmd), " invocation: ")
msg += "\n"+ indent(res.stdout or "(no output)", '|', 'stdout: |')
msg += "\n"+ indent(res.stderr or "(no output)", '|', 'stderr: |')
where = 'problematic.pdf'
msg += "\n I will copy the problematic pdf file to %s" % where
shutil.copy(tmpfile, where)
raise CmdException(msg)
except CmdException as e:
msg = 'I was not able to use Imagemagick to convert an image.'
try:
version = system_cmd_result(cwd='.', cmd=['convert', '--version'],
display_stdout=False,
display_stderr=False,
raise_on_error=True)
msg += '\n ImageMagick "convert" version:'
msg += '\n' + indent(version.stdout, ' | ')
except:
pass
raise_wrapped(ConversionError, e, msg, compact=True)
r = open(out,'rb').read()
return r
finally:
shutil.rmtree(d)
def update_graph(context, event):
print('event: %s' % event)
if event.name in ['manager-job-starting']:
job_id = event.kwargs['job_id']
Global.processing.add(job_id)
if event.name in ['manager-job-failed', 'manager-job-succeeded']:
job_id = event.kwargs['job_id']
Global.processing.remove(job_id)
print('global processing %s' % Global.processing)
if 'job_id' in event.kwargs:
what = '%s-%s' % (event.name, event.kwargs['job_id'])
else:
what = event.name
filename = os.path.join(Global.dirname,
('step-%04d-%s' % (Global.step, what)))
make_sure_dir_exists(filename)
# print('step %d: jobs = %s' % (Global.step, Global.job_list))
graph(job_list=list(Global.job_list),
context=context,
filename=filename,
processing=Global.processing,
**Global.graph_params)
Global.step += 1
# see here:
# http://stackoverflow.com/questions/14784405/how-to-set-the-output-size-in-graphviz-for-the-dot-format
png = filename + ".png"
png2 = filename + "-x.png"
size = Global.size
dpi = Global.dpi
cmd0 = [
'dot', '-Tpng',
'-Gsize=%s,%s\!' % (size[0] / dpi, size[1] / dpi),
'-Gdpi=%s' % dpi, '-o' + png, filename
]
system_cmd_result('.',
cmd0,
display_stdout=True,
display_stderr=True,
raise_on_error=True)
cmd = [
'convert', png, '-gravity', 'center', '-background', 'white',
'-extent',
'%sx%s' % (size[0], size[1]), png2
]
system_cmd_result('.',
cmd,
display_stdout=True,
display_stderr=True,
raise_on_error=True)
os.unlink(png)
def pdf2svg(f1, f2):
make_sure_dir_exists(f2)
cwd = '.'
cmd = ['pdf2svg', f1, f2]
_res = system_cmd_result(cwd, cmd,
display_stdout=False,
display_stderr=False,
raise_on_error=True)
def extract_page(filename, page, output):
make_sure_dir_exists(output)
cwd = '.'
cmd = ['pdftk', filename, 'cat', str(page), 'output', output]
_res = system_cmd_result(cwd, cmd,
display_stdout=False,
display_stderr=False,
raise_on_error=True)
def resize_icon(filename, tmppath, size):
check_isinstance(filename, str)
check_isinstance(tmppath, str)
res = os.path.join(tmppath, 'resized', str(size))
safe_makedirs(res)
resized = os.path.join(res, os.path.basename(filename))
if not os.path.exists(resized):
cmd = ['convert', filename, '-resize', '%s' % size, resized]
try:
system_cmd_result(cwd='.', cmd=cmd,
display_stdout=False,
display_stderr=False,
raise_on_error=True)
except CmdException:
raise
return resized
def mcdplib_run_make(mcdplib):
makefile = os.path.join(mcdplib, 'Makefile')
assert os.path.exists(makefile)
cwd = mcdplib
cmd = ['make', 'clean', 'all']
from system_cmd.meat import system_cmd_result
logger.debug('$ cd %s' % cwd)
env = os.environ.copy()
if all_disabled():
env['DISABLE_CONTRACTS'] = '1'
msg = ('Disabling contracts in environment by adding '
'DISABLE_CONTRACTS=%r.' % env['DISABLE_CONTRACTS'])
logger.debug(msg)
system_cmd_result(cwd, cmd,
display_stdout=True,
display_stderr=True,
raise_on_error=True,
env=env)
def pdfcrop_margins(f1, f2, margins):
make_sure_dir_exists(f2)
cwd = '.'
cmd = ['pdfjam', '--keepinfo', '--trim', margins,
'--paper', 'letter', '--landscape',
'--clip', 'true', '--outfile', f2, f1]
# cmd = ['pdfcrop', f1, '--margins', str(margins), f2]
_res = system_cmd_result(cwd, cmd,
display_stdout=False,
display_stderr=False,
raise_on_error=True)
def png_from_pdf(pdf_data, density):
""" Converts a pdf to png with the given density """
d = mkdtemp()
try:
tmpfile = os.path.join(d, 'file.pdf')
with open(tmpfile, 'wb') as f:
f.write(pdf_data)
out = os.path.join(d, 'file.png')
cmd = [
'convert',
'-density',
str(density),
tmpfile,
'-background',
'white',
'-alpha',
'remove',
'-alpha',
'off',
]
shave = True
if shave:
warnings.warn('Using shave to fix some bug in imagemagic')
cmd += ['-shave', '1']
cmd += [out]
try:
system_cmd_result(cwd='.',
cmd=cmd,
display_stdout=False,
display_stderr=False,
raise_on_error=True)
except CmdException:
raise
r = open(out, 'rb').read()
return r
finally:
shutil.rmtree(d)
def system_output(cwd, cmd):
return system_cmd_result(cwd, cmd).stdout
def update_graph(context, event):
print('event: %s' % event)
if event.name in ['manager-job-starting']:
job_id = event.kwargs['job_id']
Global.processing.add(job_id)
if event.name in ['manager-job-failed',
'manager-job-succeeded']:
job_id = event.kwargs['job_id']
Global.processing.remove(job_id)
print('global processing %s' % Global.processing)
if 'job_id' in event.kwargs:
what = '%s-%s' % (event.name, event.kwargs['job_id'])
else:
what = event.name
filename = os.path.join(Global.dirname,
('step-%04d-%s' % (Global.step, what)))
make_sure_dir_exists(filename)
# print('step %d: jobs = %s' % (Global.step, Global.job_list))
graph(job_list=list(Global.job_list),
context=context,
filename=filename,
processing=Global.processing,
**Global.graph_params)
Global.step += 1
# see here:
# http://stackoverflow.com/questions/14784405/how-to-set-the-output-size-in-graphviz-for-the-dot-format
png = filename + ".png"
png2 = filename + "-x.png"
size = Global.size
dpi = Global.dpi
cmd0 = ['dot', '-Tpng',
'-Gsize=%s,%s\!' % (size[0]/dpi, size[1]/dpi),
'-Gdpi=%s' % dpi,
'-o' + png, filename]
system_cmd_result(
'.', cmd0,
display_stdout=True,
display_stderr=True,
raise_on_error=True)
cmd=['convert',
png,
'-gravity',
'center',
'-background',
'white',
'-extent',
'%sx%s' % (size[0], size[1]),
png2]
system_cmd_result(
'.', cmd,
display_stdout=True,
display_stderr=True,
raise_on_error=True)
os.unlink(png)
import os
from conf_tools.utils.locate_files_imp import locate_files
from system_cmd.meat import system_cmd_result
d = 'src'
files = list(locate_files(directory=d, pattern='*.py'))
for f in files:
if 'libraries' in f: continue
f2 = os.path.relpath(f, d)
f2 = f2.replace('/', '.')
mod = f2.replace('.py', '')
cwd = '.'
cmd = ['python', '-c', 'import %s' % mod]
print "python -c 'import %s'" % mod
system_cmd_result(cwd,
cmd,
display_stdout=False,
display_stderr=True,
raise_on_error=False)