def process(self, iter_input):
input_file = output_file = None
stdin = iter_input
format_kwargs = {}
# Create temporary files for input and output if required
if '{infile}' in self.command:
stdin = None
if hasattr(iter_input, 'file_path'):
format_kwargs['infile'] = iter_input.file_path
else:
input_file = NamedTemporaryFile()
format_kwargs['infile'] = input_file.name
for chunk in iter_input:
input_file.write(chunk)
input_file.seek(0) # Not sure why this is needed
if '{outfile}' in self.command:
output_file = NamedTemporaryFile()
format_kwargs['outfile'] = output_file.name
command = self.command.format(**format_kwargs)
g = run_process(command, stdin=stdin, to_close=input_file)
if output_file:
# Consume the iterator into a zero length deque
collections.deque(g, maxlen=0)
return FileChunkGenerator(output_file)
else:
return g
def lint_file(bundle_type, file_path, iter_input=None):
command = bundles_settings.BUNDLES_LINTING[bundle_type]['command']
input_file = None
stdin = None
if '{infile}' in command:
if iter_input:
if hasattr(iter_input, 'file_path'):
filename = iter_input.file_path
else:
input_file = NamedTemporaryFile()
for chunk in iter_input:
input_file.write(chunk)
input_file.flush()
filename = input_file.name
else:
filename = file_path
command = command.format(infile=filename)
else:
if iter_input:
stdin = iter_input
else:
stdin = input_file = open(file_path, 'rb')
try:
# Consume the iterator into a zero length deque
collections.deque(run_process(command, stdin=stdin, to_close=input_file), maxlen=0)
except CalledProcessError as e:
return False, e.output
return True, ''
def process(self, iter_input):
input_file = output_file = None
stdin = iter_input
format_kwargs = {}
# Create temporary files for input and output if required
if '{infile}' in self.command:
stdin = None
if hasattr(iter_input, 'file_path'):
format_kwargs['infile'] = iter_input.file_path
else:
input_file = NamedTemporaryFile()
format_kwargs['infile'] = input_file.name
for chunk in iter_input:
input_file.write(chunk)
input_file.seek(0) # Not sure why this is needed
if '{outfile}' in self.command:
output_file = NamedTemporaryFile()
format_kwargs['outfile'] = output_file.name
command = self.command.format(**format_kwargs)
g = run_process(command, stdin=stdin, to_close=input_file)
if output_file:
# Consume the iterator into a zero length deque
collections.deque(g, maxlen=0)
return FileChunkGenerator(output_file)
else:
return g
def lint_file(self, full_path, bundle):
try:
iter_input = processor_pipeline(bundle[full_path].processors, FileChunkGenerator(open(full_path, 'rb')))
command = bundles_settings.BUNDLES_LINTING[bundle.bundle_type]['command']
input_file = None
stdin = None
if '{infile}' in command:
if hasattr(iter_input, 'file_path'):
filename = iter_input.file_path
else:
input_file = NamedTemporaryFile()
for chunk in iter_input:
input_file.write(chunk)
input_file.flush()
filename = input_file.name
command = command.format(infile=filename)
else:
stdin = iter_input
# Consume the iterator into a zero length deque
collections.deque(run_process(command, stdin=stdin, to_close=input_file), maxlen=0)
self.log_watch_result(full_path, True)
except CalledProcessError as e:
self.log_watch_result(full_path, False, error_message=e.output)
def test_to_close(self):
with open(__file__, 'r') as f:
collections.deque(run_process('cat',
stdin='TEST',
iterate_stdin=False,
to_close=f),
maxlen=0)
self.assertTrue(f.closed)
def test_piped_command(self):
stdin = "TEST1\nTEST2\nTEST1"
output = ''.join(
run_process('cat | grep "TEST1"', stdin=stdin,
iterate_stdin=False))
self.assertEqual(output, "TEST1\nTEST1\n")
def make_uglify_bundle(bundle, debug=False, fixed_version=None):
m = md5()
infile_list = []
source_map_processed_input_files = []
try:
for bundle_file in bundle.files:
if bundle_file.processors:
# for now preprocessed files are written to temp files and therefore won't be available in the source map
output_pipeline = processor_pipeline(bundle_file.processors, FileChunkGenerator(open(bundle_file.file_path, 'rb')), debug=debug)
tmp_input_file = NamedTemporaryFile()
source_map_processed_input_files.append(tmp_input_file)
for chunk in output_pipeline:
m.update(chunk)
tmp_input_file.write(chunk)
tmp_input_file.seek(0)
infile_list.append(tmp_input_file.name)
else:
for chunk in FileChunkGenerator(open(bundle_file.file_path, 'rb')):
m.update(chunk)
infile_list.append(bundle_file.file_path)
hash_version = fixed_version or m.hexdigest()
debug_part = '.debug' if debug else ''
output_file_name = '%s%s.%s.%s' % (os.path.join(bundle.bundle_file_root, bundle.bundle_filename), debug_part, hash_version, bundle.bundle_type)
if bundle.source_map_file_root and bundle.source_map_url_root:
source_map_file_name = '%s%s.%s.%s.map' % (os.path.join(bundle.source_map_file_root, bundle.bundle_filename), debug_part, hash_version, bundle.bundle_type)
source_map_url = '%s.%s.%s.map' % (os.path.join(bundle.source_map_url_root, bundle.bundle_filename), hash_version, bundle.bundle_type)
else:
source_map_file_name = output_file_name + '.map'
source_map_url = bundle.get_url(version=hash_version) + '.map'
source_map_options = [
'--source-map %s' % source_map_file_name,
'--source-map-root %s' % bundle.source_map_files_url_root,
'--source-map-url %s' % source_map_url,
'-p %s' % os.path.realpath(bundle.files_root).count('/'),
'-o %s' % output_file_name,
]
# Consume the iterator into a zero length deque
collections.deque(run_process(bundle.uglify_command.format(infile_list=' '.join(infile_list), source_map_options=' '.join(source_map_options))), maxlen=0)
finally:
for tmp_input_file in source_map_processed_input_files:
tmp_input_file.close()
return hash_version
def lint_file(self, bundle_type, file_path, iter_input=None):
command = bundles_settings.BUNDLES_LINTING[bundle_type]['command']
input_file = None
stdin = None
if '{infile}' in command:
if iter_input:
if hasattr(iter_input, 'file_path'):
filename = iter_input.file_path
else:
input_file = NamedTemporaryFile()
for chunk in iter_input:
input_file.write(chunk)
input_file.flush()
filename = input_file.name
else:
filename = file_path
command = command.format(infile=filename)
else:
if iter_input:
stdin = iter_input
else:
stdin = input_file = open(file_path, 'rb')
try:
# Consume the iterator into a zero length deque
collections.deque(run_process(command,
stdin=stdin,
to_close=input_file),
maxlen=0)
except CalledProcessError as e:
return False, e.output
return True, ''
def test_command_runs(self):
stdin = 'TEST'
output = ''.join(run_process('cat', stdin=stdin, iterate_stdin=False))
self.assertEqual(stdin, output)
def test_stdin_iter(self):
stdin_iter = ("TEST%s" % x for x in xrange(0, 3))
output = ''.join(run_process('cat', stdin=stdin_iter))
self.assertEqual(output, "TEST0TEST1TEST2")
def test_to_close(self):
with open(__file__, "r") as f:
collections.deque(run_process("cat", stdin="TEST", iterate_stdin=False, to_close=f), maxlen=0)
self.assertTrue(f.closed)
def test_stdin_iter(self):
stdin_iter = ("TEST%s" % x for x in xrange(0, 3))
output = "".join(run_process("cat", stdin=stdin_iter))
self.assertEqual(output, "TEST0TEST1TEST2")
def test_piped_command(self):
stdin = "TEST1\nTEST2\nTEST1"
output = "".join(run_process('cat | grep "TEST1"', stdin=stdin, iterate_stdin=False))
self.assertEqual(output, "TEST1\nTEST1\n")
def test_command_runs(self):
stdin = "TEST"
output = "".join(run_process("cat", stdin=stdin, iterate_stdin=False))
self.assertEqual(stdin, output)
