def process(self, iter_input): input_file = output_file = None stdin = iter_input format_kwargs = {} # Create temporary files for input and output if required if '{infile}' in self.command: stdin = None if hasattr(iter_input, 'file_path'): format_kwargs['infile'] = iter_input.file_path else: input_file = NamedTemporaryFile() format_kwargs['infile'] = input_file.name for chunk in iter_input: input_file.write(chunk) input_file.seek(0) # Not sure why this is needed if '{outfile}' in self.command: output_file = NamedTemporaryFile() format_kwargs['outfile'] = output_file.name command = self.command.format(**format_kwargs) g = run_process(command, stdin=stdin, to_close=input_file) if output_file: # Consume the iterator into a zero length deque collections.deque(g, maxlen=0) return FileChunkGenerator(output_file) else: return g
def lint_file(bundle_type, file_path, iter_input=None): command = bundles_settings.BUNDLES_LINTING[bundle_type]['command'] input_file = None stdin = None if '{infile}' in command: if iter_input: if hasattr(iter_input, 'file_path'): filename = iter_input.file_path else: input_file = NamedTemporaryFile() for chunk in iter_input: input_file.write(chunk) input_file.flush() filename = input_file.name else: filename = file_path command = command.format(infile=filename) else: if iter_input: stdin = iter_input else: stdin = input_file = open(file_path, 'rb') try: # Consume the iterator into a zero length deque collections.deque(run_process(command, stdin=stdin, to_close=input_file), maxlen=0) except CalledProcessError as e: return False, e.output return True, ''
def lint_file(self, full_path, bundle): try: iter_input = processor_pipeline(bundle[full_path].processors, FileChunkGenerator(open(full_path, 'rb'))) command = bundles_settings.BUNDLES_LINTING[bundle.bundle_type]['command'] input_file = None stdin = None if '{infile}' in command: if hasattr(iter_input, 'file_path'): filename = iter_input.file_path else: input_file = NamedTemporaryFile() for chunk in iter_input: input_file.write(chunk) input_file.flush() filename = input_file.name command = command.format(infile=filename) else: stdin = iter_input # Consume the iterator into a zero length deque collections.deque(run_process(command, stdin=stdin, to_close=input_file), maxlen=0) self.log_watch_result(full_path, True) except CalledProcessError as e: self.log_watch_result(full_path, False, error_message=e.output)
def test_to_close(self): with open(__file__, 'r') as f: collections.deque(run_process('cat', stdin='TEST', iterate_stdin=False, to_close=f), maxlen=0) self.assertTrue(f.closed)
def test_piped_command(self): stdin = "TEST1\nTEST2\nTEST1" output = ''.join( run_process('cat | grep "TEST1"', stdin=stdin, iterate_stdin=False)) self.assertEqual(output, "TEST1\nTEST1\n")
def make_uglify_bundle(bundle, debug=False, fixed_version=None): m = md5() infile_list = [] source_map_processed_input_files = [] try: for bundle_file in bundle.files: if bundle_file.processors: # for now preprocessed files are written to temp files and therefore won't be available in the source map output_pipeline = processor_pipeline(bundle_file.processors, FileChunkGenerator(open(bundle_file.file_path, 'rb')), debug=debug) tmp_input_file = NamedTemporaryFile() source_map_processed_input_files.append(tmp_input_file) for chunk in output_pipeline: m.update(chunk) tmp_input_file.write(chunk) tmp_input_file.seek(0) infile_list.append(tmp_input_file.name) else: for chunk in FileChunkGenerator(open(bundle_file.file_path, 'rb')): m.update(chunk) infile_list.append(bundle_file.file_path) hash_version = fixed_version or m.hexdigest() debug_part = '.debug' if debug else '' output_file_name = '%s%s.%s.%s' % (os.path.join(bundle.bundle_file_root, bundle.bundle_filename), debug_part, hash_version, bundle.bundle_type) if bundle.source_map_file_root and bundle.source_map_url_root: source_map_file_name = '%s%s.%s.%s.map' % (os.path.join(bundle.source_map_file_root, bundle.bundle_filename), debug_part, hash_version, bundle.bundle_type) source_map_url = '%s.%s.%s.map' % (os.path.join(bundle.source_map_url_root, bundle.bundle_filename), hash_version, bundle.bundle_type) else: source_map_file_name = output_file_name + '.map' source_map_url = bundle.get_url(version=hash_version) + '.map' source_map_options = [ '--source-map %s' % source_map_file_name, '--source-map-root %s' % bundle.source_map_files_url_root, '--source-map-url %s' % source_map_url, '-p %s' % os.path.realpath(bundle.files_root).count('/'), '-o %s' % output_file_name, ] # Consume the iterator into a zero length deque collections.deque(run_process(bundle.uglify_command.format(infile_list=' '.join(infile_list), source_map_options=' '.join(source_map_options))), maxlen=0) finally: for tmp_input_file in source_map_processed_input_files: tmp_input_file.close() return hash_version
def lint_file(self, bundle_type, file_path, iter_input=None): command = bundles_settings.BUNDLES_LINTING[bundle_type]['command'] input_file = None stdin = None if '{infile}' in command: if iter_input: if hasattr(iter_input, 'file_path'): filename = iter_input.file_path else: input_file = NamedTemporaryFile() for chunk in iter_input: input_file.write(chunk) input_file.flush() filename = input_file.name else: filename = file_path command = command.format(infile=filename) else: if iter_input: stdin = iter_input else: stdin = input_file = open(file_path, 'rb') try: # Consume the iterator into a zero length deque collections.deque(run_process(command, stdin=stdin, to_close=input_file), maxlen=0) except CalledProcessError as e: return False, e.output return True, ''
def test_command_runs(self): stdin = 'TEST' output = ''.join(run_process('cat', stdin=stdin, iterate_stdin=False)) self.assertEqual(stdin, output)
def test_stdin_iter(self): stdin_iter = ("TEST%s" % x for x in xrange(0, 3)) output = ''.join(run_process('cat', stdin=stdin_iter)) self.assertEqual(output, "TEST0TEST1TEST2")
def test_to_close(self): with open(__file__, "r") as f: collections.deque(run_process("cat", stdin="TEST", iterate_stdin=False, to_close=f), maxlen=0) self.assertTrue(f.closed)
def test_stdin_iter(self): stdin_iter = ("TEST%s" % x for x in xrange(0, 3)) output = "".join(run_process("cat", stdin=stdin_iter)) self.assertEqual(output, "TEST0TEST1TEST2")
def test_piped_command(self): stdin = "TEST1\nTEST2\nTEST1" output = "".join(run_process('cat | grep "TEST1"', stdin=stdin, iterate_stdin=False)) self.assertEqual(output, "TEST1\nTEST1\n")
def test_command_runs(self): stdin = "TEST" output = "".join(run_process("cat", stdin=stdin, iterate_stdin=False)) self.assertEqual(stdin, output)