def _get_push_info(self, proc, progress): progress = to_progress_instance(progress) # read progress information from stderr # we hope stdout can hold all the data, it should ... # read the lines manually as it will use carriage returns between the messages # to override the previous one. This is why we read the bytes manually progress_handler = progress.new_message_handler() output = IterableList('name') def stdout_handler(line): try: output.append(PushInfo._from_line(self, line)) except ValueError: # If an error happens, additional info is given which we parse below. pass handle_process_output(proc, stdout_handler, progress_handler, finalizer=None, decode_streams=False) stderr_text = progress.error_lines and '\n'.join( progress.error_lines) or '' try: proc.wait(stderr=stderr_text) except Exception: if not output: raise elif stderr_text: log.warning("Error lines received while fetching: %s", stderr_text) return output
def _get_push_info(self, proc, progress): progress = to_progress_instance(progress) # read progress information from stderr # we hope stdout can hold all the data, it should ... # read the lines manually as it will use carriage returns between the messages # to override the previous one. This is why we read the bytes manually progress_handler = progress.new_message_handler() output = IterableList('name') def stdout_handler(line): try: output.append(PushInfo._from_line(self, line)) except ValueError: # if an error happens, additional info is given which we cannot parse pass # END exception handling # END for each line try: handle_process_output(proc, stdout_handler, progress_handler, finalize_process) except Exception: if len(output) == 0: raise return output
def run_commit_hook(name, index): """Run the commit hook of the given name. Silently ignores hooks that do not exist. :param name: name of hook, like 'pre-commit' :param index: IndexFile instance :raises HookExecutionError: """ hp = hook_path(name, index.repo.git_dir) if not os.access(hp, os.X_OK): return env = os.environ.copy() env['GIT_INDEX_FILE'] = safe_decode(index.path) if PY3 else safe_encode(index.path) env['GIT_EDITOR'] = ':' try: cmd = subprocess.Popen(hp, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=index.repo.working_dir, close_fds=is_posix, creationflags=PROC_CREATIONFLAGS,) except Exception as ex: raise HookExecutionError(hp, ex) else: stdout = [] stderr = [] handle_process_output(cmd, stdout.append, stderr.append, finalize_process) stdout = ''.join(stdout) stderr = ''.join(stderr) if cmd.returncode != 0: stdout = force_text(stdout, defenc) stderr = force_text(stderr, defenc) raise HookExecutionError(hp, cmd.returncode, stdout, stderr)
def _index_from_raw_format(cls, repo, proc): """Create a new DiffIndex from the given stream which must be in raw format. :return: git.DiffIndex""" # handles # :100644 100644 687099101... 37c5e30c8... M .gitignore index = DiffIndex() def handle_diff_line(line): line = line.decode(defenc) if not line.startswith(":"): return meta, _, path = line[1:].partition('\t') old_mode, new_mode, a_blob_id, b_blob_id, _change_type = meta.split( None, 4) # Change type can be R100 # R: status letter # 100: score (in case of copy and rename) change_type = _change_type[0] score_str = ''.join(_change_type[1:]) score = int(score_str) if score_str.isdigit() else None path = path.strip() a_path = path.encode(defenc) b_path = path.encode(defenc) deleted_file = False new_file = False rename_from = None rename_to = None # NOTE: We cannot conclude from the existence of a blob to change type # as diffs with the working do not have blobs yet if change_type == 'D': b_blob_id = None deleted_file = True elif change_type == 'A': a_blob_id = None new_file = True elif change_type == 'R': a_path, b_path = path.split('\t', 1) a_path = a_path.encode(defenc) b_path = b_path.encode(defenc) rename_from, rename_to = a_path, b_path elif change_type == 'T': # Nothing to do pass # END add/remove handling diff = Diff(repo, a_path, b_path, a_blob_id, b_blob_id, old_mode, new_mode, new_file, deleted_file, rename_from, rename_to, '', change_type, score) index.append(diff) handle_process_output(proc, handle_diff_line, None, finalize_process, decode_streams=False) return index
def _get_push_info(self, proc, progress): progress = to_progress_instance(progress) # read progress information from stderr # we hope stdout can hold all the data, it should ... # read the lines manually as it will use carriage returns between the messages # to override the previous one. This is why we read the bytes manually progress_handler = progress.new_message_handler() output = IterableList('name') def stdout_handler(line): try: output.append(PushInfo._from_line(self, line)) except ValueError: # If an error happens, additional info is given which we parse below. pass handle_process_output(proc, stdout_handler, progress_handler, finalizer=None, decode_streams=False) stderr_text = progress.error_lines and '\n'.join(progress.error_lines) or '' try: proc.wait(stderr=stderr_text) except Exception: if not output: raise elif stderr_text: log.warning("Error lines received while fetching: %s", stderr_text) return output
def test_handle_process_output(self): from git.cmd import handle_process_output line_count = 5002 count = [None, 0, 0] def counter_stdout(line): count[1] += 1 def counter_stderr(line): count[2] += 1 proc = subprocess.Popen([ sys.executable, fixture_path('cat_file.py'), str(fixture_path('issue-301_stderr')) ], stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False) handle_process_output(proc, counter_stdout, counter_stderr, lambda proc: proc.wait()) assert count[1] == line_count assert count[2] == line_count
def _clone(cls, git, url, path, odb_default_type, progress, **kwargs): # special handling for windows for path at which the clone should be # created. # tilde '~' will be expanded to the HOME no matter where the ~ occours. Hence # we at least give a proper error instead of letting git fail prev_cwd = None prev_path = None odbt = kwargs.pop('odbt', odb_default_type) if os.name == 'nt': if '~' in path: raise OSError("Git cannot handle the ~ character in path %r correctly" % path) # on windows, git will think paths like c: are relative and prepend the # current working dir ( before it fails ). We temporarily adjust the working # dir to make this actually work match = re.match("(\w:[/\\\])(.*)", path) if match: prev_cwd = os.getcwd() prev_path = path drive, rest_of_path = match.groups() os.chdir(drive) path = rest_of_path kwargs['with_keep_cwd'] = True # END cwd preparation # END windows handling try: proc = git.clone(url, path, with_extended_output=True, as_process=True, v=True, **add_progress(kwargs, git, progress)) if progress: handle_process_output(proc, None, progress.new_message_handler(), finalize_process) else: (stdout, stderr) = proc.communicate() finalize_process(proc, stderr=stderr) # end handle progress finally: if prev_cwd is not None: os.chdir(prev_cwd) path = prev_path # END reset previous working dir # END bad windows handling # our git command could have a different working dir than our actual # environment, hence we prepend its working dir if required if not os.path.isabs(path) and git.working_dir: path = join(git._working_dir, path) # adjust remotes - there may be operating systems which use backslashes, # These might be given as initial paths, but when handling the config file # that contains the remote from which we were clones, git stops liking it # as it will escape the backslashes. Hence we undo the escaping just to be # sure repo = cls(os.path.abspath(path), odbt=odbt) if repo.remotes: writer = repo.remotes[0].config_writer writer.set_value('url', repo.remotes[0].url.replace("\\\\", "\\").replace("\\", "/")) # PY3: be sure cleanup is performed and lock is released writer.release() # END handle remote repo return repo
def _clone(cls, git, url, path, odb_default_type, progress, **kwargs): if progress is not None: progress = to_progress_instance(progress) odbt = kwargs.pop('odbt', odb_default_type) # when pathlib.Path or other classbased path is passed if not isinstance(path, str): path = str(path) ## A bug win cygwin's Git, when `--bare` or `--separate-git-dir` # it prepends the cwd or(?) the `url` into the `path, so:: # git clone --bare /cygwin/d/foo.git C:\\Work # becomes:: # git clone --bare /cygwin/d/foo.git /cygwin/d/C:\\Work # clone_path = (Git.polish_url(path) if Git.is_cygwin() and 'bare' in kwargs else path) sep_dir = kwargs.get('separate_git_dir') if sep_dir: kwargs['separate_git_dir'] = Git.polish_url(sep_dir) proc = git.clone(Git.polish_url(url), clone_path, with_extended_output=True, as_process=True, v=True, universal_newlines=True, **add_progress(kwargs, git, progress)) if progress: handle_process_output(proc, None, progress.new_message_handler(), finalize_process, decode_streams=False) else: (stdout, stderr) = proc.communicate() log.debug("Cmd(%s)'s unused stdout: %s", getattr(proc, 'args', ''), stdout) finalize_process(proc, stderr=stderr) # our git command could have a different working dir than our actual # environment, hence we prepend its working dir if required if not osp.isabs(path) and git.working_dir: path = osp.join(git._working_dir, path) repo = cls(path, odbt=odbt) # retain env values that were passed to _clone() repo.git.update_environment(**git.environment()) # adjust remotes - there may be operating systems which use backslashes, # These might be given as initial paths, but when handling the config file # that contains the remote from which we were clones, git stops liking it # as it will escape the backslashes. Hence we undo the escaping just to be # sure if repo.remotes: with repo.remotes[0].config_writer as writer: writer.set_value('url', Git.polish_url(repo.remotes[0].url)) # END handle remote repo return repo
def test_handle_process_output(self): from git.cmd import handle_process_output line_count = 5002 count = [None, 0, 0] def counter_stdout(line): count[1] += 1 def counter_stderr(line): count[2] += 1 cmdline = [sys.executable, fixture_path('cat_file.py'), str(fixture_path('issue-301_stderr'))] proc = subprocess.Popen(cmdline, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False, creationflags=cmd.PROC_CREATIONFLAGS, ) handle_process_output(proc, counter_stdout, counter_stderr, finalize_process) self.assertEqual(count[1], line_count) self.assertEqual(count[2], line_count)
def _index_from_patch_format(cls, repo, proc): """Create a new DiffIndex from the given text which must be in patch format :param repo: is the repository we are operating on - it is required :param stream: result of 'git diff' as a stream (supporting file protocol) :return: git.DiffIndex """ ## FIXME: Here SLURPING raw, need to re-phrase header-regexes linewise. text = [] handle_process_output(proc, text.append, None, finalize_process, decode_streams=False) # for now, we have to bake the stream text = b''.join(text) index = DiffIndex() previous_header = None header = None for _header in cls.re_header.finditer(text): a_path_fallback, b_path_fallback, \ old_mode, new_mode, \ rename_from, rename_to, \ new_file_mode, deleted_file_mode, copied_file_name, \ a_blob_id, b_blob_id, b_mode, \ a_path, b_path = _header.groups() new_file, deleted_file, copied_file = \ bool(new_file_mode), bool(deleted_file_mode), bool(copied_file_name) a_path = cls._pick_best_path(a_path, rename_from, a_path_fallback) b_path = cls._pick_best_path(b_path, rename_to, b_path_fallback) # Our only means to find the actual text is to see what has not been matched by our regex, # and then retro-actively assign it to our index if previous_header is not None: index[-1].diff = text[previous_header.end():_header.start()] # end assign actual diff # Make sure the mode is set if the path is set. Otherwise the resulting blob is invalid # We just use the one mode we should have parsed a_mode = old_mode or deleted_file_mode or ( a_path and (b_mode or new_mode or new_file_mode)) b_mode = b_mode or new_mode or new_file_mode or (b_path and a_mode) index.append( Diff(repo, a_path, b_path, a_blob_id and a_blob_id.decode(defenc), b_blob_id and b_blob_id.decode(defenc), a_mode and a_mode.decode(defenc), b_mode and b_mode.decode(defenc), new_file, deleted_file, copied_file, rename_from, rename_to, None, None, None)) previous_header = _header header = _header # end for each header we parse if index: index[-1].diff = text[header.end():] # end assign last diff return index
def _get_push_info(self, proc, progress): # read progress information from stderr # we hope stdout can hold all the data, it should ... # read the lines manually as it will use carriage returns between the messages # to override the previous one. This is why we read the bytes manually # TODO: poll() on file descriptors to know what to read next, process streams concurrently progress_handler = progress.new_message_handler() output = IterableList('name') def stdout_handler(line): try: output.append(PushInfo._from_line(self, line)) except ValueError: # if an error happens, additional info is given which we cannot parse pass # END exception handling # END for each line try: handle_process_output(proc, stdout_handler, progress_handler, finalize_process) except Exception: if len(output) == 0: raise return output
def _index_from_name_only_format(cls, repo, proc): """Create a new DiffIndex from the given text which must be in name only format :param repo: is the repository we are operating on - it is required :param stream: result of 'git diff' as a stream (supporting file protocol) :return: git.DiffIndex """ index = DiffIndex() def handle_diff_line_name_only(lines): lines = lines.decode(defenc) for line in lines.split('\x00'): path = line.strip() if len(path) == 0: continue if cls.is_first: cls.is_first = False continue a_path = path.encode(defenc) b_path = path.encode(defenc) index.append( Diff(repo, a_path, b_path, None, None, None, None, False, False, None, None, None, '', None, None)) handle_process_output(proc, handle_diff_line_name_only, None, finalize_process, decode_streams=False) return index
def test_handle_process_output(self): from git.cmd import handle_process_output line_count = 5002 count = [None, 0, 0] def counter_stdout(line): count[1] += 1 def counter_stderr(line): count[2] += 1 cmdline = [ sys.executable, fixture_path('cat_file.py'), str(fixture_path('issue-301_stderr')) ] proc = subprocess.Popen( cmdline, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False, creationflags=cmd.PROC_CREATIONFLAGS, ) handle_process_output(proc, counter_stdout, counter_stderr, finalize_process) self.assertEqual(count[1], line_count) self.assertEqual(count[2], line_count)
def run_commit_hook(name, index, *args): """Run the commit hook of the given name. Silently ignores hooks that do not exist. :param name: name of hook, like 'pre-commit' :param index: IndexFile instance :param args: arguments passed to hook file :raises HookExecutionError: """ hp = hook_path(name, index.repo.git_dir) if not os.access(hp, os.X_OK): return env = os.environ.copy() env['GIT_INDEX_FILE'] = safe_decode(index.path) if PY3 else safe_encode(index.path) env['GIT_EDITOR'] = ':' try: cmd = subprocess.Popen([hp] + list(args), env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=index.repo.working_dir, close_fds=is_posix, creationflags=PROC_CREATIONFLAGS,) except Exception as ex: raise HookExecutionError(hp, ex) else: stdout = [] stderr = [] handle_process_output(cmd, stdout.append, stderr.append, finalize_process) stdout = ''.join(stdout) stderr = ''.join(stderr) if cmd.returncode != 0: stdout = force_text(stdout, defenc) stderr = force_text(stderr, defenc) raise HookExecutionError(hp, cmd.returncode, stderr, stdout)
def _get_push_info(self, proc: 'Git.AutoInterrupt', progress: Union[Callable[..., Any], RemoteProgress, None], kill_after_timeout: Union[None, float] = None) -> PushInfoList: progress = to_progress_instance(progress) # read progress information from stderr # we hope stdout can hold all the data, it should ... # read the lines manually as it will use carriage returns between the messages # to override the previous one. This is why we read the bytes manually progress_handler = progress.new_message_handler() output: PushInfoList = PushInfoList() def stdout_handler(line: str) -> None: try: output.append(PushInfo._from_line(self, line)) except ValueError: # If an error happens, additional info is given which we parse below. pass handle_process_output(proc, stdout_handler, progress_handler, finalizer=None, decode_streams=False, kill_after_timeout=kill_after_timeout) stderr_text = progress.error_lines and '\n'.join(progress.error_lines) or '' try: proc.wait(stderr=stderr_text) except Exception as e: # This is different than fetch (which fails if there is any std_err # even if there is an output) if not output: raise elif stderr_text: log.warning("Error lines received while fetching: %s", stderr_text) output.error = e return output
def _get_fetch_info_from_stderr(self, proc, progress): progress = to_progress_instance(progress) # skip first line as it is some remote info we are not interested in output = IterableList('name') # lines which are no progress are fetch info lines # this also waits for the command to finish # Skip some progress lines that don't provide relevant information fetch_info_lines = [] # Basically we want all fetch info lines which appear to be in regular form, and thus have a # command character. Everything else we ignore, cmds = set(FetchInfo._flag_map.keys()) progress_handler = progress.new_message_handler() handle_process_output(proc, None, progress_handler, finalizer=None, decode_streams=False) stderr_text = progress.error_lines and '\n'.join( progress.error_lines) or '' proc.wait(stderr=stderr_text) if stderr_text: log.warning("Error lines received while fetching: %s", stderr_text) for line in progress.other_lines: line = force_text(line) for cmd in cmds: if len(line) > 1 and line[0] == ' ' and line[1] == cmd: fetch_info_lines.append(line) continue # read head information with open(osp.join(self.repo.common_dir, 'FETCH_HEAD'), 'rb') as fp: fetch_head_info = [l.decode(defenc) for l in fp.readlines()] l_fil = len(fetch_info_lines) l_fhi = len(fetch_head_info) if l_fil != l_fhi: msg = "Fetch head lines do not match lines provided via progress information\n" msg += "length of progress lines %i should be equal to lines in FETCH_HEAD file %i\n" msg += "Will ignore extra progress lines or fetch head lines." msg %= (l_fil, l_fhi) log.debug(msg) log.debug("info lines: " + str(fetch_info_lines)) log.debug("head info : " + str(fetch_head_info)) if l_fil < l_fhi: fetch_head_info = fetch_head_info[:l_fil] else: fetch_info_lines = fetch_info_lines[:l_fhi] # end truncate correct list # end sanity check + sanitization output.extend( FetchInfo._from_line(self.repo, err_line, fetch_line) for err_line, fetch_line in zip(fetch_info_lines, fetch_head_info)) return output
def _clone(cls, git, url, path, odb_default_type, progress, **kwargs): # special handling for windows for path at which the clone should be # created. # tilde '~' will be expanded to the HOME no matter where the ~ occours. Hence # we at least give a proper error instead of letting git fail prev_cwd = None prev_path = None odbt = kwargs.pop('odbt', odb_default_type) if os.name == 'nt': if '~' in path: raise OSError("Git cannot handle the ~ character in path %r correctly" % path) # on windows, git will think paths like c: are relative and prepend the # current working dir ( before it fails ). We temporarily adjust the working # dir to make this actually work match = re.match("(\w:[/\\\])(.*)", path) if match: prev_cwd = os.getcwd() prev_path = path drive, rest_of_path = match.groups() os.chdir(drive) path = rest_of_path kwargs['with_keep_cwd'] = True # END cwd preparation # END windows handling try: proc = git.clone(url, path, with_extended_output=True, as_process=True, v=True, **add_progress(kwargs, git, progress)) if progress: handle_process_output(proc, None, progress.new_message_handler(), finalize_process) else: finalize_process(proc) # end handle progress finally: if prev_cwd is not None: os.chdir(prev_cwd) path = prev_path # END reset previous working dir # END bad windows handling # our git command could have a different working dir than our actual # environment, hence we prepend its working dir if required if not os.path.isabs(path) and git.working_dir: path = join(git._working_dir, path) # adjust remotes - there may be operating systems which use backslashes, # These might be given as initial paths, but when handling the config file # that contains the remote from which we were clones, git stops liking it # as it will escape the backslashes. Hence we undo the escaping just to be # sure repo = cls(os.path.abspath(path), odbt=odbt) if repo.remotes: writer = repo.remotes[0].config_writer writer.set_value('url', repo.remotes[0].url.replace("\\\\", "\\").replace("\\", "/")) # PY3: be sure cleanup is performed and lock is released writer.release() # END handle remote repo return repo
def _index_from_patch_format(cls, repo, proc): """Create a new DiffIndex from the given text which must be in patch format :param repo: is the repository we are operating on - it is required :param stream: result of 'git diff' as a stream (supporting file protocol) :return: git.DiffIndex """ ## FIXME: Here SLURPING raw, need to re-phrase header-regexes linewise. text = [] handle_process_output(proc, text.append, None, finalize_process, decode_streams=False) # for now, we have to bake the stream text = b''.join(text) index = DiffIndex() previous_header = None for header in cls.re_header.finditer(text): a_path_fallback, b_path_fallback, \ old_mode, new_mode, \ rename_from, rename_to, \ new_file_mode, deleted_file_mode, \ a_blob_id, b_blob_id, b_mode, \ a_path, b_path = header.groups() new_file, deleted_file = bool(new_file_mode), bool(deleted_file_mode) a_path = cls._pick_best_path(a_path, rename_from, a_path_fallback) b_path = cls._pick_best_path(b_path, rename_to, b_path_fallback) # Our only means to find the actual text is to see what has not been matched by our regex, # and then retro-actively assign it to our index if previous_header is not None: index[-1].diff = text[previous_header.end():header.start()] # end assign actual diff # Make sure the mode is set if the path is set. Otherwise the resulting blob is invalid # We just use the one mode we should have parsed a_mode = old_mode or deleted_file_mode or (a_path and (b_mode or new_mode or new_file_mode)) b_mode = b_mode or new_mode or new_file_mode or (b_path and a_mode) index.append(Diff(repo, a_path, b_path, a_blob_id and a_blob_id.decode(defenc), b_blob_id and b_blob_id.decode(defenc), a_mode and a_mode.decode(defenc), b_mode and b_mode.decode(defenc), new_file, deleted_file, rename_from, rename_to, None, None)) previous_header = header # end for each header we parse if index: index[-1].diff = text[header.end():] # end assign last diff return index
def _get_fetch_info_from_stderr(self, proc, progress): progress = to_progress_instance(progress) # skip first line as it is some remote info we are not interested in output = IterableList('name') # lines which are no progress are fetch info lines # this also waits for the command to finish # Skip some progress lines that don't provide relevant information fetch_info_lines = [] # Basically we want all fetch info lines which appear to be in regular form, and thus have a # command character. Everything else we ignore, cmds = set(FetchInfo._flag_map.keys()) progress_handler = progress.new_message_handler() handle_process_output(proc, None, progress_handler, finalizer=None, decode_streams=False) stderr_text = progress.error_lines and '\n'.join(progress.error_lines) or '' proc.wait(stderr=stderr_text) if stderr_text: log.warning("Error lines received while fetching: %s", stderr_text) for line in progress.other_lines: line = force_text(line) for cmd in cmds: if len(line) > 1 and line[0] == ' ' and line[1] == cmd: fetch_info_lines.append(line) continue # read head information with open(osp.join(self.repo.common_dir, 'FETCH_HEAD'), 'rb') as fp: fetch_head_info = [l.decode(defenc) for l in fp.readlines()] l_fil = len(fetch_info_lines) l_fhi = len(fetch_head_info) if l_fil != l_fhi: msg = "Fetch head lines do not match lines provided via progress information\n" msg += "length of progress lines %i should be equal to lines in FETCH_HEAD file %i\n" msg += "Will ignore extra progress lines or fetch head lines." msg %= (l_fil, l_fhi) log.debug(msg) log.debug("info lines: " + str(fetch_info_lines)) log.debug("head info : " + str(fetch_head_info)) if l_fil < l_fhi: fetch_head_info = fetch_head_info[:l_fil] else: fetch_info_lines = fetch_info_lines[:l_fhi] # end truncate correct list # end sanity check + sanitization output.extend(FetchInfo._from_line(self.repo, err_line, fetch_line) for err_line, fetch_line in zip(fetch_info_lines, fetch_head_info)) return output
def _index_from_raw_format(cls, repo, proc): """Create a new DiffIndex from the given stream which must be in raw format. :return: git.DiffIndex""" # handles # :100644 100644 687099101... 37c5e30c8... M .gitignore index = DiffIndex() def handle_diff_line(line): line = line.decode(defenc) if not line.startswith(":"): return meta, _, path = line[1:].partition('\t') old_mode, new_mode, a_blob_id, b_blob_id, change_type = meta.split( None, 4) path = path.strip() a_path = path.encode(defenc) b_path = path.encode(defenc) deleted_file = False new_file = False rename_from = None rename_to = None # NOTE: We cannot conclude from the existence of a blob to change type # as diffs with the working do not have blobs yet if change_type == 'D': b_blob_id = None deleted_file = True elif change_type == 'A': a_blob_id = None new_file = True elif change_type[ 0] == 'R': # parses RXXX, where XXX is a confidence value a_path, b_path = path.split('\t', 1) a_path = a_path.encode(defenc) b_path = b_path.encode(defenc) rename_from, rename_to = a_path, b_path # END add/remove handling diff = Diff(repo, a_path, b_path, a_blob_id, b_blob_id, old_mode, new_mode, new_file, deleted_file, rename_from, rename_to, '', change_type) index.append(diff) handle_process_output(proc, handle_diff_line, None, finalize_process, decode_streams=False) return index
def _index_from_raw_format(cls, repo: 'Repo', proc: TBD) -> DiffIndex: """Create a new DiffIndex from the given stream which must be in raw format. :return: git.DiffIndex""" # handles # :100644 100644 687099101... 37c5e30c8... M .gitignore index = DiffIndex() handle_process_output( proc, lambda bytes: cls._handle_diff_line(bytes, repo, index), None, finalize_process, decode_streams=False) return index
def _get_fetch_info_from_stderr(self, proc, progress): # skip first line as it is some remote info we are not interested in output = IterableList('name') # lines which are no progress are fetch info lines # this also waits for the command to finish # Skip some progress lines that don't provide relevant information fetch_info_lines = list() # Basically we want all fetch info lines which appear to be in regular form, and thus have a # command character. Everything else we ignore, cmds = set(PushInfo._flag_map.keys()) & set(FetchInfo._flag_map.keys()) progress_handler = progress.new_message_handler() def my_progress_handler(line): for pline in progress_handler(line): if line.startswith('fatal:') or line.startswith('error:'): raise GitCommandError(("Error when fetching: %s" % line, ), 2) # END handle special messages for cmd in cmds: if len(line) > 1 and line[0] == ' ' and line[1] == cmd: fetch_info_lines.append(line) continue # end find command code # end for each comand code we know # end for each line progress didn't handle # end # We are only interested in stderr here ... handle_process_output(proc, None, my_progress_handler, finalize_process) # read head information fp = open(join(self.repo.git_dir, 'FETCH_HEAD'), 'rb') fetch_head_info = [l.decode(defenc) for l in fp.readlines()] fp.close() # NOTE: We assume to fetch at least enough progress lines to allow matching each fetch head line with it. assert len(fetch_info_lines) >= len( fetch_head_info), "len(%s) <= len(%s)" % (fetch_head_info, fetch_info_lines) output.extend( FetchInfo._from_line(self.repo, err_line, fetch_line) for err_line, fetch_line in zip(fetch_info_lines, fetch_head_info)) return output
def _index_from_raw_format(cls, repo, proc): """Create a new DiffIndex from the given stream which must be in raw format. :return: git.DiffIndex""" # handles # :100644 100644 687099101... 37c5e30c8... M .gitignore index = DiffIndex() def handle_diff_line(line): line = line.decode(defenc) if not line.startswith(":"): return meta, _, path = line[1:].partition('\t') old_mode, new_mode, a_blob_id, b_blob_id, change_type = meta.split(None, 4) path = path.strip() a_path = path.encode(defenc) b_path = path.encode(defenc) deleted_file = False new_file = False rename_from = None rename_to = None # NOTE: We cannot conclude from the existence of a blob to change type # as diffs with the working do not have blobs yet if change_type == 'D': b_blob_id = None deleted_file = True elif change_type == 'A': a_blob_id = None new_file = True elif change_type[0] == 'R': # parses RXXX, where XXX is a confidence value a_path, b_path = path.split('\t', 1) a_path = a_path.encode(defenc) b_path = b_path.encode(defenc) rename_from, rename_to = a_path, b_path # END add/remove handling diff = Diff(repo, a_path, b_path, a_blob_id, b_blob_id, old_mode, new_mode, new_file, deleted_file, rename_from, rename_to, '', change_type) index.append(diff) handle_process_output(proc, handle_diff_line, None, finalize_process, decode_streams=False) return index
def run_commit_hook(name: str, index: 'IndexFile', *args: str) -> None: """Run the commit hook of the given name. Silently ignores hooks that do not exist. :param name: name of hook, like 'pre-commit' :param index: IndexFile instance :param args: arguments passed to hook file :raises HookExecutionError: """ hp = hook_path(name, index.repo.git_dir) if not os.access(hp, os.X_OK): return None env = os.environ.copy() env['GIT_INDEX_FILE'] = safe_decode(str(index.path)) env['GIT_EDITOR'] = ':' cmd = [hp] try: if is_win and not _has_file_extension(hp): # Windows only uses extensions to determine how to open files # (doesn't understand shebangs). Try using bash to run the hook. relative_hp = Path(hp).relative_to( index.repo.working_dir).as_posix() cmd = ["bash.exe", relative_hp] cmd = subprocess.Popen( cmd + list(args), env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=index.repo.working_dir, close_fds=is_posix, creationflags=PROC_CREATIONFLAGS, ) except Exception as ex: raise HookExecutionError(hp, ex) from ex else: stdout_list: List[str] = [] stderr_list: List[str] = [] handle_process_output(cmd, stdout_list.append, stderr_list.append, finalize_process) stdout = ''.join(stdout_list) stderr = ''.join(stderr_list) if cmd.returncode != 0: stdout = force_text(stdout, defenc) stderr = force_text(stderr, defenc) raise HookExecutionError(hp, cmd.returncode, stderr, stdout)
def _get_fetch_info_from_stderr(self, proc, progress): # skip first line as it is some remote info we are not interested in output = IterableList('name') # lines which are no progress are fetch info lines # this also waits for the command to finish # Skip some progress lines that don't provide relevant information fetch_info_lines = list() # Basically we want all fetch info lines which appear to be in regular form, and thus have a # command character. Everything else we ignore, cmds = set(PushInfo._flag_map.keys()) & set(FetchInfo._flag_map.keys()) progress_handler = progress.new_message_handler() def my_progress_handler(line): for pline in progress_handler(line): if line.startswith('fatal:') or line.startswith('error:'): raise GitCommandError(("Error when fetching: %s" % line,), 2) # END handle special messages for cmd in cmds: if len(line) > 1 and line[0] == ' ' and line[1] == cmd: fetch_info_lines.append(line) continue # end find command code # end for each comand code we know # end for each line progress didn't handle # end # We are only interested in stderr here ... handle_process_output(proc, None, my_progress_handler, finalize_process) # read head information fp = open(join(self.repo.git_dir, 'FETCH_HEAD'), 'rb') fetch_head_info = [l.decode(defenc) for l in fp.readlines()] fp.close() # NOTE: We assume to fetch at least enough progress lines to allow matching each fetch head line with it. assert len(fetch_info_lines) >= len(fetch_head_info), "len(%s) <= len(%s)" % (fetch_head_info, fetch_info_lines) output.extend(FetchInfo._from_line(self.repo, err_line, fetch_line) for err_line, fetch_line in zip(fetch_info_lines, fetch_head_info)) return output
def _get_push_info(self, proc, progress): # read progress information from stderr # we hope stdout can hold all the data, it should ... # read the lines manually as it will use carriage returns between the messages # to override the previous one. This is why we read the bytes manually # TODO: poll() on file descriptors to know what to read next, process streams concurrently progress_handler = progress.new_message_handler() output = IterableList('name') def stdout_handler(line): try: output.append(PushInfo._from_line(self, line)) except ValueError: # if an error happens, additional info is given which we cannot parse pass # END exception handling # END for each line handle_process_output(proc, stdout_handler, progress_handler, finalize_process) return output
def test_handle_process_output(self): from git.cmd import handle_process_output line_count = 5002 count = [None, 0, 0] def counter_stdout(line): count[1] += 1 def counter_stderr(line): count[2] += 1 proc = subprocess.Popen([sys.executable, fixture_path('cat_file.py'), str(fixture_path('issue-301_stderr'))], stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False) handle_process_output(proc, counter_stdout, counter_stderr, lambda proc: proc.wait()) assert count[1] == line_count assert count[2] == line_count
def _clone(cls, git, url, path, odb_default_type, progress, **kwargs): if progress is not None: progress = to_progress_instance(progress) odbt = kwargs.pop('odbt', odb_default_type) proc = git.clone(url, path, with_extended_output=True, as_process=True, v=True, **add_progress(kwargs, git, progress)) if progress: handle_process_output(proc, None, progress.new_message_handler(), finalize_process) else: (stdout, stderr ) = proc.communicate() # FIXME: Will block of outputs are big! log.debug("Cmd(%s)'s unused stdout: %s", getattr(proc, 'args', ''), stdout) finalize_process(proc, stderr=stderr) # our git command could have a different working dir than our actual # environment, hence we prepend its working dir if required if not os.path.isabs(path) and git.working_dir: path = join(git._working_dir, path) # adjust remotes - there may be operating systems which use backslashes, # These might be given as initial paths, but when handling the config file # that contains the remote from which we were clones, git stops liking it # as it will escape the backslashes. Hence we undo the escaping just to be # sure repo = cls(path, odbt=odbt) if repo.remotes: with repo.remotes[0].config_writer as writer: writer.set_value('url', Git.polish_url(repo.remotes[0].url)) # END handle remote repo return repo
def run_commit_hook(name: str, index: 'IndexFile', *args: str) -> None: """Run the commit hook of the given name. Silently ignores hooks that do not exist. :param name: name of hook, like 'pre-commit' :param index: IndexFile instance :param args: arguments passed to hook file :raises HookExecutionError: """ hp = hook_path(name, index.repo.git_dir) if not os.access(hp, os.X_OK): return None env = os.environ.copy() env['GIT_INDEX_FILE'] = safe_decode(str(index.path)) env['GIT_EDITOR'] = ':' try: cmd = subprocess.Popen( [hp] + list(args), env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=index.repo.working_dir, close_fds=is_posix, creationflags=PROC_CREATIONFLAGS, ) except Exception as ex: raise HookExecutionError(hp, ex) from ex else: stdout_list = [] # type: List[str] stderr_list = [] # type: List[str] handle_process_output(cmd, stdout_list.append, stderr_list.append, finalize_process) stdout = ''.join(stdout_list) stderr = ''.join(stderr_list) if cmd.returncode != 0: stdout = force_text(stdout, defenc) stderr = force_text(stderr, defenc) raise HookExecutionError(hp, cmd.returncode, stderr, stdout)
def _get_fetch_info_from_stderr(self, proc: 'Git.AutoInterrupt', progress: Union[Callable[..., Any], RemoteProgress, None], kill_after_timeout: Union[None, float] = None, ) -> IterableList['FetchInfo']: progress = to_progress_instance(progress) # skip first line as it is some remote info we are not interested in output: IterableList['FetchInfo'] = IterableList('name') # lines which are no progress are fetch info lines # this also waits for the command to finish # Skip some progress lines that don't provide relevant information fetch_info_lines = [] # Basically we want all fetch info lines which appear to be in regular form, and thus have a # command character. Everything else we ignore, cmds = set(FetchInfo._flag_map.keys()) progress_handler = progress.new_message_handler() handle_process_output(proc, None, progress_handler, finalizer=None, decode_streams=False, kill_after_timeout=kill_after_timeout) stderr_text = progress.error_lines and '\n'.join(progress.error_lines) or '' proc.wait(stderr=stderr_text) if stderr_text: log.warning("Error lines received while fetching: %s", stderr_text) for line in progress.other_lines: line = force_text(line) for cmd in cmds: if len(line) > 1 and line[0] == ' ' and line[1] == cmd: fetch_info_lines.append(line) continue # read head information fetch_head = SymbolicReference(self.repo, "FETCH_HEAD") with open(fetch_head.abspath, 'rb') as fp: fetch_head_info = [line.decode(defenc) for line in fp.readlines()] l_fil = len(fetch_info_lines) l_fhi = len(fetch_head_info) if l_fil != l_fhi: msg = "Fetch head lines do not match lines provided via progress information\n" msg += "length of progress lines %i should be equal to lines in FETCH_HEAD file %i\n" msg += "Will ignore extra progress lines or fetch head lines." msg %= (l_fil, l_fhi) log.debug(msg) log.debug(b"info lines: " + str(fetch_info_lines).encode("UTF-8")) log.debug(b"head info: " + str(fetch_head_info).encode("UTF-8")) if l_fil < l_fhi: fetch_head_info = fetch_head_info[:l_fil] else: fetch_info_lines = fetch_info_lines[:l_fhi] # end truncate correct list # end sanity check + sanitization for err_line, fetch_line in zip(fetch_info_lines, fetch_head_info): try: output.append(FetchInfo._from_line(self.repo, err_line, fetch_line)) except ValueError as exc: log.debug("Caught error while parsing line: %s", exc) log.warning("Git informed while fetching: %s", err_line.strip()) return output