def _split_file(abspath: ExPath, cwd, dry_run: bool):
try:
splits = abspath.split(verify_integrity=True)
except FileExistsError as e:
# splits already existed
misc.whiteprint('found pre-existing splits')
splits = abspath.getsplits()
splits_ok = abspath.splits_integrity_ok()
if not splits_ok:
prompt.generic(
f"splits of '{abspath}' aren't good, joined file at /tmp/joined",
flowopts=('quit', 'debug'))
return
misc.greenprint('splits are good')
biggest_split = max([split.size('mb') for split in splits])
promptstr = f'found {len(splits)} pre-existing splits, biggest is {biggest_split}MB.'
_handle_already_split_file(promptstr, abspath, cwd, dry_run=dry_run)
return
except OSError:
prompt.generic(
f"Just split {abspath} but splits aren't good, joined file at /tmp/joined",
flowopts=('quit', 'debug'))
else:
biggest_split = max([split.size('mb') for split in splits])
promptstr = f'no pre-existing splits found. split file to {len(splits)} good splits, biggest is {biggest_split}MB.'
_handle_already_split_file(promptstr, abspath, cwd, dry_run=dry_run)
def test__usr__1(self):
# because '/usr/*' is parent of '/usr/local',
# '/usr/' isn't parent of '/usr/*'
# (because they function the same)
usr_wc = ExPath('/usr/')
for path in path_ctor_permutations('/usr/*'):
assert not usr_wc.parent_of(path)
def test__parent_is_actually_file(self):
# both really exist but a file isn't a parent of anything
file = ExPath('/home/gilad/.local/bin/pip*')
for otherfile in path_ctor_permutations('/home/gilad/.local/bin/pip3'):
try:
assert not file.parent_of(otherfile)
except AssertionError as e:
misc.brightredprint(f'file: {repr(file)} ({type(file)}) is not parent of otherfile: {repr(otherfile)} ({type(otherfile)})')
raise
def build_paths(exclude_parent, exclude_paths, ignore_paths) -> List[ExPath]:
logger.debug(f'exclude_parent:', exclude_parent, 'exclude_paths:',
exclude_paths, 'ignore_paths:', ignore_paths)
statusfiles = None
paths: List[ExPath] = []
skip_non_existent = False
ignore_non_existent = False
for f in ignore_paths:
# * wildcard
if "*" in f:
paths.append(f) # TODO: now that ExPath supports glob, ...
continue
# * index
try:
if not statusfiles:
statusfiles = Status().files
f = statusfiles[int(str(f))]
except IndexError:
i = input(
f'Index error with {f} (len of files: {len(statusfiles)}), try again:\t'
)
f = statusfiles[int(i)]
except ValueError:
pass # not a number
path = ExPath(f)
if not path.exists():
if skip_non_existent:
continue
if not ignore_non_existent:
key, choice = prompt.action(f'{path} does not exist',
'skip',
'ignore anyway',
sA='skip all',
iA='ignore all')
if key == 'skip':
continue
if key == 'sA':
skip_non_existent = True
elif key == 'iA':
ignore_non_existent = True
continue
if path == exclude_parent:
# path: '.config', exclude_paths: ['.config/dconf']
for sub in filter(lambda p: p not in exclude_paths,
path.iterdir()):
paths.append(sub)
else:
paths.append(path)
if not paths:
sys.exit(colors.brightred(f'no paths in {ExPath(".").absolute()}'))
return paths
def values(self) -> List[ExPath]:
with self.file.open(mode='r') as file:
data = file.read()
lines = data.splitlines()
paths = []
for x in lines:
if not bool(x) or '#' in x or not WORD_RE.search(x):
continue
paths.append(ExPath(x))
return [*paths, ExPath('.git')]
def test__doesnt_exist__0(self):
bad = ExPath('/uzer/*')
assert not bad.exists()
assert not bad.is_dir()
assert not bad.is_file()
bad2 = ExPath('/*DOESNTEXIST')
assert not bad2.exists()
assert not bad2.is_dir()
assert not bad2.is_file()
def test____eq__():
gilad = ExPath(giladdirstr)
for path in path_ctor_permutations(giladdirstr):
assert gilad == path
for path in path_ctor_permutations('~/'):
assert gilad == path
gilad = ExPath('~/')
for path in path_ctor_permutations('~/'):
assert gilad == path
for path in path_ctor_permutations(giladdirstr):
assert gilad == path
def _handle_already_split_file(promptstr, main_file: ExPath, cwd,
dry_run: bool):
key, answer = prompt.action(promptstr,
'ignore main file',
'move main file to trash://',
flowopts='quit')
if key == 'i':
ignore(main_file.relative_to(cwd), dry_run=dry_run)
return
# key == 'm' for move to trash
if dry_run:
misc.whiteprint('dry run, not trashing file')
return
main_file.trash()
def is_subpath_of_ignored(self, p) -> bool:
"""Returns True if `p` is strictly a subpath of a path in .gitignore"""
path = ExPath(p)
for ignored in self:
if ignored.parent_of(path):
return True
return False
def handle_exclude_paths(
exclude_paths_str: str) -> Tuple[ExPath, List[ExPath]]:
"""Does the '.config/dconf copyq' trick"""
if not exclude_paths_str:
return None, None
if '/' in exclude_paths_str:
exclude_parent, _, exclude_paths = exclude_paths_str.rpartition('/')
exclude_parent = ExPath(exclude_parent)
exclude_paths = [
exclude_parent / ExPath(ex) for ex in exclude_paths.split(' ')
]
else:
raise NotImplementedError(
f'only currently handled format is `.ipython/profile_default/startup ipython_config.py`'
)
return exclude_parent, exclude_paths
def test__single_wildcard__part_of_part(self):
assert ExPath('/home/gilad/.bashr*').is_file()
assert ExPath('/home/gil*d').is_dir()
assert ExPath('/home/gil*d').is_file() is False
assert ExPath('/*ome/gilad').is_dir()
assert ExPath('/*ome/gilad').is_file() is False
assert ExPath('/home/gil*d/.bashrc').is_dir() is False
assert ExPath('/home/gil*d/.bashrc').is_file()
assert ExPath('/*ome/gilad/.bashrc').is_file()
def backup(self, confirm: bool):
if confirm and not prompt.confirm(f'Backup .gitignore to .gitignore.backup?'):
print('aborting')
return False
absolute = self.file.absolute()
try:
shell.run(f'cp {absolute} {absolute}.backup', raiseexc='summary')
except Exception as e:
if not prompt.confirm('Backup failed, overwrite .gitignore anyway?', flowopts='debug'):
print('aborting')
return False
return True
else:
backup = ExPath(f'{absolute}.backup')
if not backup.exists() and not prompt.confirm(f'Backup command completed without error, but {backup} doesnt exist. overwrite .gitignore anyway?', flowopts='debug'):
print('aborting')
return False
return True
def is_ignored(self, p) -> bool:
"""Returns True if `p` in .gitignore, or if `p` is a subpath of a path in .gitignore,
or if `p` fullmatches any part of a path in .gitignore (i.e. if 'env' is ignored, then 'src/env' returns True)"""
path = ExPath(p)
for ignored in self:
if ignored == path:
return True
if ignored.parent_of(path):
return True
if any(re.fullmatch('env', part) for part in path.parts):
return True
return False
def test__no_suffix__endswith_path_reg(self):
# e.g. 'py_venv.*/'. should return no suffix (False)
no_suffix = ExPath('py_venv')
assert no_suffix.has_file_suffix() is False
for reg in path_regexes():
val = ExPath(f'{no_suffix}{reg}')
actual = val.has_file_suffix()
assert actual is False
def test__with_suffix__startswith_path_reg(self):
# e.g. '.*/py_venv.xml'. should return has suffix (True)
with_suffix = ExPath('py_venv.xml')
assert with_suffix.has_file_suffix() is True
for reg in path_regexes():
val = ExPath(f'{reg}{with_suffix}')
actual = val.has_file_suffix()
assert actual is True
def _get_large_files_in_dir(path: ExPath) -> Dict[ExPath, float]:
large_subfiles = {}
for p in path.iterdir():
if not p.exists():
misc.yellowprint(
f'_get_large_files_in_dir() | does not exist: {repr(p)}')
continue
if p.is_file():
mb = p.size('mb')
if mb >= 50:
large_subfiles[p] = mb
else:
large_subfiles.update(_get_large_files_in_dir(p))
return large_subfiles
def test__no_suffix__surrounded_by_path_reg(self):
# e.g. '.*/py_venv.*/'. should return no suffix (False)
no_suffix = ExPath('py_venv')
assert no_suffix.has_file_suffix() is False
for reg in path_regexes():
for morereg in path_regexes():
val = ExPath(f'{morereg}{no_suffix}{reg}')
actual = val.has_file_suffix()
assert actual is False
def test__everything_mixed_with_regex(self):
# e.g. '.*/py_v[en]*v.xm?l'. should return has suffix (True)
assert ExPath('.*/py_v[en]*v.xm?l').has_file_suffix() is True
mixed_stems = get_permutations_in_size_range(f'{REGEX_CHAR}.py_venv-1257', slice(5), has_letters_and_punc)
for stem in mixed_stems:
for suffix in mixed_suffixes():
name = ExPath(f'{stem}.{suffix}')
actual = name.has_file_suffix()
assert actual is True
for reg in path_regexes():
val = ExPath(f'{reg}{name}')
actual = val.has_file_suffix()
assert actual is True
def unignore(self, path, *, confirm=False, dry_run=False, backup=True):
path = ExPath(path)
newvals = []
found = False
for ignored in self:
if ignored == path:
breakpoint()
found = True
continue
newvals.append(ignored)
if not found:
logging.warning(f'Gitignore.unignore(path={path}): not in self. returning')
return
if confirm and not prompt.confirm(f'Remove {path} from .gitignore?'):
print('aborting')
return
self.write(newvals, verify_paths=False, dry_run=dry_run, backup=backup)
def _clean_shortstatus(
_statusline: str) -> Optional[Tuple[ExPath, str]]:
if _statusline.startswith("#"):
return None
_status, _file = map(misc.unquote,
map(str.strip, _statusline.split(maxsplit=1)))
if 'R' in _status:
if '->' not in _file:
raise ValueError(
f"'R' in status but '->' not in file. file: {_file}, status: {_status}",
locals())
_, _, _file = _file.partition(' -> ') # return only existing
else:
if '->' in _file:
raise ValueError(
f"'R' not in status but '->' in file. file: {_file}, status: {_status}",
locals())
return ExPath(_file), _status
def search(self, keyword: Union[str, ExPath], *, noprompt=True) -> ExPath:
"""Tries to return an ExPath in status.
First assumes `keyword` is an exact file (str or ExPath), and if fails, uses `search` module.
@param noprompt: specify False to allow using search_and_prompt(keyword, file) in case nothing matched earlier.
"""
darkprint(f'Status.search({repr(keyword)}) |')
path = ExPath(keyword)
for file in self.files:
if file == path:
return file
has_suffix = path.has_file_suffix()
has_slash = '/' in keyword
has_regex = regex.has_regex(keyword)
darkprint(f'\t{has_suffix = }, {has_slash = }, {has_regex = }')
if has_suffix:
files = self.files
else:
files = [f.with_suffix('') for f in self.files]
if has_regex:
for expath in files:
if re.search(keyword, str(expath)):
return expath
if has_slash:
darkprint(
f"looking for the nearest match among status paths for: '{keyword}'"
)
return ExPath(search.nearest(keyword, files))
darkprint(
f"looking for a matching part among status paths ({'with' if has_suffix else 'without'} suffixes...) for: '{keyword}'"
)
for f in files:
# TODO: integrate into git.search somehow
for i, part in enumerate(f.parts):
if part == keyword:
ret = ExPath(os.path.join(*f.parts[0:i + 1]))
return ret
if noprompt:
return None
darkprint(
f"didn't find a matching part, calling search_and_prompt()...")
choice = search_and_prompt(keyword, [str(f) for f in files],
criterion='substring')
if choice:
return ExPath(choice)
prompt.generic(colors.red(f"'{keyword}' didn't match anything"),
flowopts=True)
def main(commitmsg: str, dry_run: bool = False):
status = Status()
if not status.files:
if prompt.confirm('No files in status, just push?'):
if dry_run:
misc.whiteprint('dry run, not pushing. returning')
return
return git.push()
cwd = ExPath(os.getcwd())
largefiles: Dict[ExPath, float] = get_large_files_from_status(cwd, status)
if largefiles:
handle_large_files(cwd, largefiles, dry_run)
if '.zsh_history' in status:
pwd = misc.getsecret('general')
with open('/home/gilad/.zsh_history', errors='backslashreplace') as f:
history = f.readlines()
for i, line in enumerate(history):
if pwd in line:
misc.redprint(
f"password found in .zsh_history in line {i}. exiting")
return
if not commitmsg:
if len(status.files) == 1:
commitmsg = status.files[0]
elif len(status.files) < 3:
commitmsg = ', '.join([f.name for f in status.files])
else:
os.system('git status -s')
commitmsg = input('commit msg:\t')
commitmsg = misc.unquote(commitmsg)
if dry_run:
misc.whiteprint('dry run. exiting')
return
shell.run('git add .', f'git commit -am "{commitmsg}"')
git.push()
def test__single_wildcard__whole_part__0(self):
assert ExPath('/usr/*').is_dir()
assert ExPath('/usr/*/share').is_dir()
assert ExPath('/*/local/share').is_dir()
assert ExPath('/usr/*/BAD').is_dir() is False
assert ExPath('/usr/*/shar').is_dir() is False
assert ExPath('/usr/*/shar').is_file() is False
assert ExPath('/usr/*/shar').exists() is False
assert ExPath('/*/local/shar').is_dir() is False
assert ExPath('/*/local/shar').is_file() is False
assert ExPath('/*/local/shar').exists() is False
assert ExPath('/home/gilad/*').is_dir()
assert ExPath('/home/gilad/*').is_file() is False
assert ExPath('/home/*/.bashrc').is_dir() is False
assert ExPath('/home/*/.bashrc').is_file()
assert ExPath('/*/gilad/.bashrc').is_file()
def test__single_wildcard__whole_part__1(self):
assert ExPath('/*/gilad').is_dir()
assert ExPath('/*/gilad').is_file() is False
def test__multiple_wildcards__whole_part(self):
assert ExPath('/*/gilad/*').is_dir()
assert ExPath('/*/gilad/*').is_file() is False
def test__multiple_wildcards__part_of_part(self):
assert ExPath('/home/*/.b*shrc').is_dir() is False
assert ExPath('/home/*/.b*shrc').is_file()
assert ExPath('/*ome/*/.b*shrc').is_dir() is False
assert ExPath('/*ome/*/.b*shrc').is_file()
assert ExPath('/*/*/.b*shrc').is_file()
assert ExPath('/*/*/.b*shrc').is_dir() is False
assert ExPath('/home/gil*d/.b*shrc').is_dir() is False
assert ExPath('/home/gil*d/.b*shrc').is_file()
assert ExPath('/home/gil*d/*').is_file() is False
assert ExPath('/home/gil*d/*').is_dir()
assert ExPath('/*/gil*d/*').is_dir()
assert ExPath('/*/gil*d/*').is_file() is False
def __contains__(self, item):
path = ExPath(item)
for ignored in self:
if ignored == path:
return True
return False
def __init__(self):
self.file = ExPath('.gitignore')
if not self.file.exists():
raise FileNotFoundError(f'Gitignore.__init__(): {self.file.absolute()} does not exist')
class Gitignore:
# https://git-scm.com/book/en/v2/Git-Basics-Recording-Changes-to-the-Repository#Ignoring-Files
def __init__(self):
self.file = ExPath('.gitignore')
if not self.file.exists():
raise FileNotFoundError(f'Gitignore.__init__(): {self.file.absolute()} does not exist')
def __getitem__(self, item):
return self.values[item]
def __contains__(self, item):
path = ExPath(item)
for ignored in self:
if ignored == path:
return True
return False
def __getattribute__(self, name: str) -> Any:
try:
return super().__getattribute__(name)
except AttributeError as e:
# support all ExPath methods
return getattr(self.file, name)
def __iter__(self):
yield from self.values
# @cachedprop # TODO: uncomment when possible to clear cache
@property
@memoize
def values(self) -> List[ExPath]:
with self.file.open(mode='r') as file:
data = file.read()
lines = data.splitlines()
paths = []
for x in lines:
if not bool(x) or '#' in x or not WORD_RE.search(x):
continue
paths.append(ExPath(x))
return [*paths, ExPath('.git')]
def should_add_to_gitignore(self, p: ExPath, quiet=False) -> bool:
for ignored in self.values:
if ignored == p:
if not quiet:
logging.warning(f'{p} already in gitignore, continuing')
return False
if ignored.parent_of(p):
if quiet:
return False
msg = colors.yellow(f"parent '{ignored}' of '{p}' already in gitignore")
key, action = prompt.action(msg, 'skip', 'ignore anyway', flowopts=('debug', 'quit'))
if action.value == 'skip':
print('skipping')
return False
return True
def backup(self, confirm: bool):
if confirm and not prompt.confirm(f'Backup .gitignore to .gitignore.backup?'):
print('aborting')
return False
absolute = self.file.absolute()
try:
shell.run(f'cp {absolute} {absolute}.backup', raiseexc='summary')
except Exception as e:
if not prompt.confirm('Backup failed, overwrite .gitignore anyway?', flowopts='debug'):
print('aborting')
return False
return True
else:
backup = ExPath(f'{absolute}.backup')
if not backup.exists() and not prompt.confirm(f'Backup command completed without error, but {backup} doesnt exist. overwrite .gitignore anyway?', flowopts='debug'):
print('aborting')
return False
return True
def write(self, paths, *, verify_paths=True, confirm=False, dry_run=False, backup=True):
logging.debug(f'paths:', paths, 'verify_paths:', verify_paths, 'confirm:', confirm, 'dry_run:', dry_run, 'backup:', backup)
writelines = []
if verify_paths:
should_add_to_gitignore = self.should_add_to_gitignore
else:
should_add_to_gitignore = lambda _p: True
for p in filter(should_add_to_gitignore, paths):
to_write = f'\n{p}'
if confirm and not prompt.confirm(f'Add {p} to .gitignore?'):
continue
logging.info(f'Adding {p} to .gitignore. dry_run={dry_run}, backup={backup}, confirm={confirm}')
writelines.append(to_write)
if dry_run:
print('dry_run, returning')
return
if backup:
backup_ok = self.backup(confirm)
if not backup_ok:
return
with self.file.open(mode='a') as file:
file.write(''.join(sorted(writelines)))
Gitignore.values.fget.clear_cache()
def unignore(self, path, *, confirm=False, dry_run=False, backup=True):
path = ExPath(path)
newvals = []
found = False
for ignored in self:
if ignored == path:
breakpoint()
found = True
continue
newvals.append(ignored)
if not found:
logging.warning(f'Gitignore.unignore(path={path}): not in self. returning')
return
if confirm and not prompt.confirm(f'Remove {path} from .gitignore?'):
print('aborting')
return
self.write(newvals, verify_paths=False, dry_run=dry_run, backup=backup)
def is_subpath_of_ignored(self, p) -> bool:
"""Returns True if `p` is strictly a subpath of a path in .gitignore"""
path = ExPath(p)
for ignored in self:
if ignored.parent_of(path):
return True
return False
def is_ignored(self, p) -> bool:
"""Returns True if `p` in .gitignore, or if `p` is a subpath of a path in .gitignore,
or if `p` fullmatches any part of a path in .gitignore (i.e. if 'env' is ignored, then 'src/env' returns True)"""
path = ExPath(p)
for ignored in self:
if ignored == path:
return True
if ignored.parent_of(path):
return True
if any(re.fullmatch('env', part) for part in path.parts):
return True
return False
def paths_where(self, predicate: Callable[[Any], bool]) -> Generator[ExPath, None, None]:
for ignored in self:
if predicate(ignored):
yield ignored
def test__mixed_dirs_and_files__multiple_wildcards(self):
# assumes .yarn/ and .yarnrc exist
yarn = ExPath('/home/*/.yarn*')
assert yarn.exists()
assert not yarn.is_dir()
assert not yarn.is_file()