def github_backup_reposities(repositories, username, dest_dir_path):
app_settings.info('backing up repositores...')
for repository in repositories:
app_settings.info(f'backing up github repository {repository.name} for {username}')
if not github_backup_repository(repository, username, dest_dir_path):
app_settings.error(f'quitting backup process due to error while backing up {repository.name}')
return False
return True
def _get_invalid_subtitle_filenames(dir_name):
for filename in _get_subtitle_filenames(dir_name):
app_settings.debug(f'File: {filename}')
line_count = _get_subtitle_line_count(filename)
app_settings.debug(f'File: {filename}\n : {line_count} lines')
if line_count < 1000:
app_settings.info(f'{line_count} - {filename} INVALID')
yield filename
def _get_user_repository_info(username, repository):
app_settings.info(f'quering for user ({username}) repository ({repository}) information')
uri = f'https://api.github.com/repos/{username}/{repository}'
response_str = _call_uri(uri)
if not response_str:
app_settings.error(f'while trying to query {repository} under {username} account')
return None
repository_json = json.loads(response_str)
return RepositoryInfo(repository_json)
def generate(self):
for i in range(1, 100):
password = secrets.choice(string.ascii_lowercase +
string.ascii_uppercase)
password += ''.join(
secrets.choice(self._alphabet)
for i in range(self._length - 1))
if self._is_valid(password):
app_settings.info(f'returning the {i} generated password')
return password
def github_backup_git_repository(repository, username, dest_dir_path, timestamp):
app_settings.info(f'backing up git repository {repository.name} {repository.git_url}')
repo_backup_path = dest_dir_path.joinpath(f'{username}-{repository.name}-{timestamp}.git')
repo_backup_tar_file = dest_dir_path.joinpath(f'{username}-{repository.name}-{timestamp}.git.tar.bz2')
command = f'git clone --quiet --mirror {repository.git_url} {repo_backup_path}'
app_settings.info(f'calling {command}')
result = _call_external_process(command)
if result:
result = _archive_and_compress(repo_backup_tar_file, repo_backup_path)
_remove_filesystem_object(repo_backup_path)
return result
def _call_uri(uri):
try:
app_settings.info(f'calling {uri} with a {_TIME_OUT} second timeout')
response = requests.get(uri, timeout=_TIME_OUT)
response.raise_for_status()
response_str = response.text.strip()
return None if not response_str else response_str
except requests.exceptions.RequestException as e:
app_settings.error(f'error while calling {uri}')
app_settings.error(str(e))
return None
def github_backup_repository(repository, username, dest_dir_path):
timestamp = _get_backup_timestamp()
if not github_backup_git_repository(repository , username, dest_dir_path, timestamp):
app_settings.error(f'backing up github repository {repository.git_url}')
return False
if not github_backup_git_wiki(repository, username, dest_dir_path, timestamp):
app_settings.error(f'backing up github wiki repository {repository.wiki_url}')
return False
if not github_backup_git_issues(repository, username, dest_dir_path, timestamp):
app_settings.error(f'backing up github issues repository {repository.issues_url}')
return False
app_settings.info(f'successfully backed up {username} github repository {repository.git_url}')
return True
def github_backup_git_issues(repository, username, dest_dir_path, timestamp):
app_settings.info(f'backing up git issues {repository.name} {repository.git_url}')
if not repository.issues_url:
app_settings.info(f'no git issues found {repository.name} {repository.git_url}')
return True
issues_backup_path = dest_dir_path.joinpath(f'{username}-{repository.name}-{timestamp}.issues.git')
issues_backup_tar_file = dest_dir_path.joinpath(f'{username}-{repository.name}-{timestamp}.issues.git.tar.bz2')
command = f'http -b {repository.issues_url} --output {issues_backup_path}'
result = _call_external_process(command)
if result:
result = _archive_and_compress(issues_backup_tar_file, issues_backup_path)
_remove_filesystem_object(issues_backup_path)
return result
def github_backup_git_wiki(repository, username, dest_dir_path, timestamp):
app_settings.info(f'backing up git wiki {repository.name} {repository.git_url}')
if not repository.wiki_url:
app_settings.info(f'no git wiki found {repository.name} {repository.git_url}')
return True
wiki_backup_path = dest_dir_path.joinpath(f'{username}-{repository.name}-{timestamp}.wiki.git')
wiki_backup_tar_file = dest_dir_path.joinpath(f'{username}-{repository.name}-{timestamp}.wiki.git.tar.bz2')
command = f'git clone --quiet --mirror {repository.wiki_url} {wiki_backup_path}'
result = _call_external_process(command, ignore_errors=True)
if result and wiki_backup_path.exists():
result = _archive_and_compress(wiki_backup_tar_file, wiki_backup_path)
_remove_filesystem_object(wiki_backup_path)
return result
def _archive_and_compress(tar_path, backup_path):
try:
_remove_filesystem_object(tar_path)
with tarfile.open(tar_path, 'w:bz2') as tar:
tar.add(backup_path)
app_settings.info(f'successfully created and populated bz2 tar file {tar_path}')
return True
except tarfile.TarError as err:
app_settings.error(f'creating or populating {tar_path} with {backup_path}: {err}')
return False
except Exception as err:
app_settings.error(f'creating or populating {tar_path} with {backup_path}: {err}')
return False
def _get_user_repository_infos(username):
app_settings.info(f'quering for user ({username}) repositories information')
uri = f'https://api.github.com/users/{username}/repos'
response_str = _call_uri(uri)
if not response_str:
app_settings.error(f'while trying to query {repository} under {username} account')
return None
repositories_json = json.loads(response_str)
repository_infos = []
for repository_json in repositories_json:
repository_infos.append(RepositoryInfo(repository_json))
app_settings.info(f'found repository {repository_infos[-1].name} at {repository_infos[-1].git_url}')
return repository_infos
def remove_invalid_subtitles(dir_name):
for filename in _get_invalid_subtitle_filenames(dir_name):
if app_settings.live_run:
try:
app_settings.info(f'{filename} live run -- deleting')
os.remove(path)
app_settings.info(f'{filename} live run -- deleted')
except OSError as e:
app_settings.error(f'{e.code} - {e.stderr} file: {filename}')
else:
app_settings.info(f'{filename} dry run -- doing nothing')
def _parse_ignore_data(data):
for line_number, pattern in enumerate(data.splitlines(), 1):
pattern = pattern.strip()
if not pattern:
app_settings.info(f'{line_number:2}: empty, skipping')
continue
if pattern.startswith('#'):
app_settings.info(
f'{line_number:2}: comment, skipping -- {pattern}')
continue
app_settings.info(f'{line_number:2}: adding -- {pattern}')
# the only modification needed is to strip the trailing '/' which is sometimes added to directories
yield pattern.rstrip('/')
def _call_external_process(command, ignore_errors=False):
args = shlex.split(command)
try:
app_settings.info(f'executing \'{command}\'')
#subprocess.run(args, check=True, encoding='utf-8', shell=True, env=os.environ, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
subprocess.run(args, check=True, encoding='utf-8', stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
return True
except subprocess.SubprocessError as err:
if ignore_errors:
app_settings.info(f'\'{command}\' returned: {err}')
return True
else:
app_settings.error(f'\'{command}\' returned: {err}')
return False
except Exception as err:
if ignore_errors:
app_settings.info(f'\'{command}\' returned: {err}')
return True
else:
app_settings.error(f'\'{command}\' returned: {err}')
return False