def run(self):
preferences = get_preferences(self.ui_type)
backup_directory = preferences.backup_directory
dry_run = preferences.dry_run
exclusions = ExclusionPreferences(
self.backup_type.name).get_no_comments()
tables = DynamoDBAccess(
profile_name=preferences.aws_profile).get_table_names()
self.info_out(f"found {len(tables)} DynamoDB tables")
count = 0
for table_name in tables:
# awsimple will update immediately if number of table rows changes, but backup from scratch every so often to be safe
cache_life = timedelta(days=1).total_seconds()
if table_name in exclusions:
self.info_out(f"excluding {table_name}")
elif dry_run:
self.info_out(f"dry run {table_name}")
else:
self.info_out(f"{table_name}")
table = DynamoDBAccess(table_name, cache_life=cache_life)
table_contents = table.scan_table_cached()
dir_path = Path(backup_directory, "dynamodb")
dir_path.mkdir(parents=True, exist_ok=True)
with Path(dir_path, f"{table_name}.pickle").open("wb") as f:
pickle.dump(table_contents, f)
with Path(dir_path, f"{table_name}.json").open("w") as f:
f.write(dynamodb_to_json(table_contents, indent=4))
count += 1
self.info_out(
f"{count} tables, {count} backed up, {len(exclusions)} excluded")
def cli_main(args):
ui_type = UITypes.cli
balsa = Balsa(__application_name__, __author__)
balsa.log_console_prefix = "\r"
balsa.init_logger_from_args(args)
log.info(f"__application_name__={__application_name__}")
log.info(f"__author__={__author__}")
log.info(f"__version__={__version__}")
try:
preferences = get_preferences(ui_type)
preferences.backup_directory = args.path # backup classes will read the preferences DB directly
preferences.github_token = args.token
preferences.aws_profile = args.profile
# If setting the exclusions, just do it for one backup type at a time. The values are stored for subsequent runs.
if args.exclude is not None and len(args.exclude) > 0:
if args.s3:
ExclusionPreferences(BackupTypes.S3.name).set(args.exclude)
elif args.dynamodb:
ExclusionPreferences(BackupTypes.DynamoDB.name).set(
args.exclude)
elif args.github:
ExclusionPreferences(BackupTypes.github.name).set(args.exclude)
did_something = False
dynamodb_local_backup = None
s3_local_backup = None
github_local_backup = None
if args.s3 or args.aws:
s3_local_backup = S3Backup(ui_type, log.info, log.warning,
log.error)
s3_local_backup.start()
did_something = True
if args.dynamodb or args.aws:
dynamodb_local_backup = DynamoDBBackup(ui_type, log.info,
log.warning, log.error)
dynamodb_local_backup.start()
did_something = True
if args.github:
github_local_backup = GithubBackup(ui_type, log.info, log.warning,
log.error)
github_local_backup.start()
did_something = True
if not did_something:
print(
"nothing to do - please specify a backup to do or -h/--help for help"
)
if dynamodb_local_backup is not None:
dynamodb_local_backup.join()
if s3_local_backup is not None:
s3_local_backup.join()
if github_local_backup is not None:
github_local_backup.join()
except Exception as e:
log.exception(e)
def run(self):
preferences = get_preferences(self.ui_type)
dry_run = preferences.dry_run
exclusions = ExclusionPreferences(BackupTypes.github.name).get_no_comments()
backup_dir = Path(preferences.backup_directory, "github")
gh = github3.login(token=preferences.github_token)
repositories = list(gh.repositories())
clone_count = 0
pull_count = 0
for github_repo in repositories:
repo_owner_and_name = str(github_repo)
repo_name = repo_owner_and_name.split("/")[-1]
if any([e == repo_name for e in exclusions]):
self.info_out(f"{repo_owner_and_name} excluded")
elif dry_run:
self.info_out(f'dry run {repo_owner_and_name}')
else:
repo_dir = Path(backup_dir, repo_owner_and_name).absolute()
branches = github_repo.branches()
# if we've cloned previously, just do a pull
pull_success = False
if repo_dir.exists():
try:
if pull_success := self.pull_branches(repo_owner_and_name, branches, repo_dir):
pull_count += 1
except GitCommandError as e:
self.warning_out(f'could not pull "{repo_dir}" - will try to start over and do a clone of "{repo_owner_and_name} {e}"')
# new to us - clone the repo
if not pull_success:
try:
if repo_dir.exists():
rmdir(repo_dir)
self.info_out(f'git clone "{repo_owner_and_name}"')
Repo.clone_from(github_repo.clone_url, repo_dir)
time.sleep(1.0)
self.pull_branches(repo_owner_and_name, branches, repo_dir)
clone_count += 1
except PermissionError as e:
self.warning_out(f"{repo_owner_and_name} : {e}")
self.info_out(f"{len(repositories)} repos, {pull_count} pulls, {clone_count} clones, {len(exclusions)} excluded")
def __init__(self):
super().__init__()
self.setWindowIcon(QIcon(str(get_icon_path("png"))))
# set task bar icon (Windows only)
bup_app_id = f'{__author__}.{__application_name__}.{__version__}' # arbitrary string
ctypes.windll.shell32.SetCurrentProcessExplicitAppUserModelID(
bup_app_id)
self.setWindowTitle(f"{__application_name__} ({__version__})")
self.setWindowFlag(Qt.WindowMinimizeButtonHint, True)
self.setWindowFlag(Qt.WindowMaximizeButtonHint, True)
self.setLayout(QVBoxLayout())
self.tab_widget = QTabWidget()
self.layout().addWidget(self.tab_widget)
self.run_backup_widget = RunBackupWidget()
self.preferences_widget = PreferencesWidget()
self.about_widget = BupAbout()
self.tab_widget.addTab(self.run_backup_widget, "Backup")
self.tab_widget.addTab(self.preferences_widget, "Preferences")
self.tab_widget.addTab(self.about_widget, "About")
preferences = get_preferences(UITypes.gui)
width = preferences.width
height = preferences.height
if width is not None and width > 0 and height is not None and height > 0:
self.resize(preferences.width, preferences.height)
self.autobackup_timer = QTimer()
self.autobackup_timer.timeout.connect(self.autobackup_tick)
self.autobackup_timer.start(1000) # once a second
self.tick_count = 0
def get_gui_preferences():
return get_preferences(UITypes.gui)
def run(self):
preferences = get_preferences(self.ui_type)
dry_run = preferences.dry_run
backup_directory = os.path.join(preferences.backup_directory, "s3")
os.makedirs(backup_directory, exist_ok=True)
s3_access = S3Access(profile_name=preferences.aws_profile)
decoding = "utf-8"
# we delete all whitespace below
ls_re = re.compile(r"TotalObjects:([0-9]+)TotalSize:([0-9]+)")
buckets = s3_access.bucket_list()
self.info_out(f"found {len(buckets)} buckets")
count = 0
exclusions_no_comments = ExclusionPreferences(
BackupTypes.S3.name).get_no_comments()
for bucket_name in buckets:
# do the sync
if bucket_name in exclusions_no_comments:
self.info_out(f"excluding {bucket_name}")
else:
if dry_run:
self.info_out(f"dry run {bucket_name}")
else:
self.info_out(f"{bucket_name}")
# try to find the AWS CLI app
paths = [
(Path("venv", "Scripts", "python.exe").absolute(),
Path("venv", "Scripts", "aws").absolute()), # local venv
(Path("python.exe").absolute(),
Path("Scripts", "aws").absolute()) # installed app
]
aws_cli_path = None
python_path = None
for p, a in paths:
if p.exists() and a.exists():
aws_cli_path = a
python_path = p
break
if aws_cli_path is None:
log.error(f"AWS CLI executable not found ({paths=})")
elif python_path is None:
log.error(f"Python executable not found ({paths=})")
else:
aws_cli_path = f'"{str(aws_cli_path)}"' # from Path to str, with quotes for installed app
# AWS CLI app also needs the python executable to be in the path if it's not in the same dir, which happens when this program is installed.
# Make the directory of our python.exe the first in the list so it's found and not any of the others that may or may not be in the PATH.
env_var = deepcopy(os.environ)
env_var[
"path"] = f"{str(python_path.parent)};{env_var.get('path', '')}"
destination = Path(backup_directory, bucket_name)
os.makedirs(destination, exist_ok=True)
s3_bucket_path = f"s3://{bucket_name}"
# Don't use --delete. We want to keep 'old' files locally.
sync_command_line = [
aws_cli_path, "s3", "sync", s3_bucket_path,
str(destination.absolute())
]
if dry_run:
sync_command_line.append("--dryrun")
sync_command_line_str = " ".join(sync_command_line)
log.info(sync_command_line_str)
try:
sync_result = subprocess.run(sync_command_line_str,
stdout=subprocess.PIPE,
shell=True,
env=env_var)
except FileNotFoundError as e:
self.error_out(
f'error executing {" ".join(sync_command_line)} {e}'
)
return
for line in sync_result.stdout.decode(
decoding).splitlines():
log.info(line.strip())
# check the results
ls_command_line = [
aws_cli_path, "s3", "ls", "--summarize", "--recursive",
s3_bucket_path
]
ls_command_line_str = " ".join(ls_command_line)
log.info(ls_command_line_str)
ls_result = subprocess.run(ls_command_line_str,
stdout=subprocess.PIPE,
shell=True,
env=env_var)
ls_stdout = "".join([
c for c in ls_result.stdout.decode(decoding)
if c not in " \r\n"
]) # remove all whitespace
if len(ls_stdout) == 0:
self.error_out(
f'"{ls_command_line_str}" failed ({ls_stdout=}) - check internet connection'
)
else:
ls_parsed = ls_re.search(ls_stdout)
if ls_parsed is None:
self.error_out(
f"parse error:\n{ls_command_line_str=}\n{ls_stdout=}"
)
else:
count += 1
s3_object_count = int(ls_parsed.group(1))
s3_total_size = int(ls_parsed.group(2))
local_size, local_count = get_dir_size(destination)
# rough check that the sync worked
if s3_total_size > local_size:
# we're missing files
message = "not all files backed up"
output_routine = self.error_out
elif s3_total_size != local_size:
# Compare size, not number of files, since aws s3 sync does not copy files of zero size.
message = "mismatch"
output_routine = self.warning_out
else:
message = "match"
output_routine = log.info
output_routine(
f"{bucket_name} : {message} (s3_count={s3_object_count}, local_count={local_count}; s3_total_size={s3_total_size}, local_size={local_size})"
)
self.info_out(
f"{len(buckets)} buckets, {count} backed up, {len(exclusions_no_comments)} excluded"
)
def closeEvent(self, close_event: QCloseEvent) -> None:
self.run_backup_widget.save_state()
preferences = get_preferences(UITypes.gui)
preferences.width = self.width()
preferences.height = self.height()
def test_preferences():
# todo: this currently just tests what's ever in the preferences, which can be nothing - in the future add writing test data
gui_preferences = get_preferences(UITypes.cli)
pprint(gui_preferences)