def sync():
"""Synchronize files modified in last x days from the bucket to the media folder"""
b2 = B2Api(InMemoryAccountInfo())
app_key_id = get_snap_config('b2-application-key-id')
app_key = get_snap_config('b2-application-key')
b2.authorize_account('production', app_key_id, app_key)
exclude_before_timestamp = int(time.time()) - (int(get_snap_config('remove-after-days')) * 86400)
policies_manager = ScanPoliciesManager(
exclude_file_regexes=("index.html",),
exclude_modified_before=exclude_before_timestamp * 1000 # in ms
)
synchronizer = Synchronizer(
max_workers=5,
newer_file_mode=NewerFileSyncMode.SKIP,
policies_manager=policies_manager
)
bucket_uri = 'b2://' + get_snap_config('b2-bucket')
source = parse_sync_folder(bucket_uri, b2)
destination = parse_sync_folder(MEDIA_DIR, b2)
if not os.path.isdir(MEDIA_DIR):
os.makedirs(MEDIA_DIR)
with SyncReport(sys.stdout, False) as reporter:
synchronizer.sync_folders(
source_folder=source,
dest_folder=destination,
now_millis=time.time() * 1000, # in ms
reporter=reporter
)
def sync_down(local, remote, **kwargs):
bucket, path = str(remote).split(':')
workers = multiprocessing.cpu_count()
syncer = b2.Synchronizer(workers, **kwargs)
with b2.SyncReport(sys.stdout, False) as reporter:
syncer.sync_folders(
source_folder=b2.parse_sync_folder(f'b2://{bucket}/{path}', api(bucket)),
dest_folder=b2.parse_sync_folder(str(local), api(bucket)),
now_millis=int(round(time.time() * 1000)),
reporter=reporter)
def upload(self, src: path_like_obj, dst: str) -> bool:
api = self._get_api()
parsed_destination = b2.parse_sync_folder(dst, api)
if not isinstance(parsed_destination, b2.B2Folder):
raise ValueError('dst does not start with b2://')
bucket: b2.Bucket = parsed_destination.bucket
us = b2.UploadSourceLocalFile(os.fspath(src))
sha1 = us.get_content_sha1()
# Try to get file info of the existing file
try:
file_info = bucket.get_file_info_by_name(parsed_destination.folder_name)
if file_info.content_sha1 == sha1:
# No need to upload
return False
except b2exc.FileNotPresent:
pass
parsed_destination.bucket.upload(
us,
parsed_destination.folder_name,
file_info={'large_file_sha1': sha1})
return True
def sync_dir(self, srcdir: Path, basedir: Optional[Path]) -> str:
"""Sync the entire tmpdir (see rsync_files) to Backblaze
"""
destpath = Path(self.bucket.name) / basedir
dest = f'b2://{destpath}'
source = parse_sync_folder(str(srcdir), self.b2_api)
destination = parse_sync_folder(dest, self.b2_api)
synchronizer = Synchronizer(
max_workers=8,
policies_manager=ScanPoliciesManager(exclude_all_symlinks=True),
dry_run=False,
allow_empty_source=True,
compare_version_mode=CompareVersionMode.SIZE,
compare_threshold=1
)
# return stdout as synchronizer
with SyncReport(sys.stdout, no_progress=0) as reporter:
synchronizer.sync_folders(
source_folder=source,
dest_folder=destination,
now_millis=int(round(time.time() * 1000)),
reporter=reporter,
)
yield reporter
def download_if_exists(self, src: str, dst: path_like_obj) -> bool:
api = self._get_api()
parsed_src = b2.parse_sync_folder(src, api)
if not isinstance(parsed_src, b2.B2Folder):
raise ValueError('src does not start with b2://')
bucket: b2.Bucket = parsed_src.bucket
download_dest = b2.DownloadDestLocalFile(dst)
try:
bucket.download_file_by_name(parsed_src.folder_name, download_dest)
except b2exc.FileNotPresent:
return False
return True
b2_api.authorize_account("production", application_key_id, application_key)
source = cfg.SCRIPT_DIR
source_tmp = os.path.join(cfg.SCRIPT_DIR, '.tmp_BU')
if os.path.exists(source_tmp):
shutil.rmtree(source_tmp)
shutil.copytree(source,
source_tmp,
symlinks=False,
ignore=shutil.ignore_patterns(
'.*', 'bot-env', '__pycache__'
))
source = b2.parse_sync_folder(source_tmp, b2_api)
destination = b2.parse_sync_folder(destination, b2_api)
policies_manager = b2.ScanPoliciesManager(
exclude_dir_regexes=("bot-env", ".git", ".*__(.*)__.*", "^\\.(.*)"),
exclude_file_regexes=("^\\.(.*)", ".*__pycache__.*", ".*\\.txt"),
exclude_all_symlinks=True
)
synchronizer = b2.Synchronizer(
max_workers=10,
policies_manager=policies_manager,
dry_run=False,
allow_empty_source=True,
newer_file_mode=b2.NewerFileSyncMode.REPLACE,
keep_days_or_delete=b2.KeepOrDeleteMode.DELETE,