def join_files(fpath, output_fpath, verbose, notify):
"""
Concatenate multiple audio files with FFmpeg.
"""
args, result = pydoni.__pydonicli_declare_args__(locals()), dict()
assert len(fpath) > 1, 'Must pass more than one file to join!'
vb = Verbose(verbose)
fpaths = list(fpath)
vb.info(
f'Joining {len(fpaths)-1} additional audio files onto base file "{fpaths[0]}"...'
)
if output_fpath is None:
output_fpath = pydoni.append_filename_suffix(fpaths[0], '-JOINED')
audio = pydoni.AudioFile(audio_fpath=fpaths[0])
ffmpeg_output = audio.join(additional_audio_files=fpaths[1:],
output_fpath=output_fpath)
vb.info(f'Output audiofile created "{output_fpath}"')
if notify:
pydoni.macos_notify(title='Audiofile Join Complete!')
outfile_size_in_bytes = stat(output_fpath).st_size
result = dict(joined_files=fpaths,
output_fpath=output_fpath,
outfile_size_in_bytes=outfile_size_in_bytes,
ffmpeg_output=ffmpeg_output)
pydoni.__pydonicli_register__(
dict(args=args, result=result, command_name='audio.join_files'))
def remove_empty_subfolders(root, recursive, true_remove, count_hidden_files,
verbose):
"""
Scan a directory and delete any empty bottom-level directories.
"""
args, result = pydoni.__pydonicli_declare_args__(locals()), dict()
dpath = list(root)
vb = Verbose(verbose)
removed_dirs = []
for root in dpath:
pydoni.remove_empty_subfolders(root=root,
recursive=recursive,
true_remove=true_remove,
count_hidden_files=count_hidden_files)
removed_dirs += root
if verbose:
if len(removed_dirs):
for dir in removed_dirs:
vb.info('Removed: ' + dir)
else:
vb.info('No empty directories found', level='warn')
result['removed_dirs'] = removed_dirs
pydoni.__pydonicli_register__(
dict(args=args,
result=result,
command_name='opsys.remove_empty_subfolders'))
def to_mp3(fpath, verbose, notify):
"""
Convert an audio file to .mp3 format.
"""
args, result = pydoni.__pydonicli_declare_args__(locals()), dict()
vb = Verbose(verbose)
fpaths = list(fpath)
output_fpaths = []
for f in fpaths:
vb.info(f'Converting file "{f}" to MP3')
audio = pydoni.AudioFile(f)
output_fpath = output_fpath = splitext(f)[0] + '-CONVERTED.mp3'
audio.convert(output_format='mp3', output_fpath=output_fpath)
vb.info(f'Outputted .mp3 file "{output_fpath}"')
output_fpaths.append(output_fpath)
if notify:
pydoni.macos_notify(message='Audiofile compression complete',
title='Pydoni-CLI')
result = {'fpaths': fpaths, 'output_fpaths': output_fpaths}
result = dict(fpaths=fpaths)
pydoni.__pydonicli_register__(
dict(args=args, result=result, command_name='audio.to_mp3'))
def du_by_filetype(dpath, output_fpath, recursive, quiet, human_readable,
progress):
"""
List the total filesize in a directory by file type.
"""
args, result = pydoni.__pydonicli_declare_args__(locals()), dict()
dpaths = list(dpath)
write_lst = []
for d in dpaths:
write_lst.append(f'Directory "{d}"')
filesize_dct = pydoni.du_by_filetype(dpath=d,
recursive=recursive,
human_readable=human_readable,
progress=progress)
for ftype, fsize in filesize_dct.items():
write_lst.append(f'{ftype}: {fsize}')
# Print to console
if not quiet:
for item in write_lst:
print(item)
# Write output file
write_output = True if output_fpath is not None else False
if write_output:
with open(output_fpath, 'a') as f:
for item in write_lst:
f.write(item + '\n')
result['result'] = filesize_dct
pydoni.__pydonicli_register__(
dict(args=args, result=result, command_name='opsys.du_by_filetype'))
def instagram_hashtags(austin,
sf,
sony,
landscape,
long,
night,
dji,
limit=28):
"""
Create Instagram hashtag comment string given lists of instagram keywords. Allow user
to choose which keyword groups to use, and to set a limit on the number of hashtags
returned. Then generate a comment string from those selected keywords.
"""
params = locals()
options = {k: v for k, v in params.items() if k not in ['limit']}
args, result = pydoni.__pydonicli_declare_args__(locals()), dict()
if sum([v for k, v in options.items()]) == 0:
raise Exception(
'Must select at least one option! Possibilities: {}'.format(
[k for k, v in options.items()]))
hashtags = define_hashtags()
keyword_groups = list({k for k, v in params.items() if v is True})
keywords = [v for k, v in hashtags.items() if k in keyword_groups]
keywords = [item for sublist in keywords for item in sublist]
random.shuffle(keywords)
if limit < len(keywords):
keywords = keywords[:limit]
keywords = ['#' + x for x in keywords]
spaces = []
space_chars = ['·', '.', '-', '*', '~']
for i in range(1, 6):
spaces.append(''.join(random.choices(space_chars, k=1)) * i)
keyword_string = '\n'.join(spaces) + '\n' + ' '.join(keywords)
print('Here are your selected Instagram keywords:')
print()
print(keyword_string)
result = {
'keyword_groups': keyword_groups,
'keyword_string': keyword_string
}
pydoni.__pydonicli_register__(
dict(args=args, result=result,
command_name='photo.instagram_hashtags'))
def text_to_speech(input, output_fpath, verbose, notify):
"""
Convert raw text, either as commandline input or file input, to speech using gTTS.
"""
args, result = pydoni.__pydonicli_declare_args__(locals()), dict()
vb = Verbose(verbose)
input = list(input)
output = list(output_fpath)
assert len(input), 'Must pass --input'
assert len(output), 'Must pass --output'
if output[0] is None:
output = [None] * len(input)
assert len(input) == len(
output), '--input and --output-fpath must be of the same length'
for i, o in zip(input, output):
if isfile(i):
# Input is file
assert splitext(i)[1].lower(
) == '.txt', f'Input file "{i}" must have extension ".txt"'
with open(i, 'r') as f:
text_to_speechify = f.read()
if o is None:
# Default output file is the same as the input file but with .mp3 extension
o = splitext(i)[0] + '.mp3'
else:
# Input is text
text_to_speechify = i
if o is None:
# Default output file is a generic file on desktop
o = join(
expanduser('~'), 'Desktop',
f'text_to_speech_{pydoni.systime(as_string=True)}.mp3')
speech = gTTS(text=text_to_speechify, lang='en', slow=False)
speech.save(o)
if notify:
pydoni.macos_notify(message='Text to speech complete',
title='Pydoni-CLI')
pydoni.__pydonicli_register__(
dict(args=args, result=result, command_name='audio.text_to_sp'))
def append_backup_log_table(table_schema,
table_name,
source,
source_size_bytes,
target,
target_size_before_bytes,
target_size_after_bytes,
start_ts,
end_ts,
is_completed,
verbose):
"""
Append a record to directory backup Postgres table. To be used if a backup is carried
out without the use of the `pydoni data backup` command which handles the table insert
automatically, but when the backup would still like to be logged in the log table.
"""
args, result = pydoni.__pydonicli_declare_args__(locals()), dict()
vb = Verbose(verbose)
pg = pydoni.Postgres()
sql_value_dct = dict(source=source,
source_size_bytes=source_size_bytes,
target=target,
target_size_before_bytes=target_size_before_bytes,
target_size_after_bytes=target_size_after_bytes,
start_ts=datetime.datetime.fromtimestamp(start_ts),
end_ts=datetime.datetime.fromtimestamp(end_ts),
is_completed=is_completed)
vb.info(f'table_schema: {table_schema}')
vb.info(f'table_name: {table_name}')
for k, v in sql_value_dct.items():
vb.info(f'{k}: {v}')
insert_sql = pg.build_insert(schema_name=table_schema,
table_name=table_name,
columns=[k for k, v in sql_value_dct.items()],
values=[v for k, v in sql_value_dct.items()])
pg.execute(insert_sql)
vb.info(f'Appended record to {table_schema}.{table_name}')
result['sql_value_dct'] = sql_value_dct
vb.program_complete('Append to backup log table complete')
pydoni.__pydonicli_register__(dict(args=args, result=result, command_name='data.append_backup_log_table'))
def workflow(source_dpath, pg_schema, sample, full_rebuild, verbose, dry_run,
no_startup_message, no_pipeline):
"""
Refresh Postgres Photo schema from source photo metadata.
"""
args = pydoni.__pydonicli_declare_args__(locals())
pydoni.__pydonicli_register__({
'command_name':
pydoni.what_is_my_name(with_modname=True),
'args':
args
})
# Begin pipeline stopwatch
start_ts = time.time()
# Set up variables used throughout entire pipeline
vb = Verbose(verbose)
pg = pydoni.Postgres()
if vb.verbose:
if not no_startup_message:
print_startup_message()
# Extract source media file metadata and load into Postgres
pipeline_args = dict(pg=pg,
vb=vb,
source_dpath=source_dpath,
pg_schema=pg_schema,
sample=sample,
full_rebuild=full_rebuild)
if not no_pipeline:
source_to_postgres_pipeline(**pipeline_args)
# Apply transformations on data once loaded into Postgres
db_transform(pg, vb, pg_schema, sample)
# End workflow
pydoni.__pydonicli_register__(
{k: v
for k, v in locals().items() if k in ['result']})
msg = f'Photo database refresh complete'
vb.program_complete(msg, start_ts=start_ts)
def split_batch_exported_timelapse(dpath):
"""
Split a directory of exported timelapse stills into their respective folders.
Accept a directory of exported timelapse stills from Lightroom following Andoni's
photo file naming convention. Those timelapse files will be in sequences numbered
from 1 to the final # of the timelapse series, then will reset to 1 for the next
timelapse. There could be an arbitrary number of timelapse stills in this directory.
This program will take all the files in that directory and split them into folders
for easy categorization.
"""
args, result = pydoni.__pydonicli_declare_args__(locals()), dict()
os.chdir(dpath)
files = pydoni.listfiles(ext='jpg')
assert len(files), "No timelapse stills found!"
seqnums = [int(os.path.splitext(f.split('_')[4])[0]) for f in files]
differences = []
for i, num in enumerate(seqnums):
last_idx = i - 1
last_idx = last_idx if last_idx >= 0 else 0
last_num = seqnums[last_idx]
differences.append(num - last_num)
delimiters = [i for i, x in enumerate(differences) if x not in [0, 1]]
files_list_of_lists = pydoni.split_at(files, delimiters)
for i, list_of_files in enumerate(files_list_of_lists):
dname = 'timelapse_%s_of_%s' % (str(i + 1),
str(len(files_list_of_lists)))
if not os.path.isdir(dname):
os.mkdir(dname)
for fname in list_of_files:
newfname = os.path.join(dname, fname)
os.rename(fname, newfname)
result['directories_created'] = len(files_list_of_lists)
pydoni.__pydonicli_register__(
dict(args=args,
result=result,
command_name='photo.split_batch_exported_timelapse'))
def ocr(fpath, verbose):
"""
OCR an image using pytesseract.
"""
args, result = pydoni.__pydonicli_declare_args__(locals()), dict()
vb = Verbose(verbose)
fpaths = list(fpath)
for fpath in fpaths:
vb.info(f'Applying OCR to file "{fpath}"...')
text = pydoni.ocr_image(fpath)
with open(splitext(fpath)[0] + '.txt', 'w') as f:
f.write(text)
vb.info("Successfully OCR'd file")
result['ocr_files'] = fpaths
pydoni.__pydonicli_register__(
dict(args=args, result=result, command_name='image.ocr'))
def compress(fpath, verbose, notify):
"""
Compress one or more audiofiles.
"""
args, result = pydoni.__pydonicli_declare_args__(locals()), dict()
vb = Verbose(verbose)
fpaths = list(fpath)
for fpath in fpaths:
ffmpeg_output = pydoni.AudioFile(fpath).compress()
vb.info(f'Successfully compressed file "{fpath}"')
if notify:
pydoni.macos_notify(title='M4A to MP3 Conversion Complete!')
result['compressed_files'] = fpaths
result['ffmpeg_output'] = ffmpeg_output
pydoni.__pydonicli_register__(
dict(args=args, result=result, command_name='audio.compress'))
def to_gif(fpath, verbose, notify):
"""
Convert video file(s) to GIF.
"""
args, result = pydoni.__pydonicli_declare_args__(locals()), dict()
vb = Verbose(verbose)
fpaths = list(fpath)
for f in fpaths:
gif_fpath = os.path.splitext(f)[0] + '.gif'
pydoni.video_to_gif(video_fpath=f, gif_fpath=gif_fpath, fps=10)
vb.echo(f'Outputted .gif file "{gif_fpath}"')
result[f] = gif_fpath
if notify:
pydoni.macos_notify(message='Conversion complete',
title='Video to GIF')
pydoni.__pydonicli_register__(
dict(args=args, result=result, command_name='video.to_gif'))
def refresh_imdb_table(schema_name, table_name, omdbapikey, verbose=False):
"""
Query Postgres table containing IMDB metadata and refresh any values that need updating.
"""
args, result = pydoni.__pydonicli_declare_args__(locals()), dict()
# 'result' will be a dictionary where the movie names are the keys, and the values are
# dictionaries with items: 'status', 'message', 'updated_values' (dictionary of
# updated values, if any).
result_items = ['status', 'message', 'updated_values']
pg = pydoni.Postgres()
pkey_name = 'movie_id'
df = pg.read_table(schema_name, table_name).sort_values(pkey_name)
cols = pg.col_names(schema_name, table_name)
if verbose:
pbar = tqdm(total=len(df), unit='movie')
for i, row in df.iterrows():
movie_name = f"{row['title']} ({str(row['release_year'])})"
try:
omdbresp = query_omdb(title=row['title'],
release_year=row['release_year'],
omdbapikey=omdbapikey)
except Exception as e:
err_str = f"{click.style('ERROR', fg='red')} in {movie_name}: {str(e)}"
tqdm.write(err_str)
result[movie_name] = {
k: v
for k, v in zip(result_items, ['Error', str(e), None])
}
if verbose:
pbar.update(1)
continue
omdbresp = {k: v for k, v in omdbresp.items() if k in cols}
omdbresp = {k: replace_null(v) for k, v in omdbresp.items()}
color_map = {
'No change': 'yellow',
'Updated': 'green',
'Not found': 'red'
}
change = 'Not found' if not len(omdbresp) else 'No change'
# Filter out columns and values that do not require an update
if change != 'Not found':
upd = filter_updated_values(omdbresp, row)
change = 'Updated' if len(upd) else change
upd['imdb_update_ts'] = datetime.datetime.now()
stmt = pg.build_update(schema_name,
table_name,
pkey_name=pkey_name,
pkey_value=row[pkey_name],
columns=[k for k, v in upd.items()],
values=[v for k, v in upd.items()],
validate=True)
pg.execute(stmt)
upd_backend = {
k: v
for k, v in upd.items() if k != 'imdb_update_ts'
}
upd_backend = upd_backend if len(upd_backend) else None
result[movie_name] = {
k: v
for k, v in zip(result_items, [change, None, upd_backend])
}
else:
result[movie_name] = {
k: v
for k, v in zip(result_items, [change, None, None])
}
if verbose:
pbar.update(1)
space = ' ' if change == 'Updated' else ''
tqdm.write(
click.style(change, fg=color_map[change]) + space + ': ' +
movie_name)
if verbose:
pbar.close()
pydoni.program_complete('Movie refresh complete!')
pydoni.__pydonicli_register__(
dict(args=args, result=result,
command_name='movie.refresh_imdb_table'))
def website_extract_image_titles(website_export_dpath, outfile, verbose):
"""
Scan photo files exported for andonisooklaris.com and construct list of image filenames
and titles, separated by collection.
"""
args, result = pydoni.__pydonicli_declare_args__(locals()), dict()
def echo(*args, **kwargs):
kwargs['timestamp'] = True
pydoni.echo(*args, **kwargs)
website_export_dpath = expanduser(website_export_dpath)
if outfile == 'auto':
outfile = os.path.join(
website_export_dpath,
'Image Titles %s.txt' % pydoni.sysdate(stripchars=True))
elif outfile is not None:
assert not os.path.isfile(outfile)
files = pydoni.listfiles(path=website_export_dpath,
recursive=True,
full_names=True)
files = [f for f in files if os.path.splitext(f)[1].lower() != '.txt']
if verbose:
echo('Files found: ' + str(len(files)))
echo('Extracting EXIF metadata...')
exifd = pydoni.EXIF(files).extract()
echo('EXIF metadata successfully extracted')
if outfile is not None:
echo('Writing output datafile: ' + outfile)
else:
exifd = pydoni.EXIF(files).extract()
i = 0
tracker = pd.DataFrame(columns=['collection', 'file', 'title'])
for file in files:
elements = file.replace(website_export_dpath,
'').lstrip('/').split('/')
subcollection = None
collection = elements[0]
fname = elements[-1]
if len(elements) == 3:
subcollection = elements[1]
collection += ' - ' + subcollection
exif = exifd[os.path.join(website_export_dpath, file)]
title = exif['Title'] if 'Title' in exif.keys() else ''
year = fname[0:4]
title = str(year) + ' ' + str(title)
tracker.loc[i] = [collection, fname, title]
i += 1
print_lst = []
for collection in tracker['collection'].unique():
print_lst.append('\nCollection: %s\n' % collection)
df_print = tracker.loc[tracker['collection'] == collection].drop(
'collection', axis=1)
print_lst.append(
tabulate(df_print, showindex=False, headers=df_print.columns))
print_str = '\n'.join(print_lst).strip()
if outfile is None:
print(print_str)
else:
with open(outfile, 'w') as f:
f.write(print_str)
if verbose:
pydoni.program_complete()
result['n_collections'] = len(tracker['collection'].unique())
pydoni.__pydonicli_register__(
dict(args=args,
result=result,
command_name='photo.website_extract_image_titles'))
def rename_mediafile(fpath, initials, tz_adjust, verbose, notify):
"""
Rename a photo or video file according to a specified file naming convention.
fpath {str} or {list}: filename or list of filenames to rename
initials {str}: 2 or 3 letter initials string
notify {bool}: execute `pydoni.opsys.macos_notify()` on program completion
tz_adjust {int}: adjust file creation times by a set number of hours
verbose {bool}: print messages and progress bar to console
"""
args, result = pydoni.__pydonicli_declare_args__(locals()), dict()
assert len(initials) in [2,
3], 'Initials must be a 2- or 3-character string'
vb = Verbose(verbose)
mediafiles = list(fpath)
assert len(mediafiles), 'No mediafiles to rename!'
assert isinstance(mediafiles[0], str), \
f'First element of variable `mediafiles` is of type {type(mediafiles[0]).__name__}, expected string'
CONV = Convention()
EXT = Extension()
mediafiles = [os.path.abspath(x) for x in mediafiles \
if not re.match(CONV.photo, os.path.basename(x)) \
and not re.match(CONV.video, os.path.basename(x))]
msg = f'Renaming {len(mediafiles)} media files'
if verbose:
vb.section_header(msg)
pbar = tqdm(total=len(mediafiles), unit='mediafile')
if not len(mediafiles):
if verbose:
pydoni.echo('No files to rename!', fg='green')
for mfile in mediafiles:
if verbose:
vb.stabilize_postfix(mfile, max_len=15)
mf = MediaFile(mfile)
newfname = mf.build_fname(initials=initials, tz_adjust=tz_adjust)
newfname = os.path.join(os.path.dirname(mfile),
os.path.basename(newfname))
if os.path.basename(mfile) != os.path.basename(newfname):
os.rename(mfile, newfname)
result[os.path.basename(mfile)] = os.path.basename(newfname)
if verbose:
tqdm.write('{}: {} -> {}'.format(
click.style('Renamed', fg='green'),
os.path.basename(mfile), os.path.basename(newfname)))
else:
result[os.path.basename(
mfile)] = '<not renamed, new filename identical>'
if verbose:
tqdm.write('{}: {}'.format(
click.style('Not renamed', fg='red'),
os.path.basename(mfile)))
if verbose:
pbar.update(1)
if verbose:
pbar.close()
pydoni.echo(f'Renamed media files: {len(mediafiles)}', indent=2)
if verbose or notify:
pydoni.macos_notify(title='Mediafile Rename',
message='Completed successfully!')
pydoni.__pydonicli_register__(
dict(args=args, result=result, command_name='photo.rename_mediafile'))
def backup(source, target, update_log_table, use_rsync, verbose, debug, dry_run):
"""
Back up a source directory to a target directory.
This function will accept a source and target directories, most often
on separate external hard drives, and copy all files from the source
to the target that are either:
(1) Not in the target directory
(2) Are in the target directory, but have been updated
Files in the target that have been deleted in the source will also be deleted.
"""
args, result = pydoni.__pydonicli_declare_args__(locals()), dict()
start_ts = time.time()
vb = Verbose(verbose=verbose, debug=debug)
ws = ' '
ignore_files = [
'The Office S09E16 Moving On.mkv',
'The Office S09E20 Paper Airplanes.mkv',
]
if update_log_table:
start_ts_utc = datetime.datetime.utcnow()
pg = pydoni.Postgres()
directory_backup_table_schema = 'pydonicli'
directory_backup_table_name = 'directory_backup'
insert_dict = dict(source=source,
source_size_bytes=stat(source).st_size,
target=target,
target_size_before_bytes=stat(target).st_size,
target_size_after_bytes=None,
start_ts=start_ts_utc,
is_completed=False)
insert_sql = pg.build_insert(schema_name=directory_backup_table_schema,
table_name=directory_backup_table_name,
columns=list(insert_dict.keys()),
values=list(insert_dict.values()),
validate=True)
if not dry_run:
pg.execute(insert_sql)
directory_backup_id = pg.read_sql(f"""
select directory_backup_id
from {directory_backup_table_schema}.{directory_backup_table_name}
order by gen_ts desc
limit 1""").squeeze()
assert source != target, 'Source and target directories must be different'
if use_rsync:
cmd_lst = ['rsync', '--delete-before', '-a', '-h', '-u']
if verbose:
cmd_lst = cmd_lst + ['-v', '--progress']
cmd_lst = cmd_lst + [f'"{source}"'] + [f'"{target}"']
cmd = ' '.join(cmd_lst)
subprocess.call(cmd, shell=True)
# progress_flag = ' --progress' if verbose else ''
# backup_cmd = f'rsync -avhu{progress_flag} --delete-before "{source}" "{target}"'
# subprocess.call(backup_cmd, shell=True)
else:
vb.info(f'Listing files at source: {source}')
files_source = pydoni.listfiles(path=source, recursive=True, full_names=True)
vb.debug('Found files at source: ' + str(len(files_source)))
files_source = [x for x in files_source if x not in ignore_files]
vb.debug(f'Found files at source after filtering out manually ignored files: {len(files_source)}')
vb.info(f'Listing files at target: {target}')
files_target = pydoni.listfiles(path=target, recursive=True, full_names=True)
vb.debug('Found files at target: ' + str(len(files_target)))
files_target = [x for x in files_target if x not in ignore_files]
vb.debug(f'Found files at target after filtering out manually ignored files: {len(files_target)}')
# Scan source files and for each determine whether to do nothing, copy to target,
# or replace at target
copied_files = []
replaced_files = []
vb.info('Scanning for new, updated or deleted files at source')
vb.pbar_init(total=len(files_source), unit='file')
for sourcefile in files_source:
vb.pbar_write(f'Sourcefile: {sourcefile}', refer_debug=True)
vb.pbar.set_postfix({'file': basename(sourcefile)})
targetfile = sourcefile.replace(source, target)
vb.pbar_write(f'{ws}Expected mirrored targetfile: {targetfile}', refer_debug=True)
if not isfile(targetfile):
# Copy file to target. Create parent directory at target if not exists
vb.pbar_write(f'{ws}(Copy) attempting to copy file "{sourcefile}" to "{targetfile}"', refer_debug=True)
targetdpath = dirname(targetfile)
if not isdir(targetdpath):
vb.pbar_write(f'{ws}{ws}Parent directory of targetfile does not exist, creating it at: ' + targetdpath, refer_debug=True)
if not dry_run:
makedirs(targetdpath)
vb.pbar_write(f'{ws}{ws}Successful', refer_debug=True)
if not dry_run:
shutil.copy2(sourcefile, targetfile)
vb.pbar_write(f'{ws}Successful', refer_debug=True)
copied_files.append(sourcefile)
elif isfile(targetfile) and is_file_changed(sourcefile, targetfile):
# Replace file at target (same action as copy, but parent directory must exist)
vb.pbar_write(f'(Replace) attempting to copy file "{sourcefile}" to "{targetfile}"', refer_debug=True)
if not dry_run:
shutil.copy2(sourcefile, targetfile)
vb.pbar_write(f'Successful', refer_debug=True)
replaced_files.append(sourcefile)
else:
vb.pbar_write(f'{ws}Targetfile already exists and is unchanged', refer_debug=True)
vb.pbar_update(1)
vb.pbar_close()
# Scam target files and for each determine whether that file has been since
# deleted from source
deleted_files = []
vb.info('Scanning for files at target since deleted from source')
vb.pbar_init(total=len(files_target))
for targetfile in files_target:
sourcefile = targetfile.replace(target, source)
vb.pbar.set_postfix({'file': basename(targetfile)})
if not isfile(sourcefile) and not isdir(sourcefile):
vb.pbar_write(f'(Delete) attempting to delete "{targetfile}"', refer_debug=True)
if not dry_run:
send2trash(targetfile)
vb.pbar_write(f'{ws}Successful', refer_debug=True)
deleted_files.append(targetfile)
vb.pbar_update(1)
vb.pbar_close()
# Record number of files copied, replaced and deleted
vb.info(f'Copied {len(copied_files)} files')
vb.info(f'Replaced {len(replaced_files)} files')
vb.info(f'Deleted {len(deleted_files)} files')
vb.info(f'Unchanged {len(files_source) - len(copied_files) - len(replaced_files) - len(deleted_files)} files')
result = dict(copied=len(copied_files),
replaced=len(replaced_files),
deleted=len(deleted_files),
unchanged=len(files_source) - len(copied_files) - len(replaced_files) - len(deleted_files))
if update_log_table:
vb.debug('Attempting to update log table with results...')
update_dict = dict(target_size_after_bytes=pydoni.dirsize(target),
end_ts=datetime.datetime.utcnow(),
is_completed=True)
update_sql = pg.build_update(schema_name=directory_backup_table_schema,
table_name=directory_backup_table_name,
pkey_name='directory_backup_id',
pkey_value=directory_backup_id,
columns=list(update_dict.keys()),
values=list(update_dict.values()),
validate=True)
if not dry_run:
pg.execute(update_sql)
vb.debug(f'{ws}Successful')
vb.program_complete('Backup complete', start_ts=start_ts)
pydoni.__pydonicli_register__(dict(args=args, result=result, command_name='data.backup'))
def pg_dump(backup_dir,
db_name,
pg_user,
sep,
pgdump,
csvdump,
max_dir_size,
dry_run,
verbose):
"""
Dump a local Postgres database. Looks for ~/.pgpass by default.
"""
args, result = pydoni.__pydonicli_declare_args__(locals()), dict()
vb = Verbose(verbose)
if dry_run:
vb.info('Not executing any code (dry run)')
if pg_user is not None and db_name is not None:
pg = pydoni.Postgres(pg_user=pg_user, db_name=db_name)
else:
# Attempt to parse ~/.pgpass file. Fail if this file does not exist or is not
# able to be parsed
pg = pydoni.Postgres()
# Define subfolder to dump files to within dump directory
subdir = pydoni.systime(compact=True) + '_' + pg.db_name
backup_subdir = join(expanduser(backup_dir), subdir)
mkdir(backup_subdir)
vb.info('Database: ' + pg.db_name)
vb.info('Destination folder: ' + backup_subdir)
# Dump database based on user's preference
# May dump using pg_dump, export tables to CSV, or both
dumped_files = []
if pgdump:
vb.info('Executing `pg_dump`')
if not dry_run:
dumped_dbfile = pg.dump(backup_dir=backup_subdir)
dumped_files += [dumped_dbfile]
if csvdump:
# Dump each file to textfile
vb.info('Executing CSV dump to tables')
if not dry_run:
dumped_csvfiles = pg.dump_tables(backup_dir=backup_subdir, sep=sep, coerce_csv=False)
dumped_files += dumped_csvfiles
result['backup_directory'] = backup_subdir
result['dumped_files'] = {}
for f in dumped_files:
result['dumped_files'][basename(f)] = dict(
filesize=stat(f).st_size,
filesize_readable=pydoni.human_filesize(stat(f).st_size),
created=datetime.datetime.fromtimestamp(getctime(f)).strftime('%Y-%m-%d %H:%M:%S.%f'),
rows=pydoni.textfile_len(f))
if verbose:
vb.line_break()
tt_list = [[basename(file),
infodict['created'],
pydoni.human_filesize(infodict['filesize']),
str(infodict['rows'])
] for file, infodict in result['dumped_files'].items()]
if len(tt_list):
if verbose:
print(tt.to_string(
tt_list,
header=[click.style(x, bold=True) for x in ['File', 'Created', 'Size', 'Rows']],
style=tt.styles.ascii_thin_double,
padding=(0, 1),
alignment='ccrr'))
else:
vb.warn('No database files were dumped!')
if dry_run:
rmdir(backup_subdir)
max_dir_size_enforced = False
removed_old_backup_dirs = []
if max_dir_size:
# Check size of `backup_dir` and clear any backup directories until the total size
# is less than max_dir_size (upper GB limit)
subdirs = sorted([x for x in pathlib.Path(backup_dir).iterdir() if isdir(x)], key=getmtime)
subdirs_size = zip(subdirs, [pydoni.dirsize(x) / 1e9 for x in subdirs])
total_size = sum([y for x, y in subdirs_size])
if total_size > max_dir_size:
vb.warn(f'Enforcing maximum directory size: {str(max_dir_size)} GB')
max_dir_size_enforced = True
while total_size > max_dir_size:
dir_to_remove = str(subdirs[0])
shutil.rmtree(dir_to_remove)
removed_old_backup_dirs.append(dir_to_remove)
subdirs = sorted([x for x in pathlib.Path(backup_dir).iterdir() if isdir(x)], key=getmtime)
subdirs_size = zip(subdirs, [pydoni.dirsize(x) / 1e9 for x in subdirs])
total_size = sum([y for x, y in subdirs_size])
vb.warn(f'Removed "{basename(dir_to_remove)}"')
vb.program_complete('Postgres dump complete')
result['max_dir_size_enforced'] = max_dir_size_enforced
result['removed_old_backup_dirs'] = [basename(x) for x in removed_old_backup_dirs]
pydoni.__pydonicli_register__(dict(args=args, result=result, command_name='data.pg_dump'))
