def remove_helper(args, metadata):
import humanize, sdsimplefilter, sdconst, sdutils
sdlog.info("SDREMOVE-001", "Remove operation running..")
# filtering
metadata = sdsimplefilter.run(metadata, 'status',
sdconst.TRANSFER_STATUS_NEW, 'remove')
metadata = sdsimplefilter.run(
metadata, 'status', sdconst.TRANSFER_STATUS_DELETE,
'remove') # maybe not needed as we now do immediate delete
count_delete = metadata.count()
metadata_done = sdsimplefilter.run(metadata.copy(), 'status',
sdconst.TRANSFER_STATUS_DONE, 'keep')
size_delete = metadata_done.size
if count_delete > 0:
print_stderr('%i file(s) will be removed.' % count_delete)
print_stderr('After this operation, %s of disk space will be freed.' %
humanize.naturalsize(size_delete, gnu=False))
# ask user for confirmation
interactive = not args.yes
if interactive:
if sdutils.query_yes_no('Do you want to continue?', default="no"):
suppression_confirmed = True
else:
print_stderr('Abort.')
return 1
else:
suppression_confirmed = True
# perform deletion
if suppression_confirmed:
remove(metadata, (not args.keep_data))
return 0
else:
print_stderr('Nothing to delete.')
return 0
def remove_helper(args,metadata):
import humanize, sdsimplefilter, sdconst, sdutils
sdlog.info("SDREMOVE-001","Remove operation running..")
# filtering
metadata=sdsimplefilter.run(metadata,'status',sdconst.TRANSFER_STATUS_NEW,'remove')
metadata=sdsimplefilter.run(metadata,'status',sdconst.TRANSFER_STATUS_DELETE,'remove') # maybe not needed as we now do immediate delete
count_delete=metadata.count()
metadata_done=sdsimplefilter.run(metadata.copy(),'status',sdconst.TRANSFER_STATUS_DONE,'keep')
size_delete=metadata_done.size
if count_delete>0:
print_stderr('%i file(s) will be removed.'%count_delete)
print_stderr('After this operation, %s of disk space will be freed.'%humanize.naturalsize(size_delete,gnu=False))
# ask user for confirmation
interactive=not args.yes
if interactive:
if sdutils.query_yes_no('Do you want to continue?', default="no"):
suppression_confirmed=True
else:
print_stderr('Abort.')
return 1
else:
suppression_confirmed=True
# perform deletion
if suppression_confirmed:
remove(metadata,(not args.keep_data))
return 0
else:
print_stderr('Nothing to delete.')
return 0
def remove(files,args):
import sddelete,sddeletefile
if not args.dry_run:
import humanize, sdsimplefilter, sdconst, sdutils, sdoperation, sddeletedataset
# Compute deleted stat for files
files=sdsimplefilter.run(files,'status',sdconst.TRANSFER_STATUS_NEW,'remove')
files=sdsimplefilter.run(files,'status',sdconst.TRANSFER_STATUS_DELETE,'remove') # maybe not needed as we do immediate delete from now...
count_delete=len(files)
size_delete=sum(int(f['size']) for f in files if f['status']==sdconst.TRANSFER_STATUS_DONE)
if count_delete>0:
print_stderr('%i file(s) will be removed.'%count_delete)
print_stderr('After this operation, %s of disk space will be freed.'%humanize.naturalsize(size_delete,gnu=False))
if sdutils.query_yes_no('Do you want to continue?', default="no"):
# first step, change the files status from 'done' to 'delete' (update metadata)
nbr=sddelete.run(files)
print_stderr("%i file(s) removed"%nbr)
# second step, do the deletion (remove files on filesystem and remove files metadata)
# (to do a deferred deletion (i.e. by the daemon), comment line below)
sddeletefile.delete_transfers()
# third step is to remove orphan dataset
sddeletedataset.purge_orphan_datasets()
# fourth step is to remove orphan folder.
sdoperation.cleanup_tree()
else:
print_stderr('Abort.')
else:
print_stderr('Nothing to delete.')
# WARNING
#
# This code does not work for all parameters (e.g. url), as:
# - type issue in the xml returned by search-api (i.e. some parameter
# are not scalar (e.g. url))
# - call_web_service() method returns not the exact search-api xml, but
# altered result (e.g. url_http is returned instead of url)
for attribute_name in result.files[0]: # indice 0 is because we retrieve one file only (with 'limit=1')
if attribute_name not in params:
params[attribute_name]=[None]
#sdprogress.SDProgressDot.print_char()
#sdprogress.SDProgressDot.progress_complete()
return params
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('-i','--index_host')
parser.add_argument('-p','--project')
parser.add_argument('-r','--reload',action='store_true',help='Recreate cache from scratch')
args = parser.parse_args()
answer=sdutils.query_yes_no('Retrieve parameters from ESGF ?', default="no")
if answer:
run(host=args.index_host,reload=args.reload,project=args.project)
sdtools.print_stderr("Parameters are up-to-date.")
def _install(metadata, interactive, timestamp_right_boundary=None):
import sddaemon
# Compute total files stat
count_total = metadata.count()
size_total = metadata.size
sdlog.info("SYNDINST-001", "'keep new status' process begins")
# Compute new files stat
#
# (yes, block below is a duplicate of what is done inside sdenqueue.run()
# method, but safer to keep it there too, and should be no harm in term of
# perfomance)
#
import sdsimplefilter, sdconst
metadata = sdsimplefilter.run(metadata, 'status',
sdconst.TRANSFER_STATUS_NEW, 'keep')
metadata = sdsimplefilter.run(metadata, 'url', "//None", 'remove_substr')
count_new = metadata.count()
size_new = metadata.size
sdlog.info("SYNDINST-024", "'keep new status' process ends")
# what to do if no match
if count_new < 1:
if count_total > 0:
sdlog.info(
"SYNDINST-027",
"Nothing to install (matching files are already installed or waiting in the download queue). To monitor transfers status and progress, use 'synda queue' command.",
stderr=interactive)
else:
sdlog.info("SYNDINST-028",
'Nothing to install (0 file found).',
stderr=interactive)
return (0, 0)
# ask user for confirmation
if interactive:
import humanize
print_stderr('%i file(s) will be added to the download queue.' %
count_new)
print_stderr(
'Once downloaded, %s of additional disk space will be used.' %
humanize.naturalsize(size_new, gnu=False))
import sdutils
if sdutils.query_yes_no('Do you want to continue?', default="yes"):
installation_confirmed = True
else:
installation_confirmed = False
else:
installation_confirmed = True
sdlog.info("SYNDINST-002", "Store metadata in database..")
# install
if installation_confirmed:
import sdenqueue
sdenqueue.run(metadata, timestamp_right_boundary)
if interactive:
print_stderr("%i file(s) enqueued" % count_new)
print_stderr(
"You can follow the download using 'synda watch' and 'synda queue' commands"
)
if not sddaemon.is_running():
msg = sdi18n.m0025 if sdconfig.system_pkg_install else sdi18n.m0026
print_stderr(
"The daemon is not running. To start it, use '%s'." % msg)
else:
if interactive:
print_stderr('Abort.')
sdlog.info("SYNDINST-025", "Task complete")
return (0, count_new)
def _install(metadata,interactive,timestamp_right_boundary=None):
import sddaemon
# Compute total files stat
count_total=metadata.count()
size_total=metadata.size
sdlog.info("SYNDINST-001","'keep new status' process begins")
# Compute new files stat
#
# (yes, block below is a duplicate of what is done inside sdenqueue.run()
# method, but safer to keep it there too, and should be no harm in term of
# perfomance)
#
import sdsimplefilter, sdconst
metadata=sdsimplefilter.run(metadata,'status',sdconst.TRANSFER_STATUS_NEW,'keep')
count_new=metadata.count()
size_new=metadata.size
sdlog.info("SYNDINST-024","'keep new status' process ends")
# what to do if no match
if count_new<1:
if count_total>0:
sdlog.info("SYNDINST-027","Nothing to install (matching files are already installed or waiting in the download queue). To monitor transfers status and progress, use 'synda queue' command.",stderr=interactive)
else:
sdlog.info("SYNDINST-028",'Nothing to install (0 file found).',stderr=interactive)
return (0,0)
# ask user for confirmation
if interactive:
import humanize
print_stderr('%i file(s) will be added to the download queue.'%count_new)
print_stderr('Once downloaded, %s of additional disk space will be used.'%humanize.naturalsize(size_new,gnu=False))
import sdutils
if sdutils.query_yes_no('Do you want to continue?', default="yes"):
installation_confirmed=True
else:
installation_confirmed=False
else:
installation_confirmed=True
sdlog.info("SYNDINST-002","Store metadata in database..")
# install
if installation_confirmed:
import sdenqueue
sdenqueue.run(metadata,timestamp_right_boundary)
if interactive:
print_stderr("%i file(s) enqueued"%count_new)
print_stderr("You can follow the download using 'synda watch' and 'synda queue' commands")
if not sddaemon.is_running():
msg=sdi18n.m0025 if sdconfig.system_pkg_install else sdi18n.m0026
print_stderr("The daemon is not running. To start it, use '%s'."%msg)
else:
if interactive:
print_stderr('Abort.')
sdlog.info("SYNDINST-025","Task complete")
return (0,count_new)
# are not scalar (e.g. url))
# - call_web_service() method returns not the exact search-api xml, but
# altered result (e.g. url_http is returned instead of url)
for attribute_name in file_:
if attribute_name not in params:
params[attribute_name] = [None]
#sdprogress.SDProgressDot.print_char()
#sdprogress.SDProgressDot.progress_complete()
return params
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('-i', '--index_host')
parser.add_argument('-p', '--project')
parser.add_argument('-r',
'--reload',
action='store_true',
help='Recreate cache from scratch')
args = parser.parse_args()
answer = sdutils.query_yes_no('Retrieve parameters from ESGF ?',
default="no")
if answer:
run(host=args.index_host, reload=args.reload, project=args.project)
sdtools.print_stderr("Parameters are up-to-date.")
if interactive:
if count_total>0:
print_stderr("Nothing to install (matching files are already installed or waiting in the download queue). To monitor transfers status and progress, use 'synda queue' command.")
else:
print_stderr('Nothing to install (0 file found).')
return 0
# ask user for confirmation
if interactive:
import humanize
print_stderr('%i file(s) will be added to the download queue.'%count_new)
print_stderr('Once downloaded, %s of additional disk space will be used.'%humanize.naturalsize(size_new,gnu=False))
import sdutils
if sdutils.query_yes_no('Do you want to continue?', default="yes"):
installation_confirmed=True
else:
installation_confirmed=False
else:
installation_confirmed=True
# install
if installation_confirmed:
import sdenqueue
sdenqueue.run(files)
if interactive:
print_stderr("%i file(s) enqueued"%count_new)
print_stderr("You can follow the download using 'synda watch' and 'synda queue' commands")
def install(files,args):
"""
Returns
number of newly installed files
"""
if args.dry_run:
return 0
# TODO
interactive=not args.non_interactive
# Compute total files stat
count_total=len(files)
size_total=sum(int(f['size']) for f in files)
# Compute new files stat
#
# (yes, block below is a duplicate of what is done inside sdenqueue.run()
# method, but safer to keep it there too, and should be no harm in term of
# perfomance)
#
import sdsimplefilter, sdconst
files=sdsimplefilter.run(files,'status',sdconst.TRANSFER_STATUS_NEW,'keep')
count_new=len(files)
size_new=sum(int(f['size']) for f in files)
# what to do if no match
if count_new<1:
if interactive:
if count_total>0:
print_stderr('Nothing to install (files already installed).')
else:
print_stderr('Nothing to install (0 file found).')
return 0
# ask user for confirmation
if interactive:
import humanize
print_stderr('%i file(s) will be added to the download queue.'%count_new)
print_stderr('Once downloaded, %s of additional disk space will be used.'%humanize.naturalsize(size_new,gnu=False))
import sdutils
if sdutils.query_yes_no('Do you want to continue?', default="yes"):
installation_confirmed=True
else:
installation_confirmed=False
else:
installation_confirmed=True
# install
if installation_confirmed:
import sdenqueue
sdenqueue.run(files)
if interactive:
print_stderr("%i file(s) enqueued"%count_new)
print_stderr('You can now start the daemon to begin the download.') # TODO: ask for confirm and do the start here
else:
if interactive:
print_stderr('Abort.')
return count_new