def main(argv):
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('-k', '--secret-key', help='Key to convert pages with', default='')
parser.add_argument('-d', '--dry-run', help='No changes, just test it.', action='store_true')
populate_config_args(parser)
args = parser.parse_args()
properties = app_properties_from_args(args)
config = galaxy.config.Configuration(**properties)
secret = args.secret_key or config.id_secret
security_helper = IdEncodingHelper(id_secret=secret)
object_store = build_object_store_from_config(config)
if not config.database_connection:
print("The database connection is empty. If you are using the default value, please uncomment that in your galaxy.yml")
model = galaxy.config.init_models_from_config(config, object_store=object_store)
session = model.context.current
pagerevs = session.query(model.PageRevision).all()
mock_trans = Bunch(app=Bunch(security=security_helper), model=model, user_is_admin=lambda: True, sa_session=session)
for p in pagerevs:
try:
processor = _PageContentProcessor(mock_trans, _placeholderRenderForSave)
processor.feed(p.content)
newcontent = unicodify(processor.output(), 'utf-8')
if p.content != newcontent:
if not args.dry_run:
p.content = unicodify(processor.output(), 'utf-8')
session.add(p)
session.flush()
else:
print("Modifying revision %s." % p.id)
print(difflib.unified_diff(p.content, newcontent))
except Exception:
logging.exception("Error parsing page, rolling changes back and skipping revision %s. Please report this error." % p.id)
session.rollback()
def init():
app_properties = app_properties_from_args(args)
config = galaxy.config.Configuration(**app_properties)
object_store = build_object_store_from_config(config)
model = init_models_from_config(config, object_store=object_store)
return model, object_store
def init():
app_properties = app_properties_from_args(args)
config = galaxy.config.Configuration(**app_properties)
object_store = build_object_store_from_config(config)
model = galaxy.config.init_models_from_config(config, object_store=object_store)
return model, object_store
def main(argv):
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('-k', '--secret-key', help='Key to convert pages with', default='')
parser.add_argument('-d', '--dry-run', help='No changes, just test it.', action='store_true')
populate_config_args(parser)
args = parser.parse_args()
properties = app_properties_from_args(args)
config = galaxy.config.Configuration(**properties)
secret = args.secret_key or config.id_secret
security_helper = SecurityHelper(id_secret=secret)
object_store = build_object_store_from_config(config)
if not config.database_connection:
print("The database connection is empty. If you are using the default value, please uncomment that in your galaxy.yml")
model = galaxy.config.init_models_from_config(config, object_store=object_store)
session = model.context.current
pagerevs = session.query(model.PageRevision).all()
mock_trans = Bunch(app=Bunch(security=security_helper), model=model, user_is_admin=lambda: True, sa_session=session)
for p in pagerevs:
try:
processor = _PageContentProcessor(mock_trans, _placeholderRenderForSave)
processor.feed(p.content)
newcontent = unicodify(processor.output(), 'utf-8')
if p.content != newcontent:
if not args.dry_run:
p.content = unicodify(processor.output(), 'utf-8')
session.add(p)
session.flush()
else:
print("Modifying revision %s." % p.id)
print(difflib.unified_diff(p.content, newcontent))
except Exception:
logging.exception("Error parsing page, rolling changes back and skipping revision %s. Please report this error." % p.id)
session.rollback()
def parse_arguments():
parser = argparse.ArgumentParser(
description='Generate walltime statistics')
parser.add_argument('tool_id', help='Tool (by ID) to collect stats about')
parser.add_argument('--like',
action='store_true',
default=False,
help='Use SQL `LIKE` operator to find '
'a shed-installed tool using the tool\'s '
'"short" id')
populate_config_args(parser)
parser.add_argument('-d',
'--debug',
action='store_true',
default=False,
help='Print extra info')
parser.add_argument('-m',
'--min',
type=int,
default=-1,
help='Ignore runtimes less than MIN seconds')
parser.add_argument('-M',
'--max',
type=int,
default=-1,
help='Ignore runtimes greater than MAX seconds')
parser.add_argument('-u',
'--user',
help='Return stats for only this user (id, email, '
'or username)')
parser.add_argument('-s',
'--source',
default='metrics',
help='Runtime data source (SOURCES: %s)' %
', '.join(DATA_SOURCES))
args = parser.parse_args()
if args.like and '/' in args.tool_id:
print('ERROR: Do not use --like with a tool shed tool id (the tool '
'id should not contain `/` characters)')
sys.exit(2)
args.source = args.source.lower()
if args.source not in ('metrics', 'history'):
print('ERROR: Data source `%s` unknown, valid source are: %s' %
(args.source, ', '.join(DATA_SOURCES)))
app_properties = app_properties_from_args(args)
config = galaxy.config.Configuration(**app_properties)
uri = args.config.get_database_url(config)
names = {'database': 'dbname', 'username': '******'}
args.connect_args = url.make_url(uri).translate_connect_args(**names)
if args.debug:
print('Got options:')
for i in vars(args).items():
print('%s: %s' % i)
return args
def parse_arguments():
parser = argparse.ArgumentParser(
description='Generate walltime statistics')
parser.add_argument('tool_id', help='Tool (by ID) to collect stats about')
parser.add_argument('--like',
action='store_true',
default=False,
help='Use SQL `LIKE` operator to find '
'a shed-installed tool using the tool\'s '
'"short" id')
populate_config_args(parser)
parser.add_argument('-d', '--debug',
action='store_true',
default=False,
help='Print extra info')
parser.add_argument('-m', '--min',
type=int,
default=-1,
help='Ignore runtimes less than MIN seconds')
parser.add_argument('-M', '--max',
type=int,
default=-1,
help='Ignore runtimes greater than MAX seconds')
parser.add_argument('-u', '--user',
help='Return stats for only this user (id, email, '
'or username)')
parser.add_argument('-s', '--source',
default='metrics',
help='Runtime data source (SOURCES: %s)'
% ', '.join(DATA_SOURCES))
args = parser.parse_args()
if args.like and '/' in args.tool_id:
print('ERROR: Do not use --like with a tool shed tool id (the tool '
'id should not contain `/` characters)')
sys.exit(2)
args.source = args.source.lower()
if args.source not in ('metrics', 'history'):
print('ERROR: Data source `%s` unknown, valid source are: %s'
% (args.source, ', '.join(DATA_SOURCES)))
app_properties = app_properties_from_args(args)
config = galaxy.config.Configuration(**app_properties)
uri = args.config.get_database_url(config)
names = {'database': 'dbname', 'username': '******'}
args.connect_args = url.make_url(uri).translate_connect_args(**names)
if args.debug:
print('Got options:')
for i in vars(args).items():
print('%s: %s' % i)
return args
def init():
if args.username == 'all':
args.username = None
if args.email == 'all':
args.email = None
app_properties = app_properties_from_args(args)
config = galaxy.config.Configuration(**app_properties)
object_store = build_object_store_from_config(config)
engine = galaxy.config.get_database_url(config).split(":")[0]
return galaxy.config.init_models_from_config(config, object_store=object_store), object_store, engine
def _init(args):
properties = app_properties_from_args(args)
config = galaxy.config.Configuration(**properties)
object_store = build_object_store_from_config(config)
if not config.database_connection:
logging.warning("The database connection is empty. If you are using the default value, please uncomment that in your galaxy.yml")
model = galaxy.config.init_models_from_config(config, object_store=object_store)
return (
model,
object_store,
config,
)
def _init(args):
properties = app_properties_from_args(args)
config = galaxy.config.Configuration(**properties)
object_store = build_object_store_from_config(config)
if not config.database_connection:
logging.warning("The database connection is empty. If you are using the default value, please uncomment that in your galaxy.yml")
model = init_models_from_config(config, object_store=object_store)
return (
model,
object_store,
config,
)
def _init(args, need_app=False):
properties = app_properties_from_args(args)
config = galaxy.config.Configuration(**properties)
object_store = build_object_store_from_config(config)
if not config.database_connection:
logging.warning("The database connection is empty. If you are using the default value, please uncomment that in your galaxy.yml")
if need_app:
config_file = config_file_from_args(args)
app = galaxy.app.UniverseApplication(global_conf={'__file__': config_file, 'here': os.getcwd()})
else:
app = None
model = galaxy.config.init_models_from_config(config, object_store=object_store)
return (
model,
object_store,
config,
app
)
def _init(args, need_app=False):
properties = app_properties_from_args(args)
config = galaxy.config.Configuration(**properties)
object_store = build_object_store_from_config(config)
if not config.database_connection:
logging.warning(
"The database connection is empty. If you are using the default value, please uncomment that in your galaxy.yml"
)
if need_app:
config_file = config_file_from_args(args)
app = galaxy.app.UniverseApplication(global_conf={
'__file__': config_file,
'here': os.getcwd()
})
else:
app = None
model = galaxy.config.init_models_from_config(config,
object_store=object_store)
return (model, object_store, config, app)
def main():
parser = argparse.ArgumentParser(DESCRIPTION)
populate_config_args(parser)
args = parser.parse_args()
app_properties = app_properties_from_args(args)
config = galaxy.config.Configuration(**app_properties)
model = galaxy.config.init_models_from_config(config)
for row in model.context.query(model.Dataset):
if row.uuid is None:
row.uuid = uuid.uuid4()
print("Setting dataset:", row.id, " UUID to ", row.uuid)
model.context.flush()
for row in model.context.query(model.Workflow):
if row.uuid is None:
row.uuid = uuid.uuid4()
print("Setting Workflow:", row.id, " UUID to ", row.uuid)
model.context.flush()
print("Complete")
def parse_arguments():
parser = argparse.ArgumentParser(
description=
'Build a disk-backed Toolshed repository index and tool index for searching.'
)
populate_config_args(parser)
parser.add_argument('-d',
'--debug',
action='store_true',
default=False,
help='Print extra info')
args = parser.parse_args()
app_properties = app_properties_from_args(args)
config = ts_config.ToolShedAppConfiguration(**app_properties)
args.dburi = config.database_connection
args.hgweb_config_dir = config.hgweb_config_dir
args.whoosh_index_dir = config.whoosh_index_dir
args.file_path = config.file_path
if args.debug:
log.setLevel(logging.DEBUG)
log.debug('Full options:')
for i in vars(args).items():
log.debug('%s: %s' % i)
return args
def __load_config(self):
app_properties = app_properties_from_args(self.args)
self.config = galaxy.config.Configuration(**app_properties)
def main():
"""
Datasets that are older than the specified cutoff and for which the tool_id
contains the specified text will be marked as deleted in user's history and
the user will be notified by email using the specified template file.
"""
parser = argparse.ArgumentParser()
parser.add_argument('legacy_config', metavar='CONFIG', type=str,
default=None,
nargs='?',
help='config file (legacy, use --config instead)')
parser.add_argument("-d", "--days", dest="days", action="store",
type=int, help="number of days (60)", default=60)
parser.add_argument("--tool_id", default=None,
help="Text to match against tool_id"
"Default: match all")
parser.add_argument("--template", default=None,
help="Mako Template file to use as email "
"Variables are 'cutoff' for the cutoff in days, "
"'email' for users email and "
"'datasets' which is a list of tuples "
"containing 'dataset' and 'history' names. "
"Default: admin_cleanup_deletion_template.txt")
parser.add_argument("-i", "--info_only", action="store_true",
dest="info_only", help="info about the requested action",
default=False)
parser.add_argument("-e", "--email_only", action="store_true",
dest="email_only", help="Send emails only, don't delete",
default=False)
parser.add_argument("--smtp", default=None,
help="SMTP Server to use to send email. "
"Default: [read from galaxy ini file]")
parser.add_argument("--fromaddr", default=None,
help="From address to use to send email. "
"Default: [read from galaxy ini file]")
populate_config_args(parser)
args = parser.parse_args()
config_override = None
if args.legacy_config:
config_override = args.legacy_config
app_properties = app_properties_from_args(args, legacy_config_override=config_override)
if args.smtp is not None:
app_properties['smtp_server'] = args.smtp
if app_properties.get('smtp_server') is None:
parser.error("SMTP Server must be specified as an option (--smtp) "
"or in the config file (smtp_server)")
if args.fromaddr is not None:
app_properties['email_from'] = args.fromaddr
if app_properties.get('email_from') is None:
parser.error("From address must be specified as an option "
"(--fromaddr) or in the config file "
"(email_from)")
scriptdir = os.path.dirname(os.path.abspath(__file__))
template_file = args.template
if template_file is None:
default_template = os.path.join(scriptdir,
'admin_cleanup_deletion_template.txt')
sample_template_file = "%s.sample" % default_template
if os.path.exists(default_template):
template_file = default_template
elif os.path.exists(sample_template_file):
print("Copying %s to %s" % (sample_template_file, default_template))
shutil.copyfile(sample_template_file, default_template)
template_file = default_template
else:
parser.error("Default template (%s) or sample template (%s) not "
"found, please specify template as an option "
"(--template)." % default_template,
sample_template_file)
elif not os.path.exists(template_file):
parser.error("Specified template file (%s) not found." % template_file)
config = galaxy.config.Configuration(**app_properties)
app = CleanupDatasetsApplication(config)
cutoff_time = datetime.utcnow() - timedelta(days=args.days)
now = strftime("%Y-%m-%d %H:%M:%S")
print("##########################################")
print("\n# %s - Handling stuff older than %i days" % (now, args.days))
if args.info_only:
print("# Displaying info only ( --info_only )\n")
elif args.email_only:
print("# Sending emails only, not deleting ( --email_only )\n")
administrative_delete_datasets(
app, cutoff_time, args.days, tool_id=args.tool_id,
template_file=template_file, config=config,
email_only=args.email_only, info_only=args.info_only)
app.shutdown()
sys.exit(0)
def main():
"""
Datasets that are older than the specified cutoff and for which the tool_id
contains the specified text will be marked as deleted in user's history and
the user will be notified by email using the specified template file.
"""
parser = argparse.ArgumentParser()
parser.add_argument('legacy_config',
metavar='CONFIG',
type=str,
default=None,
nargs='?',
help='config file (legacy, use --config instead)')
parser.add_argument("-d",
"--days",
dest="days",
action="store",
type=int,
help="number of days (60)",
default=60)
parser.add_argument("--tool_id",
default=None,
help="Text to match against tool_id"
"Default: match all")
parser.add_argument("--template",
default=None,
help="Mako Template file to use as email "
"Variables are 'cutoff' for the cutoff in days, "
"'email' for users email and "
"'datasets' which is a list of tuples "
"containing 'dataset' and 'history' names. "
"Default: admin_cleanup_deletion_template.txt")
parser.add_argument("-i",
"--info_only",
action="store_true",
dest="info_only",
help="info about the requested action",
default=False)
parser.add_argument("-e",
"--email_only",
action="store_true",
dest="email_only",
help="Send emails only, don't delete",
default=False)
parser.add_argument("--smtp",
default=None,
help="SMTP Server to use to send email. "
"Default: [read from galaxy ini file]")
parser.add_argument("--fromaddr",
default=None,
help="From address to use to send email. "
"Default: [read from galaxy ini file]")
populate_config_args(parser)
args = parser.parse_args()
config_override = None
if args.legacy_config:
config_override = args.legacy_config
app_properties = app_properties_from_args(
args, legacy_config_override=config_override)
if args.smtp is not None:
app_properties['smtp_server'] = args.smtp
if app_properties.get('smtp_server') is None:
parser.error("SMTP Server must be specified as an option (--smtp) "
"or in the config file (smtp_server)")
if args.fromaddr is not None:
app_properties['email_from'] = args.fromaddr
if app_properties.get('email_from') is None:
parser.error("From address must be specified as an option "
"(--fromaddr) or in the config file "
"(email_from)")
scriptdir = os.path.dirname(os.path.abspath(__file__))
template_file = args.template
if template_file is None:
default_template = os.path.join(scriptdir,
'admin_cleanup_deletion_template.txt')
sample_template_file = "%s.sample" % default_template
if os.path.exists(default_template):
template_file = default_template
elif os.path.exists(sample_template_file):
print("Copying %s to %s" %
(sample_template_file, default_template))
shutil.copyfile(sample_template_file, default_template)
template_file = default_template
else:
parser.error(
"Default template (%s) or sample template (%s) not "
"found, please specify template as an option "
"(--template)." % default_template, sample_template_file)
elif not os.path.exists(template_file):
parser.error("Specified template file (%s) not found." % template_file)
config = galaxy.config.Configuration(**app_properties)
app = CleanupDatasetsApplication(config)
cutoff_time = datetime.utcnow() - timedelta(days=args.days)
now = strftime("%Y-%m-%d %H:%M:%S")
print("##########################################")
print("\n# %s - Handling stuff older than %i days" % (now, args.days))
if args.info_only:
print("# Displaying info only ( --info_only )\n")
elif args.email_only:
print("# Sending emails only, not deleting ( --email_only )\n")
administrative_delete_datasets(app,
cutoff_time,
args.days,
tool_id=args.tool_id,
template_file=template_file,
config=config,
email_only=args.email_only,
info_only=args.info_only)
app.shutdown()
sys.exit(0)
def __load_config(self):
app_properties = app_properties_from_args(self.args)
self.config = galaxy.config.Configuration(**app_properties)
def main():
"""
Managing library datasets is a bit complex, so here is a scenario that hopefully provides clarification. The complexities
of handling library datasets is mostly contained in the delete_datasets() method in this script.
Assume we have 1 library dataset with: LibraryDatasetDatasetAssociation -> LibraryDataset and Dataset
At this point, we have the following database column values:
LibraryDatasetDatasetAssociation deleted: False
LibraryDataset deleted: False, purged: False
Dataset deleted: False purged: False
1. A user deletes the assumed dataset above from a data library via a UI menu option.
This action results in the following database column values (changes from previous step marked with *):
LibraryDatasetDatasetAssociation deleted: False
LibraryDataset deleted: True*, purged: False
Dataset deleted: False, purged: False
2. After the number of days configured for the delete_datasets() method (option -6 below) have passed, execution
of the delete_datasets() method results in the following database column values (changes from previous step marked with *):
LibraryDatasetDatasetAssociation deleted: True*
LibraryDataset deleted: True, purged: True*
Dataset deleted: True*, purged: False
3. After the number of days configured for the purge_datasets() method (option -3 below) have passed, execution
of the purge_datasets() method results in the following database column values (changes from previous step marked with *):
LibraryDatasetDatasetAssociation deleted: True
LibraryDataset deleted: True, purged: True
Dataset deleted: True, purged: True* (dataset file removed from disk if -r flag is used)
This scenario is about as simple as it gets. Keep in mind that a Dataset object can have many HistoryDatasetAssociations
and many LibraryDatasetDatasetAssociations, and a LibraryDataset can have many LibraryDatasetDatasetAssociations.
Another way of stating it is: LibraryDatasetDatasetAssociation objects map LibraryDataset objects to Dataset objects,
and Dataset objects may be mapped to History objects via HistoryDatasetAssociation objects.
"""
parser = argparse.ArgumentParser()
parser.add_argument('legacy_config',
metavar='CONFIG',
type=str,
default=None,
nargs='?',
help='config file (legacy, use --config instead)')
parser.add_argument("-d",
"--days",
dest="days",
action="store",
type=int,
help="number of days (60)",
default=60)
parser.add_argument("-r",
"--remove_from_disk",
action="store_true",
dest="remove_from_disk",
help="remove datasets from disk when purged",
default=False)
parser.add_argument("-i",
"--info_only",
action="store_true",
dest="info_only",
help="info about the requested action",
default=False)
parser.add_argument(
"-f",
"--force_retry",
action="store_true",
dest="force_retry",
help=
"performs the requested actions, but ignores whether it might have been done before. Useful when -r wasn't used, but should have been",
default=False)
parser.add_argument("-1",
"--delete_userless_histories",
action="store_true",
dest="delete_userless_histories",
default=False,
help="delete userless histories and datasets")
parser.add_argument("-2",
"--purge_histories",
action="store_true",
dest="purge_histories",
default=False,
help="purge deleted histories")
parser.add_argument("-3",
"--purge_datasets",
action="store_true",
dest="purge_datasets",
default=False,
help="purge deleted datasets")
parser.add_argument("-4",
"--purge_libraries",
action="store_true",
dest="purge_libraries",
default=False,
help="purge deleted libraries")
parser.add_argument("-5",
"--purge_folders",
action="store_true",
dest="purge_folders",
default=False,
help="purge deleted library folders")
parser.add_argument(
"-6",
"--delete_datasets",
action="store_true",
dest="delete_datasets",
default=False,
help=
"mark deletable datasets as deleted and purge associated dataset instances"
)
populate_config_args(parser)
args = parser.parse_args()
config_override = None
if args.legacy_config:
config_override = args.legacy_config
if not (args.purge_folders ^ args.delete_userless_histories
^ args.purge_libraries ^ args.purge_histories ^ args.purge_datasets
^ args.delete_datasets):
parser.print_help()
sys.exit(0)
if args.remove_from_disk and args.info_only:
parser.error("remove_from_disk and info_only are mutually exclusive")
app_properties = app_properties_from_args(
args, legacy_config_override=config_override)
config = galaxy.config.Configuration(**app_properties)
app = CleanupDatasetsApplication(config)
cutoff_time = datetime.utcnow() - timedelta(days=args.days)
now = strftime("%Y-%m-%d %H:%M:%S")
log.info("##########################################")
log.info("\n# %s - Handling stuff older than %i days" % (now, args.days))
if args.info_only:
log.info("# Displaying info only ( --info_only )\n")
elif args.remove_from_disk:
log.info("Datasets will be removed from disk.\n")
else:
log.info("Datasets will NOT be removed from disk.\n")
if args.delete_userless_histories:
delete_userless_histories(app,
cutoff_time,
info_only=args.info_only,
force_retry=args.force_retry)
elif args.purge_histories:
purge_histories(app,
cutoff_time,
args.remove_from_disk,
info_only=args.info_only,
force_retry=args.force_retry)
elif args.purge_datasets:
purge_datasets(app,
cutoff_time,
args.remove_from_disk,
info_only=args.info_only,
force_retry=args.force_retry)
elif args.purge_libraries:
purge_libraries(app,
cutoff_time,
args.remove_from_disk,
info_only=args.info_only,
force_retry=args.force_retry)
elif args.purge_folders:
purge_folders(app,
cutoff_time,
args.remove_from_disk,
info_only=args.info_only,
force_retry=args.force_retry)
elif args.delete_datasets:
delete_datasets(app,
cutoff_time,
args.remove_from_disk,
info_only=args.info_only,
force_retry=args.force_retry)
app.shutdown()
sys.exit(0)
from galaxy.web.security import SecurityHelper
logging.basicConfig()
log = logging.getLogger(__name__)
parser = argparse.ArgumentParser()
parser.add_argument('action', metavar='ACTION', type=str,
default=None,
help='decode|encode')
parser.add_argument('value', metavar='VALUE', type=str,
default=None,
help='value to encode or decode')
populate_config_args(parser)
args = parser.parse_args()
app_properties = app_properties_from_args(args)
helper = SecurityHelper(id_secret=app_properties.get('id_secret'))
# We need the ID secret for configuring the security helper to decrypt
# galaxysession cookies.
if "id_secret" not in app_properties:
log.warning('No ID_SECRET specified. Please set the "id_secret" in your galaxy.yml.')
id_secret = app_properties.get('id_secret', 'dangerous_default')
security_helper = SecurityHelper(id_secret=id_secret)
# And get access to the models
# Login manager to manage current_user functionality
if args.action == 'decode':
sys.stdout.write(security_helper.decode_guid(args.value.lstrip('F')))
sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir, 'lib'))
import galaxy.config
from galaxy.security import idencoding
from galaxy.util.script import app_properties_from_args, populate_config_args
parser = argparse.ArgumentParser()
populate_config_args(parser)
parser.add_argument('-e', '--encode-id', dest='encode_id', help='Encode an ID')
parser.add_argument('-d', '--decode-id', dest='decode_id', help='Decode an ID')
parser.add_argument('--hda', dest='hda_id', help='Display HistoryDatasetAssociation info')
parser.add_argument('--ldda', dest='ldda_id', help='Display LibraryDatasetDatasetAssociation info')
args = parser.parse_args()
app_properties = app_properties_from_args(args)
config = galaxy.config.Configuration(**app_properties)
helper = idencoding.IdEncodingHelper(id_secret=app_properties.get('id_secret'))
model = galaxy.config.init_models_from_config(config)
if args.encode_id:
print('Encoded "{}": {}'.format(args.encode_id, helper.encode_id(args.encode_id)))
if args.decode_id:
print('Decoded "{}": {}'.format(args.decode_id, helper.decode_id(args.decode_id)))
if args.hda_id:
try:
hda_id = int(args.hda_id)
except Exception:
hda_id = int(helper.decode_id(args.hda_id))
def main():
"""
Managing library datasets is a bit complex, so here is a scenario that hopefully provides clarification. The complexities
of handling library datasets is mostly contained in the delete_datasets() method in this script.
Assume we have 1 library dataset with: LibraryDatasetDatasetAssociation -> LibraryDataset and Dataset
At this point, we have the following database column values:
LibraryDatasetDatasetAssociation deleted: False
LibraryDataset deleted: False, purged: False
Dataset deleted: False purged: False
1. A user deletes the assumed dataset above from a data library via a UI menu option.
This action results in the following database column values (changes from previous step marked with *):
LibraryDatasetDatasetAssociation deleted: False
LibraryDataset deleted: True*, purged: False
Dataset deleted: False, purged: False
2. After the number of days configured for the delete_datasets() method (option -6 below) have passed, execution
of the delete_datasets() method results in the following database column values (changes from previous step marked with *):
LibraryDatasetDatasetAssociation deleted: True*
LibraryDataset deleted: True, purged: True*
Dataset deleted: True*, purged: False
3. After the number of days configured for the purge_datasets() method (option -3 below) have passed, execution
of the purge_datasets() method results in the following database column values (changes from previous step marked with *):
LibraryDatasetDatasetAssociation deleted: True
LibraryDataset deleted: True, purged: True
Dataset deleted: True, purged: True* (dataset file removed from disk if -r flag is used)
This scenario is about as simple as it gets. Keep in mind that a Dataset object can have many HistoryDatasetAssociations
and many LibraryDatasetDatasetAssociations, and a LibraryDataset can have many LibraryDatasetDatasetAssociations.
Another way of stating it is: LibraryDatasetDatasetAssociation objects map LibraryDataset objects to Dataset objects,
and Dataset objects may be mapped to History objects via HistoryDatasetAssociation objects.
"""
parser = argparse.ArgumentParser()
parser.add_argument('legacy_config', metavar='CONFIG', type=str,
default=None,
nargs='?',
help='config file (legacy, use --config instead)')
parser.add_argument("-d", "--days", dest="days", action="store", type=int, help="number of days (60)", default=60)
parser.add_argument("-r", "--remove_from_disk", action="store_true", dest="remove_from_disk", help="remove datasets from disk when purged", default=False)
parser.add_argument("-i", "--info_only", action="store_true", dest="info_only", help="info about the requested action", default=False)
parser.add_argument("-f", "--force_retry", action="store_true", dest="force_retry", help="performs the requested actions, but ignores whether it might have been done before. Useful when -r wasn't used, but should have been", default=False)
parser.add_argument("-1", "--delete_userless_histories", action="store_true", dest="delete_userless_histories", default=False, help="delete userless histories and datasets")
parser.add_argument("-2", "--purge_histories", action="store_true", dest="purge_histories", default=False, help="purge deleted histories")
parser.add_argument("-3", "--purge_datasets", action="store_true", dest="purge_datasets", default=False, help="purge deleted datasets")
parser.add_argument("-4", "--purge_libraries", action="store_true", dest="purge_libraries", default=False, help="purge deleted libraries")
parser.add_argument("-5", "--purge_folders", action="store_true", dest="purge_folders", default=False, help="purge deleted library folders")
parser.add_argument("-6", "--delete_datasets", action="store_true", dest="delete_datasets", default=False, help="mark deletable datasets as deleted and purge associated dataset instances")
populate_config_args(parser)
args = parser.parse_args()
config_override = None
if args.legacy_config:
config_override = args.legacy_config
if not (args.purge_folders ^ args.delete_userless_histories ^
args.purge_libraries ^ args.purge_histories ^
args.purge_datasets ^ args.delete_datasets):
parser.print_help()
sys.exit(0)
if args.remove_from_disk and args.info_only:
parser.error("remove_from_disk and info_only are mutually exclusive")
app_properties = app_properties_from_args(args, legacy_config_override=config_override)
config = galaxy.config.Configuration(**app_properties)
app = CleanupDatasetsApplication(config)
cutoff_time = datetime.utcnow() - timedelta(days=args.days)
now = strftime("%Y-%m-%d %H:%M:%S")
print("##########################################")
print("\n# %s - Handling stuff older than %i days" % (now, args.days))
if args.info_only:
print("# Displaying info only ( --info_only )\n")
elif args.remove_from_disk:
print("Datasets will be removed from disk.\n")
else:
print("Datasets will NOT be removed from disk.\n")
if args.delete_userless_histories:
delete_userless_histories(app, cutoff_time, info_only=args.info_only, force_retry=args.force_retry)
elif args.purge_histories:
purge_histories(app, cutoff_time, args.remove_from_disk, info_only=args.info_only, force_retry=args.force_retry)
elif args.purge_datasets:
purge_datasets(app, cutoff_time, args.remove_from_disk, info_only=args.info_only, force_retry=args.force_retry)
elif args.purge_libraries:
purge_libraries(app, cutoff_time, args.remove_from_disk, info_only=args.info_only, force_retry=args.force_retry)
elif args.purge_folders:
purge_folders(app, cutoff_time, args.remove_from_disk, info_only=args.info_only, force_retry=args.force_retry)
elif args.delete_datasets:
delete_datasets(app, cutoff_time, args.remove_from_disk, info_only=args.info_only, force_retry=args.force_retry)
app.shutdown()
sys.exit(0)
