def purge_histories(app, cutoff_time, remove_from_disk, info_only=False, force_retry=False):
# Purges deleted histories whose update_time is older than the cutoff_time.
# The dataset associations of each history are also marked as deleted.
# The Purge Dataset method will purge each Dataset as necessary
# history.purged == True simply means that it can no longer be undeleted
# i.e. all associated datasets are marked as deleted
history_count = 0
start = time.time()
if force_retry:
histories = (
app.sa_session.query(app.model.History)
.filter(
and_(app.model.History.table.c.deleted == True, app.model.History.table.c.update_time < cutoff_time)
)
.options(eagerload("datasets"))
)
else:
histories = (
app.sa_session.query(app.model.History)
.filter(
and_(
app.model.History.table.c.deleted == True,
app.model.History.table.c.purged == False,
app.model.History.table.c.update_time < cutoff_time,
)
)
.options(eagerload("datasets"))
)
for history in histories:
print ("### Processing history id %d (%s)" % (history.id, history.name)).encode("utf-8")
for dataset_assoc in history.datasets:
_purge_dataset_instance(
dataset_assoc, app, remove_from_disk, info_only=info_only
) # mark a DatasetInstance as deleted, clear associated files, and mark the Dataset as deleted if it is deletable
if not info_only:
# TODO: should the Delete DefaultHistoryPermissions be deleted here? This was incorrectly
# done in the _list_delete() method of the history controller, so copied it here. Not sure
# if we should ever delete info like this from the db though, so commented out for now...
# for dhp in history.default_permissions:
# dhp.delete()
print "Purging history id ", history.id
history.purged = True
app.sa_session.add(history)
app.sa_session.flush()
else:
print "History id %d will be purged (without 'info_only' mode)" % history.id
history_count += 1
stop = time.time()
print "Purged %d histories." % history_count
print "Elapsed time: ", stop - start
print "##########################################"
def deleted_histories( self, trans, **kwd ):
"""
The number of histories that were deleted more than the specified number of days ago, but have not yet been purged.
Also included is the number of datasets associated with the histories.
"""
params = util.Params( kwd )
message = ''
if params.deleted_histories_days:
deleted_histories_days = int( params.deleted_histories_days )
cutoff_time = datetime.utcnow() - timedelta( days=deleted_histories_days )
history_count = 0
dataset_count = 0
disk_space = 0
histories = trans.sa_session.query( model.History ) \
.filter( and_( model.History.table.c.deleted == True,
model.History.table.c.purged == False,
model.History.table.c.update_time < cutoff_time ) ) \
.options( eagerload( 'datasets' ) )
for history in histories:
for hda in history.datasets:
if not hda.dataset.purged:
dataset_count += 1
try:
disk_space += hda.dataset.file_size
except:
pass
history_count += 1
message = "%d histories ( including a total of %d datasets ) were deleted more than %d days ago, but have not yet been purged, " \
"disk space: %s." % ( history_count, dataset_count, deleted_histories_days, nice_size( disk_space, True ) )
else:
message = "Enter the number of days."
return str( deleted_histories_days ), message
def deleted_histories( self, trans, **kwd ):
"""
The number of histories that were deleted more than the specified number of days ago, but have not yet been purged.
Also included is the number of datasets associated with the histories.
"""
params = util.Params( kwd )
message = ''
if params.deleted_histories_days:
deleted_histories_days = int( params.deleted_histories_days )
cutoff_time = datetime.utcnow() - timedelta( days=deleted_histories_days )
history_count = 0
dataset_count = 0
disk_space = 0
histories = trans.sa_session.query( model.History ) \
.filter( and_( model.History.table.c.deleted == True,
model.History.table.c.purged == False,
model.History.table.c.update_time < cutoff_time ) ) \
.options( eagerload( 'datasets' ) )
for history in histories:
for hda in history.datasets:
if not hda.dataset.purged:
dataset_count += 1
try:
disk_space += hda.dataset.file_size
except:
pass
history_count += 1
message = "%d histories ( including a total of %d datasets ) were deleted more than %d days ago, but have not yet been purged. Disk space: " %( history_count, dataset_count, deleted_histories_days ) + str( disk_space )
else:
message = "Enter the number of days."
return str( deleted_histories_days ), message
def purge_histories( app, cutoff_time, remove_from_disk, info_only = False ):
# Purges deleted histories whose update_time is older than the cutoff_time.
# The dataset associations of each history are also marked as deleted.
# The Purge Dataset method will purge each Dataset as necessary
# history.purged == True simply means that it can no longer be undeleted
# i.e. all associated datasets are marked as deleted
history_count = 0
print '# The following datasets and associated deleted histories have been purged'
start = time.clock()
histories = app.model.History.filter( and_( app.model.History.table.c.deleted==True,
app.model.History.table.c.purged==False,
app.model.History.table.c.update_time < cutoff_time ) ) \
.options( eagerload( 'datasets' ) ).all()
for history in histories:
for dataset_assoc in history.datasets:
_purge_dataset_instance( dataset_assoc, app, remove_from_disk, info_only = info_only ) #mark a DatasetInstance as deleted, clear associated files, and mark the Dataset as deleted if it is deletable
if not info_only:
# TODO: should the Delete DefaultHistoryPermissions be deleted here? This was incorrectly
# done in the _list_delete() method of the history controller, so copied it here. Not sure
# if we should ever delete info like this from the db though, so commented out for now...
#for dhp in history.default_permissions:
# dhp.delete()
history.purged = True
print "%d" % history.id
history_count += 1
app.model.flush()
stop = time.clock()
print '# Purged %d histories.' % ( history_count ), '\n'
print "Elapsed time: ", stop - start, "\n"
def purge_histories(app,
cutoff_time,
remove_from_disk,
info_only=False,
force_retry=False):
# Purges deleted histories whose update_time is older than the cutoff_time.
# The dataset associations of each history are also marked as deleted.
# The Purge Dataset method will purge each Dataset as necessary
# history.purged == True simply means that it can no longer be undeleted
# i.e. all associated datasets are marked as deleted
history_count = 0
start = time.time()
if force_retry:
histories = app.sa_session.query( app.model.History ) \
.filter( and_( app.model.History.table.c.deleted==True,
app.model.History.table.c.update_time < cutoff_time ) ) \
.options( eagerload( 'datasets' ) )
else:
histories = app.sa_session.query( app.model.History ) \
.filter( and_( app.model.History.table.c.deleted==True,
app.model.History.table.c.purged==False,
app.model.History.table.c.update_time < cutoff_time ) ) \
.options( eagerload( 'datasets' ) )
for history in histories:
print("### Processing history id %d (%s)" %
(history.id, history.name)).encode('utf-8')
for dataset_assoc in history.datasets:
_purge_dataset_instance(
dataset_assoc, app, remove_from_disk, info_only=info_only
) #mark a DatasetInstance as deleted, clear associated files, and mark the Dataset as deleted if it is deletable
if not info_only:
# TODO: should the Delete DefaultHistoryPermissions be deleted here? This was incorrectly
# done in the _list_delete() method of the history controller, so copied it here. Not sure
# if we should ever delete info like this from the db though, so commented out for now...
#for dhp in history.default_permissions:
# dhp.delete()
print "Purging history id ", history.id
history.purged = True
app.sa_session.add(history)
app.sa_session.flush()
else:
print "History id %d will be purged (without 'info_only' mode)" % history.id
history_count += 1
stop = time.time()
print 'Purged %d histories.' % history_count
print "Elapsed time: ", stop - start
print "##########################################"
def get_history_datasets( self, trans, history ):
"""
Returns history's datasets.
"""
query = ( trans.sa_session.query( trans.model.HistoryDatasetAssociation )
.filter( trans.model.HistoryDatasetAssociation.history == history )
.options( eagerload( "children" ) )
.join( "dataset" )
.options( eagerload_all( "dataset.actions" ) )
.order_by( trans.model.HistoryDatasetAssociation.hid )
.filter( trans.model.HistoryDatasetAssociation.deleted == False ) #noqa
.filter( trans.model.Dataset.purged == False ) )
return query.all()