def DeleteStaticContent(urls):
TAG = Const.Tags.Events.PRUNE_STATIC_CONTENT
try:
if(settings.PRUNE_STATIC_CONTENT):
s3 = boto3.resource('s3')
bucket = s3.Bucket(settings.AWS_BUCKET_NAME)
objectsToDelete = []
# build up the list of objects to delete to we can pass to the delete
for url in urls:
object = { 'Key': url}
objectsToDelete.append(object)
delete = {'Objects': objectsToDelete,
'Quiet': False }
# Delete the objects from the bucket
# TODO: parse this response object and do stuff with it
response = bucket.delete_objects(Delete=delete)
DataCollector.logServerEvent(TAG, {
Const.DataCollection.ParamNames.MESSAGE_CODE: Const.DataCollection.MessageCodes.Events.PruneStaticContent.SUCCESS,
Const.DataCollection.ParamNames.NUM_RECEIVED: len(urls),
Const.DataCollection.ParamNames.NUM_DELETED: len(urls) }) # note: we need to get this info from response object later
except Exception as e:
DataCollector.logServerError(e)
def handle(self, *args, **options):
TAG = Const.Tags.Events.PRUNE_LOCALPOSTS
try:
# Only run the pruning if the setting is set
if (settings.PRUNE_OLD_LOCALPOSTS):
currentTime = time.time()
oldestTimeAllowed = currentTime - (
Const.Pruning.LOCALPOST_MAX_AGE_HOURS *
Const.SECONDS_IN_HOUR)
numPostsDeleted = 0
# Get all localposts where timeCreated < oldestTimeAllowed
localPostsToDelete = LocalPost.objects.filter(
timeCreated__lt=oldestTimeAllowed)
# remove static content if enabled
if (settings.PRUNE_STATIC_CONTENT):
localPostUrls = []
for lp in localPostsToDelete:
localPostUrls.append(lp.url)
StaticContentUtils.DeleteStaticContent(
localPostUrls, 'local')
# Remove each local post
for lp in localPostsToDelete:
lp.delete()
numPostsDeleted += 1
# log info on success
DataCollector.logServerEvent(
TAG, {
Const.DataCollection.ParamNames.MESSAGE_CODE:
Const.DataCollection.MessageCodes.Events.PruneLocalPosts.
SUCCESS,
Const.DataCollection.ParamNames.NUM_RECEIVED:
len(list(localPostsToDelete)),
Const.DataCollection.ParamNames.NUM_DELETED:
numPostsDeleted
})
except Exception as e:
# log error and info on error
DataCollector.logServerError(e)
DataCollector.logServerEvent(
TAG, {
Const.DataCollection.ParamNames.MESSAGE_CODE:
Const.DataCollection.MessageCodes.Events.PruneLocalPosts.
SERVER_ERROR,
Const.DataCollection.ParamNames.NUM_RECEIVED:
len(list(localPostsToDelete)),
Const.DataCollection.ParamNames.NUM_DELETED:
numPostsDeleted
})
def handle(self, *args, **options):
TAG = Const.Tags.Events.PRUNE_MESSAGES
try:
if (settings.PRUNE_OLD_MESSAGES):
currentTime = time.time()
oldestTimeAllowed = currentTime - (
Const.Pruning.MESSAGE_MAX_AGE_HOURS *
Const.SECONDS_IN_HOUR)
numMessagesDeleted = 0
# Get all messages where timeCreated < oldestTimeAllowed
messagesToDelete = Message.objects.filter(
timeCreated__lt=oldestTimeAllowed)
# remove static content if enabled
if (settings.PRUNE_STATIC_CONTENT):
messageUrls = []
for m in messagesToDelete:
messageUrls.append(m.url)
StaticContentUtils.DeleteStaticContent(
messageUrls, 'message')
# Remove each one
for m in messagesToDelete:
m.delete()
numMessagesDeleted += 1
# log info on success
DataCollector.logServerEvent(
TAG, {
Const.DataCollection.ParamNames.MESSAGE_CODE:
Const.DataCollection.MessageCodes.Events.PruneLocalPosts.
SUCCESS,
Const.DataCollection.ParamNames.NUM_RECEIVED:
len(list(messagesToDelete)),
Const.DataCollection.ParamNames.NUM_DELETED:
numMessagesDeleted
})
except Exception as e:
# log error and info on error
DataCollector.logServerError(e)
DataCollector.logServerEvent(
TAG, {
Const.DataCollection.ParamNames.MESSAGE_CODE:
Const.DataCollection.MessageCodes.Events.PruneLocalPosts.
SERVER_ERROR,
Const.DataCollection.ParamNames.NUM_RECEIVED:
len(list(messagesToDelete)),
Const.DataCollection.ParamNames.NUM_DELETED:
numMessagesDeleted
})
def CheckAndPruneThreads():
import JokrBackend.DataCollection.ContentManager as ContentManager
import JokrBackend.DataCollection.DataCollector as DataCollector
# get the thread count
numOfCurrentThreads = Thread.objects.count()
# If the board is full already, prune a thread
if numOfCurrentThreads > settings.BOARD_THREAD_LIMIT:
numThreadsToPrune = numOfCurrentThreads - settings.BOARD_THREAD_LIMIT
# Get the lowest bump order threads to be pruned
threadsToPrune = GetLiveThreadsByTimeLastActive(
ascending=True, limit=numThreadsToPrune)
for thread in threadsToPrune:
# log info about the pruned thread
DataCollector.logServerEvent(
Const.Tags.Events.PRUNE_THREAD,
{Const.DataCollection.ParamNames.THREAD_ID: thread.id})
ContentManager.DeleteContent(thread.id)