def tearDown(self):
shutil.rmtree(self.workdir)
with closing(S3Connection()) as s3:
bucket = Bucket(s3, self.output_dir.netloc)
prefix = self.output_dir.path[1:]
for key in bucket.list(prefix=prefix):
assert key.name.startswith(prefix)
key.delete()
def cleanup_and_validate():
valid_output = False
expected_output = posixpath.basename(sample.path)
with closing(S3Connection()) as s3:
b = Bucket(s3, output_dir.netloc)
path = output_dir.path[1:]
for k in b.list(prefix=path):
assert k.name.startswith(path)
if k.name[len(path):] == '/' + expected_output:
# FIXME: We may want to validate the output a bit more
valid_output = True
else:
log.warn('Unexpected output file %s/%s', output_dir.geturl(), k.name)
k.delete()
assert valid_output, 'Did not find expected output file'
def checkS3FileExists(conf_file,bucket,path): conf = json.load(open(conf_file)) key = conf['s3_key'] skey = conf['s3_skey'] conn = S3Connection(key, skey) b = Bucket(conn, bucket) found_file = 'false' for x in b.list(prefix = path[1:]): if(len(str(x)) > 0): print(path) found_file = 'true' break return found_file
def handle(self, *args, **options):
conn = S3Connection(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY)
bucket = Bucket(conn, settings.AWS_STORAGE_BUCKET_AI_NAME)
s3_files = set()
for key in bucket.list():
s3_files.add(key.name)
startdate = timezone.now() - timedelta(days=int(options["no_of_days_back"]))
attachments = AIAttachment.objects.select_related('ai_pics').filter(ai_pics__created_at__gte=startdate)
for attachment in attachments:
if attachment.attachment not in s3_files:
print(attachment.attachment)
attachment.delete()
with connection.cursor() as cursor:
cursor.execute(
'delete from ai_pics_aipics WHERE '
'(select count(*) from ai_pics_aiattachment where ai_pics_id=ai_pics_aipics.id) =0'
)
def remove_s3_file(filePath):
if settings.ENVIRONMENT_PRODUCTION:
conn = S3Connection(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY)
bucket = Bucket(conn,settings.AWS_STORAGE_BUCKET_NAME)
for key in bucket.list(prefix=filePath):
key.delete()
def process(access_key=None, secret_key=None, bucket_name=None, video_path=None, video=None):
for_deleting = False
if len(sys.argv) > 1:
a = sys.argv[1]
ACCESS_KEY = sys.argv[2]
SECRET_KEY = sys.argv[3]
BUCKET_NAME = sys.argv[4]
database.createDatabase(a)
else:
for_deleting = True
ACCESS_KEY = access_key
SECRET_KEY = secret_key
BUCKET_NAME = bucket_name
database.createDatabase()
aws_s3_connection = S3Connection(ACCESS_KEY, SECRET_KEY)
aws_s3_bucket = Bucket(aws_s3_connection, BUCKET_NAME)
session = database.DBSession()
object_keys = []
# This is if the CRON job is running and is removing flagged videos
if for_deleting is False:
flagged_content = session.query(FlaggedVideoModel).all()
if len(flagged_content) > 0:
time_stamp_now = calendar.timegm(datetime.utcnow().timetuple())
for content in flagged_content:
if content.timeStamp <= time_stamp_now:
video = content.video
favourites_of_video = session.query(VideoFavourite).filter(VideoFavourite.video_id == video.video_id).all()
for key in aws_s3_bucket.list(prefix=content.video_path):
object_keys.append(key)
if len(favourites_of_video) > 0:
for fv in favourites_of_video:
session.delete(fv)
session.delete(content)
session.delete(video)
# This is for when you are deleting a video from the timeline
elif for_deleting is True and video is not None and video_path is not '' and video_path is not None:
favourites_of_video = session.query(VideoFavourite).filter(VideoFavourite.video_id == video.video_id).all()
flags_for_video = session.query(FlaggedVideoModel).filter(FlaggedVideoModel.video_id == video.video_id).all()
# Collect the AWS S3 objects to delete
for key in aws_s3_bucket.list(prefix=video_path):
object_keys.append(key)
# Collect the Video Favourites
if len(favourites_of_video) > 0:
for fv in favourites_of_video:
session.delete(fv)
# Collect the Video Flags
if len(flags_for_video) > 0:
for fv in flags_for_video:
session.delete(fv)
try:
if len(object_keys) > 0:
aws_s3_bucket.delete_keys(object_keys)
session.commit()
session.close()
return True
except exc.SQLAlchemyError:
session.close()
return False
