def on_any_event(event):
bucket_name = "sn-dump-collector"
pod_name = os.getenv("POD_NAME") if os.getenv(
"POD_NAME") is not None else "POD_NAME"
namespace_name = os.getenv("NAMESPACE_NAME") if os.getenv(
"NAMESPACE_NAME") is not None else "POD_NAME"
key_name = datetime.today().strftime(
'%Y-%m-%d') + "_" + namespace_name + "_" + pod_name
if event.is_directory:
return None
elif event.event_type == 'created':
# Event is created, you can process it now
print("Watchdog received created event - % s." % event.src_path)
S3Utils.write_to_s3(bucket=bucket_name,
keyname=key_name,
file_path=event.src_path)
elif event.event_type == 'modified':
# Event is modified, you can process it now
print("Watchdog received modified event - % s." % event.src_path)
S3Utils.write_to_s3(bucket=bucket_name,
keyname=key_name,
file_path=event.src_path)
def LoadData():
target_url = "https://s3-us-west-2.amazonaws.com/css490/input.txt"
headers = {}
headers['User-Agent'] = 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.27 Safari/537.17'
# get data from txt file in s3 bucket
try:
request = urllib.request.Request(target_url, headers=headers)
resp = urllib.request.urlopen(request).read().decode('UTF-8')
except Exception as e:
errorString = "Error: %s" %e
return errorString
# creation of dynamodb table
currTableName = DynamoUtils.CreateTable(GetDynamoDbTableName())
# attempt to create s3 bucket to store parsed data
bucketCreationName = S3Utils.CreateS3Bucket(GetS3BucketName())
# create list of lines of member data from txt file
textLines = resp.split('\r\n')
# split each line of data into lists of individual strings
dataList = [[line.split(' ')[i] for i in range(len(line.split(' ')))] for line in textLines]
cleanedDataList = S3Utils.CleanDataString(dataList)
S3Utils.WriteDataToS3(cleanedDataList, GetS3BucketName())
# upload data from local file to dynamodb
localFileToDynamo = DynamoUtils.InputLocalFileDataToDynamoDB(cleanedDataList, currTableName)
message = "Loaded data to s3 bucket, then dynamodb."
return render_template('index.html', loadMessage=message)
def get_link():
random = randomString(20)
if not id in S3Utils.get_events_id():
rd = '/tmp/' + randomString(20) + '/'
os.makedirs(rd)
zf = zipfile.ZipFile(rd + 'archive-1.zip', mode='w')
zf.close()
S3Utils.addEmptyArch('archives/' + random + '/archive-1.zip',
rd + 'archive-1.zip')
return random
else:
return get_link()
def event(request, id):
if not exist(id):
return redirect('/')
template = loader.get_template('event.html')
context = {}
context['id'] = id
context['archives'] = S3Utils.get_available_archives(id)
context['region'] = settings.REGION
context['static'] = settings.BUCKET_STATIC
context['bucket_archives'] = settings.BUCKET_ARCHIVES
context['url_static'] = settings.URL_STATIC
context['url_archives'] = settings.URL_ARCHIVES
if request.method == 'POST':
form = PictureForm(request.POST, request.FILES)
if form.is_valid():
context['filenames'], context['errors'] = handle_uploaded_file(
id, 'upload/' + str(id) + '/', request.FILES['file'],
request.FILES['file'].name)
form = PictureForm()
context['form'] = form
return render_to_response('event.html',
context,
context_instance=RequestContext(request))
else:
form = PictureForm()
context['form'] = form
return render_to_response('event.html',
context,
context_instance=RequestContext(request))
def archive(request, id):
print('archive')
localpath = S3Utils.getArchive(id)
response = StreamingHttpResponse((line for line in open(localpath, 'r')))
response['Content-Disposition'] = "attachment; filename=Archive.zip"
response['Content-Length'] = os.path.getsize(localpath)
return response
def DeleteData():
noDataMessage = "No data to delete yet"
try:
response = DynamoUtils.DeleteDynamoTable(GetDynamoDbTableName())
if not response:
return render_template('index.html', deleteMessage=noDataMessage)
except Exception as e:
print("Error deleting dynamodb from appl.py: ", e)
return render_template('index.html', deleteMessage=noDataMessage)
try:
S3Utils.DeleteS3BucketObject(GetS3BucketName(), GetS3FileName())
except Exception as e:
print("Error deleting s3 bucket from app.py: ", e)
return render_template('index.html', deleteMessage=noDataMessage)
message = "Deleted s3 bucket object, and dynamodb."
return render_template('index.html', deleteMessage=message)
def handle_uploaded_file(event, folder, file, filename):
randFolder = randomString(20) + '/'
localTempPath = folder + randFolder + filename
filepath = folder + filename
pictures = []
returned = []
error = []
if not os.path.exists(folder + randFolder):
os.makedirs(folder + randFolder)
spl = filename.split('.')
ext = spl[len(spl) - 1]
if ext == 'zip':
destination = open(localTempPath, 'wb+')
for chunk in file.chunks():
destination.write(chunk)
destination.close()
size = os.path.getsize(localTempPath)
if zip_error(localTempPath):
error.append(filename + ' is not a valid zip file')
os.remove(localTempPath)
return returned, error
elif size > 100000000:
error.append(filename + ' is too big (' + str(size / 1000) +
' KB)')
os.remove(localTempPath)
return returned, error
fh = open(localTempPath, 'rb')
z = zipfile.ZipFile(fh)
newzippath = folder + randFolder + randomString(20) + '.zip'
for name in z.namelist():
if not '/' in name:
outpath = folder + randFolder
z.extract(name, outpath)
if imghdr.what(outpath + name) == None:
print(name + ' is not an image')
error.append(name + ' is not an image')
os.remove(outpath + name)
else:
#call(["convert", outpath + name, "-resize", "2000x2000>", outpath + name])
pictures.append(outpath + name)
returned.append(name)
elif '__MACOSX' == name.split('/')[0]:
pass
elif '/' != name[-1]:
outpath = folder + randFolder
z.extract(name, outpath)
src = outpath + name
newname = name.replace('/', '_')
dst = outpath + newname
copyfile(src, dst)
rmtree(folder + randFolder + name.split('/')[0])
if imghdr.what(dst) == None:
print(newname + ' is not an image')
error.append(newname + ' is not an image')
os.remove(dst)
else:
#call(["convert", dst, "-resize", "2000x2000>", dst])
pictures.append(dst)
returned.append(newname)
fh.close()
print("Zip extracted")
os.remove(localTempPath)
try:
S3Utils.addPictures(event, pictures)
print("Files sended")
except:
print("ERREUR LORS DE L'UPLOAD")
raise
return [], ["ERREUR LORS DE L'UPLOAD"]
else:
write_file(localTempPath, file)
size = os.path.getsize(localTempPath)
if imghdr.what(localTempPath) == None:
print(filename + ' is not an image')
error.append(filename + ' is not an image')
os.remove(localTempPath)
elif size > 100000000:
error.append(filename + ' is too big (' + str(size / 1000) +
' KB)')
os.remove(localTempPath)
return returned, error
else:
#call(["convert", localTempPath, "-resize", "2000x2000>", localTempPath])
pictures.append(localTempPath)
returned = [filename]
try:
S3Utils.addPictures(event, pictures)
except:
print("ERREUR LORS DE L'UPLOAD")
raise
return [], ["ERREUR LORS DE L'UPLOAD"]
return returned, error
def exist(id):
return (id in S3Utils.get_events_id())