def on_any_event(event): bucket_name = "sn-dump-collector" pod_name = os.getenv("POD_NAME") if os.getenv( "POD_NAME") is not None else "POD_NAME" namespace_name = os.getenv("NAMESPACE_NAME") if os.getenv( "NAMESPACE_NAME") is not None else "POD_NAME" key_name = datetime.today().strftime( '%Y-%m-%d') + "_" + namespace_name + "_" + pod_name if event.is_directory: return None elif event.event_type == 'created': # Event is created, you can process it now print("Watchdog received created event - % s." % event.src_path) S3Utils.write_to_s3(bucket=bucket_name, keyname=key_name, file_path=event.src_path) elif event.event_type == 'modified': # Event is modified, you can process it now print("Watchdog received modified event - % s." % event.src_path) S3Utils.write_to_s3(bucket=bucket_name, keyname=key_name, file_path=event.src_path)
def LoadData(): target_url = "https://s3-us-west-2.amazonaws.com/css490/input.txt" headers = {} headers['User-Agent'] = 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.27 Safari/537.17' # get data from txt file in s3 bucket try: request = urllib.request.Request(target_url, headers=headers) resp = urllib.request.urlopen(request).read().decode('UTF-8') except Exception as e: errorString = "Error: %s" %e return errorString # creation of dynamodb table currTableName = DynamoUtils.CreateTable(GetDynamoDbTableName()) # attempt to create s3 bucket to store parsed data bucketCreationName = S3Utils.CreateS3Bucket(GetS3BucketName()) # create list of lines of member data from txt file textLines = resp.split('\r\n') # split each line of data into lists of individual strings dataList = [[line.split(' ')[i] for i in range(len(line.split(' ')))] for line in textLines] cleanedDataList = S3Utils.CleanDataString(dataList) S3Utils.WriteDataToS3(cleanedDataList, GetS3BucketName()) # upload data from local file to dynamodb localFileToDynamo = DynamoUtils.InputLocalFileDataToDynamoDB(cleanedDataList, currTableName) message = "Loaded data to s3 bucket, then dynamodb." return render_template('index.html', loadMessage=message)
def get_link(): random = randomString(20) if not id in S3Utils.get_events_id(): rd = '/tmp/' + randomString(20) + '/' os.makedirs(rd) zf = zipfile.ZipFile(rd + 'archive-1.zip', mode='w') zf.close() S3Utils.addEmptyArch('archives/' + random + '/archive-1.zip', rd + 'archive-1.zip') return random else: return get_link()
def event(request, id): if not exist(id): return redirect('/') template = loader.get_template('event.html') context = {} context['id'] = id context['archives'] = S3Utils.get_available_archives(id) context['region'] = settings.REGION context['static'] = settings.BUCKET_STATIC context['bucket_archives'] = settings.BUCKET_ARCHIVES context['url_static'] = settings.URL_STATIC context['url_archives'] = settings.URL_ARCHIVES if request.method == 'POST': form = PictureForm(request.POST, request.FILES) if form.is_valid(): context['filenames'], context['errors'] = handle_uploaded_file( id, 'upload/' + str(id) + '/', request.FILES['file'], request.FILES['file'].name) form = PictureForm() context['form'] = form return render_to_response('event.html', context, context_instance=RequestContext(request)) else: form = PictureForm() context['form'] = form return render_to_response('event.html', context, context_instance=RequestContext(request))
def archive(request, id): print('archive') localpath = S3Utils.getArchive(id) response = StreamingHttpResponse((line for line in open(localpath, 'r'))) response['Content-Disposition'] = "attachment; filename=Archive.zip" response['Content-Length'] = os.path.getsize(localpath) return response
def DeleteData(): noDataMessage = "No data to delete yet" try: response = DynamoUtils.DeleteDynamoTable(GetDynamoDbTableName()) if not response: return render_template('index.html', deleteMessage=noDataMessage) except Exception as e: print("Error deleting dynamodb from appl.py: ", e) return render_template('index.html', deleteMessage=noDataMessage) try: S3Utils.DeleteS3BucketObject(GetS3BucketName(), GetS3FileName()) except Exception as e: print("Error deleting s3 bucket from app.py: ", e) return render_template('index.html', deleteMessage=noDataMessage) message = "Deleted s3 bucket object, and dynamodb." return render_template('index.html', deleteMessage=message)
def handle_uploaded_file(event, folder, file, filename): randFolder = randomString(20) + '/' localTempPath = folder + randFolder + filename filepath = folder + filename pictures = [] returned = [] error = [] if not os.path.exists(folder + randFolder): os.makedirs(folder + randFolder) spl = filename.split('.') ext = spl[len(spl) - 1] if ext == 'zip': destination = open(localTempPath, 'wb+') for chunk in file.chunks(): destination.write(chunk) destination.close() size = os.path.getsize(localTempPath) if zip_error(localTempPath): error.append(filename + ' is not a valid zip file') os.remove(localTempPath) return returned, error elif size > 100000000: error.append(filename + ' is too big (' + str(size / 1000) + ' KB)') os.remove(localTempPath) return returned, error fh = open(localTempPath, 'rb') z = zipfile.ZipFile(fh) newzippath = folder + randFolder + randomString(20) + '.zip' for name in z.namelist(): if not '/' in name: outpath = folder + randFolder z.extract(name, outpath) if imghdr.what(outpath + name) == None: print(name + ' is not an image') error.append(name + ' is not an image') os.remove(outpath + name) else: #call(["convert", outpath + name, "-resize", "2000x2000>", outpath + name]) pictures.append(outpath + name) returned.append(name) elif '__MACOSX' == name.split('/')[0]: pass elif '/' != name[-1]: outpath = folder + randFolder z.extract(name, outpath) src = outpath + name newname = name.replace('/', '_') dst = outpath + newname copyfile(src, dst) rmtree(folder + randFolder + name.split('/')[0]) if imghdr.what(dst) == None: print(newname + ' is not an image') error.append(newname + ' is not an image') os.remove(dst) else: #call(["convert", dst, "-resize", "2000x2000>", dst]) pictures.append(dst) returned.append(newname) fh.close() print("Zip extracted") os.remove(localTempPath) try: S3Utils.addPictures(event, pictures) print("Files sended") except: print("ERREUR LORS DE L'UPLOAD") raise return [], ["ERREUR LORS DE L'UPLOAD"] else: write_file(localTempPath, file) size = os.path.getsize(localTempPath) if imghdr.what(localTempPath) == None: print(filename + ' is not an image') error.append(filename + ' is not an image') os.remove(localTempPath) elif size > 100000000: error.append(filename + ' is too big (' + str(size / 1000) + ' KB)') os.remove(localTempPath) return returned, error else: #call(["convert", localTempPath, "-resize", "2000x2000>", localTempPath]) pictures.append(localTempPath) returned = [filename] try: S3Utils.addPictures(event, pictures) except: print("ERREUR LORS DE L'UPLOAD") raise return [], ["ERREUR LORS DE L'UPLOAD"] return returned, error
def exist(id): return (id in S3Utils.get_events_id())