コード例 #1
0
    def on_any_event(event):
        bucket_name = "sn-dump-collector"
        pod_name = os.getenv("POD_NAME") if os.getenv(
            "POD_NAME") is not None else "POD_NAME"
        namespace_name = os.getenv("NAMESPACE_NAME") if os.getenv(
            "NAMESPACE_NAME") is not None else "POD_NAME"

        key_name = datetime.today().strftime(
            '%Y-%m-%d') + "_" + namespace_name + "_" + pod_name

        if event.is_directory:
            return None

        elif event.event_type == 'created':
            # Event is created, you can process it now
            print("Watchdog received created event - % s." % event.src_path)
            S3Utils.write_to_s3(bucket=bucket_name,
                                keyname=key_name,
                                file_path=event.src_path)
        elif event.event_type == 'modified':
            # Event is modified, you can process it now
            print("Watchdog received modified event - % s." % event.src_path)
            S3Utils.write_to_s3(bucket=bucket_name,
                                keyname=key_name,
                                file_path=event.src_path)
コード例 #2
0
def LoadData():
    target_url = "https://s3-us-west-2.amazonaws.com/css490/input.txt"
    
    headers = {}
    headers['User-Agent'] = 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.27 Safari/537.17'

    # get data from txt file in s3 bucket
    try:
        request = urllib.request.Request(target_url, headers=headers)
        resp = urllib.request.urlopen(request).read().decode('UTF-8')
    except Exception as e:
        errorString = "Error: %s" %e
        return errorString

    # creation of dynamodb table
    currTableName = DynamoUtils.CreateTable(GetDynamoDbTableName())

    # attempt to create s3 bucket to store parsed data
    bucketCreationName = S3Utils.CreateS3Bucket(GetS3BucketName())

    # create list of lines of member data from txt file
    textLines = resp.split('\r\n')

    # split each line of data into lists of individual strings
    dataList = [[line.split(' ')[i] for i in range(len(line.split(' ')))] for line in textLines]
    
    cleanedDataList = S3Utils.CleanDataString(dataList)

    S3Utils.WriteDataToS3(cleanedDataList, GetS3BucketName())

    # upload data from local file to dynamodb
    localFileToDynamo = DynamoUtils.InputLocalFileDataToDynamoDB(cleanedDataList, currTableName)

    message = "Loaded data to s3 bucket, then dynamodb."
    return render_template('index.html', loadMessage=message)
コード例 #3
0
def get_link():
    random = randomString(20)
    if not id in S3Utils.get_events_id():
        rd = '/tmp/' + randomString(20) + '/'
        os.makedirs(rd)
        zf = zipfile.ZipFile(rd + 'archive-1.zip', mode='w')
        zf.close()
        S3Utils.addEmptyArch('archives/' + random + '/archive-1.zip',
                             rd + 'archive-1.zip')
        return random
    else:
        return get_link()
コード例 #4
0
def event(request, id):
    if not exist(id):
        return redirect('/')
    template = loader.get_template('event.html')
    context = {}
    context['id'] = id
    context['archives'] = S3Utils.get_available_archives(id)
    context['region'] = settings.REGION
    context['static'] = settings.BUCKET_STATIC
    context['bucket_archives'] = settings.BUCKET_ARCHIVES
    context['url_static'] = settings.URL_STATIC
    context['url_archives'] = settings.URL_ARCHIVES
    if request.method == 'POST':
        form = PictureForm(request.POST, request.FILES)
        if form.is_valid():
            context['filenames'], context['errors'] = handle_uploaded_file(
                id, 'upload/' + str(id) + '/', request.FILES['file'],
                request.FILES['file'].name)

        form = PictureForm()
        context['form'] = form
        return render_to_response('event.html',
                                  context,
                                  context_instance=RequestContext(request))
    else:
        form = PictureForm()
        context['form'] = form
        return render_to_response('event.html',
                                  context,
                                  context_instance=RequestContext(request))
コード例 #5
0
def archive(request, id):
    print('archive')
    localpath = S3Utils.getArchive(id)

    response = StreamingHttpResponse((line for line in open(localpath, 'r')))
    response['Content-Disposition'] = "attachment; filename=Archive.zip"
    response['Content-Length'] = os.path.getsize(localpath)
    return response
コード例 #6
0
def DeleteData():
    noDataMessage = "No data to delete yet"
    
    try:
        response = DynamoUtils.DeleteDynamoTable(GetDynamoDbTableName())
        if not response:
            return render_template('index.html', deleteMessage=noDataMessage)
    except Exception as e:
        print("Error deleting dynamodb from appl.py: ", e)
        return render_template('index.html', deleteMessage=noDataMessage)
    try:
        S3Utils.DeleteS3BucketObject(GetS3BucketName(), GetS3FileName())
    except Exception as e:
        print("Error deleting s3 bucket from app.py: ", e)
        return render_template('index.html', deleteMessage=noDataMessage)
    message = "Deleted s3 bucket object, and dynamodb."
    return render_template('index.html', deleteMessage=message)
コード例 #7
0
def handle_uploaded_file(event, folder, file, filename):
    randFolder = randomString(20) + '/'
    localTempPath = folder + randFolder + filename
    filepath = folder + filename
    pictures = []
    returned = []
    error = []

    if not os.path.exists(folder + randFolder):
        os.makedirs(folder + randFolder)

    spl = filename.split('.')
    ext = spl[len(spl) - 1]

    if ext == 'zip':

        destination = open(localTempPath, 'wb+')
        for chunk in file.chunks():
            destination.write(chunk)
        destination.close()
        size = os.path.getsize(localTempPath)
        if zip_error(localTempPath):
            error.append(filename + ' is not a valid zip file')
            os.remove(localTempPath)
            return returned, error
        elif size > 100000000:
            error.append(filename + ' is too big (' + str(size / 1000) +
                         ' KB)')
            os.remove(localTempPath)
            return returned, error

        fh = open(localTempPath, 'rb')
        z = zipfile.ZipFile(fh)
        newzippath = folder + randFolder + randomString(20) + '.zip'
        for name in z.namelist():
            if not '/' in name:
                outpath = folder + randFolder
                z.extract(name, outpath)
                if imghdr.what(outpath + name) == None:
                    print(name + ' is not an image')
                    error.append(name + ' is not an image')
                    os.remove(outpath + name)
                else:
                    #call(["convert", outpath + name, "-resize", "2000x2000>", outpath + name])
                    pictures.append(outpath + name)
                    returned.append(name)
            elif '__MACOSX' == name.split('/')[0]:
                pass
            elif '/' != name[-1]:
                outpath = folder + randFolder
                z.extract(name, outpath)
                src = outpath + name
                newname = name.replace('/', '_')
                dst = outpath + newname
                copyfile(src, dst)
                rmtree(folder + randFolder + name.split('/')[0])

                if imghdr.what(dst) == None:
                    print(newname + ' is not an image')
                    error.append(newname + ' is not an image')
                    os.remove(dst)
                else:
                    #call(["convert", dst, "-resize", "2000x2000>", dst])
                    pictures.append(dst)
                    returned.append(newname)
        fh.close()
        print("Zip extracted")
        os.remove(localTempPath)
        try:
            S3Utils.addPictures(event, pictures)
            print("Files sended")
        except:
            print("ERREUR LORS DE L'UPLOAD")
            raise
            return [], ["ERREUR LORS DE L'UPLOAD"]
    else:
        write_file(localTempPath, file)
        size = os.path.getsize(localTempPath)
        if imghdr.what(localTempPath) == None:
            print(filename + ' is not an image')
            error.append(filename + ' is not an image')
            os.remove(localTempPath)
        elif size > 100000000:
            error.append(filename + ' is too big (' + str(size / 1000) +
                         ' KB)')
            os.remove(localTempPath)
            return returned, error
        else:
            #call(["convert", localTempPath, "-resize", "2000x2000>", localTempPath])
            pictures.append(localTempPath)
            returned = [filename]
        try:
            S3Utils.addPictures(event, pictures)
        except:
            print("ERREUR LORS DE L'UPLOAD")
            raise
            return [], ["ERREUR LORS DE L'UPLOAD"]

    return returned, error
コード例 #8
0
def exist(id):
    return (id in S3Utils.get_events_id())