from urllib.parse import urljoin
from flask import abort, url_for, request, jsonify, Response, make_response, Blueprint
from cache import get_cache
import config
from errors import BaseException, APIUnauthorized, APIForbidden, APINotFound
from loggers import getLogger
from models import User
logger = getLogger(__name__)
class RestBase():
auth_required = False
pre_methods = []
def __init__(self):
self.map = {
'GET': getattr(self, 'get', None),
'HEAD': getattr(self, 'get', None),
'POST': getattr(self, 'post', None),
'DELETE': getattr(self, 'delete', None),
'PUT': getattr(self, 'put', None),
}
def get_methods(self):
methods = [
method for method, func in self.map.items() if func is not None
]
methods.append('OPTIONS')
(although it's hard to think someone would be desperate enough to rip
off such hastily written crap code. At some pt I'll clean it up, meanwhile
Use at your own risk)
"""
import sys
import loggers
import inspect
import datetime
from time import sleep
from libtools.js import export_json_object
import boto3
from botocore.exceptions import ClientError
logger = loggers.getLogger('1.0')
DEBUGMODE = True # will not retag any resources
SUMMARY_REPORT = False # print summary report only
regions = ['ap-southeast-1', 'eu-west-1', 'us-east-1']
profiles = [
'gcreds-phht-gen-ra1-pr',
'gcreds-phht-gen-ra2-pr',
'gcreds-phht-gen-ra3-pr',
'gcreds-phht-gen-ra4-pr',
'gcreds-phht-gen-ra5-pr',
]
profiles = ['gcreds-phht-gen-ra3-pr']
import unittest
import numpy as np
import sklearn
from sklearn import model_selection
import sys, os
sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
import pretrainedTransformersPipeline
import loggers
import inputFunctions
logger = loggers.getLogger("PretrainedTransformersPipeLineTest", debug=True)
class PretrainedTransformersPipeLineTest(unittest.TestCase):
def __init__(self, *args, **kwargs):
super(PretrainedTransformersPipeLineTest,
self).__init__(*args, **kwargs)
def assertArrayEqual(self, arr1: np.ndarray, arr2: np.ndarray):
for d1, d2 in zip(arr1.shape, arr2.shape):
self.assertEqual(d1, d2)
for i in range(arr1.size):
# print(arr1.take(i))
self.assertEqual(arr1.take(i), arr2.take(i))
# def test_loading(self):
# newBowp = pretrainedTransformersPipeline.PretrainedTransformersPipeLine(loadFunction=inputFunctions.loadDataForUnitTesting)
# newBowp.loadData()
# def test_train(self):
supporting documentation
Additional terms may be found in the complete license agreement:
https://bitbucket.org/blakeca00/lambda-library-python/src/master/LICENSE.md
"""
import inspect
import boto3
from botocore.exceptions import ClientError
import loggers
from _version import __version__
# lambda custom log object
logger = loggers.getLogger(__version__)
class AssumeAWSRole():
""" class def for assuming roles in AWS """
def __init__(self, account, role_name, profile=None):
self.role = role_name
self.account_number = str(account)
self.profile = profile
self.credentials = self.assume_role(role_name, self.account_number)
self.status = {'STATUS': ''}
def assume_role(self, account, role):
"""
Summary:
Assumes a DynamoDB role in 'destination' AWS account
import loggers
# LOGGING CONFIGURATION
# ---------------------
# create logger
logger = loggers.getLogger(__name__)
logger.setLevel(loggers.DEBUG)
# create console handler and set level to debug
ch = loggers.StreamHandler()
ch.setLevel(loggers.DEBUG)
# create formatter
formatter = loggers.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
# add formatter to ch
ch.setFormatter(formatter)
# add ch to logger
logger.addHandler(ch)
# 'application' code
logger.debug('debug message')
logger.info('info message')
logger.warn('warn message')
logger.error('error message')
logger.critical('critical message')
import unittest
import transformers
import transformersModel
import numpy as np
import tensorflow as tf
import sys, os
sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
import inputFunctions
from preprocessing.pretrainedTransformersPipeline import PretrainedTransformersPipeLine
from models.modelMaps import getModelMapAvailableNames
import loggers
logger = loggers.getLogger("RobertaModelTest", debug=True)
class TransformersModelTest(unittest.TestCase):
def __init__(self, *args, **kwargs):
super(TransformersModelTest, self).__init__(*args, **kwargs)
def test_createModelRoberta(self):
logger.debug("Testing createModel Roberta")
roberta = transformersModel.TransformersModel()
roberta.pipeLine.loadFunction = inputFunctions.loadDataForUnitTesting
roberta.loadData()
roberta.createModel()
def test_createAllTransformerModels(self):
for name in getModelMapAvailableNames():
logger.debug(f"Testing createModel for {name}")
transfModel = transformersModel.TransformersModel(
pipeLine={'modelName': name}, modelName=name)
import tensorflow as tf
import numpy as np
import sklearn
from sklearn import model_selection
import typing
import os, sys
sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
from preprocessing.bagOfWordsPipeline import BagOfWordsPipeLine
from models.Model import ModelConstruction
import inputFunctions
import loggers
logger = loggers.getLogger("BagOfWordsModel", True)
class BagOfWords2LayerModel(ModelConstruction):
def __init__(self, dataPath: str = None, pipeLine=BagOfWordsPipeLine()):
logger.info("BagOfWordsModel created")
self.dataPath = dataPath
self._model = None
self.all_data = []
self.pipeLine = pipeLine
self.paddedSequencesPos = []
self.paddedSequencesNeg = []
self._registeredMetrics = []
self._dataLoaded = False
def loadData(self):
self.pipeLine.loadData()
self.pipeLine.trainTokenizer()
self.paddedSequencesPos = self.pipeLine.textsPosToPaddedSequences()
import tensorflow as tf
import os, sys
sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
from preprocessing import InputPipeline
import inputFunctions
import loggers
import numpy as np
import typing
logger = loggers.getLogger("BagOfWordsPipeline", debug=True)
class BagOfWordsPipeLine(InputPipeline.InputPipeline):
def __init__(self, dataPath=None, loadFunction:callable=None):
logger.info("BagOfWordsPipeline created")
self.dataPath = dataPath
self._tokenizer = None
self.allData = []
self.dataPos = []
self.dataNeg = []
self.num_words:int = None
if loadFunction == None:
self.loadFunction = inputFunctions.loadData
else:
self.loadFunction = loadFunction
def loadData(self):
train_pos, train_neg, test_data = self.loadFunction(self.dataPath)
self.dataPos = train_pos
self.dataNeg = train_neg
self.allData = train_pos + train_neg
import numpy as np
import sklearn
from sklearn import model_selection
from sklearn.metrics import accuracy_score
import typing
import os, sys
sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
from preprocessing.pretrainedTransformersPipeline import PretrainedTransformersPipeLine, torchOrTFEnum
from models.Model import ModelConstruction
from modelMaps import mapStrToTransformerModel
from preprocessing.pipelineMaps import mapStrToTransformersTokenizer
import inputFunctions
import loggers
import pdb
logger = loggers.getLogger("RobertaModel", True)
def getDefaultTokenizer(
loadFunction: typing.Callable[[str], typing.Tuple[list, list,
list]] = None):
if loadFunction == None:
return PretrainedTransformersPipeLine(
tokenizer=transformers.RobertaTokenizer,
pretrainedTokenizerName='roberta-base')
else:
return PretrainedTransformersPipeLine(
loadFunction=loadFunction,
tokenizer=transformers.RobertaTokenizer,
pretrainedTokenizerName='roberta-base')