def updateWithS3(self, data,
inputKeyName = 'input-data-name',
invocationType = InvocationType.event,
user= None, pw= None):
# put users if not specified
user = user or self.user; pw = pw or self.pw
# extract function name and inputbucket name
inputBucketName = self.functionNames.inputBucket()
functionName = self.functionNames.updateS3()
logging.info(f'bucket is {inputBucketName}')
# save data to s3
S3.save(key = inputKeyName,
objectToSave = data ,
bucket = inputBucketName,
user=user, pw=pw)
logging.info(f'data is saved to s3, invoking ingestion function')
# call lambda function
inputValue = Event(body = json.dumps({ 'key': inputKeyName })).to_dict()
logging.info(f'input to lambda is {inputValue}')
lambdaResponse = self.lambdaClient.invoke(
functionName= functionName ,
input=inputValue,
invocationType= invocationType )
logging.info(f'lambdaResponse is {lambdaResponse}')
if invocationType == 'Event': return "successfully sent event, please watch your slack"
if lambdaResponse: return self.returnLambdaResponse(lambdaResponse)
def updateWithS3(self,
data,
inputKeyName='input-data-name',
invocationType='RequestResponse',
user=None,
pw=None):
# put users if not specified
user = user or self.user
pw = pw or self.pw
# extract function name and inputbucket name
inputBucketName = self.functionNames.inputBucket()
functionName = self.functionNames.updateS3()
logging.info(f'bucket is {inputBucketName}')
# save data to s3
S3.save(key=inputKeyName,
objectToSave=data,
bucket=inputBucketName,
user=user,
pw=pw)
logging.info(f'data is saved to s3, invoking ingestion function')
# call lambda function
lambdaPayload = {'key': inputKeyName}
logging.info(f'input to lambda is {lambdaPayload}')
return self.lambdaClient.invoke(functionName=functionName,
input=lambdaPayload,
invocationType=invocationType)
def lambdaDumpOnlineS3(event, *args):
print(f'ecommece col list is {ECOMMERCE_COL_LIST}')
# get all products from db
df: pd.DataFrame = pd.DataFrame([i.data for i in ProductDatabase.scan()])
# get online list from ECOMMERCE_COL_LIST
onlineList: List[str] = yaml.load(requests.get(ECOMMERCE_COL_LIST).content)
# filter df for item
## condition 1 master online is true
condition1 = df['master_online'] == True
## condition 2 hema_name_en is not blank
condition2 = df['hema_name_en'] != ''
## filtered df
onlineDf: pd.DataFrame = df[condition1 & condition2].loc[:, onlineList]
### log shape and size
print('shape is:', onlineDf.shape)
print('size is:',
sys.getsizeof(onlineDf.to_json(orient='split')) / 1e6, 'Mb')
# save file as gzip
key = 'onlineData'
bucket = INVENTORY_BUCKET_NAME
path = '/tmp/inventory.json'
## export to gzip
onlineDf.to_json(path, orient='split', compression='gzip')
## upload file to s3
S3.saveFile(key=key,
path=path,
bucket=bucket,
ExtraArgs={
**ExtraArgs.gzip,
**ExtraArgs.publicRead
})
return Response.returnSuccess()
def loadRemoteCache(key='', bucket='', **kwargs):
'''
load cache of the dataframe from S3 bucket \n
key: str: the name (key) of the dataframe to be loaded from the S3 bucket \n
bucket: str: the name of the bucket to load the dataframe
'''
path = '/tmp/tmpPath'
S3.loadFile(key, path=path, bucket=bucket, **kwargs)
df = pd.read_feather(path)
return df
def imageToS3(image: Image, bucket: str, key: str):
'''
saves the image inputted to the S3 bucket \n
image: Image: the image that is going to be saved \n
bucket: str: the name of the bucket where the image will be saved \n,
key: str: the key of the image
'''
path = '/tmp/tmpImage'
print(f'saving image to {bucket}/{key}')
image.save(path, format='png')
S3.saveFile(key, path, bucket)
return True
def saveRemoteHash(data: pd.DataFrame, key='', bucket='', **kwargs):
'''
save hash of the dataframe to S3 bucket \n
data:pd.DataFrame: dataframe to save \n
key: str: the name (key) of the dataframe to be saved in the S3 bucket \n
bucket: str: the name of the bucket to store the dataframe
'''
hashKey = f'{key}-hash'
hashString = getDfHash(data)
dictToSave = {'hash': hashString}
print(f'hashKey is {hashKey}')
print('saving hash to s3')
S3.save(key=hashKey, objectToSave=dictToSave, bucket=bucket, **kwargs)
print(f'saved hash {hashString}')
def updateS3Input(cls,
inputBucketName: str = INPUT_BUCKET_NAME,
key: str = '',
**kwargs):
products = S3.load(key=key, bucket=inputBucketName, **kwargs)
updateResult = cls.valueUpdate2({'items': cls.convertIntCols(products)})
return updateResult
def imageFromS3(bucket: str, key: str, **kwargs):
'''
obtain the image url from the S3 bucket and returns it \n
bucket: str: the name of the bucket containing the image \n
key: str: the key of the image
'''
url = S3.presign(key, bucket=bucket, expiry=10, **kwargs)
return imageFromUrl(url)
def saveRemoteCache(data: pd.DataFrame,
key='',
bucket='',
localCachePath='/tmp/cache',
localHashPath='/tmp/hash',
**kwargs):
'''
save cache and hash of the dataframe to both local location and S3 bucket \n
data:pd.DataFrame: dataframe to save \n
key: str: the name (key) of the dataframe to be saved in the S3 bucket \n
bucket: str: the name of the bucket to store the dataframe \n
localCachePath: str:path to save cache locally \n
localHashPath: str: path to save hash locally
'''
saveLocalCache(data=data, path=localCachePath)
saveLocalHash(data=data, path=localHashPath)
saveRemoteHash(data=data, key=key, bucket=bucket)
S3.saveFile(key=key, path=localCachePath, bucket=bucket, **kwargs)
def showImgS3(bucket: str, key: str):
'''
show the image obtained from the S3 bucket \n
bucket: str: the name of the bucket containing the image \n
key: str: the key of the image
'''
from matplotlib.pyplot import imshow
import time
imageFound = False
while imageFound == False:
if S3.exist(key, bucket=bucket):
url = S3.presign(key, expiry='10', bucket=bucket)
img = imageFromUrl(url)
imshow(img)
imageFound = True
else:
print('conversion failed or didnt happen')
time.sleep(1)
def saveRemoteCache(cls ,db:pd.DataFrame, key= ALLDATAKEY,
bucket = INVENTORY_BUCKET_NAME, localCachePath=DBCACHELOCATION,
localHashPath=DBHASHLOCATION):
if db.empty:
db = pd.DataFrame([DUMMYDATA])
db.columns = db.columns.astype(str)
db = db.reset_index(drop=True)
# print(db)
pdUtils.saveRemoteCache(data=db, key= key,
bucket=bucket, localCachePath=localCachePath,
localHashPath=localHashPath)
jsonDb = db.to_json(orient='split')
zlibArc = zlib.compress(jsonDb.encode())
tmpPath = '/tmp/zlibJsonCache.zl'
with open(tmpPath, 'wb') as f:
f.write(zlibArc)
S3.saveFile(key=f'{key}-json.zl',path=tmpPath,bucket=bucket)
def loadRemoteHash(key='', bucket='', **kwargs):
'''
load hash of the dataframe from S3 bucket \n
key: str: the name (key) of the dataframe to be loaded from the S3 bucket \n
bucket: str: the name of the bucket to load the dataframe
'''
hashKey = f'{key}-hash'
print(f'loading hashkey {hashKey}')
loadedHash = S3.load(hashKey, bucket=bucket, **kwargs).get('hash')
print(f'loaded hash is {loadedHash}')
return loadedHash
def allQuery(cls, key = 'allData', bucket = os.environ.get('INVENTORY_BUCKET_NAME'), **kwargs):
print(bucket)
result = S3.presign(key, bucket = bucket, checkExist=False, **kwargs)
return result
ACCESSTOKEN = 'TQ393qRcgP5+onjGEJQmmuwxpvelLdBGh5+5hA8haI6Jmn2i+KXyDmg8vASWWNJh3Jl22x/w0tRx8E+LqespD7qQS5VZtmCL1OLbrlrp4YNllymiJhGuGaQgH1zy2UlNnWR5+zQ86fwRXc/x3zsX3gdB04t89/1O/w1cDnyilFU='
## crate chatbot object
bot = ChatBot('robot',
storage_adapter='chatterbot.storage.SQLStorageAdapter',
preprocessors=[
'chatterbot.preprocessors.clean_whitespace',
],
database_uri='sqlite:////tmp/database.db',
read_only=True)
## download chatbot file
key = 'chatterbotdb.db'
path = '/tmp/database.db'
bucket = 'chatterbot-data'
S3.loadFile(key=key, path=path, bucket=bucket)
def getReplyToken(body):
return body["events"][0]['replyToken']
def sendReply(replyToken, accessToken):
'''send line reply message'''
line_bot_api = LineBotApi(accessToken)
line_bot_api.reply_message(replyToken,
TextSendMessage(text='Hello World!'))
def chatterbotResponse(inputText):
response = bot.get_response(inputText)