def lambda_handler(event, context):
templateID = event['queryStringParameters']['id']
PDF_Name = GetTemplateFileInfo(templateID)
response = s3.get_object(Bucket="techtalk-legalformgeneration", Key="Docs/"+templateID+"/" + PDF_Name + ".pdf", "/tmp/source.pdf")
temp_dir = "/tmp"
PDF_file = temp_dir + "/source.pdf"
save_pdf_pages(PDF_file, temp_dir)
final_contours = []
pages = convert_from_path(pdf_path=PDF_file, dpi=500)
for page_no, page in enumerate(pages):
filename = temp_dir + "/page_" + str(page_no + 1) + ".jpg"
final_contours = box_extraction(filename, temp_dir, final_contours)
s3.Bucket("techtalk-legalformgeneration")
json.dump_s3 = lambda obj, f: s3.Object(key=f).put(Body=json.dumps(obj))
json.dump_s3(final_contours, "blank_boxes.json")
return {
'statusCode':200,
'body':json.dumps(final_contours),
'headers':{
'Content-Type': 'application/json',
'Access-Control-Allow-Origin': event['headers']['origin'],
'Access-Control-Allow-Headers':'Content-Type,X-Amz-Date,Authorization,X-Api-Key,X-Amz-Security-Token'
}
}
def __init__(self, logger, object_list, event):
self.logger = logger.global_log
bucket = event['bucket-input']
s3 = boto3.resource("s3").Bucket(bucket)
json.dump_s3 = lambda obj, path: s3.Object(key=path).put(Body=json.
dumps(obj))
json.dump_s3(object_list, event['file-objects'])
self.logger.info('Write file')
def load_triumph_manifest():
try:
triumph_manifest = json.load_s3(MANIFEST_JSON)
except:
data = {"triumphs": {}, "objectives": {}}
json.dump_s3(MANIFEST_JSON)
triumph_manifest = json.load_s3(MANIFEST_JSON)
return triumph_manifest
def perform_manifest_updates(self):
for triumph in self.MANIFEST_UPDATE['triumphs']:
self.TRIUMPH_MANIFEST['triumphs'][triumph] = self.MANIFEST_UPDATE[
'triumphs'][triumph]
for objective in self.MANIFEST_UPDATE['objectives']:
self.TRIUMPH_MANIFEST['objectives'][
objective] = self.MANIFEST_UPDATE['objectives'][objective]
json.dump_s3(self.TRIUMPH_MANIFEST, MANIFEST_JSON)
def save_file(file_name, dict_obj):
file_path = 'structured/' + file_name + '.json'
s3 = boto3.resource("s3").Bucket(S3_BUCKET)
json.dump_s3 = lambda obj, f: s3.Object(key=f).put(Body=json.dumps(obj)
)
json.dump_s3(dict_obj, file_path)
return
def load_stored_scores():
try:
# with open(json_path, 'r') as stored_scores:
# triumph_scores = json.load(stored_scores)
triumph_scores = json.load_s3(JSONFILE)
except:
# with open(json_path, 'w+') as stored_scores:
# data = {"discord_users":{}, "triumph_scores":{}}
# json.dump(data, stored_scores)
# triumph_scores = json.load(stored_scores)
data = {"discord_users": {}, "triumph_scores": {}}
json.dump_s3(data, JSONFILE)
triumph_scores = json.load_s3(JSONFILE)
return triumph_scores
def save(url):
track_or_not = False
session = boto3.Session(
aws_access_key_id=settings.AWS_SERVER_PUBLIC_KEY,
aws_secret_access_key=settings.AWS_SERVER_SECRET_KEY,
)
s3 = session.resource('s3').Bucket("dont-track-url-bucket")
json.load_s3 = lambda f: json.load(s3.Object(key=f).get()["Body"])
json.dump_s3 = lambda obj, f: s3.Object(key=f).put(Body=json.dumps(obj))
try:
urls = json.load_s3("url")
if url not in urls['url']:
urls['url'].append(url)
else:
urls['url'].remove(url)
track_or_not = True
except:
urls = {'url': [url]}
json.dump_s3(urls, "url")
make_reco_model.reco_model.update_model()
return track_or_not
def reco():
if request.headers.get('recoRequest'):
recommended_url = get_reco_site.get_reco()
print(datetime.now())
print("recommended_url:", recommended_url)
return recommended_url
elif request.headers.get('sendHistory'):
json_history = json.loads(request.data)
history_data_dump = json.dump_s3(json_history, "history1")
return "history_saved"
elif request.headers.get('dontTrackRequest'):
print(request.data)
track_or_not = save_dont_track_url.save(
parse((request.data).decode("utf-8")))
return "We track this page" if track_or_not else "We don't track this page"
else:
return "this???"
def index():
if request.method == 'GET':
return 'OK'
elif request.method == 'POST':
# Store the IP address of the requester
request_ip = ipaddress.ip_address(u'{0}'.format(request.remote_addr))
# If VALIDATE_SOURCEIP is set to false, do not validate source IP
if os.environ.get('VALIDATE_SOURCEIP', None) != 'false':
# If GHE_ADDRESS is specified, use it as the hook_blocks.
if os.environ.get('GHE_ADDRESS', None):
hook_blocks = [unicode(os.environ.get('GHE_ADDRESS'))]
# Otherwise get the hook address blocks from the API.
else:
hook_blocks = requests.get(
'https://api.github.com/meta').json()['hooks']
# Check if the POST request is from github.com or GHE
for block in hook_blocks:
if ipaddress.ip_address(request_ip) in ipaddress.ip_network(
block):
break # the remote_addr is within the network range of github.
else:
if str(request_ip) != '127.0.0.1':
abort(403)
if request.headers.get('X-GitHub-Event') == "ping":
return json.dumps({'msg': 'Hi!'})
if request.headers.get('X-GitHub-Event') != "push":
return json.dumps({'msg': "wrong event type"})
repos = json.loads(io.open(REPOS_JSON_PATH, 'r').read())
payload = json.loads(request.data)
repo_meta = {
'name': payload['repository']['name'],
'owner': payload['repository']['owner']['name'],
}
# Try to match on branch as configured in repos.json
match = re.match(r"refs/heads/(?P<branch>.*)", payload['ref'])
if match:
repo_meta['branch'] = match.groupdict()['branch']
repo = repos.get(
'{owner}/{name}/branch:{branch}'.format(**repo_meta), None)
# Fallback to plain owner/name lookup
if not repo:
repo = repos.get('{owner}/{name}'.format(**repo_meta), None)
if repo and repo.get('path', None):
# Check if POST request signature is valid
key = repo.get('key', None)
if key:
signature = request.headers.get('X-Hub-Signature').split(
'=')[1]
if type(key) == unicode:
key = key.encode()
mac = hmac.new(key, msg=request.data, digestmod=sha1)
if not compare_digest(mac.hexdigest(), signature):
abort(403)
if repo.get('action', None):
for action in repo['action']:
subp = subprocess.Popen(action, cwd=repo.get('path', '.'))
subp.wait()
if repo.get('s3bucket', None):
s3bucketname = repo.get('s3bucket')
else:
print('missing s3 bucketname')
abort(500)
if repo.get('s3key', None):
s3key = repo.get('s3key')
else:
print('missing s3 filename')
abort(500)
print('s3 connection')
if os.environ.get('USE_EC2', None) == 'true':
provider = InstanceMetadataProvider(
iam_role_fetcher=InstanceMetadataFetcher(timeout=1000,
num_attempts=2))
creds = provider.load()
session = boto3.Session(aws_access_key_id=creds.access_key,
aws_secret_access_key=creds.secret_key,
aws_session_token=creds.token)
s3 = session.resource('s3').Bucket(s3bucketname)
else:
s3 = boto3.resource('s3')
bucket = s3.Bucket(s3bucketname)
json.load_s3 = lambda f: json.load(bucket.Object(key=f).get()['Body'])
json.dump_s3 = lambda obj, f: bucket.Object(key=f).put(Body=json.dumps(
obj))
#s3 fetch
s3data = json.load_s3(s3key)
datad = FilehashMap(s3data)
commithash = payload['after']
for commit in payload['commits']:
for z in commit['added']:
print(z)
datad.additem(z, commithash)
for z in commit['modified']:
print(z)
datad.additem(z, commithash)
for z in commit['removed']:
datad.delitem(z)
print(z)
print('s3 upload')
json.dump_s3(datad.displayhashmap(), s3key)
#set perms
s3objacl = s3.ObjectAcl(s3bucketname, s3key)
response = s3objacl.put(ACL='public-read')
print('s3 done')
return 'OK'
def retail_prophet():
json.load_s3 = lambda f: json.load(s3.Object(key=f).get()["Body"])
json.dump_s3 = lambda obj, f: s3.Object(key=f).put(Body=json.dumps(obj))
try:
lastrows = int(os.getenv('LASTROWS'))
print(lastrows)
except:
lastrows = None
try:
canales = os.getenv('CANALES').split(',')
print(canales)
except:
canales = None
cache = os.getenv('CACHE')
if cache:
if cache == 'False':
cache = False
elif cache == 'True':
cache = True
else:
cache = False
print("CACHE IS " + str(cache))
bucket = os.getenv('BUCKET')
key = os.getenv('KEY')
path = '/'.join([bucket, key])
df = pd_read_csv_s3(path, compression="gzip")
print(df)
if df is None:
print("Can't read file")
return {}
if canales is None or canales == 'All':
canales = [
"directo", "google", "google seo", "mailing", "newsroom",
"facebook", "referrers", "paid_social_samsung", "totales"
]
result = {}
s3 = boto3.resource("s3").Bucket(bucket)
if cache:
try:
print("Entered first try")
body = json.load_s3("prophet.json")
response = {"statusCode": 200, "body": json.dumps(body)}
return response
except:
print("Entered except")
response = {
"statusCode":
404,
"error": ("there is no previous prophet result, please run"
"without cache")
}
return response
else:
result = {}
print("Entered success:")
for canal in canales:
if lastrows:
if lastrows != 'All':
canal_df = df[['fecha', canal]].tail(lastrows)
canal_df = df[['fecha', canal]]
result.update({canal: forecast_total(canal_df)})
response = {"statusCode": 200, "body": json.dumps(result)}
json.dump_s3(result, "prophet.json")
print("Initializing manifest")
manifest = {
"uuid": __UUID__,
"timestamp": timestamp,
"num_stacks": 4,
"stack_size": 2,
"captures": []
}
pass
# Upload a set of images
for stack in range(manifest["num_stacks"]):
for z in range(manifest["stack_size"]):
text = "{} {} {}".format(timestamp, stack, z)
key = "{}/{}/images/{}/{}/{}.jpg".format(
__BASE__, __UUID__, timestamp, stack, z)
generate_and_upload_image(text, bucket, key)
print("Uploaded", text, key)
# Add this capture to manifest and update
manifest["captures"].append(timestamp)
json.dump_s3(
manifest,
"{}/{}/images/manifest.json".format(__BASE__, __UUID__))
except KeyboardInterrupt:
break
client.disconnect()
print("Stopped listening to events and disconnected.")
def update_image_index(release_label, apt_repo, common_config, image_name):
"""Updates the index file used to track bare metal images
Current format:
{
"<timestamp>": {
"raw": {
"bot": {
"<distribution>": {
"file": "<organization>_<flavour>_<distribution>_<release_label>_<date><time>",
"checksum": <md5sum_of_image>
}
}
}
},
...
}
"""
s3 = boto3.client('s3')
# Helper methods
json.load_s3 = lambda f: json.load(
s3.get_object(Bucket=apt_repo, Key=f)['Body'])
json.dump_s3 = lambda obj, f: s3.put_object(
Bucket=apt_repo, Key=f, Body=json.dumps(obj, indent=2))
index_key = release_label + '/images/index'
_, flavour, distribution, release_label, timestamp = image_name.split('_')
# Read checksum from generated file
with open(f'/tmp/{image_name}', 'r') as checksum_file:
checksum = checksum_file.read().replace('\n', '').split(' ')[0]
os.remove(f'/tmp/{image_name}')
image_data = {
'raw': {
flavour: {
distribution: {
'file': image_name,
'checksum': checksum
}
}
}
}
data = {}
try:
# Wait for file to be ready to write
wait_for_index(s3, apt_repo, index_key)
data = json.load_s3(index_key)
except botocore.exceptions.ClientError as error:
# If file doesn't exists, we'll create a new one
if error.response['Error']['Code'] == 'NoSuchKey':
click.echo('Index file doesn\'t exist, creating a new one')
try:
data[timestamp] = merge_dicts(data[timestamp], image_data)
except KeyError:
data[timestamp] = image_data
# Write data to index file
json.dump_s3(data, index_key)
tag_file(s3, apt_repo, index_key, 'Lock', 'False')
# Invalidate image index cache
if 'cloudfront_distribution_id' in common_config:
invalidate_file_cloudfront(common_config['cloudfront_distribution_id'],
index_key)
def upload_json(self, json_file, filename):
s3 = boto3.resource('s3')
json.dump_s3 = lambda obj, f: s3.Object(key=f).put(Body=json.dumps(obj)
)
json.dump_s3(json_file, filename)
def ping_handler(event, context):
config = yaml.load_s3('config.yaml')
if config['monitor']['urls'] != None:
ping_urls(config['monitor']['urls'])
json.dump_s3(state_machine, 'state_machine')
def write_json_s3(data, s3_file_key):
"""Writes a dictionary into file in s3"""
print("write_file_s3() writing file: " + s3_file_key)
json.dump_s3(data, s3_file_key)
print(contents)
# uploading a file directly to S3 https://stackoverflow.com/questions/40336918/how-to-write-a-file-or-data-to-an-s3-object-using-boto3
s3.Bucket('bucketname').upload_file('/local/file/here.txt','folder/sub/path/to/s3key')
# reading the read json from s3
import json, boto3
s3 = boto3.resource("s3").Bucket("bucket")
json.load_s3 = lambda f: json.load(s3.Object(key=f).get()["Body"])
json.dump_s3 = lambda obj, f: s3.Object(key=f).put(Body=json.dumps(obj))
# Now you can use json.load_s3 and json.dump_s3 with the same API as load and dump
data = {"test":0}
json.dump_s3(data, "key") # saves json to s3://bucket/key
data = json.load_s3("key") # read json from s3://bucket/key
#-----------------------------------------------------------------------------------------------------
# storing a list in S3 bucket https://dzone.com/articles/boto3-amazon-s3-as-python-object-store
import boto3
import pickle
s3 = boto3.client('s3')
myList=[1,2,3,4,5]
#Serialize the object
serializedListObject = pickle.dumps(myList)
#Write to Bucket named 'mytestbucket' and
#Store the list using key myList001
s3.put_object(Bucket='mytestbucket',Key='myList001',Body=serializedListObject)
# retrieving a list from S3 Bucket
def save_triumph_score(self, scores):
for player in scores:
self.TRIUMPH_SCORES["triumph_scores"][player] = scores[player]
# with open(self.JSON_PATH, 'w') as stored_scores:
# json.dump(self.TRIUMPH_SCORES, stored_scores)
json.dump_s3(self.TRIUMPH_SCORES, JSONFILE)
