def lambda_handler(event, context): templateID = event['queryStringParameters']['id'] PDF_Name = GetTemplateFileInfo(templateID) response = s3.get_object(Bucket="techtalk-legalformgeneration", Key="Docs/"+templateID+"/" + PDF_Name + ".pdf", "/tmp/source.pdf") temp_dir = "/tmp" PDF_file = temp_dir + "/source.pdf" save_pdf_pages(PDF_file, temp_dir) final_contours = [] pages = convert_from_path(pdf_path=PDF_file, dpi=500) for page_no, page in enumerate(pages): filename = temp_dir + "/page_" + str(page_no + 1) + ".jpg" final_contours = box_extraction(filename, temp_dir, final_contours) s3.Bucket("techtalk-legalformgeneration") json.dump_s3 = lambda obj, f: s3.Object(key=f).put(Body=json.dumps(obj)) json.dump_s3(final_contours, "blank_boxes.json") return { 'statusCode':200, 'body':json.dumps(final_contours), 'headers':{ 'Content-Type': 'application/json', 'Access-Control-Allow-Origin': event['headers']['origin'], 'Access-Control-Allow-Headers':'Content-Type,X-Amz-Date,Authorization,X-Api-Key,X-Amz-Security-Token' } }
def __init__(self, logger, object_list, event): self.logger = logger.global_log bucket = event['bucket-input'] s3 = boto3.resource("s3").Bucket(bucket) json.dump_s3 = lambda obj, path: s3.Object(key=path).put(Body=json. dumps(obj)) json.dump_s3(object_list, event['file-objects']) self.logger.info('Write file')
def load_triumph_manifest(): try: triumph_manifest = json.load_s3(MANIFEST_JSON) except: data = {"triumphs": {}, "objectives": {}} json.dump_s3(MANIFEST_JSON) triumph_manifest = json.load_s3(MANIFEST_JSON) return triumph_manifest
def perform_manifest_updates(self): for triumph in self.MANIFEST_UPDATE['triumphs']: self.TRIUMPH_MANIFEST['triumphs'][triumph] = self.MANIFEST_UPDATE[ 'triumphs'][triumph] for objective in self.MANIFEST_UPDATE['objectives']: self.TRIUMPH_MANIFEST['objectives'][ objective] = self.MANIFEST_UPDATE['objectives'][objective] json.dump_s3(self.TRIUMPH_MANIFEST, MANIFEST_JSON)
def save_file(file_name, dict_obj): file_path = 'structured/' + file_name + '.json' s3 = boto3.resource("s3").Bucket(S3_BUCKET) json.dump_s3 = lambda obj, f: s3.Object(key=f).put(Body=json.dumps(obj) ) json.dump_s3(dict_obj, file_path) return
def load_stored_scores(): try: # with open(json_path, 'r') as stored_scores: # triumph_scores = json.load(stored_scores) triumph_scores = json.load_s3(JSONFILE) except: # with open(json_path, 'w+') as stored_scores: # data = {"discord_users":{}, "triumph_scores":{}} # json.dump(data, stored_scores) # triumph_scores = json.load(stored_scores) data = {"discord_users": {}, "triumph_scores": {}} json.dump_s3(data, JSONFILE) triumph_scores = json.load_s3(JSONFILE) return triumph_scores
def save(url): track_or_not = False session = boto3.Session( aws_access_key_id=settings.AWS_SERVER_PUBLIC_KEY, aws_secret_access_key=settings.AWS_SERVER_SECRET_KEY, ) s3 = session.resource('s3').Bucket("dont-track-url-bucket") json.load_s3 = lambda f: json.load(s3.Object(key=f).get()["Body"]) json.dump_s3 = lambda obj, f: s3.Object(key=f).put(Body=json.dumps(obj)) try: urls = json.load_s3("url") if url not in urls['url']: urls['url'].append(url) else: urls['url'].remove(url) track_or_not = True except: urls = {'url': [url]} json.dump_s3(urls, "url") make_reco_model.reco_model.update_model() return track_or_not
def reco(): if request.headers.get('recoRequest'): recommended_url = get_reco_site.get_reco() print(datetime.now()) print("recommended_url:", recommended_url) return recommended_url elif request.headers.get('sendHistory'): json_history = json.loads(request.data) history_data_dump = json.dump_s3(json_history, "history1") return "history_saved" elif request.headers.get('dontTrackRequest'): print(request.data) track_or_not = save_dont_track_url.save( parse((request.data).decode("utf-8"))) return "We track this page" if track_or_not else "We don't track this page" else: return "this???"
def index(): if request.method == 'GET': return 'OK' elif request.method == 'POST': # Store the IP address of the requester request_ip = ipaddress.ip_address(u'{0}'.format(request.remote_addr)) # If VALIDATE_SOURCEIP is set to false, do not validate source IP if os.environ.get('VALIDATE_SOURCEIP', None) != 'false': # If GHE_ADDRESS is specified, use it as the hook_blocks. if os.environ.get('GHE_ADDRESS', None): hook_blocks = [unicode(os.environ.get('GHE_ADDRESS'))] # Otherwise get the hook address blocks from the API. else: hook_blocks = requests.get( 'https://api.github.com/meta').json()['hooks'] # Check if the POST request is from github.com or GHE for block in hook_blocks: if ipaddress.ip_address(request_ip) in ipaddress.ip_network( block): break # the remote_addr is within the network range of github. else: if str(request_ip) != '127.0.0.1': abort(403) if request.headers.get('X-GitHub-Event') == "ping": return json.dumps({'msg': 'Hi!'}) if request.headers.get('X-GitHub-Event') != "push": return json.dumps({'msg': "wrong event type"}) repos = json.loads(io.open(REPOS_JSON_PATH, 'r').read()) payload = json.loads(request.data) repo_meta = { 'name': payload['repository']['name'], 'owner': payload['repository']['owner']['name'], } # Try to match on branch as configured in repos.json match = re.match(r"refs/heads/(?P<branch>.*)", payload['ref']) if match: repo_meta['branch'] = match.groupdict()['branch'] repo = repos.get( '{owner}/{name}/branch:{branch}'.format(**repo_meta), None) # Fallback to plain owner/name lookup if not repo: repo = repos.get('{owner}/{name}'.format(**repo_meta), None) if repo and repo.get('path', None): # Check if POST request signature is valid key = repo.get('key', None) if key: signature = request.headers.get('X-Hub-Signature').split( '=')[1] if type(key) == unicode: key = key.encode() mac = hmac.new(key, msg=request.data, digestmod=sha1) if not compare_digest(mac.hexdigest(), signature): abort(403) if repo.get('action', None): for action in repo['action']: subp = subprocess.Popen(action, cwd=repo.get('path', '.')) subp.wait() if repo.get('s3bucket', None): s3bucketname = repo.get('s3bucket') else: print('missing s3 bucketname') abort(500) if repo.get('s3key', None): s3key = repo.get('s3key') else: print('missing s3 filename') abort(500) print('s3 connection') if os.environ.get('USE_EC2', None) == 'true': provider = InstanceMetadataProvider( iam_role_fetcher=InstanceMetadataFetcher(timeout=1000, num_attempts=2)) creds = provider.load() session = boto3.Session(aws_access_key_id=creds.access_key, aws_secret_access_key=creds.secret_key, aws_session_token=creds.token) s3 = session.resource('s3').Bucket(s3bucketname) else: s3 = boto3.resource('s3') bucket = s3.Bucket(s3bucketname) json.load_s3 = lambda f: json.load(bucket.Object(key=f).get()['Body']) json.dump_s3 = lambda obj, f: bucket.Object(key=f).put(Body=json.dumps( obj)) #s3 fetch s3data = json.load_s3(s3key) datad = FilehashMap(s3data) commithash = payload['after'] for commit in payload['commits']: for z in commit['added']: print(z) datad.additem(z, commithash) for z in commit['modified']: print(z) datad.additem(z, commithash) for z in commit['removed']: datad.delitem(z) print(z) print('s3 upload') json.dump_s3(datad.displayhashmap(), s3key) #set perms s3objacl = s3.ObjectAcl(s3bucketname, s3key) response = s3objacl.put(ACL='public-read') print('s3 done') return 'OK'
def retail_prophet(): json.load_s3 = lambda f: json.load(s3.Object(key=f).get()["Body"]) json.dump_s3 = lambda obj, f: s3.Object(key=f).put(Body=json.dumps(obj)) try: lastrows = int(os.getenv('LASTROWS')) print(lastrows) except: lastrows = None try: canales = os.getenv('CANALES').split(',') print(canales) except: canales = None cache = os.getenv('CACHE') if cache: if cache == 'False': cache = False elif cache == 'True': cache = True else: cache = False print("CACHE IS " + str(cache)) bucket = os.getenv('BUCKET') key = os.getenv('KEY') path = '/'.join([bucket, key]) df = pd_read_csv_s3(path, compression="gzip") print(df) if df is None: print("Can't read file") return {} if canales is None or canales == 'All': canales = [ "directo", "google", "google seo", "mailing", "newsroom", "facebook", "referrers", "paid_social_samsung", "totales" ] result = {} s3 = boto3.resource("s3").Bucket(bucket) if cache: try: print("Entered first try") body = json.load_s3("prophet.json") response = {"statusCode": 200, "body": json.dumps(body)} return response except: print("Entered except") response = { "statusCode": 404, "error": ("there is no previous prophet result, please run" "without cache") } return response else: result = {} print("Entered success:") for canal in canales: if lastrows: if lastrows != 'All': canal_df = df[['fecha', canal]].tail(lastrows) canal_df = df[['fecha', canal]] result.update({canal: forecast_total(canal_df)}) response = {"statusCode": 200, "body": json.dumps(result)} json.dump_s3(result, "prophet.json")
print("Initializing manifest") manifest = { "uuid": __UUID__, "timestamp": timestamp, "num_stacks": 4, "stack_size": 2, "captures": [] } pass # Upload a set of images for stack in range(manifest["num_stacks"]): for z in range(manifest["stack_size"]): text = "{} {} {}".format(timestamp, stack, z) key = "{}/{}/images/{}/{}/{}.jpg".format( __BASE__, __UUID__, timestamp, stack, z) generate_and_upload_image(text, bucket, key) print("Uploaded", text, key) # Add this capture to manifest and update manifest["captures"].append(timestamp) json.dump_s3( manifest, "{}/{}/images/manifest.json".format(__BASE__, __UUID__)) except KeyboardInterrupt: break client.disconnect() print("Stopped listening to events and disconnected.")
def update_image_index(release_label, apt_repo, common_config, image_name): """Updates the index file used to track bare metal images Current format: { "<timestamp>": { "raw": { "bot": { "<distribution>": { "file": "<organization>_<flavour>_<distribution>_<release_label>_<date><time>", "checksum": <md5sum_of_image> } } } }, ... } """ s3 = boto3.client('s3') # Helper methods json.load_s3 = lambda f: json.load( s3.get_object(Bucket=apt_repo, Key=f)['Body']) json.dump_s3 = lambda obj, f: s3.put_object( Bucket=apt_repo, Key=f, Body=json.dumps(obj, indent=2)) index_key = release_label + '/images/index' _, flavour, distribution, release_label, timestamp = image_name.split('_') # Read checksum from generated file with open(f'/tmp/{image_name}', 'r') as checksum_file: checksum = checksum_file.read().replace('\n', '').split(' ')[0] os.remove(f'/tmp/{image_name}') image_data = { 'raw': { flavour: { distribution: { 'file': image_name, 'checksum': checksum } } } } data = {} try: # Wait for file to be ready to write wait_for_index(s3, apt_repo, index_key) data = json.load_s3(index_key) except botocore.exceptions.ClientError as error: # If file doesn't exists, we'll create a new one if error.response['Error']['Code'] == 'NoSuchKey': click.echo('Index file doesn\'t exist, creating a new one') try: data[timestamp] = merge_dicts(data[timestamp], image_data) except KeyError: data[timestamp] = image_data # Write data to index file json.dump_s3(data, index_key) tag_file(s3, apt_repo, index_key, 'Lock', 'False') # Invalidate image index cache if 'cloudfront_distribution_id' in common_config: invalidate_file_cloudfront(common_config['cloudfront_distribution_id'], index_key)
def upload_json(self, json_file, filename): s3 = boto3.resource('s3') json.dump_s3 = lambda obj, f: s3.Object(key=f).put(Body=json.dumps(obj) ) json.dump_s3(json_file, filename)
def ping_handler(event, context): config = yaml.load_s3('config.yaml') if config['monitor']['urls'] != None: ping_urls(config['monitor']['urls']) json.dump_s3(state_machine, 'state_machine')
def write_json_s3(data, s3_file_key): """Writes a dictionary into file in s3""" print("write_file_s3() writing file: " + s3_file_key) json.dump_s3(data, s3_file_key)
print(contents) # uploading a file directly to S3 https://stackoverflow.com/questions/40336918/how-to-write-a-file-or-data-to-an-s3-object-using-boto3 s3.Bucket('bucketname').upload_file('/local/file/here.txt','folder/sub/path/to/s3key') # reading the read json from s3 import json, boto3 s3 = boto3.resource("s3").Bucket("bucket") json.load_s3 = lambda f: json.load(s3.Object(key=f).get()["Body"]) json.dump_s3 = lambda obj, f: s3.Object(key=f).put(Body=json.dumps(obj)) # Now you can use json.load_s3 and json.dump_s3 with the same API as load and dump data = {"test":0} json.dump_s3(data, "key") # saves json to s3://bucket/key data = json.load_s3("key") # read json from s3://bucket/key #----------------------------------------------------------------------------------------------------- # storing a list in S3 bucket https://dzone.com/articles/boto3-amazon-s3-as-python-object-store import boto3 import pickle s3 = boto3.client('s3') myList=[1,2,3,4,5] #Serialize the object serializedListObject = pickle.dumps(myList) #Write to Bucket named 'mytestbucket' and #Store the list using key myList001 s3.put_object(Bucket='mytestbucket',Key='myList001',Body=serializedListObject) # retrieving a list from S3 Bucket
def save_triumph_score(self, scores): for player in scores: self.TRIUMPH_SCORES["triumph_scores"][player] = scores[player] # with open(self.JSON_PATH, 'w') as stored_scores: # json.dump(self.TRIUMPH_SCORES, stored_scores) json.dump_s3(self.TRIUMPH_SCORES, JSONFILE)