def post(self, request, id):
reservation = self.get_reservation(id)
data = json.loads(request.data['data'])
if (request.user.id == reservation.hall.user.id):
if reservation.date.date() + timedelta(days=60) < date.today():
return Response(status=status.HTTP_304_NOT_MODIFIED)
else:
if reservation.hall.num_of_messages < reservation.hall.quota_of_messages:
session = Session(
aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY,
region_name="us-east-2")
s3_client = session.client(
's3',
region_name="us-east-2",
config=Config(signature_version='s3v4'))
s3_resource = session.resource(
's3',
region_name="us-east-2",
config=Config(signature_version='s3v4'))
my_bucket = s3_resource.Bucket(
settings.AWS_STORAGE_BUCKET_NAME)
my_bucket_resized = s3_resource.Bucket(
settings.AWS_STORAGE_BUCKET_NAME_RESIZED)
obj = []
obj_resized = []
if data['deletedImages'] != []:
for image in data['deletedImages']:
obj.append({
'Key':
'photos/' + str(reservation.id) + '/' + image +
"/image.jpg"
})
obj_resized.append({
'Key':
'resized-photos/' + str(reservation.id) + '/' +
image + "/image.jpg"
})
response = my_bucket.delete_objects(Delete={
'Objects': obj,
})
response = my_bucket_resized.delete_objects(
Delete={
'Objects': obj_resized,
})
images_to_delete = ReservationImage.objects.filter(
reservation=reservation).filter(
name__in=data['deletedImages'])
images_to_delete.delete()
images = iter(request.data)
next(images)
responses = []
num_images = 0
for image, idx in enumerate(images):
num_images = num_images + 1
try:
name = uuid.uuid4()
s3_object_name = "photos" + "/" + str(
id) + "/" + str(name) + "/" + "image.jpg"
response = s3_client.generate_presigned_post(
Bucket=settings.AWS_STORAGE_BUCKET_NAME,
Key=s3_object_name,
ExpiresIn=3600)
responses.append(response)
reservation_image = ReservationImage()
reservation_image.name = name
reservation_image.reservation = reservation
reservation_image.save()
except ClientError as e:
reservation.hall.num_of_images = reservation.hall.num_of_images + idx
reservation.num_of_unpaid = reservation.num_of_unpaid + idx
reservation.save()
reservation.hall.save()
return Response(
status=status.HTTP_503_SERVICE_UNAVAILABLE)
phoneNumber = "+90" + reservation.phone
message = "Merhabalar,Linkten Ulaşabileceğiniz Albümünüz Güncellenmiştir.\nBizi Tercih Ettiğiniz için Teşekkür Ederiz.\n" + "Albüm Şifresi:" + str(
reservation.code) + "\nsalonayır.com/photos/" + str(
reservation.id)
if phoneNumber != "+90":
s3_client = session.client('sns', 'us-east-2')
s3_client.publish(PhoneNumber=phoneNumber,
Message=message)
reservation.hall.num_of_messages = reservation.hall.num_of_messages + 1
if data['duration'] != 0 and data['duration'] < 21:
reservation.duration = data['duration']
reservation.save()
reservation.hall.num_of_images = reservation.hall.num_of_images + num_images
reservation.hall.save()
reservation.num_of_unpaid += (num_images -
len(data['deletedImages']))
reservation.save()
return Response(responses, status=status.HTTP_200_OK)
else:
return Response([], status.HTTP_417_EXPECTATION_FAILED)
return Response(status=status.HTTP_401_UNAUTHORIZED)
def archive_to_glacier():
config = configparser.ConfigParser()
config.read('configini.ini')
aws_config = config['AWS']
canceled_archive = []
# Connect to SQS and get the message queue
try:
sqs = boto3.resource('sqs', region_name=aws_config['AWS_REGION_NAME'])
huangxy_queue = sqs.get_queue_by_name(
QueueName=aws_config['AWS_SQS_GLACIER'])
except Exception as e:
print(e)
return
# Poll the message queue in a loop
while True:
# Attempt to read a message from the queue
response = huangxy_queue.receive_messages(WaitTimeSeconds=20)
# If message read, extract job parameters from the message body as before
if response:
print('Get response successfully.')
try:
msg = json.loads(json.loads(response[0].body)['Message'])
job_id = msg['job_id']
except Exception as e:
raise e
return
# if the job should be canceled to put into archive, continue to the next while loop.
if 'canceled_archive' in msg:
canceled_archive.append(msg['canceled_archive'])
response[0].delete()
print('This job should not be moved to glacier.')
continue
# intercept the canceled archive job
if job_id in canceled_archive:
canceled_archive.remove(job_id)
response[0].delete()
print('Avoid moving to glacier.')
continue
try:
dynamodb = boto3.resource(
'dynamodb', region_name=aws_config['AWS_REGION_NAME'])
annotation_table = dynamodb.Table(
aws_config['AWS_DYNAMODB_ANNOTATIONS_TABLE'])
job = annotation_table.query(
Select='ALL_ATTRIBUTES',
KeyConditionExpression=Key('job_id').eq(
job_id))['Items'][0]
print('Get job successfully.')
except Exception as e:
raise e
return
if 'complete_time' in job and (
time.time() - float(job['complete_time'])) > float(
aws_config['FREE_USER_DATA_RETENTION']):
try:
key = msg['s3_key_input_file'].replace(
'.vcf', '.annot.vcf')
s3 = boto3.resource('s3')
bucket = s3.Bucket(aws_config['AWS_S3_RESULTS_BUCKET'])
body = bucket.Object(key).get()['Body'].read()
# print(body)
client_glacier = boto3.client(
'glacier', aws_config['AWS_REGION_NAME'])
# Response Syntax
# {
# 'location': 'string',
# 'checksum': 'string',
# 'archiveId': 'string'
# }
glacier_upload_response = client_glacier.upload_archive(
vaultName=aws_config['AWS_GLACIER_VAULT'], body=body)
print('Upload glacier successfully.')
except Exception as e:
raise e
return
try:
client_s3 = boto3.client(
's3',
region_name=aws_config['AWS_REGION_NAME'],
config=Config(signature_version='s3v4'))
# Response Syntax
# {
# 'DeleteMarker': True|False,
# 'VersionId': 'string',
# 'RequestCharged': 'requester'
# }
s3_delete_response = client_s3.delete_object(
Bucket=aws_config['AWS_S3_RESULTS_BUCKET'], Key=key)
print('Delete from s3 successfully.')
except Exception as e:
raise e
return
annotation_table.update_item(
Key={'job_id': job_id},
AttributeUpdates={
'archive_id': {
'Value': glacier_upload_response['archiveId'],
'Action': 'PUT'
}
})
print('Update database successfully.')
# After all done, delete SQS
response[0].delete()
print('Delete SQS successfully.')
else:
print('There is no response.')
def main(args):
DOMAIN = 'ITD'
VERSION = '1'
taskName = os.path.basename(__file__)[:-3]
logging.config.fileConfig('/Assets/sharedLibraries/logging_config.ini')
logging.debug("Creating SWF boto client")
botoConfig = Config(
connect_timeout=50,
read_timeout=70) # suggestion is the read is higher than connect
swf = boto3.client('swf', config=botoConfig)
logging.debug("Created SWF boto client: %s", swf)
BUCKETNAME = "schulerfiles"
workingStorage = "/Assets/working/"
while True:
task = swf.poll_for_activity_task(domain=DOMAIN,
taskList={'name': taskName},
identity='%s-01' % (taskName))
if 'taskToken' not in task:
logging.info("%s - Poll timed out, no new task. Repoll", taskName)
# Run the operation
else:
taskToken = task['taskToken']
workID = task['workflowExecution']['workflowId']
logging.info("[%s] New request for %s", workID, taskName)
INPUT = json.loads(task['input'])
source = INPUT['locationSource']
destination = INPUT['locationDestination']
dbPrimaryKey = INPUT['dbPrimaryKey']
fileKey = INPUT['fileKey'] + '/'
# Bucket object is necessary in all cases
logging.debug("[%s] Creating S3 bucket boto client", workID)
s3 = boto3.resource('s3')
bucket = s3.Bucket(BUCKETNAME)
logging.debug("[%s] Created S3 bucket boto client", workID)
# Setting the storage class to be used for later
s3StorageClass = 'STANDARD'
if destination == 'near_line':
s3StorageClass = 'STANDARD_IA'
logging.info("[%s] Moving %s from %s to %s", workID, fileKey,
source, destination)
# CDN and near_line are both S3 tiers, so all we are doing is changing the Storage Class with a PUT
if (source == 'CDN' and destination == 'near_line') or (
source == 'near_line' and destination == 'CDN'):
logging.info("[%s] Moving objects between S3 and S3IA", workID)
for obj in bucket.objects.filter(Prefix=fileKey):
logging.debug(
"[%s] Moving object %s from %s to %s object: ", workID,
obj.key, source, destination)
copy_source = {'Bucket': bucket.name, 'Key': obj.key}
response = s3.meta.client.copy_object(
CopySource=copy_source,
Bucket=bucket.name,
Key=obj.key,
StorageClass=s3StorageClass)
logging.debug("[%s] Object moved: ", workID, response)
OUTPUT = {
'result': 'success',
}
# If we need to move to or restore from archive, we need to run the whole gamut
elif 'archive' in [source, destination]: #Glacier
# Create Glacier object
# Create directory in working storage
subDir = parseHelper.createDir(workingStorage, fileKey)
# Pull down from glacier
if source == 'archive':
logging.info("[%s] Moving asset from Glacier", workID)
else:
logging.info("[%s] Begin moving objects to Glacier",
workID)
logging.info("[%s] Begin object download", workID)
# Download object to the working storage subdirectory
# Upload files back up to the same fileKey (this takes Accounts into consideration as well)
for obj in bucket.objects.filter(Prefix=fileKey):
logging.info(
"[%s] Downloading %s to temporary storage", workID,
obj.key)
fileName = os.path.join(workingStorage, obj.key)
if not os.path.exists(os.path.dirname(fileName)):
try:
os.makedirs(os.path.dirname(fileName))
except OSError as exc: # Guard against race condition
if exc.errno != errno.EEXIST:
raise
s3.Object(bucket.name, obj.key).download_file(
fileName) # Create directories as needed here
logging.info("[%s] Begin object upload to glacier", workID)
# Output needs the temporary storage location to clean up
# cleanUpLandingPads expects an ASSET (e.g., /Assets/working/file.ext), and not just a path. We will provide a dummy asset
OUTPUT = {
'result': 'success',
'asset': '%sdummy.file' % (subDir)
}
AUDIT = {}
AUDIT['User'] = '******'
AUDIT['Timestamp'] = time.strftime("%Y-%m-%dT%H:%M:%S+0000",
time.gmtime())
AUDIT['Action'] = 'Asset moved from %s from %s' % (source,
destination)
AUDIT['Notes'] = workID
# Add the Audit Dictionary to a list so that we can append it
aLIST = []
aLIST.append(AUDIT)
updateExpression = 'set File_Location = :d, Audit = list_append(Audit, :a)'
expressionValues = {':d': destination, ':a': aLIST}
# Call the update function
logging.debug("[%s] Updating the asset location and history: %s",
workID, destination)
response = databaseHelper.updateEntry(dbPrimaryKey,
updateExpression,
expressionValues)
OUTPUT.update(INPUT)
swf.respond_activity_task_completed(taskToken=taskToken,
result=json.dumps(OUTPUT))
logging.info("[%s] %s Complete", workID, taskName)
RECORD_TYPE_CONTENT,
RECORD_TYPE_CREATE,
RECORD_TYPE_SPECIAL,
BaseAggregator,
BaseListener,
BaseParams,
)
from .parquet_schema import PQ_SCHEMAS
CACHE_SIZE = 500
SITE_VISITS_INDEX = "_site_visits_index"
CONTENT_DIRECTORY = "content"
CONFIG_DIR = "config"
BATCH_COMMIT_TIMEOUT = 30 # commit a batch if no new records for N seconds
S3_CONFIG_KWARGS = {"retries": {"max_attempts": 20}}
S3_CONFIG = Config(**S3_CONFIG_KWARGS)
def listener_process_runner(base_params: BaseParams, manager_params: Dict[str,
Any],
instance_id: int) -> None:
"""S3Listener runner. Pass to new process"""
listener = S3Listener(base_params, manager_params, instance_id)
listener.startup()
while True:
listener.update_status_queue()
listener.save_batch_if_past_timeout()
if listener.should_shutdown():
break
try:
DEFAULT_TTL = 60 * 60 * 48
# Default size of the bucket before checking for inventory
DEFAULT_INVENTORY_BUCKET_SIZE_THRESHOLD = \
int(os.environ.get("SALACTUS_INVENTORY_THRESHOLD", 100000))
BUCKET_OBJ_DESC = {
True: ('Versions', 'list_object_versions', ('NextKeyMarker',
'NextVersionIdMarker')),
False: ('Contents', 'list_objects_v2', ('NextContinuationToken', ))
}
connection = redis.Redis(host=REDIS_HOST)
# Increase timeouts to assist with non local regions, also
# seeing some odd net slowness all around.
s3config = Config(read_timeout=420, connect_timeout=90)
keyconfig = {
'report-only': not os.environ.get('SALACTUS_ENCRYPT') and True or False,
'glacier': False,
'large': True,
'key-id': os.environ.get('SALACTUS_KEYID'),
'crypto': os.environ.get('SALACTUS_CRYPTO', 'AES256')
}
log = logging.getLogger("salactus")
def get_session(account_info):
"""Get a boto3 sesssion potentially cross account sts assumed
assumed sessions are automatically refreshed.
import boto3
from botocore.client import Config
s3 = boto3.client('s3', config=Config(signature_version='s3v4'))
def lambda_handler(event, context):
# link to download certificate
key = 'certs/' + str(event['params']['path']['id']) + '.json'
url = s3.generate_presigned_url(ClientMethod='get_object',
Params={
'Bucket': 'fs.blockcert.poc',
'Key': key,
'ResponseContentDisposition':
'attachment'
})
return {"location": url}
def create_client(self):
client_config = Config(signature_version='s3v4')
return self.session.create_client('s3', self.region,
config=client_config)
def handler_wrapper(event: dict, context: dict = None):
nonlocal redactConfig
nonlocal timeoutFunction
logger.info('Request received, processing...')
if not is_valid_event(event):
# If it is not a valid event we need to raise an exception
message = 'The event object passed is not a valid Request Object as per ' + \
'https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/crpg-ref-requests.html'
logger.error(message)
raise NotValidRequestObjectException(message)
# Timeout Function Handler
if 'LambdaParentRequestId' in event:
logger.info(
'This request has been invoked as a child, for parent logs please see request ID: %s'
% event['LambdaParentRequestId'])
elif context is None and timeoutFunction:
logger.warning(
'You cannot use the timeoutFunction option outside of Lambda. To suppress this warning'
+ ', set timeoutFunction to False')
elif timeoutFunction:
# Attempt to invoke the function. Depending on the error we get may continue execution or return
logger.info(
'Request has been invoked in Lambda with timeoutFunction set, attempting to invoke self'
)
pevent = event.copy()
pevent['LambdaParentRequestId'] = context.aws_request_id
payload = json.dumps(pevent).encode('UTF-8')
timeout = (context.get_remaining_time_in_millis() -
TIMEOUT_THRESHOLD) / 1000
# Edge case where time is set to very low timeout, use half the timeout threshold as the timeout for the
# the Lambda Function
if timeout <= 0: timeout = TIMEOUT_THRESHOLD / 2000
config = Config(connect_timeout=2,
read_timeout=timeout,
retries={'max_attempts': 0})
b_lambda = client('lambda', config=config)
# Normally we would just do a catch all error handler but in this case we want to be paranoid
try:
response = b_lambda.invoke(
FunctionName=context.invoked_function_arn,
InvocationType='RequestResponse',
Payload=payload)
# Further checks
if 'FunctionError' in response:
response.get('Payload', ''.encode('UTF-8'))
message = 'Invocation got an error: %s' % payload.decode(
)
logger.error(message)
return ResponseObject(
reason=message,
responseStatus=Status.FAILED).send(event, context)
else:
# In this case the function returned without error which means we can assume the chained
# invokation sent a response, so we do not have too.
logger.info(
'Compeleted execution of chained invocation, returning payload'
)
response.get('Payload', ''.encode('UTF-8'))
return payload.decode()
except bexceptions.ClientError as e:
logger.warning(
'Caught exception %s while trying to invoke function. Running handler locally.'
% str(e))
logger.warning(
'You cannot use the timeoutFunction option without the ability for the function to'
+
' invoke itself. To suppress this warning, set timeoutFunction to False'
)
except bexceptions.ConnectionError as e:
logger.error(
'Got error %s while trying to invoke function. Running handler locally'
% str(e))
logger.error(
'You cannot use the timeoutFunction option without the ability to connect to the '
+
'Lambda API from within the function. As we may not have time to execute the '
+ 'function, returning failure.')
return ResponseObject(
reason=
'Unable to call Lambda to do chained invoke, returning failure.',
responseStatus=Status.FAILED).send(event, context)
except bexceptions.ReadTimeoutError:
# This should be a critical failure
logger.error(
'Waited the read timeout and function did not return, returning an error'
)
return ResponseObject(
reason='Lambda function timed out, returning failure.',
responseStatus=Status.FAILED).send(event, context)
except Exception as e:
message = 'Got an %s I did not understand while trying to invoke child function: %s' % (
e.__class__, str(e))
logger.error(message)
return ResponseObject(reason=message,
responseStatus=Status.FAILED).send(
event, context)
# Debug Logging Handler
if logger.getEffectiveLevel() <= logging.DEBUG:
if context is not None:
logger.debug('Running request with Lambda RequestId: %s' %
context.aws_request_id)
if redactConfig is not None and isinstance(
redactConfig,
(StandaloneRedactionConfig, RedactionConfig)):
logger.debug('Request Body:\n' +
json.dumps(redactConfig._redact(event)))
elif redactConfig is not None:
logger.warning(
'A non valid RedactionConfig was provided, and ignored'
)
logger.debug('Request Body:\n' + json.dumps(event))
else:
logger.debug('Request Body:\n' + json.dumps(event))
try:
logger.info('Running CloudFormation request %s for stack: %s' %
(event['RequestId'], event['StackId']))
# Run the function
if context is not None:
result = func(event, context)
else:
result = func(event)
except Exception as e:
# If there was an exception thrown by the function, send a failure response
result = ResponseObject(
physicalResourceId=str(uuid4())
if context is None else None,
reason='Function %s failed due to exception "%s"' %
(func.__name__, str(e)),
responseStatus=Status.FAILED)
logger.error(result.reason)
if not isinstance(result, ResponseObject):
# If a ResponseObject is not provided, work out what kind of response object to pass, or return a
# failure if it is an invalid response type, or if the enforceUseOfClass is explicitly or implicitly set
if context is None:
result = ResponseObject(
reason=
'Response Object of type %s was not a ResponseObject and there is no Lambda Context'
% result.__class__,
responseStatus=Status.FAILED)
logger.error(result.reason)
elif enforceUseOfClass:
result = ResponseObject(
reason=
'Response Object of type %s was not a ResponseObject instance and '
+ 'enforceUseOfClass set to true' % result.__class__,
responseStatus=Status.FAILED)
logger.error(result.reason)
elif result is False:
result = ResponseObject(
reason='Function %s returned False.' % func.__name__,
responseStatus=Status.FAILED)
logger.debug(result.reason)
elif isinstance(result, dict):
result = ResponseObject(data=result)
elif isinstance(result, six.string_types):
result = ResponseObject(data={'Return': result})
elif result is None or result is True:
result = ResponseObject()
else:
result = ResponseObject(
reason=
'Return value from Function %s is of unsupported type %s'
% (func.__name__, result.__class__),
responseStatus=Status.FAILED)
logger.error(result.reason)
# This block will hide resources on delete failure if the flag is set to true
if event['RequestType'] == RequestType.DELETE and result.responseStatus == Status.FAILED \
and hideResourceDeleteFailure:
logger.warning('Hiding Resource DELETE request failure')
if result.data is not None:
if not result.squashPrintResponse:
logger.debug('Data:\n' + json.dumps(result.data))
else:
logger.debug('Data: [REDACTED]')
if result.reason is not None:
logger.debug('Reason: %s' % result.reason)
if result.physicalResourceId is not None:
logger.debug('PhysicalResourceId: %s' %
result.physicalResourceId)
result = ResponseObject(
reason=
'There may be resources created by this Custom Resource that have not been cleaned'
+
'up despite the fact this resource is in DELETE_COMPLETE',
physicalResourceId=result.physicalResourceId,
responseStatus=Status.SUCCESS)
try:
return_value = result.send(event, context)
except Exception as e:
if isinstance(e, FailedToSendResponseException):
raise e
logger.error('Malformed request, Exception: %s' % str(e))
if result.data is not None and not isinstance(
e, DataIsNotDictException):
if not result.squashPrintResponse:
logger.debug('Data:\n' + json.dumps(result.data))
else:
logger.debug('Data: [REDACTED]')
if result.reason is not None:
logger.debug('Reason: %s' % result.reason)
if result.physicalResourceId is not None:
logger.debug('PhysicalResourceId: %s' %
result.physicalResourceId)
if not isinstance(e, InvalidResponseStatusException):
logger.debug('Status: %s' % result.responseStatus)
result = ResponseObject(
reason='Malformed request, Exception: %s' % str(e),
physicalResourceId=result.physicalResourceId,
responseStatus=Status.FAILED)
return_value = result.send(event, context)
return return_value
def upload_file():
if request.method == 'POST':
if 'file' not in request.files:
flash('No file part')
return redirect(request.referrer)
file = request.files['file']
if file.filename == '':
flash('No selected file')
return redirect(request.referrer)
if file and allowed_file(secure_filename(file.filename)):
filename = secure_filename(file.filename)
if not os.path.isfile(
os.path.join(app.config['UPLOAD_FOLDER'], filename)):
new_name = str(uuid.uuid4()) + filename[
(filename.index('.')):] # Appends file extension to UUID
# Check for name collisions to ensure unique filename
while os.path.isfile(
os.path.join(app.config['UPLOAD_FOLDER'], new_name)):
new_name = str(
uuid.uuid4()) + filename[(filename.index('.')):]
file.save(os.path.join(app.config['UPLOAD_FOLDER'], new_name))
# Upload file to S3 bucket
s3_upload_file(
os.path.join(app.config['UPLOAD_FOLDER'], new_name),
BUCKET, new_name)
product = Product(
new_name,
os.path.join(app.config['UPLOAD_FOLDER'], new_name))
text = image_recognition.get_text(product.img_path,
config.PREPROCESSOR)
if text is None or text == '':
os.remove(
os.path.join(
os.path.join(app.config['UPLOAD_FOLDER'],
new_name)))
return render_template(
'error.html',
error="Unable to process the nutrition facts!")
n = NutritionFacts()
product.facts = n.process_text(text)
facts = product.facts
if facts['Calories'] < 0 or facts[
'Carbohydrates'] < 0 or facts['Protein'] < 0:
os.remove(
os.path.join(
os.path.join(app.config['UPLOAD_FOLDER'],
new_name)))
return render_template(
'error.html',
error="Unable to correctly parse image data,"
" please upload a higher quality image")
# Add new image to database
n_facts = database.product_data(
file_name=product.name,
product_name=request.form['product_name'],
calories=int(facts['Calories']),
fat=int(facts['Fat']),
carbohydrates=int(facts['Carbohydrates']),
protein=int(facts['Protein']))
db_handler.add_model(n_facts)
s3 = boto3.client(
's3',
aws_access_key_id=os.environ['S3_ACCESS_KEY'],
aws_secret_access_key=os.environ['S3_SECRET_KEY'],
config=Config(signature_version='s3v4'),
region_name='us-east-2')
url = s3.generate_presigned_url('get_object',
Params={
'Bucket': BUCKET,
'Key': new_name
},
ExpiresIn=100)
print("URL: " + url)
return render_template(
'results.html',
message=facts,
resource=url,
productName=request.form['product_name'])
else:
return render_template('error.html',
error="File already exists")
##############
## Vars init #
##############
# Object storage
access_key = os.getenv('AWS_ACCESS_KEY_ID', None)
secret_key = os.getenv('AWS_SECRET_ACCESS_KEY', None)
service_point = os.getenv('S3_URL_ENDPOINT', 'http://ceph-nano-0/')
s3client = boto3.client('s3',
'us-east-1',
endpoint_url=service_point,
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
use_ssl=True if 'https' in service_point else False)
s3sourceclient = boto3.client('s3', config=Config(signature_version=UNSIGNED))
# Buckets
bucket_source = os.getenv(
'BUCKET_SOURCE',
'https://polyglot-academy-pub.nyc3.digitaloceanspaces.com/liquor-ml')
bucket_source_name = bucket_source.split('/')[-1]
bucket_destination = os.getenv('BUCKET_BASE_NAME', 'liquor-images')
# Helper database
db_user = os.getenv('DATABASE_USER', 'liquorlab')
db_password = os.getenv('DATABASE_PASSWORD', 'liquorlab')
db_host = os.getenv('DATABASE_HOST', 'liquorlabdb')
db_db = os.getenv('DATABASE_DB', 'liquorlabdb')
# Delay between images
def _build_unsigned_client(self):
s3_client = self._build_client(lambda session: Config(signature_version=UNSIGNED))
self.register_signals(s3_client)
self._unsigned_client = s3_client
def __init__(self,
session=None,
region_name=None,
api_version=None,
use_ssl=None,
verify=None,
endpoint_url=None,
aws_access_key_id=None,
aws_secret_access_key=None,
aws_session_token=None,
config=None,
endpoints=None):
if session is None:
session = botocore.session.get_session()
else:
if not isinstance(session, botocore.session.Session):
try:
# Check for a boto3 session-ish object and get the internal botocore session
_session = session._session
except AttributeError:
raise ValueError(
'session must be a botocore or boto3 session')
else:
session = _session
if use_ssl:
raise ValueError('SSL/TLS is not supported. Set use_ssl=False.')
self._session = session
default_client_config = self._session.get_default_client_config()
if config is not None and default_client_config is not None:
# If a config is provided and a default config is set, then
# use the config resulting from merging the two.
config = default_client_config.merge(config)
elif default_client_config is not None:
# If a config was not provided then use the default
# client config from the session
config = default_client_config
self._client_config = config if config is not None else Config(
region_name=region_name)
# resolve the region name
if region_name is None:
if config and config.region_name:
region_name = config.region_name
else:
region_name = self._session.get_config_variable('region')
self._region_name = region_name
# Figure out the verify value base on the various
# configuration options.
if verify is None:
verify = self._session.get_config_variable('ca_bundle')
self._verify = verify
# Gather endpoints
self._endpoints = endpoints or []
if endpoint_url and endpoint_url not in self._endpoints:
# If endpoint_url is provided, include it
self._endpoints.insert(0, endpoint_url)
if not self._endpoints:
raise ValueError('No endpoints provided')
# Resolve credentials
if aws_access_key_id is not None and aws_secret_access_key is not None:
self._credentials = Credentials(aws_access_key_id,
aws_secret_access_key,
aws_session_token)
elif self._session._missing_cred_vars(aws_access_key_id,
aws_secret_access_key):
raise PartialCredentialsError(
provider='explicit',
cred_var=self._session._missing_cred_vars(
aws_access_key_id, aws_secret_access_key))
else:
self._credentials = self._session.get_credentials()
# Fake out the meta information as much as possible
loader = session.get_component('data_loader')
json_model = loader.load_service_model('dynamodb',
'service-2',
api_version=api_version)
service_model = ServiceModel(json_model, service_name='dynamodb')
event_emitter = session.get_component('event_emitter')
partition = None
self.meta = ClientMeta(event_emitter, self._client_config,
self._endpoints[0], service_model,
self._PY_TO_OP_NAME, partition)
# Check signing version
if self._client_config.signature_version and self._client_config.signature_version != 'v4':
logger.warning(
'DAX only supports SigV4 signing; given signature_version "%s" ignored.',
self._client_config.signature_version)
# Start cluster connection & background tasks
self._cluster = Cluster(self._region_name, self._endpoints,
self._credentials,
self._client_config.user_agent,
self._client_config.user_agent_extra,
self._client_config.connect_timeout,
self._client_config.read_timeout)
self._cluster.start()
def create_cisco_config(bucket_name, bucket_key, s3_url, bgp_asn, ssh):
log.info("Processing %s/%s", bucket_name, bucket_key)
#Download the VPN configuration XML document
s3 = boto3.client('s3',
endpoint_url=s3_url,
config=Config(s3={'addressing_style': 'virtual'},
signature_version='s3v4'))
config = s3.get_object(Bucket=bucket_name, Key=bucket_key)
xmldoc = minidom.parseString(config['Body'].read())
#Extract transit_vpc_configuration values
vpn_config = xmldoc.getElementsByTagName("transit_vpc_config")[0]
account_id = vpn_config.getElementsByTagName(
"account_id")[0].firstChild.data
vpn_endpoint = vpn_config.getElementsByTagName(
"vpn_endpoint")[0].firstChild.data
vpn_status = vpn_config.getElementsByTagName("status")[0].firstChild.data
preferred_path = vpn_config.getElementsByTagName(
"preferred_path")[0].firstChild.data
#Extract VPN connection information
vpn_connection = xmldoc.getElementsByTagName('vpn_connection')[0]
vpn_connection_id = vpn_connection.attributes['id'].value
customer_gateway_id = vpn_connection.getElementsByTagName(
"customer_gateway_id")[0].firstChild.data
vpn_gateway_id = vpn_connection.getElementsByTagName(
"vpn_gateway_id")[0].firstChild.data
vpn_connection_type = vpn_connection.getElementsByTagName(
"vpn_connection_type")[0].firstChild.data
#Determine the VPN tunnels to work with
if vpn_status == 'create':
tunnelId = getNextTunnelId(ssh)
else:
tunnelId = getExistingTunnelId(ssh, vpn_connection_id)
if tunnelId == 0:
return
log.info("%s %s with tunnel #%s and #%s.", vpn_status, vpn_connection_id,
tunnelId, tunnelId + 1)
# Create or delete the VRF for this connection
if vpn_status == 'delete':
log.info("we're not doing deletes yet")
raise Exception
#ipsec_tunnel = vpn_connection.getElementsByTagName("ipsec_tunnel")[0]
#customer_gateway=ipsec_tunnel.getElementsByTagName("customer_gateway")[0]
#customer_gateway_bgp_asn=customer_gateway.getElementsByTagName("bgp")[0].getElementsByTagName("asn")[0].firstChild.data
##Remove VPN configuration for both tunnels
#config_text = ['router bgp {}'.format(customer_gateway_bgp_asn)]
#config_text.append(' no address-family ipv4 vrf {}'.format(vpn_connection_id))
#config_text.append('exit')
#config_text.append('no ip vrf {}'.format(vpn_connection_id))
#config_text.append('interface Tunnel{}'.format(tunnelId))
#config_text.append(' shutdown')
#config_text.append('exit')
#config_text.append('no interface Tunnel{}'.format(tunnelId))
#config_text.append('interface Tunnel{}'.format(tunnelId+1))
#config_text.append(' shutdown')
#config_text.append('exit')
#config_text.append('no interface Tunnel{}'.format(tunnelId+1))
#config_text.append('no route-map rm-{} permit'.format(vpn_connection_id))
## Cisco requires waiting 60 seconds before removing the isakmp profile
#config_text.append('WAIT')
#config_text.append('WAIT')
#config_text.append('no crypto isakmp profile isakmp-{}-{}'.format(vpn_connection_id,tunnelId))
#config_text.append('no crypto isakmp profile isakmp-{}-{}'.format(vpn_connection_id,tunnelId+1))
#config_text.append('no crypto keyring keyring-{}-{}'.format(vpn_connection_id,tunnelId))
#config_text.append('no crypto keyring keyring-{}-{}'.format(vpn_connection_id,tunnelId+1))
else:
# Create global tunnel configuration
config_text = ['ip vrf {}'.format(vpn_connection_id)]
config_text.append(' rd {}:{}'.format(bgp_asn, tunnelId))
config_text.append(' route-target export {}:100'.format(bgp_asn))
config_text.append(' route-target import {}:200'.format(bgp_asn))
config_text.append('exit')
# Check to see if a route map is needed for creating a preferred path
if preferred_path != 'none':
config_text.append(
'route-map rm-{} permit'.format(vpn_connection_id))
# If the preferred path is this transit VPC vpn endpoint, then set a shorter as-path prepend than if it is not
if preferred_path == vpn_endpoint:
config_text.append(' set as-path prepend {}'.format(bgp_asn))
else:
config_text.append(' set as-path prepend {} {}'.format(
bgp_asn, bgp_asn))
config_text.append('exit')
# Create tunnel specific configuration
for ipsec_tunnel in vpn_connection.getElementsByTagName(
"ipsec_tunnel"):
customer_gateway = ipsec_tunnel.getElementsByTagName(
"customer_gateway")[0]
customer_gateway_tunnel_outside_address = customer_gateway.getElementsByTagName(
"tunnel_outside_address")[0].getElementsByTagName(
"ip_address")[0].firstChild.data
customer_gateway_tunnel_inside_address_ip_address = customer_gateway.getElementsByTagName(
"tunnel_inside_address")[0].getElementsByTagName(
"ip_address")[0].firstChild.data
customer_gateway_tunnel_inside_address_network_mask = customer_gateway.getElementsByTagName(
"tunnel_inside_address")[0].getElementsByTagName(
"network_mask")[0].firstChild.data
customer_gateway_tunnel_inside_address_network_cidr = customer_gateway.getElementsByTagName(
"tunnel_inside_address")[0].getElementsByTagName(
"network_cidr")[0].firstChild.data
customer_gateway_bgp_asn = customer_gateway.getElementsByTagName(
"bgp")[0].getElementsByTagName("asn")[0].firstChild.data
customer_gateway_bgp_hold_time = customer_gateway.getElementsByTagName(
"bgp")[0].getElementsByTagName("hold_time")[0].firstChild.data
vpn_gateway = ipsec_tunnel.getElementsByTagName("vpn_gateway")[0]
vpn_gateway_tunnel_outside_address = vpn_gateway.getElementsByTagName(
"tunnel_outside_address")[0].getElementsByTagName(
"ip_address")[0].firstChild.data
vpn_gateway_tunnel_inside_address_ip_address = vpn_gateway.getElementsByTagName(
"tunnel_inside_address")[0].getElementsByTagName(
"ip_address")[0].firstChild.data
vpn_gateway_tunnel_inside_address_network_mask = vpn_gateway.getElementsByTagName(
"tunnel_inside_address")[0].getElementsByTagName(
"network_mask")[0].firstChild.data
vpn_gateway_tunnel_inside_address_network_cidr = vpn_gateway.getElementsByTagName(
"tunnel_inside_address")[0].getElementsByTagName(
"network_cidr")[0].firstChild.data
vpn_gateway_bgp_asn = vpn_gateway.getElementsByTagName(
"bgp")[0].getElementsByTagName("asn")[0].firstChild.data
vpn_gateway_bgp_hold_time = vpn_gateway.getElementsByTagName(
"bgp")[0].getElementsByTagName("hold_time")[0].firstChild.data
ike = ipsec_tunnel.getElementsByTagName("ike")[0]
ike_authentication_protocol = ike.getElementsByTagName(
"authentication_protocol")[0].firstChild.data
ike_encryption_protocol = ike.getElementsByTagName(
"encryption_protocol")[0].firstChild.data
ike_lifetime = ike.getElementsByTagName(
"lifetime")[0].firstChild.data
ike_perfect_forward_secrecy = ike.getElementsByTagName(
"perfect_forward_secrecy")[0].firstChild.data
ike_mode = ike.getElementsByTagName("mode")[0].firstChild.data
ike_pre_shared_key = ike.getElementsByTagName(
"pre_shared_key")[0].firstChild.data
ipsec = ipsec_tunnel.getElementsByTagName("ipsec")[0]
ipsec_protocol = ipsec.getElementsByTagName(
"protocol")[0].firstChild.data
ipsec_authentication_protocol = ipsec.getElementsByTagName(
"authentication_protocol")[0].firstChild.data
ipsec_encryption_protocol = ipsec.getElementsByTagName(
"encryption_protocol")[0].firstChild.data
ipsec_lifetime = ipsec.getElementsByTagName(
"lifetime")[0].firstChild.data
ipsec_perfect_forward_secrecy = ipsec.getElementsByTagName(
"perfect_forward_secrecy")[0].firstChild.data
ipsec_mode = ipsec.getElementsByTagName("mode")[0].firstChild.data
ipsec_clear_df_bit = ipsec.getElementsByTagName(
"clear_df_bit")[0].firstChild.data
ipsec_fragmentation_before_encryption = ipsec.getElementsByTagName(
"fragmentation_before_encryption")[0].firstChild.data
ipsec_tcp_mss_adjustment = ipsec.getElementsByTagName(
"tcp_mss_adjustment")[0].firstChild.data
ipsec_dead_peer_detection_interval = ipsec.getElementsByTagName(
"dead_peer_detection")[0].getElementsByTagName(
"interval")[0].firstChild.data
ipsec_dead_peer_detection_retries = ipsec.getElementsByTagName(
"dead_peer_detection")[0].getElementsByTagName(
"retries")[0].firstChild.data
config_text.append('crypto keyring keyring-{}-{}'.format(
vpn_connection_id, tunnelId))
config_text.append(' local-address GigabitEthernet1')
config_text.append(' pre-shared-key address {} key {}'.format(
vpn_gateway_tunnel_outside_address, ike_pre_shared_key))
config_text.append('exit')
config_text.append('crypto isakmp profile isakmp-{}-{}'.format(
vpn_connection_id, tunnelId))
config_text.append(' local-address GigabitEthernet1')
config_text.append(' match identity address {}'.format(
vpn_gateway_tunnel_outside_address))
config_text.append(' keyring keyring-{}-{}'.format(
vpn_connection_id, tunnelId))
config_text.append('exit')
config_text.append('interface Tunnel{}'.format(tunnelId))
config_text.append(
' description {} from {} to {} for account {}'.format(
vpn_connection_id, vpn_gateway_id, customer_gateway_id,
account_id))
config_text.append(' bandwidth 1000000')
config_text.append(' ip mtu 1340')
config_text.append(' ip tcp adjust-mss 1300')
config_text.append(
' ip vrf forwarding {}'.format(vpn_connection_id))
config_text.append(' ip address {} 255.255.255.252'.format(
customer_gateway_tunnel_inside_address_ip_address))
config_text.append(' ip virtual-reassembly')
config_text.append(' tunnel source GigabitEthernet1')
config_text.append(' tunnel destination {} '.format(
vpn_gateway_tunnel_outside_address))
config_text.append(' tunnel mode ipsec ipv4')
config_text.append(
' tunnel protection ipsec profile ipsec-vpn-aws')
config_text.append(' ip tcp adjust-mss 1387')
config_text.append(' no shutdown')
config_text.append('exit')
config_text.append(
'router bgp {}'.format(customer_gateway_bgp_asn))
config_text.append(
' address-family ipv4 vrf {}'.format(vpn_connection_id))
config_text.append(' neighbor {} remote-as {}'.format(
vpn_gateway_tunnel_inside_address_ip_address,
vpn_gateway_bgp_asn))
config_text.append(' neighbor {} maximum-prefix 2'.format(
vpn_gateway_tunnel_inside_address_ip_address))
if preferred_path != 'none':
config_text.append(' neighbor {} route-map rm-{} out'.format(
vpn_gateway_tunnel_inside_address_ip_address,
vpn_connection_id))
config_text.append(' neighbor {} timers 10 30 30'.format(
vpn_gateway_tunnel_inside_address_ip_address))
config_text.append(' neighbor {} activate'.format(
vpn_gateway_tunnel_inside_address_ip_address))
config_text.append(' neighbor {} as-override'.format(
vpn_gateway_tunnel_inside_address_ip_address))
config_text.append(
' neighbor {} soft-reconfiguration inbound'.format(
vpn_gateway_tunnel_inside_address_ip_address))
config_text.append(' neighbor {} next-hop-self'.format(
vpn_gateway_tunnel_inside_address_ip_address))
config_text.append('exit')
config_text.append('exit')
#Increment tunnel ID for going onto the next tunnel
tunnelId += 1
log.debug("Conversion complete")
return config_text
def post(self, request):
reservation = self.get_reservation(request.data['id'])
if str(reservation.code) == request.data['code']:
if not (reservation.num_of_unpaid > 0):
s3_client = session.client(
's3',
region_name="us-east-2",
config=Config(signature_version='s3v4'))
if (reservation.date + timedelta(weeks=24)) > timezone.now():
serializer = ReservationSerializer(reservation)
images = ReservationImage.objects.filter(
reservation=reservation)
responses = []
for image in images:
s3_object_name = "photos" + "/" + str(
reservation.id) + "/" + str(
image.name) + "/" + "image.jpg"
s3_object_name_resized = "resized-photos" + "/" + str(
reservation.id) + "/" + str(
image.name) + "/" + "image.jpg"
response = s3_client.generate_presigned_url(
"get_object",
Params={
"Bucket": settings.AWS_STORAGE_BUCKET_NAME,
"Key": s3_object_name
},
ExpiresIn=3600)
response_resized = s3_client.generate_presigned_url(
"get_object",
Params={
"Bucket":
settings.AWS_STORAGE_BUCKET_NAME_RESIZED,
"Key": s3_object_name_resized,
},
ExpiresIn=3600)
image_urls = {
"original": response,
"thumbnail": response_resized
}
responses.append(image_urls)
res = {"images": responses, "data": serializer.data}
return Response(res, status=status.HTTP_200_OK)
else:
if reservation.count_of_visit < 300:
serializer = ReservationSerializer(reservation)
reservation.count_of_visit += 1
reservation.save()
images = ReservationImage.objects.filter(
reservation=reservation)
responses = []
for image in images:
s3_object_name = "photos" + "/" + str(
reservation.id) + "/" + str(
image.name) + "/" + "image.jpg"
s3_object_name_resized = "resized-photos" + "/" + str(
reservation.id) + "/" + str(
image.name) + "/" + "image.jpg"
response = s3_client.generate_presigned_url(
"get_object",
Params={
"Bucket": settings.AWS_STORAGE_BUCKET_NAME,
"Key": s3_object_name,
},
ExpiresIn=3600)
response_resized = s3_client.generate_presigned_url(
"get_object",
Params={
"Bucket":
settings.AWS_STORAGE_BUCKET_NAME_RESIZED,
"Key": s3_object_name_resized,
},
ExpiresIn=3600)
image_urls = {
"original": response,
"thumbnail": response_resized
}
responses.append(image_urls)
res = {"images": responses, "data": serializer.data}
return Response(res, status=status.HTTP_200_OK)
else:
return Response(status=status.HTTP_403_FORBIDDEN)
else:
return Response(status=status.HTTP_402_PAYMENT_REQUIRED)
else:
return Response(status=status.HTTP_401_UNAUTHORIZED)
import types
import pandas as pd
from botocore.client import Config
import ibm_boto3
def __iter__(self): return 0
# @hidden_cell
# The following code accesses a file in your IBM Cloud Object Storage. It includes your credentials.
# You might want to remove those credentials before you share your notebook.
client_bba26c7923b04a17832943e1b244b58e = ibm_boto3.client(service_name='s3',
ibm_api_key_id='MTHFdZK_I6-W1zWk4b-hEUyJA2ihmrD9b61PU91_ib9B',
ibm_auth_endpoint="https://iam.bluemix.net/oidc/token",
config=Config(signature_version='oauth'),
endpoint_url='https://s3.eu-geo.objectstorage.service.networklayer.com')
body = client_bba26c7923b04a17832943e1b244b58e.get_object(Bucket='graduateadmissionprediction-donotdelete-pr-8bat18q13wgs5s',Key='Admission_Predict_Ver1.1.csv')['Body']
# add missing __iter__ method, so pandas accepts body as file-like object
if not hasattr(body, "__iter__"): body.__iter__ = types.MethodType( __iter__, body )
dataset = pd.read_csv(body)
dataset.head()
# In[54]:
dataset
def s3_client():
return boto3.client('s3',
config=Config(signature_version='s3v4'),
region_name='us-east-1',
aws_access_key_id=config.AWS_ACCESS_KEY_ID,
aws_secret_access_key=config.AWS_SECRET_ACCESS_KEY)
def CardCreator(company, phoneNumber, street, city, region, code, country, website, note):
access_key = os.environ['S3_KEY']
secret_key = os.environ['S3_SECRET']
regionS3 = "eu-central-1"
key = "contactcards/" + str(company) + '.vcf'
bucket = os.environ['S3_BUCKET']
file = tempfile.TemporaryFile("w+b")
file.write("BEGIN:VCARD\nVERSION:3.0\nPRODID:-//Apple Inc.//iPhone OS 13.0//EN\nN:;;;;\n".encode("utf-8"))
file.write(("FN:" + str(company) + "\n").encode("utf-8"))
file.write(("ORG:" + str(company) + "\n").encode("utf-8"))
if note:
file.write("NOTE:".encode("utf-8"))
for i in range(0, len(note)):
if i is len(note)-1:
file.write(str(note[i]).encode("utf-8"))
continue
file.write((str(note[i]) + "\\n").encode("utf-8"))
file.write(("\nTEL;type=WORK;type=VOICE;type=pref:" + str(phoneNumber) + "\n").encode("utf-8"))
file.write(("item1.ADR;type=WORK;type=pref:;;" + str(street) + ";" + str(city) + ";" + str(region) + ";" + str(code) + ";" + str(country) + "\n").encode("utf-8"))
file.write("item1.X-ABADR:de\n".encode("utf-8"))
file.write(("item2.URL;type=pref:" + str(website) + "\n").encode("utf-8"))
file.write("item2.X-ABLabel:_$!<HomePage>!$_\nX-ABShowAs:COMPANY\nEND:VCARD\n".encode("utf-8"))
file.seek(0)
s3_client = boto3.client('s3', region_name=regionS3, aws_access_key_id=access_key, aws_secret_access_key=secret_key, config=Config(signature_version='s3v4'))
s3 = boto3.resource('s3', region_name=regionS3, aws_access_key_id=access_key, aws_secret_access_key=secret_key, config=Config(signature_version='s3v4'))
s3.Bucket(bucket).put_object(Key=key, Body=file)
objectUrl = s3_client.generate_presigned_url(ClientMethod='get_object', Params={'Bucket': bucket, 'Key': key})
response = urllib.request.urlretrieve(str(objectUrl))
print(response)
return objectUrl
def get_s3_dcm(bucket, file_key):
"""Read DICOM from S3"""
s3_config = Config(connect_timeout=50, read_timeout=70)
s3_client = boto3.client('s3', config=s3_config) # low-level functional API
obj = s3_client.get_object(Bucket=bucket, Key=file_key)
return pydicom.read_file(BytesIO(obj['Body'].read()))
def create_client(self):
# Even though the default signature_version is s3,
# we're being explicit in case this ever changes.
client_config = Config(signature_version='s3')
return self.session.create_client('s3', self.region,
config=client_config)
def get_s3_session(self):
return boto3.client('s3',
config=Config(signature_version='s3v4',
region_name=self.S3_REGION))
def create_client(self):
return self.session.create_client(
's3', region_name=self.region,
config=Config(s3={'addressing_style': self.addressing_style}))
import boto3
from botocore.client import Config
import StringIO
import zipfile
import mimetypes
s3 = boto3.resource('s3', config=Config(signature_version='s3v4'))
portfolio_bucket = s3.Bucket('portfolio.bala.info')
build_bucket = s3.Bucket('portfolio.build.bala.info')
portfolio_zip = StringIO.StringIO()
build_bucket.download_fileobj('portfoliobuild.zip', portfolio_zip)
with zipfile.ZipFile(portfolio_zip) as myzip:
for nm in myzip.namelist():
obj = myzip.open(nm)
portfolio_bucket.upload_fileobj(
obj, nm, ExtraArgs={'ContentType': mimetypes.guess_type(nm)[0]})
portfolio_bucket.Object(nm).Acl().put(ACL='public-read')
def lambda_handler(event, context):
bucket = os.environ["BUCKET_NAME"] #Using enviroment varibles below the lambda will use your S3 bucket
DestinationPrefix = os.environ["PREFIX"]
####CODE TO GET DATA CAN BE REPLACED######
client = boto3.client('ecs')
paginator = client.get_paginator("list_clusters") #Paginator for a large list of accounts
response_iterator = paginator.paginate()
with open('/tmp/data.json', 'w') as f: # Saving in the temporay folder in the lambda
for response in response_iterator: # extracts the needed info
for cluster in response['clusterArns']:
listservices = client.list_services(cluster=cluster.split( '/')[1],maxResults=100)
for i in listservices['serviceArns']:
#print (i)
services = client.describe_services(
cluster=cluster.split( '/')[1],
services=[
i.split( '/')[2],
],
include=[
'TAGS',
]
)
for service in services['services']:
data = {'cluster':cluster.split( '/')[1], 'services':i.split( '/')[2], 'serviceName': service.get('serviceName'), 'tags':service.get('tags') }
print(data)
####CODE TO GET DATA######
jsondata = json.dumps(data) #converts datetime to be able to placed in json
f.write(jsondata)
f.write('\n')
print("respose gathered")
today = date.today()
year = today.year
month = today.month
try:
s3 = boto3.client('s3', config=Config(s3={'addressing_style': 'path'}))
s3.upload_file(
'/tmp/data.json', bucket, f"{DestinationPrefix}-data/year={year}/month={month}/{DestinationPrefix}.json") #uploading the file with the data to s3
print(f"Data in s3 - {DestinationPrefix}-data/year={year}/month={month}")
except Exception as e:
print(e)
start_crawler()
def start_crawler():
glue_client = boto3.client('glue')
os.environ['ROLE_ARN']
try:
glue_client.start_crawler(Name=os.environ['CRAWLER_NAME'])
except Exception as e:
# Send some context about this error to Lambda Logs
logging.warning("%s" % e)
def assume_role(account_id, service, region):
role_name = os.environ['ROLENAME']
role_arn = f"arn:aws:iam::{account_id}:role/{role_name}" #OrganizationAccountAccessRole
sts_client = boto3.client('sts')
try:
#region = sts_client.meta.region_name
assumedRoleObject = sts_client.assume_role(
RoleArn=role_arn,
RoleSessionName="AssumeRoleRoot"
)
credentials = assumedRoleObject['Credentials']
client = boto3.client(
service,
aws_access_key_id=credentials['AccessKeyId'],
aws_secret_access_key=credentials['SecretAccessKey'],
aws_session_token=credentials['SessionToken'],
region_name = region
)
return client
except ClientError as e:
logging.warning(f"Unexpected error Account {account_id}: {e}")
return None
def lits_regions():
from boto3.session import Session
s = Session()
ecs_regions = s.get_available_regions('ecs')
return ecs_regions
import numpy as np
from PIL import Image
from tqdm import tqdm
from joblib import Parallel, delayed, cpu_count
import math
import tifffile as tf
from util import (
tqdm_joblib,
chunks,
imgResample,
upload_file_to_s3,
S3Url,
s3_object_exists,
)
config = Config(connect_timeout=5, retries={"max_attempts": 5})
def get_out_path(in_path, outdir):
head, fname = os.path.split(in_path)
head_tmp = head.split("/")
head = f"{outdir}/" + "/".join(head_tmp[-1:])
idx = fname.find(".")
fname_new = fname[:idx] + "_corrected.tiff"
out_path = f"{head}/{fname_new}"
os.makedirs(head, exist_ok=True) # succeeds even if directory exists.
return out_path
def get_all_s3_objects(s3, **base_kwargs):
continuation_token = None
logging.basicConfig(format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S %p',
filename=logfile,
level=logging.INFO)
with open('config.json') as json_data_file:
data = json.load(json_data_file)
accesskey = data['AWSAccess']
secretkey = data['AWSSecret']
#Create a connection
s3 = boto3.resource('s3',
aws_access_key_id=accesskey,
aws_secret_access_key=secretkey,
config=Config(signature_version='s3v4'))
#boto3.set_stream_logger('boto3.resources', logging.INFO)
#Create a bucket
#logging.info('Connection created')
#s3.create_bucket(Bucket='team7pa_assignment1')
logging.info('Bucket created')
bucket = s3.Bucket('team7pa_assignment1')
bucketlen = len(list(bucket.objects.all()))
print(bucketlen)
if bucketlen == 0:
print(cwd)
#initial data uploading get files from configinitial
with open('configinitial.json') as json_data_file:
data = json.load(json_data_file)
for i in data["result"]:
url = i["link"]
import argparse
import concurrent.futures
from botocore.client import Config
import boto3
import time
import tempfile
config = Config(connect_timeout=5, read_timeout=5, retries={'max_attempts': 1})
s3 = boto3.client('s3', config=config)
parser = argparse.ArgumentParser(
description='MegaJQ runs SQL-statements through S3-prefixes parallel')
parser.add_argument("bucket", help="S3 bucket where select will be run")
parser.add_argument("prefix", help="S3 key prefix")
parser.add_argument(
"query",
help=
"SQL-query, see https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-glacier-select-sql-reference-select.html"
)
parser.add_argument("output", help="Output file")
args = parser.parse_args()
print(args)
BUCKET = args.bucket
PREFIX = args.prefix
query = args.query
OUTPUT = args.output
def _create_client(self):
return self._client_creator(
'sts', config=Config(signature_version=botocore.UNSIGNED))
def open_portal(context, args):
project_resources = context.config.project_resources
if not project_resources.has_key(constant.PROJECT_CGP_RESOURCE_NAME):
raise HandledError(
'You can not open the Cloud Gem Portal without having the Cloud Gem Portal gem installed in your project.'
)
project_config_bucket_id = context.config.configuration_bucket_name
cgp_s3_resource = project_resources[constant.PROJECT_CGP_RESOURCE_NAME]
stackid = cgp_s3_resource['StackId']
bucket_id = cgp_s3_resource['PhysicalResourceId']
expiration = args.duration_seconds if args.duration_seconds else constant.PROJECT_CGP_DEFAULT_EXPIRATION_SECONDS # default comes from argparse only on cli, gui call doesn't provide a default expiration
region = resource_manager.util.get_region_from_arn(stackid)
s3_client = context.aws.session.client(
's3', region, config=Config(signature_version='s3v4'))
user_pool_resource = project_resources[
constant.PROJECT_RESOURCE_NAME_USER_POOL]
identity_pool_resource = project_resources[
constant.PROJECT_RESOURCE_NAME_IDENTITY_POOL]
if 'CloudGemPortalApp' not in user_pool_resource['UserPoolClients']:
credentials = context.aws.load_credentials()
access_key = credentials.get(constant.DEFAULT_SECTION_NAME,
constant.ACCESS_KEY_OPTION)
raise HandledError(
'The Cognito user pool \'{}\' is missing the \'CloudGemPortalApp\' app client. Ensure the Lumberyard user \'{}\' with AWS access key identifier \'{}\' in the Lumberyard Credentials Manager has the policy \'AmazonCognitoReadOnly\' attached and a project stack has been created (Lumberyard -> AWS -> Resource Manager).'
.format(constant.PROJECT_RESOURCE_NAME_USER_POOL,
context.config.user_default_profile, access_key))
client_id = user_pool_resource['UserPoolClients']['CloudGemPortalApp'][
'ClientId']
user_pool_id = user_pool_resource['PhysicalResourceId']
identity_pool_id = identity_pool_resource['PhysicalResourceId']
#create an administrator account if one is not present
output = __validate_administrator_account(context, args)
admin_account_created = __is_first_time_usage(output)
#Request the index file
try:
s3_index_obj_request = s3_client.get_object(
Bucket=bucket_id, Key=constant.PROJECT_CGP_ROOT_FILE)
except ClientError as e:
raise HandledError(
"Could not read from the key '{}' in the S3 bucket '{}'.".format(
constant.PROJECT_CGP_ROOT_FILE, bucket_id), e)
#Does the user have access to it?
if s3_index_obj_request['ResponseMetadata']['HTTPStatusCode'] != 200:
raise HandledError(
"The user does not have access to the file index.html file. This Cloud Gem Portal site will not load."
)
content = s3_index_obj_request['Body'].read().decode('utf-8')
if args.show_current_configuration:
try:
cgp_current_bootstrap_config = s3_client.get_object(
Bucket=bucket_id, Key=constant.PROJECT_CGP_ROOT_SUPPORT_FILE)
cgp_current_bootstrap_config = cgp_current_bootstrap_config[
'Body'].read().decode('utf-8')
context.view._output_message(
cgp_current_bootstrap_config.replace(BOOTSTRAP_VARIABLE_NAME,
''))
return
except ClientError as e:
raise HandledError(
"Could not read from the key '{}' in the S3 bucket '{}'.".
format(constant.PROJECT_CGP_ROOT_SUPPORT_FILE, bucket_id), e)
cgp_bootstrap_config = {
"clientId": client_id,
"userPoolId": user_pool_id,
"identityPoolId": identity_pool_id,
"projectConfigBucketId": project_config_bucket_id,
"region": region,
"firstTimeUse": admin_account_created,
"cognitoDev": args.cognito_dev if args.cognito_dev != "''" else None,
"cognitoProd":
args.cognito_prod if args.cognito_prod != "''" else None,
"cognitoTest": args.cognito_test if args.cognito_test != "''" else None
}
content = set_presigned_urls(content, bucket_id, s3_client, expiration,
region)
result = None
try:
# TODO: write to an unique name and configure bucket to auto delete these objects after 1 hour
# the max allowed --duration-seconds value.
s3_client.put_object(Bucket=bucket_id,
Key=constant.PROJECT_CGP_ROOT_SUPPORT_FILE,
Body="var bootstrap = {}".format(
json.dumps(cgp_bootstrap_config)),
ContentType='text/html')
result = s3_client.put_object(Bucket=bucket_id,
Key=constant.PROJECT_CGP_ROOT_FILE,
Body=content,
ContentType='text/html')
except ClientError as e:
if e.response["Error"]["Code"] in ["AccessDenied"]:
credentials = context.aws.load_credentials()
access_key = credentials.get(constant.DEFAULT_SECTION_NAME,
constant.ACCESS_KEY_OPTION)
context.view._output_message(
"The Lumberyard user '{0}' associated with AWS IAM access key identifier '{1}' is missing PUT permissions on the S3 bucket '{2}'. Now attempting to use old Cloud Gem Portal pre-signed urls.\nHave the administrator grant the AWS user account with access key '{1}' S3 PUT permissions for bucket '{2}'"
.format(context.config.user_default_profile, access_key,
bucket_id))
else:
raise HandledError(
"Could write to the key '{}' in the S3 bucket '{}'.".format(
constant.PROJECT_CGP_ROOT_FILE, bucket_id), e)
if result == None or result['ResponseMetadata']['HTTPStatusCode'] == 200:
if result != None and not set_bucket_cors(
context, project_config_bucket_id, region):
raise HandledError(
"Warning: the Cross Origin Resource Sharing (CORS) policy cloud not be set: Access Denied. This may prevent the Cloud Gem Portal from accessing the projects project-settings.json file."
)
#generate presigned url
secured_url = __get_presigned_url(s3_client, bucket_id,
constant.PROJECT_CGP_ROOT_FILE,
expiration)
__updateUserPoolEmailMessage(context, secured_url,
project_config_bucket_id)
if args.show_configuration:
context.view._output_message(json.dumps(cgp_bootstrap_config))
if args.show_url_only:
context.view._output_message(secured_url)
else:
webbrowser.open_new(secured_url)
else:
raise HandledError(
"The index.html cloud not be set in the S3 bucket '{}'. This Cloud Gem Portal site will not load."
.format(bucket_id))
import os
import logging
import json
import boto3
from boto3.dynamodb.conditions import Key
from botocore.client import Config
_logger: logging.Logger = logging.getLogger(__name__)
sfn_client_config = Config(connect_timeout=50, read_timeout=70)
sfn = boto3.client('stepfunctions', config=sfn_client_config)
sts = boto3.client('sts')
crawler = boto3.client('glue')
dynamodb = boto3.resource('dynamodb')
_config = {
"sfn_activity_arn":
"arn:aws:states:{}:{}:activity:CrawlerRunnerActivity".format(
os.getenv("REGION", "eu-west-2"),
sts.get_caller_identity()['Account']),
"sfn_worker_name":
"crawlerrunner",
"crawler_name":
"openaq_curated_crawler",
"ddb_table":
"CrawlerRunnerActiveJobs",
"ddb_query_limit":
50,
"sfn_max_executions":
def write_bootstrap(
context,
customer_cognito_id,
expiration=constant.PROJECT_CGP_DEFAULT_EXPIRATION_SECONDS):
project_resources = context.config.project_resources
if not project_resources.has_key(constant.PROJECT_CGP_RESOURCE_NAME):
raise HandledError(
'You can not open the Cloud Gem Portal without having the Cloud Gem Portal gem installed in your project.'
)
cgp_s3_resource = project_resources[constant.PROJECT_CGP_RESOURCE_NAME]
stack_id = cgp_s3_resource['StackId']
bucket_id = cgp_s3_resource['PhysicalResourceId']
region = resource_manager.util.get_region_from_arn(stack_id)
s3_client = context.aws.session.client(
's3', region, config=Config(signature_version='s3v4'))
user_pool_resource = project_resources[
constant.PROJECT_RESOURCE_NAME_USER_POOL]
identity_pool_resource = project_resources[
constant.PROJECT_RESOURCE_NAME_IDENTITY_POOL]
project_handler = project_resources[
constant.PROJECT_RESOURCE_HANDLER_NAME]['PhysicalResourceId']
project_config_bucket_id = context.config.configuration_bucket_name
if 'CloudGemPortalApp' not in user_pool_resource['UserPoolClients']:
credentials = context.aws.load_credentials()
access_key = credentials.get(constant.DEFAULT_SECTION_NAME,
constant.ACCESS_KEY_OPTION)
raise HandledError(
'The Cognito user pool \'{}\' is missing the \'CloudGemPortalApp\' app client. \
Ensure the Lumberyard user \'{}\' with AWS access key identifier \'{}\' in the Lumberyard Credentials Manager \
has the policy \'AmazonCognitoReadOnly\' attached and a project stack has been created (Lumberyard -> AWS -> Resource Manager).'
.format(constant.PROJECT_RESOURCE_NAME_USER_POOL,
context.config.user_default_profile, access_key))
client_id = user_pool_resource['UserPoolClients']['CloudGemPortalApp'][
'ClientId']
user_pool_id = user_pool_resource['PhysicalResourceId']
identity_pool_id = identity_pool_resource['PhysicalResourceId']
# create an administrator account if one is not present
is_new_user, username, password = create_portal_administrator(context)
cgp_bootstrap_config = {
"clientId": client_id,
"userPoolId": user_pool_id,
"identityPoolId": identity_pool_id,
"projectConfigBucketId": project_config_bucket_id,
"region": region,
"firstTimeUse": is_new_user,
"cognitoProd":
customer_cognito_id if customer_cognito_id != "''" else None,
"projectPhysicalId": project_handler
}
if password is not None:
context.view.create_admin(
username, password,
'The Cloud Gem Portal administrator account has been created.')
content = get_index(s3_client, bucket_id)
content = content.replace(
get_bootstrap(s3_client, bucket_id),
'<script>{}{}</script>'.format(BOOTSTRAP_VARIABLE_NAME,
json.dumps(cgp_bootstrap_config)))
result = None
try:
# TODO: write to an unique name and configure bucket to auto delete these objects after 1 hour
# the max allowed --duration-seconds value.
result = s3_client.put_object(Bucket=bucket_id,
Key=constant.PROJECT_CGP_ROOT_FILE,
Body=content,
ContentType='text/html')
except ClientError as e:
if e.response["Error"]["Code"] in ["AccessDenied"]:
credentials = context.aws.load_credentials()
access_key = credentials.get(constant.DEFAULT_SECTION_NAME,
constant.ACCESS_KEY_OPTION)
context.view._output_message(
"The Lumberyard user '{0}' associated with AWS IAM access key identifier '{1}' is missing PUT permissions on the S3 bucket '{2}'. Now attempting to use old Cloud Gem Portal pre-signed urls.\nHave the administrator grant the AWS user account with access key '{1}' S3 PUT permissions for bucket '{2}'"
.format(context.config.user_default_profile, access_key,
bucket_id))
else:
raise HandledError(
"Could not write to the key '{}' in the S3 bucket '{}'.".
format(constant.PROJECT_CGP_ROOT_FILE, bucket_id), e)
if result == None or result['ResponseMetadata']['HTTPStatusCode'] == 200:
context.view._output_message(
"The Cloud Gem Portal bootstrap information has been written successfully."
)
else:
raise HandledError(
"The index.html cloud not be set in the S3 bucket '{}'. This Cloud Gem Portal site will not load."
.format(bucket_id))
updateUserPoolEmailMessage(
context,
get_index_url(s3_client, bucket_id,
constant.PROJECT_CGP_DEFAULT_EXPIRATION_SECONDS),
project_config_bucket_id)
