def __init__(self, gs_access_key_id=None, gs_secret_access_key=None, is_secure=True, port=None, proxy=None, proxy_port=None, proxy_user=None, proxy_pass=None, host=DefaultHost, debug=0, https_connection_factory=None, calling_format=SubdomainCallingFormat(), path='/'): S3Connection.__init__(self, gs_access_key_id, gs_secret_access_key, is_secure, port, proxy, proxy_port, proxy_user, proxy_pass, host, debug, https_connection_factory, calling_format, path, "google")
def __init__(self, access_key=None, secret_key=None): from pinhole.common.app import app if not access_key: access_key = app.config["AWS_ACCESS_KEY"] if not secret_key: secret_key = app.config["AWS_SECRET_KEY"] S3Connection.__init__(self, access_key, secret_key)
def get_bucket(conn: S3Connection, bucket_name: str) -> Bucket: # Calling get_bucket() with validate=True can apparently lead # to expensive S3 bills: # http://www.appneta.com/blog/s3-list-get-bucket-default/ # The benefits of validation aren't completely clear to us, and # we want to save on our bills, so we set the validate flag to False. # (We think setting validate to True would cause us to fail faster # in situations where buckets don't exist, but that shouldn't be # an issue for us.) bucket = conn.get_bucket(bucket_name, validate=False) return bucket
def __init__(self, access_key_id, secret_access_key, bucket_name, filenames, upload_options, wizard_page, parent=None): S3Connection.__init__(self, access_key_id, secret_access_key) self.upload_options = upload_options self.bucket_name = bucket_name self.bucket = None self.filenames = filenames self.s3Uploaders = [] self.threads = [] self.count_uploaded_images = 0 self.num_uploading_images = 0 #For GUI (messages and progress bars) self.wizard_page = wizard_page self.uploader_widget = QtGui.QWidget() self.uploader_widget.setWindowTitle("Upload Progress Bars") self.uploader_widget.setWindowFlags(Qt.WindowStaysOnTopHint) self.uploader_v_box = QtGui.QVBoxLayout() self.uploader_widget.setLayout(self.uploader_v_box) self.msg_bar_main = None self.msg_bar_main_content = None self.cancel_button_main = None self.msg_bars = [] self.msg_bars_content = [] self.progress_bars = [] self.cancel_buttons = []
def __init__(self, bucket_name): self.connection = S3Connection(os.environ.get('AWS_ACCESS_KEY_ID'), os.environ.get('AWS_SECRET_ACCESS_KEY')) self.bucket = self.connection.create_bucket(bucket_name)
def process(log): """Process the files""" log.append('Start Time: %s' % datetime.now()) conn = S3Connection(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY) bucket = conn.get_bucket(settings.AWS_AUTOIMPORT_BUCKET_NAME) storage_bucket = conn.get_bucket(settings.AWS_STORAGE_BUCKET_NAME) for key in bucket.list(prefix='scans/', delimiter='/'): if key.name == 'scans/': continue # strip off 'scans/' file_name = key.name[6:] try: (foia_pks, file_date, code, title, status, body, arg, id_, est_date) = parse_name(file_name) except ValueError as exc: s3_copy(bucket, key, 'review/%s' % file_name) s3_delete(bucket, key) log.append(unicode(exc)) continue for foia_pk in foia_pks: try: foia = FOIARequest.objects.get(pk=foia_pk) from_user = foia.agency.get_user() if foia.agency else None comm = FOIACommunication.objects.create( foia=foia, from_user=from_user, to_user=foia.user, response=True, date=file_date, communication=body, status=status, ) MailCommunication.objects.create( communication=comm, sent_datetime=file_date, ) foia.status = status or foia.status if foia.status in ['partial', 'done', 'rejected', 'no_docs']: foia.date_done = file_date.date() if code == 'FEE' and arg: foia.price = Decimal(arg) if id_: foia.tracking_id = id_ if est_date: foia.date_estimate = est_date if code == 'REJ-P': foia.proxy_reject() if key.name.endswith('/'): import_prefix(key, bucket, storage_bucket, comm, log) else: import_key(key, storage_bucket, comm, log, title=title) foia.save(comment='updated from autoimport files') action = generate_status_action(foia) foia.notify(action) foia.update(comm.anchor()) except FOIARequest.DoesNotExist: s3_copy(bucket, key, 'review/%s' % file_name) log.append('ERROR: %s references FOIA Request %s, but it does not exist' % (file_name, foia_pk)) except SoftTimeLimitExceeded: # if we reach the soft time limit, # re-raise so we can catch and clean up raise except Exception as exc: s3_copy(bucket, key, 'review/%s' % file_name) log.append('ERROR: %s has caused an unknown error. %s' % (file_name, exc)) logger.error('Autoimport error: %s', exc, exc_info=sys.exc_info()) # delete key after processing all requests for it s3_delete(bucket, key) log.append('End Time: %s' % datetime.now())
import pandas as pd import ssl import sys from boto.s3.connection import S3Connection def main(): ''' This pulls xml casefiles 1. Import credentals 2. Retrieve Casefiles 3. Loop Casefiles to Push to s3 ''' pass # AWS Credentials S3Conn = S3Connection('MyAWSaccessKey', 'MyAWSsecretKey') greatbucket = S3Conn.get_bucket('bucketname') ### Accessing Desktop Folder path = '/Users/mycomputerName/Desktop/myFolder/' # UPLOAD BY LOOPING THRU BY INDIVIDUAL XML FILE for i in (os.listdir(path)): file = greatbucket.new_key(i) file.set_contents_from_filename(path + '/' + i) if __name__ == '__main__': main()
def get_bucket(s3_bucket, aws_access_key_id, aws_secret_access_key, s3_connection_host): connection = S3Connection(aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, host=s3_connection_host) return connection.get_bucket(s3_bucket, validate=False)
import json import random import settings from flask import Flask, render_template, abort, redirect, request app = Flask(__name__) app.config.from_object(settings) from boto.s3.connection import S3Connection from boto.s3.key import Key conn = S3Connection(settings.S3_KEY_ID, settings.S3_SECRET) bucket = conn.get_bucket(settings.S3_BUCKET_NAME) @app.route('/sheets/<sheet_id>', methods=['PUT']) def save_sheet(sheet_id): data = request.get_json() if len(data['cells']) > 99 or len(data['cells'][0]) > 20: abort(406) sheet = { 'cells': data['cells'] } key = Key(bucket) key.key = sheet_id + '.json' key.set_contents_from_string( json.dumps(sheet), policy='public-read', replace=True, headers={ 'Content-Type': 'application/json'
def get_bucket(aws_access_key_id, aws_secret_access_key, bucket_name): connection = S3Connection(aws_access_key_id, aws_secret_access_key) return connection.get_bucket(bucket_name, validate=False)
def send_file_to_genomespace( genomespace_site, username, token, source_filename, target_directory, target_filename, file_type, content_type, log_filename ): target_filename = target_filename.replace( '/', '-' ) # Slashes no longer allowed in filenames url_opener = get_cookie_opener( username, token ) genomespace_site_dict = get_genomespace_site_urls()[ genomespace_site ] dm_url = genomespace_site_dict['dmServer'] #get default directory if target_directory and target_directory[0] == '/': directory_dict, target_directory = get_directory( url_opener, dm_url, [ "%s/%s/%s" % ( GENOMESPACE_API_VERSION_STRING, 'file', target_directory[1] ) ] + target_directory[2:] ) directory_dict = directory_dict['directory'] else: directory_dict = get_personal_directory( url_opener, dm_url )['directory'] #this is the base for the auto-generated galaxy export directories #what directory to stuff this in target_directory_dict = create_directory( url_opener, directory_dict, target_directory, dm_url ) content_length = os.path.getsize( source_filename ) input_file = open( source_filename, 'rb' ) if content_length > TARGET_SIMPLE_PUT_UPLOAD_SIZE: # Determine sizes of each part. split_count = content_length / TARGET_SPLIT_SIZE last_size = content_length - ( split_count * TARGET_SPLIT_SIZE ) sizes = [ TARGET_SPLIT_SIZE ] * split_count if last_size: if last_size < MIN_MULTIPART_UPLOAD_SIZE: if sizes: sizes[-1] = sizes[-1] + last_size else: sizes = [ last_size ] else: sizes.append( last_size ) print "Performing multi-part upload in %i parts." % ( len( sizes ) ) #get upload url upload_url = "uploadinfo" upload_url = "%s/%s/%s%s/%s" % ( dm_url, GENOMESPACE_API_VERSION_STRING, upload_url, target_directory_dict['path'], urllib.quote( target_filename, safe='' ) ) upload_request = urllib2.Request( upload_url, headers = { 'Content-Type': 'application/json', 'Accept': 'application/json' } ) upload_request.get_method = lambda: 'GET' upload_info = json.loads( url_opener.open( upload_request ).read() ) conn = S3Connection( aws_access_key_id=upload_info['amazonCredentials']['accessKey'], aws_secret_access_key=upload_info['amazonCredentials']['secretKey'], security_token=upload_info['amazonCredentials']['sessionToken'] ) # Cannot use conn.get_bucket due to permissions, manually create bucket object bucket = boto.s3.bucket.Bucket( connection=conn, name=upload_info['s3BucketName'] ) mp = bucket.initiate_multipart_upload( upload_info['s3ObjectKey'] ) for i,part_size in enumerate( sizes, start=1 ): fh = tempfile.TemporaryFile( 'wb+' ) while part_size: if CHUNK_SIZE > part_size: read_size = part_size else: read_size = CHUNK_SIZE chunk = input_file.read( read_size ) fh.write( chunk ) part_size = part_size - read_size fh.flush() fh.seek(0) mp.upload_part_from_file( fh, i ) fh.close() upload_result = mp.complete_upload() else: print 'Performing simple put upload.' upload_url = "uploadurl" content_md5 = hashlib.md5() chunk_write( input_file, content_md5, target_method="update" ) input_file.seek( 0 ) #back to start, for uploading upload_params = { 'Content-Length': content_length, 'Content-MD5': base64.standard_b64encode( content_md5.digest() ), 'Content-Type': content_type } upload_url = "%s/%s/%s%s/%s?%s" % ( dm_url, GENOMESPACE_API_VERSION_STRING, upload_url, target_directory_dict['path'], urllib.quote( target_filename, safe='' ), urllib.urlencode( upload_params ) ) new_file_request = urllib2.Request( upload_url )#, headers = { 'Content-Type': 'application/json', 'Accept': 'application/text' } ) #apparently http://www.genomespace.org/team/specs/updated-dm-rest-api:"Every HTTP request to the Data Manager should include the Accept header with a preference for the media types application/json and application/text." is not correct new_file_request.get_method = lambda: 'GET' #get url to upload to target_upload_url = url_opener.open( new_file_request ).read() #upload file to determined url upload_headers = dict( upload_params ) #upload_headers[ 'x-amz-meta-md5-hash' ] = content_md5.hexdigest() upload_headers[ 'Accept' ] = 'application/json' upload_file_request = urllib2.Request( target_upload_url, headers = upload_headers, data = input_file ) upload_file_request.get_method = lambda: 'PUT' upload_result = urllib2.urlopen( upload_file_request ).read() result_url = "%s/%s" % ( target_directory_dict['url'], urllib.quote( target_filename, safe='' ) ) #determine available gs launch apps web_tools = get_genome_space_launch_apps( genomespace_site_dict['atmServer'], url_opener, result_url, file_type ) if log_filename: log_file = open( log_filename, 'wb' ) log_file.write( "<html><head><title>File uploaded to GenomeSpace from Galaxy</title></head><body>\n" ) log_file.write( '<p>Uploaded <a href="%s">%s/%s</a> to GenomeSpace.</p>\n' % ( result_url, target_directory_dict['path'], target_filename ) ) if web_tools: log_file.write( "<p>You may open this file directly in the following applications:</p>\n" ) log_file.write( '<p><ul>\n' ) for web_tool in web_tools: log_file.write( '<li><a href="%s">%s</a></li>\n' % ( web_tool ) ) log_file.write( '</p></ul>\n' ) else: log_file.write( '<p>There are no GenomeSpace applications available for file type: %s</p>\n' % ( file_type ) ) log_file.write( "</body></html>\n" ) return upload_result
import json import time from boto.s3.connection import S3Connection from boto.s3.key import Key # get tournament schedule from AWS c = S3Connection('AKIAIQQ36BOSTXH3YEBA','cXNBbLttQnB9NB3wiEzOWLF13Xw8jKujvoFxmv3L') b = c.get_bucket('public.tenthtee') k = Key(b) k1 = Key(b) k2 = Key(b) rs = b.list() keys = [] for key in rs: keys.append(key.name) for year in [2013,2014,2015]: k.key = 'sportsData/' + str(year) + '/schedule.json' schedule_string = k.get_contents_as_string() schedule = json.loads(schedule_string) # get tournament id for tournament in schedule['tournaments']: # uncomment line below to identify the tournament names # print tournament['name'],tournament['id'] # if tournament['name'] == target_tournament: break tournament_name = tournament['name']
file = sys.argv[1] if not os.path.isfile(file): exit('File not found "' + file + '"') filename = os.path.basename(file) try: secret = check_output([ "security", "find-generic-password", "-w", "-a", "AKIAIDCNEPBLS2IMO55A" ]) secret = secret[:-1] except CalledProcessError: exit("Can't get AWS secret access key! Do you have it in your keychain?") try: connection = S3Connection(AWS_ACCESS_KEY, secret) bucket = connection.get_bucket(BUCKET) # Check if there's no such file already on the server. awsfile = bucket.get_key(filename) if awsfile: exit("%s already exists on S3. Remove first!" % (filename)) awsfile = bucket.new_key(filename) if os.isatty(1): print 'Uploading "%s"... ' % (filename) awsfile.set_contents_from_filename(file, cb=progress_callback, num_cb=100) else: awsfile.set_contents_from_filename(
d1 = datetime.strptime(d1, "%Y-%m-%d %H:%M:%S") d2 = datetime.strptime(d2, "%Y-%m-%d %H:%M:%S") return abs((d2 - d1).days) def dateFormatter(unoformattedDate): year = unoformattedDate[:5] month = unoformattedDate[5:8] day = unoformattedDate[8:10] hour = unoformattedDate[12:19] date = year+month+day+" "+hour return date location = "s3.eu-west-1.amazonaws.com" bucketName = 'kds-uav-dev' minimumDays = 7 aws_connection = S3Connection( host = location ) bucket = aws_connection.get_bucket( bucketName , validate=False) uavs = bucket.list() uavsOfIntrest = [] totalUavs = 0 for uav in uavs: lastModified = uav.__getattribute__('last_modified') date = dateFormatter(lastModified) currentDate = datetime.now().strftime("%Y-%m-%d %H:%M:%S") diff = days_between(date, currentDate) #print(lastModified) if diff <= minimumDays: uavsOfIntrest.append(uav) totalUavs += 1
def getS3connection(self): ''' Returns an S3Connection instance. ''' return S3Connection(AWS_ACCESS_KEY_ID, AWS_ACCESS_SECRET_KEY)
from keys import key, secret import os path = os.path.dirname(os.path.abspath(__file__)) version = "0.6.2" from boto.s3.key import Key from boto.s3.connection import S3Connection conn = S3Connection(key, secret) uninstall_bucket = conn.get_bucket('uninstall.amon.cx') k = Key(uninstall_bucket) distros = ['debian', 'rpm', 'macos'] uninstallers = list(distros) uninstallers.append('uninstaller') for u in uninstallers: k.key = u full_path = "{0}/uninstallers/{1}".format(path, u) k.set_contents_from_filename(full_path, headers={'Content-Type': 'text/plain'}) k.make_public() install_bucket = conn.get_bucket('install.amon.cx') k = Key(install_bucket) installers = list(distros) installers.append('installer')
def get_s3_connection(): """Connect to S3. Caches connection objects""" return S3Connection()
def _get_bucket(): conn = S3Connection(host=current_app.config['S3_REGION_ENDPOINT']) #the bucket should exists (notice we skip the request to check) bucket = conn.get_bucket(current_app.config['S3_BUCKET'], validate=False) return bucket
def __init__(self, settings): self.root_path = settings['root_path'] conn = S3Connection(settings['aws_key_id'], settings['aws_key']) self.bucket = conn.get_bucket(settings['bucket'])
#sudo pip install boto #python c3.py in the same folder of code #US West (Oregon) Region This uses Amazon S3 servers in Oregon. s3-us-west-2.amazonaws.com us-west-2 #US West (Northern California) Region This used Amazon S3 servers in Northern California. s3-us-west-1.amazonaws.com us-west-1 #EU (Ireland) Region This uses Amazon S3 servers in Ireland. s3-eu-west-1.amazonaws.com EU #Asia Pacific (Singapore) Region This uses Amazon S3 servers in Singapore. s3-ap-southeast-1.amazonaws.com ap-southeast-1 #Asia Pacific (Sydney) Region This uses Amazon S3 servers in Sydney. s3-ap-southeast-2.amazonaws.com ap-southeast-2 #Asia Pacific (Tokyo) Region This uses Amazon S3 servers in Tokyo. s3-ap-northeast-1.amazonaws.com ap-northeast-1 #South America (San Paulo) Region This uses Amazon S3 servers in San Paulo. sa-east-1 from boto.s3.connection import S3Connection, Location from boto.s3.key import Key import boto conn = S3Connection('AKIAJP3BDZM4BVZJDIEA', 'lzxyKI51aRoQf1lDB91MD5WGBG1RJrTFiVP+//GH') if __name__ == "__main__": while True: operations = ["List Buckets","Create Bucket", \ "Create Bucket with Location","Delete Bucket", \ "Write Object","List Objects","Exit"] for x in range(len(operations)): print x, " ", operations[x] op = input("Enter the number of operation you want to perform: ") if op == 0: list = conn.get_all_buckets()
def get_s3_connection(cls): if not cls.s3_connection: cls.s3_connection = S3Connection(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY) return cls.s3_connection
def getCarsonOakesMeals(): page_link = "https://nutrition.sa.ucsc.edu/menuSamp.asp?locationNum=30&locationName=Rachel+Carson+Oakes+Dining+Hall&sName=&naFlag=" #cowell - https://nutrition.sa.ucsc.edu/menuSamp.asp?locationNum=05&locationName=Cowell+Stevenson+Dining+Hall&sName=&naFlag= #crown - https://nutrition.sa.ucsc.edu/menuSamp.asp?locationNum=20&locationName=Crown+Merrill+Dining+Hall&sName=&naFlag= #porter - https://nutrition.sa.ucsc.edu/menuSamp.asp?locationNum=25&locationName=Porter+Kresge+Dining+Hall&sName=&naFlag= #rc - https://nutrition.sa.ucsc.edu/menuSamp.asp?locationNum=30&locationName=Rachel+Carson+Oakes+Dining+Hall&sName=&naFlag= #ten - https://nutrition.sa.ucsc.edu/menuSamp.asp?locationNum=40&locationName=Colleges+Nine+%26+Ten+Dining+Hall&sName=&naFlag= page_response = requests.get(page_link, timeout=5) page_content = BeautifulSoup(page_response.content, "html.parser") textcontent = [] breakfastList = [] lunchList = [] dinnerList = [] lateNightList = [] lateNightHappening = False htmlMeals = page_content.find_all("div", attrs={'class': "menusamprecipes"}) for i in range(len(htmlMeals)): textcontent.append(htmlMeals[i].text) #------ htmlMeals = page_content.find_all('td', attrs={'valign': 'top'}) meals = '' for i in range(0, len(htmlMeals)): meals = meals + htmlMeals[i].text if len(meals) == 0: return b = meals.split("Lunch") breakfast = b[0] l = b[1].split("Dinner") lunch = l[0] dinner = l[1] ln = dinner.split("Late Night") if len(ln) > 1: lateNight = ln[1] dinner = ln[0] lateNightHappening = True breakfast = re.sub(r'\W+', ' ', breakfast) lunch = re.sub(r'\W+', ' ', lunch) dinner = re.sub(r'\W+', ' ', dinner) if lateNightHappening: lateNight = re.sub(r'\W+', ' ', lateNight) for s in textcontent: if s in breakfast and s not in breakfastList: breakfastList.append(s) for s in textcontent: if s in lunch and s not in lunchList: lunchList.append(s) for s in textcontent: if s in dinner and s not in dinnerList: dinnerList.append(s) if lateNightHappening: for s in textcontent: if s in lateNight and s not in lateNightList: lateNightList.append(s) bf = open("CarsonOakesbreakfast.txt", "w") lun = open("CarsonOakeslunch.txt", "w") din = open("CarsonOakesdinner.txt", "w") if lateNightHappening: late = open("CarsonOakeslateNight.txt", "w") for i in range(len(breakfastList)): if '&' in breakfastList[i]: breakfastList[i].replace("&", "and") if (i < len(breakfastList) - 1): bf.write(breakfastList[i] + ", ") else: bf.write("and " + breakfastList[i]) for i in range(len(lunchList)): if '&' in lunchList[i]: lunchList[i].replace("&", "and") if (i < len(lunchList) - 1): lun.write(lunchList[i] + ", ") else: lun.write("and " + lunchList[i]) for i in range(len(dinnerList)): if '&' in dinnerList[i]: dinnerList[i].replace("&", "and") if (i < len(dinnerList) - 1): din.write(dinnerList[i] + ", ") else: din.write("and " + dinnerList[i]) if lateNightHappening: for i in range(len(lateNightList)): if '&' in lateNightList[i]: lateNightList[i].replace("&", "and") if (i < len(lateNightList) - 1): late.write(lateNightList[i] + ", ") else: late.write("and " + lateNightList[i]) bf.close() lun.close() din.close() if lateNightHappening: late.close() if not lateNightHappening: filenames = [ 'CarsonOakesbreakfast.txt', 'CarsonOakeslunch.txt', 'CarsonOakesdinner.txt' ] else: filenames = [ 'CarsonOakesbreakfast.txt', 'CarsonOakeslunch.txt', 'CarsonOakesdinner.txt', 'CarsonOakeslateNight.txt' ] conn = S3Connection(aws_access_key_id=ACCESS_KEY_ID, aws_secret_access_key=ACCESS_SECRET_KEY) for fname in filenames: bucket = conn.get_bucket("dininghall") content = open(fname, 'r') read_data = content.read() key = bucket.new_key(fname).set_contents_from_string(read_data) print("uploaded file %s" % fname)
import os import sys import gzip import cStringIO from boto.s3.connection import S3Connection from boto.s3.key import Key sys.path.append( os.path.join(os.path.abspath(os.path.dirname(__file__)), os.path.pardir)) import settings conn = S3Connection() bucket = conn.get_bucket(settings.S3_BUCKET) base = sys.argv[1] for j in os.listdir(base): print "uploading", j k = Key(bucket) k.key = os.path.join(sys.argv[1], j) gzdata = cStringIO.StringIO() gzfile = gzip.GzipFile(fileobj=gzdata, mode="wb") gzfile.write(open(os.path.join(base, j)).read()) gzfile.close() gzdata.seek(0)
# celeryapp.autodiscover_tasks(INSTALLED_APPS) CELERY_ACCEPT_CONTENT = ['pickle', 'json', 'msgpack', 'yaml'] # ========== # = Assets = # ========== JAMMIT = jammit.JammitAssets(NEWSBLUR_DIR) if DEBUG: MIDDLEWARE_CLASSES += ( 'utils.request_introspection_middleware.DumpRequestMiddleware', ) MIDDLEWARE_CLASSES += ( 'utils.exception_middleware.ConsoleExceptionMiddleware', ) # ======= # = AWS = # ======= S3_CONN = None if BACKED_BY_AWS.get('pages_on_s3') or BACKED_BY_AWS.get('icons_on_s3'): S3_CONN = S3Connection(S3_ACCESS_KEY, S3_SECRET) if BACKED_BY_AWS.get('pages_on_s3'): S3_PAGES_BUCKET = S3_CONN.get_bucket(S3_PAGES_BUCKET_NAME) if BACKED_BY_AWS.get('icons_on_s3'): S3_ICONS_BUCKET = S3_CONN.get_bucket(S3_ICONS_BUCKET_NAME) django.http.request.host_validation_re = re.compile( r"^([a-z0-9.-_\-]+|\[[a-f0-9]*:[a-f0-9:]+\])(:\d+)?$")
def __init__(self, aws_access_key_id, aws_secret_access_key, bucket, cache_dir): self.bucket = bucket self.cache_dir = cache_dir self.s3 = S3Connection(aws_access_key_id, aws_secret_access_key) self.s3bucket = self.s3.get_bucket(self.bucket)
def __init__(self) -> None: super().__init__() conn = S3Connection(settings.S3_KEY, settings.S3_SECRET_KEY) bucket_name = settings.S3_AVATAR_BUCKET self.bucket = conn.get_bucket(bucket_name, validate=False)
def setup_connection(self): self.conn = S3Connection(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY) self.bucket = self.conn.get_bucket(settings.STATIC_MEDIA_S3_BUCKET)
def test_other_region(): conn = S3Connection('key', 'secret', host='s3-website-ap-southeast-2.amazonaws.com') conn.create_bucket("foobar") list(conn.get_bucket("foobar").get_all_keys()).should.equal([])
def getBucket(self, aws_access, aws_secret, bucket_name): self.log.info("S3: Open Bucket, %s, %s, %s" % (bucket_name, aws_access, aws_secret)) s3 = S3Connection(aws_access, aws_secret) return s3.lookup(bucket_name)
roi_filename = 'RoiSet.zip' # File of ROI masks dim_order_for_hdf5 = 'tyx' # dim_order for HDF5 file. It states what the axes # in your data array represents. So tyx means the data array in the HDF5 file # is shaped "Time x Y pixels x X pixels". Used in sima.Sequence.create() # ------------------------------------------------------------------- # ------------------------------------------------------------------- # --------------------------------TO HERE---------------------------- # ------------------------------------------------------------------- # ------------------------------------------------------------------- # Additional global variables. Do not change any of these base_path_on_ec2 = '/mnt/analysis/DATA/' # This is the working directory where the analysis script is run. conn = S3Connection(AWS_ACCESS_KEY, AWS_SECRET_ACCESS_KEY) bucket = conn.get_bucket(s3_bucket_name) def create_sequence(full_path_to_file, individualframename): # Retrieve S3 filename without path temp = os.path.splitext(os.path.basename(full_path_to_file)) filename = temp[0] if filename[-4:] == '_CH1': dual_channel = True else: dual_channel = False if is_single_file: file_format = temp[-1]
def get_s3_bucket(): from boto.s3.connection import S3Connection conn = S3Connection(app_settings.S3_ACCESS_KEY, app_settings.S3_SECRET_KEY) return conn.get_bucket(app_settings.S3_BUCKET)
def fill_in_auth(self, http_request, **kwargs): _update_headers(http_request.headers) S3Connection.fill_in_auth(self, http_request, **kwargs) _update_headers(http_request.headers) return http_request
import os import time import json from models import db, STATUS, PRIORITY, File, Conversion, Account from config import app from semisync import semisync import requests from boto.s3.connection import S3Connection from boto.s3.key import Key AWS_S3_CONNECTION = S3Connection() bucket = AWS_S3_CONNECTION.get_bucket(app.config['S3_BUCKET']) BASE_URL = 'http://127.0.0.1:8200' JSON_POST_HEADERS = { 'Content-type': 'application/json', 'Accept': 'application/json' } # dummy_auth = requests.auth.HTTPBasicAuth(username, password) dummy_token_auth = lambda token: requests.auth.HTTPBasicAuth(token, 'unused') def is_ok_response(r): return r.status_code >= 200 and r.status_code < 400 def is_unauthorised_call(r): return r.status_code == 401
def main(): parser = argparse.ArgumentParser( description='Process incoming Telemetry data', formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("incoming_bucket", help="The S3 bucket containing incoming files") parser.add_argument("publish_bucket", help="The S3 bucket to save processed files") parser.add_argument("-n", "--num-helpers", metavar="N", help="Start N helper processes", type=int, default=1) parser.add_argument("-k", "--aws-key", help="AWS Key", required=True) parser.add_argument("-s", "--aws-secret-key", help="AWS Secret Key", required=True) parser.add_argument("-w", "--work-dir", help="Location to cache downloaded files", required=True) parser.add_argument("-o", "--output-dir", help="Base dir to store processed data", required=True) parser.add_argument("-i", "--input-files", help="File containing a list of keys to process", type=file) parser.add_argument("-t", "--telemetry-schema", help="Location of the desired telemetry schema", required=True) args = parser.parse_args() # TODO: keep track of partial success so that subsequent runs are idempotent. start = datetime.now() conn = S3Connection(args.aws_key, args.aws_secret_key) incoming_bucket = conn.get_bucket(args.incoming_bucket) incoming_filenames = [] if args.input_files: print "Fetching file list from file", args.input_files incoming_filenames = [l.strip() for l in args.input_files.readlines()] else: print "Fetching file list from S3..." for f in incoming_bucket.list(): incoming_filenames.append(f.name) print "Done" for f in incoming_filenames: print " ", f result = 0 print "Downloading", len(incoming_filenames), "files..." result = fetch_s3_files(incoming_filenames, args.work_dir, args.incoming_bucket, args.aws_key, args.aws_secret_key) if result != 0: print "Error downloading files. Return code of s3funnel was", result return result print "Done" print "Splitting raw logs..." local_filenames = [ os.path.join(args.work_dir, f) for f in incoming_filenames ] result = split_raw_logs(local_filenames, args.output_dir, args.telemetry_schema) if result != 0: print "Error splitting logs. Return code was", result return result print "Done" print "Converting split logs..." result = convert_split_logs(args.output_dir) if result != 0: print "Error converting logs. Return code was", result return result print "Done" print "Exporting converted logs back to S3..." result = export_converted_logs(args.output_dir, args.publish_bucket, args.aws_key, args.aws_secret_key) if result != 0: print "Error exporting logs. Return code was", result return result print "Done" print "Removing processed logs from S3..." for f in incoming_filenames: print " Deleting", f incoming_bucket.delete_key(f) print "Done" duration = timer.delta_sec(start) print "All done in %.2fs" % (duration) return 0
def main(): argument_spec = ec2_argument_spec() argument_spec.update( dict( bucket=dict(required=True), dest=dict(default=None), encrypt=dict(default=True, type='bool'), expiry=dict(default=600, aliases=['expiration']), headers=dict(type='dict'), marker=dict(default=None), max_keys=dict(default=1000), metadata=dict(type='dict'), mode=dict(choices=[ 'get', 'put', 'delete', 'create', 'geturl', 'getstr', 'delobj', 'list' ], required=True), object=dict(), permission=dict(type='list', default=['private']), version=dict(default=None), overwrite=dict(aliases=['force'], default='always'), prefix=dict(default=None), retries=dict(aliases=['retry'], type='int', default=0), s3_url=dict(aliases=['S3_URL']), rgw=dict(default='no', type='bool'), src=dict(), ), ) module = AnsibleModule(argument_spec=argument_spec) if not HAS_BOTO: module.fail_json(msg='boto required for this module') bucket = module.params.get('bucket') encrypt = module.params.get('encrypt') expiry = int(module.params['expiry']) if module.params.get('dest'): dest = os.path.expanduser(module.params.get('dest')) headers = module.params.get('headers') marker = module.params.get('marker') max_keys = module.params.get('max_keys') metadata = module.params.get('metadata') mode = module.params.get('mode') obj = module.params.get('object') version = module.params.get('version') overwrite = module.params.get('overwrite') prefix = module.params.get('prefix') retries = module.params.get('retries') s3_url = module.params.get('s3_url') rgw = module.params.get('rgw') src = module.params.get('src') for acl in module.params.get('permission'): if acl not in CannedACLStrings: module.fail_json(msg='Unknown permission specified: %s' % str(acl)) if overwrite not in ['always', 'never', 'different']: if module.boolean(overwrite): overwrite = 'always' else: overwrite = 'never' region, ec2_url, aws_connect_kwargs = get_aws_connection_info(module) if region in ('us-east-1', '', None): # S3ism for the US Standard region location = Location.DEFAULT else: # Boto uses symbolic names for locations but region strings will # actually work fine for everything except us-east-1 (US Standard) location = region if module.params.get('object'): obj = os.path.expanduser(module.params['object']) # allow eucarc environment variables to be used if ansible vars aren't set if not s3_url and 'S3_URL' in os.environ: s3_url = os.environ['S3_URL'] # rgw requires an explicit url if rgw and not s3_url: module.fail_json(msg='rgw flavour requires s3_url') # bucket names with .'s in them need to use the calling_format option, # otherwise the connection will fail. See https://github.com/boto/boto/issues/2836 # for more details. if '.' in bucket: aws_connect_kwargs['calling_format'] = OrdinaryCallingFormat() # Look at s3_url and tweak connection settings # if connecting to RGW, Walrus or fakes3 try: if s3_url and rgw: rgw = urlparse.urlparse(s3_url) s3 = boto.connect_s3(is_secure=rgw.scheme == 'https', host=rgw.hostname, port=rgw.port, calling_format=OrdinaryCallingFormat(), **aws_connect_kwargs) elif is_fakes3(s3_url): fakes3 = urlparse.urlparse(s3_url) s3 = S3Connection(is_secure=fakes3.scheme == 'fakes3s', host=fakes3.hostname, port=fakes3.port, calling_format=OrdinaryCallingFormat(), **aws_connect_kwargs) elif is_walrus(s3_url): walrus = urlparse.urlparse(s3_url).hostname s3 = boto.connect_walrus(walrus, **aws_connect_kwargs) else: aws_connect_kwargs['is_secure'] = True try: s3 = connect_to_aws(boto.s3, location, **aws_connect_kwargs) except AnsibleAWSError: # use this as fallback because connect_to_region seems to fail in boto + non 'classic' aws accounts in some cases s3 = boto.connect_s3(**aws_connect_kwargs) except boto.exception.NoAuthHandlerFound as e: module.fail_json(msg='No Authentication Handler found: %s ' % str(e)) except Exception as e: module.fail_json(msg='Failed to connect to S3: %s' % str(e)) if s3 is None: # this should never happen module.fail_json( msg= 'Unknown error, failed to create s3 connection, no information from boto.' ) # If our mode is a GET operation (download), go through the procedure as appropriate ... if mode == 'get': # First, we check to see if the bucket exists, we get "bucket" returned. bucketrtn = bucket_check(module, s3, bucket) if bucketrtn is False: module.fail_json(msg="Source bucket cannot be found", failed=True) # Next, we check to see if the key in the bucket exists. If it exists, it also returns key_matches md5sum check. keyrtn = key_check(module, s3, bucket, obj, version=version) if keyrtn is False: if version is not None: module.fail_json( msg="Key %s with version id %s does not exist." % (obj, version), failed=True) else: module.fail_json(msg="Key %s does not exist." % obj, failed=True) # If the destination path doesn't exist or overwrite is True, no need to do the md5um etag check, so just download. pathrtn = path_check(dest) if pathrtn is False or overwrite == 'always': download_s3file(module, s3, bucket, obj, dest, retries, version=version) # Compare the remote MD5 sum of the object with the local dest md5sum, if it already exists. if pathrtn is True: md5_remote = keysum(module, s3, bucket, obj, version=version) md5_local = module.md5(dest) if md5_local == md5_remote: sum_matches = True if overwrite == 'always': download_s3file(module, s3, bucket, obj, dest, retries, version=version) else: module.exit_json( msg= "Local and remote object are identical, ignoring. Use overwrite=always parameter to force.", changed=False) else: sum_matches = False if overwrite in ('always', 'different'): download_s3file(module, s3, bucket, obj, dest, retries, version=version) else: module.exit_json( msg= "WARNING: Checksums do not match. Use overwrite parameter to force download." ) # Firstly, if key_matches is TRUE and overwrite is not enabled, we EXIT with a helpful message. if sum_matches is True and overwrite == 'never': module.exit_json( msg= "Local and remote object are identical, ignoring. Use overwrite parameter to force.", changed=False) # if our mode is a PUT operation (upload), go through the procedure as appropriate ... if mode == 'put': # Use this snippet to debug through conditionals: # module.exit_json(msg="Bucket return %s"%bucketrtn) # Lets check the src path. pathrtn = path_check(src) if pathrtn is False: module.fail_json(msg="Local object for PUT does not exist", failed=True) # Lets check to see if bucket exists to get ground truth. bucketrtn = bucket_check(module, s3, bucket) if bucketrtn is True: keyrtn = key_check(module, s3, bucket, obj) # Lets check key state. Does it exist and if it does, compute the etag md5sum. if bucketrtn is True and keyrtn is True: md5_remote = keysum(module, s3, bucket, obj) md5_local = module.md5(src) if md5_local == md5_remote: sum_matches = True if overwrite == 'always': upload_s3file(module, s3, bucket, obj, src, expiry, metadata, encrypt, headers) else: get_download_url(module, s3, bucket, obj, expiry, changed=False) else: sum_matches = False if overwrite in ('always', 'different'): upload_s3file(module, s3, bucket, obj, src, expiry, metadata, encrypt, headers) else: module.exit_json( msg= "WARNING: Checksums do not match. Use overwrite parameter to force upload." ) # If neither exist (based on bucket existence), we can create both. if bucketrtn is False and pathrtn is True: create_bucket(module, s3, bucket, location) upload_s3file(module, s3, bucket, obj, src, expiry, metadata, encrypt, headers) # If bucket exists but key doesn't, just upload. if bucketrtn is True and pathrtn is True and keyrtn is False: upload_s3file(module, s3, bucket, obj, src, expiry, metadata, encrypt, headers) # Delete an object from a bucket, not the entire bucket if mode == 'delobj': if obj is None: module.fail_json(msg="object parameter is required", failed=True) if bucket: bucketrtn = bucket_check(module, s3, bucket) if bucketrtn is True: deletertn = delete_key(module, s3, bucket, obj) if deletertn is True: module.exit_json(msg="Object %s deleted from bucket %s." % (obj, bucket), changed=True) else: module.fail_json(msg="Bucket does not exist.", changed=False) else: module.fail_json(msg="Bucket parameter is required.", failed=True) # Delete an entire bucket, including all objects in the bucket if mode == 'delete': if bucket: bucketrtn = bucket_check(module, s3, bucket) if bucketrtn is True: deletertn = delete_bucket(module, s3, bucket) if deletertn is True: module.exit_json( msg="Bucket %s and all keys have been deleted." % bucket, changed=True) else: module.fail_json(msg="Bucket does not exist.", changed=False) else: module.fail_json(msg="Bucket parameter is required.", failed=True) # Support for listing a set of keys if mode == 'list': bucket_object = get_bucket(module, s3, bucket) # If the bucket does not exist then bail out if bucket_object is None: module.fail_json(msg="Target bucket (%s) cannot be found" % bucket, failed=True) list_keys(module, bucket_object, prefix, marker, max_keys) # Need to research how to create directories without "populating" a key, so this should just do bucket creation for now. # WE SHOULD ENABLE SOME WAY OF CREATING AN EMPTY KEY TO CREATE "DIRECTORY" STRUCTURE, AWS CONSOLE DOES THIS. if mode == 'create': if bucket and not obj: bucketrtn = bucket_check(module, s3, bucket) if bucketrtn is True: module.exit_json(msg="Bucket already exists.", changed=False) else: module.exit_json(msg="Bucket created successfully", changed=create_bucket(module, s3, bucket, location)) if bucket and obj: bucketrtn = bucket_check(module, s3, bucket) if obj.endswith('/'): dirobj = obj else: dirobj = obj + "/" if bucketrtn is True: keyrtn = key_check(module, s3, bucket, dirobj) if keyrtn is True: module.exit_json( msg="Bucket %s and key %s already exists." % (bucket, obj), changed=False) else: create_dirkey(module, s3, bucket, dirobj) if bucketrtn is False: created = create_bucket(module, s3, bucket, location) create_dirkey(module, s3, bucket, dirobj) # Support for grabbing the time-expired URL for an object in S3/Walrus. if mode == 'geturl': if bucket and obj: bucketrtn = bucket_check(module, s3, bucket) if bucketrtn is False: module.fail_json(msg="Bucket %s does not exist." % bucket, failed=True) else: keyrtn = key_check(module, s3, bucket, obj) if keyrtn is True: get_download_url(module, s3, bucket, obj, expiry) else: module.fail_json(msg="Key %s does not exist." % obj, failed=True) else: module.fail_json(msg="Bucket and Object parameters must be set", failed=True) if mode == 'getstr': if bucket and obj: bucketrtn = bucket_check(module, s3, bucket) if bucketrtn is False: module.fail_json(msg="Bucket %s does not exist." % bucket, failed=True) else: keyrtn = key_check(module, s3, bucket, obj, version=version) if keyrtn is True: download_s3str(module, s3, bucket, obj, version=version) else: if version is not None: module.fail_json( msg="Key %s with version id %s does not exist." % (obj, version), failed=True) else: module.fail_json(msg="Key %s does not exist." % obj, failed=True) module.exit_json(failed=False)