def __init__(self, gs_access_key_id=None, gs_secret_access_key=None,
is_secure=True, port=None, proxy=None, proxy_port=None,
proxy_user=None, proxy_pass=None,
host=DefaultHost, debug=0, https_connection_factory=None,
calling_format=SubdomainCallingFormat(), path='/'):
S3Connection.__init__(self, gs_access_key_id, gs_secret_access_key,
is_secure, port, proxy, proxy_port, proxy_user, proxy_pass,
host, debug, https_connection_factory, calling_format, path,
"google")
def __init__(self, access_key=None, secret_key=None):
from pinhole.common.app import app
if not access_key:
access_key = app.config["AWS_ACCESS_KEY"]
if not secret_key:
secret_key = app.config["AWS_SECRET_KEY"]
S3Connection.__init__(self, access_key, secret_key)
def get_bucket(conn: S3Connection, bucket_name: str) -> Bucket:
# Calling get_bucket() with validate=True can apparently lead
# to expensive S3 bills:
# http://www.appneta.com/blog/s3-list-get-bucket-default/
# The benefits of validation aren't completely clear to us, and
# we want to save on our bills, so we set the validate flag to False.
# (We think setting validate to True would cause us to fail faster
# in situations where buckets don't exist, but that shouldn't be
# an issue for us.)
bucket = conn.get_bucket(bucket_name, validate=False)
return bucket
def __init__(self,
access_key_id,
secret_access_key,
bucket_name,
filenames,
upload_options,
wizard_page,
parent=None):
S3Connection.__init__(self, access_key_id, secret_access_key)
self.upload_options = upload_options
self.bucket_name = bucket_name
self.bucket = None
self.filenames = filenames
self.s3Uploaders = []
self.threads = []
self.count_uploaded_images = 0
self.num_uploading_images = 0
#For GUI (messages and progress bars)
self.wizard_page = wizard_page
self.uploader_widget = QtGui.QWidget()
self.uploader_widget.setWindowTitle("Upload Progress Bars")
self.uploader_widget.setWindowFlags(Qt.WindowStaysOnTopHint)
self.uploader_v_box = QtGui.QVBoxLayout()
self.uploader_widget.setLayout(self.uploader_v_box)
self.msg_bar_main = None
self.msg_bar_main_content = None
self.cancel_button_main = None
self.msg_bars = []
self.msg_bars_content = []
self.progress_bars = []
self.cancel_buttons = []
def __init__(self, bucket_name):
self.connection = S3Connection(os.environ.get('AWS_ACCESS_KEY_ID'),
os.environ.get('AWS_SECRET_ACCESS_KEY'))
self.bucket = self.connection.create_bucket(bucket_name)
def process(log):
"""Process the files"""
log.append('Start Time: %s' % datetime.now())
conn = S3Connection(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY)
bucket = conn.get_bucket(settings.AWS_AUTOIMPORT_BUCKET_NAME)
storage_bucket = conn.get_bucket(settings.AWS_STORAGE_BUCKET_NAME)
for key in bucket.list(prefix='scans/', delimiter='/'):
if key.name == 'scans/':
continue
# strip off 'scans/'
file_name = key.name[6:]
try:
(foia_pks, file_date, code, title, status,
body, arg, id_, est_date) = parse_name(file_name)
except ValueError as exc:
s3_copy(bucket, key, 'review/%s' % file_name)
s3_delete(bucket, key)
log.append(unicode(exc))
continue
for foia_pk in foia_pks:
try:
foia = FOIARequest.objects.get(pk=foia_pk)
from_user = foia.agency.get_user() if foia.agency else None
comm = FOIACommunication.objects.create(
foia=foia,
from_user=from_user,
to_user=foia.user,
response=True,
date=file_date,
communication=body,
status=status,
)
MailCommunication.objects.create(
communication=comm,
sent_datetime=file_date,
)
foia.status = status or foia.status
if foia.status in ['partial', 'done', 'rejected', 'no_docs']:
foia.date_done = file_date.date()
if code == 'FEE' and arg:
foia.price = Decimal(arg)
if id_:
foia.tracking_id = id_
if est_date:
foia.date_estimate = est_date
if code == 'REJ-P':
foia.proxy_reject()
if key.name.endswith('/'):
import_prefix(key, bucket, storage_bucket, comm, log)
else:
import_key(key, storage_bucket, comm, log, title=title)
foia.save(comment='updated from autoimport files')
action = generate_status_action(foia)
foia.notify(action)
foia.update(comm.anchor())
except FOIARequest.DoesNotExist:
s3_copy(bucket, key, 'review/%s' % file_name)
log.append('ERROR: %s references FOIA Request %s, but it does not exist' %
(file_name, foia_pk))
except SoftTimeLimitExceeded:
# if we reach the soft time limit,
# re-raise so we can catch and clean up
raise
except Exception as exc:
s3_copy(bucket, key, 'review/%s' % file_name)
log.append('ERROR: %s has caused an unknown error. %s' % (file_name, exc))
logger.error('Autoimport error: %s', exc, exc_info=sys.exc_info())
# delete key after processing all requests for it
s3_delete(bucket, key)
log.append('End Time: %s' % datetime.now())
import pandas as pd
import ssl
import sys
from boto.s3.connection import S3Connection
def main():
'''
This pulls xml casefiles
1. Import credentals
2. Retrieve Casefiles
3. Loop Casefiles to Push to s3
'''
pass
# AWS Credentials
S3Conn = S3Connection('MyAWSaccessKey', 'MyAWSsecretKey')
greatbucket = S3Conn.get_bucket('bucketname')
### Accessing Desktop Folder
path = '/Users/mycomputerName/Desktop/myFolder/'
# UPLOAD BY LOOPING THRU BY INDIVIDUAL XML FILE
for i in (os.listdir(path)):
file = greatbucket.new_key(i)
file.set_contents_from_filename(path + '/' + i)
if __name__ == '__main__':
main()
def get_bucket(s3_bucket, aws_access_key_id, aws_secret_access_key,
s3_connection_host):
connection = S3Connection(aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
host=s3_connection_host)
return connection.get_bucket(s3_bucket, validate=False)
import json
import random
import settings
from flask import Flask, render_template, abort, redirect, request
app = Flask(__name__)
app.config.from_object(settings)
from boto.s3.connection import S3Connection
from boto.s3.key import Key
conn = S3Connection(settings.S3_KEY_ID, settings.S3_SECRET)
bucket = conn.get_bucket(settings.S3_BUCKET_NAME)
@app.route('/sheets/<sheet_id>', methods=['PUT'])
def save_sheet(sheet_id):
data = request.get_json()
if len(data['cells']) > 99 or len(data['cells'][0]) > 20:
abort(406)
sheet = {
'cells': data['cells']
}
key = Key(bucket)
key.key = sheet_id + '.json'
key.set_contents_from_string(
json.dumps(sheet),
policy='public-read',
replace=True,
headers={
'Content-Type': 'application/json'
def get_bucket(aws_access_key_id, aws_secret_access_key, bucket_name):
connection = S3Connection(aws_access_key_id, aws_secret_access_key)
return connection.get_bucket(bucket_name, validate=False)
def send_file_to_genomespace( genomespace_site, username, token, source_filename, target_directory, target_filename, file_type, content_type, log_filename ):
target_filename = target_filename.replace( '/', '-' ) # Slashes no longer allowed in filenames
url_opener = get_cookie_opener( username, token )
genomespace_site_dict = get_genomespace_site_urls()[ genomespace_site ]
dm_url = genomespace_site_dict['dmServer']
#get default directory
if target_directory and target_directory[0] == '/':
directory_dict, target_directory = get_directory( url_opener, dm_url, [ "%s/%s/%s" % ( GENOMESPACE_API_VERSION_STRING, 'file', target_directory[1] ) ] + target_directory[2:] )
directory_dict = directory_dict['directory']
else:
directory_dict = get_personal_directory( url_opener, dm_url )['directory'] #this is the base for the auto-generated galaxy export directories
#what directory to stuff this in
target_directory_dict = create_directory( url_opener, directory_dict, target_directory, dm_url )
content_length = os.path.getsize( source_filename )
input_file = open( source_filename, 'rb' )
if content_length > TARGET_SIMPLE_PUT_UPLOAD_SIZE:
# Determine sizes of each part.
split_count = content_length / TARGET_SPLIT_SIZE
last_size = content_length - ( split_count * TARGET_SPLIT_SIZE )
sizes = [ TARGET_SPLIT_SIZE ] * split_count
if last_size:
if last_size < MIN_MULTIPART_UPLOAD_SIZE:
if sizes:
sizes[-1] = sizes[-1] + last_size
else:
sizes = [ last_size ]
else:
sizes.append( last_size )
print "Performing multi-part upload in %i parts." % ( len( sizes ) )
#get upload url
upload_url = "uploadinfo"
upload_url = "%s/%s/%s%s/%s" % ( dm_url, GENOMESPACE_API_VERSION_STRING, upload_url, target_directory_dict['path'], urllib.quote( target_filename, safe='' ) )
upload_request = urllib2.Request( upload_url, headers = { 'Content-Type': 'application/json', 'Accept': 'application/json' } )
upload_request.get_method = lambda: 'GET'
upload_info = json.loads( url_opener.open( upload_request ).read() )
conn = S3Connection( aws_access_key_id=upload_info['amazonCredentials']['accessKey'],
aws_secret_access_key=upload_info['amazonCredentials']['secretKey'],
security_token=upload_info['amazonCredentials']['sessionToken'] )
# Cannot use conn.get_bucket due to permissions, manually create bucket object
bucket = boto.s3.bucket.Bucket( connection=conn, name=upload_info['s3BucketName'] )
mp = bucket.initiate_multipart_upload( upload_info['s3ObjectKey'] )
for i,part_size in enumerate( sizes, start=1 ):
fh = tempfile.TemporaryFile( 'wb+' )
while part_size:
if CHUNK_SIZE > part_size:
read_size = part_size
else:
read_size = CHUNK_SIZE
chunk = input_file.read( read_size )
fh.write( chunk )
part_size = part_size - read_size
fh.flush()
fh.seek(0)
mp.upload_part_from_file( fh, i )
fh.close()
upload_result = mp.complete_upload()
else:
print 'Performing simple put upload.'
upload_url = "uploadurl"
content_md5 = hashlib.md5()
chunk_write( input_file, content_md5, target_method="update" )
input_file.seek( 0 ) #back to start, for uploading
upload_params = { 'Content-Length': content_length, 'Content-MD5': base64.standard_b64encode( content_md5.digest() ), 'Content-Type': content_type }
upload_url = "%s/%s/%s%s/%s?%s" % ( dm_url, GENOMESPACE_API_VERSION_STRING, upload_url, target_directory_dict['path'], urllib.quote( target_filename, safe='' ), urllib.urlencode( upload_params ) )
new_file_request = urllib2.Request( upload_url )#, headers = { 'Content-Type': 'application/json', 'Accept': 'application/text' } ) #apparently http://www.genomespace.org/team/specs/updated-dm-rest-api:"Every HTTP request to the Data Manager should include the Accept header with a preference for the media types application/json and application/text." is not correct
new_file_request.get_method = lambda: 'GET'
#get url to upload to
target_upload_url = url_opener.open( new_file_request ).read()
#upload file to determined url
upload_headers = dict( upload_params )
#upload_headers[ 'x-amz-meta-md5-hash' ] = content_md5.hexdigest()
upload_headers[ 'Accept' ] = 'application/json'
upload_file_request = urllib2.Request( target_upload_url, headers = upload_headers, data = input_file )
upload_file_request.get_method = lambda: 'PUT'
upload_result = urllib2.urlopen( upload_file_request ).read()
result_url = "%s/%s" % ( target_directory_dict['url'], urllib.quote( target_filename, safe='' ) )
#determine available gs launch apps
web_tools = get_genome_space_launch_apps( genomespace_site_dict['atmServer'], url_opener, result_url, file_type )
if log_filename:
log_file = open( log_filename, 'wb' )
log_file.write( "<html><head><title>File uploaded to GenomeSpace from Galaxy</title></head><body>\n" )
log_file.write( '<p>Uploaded <a href="%s">%s/%s</a> to GenomeSpace.</p>\n' % ( result_url, target_directory_dict['path'], target_filename ) )
if web_tools:
log_file.write( "<p>You may open this file directly in the following applications:</p>\n" )
log_file.write( '<p><ul>\n' )
for web_tool in web_tools:
log_file.write( '<li><a href="%s">%s</a></li>\n' % ( web_tool ) )
log_file.write( '</p></ul>\n' )
else:
log_file.write( '<p>There are no GenomeSpace applications available for file type: %s</p>\n' % ( file_type ) )
log_file.write( "</body></html>\n" )
return upload_result
import json
import time
from boto.s3.connection import S3Connection
from boto.s3.key import Key
# get tournament schedule from AWS
c = S3Connection('AKIAIQQ36BOSTXH3YEBA','cXNBbLttQnB9NB3wiEzOWLF13Xw8jKujvoFxmv3L')
b = c.get_bucket('public.tenthtee')
k = Key(b)
k1 = Key(b)
k2 = Key(b)
rs = b.list()
keys = []
for key in rs:
keys.append(key.name)
for year in [2013,2014,2015]:
k.key = 'sportsData/' + str(year) + '/schedule.json'
schedule_string = k.get_contents_as_string()
schedule = json.loads(schedule_string)
# get tournament id
for tournament in schedule['tournaments']:
# uncomment line below to identify the tournament names
# print tournament['name'],tournament['id']
# if tournament['name'] == target_tournament: break
tournament_name = tournament['name']
file = sys.argv[1]
if not os.path.isfile(file):
exit('File not found "' + file + '"')
filename = os.path.basename(file)
try:
secret = check_output([
"security", "find-generic-password", "-w", "-a", "AKIAIDCNEPBLS2IMO55A"
])
secret = secret[:-1]
except CalledProcessError:
exit("Can't get AWS secret access key! Do you have it in your keychain?")
try:
connection = S3Connection(AWS_ACCESS_KEY, secret)
bucket = connection.get_bucket(BUCKET)
# Check if there's no such file already on the server.
awsfile = bucket.get_key(filename)
if awsfile:
exit("%s already exists on S3. Remove first!" % (filename))
awsfile = bucket.new_key(filename)
if os.isatty(1):
print 'Uploading "%s"... ' % (filename)
awsfile.set_contents_from_filename(file,
cb=progress_callback,
num_cb=100)
else:
awsfile.set_contents_from_filename(
d1 = datetime.strptime(d1, "%Y-%m-%d %H:%M:%S")
d2 = datetime.strptime(d2, "%Y-%m-%d %H:%M:%S")
return abs((d2 - d1).days)
def dateFormatter(unoformattedDate):
year = unoformattedDate[:5]
month = unoformattedDate[5:8]
day = unoformattedDate[8:10]
hour = unoformattedDate[12:19]
date = year+month+day+" "+hour
return date
location = "s3.eu-west-1.amazonaws.com"
bucketName = 'kds-uav-dev'
minimumDays = 7
aws_connection = S3Connection( host = location )
bucket = aws_connection.get_bucket( bucketName , validate=False)
uavs = bucket.list()
uavsOfIntrest = []
totalUavs = 0
for uav in uavs:
lastModified = uav.__getattribute__('last_modified')
date = dateFormatter(lastModified)
currentDate = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
diff = days_between(date, currentDate)
#print(lastModified)
if diff <= minimumDays:
uavsOfIntrest.append(uav)
totalUavs += 1
def getS3connection(self):
'''
Returns an S3Connection instance.
'''
return S3Connection(AWS_ACCESS_KEY_ID, AWS_ACCESS_SECRET_KEY)
from keys import key, secret
import os
path = os.path.dirname(os.path.abspath(__file__))
version = "0.6.2"
from boto.s3.key import Key
from boto.s3.connection import S3Connection
conn = S3Connection(key, secret)
uninstall_bucket = conn.get_bucket('uninstall.amon.cx')
k = Key(uninstall_bucket)
distros = ['debian', 'rpm', 'macos']
uninstallers = list(distros)
uninstallers.append('uninstaller')
for u in uninstallers:
k.key = u
full_path = "{0}/uninstallers/{1}".format(path, u)
k.set_contents_from_filename(full_path,
headers={'Content-Type': 'text/plain'})
k.make_public()
install_bucket = conn.get_bucket('install.amon.cx')
k = Key(install_bucket)
installers = list(distros)
installers.append('installer')
def get_s3_connection():
"""Connect to S3. Caches connection objects"""
return S3Connection()
def _get_bucket():
conn = S3Connection(host=current_app.config['S3_REGION_ENDPOINT'])
#the bucket should exists (notice we skip the request to check)
bucket = conn.get_bucket(current_app.config['S3_BUCKET'], validate=False)
return bucket
def __init__(self, settings):
self.root_path = settings['root_path']
conn = S3Connection(settings['aws_key_id'], settings['aws_key'])
self.bucket = conn.get_bucket(settings['bucket'])
#sudo pip install boto
#python c3.py in the same folder of code
#US West (Oregon) Region This uses Amazon S3 servers in Oregon. s3-us-west-2.amazonaws.com us-west-2
#US West (Northern California) Region This used Amazon S3 servers in Northern California. s3-us-west-1.amazonaws.com us-west-1
#EU (Ireland) Region This uses Amazon S3 servers in Ireland. s3-eu-west-1.amazonaws.com EU
#Asia Pacific (Singapore) Region This uses Amazon S3 servers in Singapore. s3-ap-southeast-1.amazonaws.com ap-southeast-1
#Asia Pacific (Sydney) Region This uses Amazon S3 servers in Sydney. s3-ap-southeast-2.amazonaws.com ap-southeast-2
#Asia Pacific (Tokyo) Region This uses Amazon S3 servers in Tokyo. s3-ap-northeast-1.amazonaws.com ap-northeast-1
#South America (San Paulo) Region This uses Amazon S3 servers in San Paulo. sa-east-1
from boto.s3.connection import S3Connection, Location
from boto.s3.key import Key
import boto
conn = S3Connection('AKIAJP3BDZM4BVZJDIEA',
'lzxyKI51aRoQf1lDB91MD5WGBG1RJrTFiVP+//GH')
if __name__ == "__main__":
while True:
operations = ["List Buckets","Create Bucket", \
"Create Bucket with Location","Delete Bucket", \
"Write Object","List Objects","Exit"]
for x in range(len(operations)):
print x, " ", operations[x]
op = input("Enter the number of operation you want to perform: ")
if op == 0:
list = conn.get_all_buckets()
def get_s3_connection(cls):
if not cls.s3_connection:
cls.s3_connection = S3Connection(settings.AWS_ACCESS_KEY_ID,
settings.AWS_SECRET_ACCESS_KEY)
return cls.s3_connection
def getCarsonOakesMeals():
page_link = "https://nutrition.sa.ucsc.edu/menuSamp.asp?locationNum=30&locationName=Rachel+Carson+Oakes+Dining+Hall&sName=&naFlag="
#cowell - https://nutrition.sa.ucsc.edu/menuSamp.asp?locationNum=05&locationName=Cowell+Stevenson+Dining+Hall&sName=&naFlag=
#crown - https://nutrition.sa.ucsc.edu/menuSamp.asp?locationNum=20&locationName=Crown+Merrill+Dining+Hall&sName=&naFlag=
#porter - https://nutrition.sa.ucsc.edu/menuSamp.asp?locationNum=25&locationName=Porter+Kresge+Dining+Hall&sName=&naFlag=
#rc - https://nutrition.sa.ucsc.edu/menuSamp.asp?locationNum=30&locationName=Rachel+Carson+Oakes+Dining+Hall&sName=&naFlag=
#ten - https://nutrition.sa.ucsc.edu/menuSamp.asp?locationNum=40&locationName=Colleges+Nine+%26+Ten+Dining+Hall&sName=&naFlag=
page_response = requests.get(page_link, timeout=5)
page_content = BeautifulSoup(page_response.content, "html.parser")
textcontent = []
breakfastList = []
lunchList = []
dinnerList = []
lateNightList = []
lateNightHappening = False
htmlMeals = page_content.find_all("div",
attrs={'class': "menusamprecipes"})
for i in range(len(htmlMeals)):
textcontent.append(htmlMeals[i].text)
#------
htmlMeals = page_content.find_all('td', attrs={'valign': 'top'})
meals = ''
for i in range(0, len(htmlMeals)):
meals = meals + htmlMeals[i].text
if len(meals) == 0:
return
b = meals.split("Lunch")
breakfast = b[0]
l = b[1].split("Dinner")
lunch = l[0]
dinner = l[1]
ln = dinner.split("Late Night")
if len(ln) > 1:
lateNight = ln[1]
dinner = ln[0]
lateNightHappening = True
breakfast = re.sub(r'\W+', ' ', breakfast)
lunch = re.sub(r'\W+', ' ', lunch)
dinner = re.sub(r'\W+', ' ', dinner)
if lateNightHappening:
lateNight = re.sub(r'\W+', ' ', lateNight)
for s in textcontent:
if s in breakfast and s not in breakfastList:
breakfastList.append(s)
for s in textcontent:
if s in lunch and s not in lunchList:
lunchList.append(s)
for s in textcontent:
if s in dinner and s not in dinnerList:
dinnerList.append(s)
if lateNightHappening:
for s in textcontent:
if s in lateNight and s not in lateNightList:
lateNightList.append(s)
bf = open("CarsonOakesbreakfast.txt", "w")
lun = open("CarsonOakeslunch.txt", "w")
din = open("CarsonOakesdinner.txt", "w")
if lateNightHappening:
late = open("CarsonOakeslateNight.txt", "w")
for i in range(len(breakfastList)):
if '&' in breakfastList[i]:
breakfastList[i].replace("&", "and")
if (i < len(breakfastList) - 1):
bf.write(breakfastList[i] + ", ")
else:
bf.write("and " + breakfastList[i])
for i in range(len(lunchList)):
if '&' in lunchList[i]:
lunchList[i].replace("&", "and")
if (i < len(lunchList) - 1):
lun.write(lunchList[i] + ", ")
else:
lun.write("and " + lunchList[i])
for i in range(len(dinnerList)):
if '&' in dinnerList[i]:
dinnerList[i].replace("&", "and")
if (i < len(dinnerList) - 1):
din.write(dinnerList[i] + ", ")
else:
din.write("and " + dinnerList[i])
if lateNightHappening:
for i in range(len(lateNightList)):
if '&' in lateNightList[i]:
lateNightList[i].replace("&", "and")
if (i < len(lateNightList) - 1):
late.write(lateNightList[i] + ", ")
else:
late.write("and " + lateNightList[i])
bf.close()
lun.close()
din.close()
if lateNightHappening:
late.close()
if not lateNightHappening:
filenames = [
'CarsonOakesbreakfast.txt', 'CarsonOakeslunch.txt',
'CarsonOakesdinner.txt'
]
else:
filenames = [
'CarsonOakesbreakfast.txt', 'CarsonOakeslunch.txt',
'CarsonOakesdinner.txt', 'CarsonOakeslateNight.txt'
]
conn = S3Connection(aws_access_key_id=ACCESS_KEY_ID,
aws_secret_access_key=ACCESS_SECRET_KEY)
for fname in filenames:
bucket = conn.get_bucket("dininghall")
content = open(fname, 'r')
read_data = content.read()
key = bucket.new_key(fname).set_contents_from_string(read_data)
print("uploaded file %s" % fname)
import os
import sys
import gzip
import cStringIO
from boto.s3.connection import S3Connection
from boto.s3.key import Key
sys.path.append(
os.path.join(os.path.abspath(os.path.dirname(__file__)), os.path.pardir))
import settings
conn = S3Connection()
bucket = conn.get_bucket(settings.S3_BUCKET)
base = sys.argv[1]
for j in os.listdir(base):
print "uploading", j
k = Key(bucket)
k.key = os.path.join(sys.argv[1], j)
gzdata = cStringIO.StringIO()
gzfile = gzip.GzipFile(fileobj=gzdata, mode="wb")
gzfile.write(open(os.path.join(base, j)).read())
gzfile.close()
gzdata.seek(0)
# celeryapp.autodiscover_tasks(INSTALLED_APPS)
CELERY_ACCEPT_CONTENT = ['pickle', 'json', 'msgpack', 'yaml']
# ==========
# = Assets =
# ==========
JAMMIT = jammit.JammitAssets(NEWSBLUR_DIR)
if DEBUG:
MIDDLEWARE_CLASSES += (
'utils.request_introspection_middleware.DumpRequestMiddleware', )
MIDDLEWARE_CLASSES += (
'utils.exception_middleware.ConsoleExceptionMiddleware', )
# =======
# = AWS =
# =======
S3_CONN = None
if BACKED_BY_AWS.get('pages_on_s3') or BACKED_BY_AWS.get('icons_on_s3'):
S3_CONN = S3Connection(S3_ACCESS_KEY, S3_SECRET)
if BACKED_BY_AWS.get('pages_on_s3'):
S3_PAGES_BUCKET = S3_CONN.get_bucket(S3_PAGES_BUCKET_NAME)
if BACKED_BY_AWS.get('icons_on_s3'):
S3_ICONS_BUCKET = S3_CONN.get_bucket(S3_ICONS_BUCKET_NAME)
django.http.request.host_validation_re = re.compile(
r"^([a-z0-9.-_\-]+|\[[a-f0-9]*:[a-f0-9:]+\])(:\d+)?$")
def __init__(self, aws_access_key_id, aws_secret_access_key, bucket,
cache_dir):
self.bucket = bucket
self.cache_dir = cache_dir
self.s3 = S3Connection(aws_access_key_id, aws_secret_access_key)
self.s3bucket = self.s3.get_bucket(self.bucket)
def __init__(self) -> None:
super().__init__()
conn = S3Connection(settings.S3_KEY, settings.S3_SECRET_KEY)
bucket_name = settings.S3_AVATAR_BUCKET
self.bucket = conn.get_bucket(bucket_name, validate=False)
def setup_connection(self):
self.conn = S3Connection(settings.AWS_ACCESS_KEY_ID,
settings.AWS_SECRET_ACCESS_KEY)
self.bucket = self.conn.get_bucket(settings.STATIC_MEDIA_S3_BUCKET)
def test_other_region():
conn = S3Connection('key',
'secret',
host='s3-website-ap-southeast-2.amazonaws.com')
conn.create_bucket("foobar")
list(conn.get_bucket("foobar").get_all_keys()).should.equal([])
def getBucket(self, aws_access, aws_secret, bucket_name):
self.log.info("S3: Open Bucket, %s, %s, %s" % (bucket_name, aws_access,
aws_secret))
s3 = S3Connection(aws_access, aws_secret)
return s3.lookup(bucket_name)
roi_filename = 'RoiSet.zip' # File of ROI masks
dim_order_for_hdf5 = 'tyx' # dim_order for HDF5 file. It states what the axes
# in your data array represents. So tyx means the data array in the HDF5 file
# is shaped "Time x Y pixels x X pixels". Used in sima.Sequence.create()
# -------------------------------------------------------------------
# -------------------------------------------------------------------
# --------------------------------TO HERE----------------------------
# -------------------------------------------------------------------
# -------------------------------------------------------------------
# Additional global variables. Do not change any of these
base_path_on_ec2 = '/mnt/analysis/DATA/'
# This is the working directory where the analysis script is run.
conn = S3Connection(AWS_ACCESS_KEY, AWS_SECRET_ACCESS_KEY)
bucket = conn.get_bucket(s3_bucket_name)
def create_sequence(full_path_to_file, individualframename):
# Retrieve S3 filename without path
temp = os.path.splitext(os.path.basename(full_path_to_file))
filename = temp[0]
if filename[-4:] == '_CH1':
dual_channel = True
else:
dual_channel = False
if is_single_file:
file_format = temp[-1]
def get_s3_bucket():
from boto.s3.connection import S3Connection
conn = S3Connection(app_settings.S3_ACCESS_KEY,
app_settings.S3_SECRET_KEY)
return conn.get_bucket(app_settings.S3_BUCKET)
def fill_in_auth(self, http_request, **kwargs):
_update_headers(http_request.headers)
S3Connection.fill_in_auth(self, http_request, **kwargs)
_update_headers(http_request.headers)
return http_request
import os
import time
import json
from models import db, STATUS, PRIORITY, File, Conversion, Account
from config import app
from semisync import semisync
import requests
from boto.s3.connection import S3Connection
from boto.s3.key import Key
AWS_S3_CONNECTION = S3Connection()
bucket = AWS_S3_CONNECTION.get_bucket(app.config['S3_BUCKET'])
BASE_URL = 'http://127.0.0.1:8200'
JSON_POST_HEADERS = {
'Content-type': 'application/json',
'Accept': 'application/json'
}
# dummy_auth = requests.auth.HTTPBasicAuth(username, password)
dummy_token_auth = lambda token: requests.auth.HTTPBasicAuth(token, 'unused')
def is_ok_response(r):
return r.status_code >= 200 and r.status_code < 400
def is_unauthorised_call(r):
return r.status_code == 401
def main():
parser = argparse.ArgumentParser(
description='Process incoming Telemetry data',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("incoming_bucket",
help="The S3 bucket containing incoming files")
parser.add_argument("publish_bucket",
help="The S3 bucket to save processed files")
parser.add_argument("-n",
"--num-helpers",
metavar="N",
help="Start N helper processes",
type=int,
default=1)
parser.add_argument("-k", "--aws-key", help="AWS Key", required=True)
parser.add_argument("-s",
"--aws-secret-key",
help="AWS Secret Key",
required=True)
parser.add_argument("-w",
"--work-dir",
help="Location to cache downloaded files",
required=True)
parser.add_argument("-o",
"--output-dir",
help="Base dir to store processed data",
required=True)
parser.add_argument("-i",
"--input-files",
help="File containing a list of keys to process",
type=file)
parser.add_argument("-t",
"--telemetry-schema",
help="Location of the desired telemetry schema",
required=True)
args = parser.parse_args()
# TODO: keep track of partial success so that subsequent runs are idempotent.
start = datetime.now()
conn = S3Connection(args.aws_key, args.aws_secret_key)
incoming_bucket = conn.get_bucket(args.incoming_bucket)
incoming_filenames = []
if args.input_files:
print "Fetching file list from file", args.input_files
incoming_filenames = [l.strip() for l in args.input_files.readlines()]
else:
print "Fetching file list from S3..."
for f in incoming_bucket.list():
incoming_filenames.append(f.name)
print "Done"
for f in incoming_filenames:
print " ", f
result = 0
print "Downloading", len(incoming_filenames), "files..."
result = fetch_s3_files(incoming_filenames, args.work_dir,
args.incoming_bucket, args.aws_key,
args.aws_secret_key)
if result != 0:
print "Error downloading files. Return code of s3funnel was", result
return result
print "Done"
print "Splitting raw logs..."
local_filenames = [
os.path.join(args.work_dir, f) for f in incoming_filenames
]
result = split_raw_logs(local_filenames, args.output_dir,
args.telemetry_schema)
if result != 0:
print "Error splitting logs. Return code was", result
return result
print "Done"
print "Converting split logs..."
result = convert_split_logs(args.output_dir)
if result != 0:
print "Error converting logs. Return code was", result
return result
print "Done"
print "Exporting converted logs back to S3..."
result = export_converted_logs(args.output_dir, args.publish_bucket,
args.aws_key, args.aws_secret_key)
if result != 0:
print "Error exporting logs. Return code was", result
return result
print "Done"
print "Removing processed logs from S3..."
for f in incoming_filenames:
print " Deleting", f
incoming_bucket.delete_key(f)
print "Done"
duration = timer.delta_sec(start)
print "All done in %.2fs" % (duration)
return 0
def main():
argument_spec = ec2_argument_spec()
argument_spec.update(
dict(
bucket=dict(required=True),
dest=dict(default=None),
encrypt=dict(default=True, type='bool'),
expiry=dict(default=600, aliases=['expiration']),
headers=dict(type='dict'),
marker=dict(default=None),
max_keys=dict(default=1000),
metadata=dict(type='dict'),
mode=dict(choices=[
'get', 'put', 'delete', 'create', 'geturl', 'getstr', 'delobj',
'list'
],
required=True),
object=dict(),
permission=dict(type='list', default=['private']),
version=dict(default=None),
overwrite=dict(aliases=['force'], default='always'),
prefix=dict(default=None),
retries=dict(aliases=['retry'], type='int', default=0),
s3_url=dict(aliases=['S3_URL']),
rgw=dict(default='no', type='bool'),
src=dict(),
), )
module = AnsibleModule(argument_spec=argument_spec)
if not HAS_BOTO:
module.fail_json(msg='boto required for this module')
bucket = module.params.get('bucket')
encrypt = module.params.get('encrypt')
expiry = int(module.params['expiry'])
if module.params.get('dest'):
dest = os.path.expanduser(module.params.get('dest'))
headers = module.params.get('headers')
marker = module.params.get('marker')
max_keys = module.params.get('max_keys')
metadata = module.params.get('metadata')
mode = module.params.get('mode')
obj = module.params.get('object')
version = module.params.get('version')
overwrite = module.params.get('overwrite')
prefix = module.params.get('prefix')
retries = module.params.get('retries')
s3_url = module.params.get('s3_url')
rgw = module.params.get('rgw')
src = module.params.get('src')
for acl in module.params.get('permission'):
if acl not in CannedACLStrings:
module.fail_json(msg='Unknown permission specified: %s' % str(acl))
if overwrite not in ['always', 'never', 'different']:
if module.boolean(overwrite):
overwrite = 'always'
else:
overwrite = 'never'
region, ec2_url, aws_connect_kwargs = get_aws_connection_info(module)
if region in ('us-east-1', '', None):
# S3ism for the US Standard region
location = Location.DEFAULT
else:
# Boto uses symbolic names for locations but region strings will
# actually work fine for everything except us-east-1 (US Standard)
location = region
if module.params.get('object'):
obj = os.path.expanduser(module.params['object'])
# allow eucarc environment variables to be used if ansible vars aren't set
if not s3_url and 'S3_URL' in os.environ:
s3_url = os.environ['S3_URL']
# rgw requires an explicit url
if rgw and not s3_url:
module.fail_json(msg='rgw flavour requires s3_url')
# bucket names with .'s in them need to use the calling_format option,
# otherwise the connection will fail. See https://github.com/boto/boto/issues/2836
# for more details.
if '.' in bucket:
aws_connect_kwargs['calling_format'] = OrdinaryCallingFormat()
# Look at s3_url and tweak connection settings
# if connecting to RGW, Walrus or fakes3
try:
if s3_url and rgw:
rgw = urlparse.urlparse(s3_url)
s3 = boto.connect_s3(is_secure=rgw.scheme == 'https',
host=rgw.hostname,
port=rgw.port,
calling_format=OrdinaryCallingFormat(),
**aws_connect_kwargs)
elif is_fakes3(s3_url):
fakes3 = urlparse.urlparse(s3_url)
s3 = S3Connection(is_secure=fakes3.scheme == 'fakes3s',
host=fakes3.hostname,
port=fakes3.port,
calling_format=OrdinaryCallingFormat(),
**aws_connect_kwargs)
elif is_walrus(s3_url):
walrus = urlparse.urlparse(s3_url).hostname
s3 = boto.connect_walrus(walrus, **aws_connect_kwargs)
else:
aws_connect_kwargs['is_secure'] = True
try:
s3 = connect_to_aws(boto.s3, location, **aws_connect_kwargs)
except AnsibleAWSError:
# use this as fallback because connect_to_region seems to fail in boto + non 'classic' aws accounts in some cases
s3 = boto.connect_s3(**aws_connect_kwargs)
except boto.exception.NoAuthHandlerFound as e:
module.fail_json(msg='No Authentication Handler found: %s ' % str(e))
except Exception as e:
module.fail_json(msg='Failed to connect to S3: %s' % str(e))
if s3 is None: # this should never happen
module.fail_json(
msg=
'Unknown error, failed to create s3 connection, no information from boto.'
)
# If our mode is a GET operation (download), go through the procedure as appropriate ...
if mode == 'get':
# First, we check to see if the bucket exists, we get "bucket" returned.
bucketrtn = bucket_check(module, s3, bucket)
if bucketrtn is False:
module.fail_json(msg="Source bucket cannot be found", failed=True)
# Next, we check to see if the key in the bucket exists. If it exists, it also returns key_matches md5sum check.
keyrtn = key_check(module, s3, bucket, obj, version=version)
if keyrtn is False:
if version is not None:
module.fail_json(
msg="Key %s with version id %s does not exist." %
(obj, version),
failed=True)
else:
module.fail_json(msg="Key %s does not exist." % obj,
failed=True)
# If the destination path doesn't exist or overwrite is True, no need to do the md5um etag check, so just download.
pathrtn = path_check(dest)
if pathrtn is False or overwrite == 'always':
download_s3file(module,
s3,
bucket,
obj,
dest,
retries,
version=version)
# Compare the remote MD5 sum of the object with the local dest md5sum, if it already exists.
if pathrtn is True:
md5_remote = keysum(module, s3, bucket, obj, version=version)
md5_local = module.md5(dest)
if md5_local == md5_remote:
sum_matches = True
if overwrite == 'always':
download_s3file(module,
s3,
bucket,
obj,
dest,
retries,
version=version)
else:
module.exit_json(
msg=
"Local and remote object are identical, ignoring. Use overwrite=always parameter to force.",
changed=False)
else:
sum_matches = False
if overwrite in ('always', 'different'):
download_s3file(module,
s3,
bucket,
obj,
dest,
retries,
version=version)
else:
module.exit_json(
msg=
"WARNING: Checksums do not match. Use overwrite parameter to force download."
)
# Firstly, if key_matches is TRUE and overwrite is not enabled, we EXIT with a helpful message.
if sum_matches is True and overwrite == 'never':
module.exit_json(
msg=
"Local and remote object are identical, ignoring. Use overwrite parameter to force.",
changed=False)
# if our mode is a PUT operation (upload), go through the procedure as appropriate ...
if mode == 'put':
# Use this snippet to debug through conditionals:
# module.exit_json(msg="Bucket return %s"%bucketrtn)
# Lets check the src path.
pathrtn = path_check(src)
if pathrtn is False:
module.fail_json(msg="Local object for PUT does not exist",
failed=True)
# Lets check to see if bucket exists to get ground truth.
bucketrtn = bucket_check(module, s3, bucket)
if bucketrtn is True:
keyrtn = key_check(module, s3, bucket, obj)
# Lets check key state. Does it exist and if it does, compute the etag md5sum.
if bucketrtn is True and keyrtn is True:
md5_remote = keysum(module, s3, bucket, obj)
md5_local = module.md5(src)
if md5_local == md5_remote:
sum_matches = True
if overwrite == 'always':
upload_s3file(module, s3, bucket, obj, src, expiry,
metadata, encrypt, headers)
else:
get_download_url(module,
s3,
bucket,
obj,
expiry,
changed=False)
else:
sum_matches = False
if overwrite in ('always', 'different'):
upload_s3file(module, s3, bucket, obj, src, expiry,
metadata, encrypt, headers)
else:
module.exit_json(
msg=
"WARNING: Checksums do not match. Use overwrite parameter to force upload."
)
# If neither exist (based on bucket existence), we can create both.
if bucketrtn is False and pathrtn is True:
create_bucket(module, s3, bucket, location)
upload_s3file(module, s3, bucket, obj, src, expiry, metadata,
encrypt, headers)
# If bucket exists but key doesn't, just upload.
if bucketrtn is True and pathrtn is True and keyrtn is False:
upload_s3file(module, s3, bucket, obj, src, expiry, metadata,
encrypt, headers)
# Delete an object from a bucket, not the entire bucket
if mode == 'delobj':
if obj is None:
module.fail_json(msg="object parameter is required", failed=True)
if bucket:
bucketrtn = bucket_check(module, s3, bucket)
if bucketrtn is True:
deletertn = delete_key(module, s3, bucket, obj)
if deletertn is True:
module.exit_json(msg="Object %s deleted from bucket %s." %
(obj, bucket),
changed=True)
else:
module.fail_json(msg="Bucket does not exist.", changed=False)
else:
module.fail_json(msg="Bucket parameter is required.", failed=True)
# Delete an entire bucket, including all objects in the bucket
if mode == 'delete':
if bucket:
bucketrtn = bucket_check(module, s3, bucket)
if bucketrtn is True:
deletertn = delete_bucket(module, s3, bucket)
if deletertn is True:
module.exit_json(
msg="Bucket %s and all keys have been deleted." %
bucket,
changed=True)
else:
module.fail_json(msg="Bucket does not exist.", changed=False)
else:
module.fail_json(msg="Bucket parameter is required.", failed=True)
# Support for listing a set of keys
if mode == 'list':
bucket_object = get_bucket(module, s3, bucket)
# If the bucket does not exist then bail out
if bucket_object is None:
module.fail_json(msg="Target bucket (%s) cannot be found" % bucket,
failed=True)
list_keys(module, bucket_object, prefix, marker, max_keys)
# Need to research how to create directories without "populating" a key, so this should just do bucket creation for now.
# WE SHOULD ENABLE SOME WAY OF CREATING AN EMPTY KEY TO CREATE "DIRECTORY" STRUCTURE, AWS CONSOLE DOES THIS.
if mode == 'create':
if bucket and not obj:
bucketrtn = bucket_check(module, s3, bucket)
if bucketrtn is True:
module.exit_json(msg="Bucket already exists.", changed=False)
else:
module.exit_json(msg="Bucket created successfully",
changed=create_bucket(module, s3, bucket,
location))
if bucket and obj:
bucketrtn = bucket_check(module, s3, bucket)
if obj.endswith('/'):
dirobj = obj
else:
dirobj = obj + "/"
if bucketrtn is True:
keyrtn = key_check(module, s3, bucket, dirobj)
if keyrtn is True:
module.exit_json(
msg="Bucket %s and key %s already exists." %
(bucket, obj),
changed=False)
else:
create_dirkey(module, s3, bucket, dirobj)
if bucketrtn is False:
created = create_bucket(module, s3, bucket, location)
create_dirkey(module, s3, bucket, dirobj)
# Support for grabbing the time-expired URL for an object in S3/Walrus.
if mode == 'geturl':
if bucket and obj:
bucketrtn = bucket_check(module, s3, bucket)
if bucketrtn is False:
module.fail_json(msg="Bucket %s does not exist." % bucket,
failed=True)
else:
keyrtn = key_check(module, s3, bucket, obj)
if keyrtn is True:
get_download_url(module, s3, bucket, obj, expiry)
else:
module.fail_json(msg="Key %s does not exist." % obj,
failed=True)
else:
module.fail_json(msg="Bucket and Object parameters must be set",
failed=True)
if mode == 'getstr':
if bucket and obj:
bucketrtn = bucket_check(module, s3, bucket)
if bucketrtn is False:
module.fail_json(msg="Bucket %s does not exist." % bucket,
failed=True)
else:
keyrtn = key_check(module, s3, bucket, obj, version=version)
if keyrtn is True:
download_s3str(module, s3, bucket, obj, version=version)
else:
if version is not None:
module.fail_json(
msg="Key %s with version id %s does not exist." %
(obj, version),
failed=True)
else:
module.fail_json(msg="Key %s does not exist." % obj,
failed=True)
module.exit_json(failed=False)
