def setup_s3(test):
# Test machines can set the GLANCE_TEST_S3_CONF variable
# to override the location of the config file for S3 testing
CONFIG_FILE_PATH = os.environ.get('GLANCE_TEST_S3_CONF')
if not CONFIG_FILE_PATH:
test.disabled_message = "GLANCE_TEST_S3_CONF environ not set."
test.disabled = True
return
if os.path.exists(CONFIG_FILE_PATH):
cp = ConfigParser.RawConfigParser()
try:
cp.read(CONFIG_FILE_PATH)
defaults = cp.defaults()
for key, value in defaults.items():
test.__dict__[key] = (_uniq(value)
if key == 's3_store_bucket' else value)
except ConfigParser.ParsingError as e:
test.disabled_message = ("Failed to read test_s3.conf config "
"file. Got error: %s" % e)
test.disabled = True
return
from boto.s3.connection import S3Connection
from boto.exception import S3ResponseError
try:
s3_host = test.s3_store_host
access_key = test.s3_store_access_key
secret_key = test.s3_store_secret_key
bucket_name = test.s3_store_bucket
except AttributeError as e:
test.disabled_message = ("Failed to find required configuration "
"options for S3 store. Got error: %s" % e)
test.disabled = True
return
calling_format = get_calling_format(test.s3_store_bucket_url_format)
s3_conn = S3Connection(access_key, secret_key,
host=s3_host,
is_secure=False,
calling_format=calling_format)
test.bucket = None
try:
buckets = s3_conn.get_all_buckets()
for bucket in buckets:
if bucket.name == bucket_name:
test.bucket = bucket
except S3ResponseError as e:
test.disabled_message = ("Failed to connect to S3 with "
"credentials, to find bucket. "
"Got error: %s" % e)
test.disabled = True
return
except TypeError as e:
# This hack is necessary because of a bug in boto 1.9b:
# http://code.google.com/p/boto/issues/detail?id=540
test.disabled_message = ("Failed to connect to S3 with "
"credentials. Got error: %s" % e)
test.disabled = True
return
test.s3_conn = s3_conn
if not test.bucket:
location = get_s3_location(test.s3_store_host)
try:
test.bucket = s3_conn.create_bucket(bucket_name,
location=location)
except S3ResponseError as e:
test.disabled_message = ("Failed to create bucket. "
"Got error: %s" % e)
test.disabled = True
return
else:
for key in test.bucket.list():
key.delete()
def connect(**kw):
# TODO: pool connections?
return S3Connection(settings.AWS_ACCESS_KEY, settings.AWS_SECRET_KEY,
**kw)
def main():
argument_spec = ec2_argument_spec()
argument_spec.update(
dict(
bucket=dict(required=True),
dest=dict(default=None),
encrypt=dict(default=True, type='bool'),
expiry=dict(default=600, aliases=['expiration']),
headers=dict(type='dict'),
marker=dict(default=None),
max_keys=dict(default=1000),
metadata=dict(type='dict'),
mode=dict(choices=[
'get', 'put', 'delete', 'create', 'geturl', 'getstr', 'delobj',
'list'
],
required=True),
object=dict(),
version=dict(default=None),
overwrite=dict(aliases=['force'], default='always'),
prefix=dict(default=None),
retries=dict(aliases=['retry'], type='int', default=0),
s3_url=dict(aliases=['S3_URL']),
src=dict(),
), )
module = AnsibleModule(argument_spec=argument_spec)
if not HAS_BOTO:
module.fail_json(msg='boto required for this module')
bucket = module.params.get('bucket')
encrypt = module.params.get('encrypt')
expiry = int(module.params['expiry'])
if module.params.get('dest'):
dest = os.path.expanduser(module.params.get('dest'))
headers = module.params.get('headers')
marker = module.params.get('marker')
max_keys = module.params.get('max_keys')
metadata = module.params.get('metadata')
mode = module.params.get('mode')
obj = module.params.get('object')
version = module.params.get('version')
overwrite = module.params.get('overwrite')
prefix = module.params.get('prefix')
retries = module.params.get('retries')
s3_url = module.params.get('s3_url')
src = module.params.get('src')
if overwrite not in ['always', 'never', 'different']:
if module.boolean(overwrite):
overwrite = 'always'
else:
overwrite = 'never'
if overwrite not in ['always', 'never', 'different']:
if module.boolean(overwrite):
overwrite = 'always'
else:
overwrite = 'never'
region, ec2_url, aws_connect_kwargs = get_aws_connection_info(module)
if region in ('us-east-1', '', None):
# S3ism for the US Standard region
location = Location.DEFAULT
else:
# Boto uses symbolic names for locations but region strings will
# actually work fine for everything except us-east-1 (US Standard)
location = region
if module.params.get('object'):
obj = os.path.expanduser(module.params['object'])
# allow eucarc environment variables to be used if ansible vars aren't set
if not s3_url and 'S3_URL' in os.environ:
s3_url = os.environ['S3_URL']
# Look at s3_url and tweak connection settings
# if connecting to Walrus or fakes3
try:
if is_fakes3(s3_url):
fakes3 = urlparse.urlparse(s3_url)
s3 = S3Connection(is_secure=fakes3.scheme == 'fakes3s',
host=fakes3.hostname,
port=fakes3.port,
calling_format=OrdinaryCallingFormat(),
**aws_connect_kwargs)
elif is_walrus(s3_url):
walrus = urlparse.urlparse(s3_url).hostname
s3 = boto.connect_walrus(walrus, **aws_connect_kwargs)
else:
s3 = boto.s3.connect_to_region(
location,
is_secure=True,
calling_format=OrdinaryCallingFormat(),
**aws_connect_kwargs)
# use this as fallback because connect_to_region seems to fail in boto + non 'classic' aws accounts in some cases
if s3 is None:
s3 = boto.connect_s3(**aws_connect_kwargs)
except boto.exception.NoAuthHandlerFound, e:
module.fail_json(msg='No Authentication Handler found: %s ' % str(e))
logger = logging.getLogger(log_file_name)
hdlr = logging.FileHandler(log_file_name)
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
# In[6]:
with open('config.json') as data_file:
data1 = json.load(data_file)
# In[9]:
conn = S3Connection(data1['AWSAccess'], data1['AWSSecret'])
# In[12]:
bucket_name = "assignment2info7390"
initial_file = "zillow_clean.csv"
conn.create_bucket(bucket_name)
existingbucket = conn.get_bucket(bucket_name)
initial_data = Key(existingbucket)
initial_data.key = initial_file
initial_data.set_contents_from_filename(initial_file)
# In[ ]:
try:
s3_conn = S3Connection(
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
is_secure=is_secure,
port=port,
host=host,
path=path,
calling_format=calling_format,
)
log.debug('Got boto S3 connection to %s' % ud['s3_url'])
except Exception, e:
log.error("Exception getting S3 connection: %s" % e)
else: # default to Amazon connection
try:
s3_conn = S3Connection(access_key, secret_key)
log.debug('Got boto S3 connection.')
except BotoServerError, e:
log.error("Exception getting S3 connection: %s" % e)
return s3_conn
def _bucket_exists(s3_conn, bucket_name):
bucket = None
for i in range(1, 6):
try:
# log.debug("Looking for bucket '%s'" % bucket_name)
bucket = s3_conn.lookup(bucket_name)
break
except S3ResponseError:
log.error("Bucket '%s' not found, attempt %s/5" %
def yieldS3Datasets(awsid, awskey, host, bucket, prefix):
s3 = S3Connection(awsid, awskey, host=host)
buck = s3.get_bucket(bucket)
for k in buck.list(prefix=prefix):
if k.name.endswith('_runfiles.tar.gz'):
yield k.generate_url(expires_in=-1, query_auth=False, force_http=True)
def get_all_keys_in_bucket(self):
self.conn = S3Connection(self.access_key, self.access_secret,
calling_format=(
boto.s3.connection.OrdinaryCallingFormat()))
self.bucket = self.conn.get_bucket(self.bucket, validate=False)
self.key_list = self.bucket.list(prefix=self.prefix)
def __init__(self) -> None:
self.connection = S3Connection(settings.S3_KEY, settings.S3_SECRET_KEY)
from boto.s3.connection import S3Connection
from boto.s3.key import Key
GPIO.setwarnings(False)
GPIO.setmode(GPIO.BCM)
GPIO.setup(7, GPIO.OUT)
GPIO.setup(18, GPIO.IN, pull_up_down=GPIO.PUD_UP)
AWS_ACCESS ='AKIAJVAP3ASQHYTSOA7A'
AWS_SECRET ='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'
conn = S3Connection(AWS_ACCESS,AWS_SECRET)
bucket = conn.get_bucket('imagesfrompi')
directory = '/home/pi/'
def percent_cb(complete, total):
sys.stdout.write('.')
sys.stdout.flush()
def getFiles(dir):
return [os.path.basename(x) for x in glob.glob(str(dir) + '*.jpg')]
def setPinHigh():
GPIO.output(7, GPIO.HIGH)
def setPinLow():
GPIO.output(7, GPIO.LOW)
from multiprocessing import Pool
from traceback import format_exc
from boto.s3.connection import S3Connection
from boto.s3.key import Key
import json
import cStringIO
import gzip
from ...profile import geo_profile, enhance_api_data
import logging
logging.basicConfig(level=logging.WARN)
logger = logging.getLogger(__name__)
s3 = S3Connection()
def s3_keyname(geoid):
return '/1.0/data/profiles/%s.json' % geoid
def key(geoid):
bucket = s3.get_bucket('embed.censusreporter.org')
keyname = s3_keyname(geoid)
key = Key(bucket, keyname)
return key
def write_profile_json(s3_key, data):
def get_signed_upload_url(path: str) -> str:
conn = S3Connection(settings.S3_KEY, settings.S3_SECRET_KEY)
return conn.generate_url(SIGNED_UPLOAD_URL_DURATION,
'GET',
bucket=settings.S3_AUTH_UPLOADS_BUCKET,
key=path)
def main():
argument_spec = ec2_argument_spec()
argument_spec.update(
dict(
bucket=dict(required=True),
dest=dict(default=None),
encrypt=dict(default=True, type='bool'),
expiry=dict(default=600, aliases=['expiration']),
headers=dict(type='dict'),
marker=dict(default=None),
max_keys=dict(default=1000),
metadata=dict(type='dict'),
mode=dict(choices=[
'get', 'put', 'delete', 'create', 'geturl', 'getstr', 'delobj',
'list'
],
required=True),
object=dict(),
permission=dict(type='list', default=['private']),
version=dict(default=None),
overwrite=dict(aliases=['force'], default='always'),
prefix=dict(default=None),
retries=dict(aliases=['retry'], type='int', default=0),
s3_url=dict(aliases=['S3_URL']),
rgw=dict(default='no', type='bool'),
src=dict(),
), )
module = AnsibleModule(argument_spec=argument_spec)
if not HAS_BOTO:
module.fail_json(msg='boto required for this module')
bucket = module.params.get('bucket')
encrypt = module.params.get('encrypt')
expiry = int(module.params['expiry'])
if module.params.get('dest'):
dest = os.path.expanduser(module.params.get('dest'))
headers = module.params.get('headers')
marker = module.params.get('marker')
max_keys = module.params.get('max_keys')
metadata = module.params.get('metadata')
mode = module.params.get('mode')
obj = module.params.get('object')
version = module.params.get('version')
overwrite = module.params.get('overwrite')
prefix = module.params.get('prefix')
retries = module.params.get('retries')
s3_url = module.params.get('s3_url')
rgw = module.params.get('rgw')
src = module.params.get('src')
for acl in module.params.get('permission'):
if acl not in CannedACLStrings:
module.fail_json(msg='Unknown permission specified: %s' % str(acl))
if overwrite not in ['always', 'never', 'different']:
if module.boolean(overwrite):
overwrite = 'always'
else:
overwrite = 'never'
region, ec2_url, aws_connect_kwargs = get_aws_connection_info(module)
if region in ('us-east-1', '', None):
# S3ism for the US Standard region
location = Location.DEFAULT
else:
# Boto uses symbolic names for locations but region strings will
# actually work fine for everything except us-east-1 (US Standard)
location = region
if module.params.get('object'):
obj = os.path.expanduser(module.params['object'])
# allow eucarc environment variables to be used if ansible vars aren't set
if not s3_url and 'S3_URL' in os.environ:
s3_url = os.environ['S3_URL']
# rgw requires an explicit url
if rgw and not s3_url:
module.fail_json(msg='rgw flavour requires s3_url')
# bucket names with .'s in them need to use the calling_format option,
# otherwise the connection will fail. See https://github.com/boto/boto/issues/2836
# for more details.
if '.' in bucket:
aws_connect_kwargs['calling_format'] = OrdinaryCallingFormat()
# Look at s3_url and tweak connection settings
# if connecting to RGW, Walrus or fakes3
try:
if s3_url and rgw:
rgw = urlparse.urlparse(s3_url)
s3 = boto.connect_s3(is_secure=rgw.scheme == 'https',
host=rgw.hostname,
port=rgw.port,
calling_format=OrdinaryCallingFormat(),
**aws_connect_kwargs)
elif is_fakes3(s3_url):
fakes3 = urlparse.urlparse(s3_url)
s3 = S3Connection(is_secure=fakes3.scheme == 'fakes3s',
host=fakes3.hostname,
port=fakes3.port,
calling_format=OrdinaryCallingFormat(),
**aws_connect_kwargs)
elif is_walrus(s3_url):
walrus = urlparse.urlparse(s3_url).hostname
s3 = boto.connect_walrus(walrus, **aws_connect_kwargs)
else:
aws_connect_kwargs['is_secure'] = True
try:
s3 = connect_to_aws(boto.s3, location, **aws_connect_kwargs)
except AnsibleAWSError:
# use this as fallback because connect_to_region seems to fail in boto + non 'classic' aws accounts in some cases
s3 = boto.connect_s3(**aws_connect_kwargs)
except boto.exception.NoAuthHandlerFound as e:
module.fail_json(msg='No Authentication Handler found: %s ' % str(e))
except Exception as e:
module.fail_json(msg='Failed to connect to S3: %s' % str(e))
if s3 is None: # this should never happen
module.fail_json(
msg=
'Unknown error, failed to create s3 connection, no information from boto.'
)
# If our mode is a GET operation (download), go through the procedure as appropriate ...
if mode == 'get':
# First, we check to see if the bucket exists, we get "bucket" returned.
bucketrtn = bucket_check(module, s3, bucket)
if bucketrtn is False:
module.fail_json(msg="Source bucket cannot be found", failed=True)
# Next, we check to see if the key in the bucket exists. If it exists, it also returns key_matches md5sum check.
keyrtn = key_check(module, s3, bucket, obj, version=version)
if keyrtn is False:
if version is not None:
module.fail_json(
msg="Key %s with version id %s does not exist." %
(obj, version),
failed=True)
else:
module.fail_json(msg="Key %s does not exist." % obj,
failed=True)
# If the destination path doesn't exist or overwrite is True, no need to do the md5um etag check, so just download.
pathrtn = path_check(dest)
if pathrtn is False or overwrite == 'always':
download_s3file(module,
s3,
bucket,
obj,
dest,
retries,
version=version)
# Compare the remote MD5 sum of the object with the local dest md5sum, if it already exists.
if pathrtn is True:
md5_remote = keysum(module, s3, bucket, obj, version=version)
md5_local = module.md5(dest)
if md5_local == md5_remote:
sum_matches = True
if overwrite == 'always':
download_s3file(module,
s3,
bucket,
obj,
dest,
retries,
version=version)
else:
module.exit_json(
msg=
"Local and remote object are identical, ignoring. Use overwrite=always parameter to force.",
changed=False)
else:
sum_matches = False
if overwrite in ('always', 'different'):
download_s3file(module,
s3,
bucket,
obj,
dest,
retries,
version=version)
else:
module.exit_json(
msg=
"WARNING: Checksums do not match. Use overwrite parameter to force download."
)
# Firstly, if key_matches is TRUE and overwrite is not enabled, we EXIT with a helpful message.
if sum_matches is True and overwrite == 'never':
module.exit_json(
msg=
"Local and remote object are identical, ignoring. Use overwrite parameter to force.",
changed=False)
# if our mode is a PUT operation (upload), go through the procedure as appropriate ...
if mode == 'put':
# Use this snippet to debug through conditionals:
# module.exit_json(msg="Bucket return %s"%bucketrtn)
# Lets check the src path.
pathrtn = path_check(src)
if pathrtn is False:
module.fail_json(msg="Local object for PUT does not exist",
failed=True)
# Lets check to see if bucket exists to get ground truth.
bucketrtn = bucket_check(module, s3, bucket)
if bucketrtn is True:
keyrtn = key_check(module, s3, bucket, obj)
# Lets check key state. Does it exist and if it does, compute the etag md5sum.
if bucketrtn is True and keyrtn is True:
md5_remote = keysum(module, s3, bucket, obj)
md5_local = module.md5(src)
if md5_local == md5_remote:
sum_matches = True
if overwrite == 'always':
upload_s3file(module, s3, bucket, obj, src, expiry,
metadata, encrypt, headers)
else:
get_download_url(module,
s3,
bucket,
obj,
expiry,
changed=False)
else:
sum_matches = False
if overwrite in ('always', 'different'):
upload_s3file(module, s3, bucket, obj, src, expiry,
metadata, encrypt, headers)
else:
module.exit_json(
msg=
"WARNING: Checksums do not match. Use overwrite parameter to force upload."
)
# If neither exist (based on bucket existence), we can create both.
if bucketrtn is False and pathrtn is True:
create_bucket(module, s3, bucket, location)
upload_s3file(module, s3, bucket, obj, src, expiry, metadata,
encrypt, headers)
# If bucket exists but key doesn't, just upload.
if bucketrtn is True and pathrtn is True and keyrtn is False:
upload_s3file(module, s3, bucket, obj, src, expiry, metadata,
encrypt, headers)
# Delete an object from a bucket, not the entire bucket
if mode == 'delobj':
if obj is None:
module.fail_json(msg="object parameter is required", failed=True)
if bucket:
bucketrtn = bucket_check(module, s3, bucket)
if bucketrtn is True:
deletertn = delete_key(module, s3, bucket, obj)
if deletertn is True:
module.exit_json(msg="Object %s deleted from bucket %s." %
(obj, bucket),
changed=True)
else:
module.fail_json(msg="Bucket does not exist.", changed=False)
else:
module.fail_json(msg="Bucket parameter is required.", failed=True)
# Delete an entire bucket, including all objects in the bucket
if mode == 'delete':
if bucket:
bucketrtn = bucket_check(module, s3, bucket)
if bucketrtn is True:
deletertn = delete_bucket(module, s3, bucket)
if deletertn is True:
module.exit_json(
msg="Bucket %s and all keys have been deleted." %
bucket,
changed=True)
else:
module.fail_json(msg="Bucket does not exist.", changed=False)
else:
module.fail_json(msg="Bucket parameter is required.", failed=True)
# Support for listing a set of keys
if mode == 'list':
bucket_object = get_bucket(module, s3, bucket)
# If the bucket does not exist then bail out
if bucket_object is None:
module.fail_json(msg="Target bucket (%s) cannot be found" % bucket,
failed=True)
list_keys(module, bucket_object, prefix, marker, max_keys)
# Need to research how to create directories without "populating" a key, so this should just do bucket creation for now.
# WE SHOULD ENABLE SOME WAY OF CREATING AN EMPTY KEY TO CREATE "DIRECTORY" STRUCTURE, AWS CONSOLE DOES THIS.
if mode == 'create':
if bucket and not obj:
bucketrtn = bucket_check(module, s3, bucket)
if bucketrtn is True:
module.exit_json(msg="Bucket already exists.", changed=False)
else:
module.exit_json(msg="Bucket created successfully",
changed=create_bucket(module, s3, bucket,
location))
if bucket and obj:
bucketrtn = bucket_check(module, s3, bucket)
if obj.endswith('/'):
dirobj = obj
else:
dirobj = obj + "/"
if bucketrtn is True:
keyrtn = key_check(module, s3, bucket, dirobj)
if keyrtn is True:
module.exit_json(
msg="Bucket %s and key %s already exists." %
(bucket, obj),
changed=False)
else:
create_dirkey(module, s3, bucket, dirobj)
if bucketrtn is False:
created = create_bucket(module, s3, bucket, location)
create_dirkey(module, s3, bucket, dirobj)
# Support for grabbing the time-expired URL for an object in S3/Walrus.
if mode == 'geturl':
if bucket and obj:
bucketrtn = bucket_check(module, s3, bucket)
if bucketrtn is False:
module.fail_json(msg="Bucket %s does not exist." % bucket,
failed=True)
else:
keyrtn = key_check(module, s3, bucket, obj)
if keyrtn is True:
get_download_url(module, s3, bucket, obj, expiry)
else:
module.fail_json(msg="Key %s does not exist." % obj,
failed=True)
else:
module.fail_json(msg="Bucket and Object parameters must be set",
failed=True)
if mode == 'getstr':
if bucket and obj:
bucketrtn = bucket_check(module, s3, bucket)
if bucketrtn is False:
module.fail_json(msg="Bucket %s does not exist." % bucket,
failed=True)
else:
keyrtn = key_check(module, s3, bucket, obj, version=version)
if keyrtn is True:
download_s3str(module, s3, bucket, obj, version=version)
else:
if version is not None:
module.fail_json(
msg="Key %s with version id %s does not exist." %
(obj, version),
failed=True)
else:
module.fail_json(msg="Key %s does not exist." % obj,
failed=True)
module.exit_json(failed=False)
def add_questions_to_db_and_mock_test(paper_questions, comprehensions,
mock_test_id):
"""This method adds the given questions (result of `parse_question`) to the DB
and also adds them to the questions_ids attribute of the mock test row.
"""
## Get the mock test
mock_test = MockTest.query.get(mock_test_id)
## Get the S3 buckets here (otherwise will have to make calls for every content block)
conn = S3Connection(config.S3_ACCESS_KEY, config.S3_SECRET)
question_files_final_bucket = conn.get_bucket(
app.config['S3_QUESTION_FILES_FINAL_BUCKET'])
## Upload the questions to the DB if there are no errors
added_questions = []
comprehension_ids = {}
print 'Number of Questions: {0}'.format(len(paper_questions))
for question in paper_questions:
status = dict(config.QUESTION_STATUS.items())
status['categorized'] = '1'
status['text_solution_added'] = '1'
# status['proof_read_categorization'] = '1'
# make a list of the correct options
correct_options = []
for i in range(len(question['options']['values'])):
if question['options']['values'][i]['correct']:
correct_options.append(i)
# move the images to s3 and change the markup accordingly
"""
question['body']['value'] = move_images_to_final_bucket(question['body']['value'], question_files_final_bucket)
question['text_solution']['value'] = move_images_to_final_bucket(question['text_solution']['value'], question_files_final_bucket)
for i in range(len(question['options']['values'])):
question['options']['values'][i]['value'] = move_images_to_final_bucket(question['options']['values'][i]['value'],
question_files_final_bucket)
for i in range(len(comprehensions)):
comprehensions[i]['value'] = move_images_to_final_bucket(comprehensions[i]['value'], question_files_final_bucket)
"""
# create a comprehension if needed or just pick up a comprehension ID
comprehension_id = None
if question['comprehension']:
if comprehension_ids.get(question['comprehension_index']):
comprehension_id = comprehension_ids[
question['comprehension_index']]
else:
comp_ = comprehensions[question['comprehension_index']]
comprehension = Comprehension.create(comp_['value'])
db.session.add(comprehension)
db.session.commit()
comprehension_id = comprehension.id
comprehension_ids[
question['comprehension_index']] = comprehension.id
# create the question in the DB
question_data = {
'content':
question['body']['value'],
'status':
status,
'all_options':
[option['value'] for option in question['options']['values']],
'correct_options':
correct_options,
'ontology_id':
question['ontology']['value'][-1],
'average_time':
int(question['attributes']['average_time']['value']),
'nature':
question['attributes']['nature']['value'],
'difficulty':
question['attributes']['difficulty']['value'],
'type':
question['attributes']['type']['value'],
'text_solution':
question['text_solution']['value'],
'text_solution_by_type':
1,
'text_solution_by_id':
app.config['AUTO_UPLOAD_DTP_ID'],
'comprehension_id':
comprehension_id
}
question_ = Question.create(**question_data)
added_questions.append([question_, question['ontology']['value']])
# create the attached text solution submission row in the db too
solution_submission_params = {
'submitted_by_type': 3,
'submitted_by_id': app.config['AUTO_UPLOAD_TEACHER_ID'],
'question_id': question_.id,
'solution_type': 'text',
'solution': question['text_solution']['value'],
}
SolutionSubmission.create(**solution_submission_params)
# create the attached category submission row in the db too
last_ontology_obj = Ontology.query.get(question_data['ontology_id'])
category_submission_params = {
'submitted_by_type': 3,
'submitted_by_id': app.config['AUTO_UPLOAD_TEACHER_ID'],
'question_id': question_.id,
'ontology': last_ontology_obj.absolute_path,
'nature': question_data['nature'],
'type': question_data['type'],
'difficulty': question_data['difficulty'],
'average_time': question_data['average_time']
}
CategorySubmission.create(**category_submission_params)
## Add the questions to the mock Test
mock_test_questions = {}
order = -1
for question, ontology in added_questions:
subject_id = ontology[0]
if str(subject_id) not in mock_test_questions:
print str(subject_id)
order = order + 1
mock_test_questions.update(
{str(subject_id): {
'order': order,
'q_ids': [question.id]
}})
continue
if str(subject_id) in mock_test_questions:
mock_test_questions[str(subject_id)]['q_ids'].append(question.id)
continue
print mock_test_questions.keys()
## Add the `mock_test_questions` to the mock test
mock_test.question_ids = json.dumps(mock_test_questions)
db.session.add(mock_test)
db.session.commit()
return True
def factory(cls, bucket, directory=None):
access_key, secret_key = get_s3_access()
conn = S3Connection(access_key, secret_key)
bucket = conn.get_bucket(bucket)
return cls(directory, access_key, secret_key, conn, bucket)
import requests
import json
import time
from bs4 import BeautifulSoup
from boto.s3.connection import S3Connection
from boto.s3.key import Key
tournament_string = 'rbc-heritage'
tournament = 'RBC Heritage'
year = 2015
time_string = time.time()
# create connection to bucket
c = S3Connection('AKIAIQQ36BOSTXH3YEBA',
'cXNBbLttQnB9NB3wiEzOWLF13Xw8jKujvoFxmv3L')
# create connection to bucket
b = c.get_bucket('public.tenthtee')
k1 = Key(b)
k1.key = 'playerData/oddsToPgaMapping'
player_map = k1.get_contents_as_string()
player_map = json.loads(player_map)
#k2 = Key(b)
#k2.key = 'playerData/playerLookupByName'
#player_list = k2.get_contents_as_string()
#player_list = json.loads(player_list)
#players = []
#for key in player_list.keys():
import tweepy
import os
from boto.s3.connection import S3Connection
from boto.s3.key import Key
from dl import main
# set up api
auth = tweepy.OAuthHandler(os.environ.get('CONSUMER_KEY'),
os.environ.get('CONSUMER_SECRET'))
auth.set_access_token(os.environ.get('ACCESS_TOKEN'),
os.environ.get('ACCESS_SECRET'))
api = tweepy.API(auth)
# import s3 database for line.txt
s3 = S3Connection(os.environ['AWS_ACCESS_KEY_ID'],
os.environ['AWS_SECRET_ACCESS_KEY'])
bucket = s3.get_bucket('birdcallbot-assets')
k = Key(bucket)
k.key = 'line.txt'
# read line and add one for later
line = int(k.get_contents_as_string())
k.set_contents_from_string(str(line + 1))
# make a new bird & populate folder with new media
main(line)
# what the bot will tweet
with open('tweet.txt', 'r') as text:
tweet = text.read()
# update status
from boto.s3.connection import S3Connection
AWS_KEY = ''
AWS_SECRET = ''
aws_connection = S3Connection(AWS_KEY, AWS_SECRET)
bucket = aws_connection.get_bucket('apple123')
for file_key in bucket.list():
print file_key.name
import os, sys
import time
from twython import Twython, TwythonError, TwythonRateLimitError
from boto.s3.connection import S3Connection
twitterAPI = Twython(os.environ["TWITTER_API_KEY"],
os.environ["TWITTER_API_SECRET"],
os.environ["TWITTER_TOKEN"],
os.environ["TWITTER_TOKEN_SECRET"])
s3API = S3Connection(os.environ["S3_KEY"], os.environ["S3_SECRET_KEY"])
key = s3API.get_bucket("wespooky-twittermanagement").get_key("followers.txt")
# ======================
# -=- CODE TIME -=-
# ======================
# == Load Sets ==
lastrunFollowers = set([
int(follower)
for follower in key.get_contents_as_string().decode().split("|")
])
currentFollowers = set(twitterAPI.get_followers_ids()["ids"])
# == Follow Back ==
# DESC: Follow everyone in current follower list who isn't in lastrun follower list
toFollow = currentFollowers.difference(lastrunFollowers)
for ID in toFollow:
try:
user = twitterAPI.show_user(user_id=ID)
class CloudController(BaseUIController):
def __init__(self, app):
BaseUIController.__init__(self, app)
@web.expose
def index(self, trans, share_string=None):
return trans.fill_template("cloud/index.mako",
default_keypair=DEFAULT_KEYPAIR,
share_string=share_string)
@web.expose
def get_account_info(self, trans, key_id, secret, **kwargs):
"""
Get EC2 Account Info
"""
#Keypairs
account_info = {}
try:
ec2_conn = connect_ec2(key_id, secret)
kps = ec2_conn.get_all_key_pairs()
except EC2ResponseError, e:
log.error("Problem starting an instance: %s\n%s" % (e, e.body))
account_info['keypairs'] = [akp.name for akp in kps]
#Existing Clusters
s3_conn = S3Connection(key_id,
secret,
calling_format=OrdinaryCallingFormat())
buckets = s3_conn.get_all_buckets()
clusters = []
for bucket in buckets:
try:
pd = bucket.get_key('persistent_data.yaml')
except S3ResponseError, e:
# This can fail for a number of reasons for non-us and/or CNAME'd buckets.
log.error(
"Problem fetching persistent_data.yaml from bucket: %s \n%s"
% (e, e.body))
continue
if pd:
# This is a cloudman bucket.
# We need to get persistent data, and the cluster name.
# DBTODO: Add pyyaml egg, and rewrite this.
# This will also allow for much more sophisticated rendering of existing clusters
# Currently, this zone detection is a hack.
pd_contents = pd.get_contents_as_string()
zone = ''
try:
for line in pd_contents.split('\n'):
if 'vol_id' in line:
vol_id = line.split(':')[1].strip()
v = ec2_conn.get_all_volumes(volume_ids=[vol_id])
if v:
zone = v[0].zone
else:
zone = ''
except:
#If anything goes wrong with zone detection, use the default selection.
zone = ''
for key in bucket.list():
if key.name.endswith('.clusterName'):
clusters.append({
'name':
key.name.split('.clusterName')[0],
'persistent_data':
pd_contents,
'zone':
zone
})
def s3_connection(kaccess, ksecret): from boto.s3.connection import S3Connection sconn = S3Connection(base64.b64decode(kaccess), base64.b64decode(ksecret)) return sconn
import numpy as np
import pandas as pd
import random
import pickle
from boto.s3.connection import S3Connection
from boto.s3.key import Key
import csv
key_data = np.genfromtxt(
"/media/parik/New Volume/SDM/R Lab/Final Project Ariline Delay/Data-Mining-Project-master/Secret_Keys/Key_data.txt",
delimiter=",",
dtype=str)
pic = ['Data-2003-1000000']
pictest = ['Data-2006-1000000', 'Data-2006-2000000']
conn = S3Connection(key_data[0], key_data[1])
bucket = conn.get_bucket('sdmairlines')
k = Key(bucket)
theta = []
def gradientDescent(x, y, numIterations, dimension, theta):
for i in range(1, numIterations):
randIdx = random.randint(0, len(x) - 1)
xTrans = x[randIdx][np.newaxis].transpose()
u = 1 / (1 + np.exp(
np.dot(theta.transpose().astype(float) *
(-1), xTrans.astype(float))))
loss = y[randIdx] - u
gradient = np.dot(loss[0][0], xTrans)
# update
# celeryapp.autodiscover_tasks(INSTALLED_APPS)
CELERY_ACCEPT_CONTENT = ['pickle', 'json', 'msgpack', 'yaml']
# ==========
# = Assets =
# ==========
JAMMIT = jammit.JammitAssets(PYTUNE_DIR)
if DEBUG:
MIDDLEWARE_CLASSES += (
'utils.request_introspection_middleware.DumpRequestMiddleware', )
MIDDLEWARE_CLASSES += (
'utils.exception_middleware.ConsoleExceptionMiddleware', )
# =======
# = AWS =
# =======
S3_CONN = None
if BACKED_BY_AWS.get('pages_on_s3') or BACKED_BY_AWS.get('icons_on_s3'):
S3_CONN = S3Connection(S3_ACCESS_KEY, S3_SECRET)
if BACKED_BY_AWS.get('pages_on_s3'):
S3_PAGES_BUCKET = S3_CONN.get_bucket(S3_PAGES_BUCKET_NAME)
if BACKED_BY_AWS.get('icons_on_s3'):
S3_ICONS_BUCKET = S3_CONN.get_bucket(S3_ICONS_BUCKET_NAME)
django.http.request.host_validation_re = re.compile(
r"^([a-z0-9.-_\-]+|\[[a-f0-9]*:[a-f0-9:]+\])(:\d+)?$")
def __init__(self, aws_key, aws_secret):
self.connection = S3Connection(aws_access_key_id=aws_key,
aws_secret_access_key=aws_secret)
def test_other_region():
conn = S3Connection('key',
'secret',
host='s3-website-ap-southeast-2.amazonaws.com')
conn.create_bucket("foobar")
list(conn.get_bucket("foobar").get_all_keys()).should.equal([])
# should just be one line, but you never know
with open('/home/hduser/.ec2/aws_id') as f:
lines = f.read().splitlines()
aws_id = lines[0]
print " Using aws_id:", aws_id
with open('/home/hduser/.ec2/aws_key') as f:
lines = f.read().splitlines()
aws_key = lines[0]
print " Using aws_key:", aws_key
from boto.s3.connection import S3Connection
from boto.s3.key import Key
# Initiate a S3 connection using key and secret
conn = S3Connection(aws_id, aws_key)
# The bucket name
bucket = 'home2-0xdiag-datasets'
pb = conn.get_bucket(bucket)
# Make an S3 key using the bucket
k = Key(pb)
file_name_to_use_in_s3 = "%s/%s" % (args.s3_path,
os.path.basename(args.local_file_path))
# Set the name of the file to use in S3
# S3 doesn't have the concept of directories
# Use / in the file name to mimic the directory path
k.name = file_name_to_use_in_s3
k.set_metadata(
'Cache-Control',
def get_conn():
global g_conn
from boto.s3.connection import S3Connection
if g_conn is None:
g_conn = S3Connection(g_aws_access, g_aws_secret, True)
return g_conn
import re
from boto.s3.connection import S3Connection
from boto.s3.key import Key
bucket = S3Connection().get_bucket('nlp-data')
print 'loading titles...'
titles = dict((re.search('/([0-9]+)\.gz', title.key).group(1), True)
for title in bucket.list(prefix='article_titles/')
if re.search('/([0-9]+)\.gz', title.key))
print 'loading redirects...'
redirects = dict((re.search('/([0-9]+)\.gz', redirect.key).group(1), True)
for redirect in bucket.list(prefix='article_redirects/')
if re.search('/([0-9]+)\.gz', redirect.key))
t = open('missing-titles.txt', 'w')
r = open('missing-redirects.txt', 'w')
for parsed in bucket.list(prefix='xml/', delimiter='/'):
#print parsed.name
s = re.search('([0-9]+)', parsed.name)
if s:
print 'checking', s
wid = s.group(0)
if not titles.get(wid, False):
print wid, 'not found in titles!'
t.write('%s\n' % wid)
if not redirects.get(wid, False):
print wid, 'not found in redirects!'
r.write('%s\n' % wid)
import os
import glob
from boto.s3.connection import S3Connection
import boto3
import numpy as np
import matplotlib.pyplot as plt
from imprep import *
np.random.seed(11)
conn = S3Connection()
s3 = boto3.resource('s3')
pics = []
y = []
for n, species in enumerate(
['alb', 'bet', 'dol', 'lag', 'nof', 'other', 'shark', 'yft']):
name = 'rb-fishery-' + species
bucket = conn.get_bucket(name)
key_list = bucket.get_all_keys()
keys = []
for key in key_list:
keys.append(key.key)
for key in keys:
try:
s3.meta.client.download_file(name, key, 'temp/' + str(key))
im = plt.imread('temp/' + str(key))
x = prep_image(im)
pics.append(x)
y.append(n)
# <nbformat>3.0</nbformat>
# <codecell>
from boto.s3.connection import S3Connection
from boto.s3.key import Key
from boto.emr.connection import EmrConnection
from boto.emr.step import StreamingStep
import boto.emr
# <codecell>
### Adding and deleting files from S3
#s3con = S3Connection('<aws access key>', '<aws secret key>')
s3con = S3Connection('AKIAJRV3RN6NXQTSSTBA',
'3e212d6rs99xtiPgwKnfN1QD30WZk2hJwCWjMcGc')
# <codecell>
#b = s3con.create_bucket('winteram-boto-example')
b = s3con.get_bucket('wambia660fall2013')
# <codecell>
k = Key(b)
k.key = 'mapper.py'
k.set_contents_from_filename('/Users/winteram/Documents/Teaching/mapper.py')
k.close()
# <codecell>
def main():
global s3con, dbcon
d = parse_creds(CRED_FILE)
db = EnrDb()
db.connect()
s3con = S3Connection(d['akey'], d['skey'])
bucket = s3con.get_bucket(CONFIG_BUCKET)
k = Key(bucket)
k.key = "microput/VERSION.TXT"
# help(k)
s3_version = int(k.get_contents_as_string().strip())
db_bundle = db.getLatestBundle()
db_version = 0
if not db_bundle:
db_version = 0
else:
print "Bundle in DB: " + str(db_bundle)
db_version = db_bundle[0]
print
print "In DB: %s, on S3: %s" % (db_version, s3_version)
if db_version < s3_version:
print "I have no idea what's over there on S3..."
dc_info = db.getDataCollectionInfo()
tmp_dir = tempfile.mkdtemp('microput')
os.mkdir(tmp_dir + "/lua")
log_s = ''
logdefs = db.getLogdefs()
if not logdefs:
print "No log definitions found, exiting."
sys.exit(1)
for l in logdefs:
log_s += "\n# %s (%s)\n" % l[0:2]
ld_name = l[0].lower()
ld_name = re.sub('[^0-9a-zA-Z]+', '', ld_name)
c_name = l[1].lower()
c_name = re.sub('[^0-9a-zA-Z]+', '', c_name)
segment = "%s_%s.%s_%s" % (c_name, l[2], ld_name, l[3])
# temporary hack
fields = l[4].replace('time_local', 'time_iso8601')
log_s += "log_format c%s_dc%s '$uid_got\\t$uid_set\\t$uid_reset\\t%s\\t%s';\n" % (
l[2], l[3], fields, segment)
log_s += "\n"
log_f = tmp_dir + "/nogit_logdefs.conf"
fwrite(log_f, log_s)
loc_s = generateLocations(tmp_dir, dc_info)
loc_f = tmp_dir + "/nogit_locations.conf"
fwrite(loc_f, loc_s)
bundle = "bundle"
zfname = tmp_dir + ("/%s" % bundle)
print "Creating %s" % (zfname)
zf = zipfile.ZipFile(zfname, 'w')
add_to_zip(zf, loc_f, os.path.basename(loc_f))
add_to_zip(zf, log_f, os.path.basename(log_f))
lua_list = os.listdir(tmp_dir + "/lua")
for lua_file in lua_list:
lua_file = tmp_dir + "/lua/" + lua_file
add_to_zip(zf, lua_file, "lua/" + os.path.basename(lua_file))
zf.close()
print "Now let's try this bundle..."
conf_dir = ENR_HOME + '/all/conf'
print "Changing directory to %s" % conf_dir
os.chdir(conf_dir)
print "Unzipping %s into %s" % (zfname, conf_dir)
zf = zipfile.ZipFile(zfname, 'r')
zf.extractall(conf_dir)
print "Reloading nginx"
(exit_code, out, err) = run_cmd(['/sbin/service', 'nginx', 'reload'])
print out
print err
print "Error code: %s" % exit_code
if exit_code:
print "Bad bundle, we cannot generate this."
sys.exit(exit_code)
if db_bundle and db_bundle[1] == loc_s and db_bundle[2] == log_s:
new_ver = db_version
else:
new_ver = db.addBundle(loc_s, log_s)
if new_ver == s3_version:
print "All up to date at version %s" % db_version
sys.exit()
new_zfname = zfname + "%s.zip" % new_ver
print "Renaming %s to %s" % (zfname, new_zfname)
os.rename(zfname, new_zfname)
s3con = S3Connection(d['akey'], d['skey'])
bucket = s3con.get_bucket(CONFIG_BUCKET)
k = Key(bucket)
k.key = "microput/VERSION.TXT"
print "Changing version on S3"
k.set_contents_from_string(new_ver)
print "Uploading bundle"
k.key = "microput/bundles/bundle%s.zip" % new_ver
k.set_contents_from_filename(new_zfname)
print "Done!"
