def get_obj(self, name):
"""
Returns the AWS object associated with a given option.
The heuristics used are a bit lame. If the option name contains
the word 'bucket' it is assumed to be an S3 bucket, if the name
contains the word 'queue' it is assumed to be an SQS queue and
if it contains the word 'domain' it is assumed to be a SimpleDB
domain. If the option name specified does not exist in the
config file or if the AWS object cannot be retrieved this
returns None.
"""
val = self.get(name)
if not val:
return None
if name.find('queue') >= 0:
obj = boto.lookup('sqs', val)
if obj:
obj.set_message_class(ServiceMessage)
elif name.find('bucket') >= 0:
obj = boto.lookup('s3', val)
elif name.find('domain') >= 0:
obj = boto.lookup('sdb', val)
else:
obj = None
return obj
def process_record(self, record, path, get_file=True):
self.log_message(record, path)
self.calculate_stats(record)
outputs = record['OutputKey'].split(',')
if 'OutputBucket' in record:
bucket = boto.lookup('s3', record['OutputBucket'])
else:
bucket = boto.lookup('s3', record['Bucket'])
for output in outputs:
if get_file:
key_name = output.split(';')[0]
key = bucket.lookup(key_name)
file_name = os.path.join(path, key_name)
print 'retrieving file: %s to %s' % (key_name, file_name)
key.get_contents_to_filename(file_name)
self.num_files += 1
def start(self, queue_name):
boto.log.info('Task[%s] - starting with queue: %s' % (self.name, queue_name))
queue = boto.lookup('sqs', queue_name)
msg = queue.new_message(self.id)
msg = queue.write(msg)
self.message_id = msg.id
self.put()
boto.log.info('Task[%s] - start successful' % self.name)
def get_file(self, message):
bucket_name = message['Bucket']
key_name = message['InputKey']
file_name = os.path.join(self.working_dir, message.get('OriginalFileName', 'in_file'))
boto.log.info('get_file: %s/%s to %s' % (bucket_name, key_name, file_name))
bucket = boto.lookup('s3', bucket_name)
key = bucket.new_key(key_name)
key.get_contents_to_filename(os.path.join(self.working_dir, file_name))
return file_name
def get_file(self, message):
bucket_name = message['Bucket']
key_name = message['InputKey']
file_name = os.path.join(self.working_dir,
message.get('OriginalFileName', 'in_file'))
boto.log.info('get_file: %s/%s to %s' %
(bucket_name, key_name, file_name))
bucket = boto.lookup('s3', bucket_name)
key = bucket.new_key(key_name)
key.get_contents_to_filename(os.path.join(self.working_dir, file_name))
return file_name
def connect(self):
self.queue = boto.lookup('sqs', self.queue_name)
self.queue.set_mesasge_class(JSONMessage)
def put_file(self, bucket_name, file_path, key_name=None):
boto.log.info('putting file %s as %s.%s' % (file_path, bucket_name, key_name))
bucket = boto.lookup('s3', bucket_name)
key = bucket.new_key(key_name)
key.set_contents_from_filename(file_path)
return key
def connect(self):
self.queue = boto.lookup("sqs", self.queue_name)
self.queue.set_mesasge_class(JSONMessage)
