def test_feeding_sends_an_sqs_message(self, now):
now.return_value = datetime.strptime('2016-01-01', '%Y-%m-%d')
queue = MagicMock()
key = Key()
key.bucket = Bucket()
key.bucket.name = 'ct-elife-production-final'
key.name = 'elife-12345-vor-r1.zip'
key.etag = '...'
key.size = 2 * 1024 * 1024
econ_article_feeder.initiate_econ_feed(queue, key, 'MyArticleWorkflow')
self.assertEqual(len(queue.method_calls), 1)
(_, args, _) = queue.method_calls[0]
message_body = args[0].get_body()
self.assertEqual(
json.loads(message_body),
{
'workflow_name': 'MyArticleWorkflow',
'workflow_data': {
'event_time': '2016-01-01T00:00:00Z',
'event_name': 'ObjectCreated:Put',
'file_name': 'elife-12345-vor-r1.zip',
'file_etag': '...',
'bucket_name': 'ct-elife-production-final',
'file_size': 2 * 1024 * 1024,
},
}
)
def test_key_with_files_and_callback(self):
"""
test simple key 'from_file' and 'to_file' functions
"""
def _archive_callback(bytes_sent, total_bytes):
print("archived {0} out of {1}".format(bytes_sent, total_bytes))
def _retrieve_callback(bytes_sent, total_bytes):
print("retrieved {0} out of {1}".format(bytes_sent, total_bytes))
log = logging.getLogger("test_key_with_files")
key_name = "A" * 1024
test_file_path = os.path.join(test_dir_path, "test_key_with_files-orignal")
test_file_size = 1024 ** 2
buffer_size = 1024
log.debug("writing {0} bytes to {1}".format(test_file_size, test_file_path))
bytes_written = 0
with open(test_file_path, "wb") as output_file:
while bytes_written < test_file_size:
output_file.write(os.urandom(buffer_size))
bytes_written += buffer_size
# create the bucket
bucket = self._s3_connection.create_unique_bucket()
self.assertTrue(bucket is not None)
# create an empty key
write_key = Key(bucket)
# set the name
write_key.name = key_name
self.assertFalse(write_key.exists())
# upload some data
with open(test_file_path, "rb") as archive_file:
write_key.set_contents_from_file(archive_file, cb=_archive_callback)
self.assertTrue(write_key.exists())
# create another key with the same name
read_key = Key(bucket, key_name)
# read back the data
retrieve_file_path = os.path.join(test_dir_path, "test_key_with_files-orignal")
# 2011-08-08 dougfort boto aborts if you don't tell it the size
read_key.size = test_file_size
with open(retrieve_file_path, "wb") as retrieve_file:
read_key.get_contents_to_file(retrieve_file, cb=_retrieve_callback)
self.assertTrue(filecmp.cmp(test_file_path, retrieve_file_path, shallow=False))
# delete the key
read_key.delete()
self.assertFalse(write_key.exists())
# delete the bucket
self._s3_connection.delete_bucket(bucket.name)
def test_should_list_backup_files(self, mock_print):
connection = boto.connect_s3()
bucket = connection.create_bucket('backup-test-bucket')
k = Key(bucket, name='backup-listtest-20150725062606.tar.gz')
k.size = '1234567890'
backup = S3Backup('test_backup',
config=self.s3_based_config,
workdir=self.backup_test_workdir)
backup.s3_keys = [k]
backup.list()
def test_should_list_backup_files(self, mock_print):
connection = boto.connect_s3()
bucket = connection.create_bucket('backup-test-bucket')
k = Key(bucket, name='backup-listtest-20150725062606.tar.gz')
k.size = '1234567890'
backup = S3Backup('test_backup',
config=self.s3_based_config,
workdir=self.backup_test_workdir)
backup.s3_keys = [k]
backup.list()
def Upload_Object(self, path, bucket):
"""
Uploads given file into bucket
_path_: local path to file to upload. if _path_ is a directory, then
an exception will be raised
_bucket_: target bucket
"""
if os.path.isdir(path):
raise NotImplementedError("Can't upload directories")
fp = open(path, 'r')
s3bucket = self._conn.get_bucket(bucket)
s3obj = Key(s3bucket)
s3obj.key = path
s3obj.size = os.stat(path).st_size
logger.debug("Calling send_file for object %s" % path)
s3obj.send_file(fp)
def uploadImage(imageFile):
try:
# create unqiue name and get extension
unique_name = str(uuid.uuid1())
fileName, fileExtension = os.path.splitext(imageFile.name)
# validate fileExtension
if not re.match('\.(gif|png|jpg|jpeg)$', fileExtension.lower()):
raise Exception
# get bucket and upload image file
bucket = getS3Bucket()
key = Key(bucket)
key.key = unique_name + fileExtension
key.size = imageFile.size # size must be set in order for this to work
key.send_file(imageFile)
return True, unique_name
except Exception, e:
print e
return False, None
def uploadImage(imageFile):
try:
# create unqiue name and get extension
unique_name = str(uuid.uuid1())
fileName, fileExtension = os.path.splitext(imageFile.name)
# validate fileExtension
if not re.match('\.(gif|png|jpg|jpeg)$', fileExtension.lower()):
raise Exception
# get bucket and upload image file
bucket = getS3Bucket()
key = Key(bucket)
key.key = unique_name + fileExtension
key.size = imageFile.size # size must be set in order for this to work
key.send_file(imageFile)
return True, unique_name
except Exception, e:
print e
return False, None
else : # hadoop
try :
hadoopName = s3FileName
if (not hadoopName.startswith(getHadoopDir())) :
if (not hadoopName.startswith("/")) :
hadoopName = "/"+hadoopName
hadoopName = getHadoopDir()+hadoopName
cmd = getHadoopBinary() + " dfs -ls "+hadoopName+" 2> /dev/null" # ignore "file not found" messages
debug(cmd)
lines = os.popen(cmd).readlines()
testKey = None
for t in lines :
if ( t.find(hadoopName) > -1 ) :
items = t.split()
testKey = Key()
testKey.size = int(items[4])
result = 0
except Exception, e :
log( e )
testKey = None
result = 0
if ( None != testKey ):
# file is on S3 or hadoop
# if we have a local copy, compare against it
needsWrite = False # probably won't need to write
debug( "ok -- file found on target system" )
# we have a local file to compare against
def test_key_with_files_and_callback(self):
"""
test simple key 'from_file' and 'to_file' functions
"""
def _archive_callback(bytes_sent, total_bytes):
print("archived {0} out of {1}".format(bytes_sent, total_bytes))
def _retrieve_callback(bytes_sent, total_bytes):
print("retrieved {0} out of {1}".format(bytes_sent, total_bytes))
log = logging.getLogger("test_key_with_files")
key_name = "A" * 1024
test_file_path = os.path.join(test_dir_path,
"test_key_with_files-orignal")
test_file_size = 1024**2
buffer_size = 1024
log.debug("writing {0} bytes to {1}".format(test_file_size,
test_file_path))
bytes_written = 0
with open(test_file_path, "wb") as output_file:
while bytes_written < test_file_size:
output_file.write(os.urandom(buffer_size))
bytes_written += buffer_size
# create the bucket
bucket = self._s3_connection.create_unique_bucket()
self.assertTrue(bucket is not None)
# create an empty key
write_key = Key(bucket)
# set the name
write_key.name = key_name
self.assertFalse(write_key.exists())
# upload some data
with open(test_file_path, "rb") as archive_file:
write_key.set_contents_from_file(archive_file,
cb=_archive_callback)
self.assertTrue(write_key.exists())
# create another key with the same name
read_key = Key(bucket, key_name)
# read back the data
retrieve_file_path = os.path.join(test_dir_path,
"test_key_with_files-orignal")
# 2011-08-08 dougfort boto aborts if you don't tell it the size
read_key.size = test_file_size
with open(retrieve_file_path, "wb") as retrieve_file:
read_key.get_contents_to_file(retrieve_file, cb=_retrieve_callback)
self.assertTrue(
filecmp.cmp(test_file_path, retrieve_file_path, shallow=False))
# delete the key
read_key.delete()
self.assertFalse(write_key.exists())
# delete the bucket
self._s3_connection.delete_bucket(bucket.name)
