def test_run(self):
client = S3Client(AWS_ACCESS_KEY, AWS_SECRET_KEY)
bucket = client.s3.create_bucket(BUCKET)
for key in FILES:
k = Key(bucket)
k.key = '%s/%s' % (KEY, key)
k.set_contents_from_string('')
folder_path = 's3://%s/%s' % (BUCKET, KEY)
k = Key(bucket)
k.key = 'manifest'
path = 's3://%s/%s/%s' % (BUCKET, k.key, 'test.manifest')
folder_paths = [folder_path]
t = redshift.RedshiftManifestTask(path, folder_paths)
luigi.build([t], local_scheduler=True)
output = t.output().open('r').read()
expected_manifest_output = json.dumps(generate_manifest_json(folder_paths, FILES))
self.assertEqual(output, expected_manifest_output)
def test_run(self):
with mock_s3():
client = S3Client()
client.s3.meta.client.create_bucket(Bucket=BUCKET)
for key in FILES:
k = '%s/%s' % (KEY, key)
client.put_string('', 's3://%s/%s' % (BUCKET, k))
folder_path = 's3://%s/%s' % (BUCKET, KEY)
path = 's3://%s/%s/%s' % (BUCKET, 'manifest', 'test.manifest')
folder_paths = [folder_path]
m = mock.mock_open()
with mock.patch('luigi.contrib.s3.S3Target.open', m, create=True):
t = redshift.RedshiftManifestTask(path, folder_paths)
luigi.build([t], local_scheduler=True)
expected_manifest_output = json.dumps(
generate_manifest_json(folder_paths, FILES))
handle = m()
handle.write.assert_called_with(expected_manifest_output)