def test_writting_dataframe_with_records(self, pandas_dataframe_with_data):
buffer = StringIO()
csv_writer = CsvFormatter(specification={})
csv_writer.format(dataframe=pandas_dataframe_with_data,
path_or_buffer=buffer)
assert isinstance(buffer.getvalue(), str) is True
assert len(buffer.getvalue()) > 0
def test_writting_dataframe_without_records(self,
pandas_dataframe_without_data,
specification):
test_file_path = specification['uri']
csv_writer = CsvFormatter(specification={"options": {"index": False}})
csv_writer.format(dataframe=pandas_dataframe_without_data,
path_or_buffer=test_file_path)
assert exists(test_file_path) is False
assert isfile(test_file_path) is False
def test_date_format_iso(self, dataframe_with_datetime):
buffer = StringIO()
spec = {'options': {'date_format': 'iso'}}
csv_writer = CsvFormatter(specification=spec)
csv_writer.format(dataframe=dataframe_with_datetime,
path_or_buffer=buffer)
csv_text = buffer.getvalue()
assert isinstance(csv_text, str) is True
assert len(csv_text) > 0
for date_ in dataframe_with_datetime['at'].tolist():
assert date_.isoformat() in csv_text
def test_without_url_and_valid_input(self,
mocker,
pandas_dataframe_without_data,
gcs_without_url,
specification_url_gcs):
expected = specification_url_gcs['uri']
response = requests.Response()
response.status_code = 200
response.url = expected
mock_put = mocker.patch.object(requests, 'put')
mock_put.return_value = response
mock_input = mocker.patch('builtins.input')
mock_input.return_value = expected
formatter = CsvFormatter(specification={})
writer = \
GCSPresignedUrlRemoteWriter(formatter=formatter,
specification=gcs_without_url)
writer.before_write()
signed_url = writer.write(dataframe=pandas_dataframe_without_data)
mock_put.assert_called()
assert signed_url == "gs://"
def test_writting_dataframe_without_records(self,
pandas_dataframe_without_data,
specification):
formatter = CsvFormatter(specification={})
writer = FileWriter(formatter=formatter, specification=specification)
writer.write(dataframe=pandas_dataframe_without_data)
assert exists(specification['uri']) is False
assert isfile(specification['uri']) is False
def test_writting_raised(self,
mocker,
pandas_dataframe_without_data,
specification_url_gcs):
formatter = CsvFormatter(specification={})
writer = \
GCSPresignedUrlRemoteWriter(formatter=formatter,
specification=specification_url_gcs)
with pytest.raises(requests.RequestException):
writer.before_write()
writer.write(dataframe=pandas_dataframe_without_data)
def test_writting_dataframe_with_records(self, pandas_dataframe_with_data,
specification):
test_file_path = specification['uri']
csv_writer = CsvFormatter(specification={"options": {"index": False}})
csv_writer.format(dataframe=pandas_dataframe_with_data,
path_or_buffer=test_file_path)
assert exists(test_file_path) is True
assert isfile(test_file_path) is True
with open(test_file_path) as csv_file:
csv_reader = reader(csv_file, delimiter=',')
csv_reader = [x for x in csv_reader]
headers = csv_reader[0]
rows = csv_reader[1:len(csv_reader)]
csv_reader = {
headers[index]: [row[index] for row in rows]
for index in range(0, len(headers))
}
expected = pandas_dataframe_with_data.to_dict(orient="list")
assert csv_reader == expected
remove(test_file_path)
assert exists(test_file_path) is False
def test_writting_csv_without_records(self,
mocker,
pandas_dataframe_without_data,
specification_url_gcs):
mock = mocker.patch.object(requests, 'put')
resp = requests.Response()
resp.status_code = 200
mock.return_value = resp
formatter = CsvFormatter(specification={})
writer = \
GCSPresignedUrlRemoteWriter(formatter=formatter,
specification=specification_url_gcs)
writer.before_write()
writer.write(dataframe=pandas_dataframe_without_data)
mock.assert_called()
def test_writting_csv_without_records(self, pandas_dataframe_without_data,
specification_s3):
with mock_s3():
bucket = specification_s3['options']['bucket']
key = specification_s3['options']['key']
conn = boto3.resource('s3', region_name='us-east-1')
conn.create_bucket(Bucket=bucket)
formatter = CsvFormatter(specification={})
writer = S3RemoteWriter(formatter=formatter,
specification=specification_s3)
writer.write(dataframe=pandas_dataframe_without_data)
object_ = conn.Object(bucket, key).get()
assert object_['ContentLength'] == 0
def test_writting_csv_with_records(self, mocker, specification_with_url,
pandas_dataframe_with_data,
signed_post_file):
mock_post = mocker.patch.object(requests, 'post')
mock_post.return_value.raise_for_status = lambda: None
with patch('builtins.open',
mock_open(read_data=signed_post_file)) as mock_file:
formatter = CsvFormatter(specification={})
writer = S3PresignedUrlRemoteWriter(
formatter=formatter, specification=specification_with_url)
signed_url = writer.write(dataframe=pandas_dataframe_with_data)
mock_post.assert_called()
mock_file.assert_called()
expected = json.loads(signed_post_file)['url']
assert signed_url == expected
def test_writting_csv_with_records(self, pandas_dataframe_with_data,
specification_s3):
with mock_s3():
bucket = specification_s3['options']['bucket']
key = specification_s3['options']['key']
conn = boto3.resource('s3', region_name='us-east-1')
conn.create_bucket(Bucket=bucket)
formatter = CsvFormatter(specification={})
writer = S3RemoteWriter(formatter=formatter,
specification=specification_s3)
writer.write(dataframe=pandas_dataframe_with_data)
body = conn.Object(bucket, key).get()['Body']
csv_reader = pd.read_csv(body).to_dict(orient="list")
expected = pandas_dataframe_with_data.to_dict(orient="list")
assert csv_reader == expected
def test_without_url_and_invalid_input(self, mocker,
pandas_dataframe_without_data,
specification_without_url,
signed_post_file):
expected = json.loads(signed_post_file)['url']
mock_post = mocker.patch.object(requests, 'post')
mock_post.return_value.raise_for_status = lambda: None
mock_input = mocker.patch('builtins.input')
mock_input.return_value = expected
mock_input = mocker.patch('os.path.isfile')
mock_input.return_value = False
with patch('builtins.open',
mock_open(read_data=signed_post_file)) as mock_file:
formatter = CsvFormatter(specification={})
writer = S3PresignedUrlRemoteWriter(
formatter=formatter, specification=specification_without_url)
with raises(ValueError):
writer.before_write()
mock_post.assert_called()
mock_file.assert_called()
def test_writting_csv_with_records(self, pandas_dataframe_with_data,
specification):
formatter = CsvFormatter(specification={})
writer = FileWriter(formatter=formatter, specification=specification)
writer.write(dataframe=pandas_dataframe_with_data)
assert exists(specification['uri']) is True
assert isfile(specification['uri']) is True
with open(specification['uri']) as csv_file:
csv_reader = reader(csv_file, delimiter=',')
csv_reader = [x for x in csv_reader]
headers = csv_reader[0]
rows = csv_reader[1:len(csv_reader)]
csv_reader = {
headers[index]: [row[index] for row in rows]
for index in range(0, len(headers))
}
expected = pandas_dataframe_with_data.to_dict(orient="list")
assert csv_reader == expected
remove(specification['uri'])
assert exists(specification['uri']) is False