def test_x_translation():
t = Transform()
p = np.array([
[0, 0, 0, 1],
], dtype="float64")
expect_p = np.array([
[1, 0, 0, 1],
], dtype="float64")
t.set_translation(x=1)
new_p = p @ t.view_matrix
assert np.all(new_p == expect_p), "Translation for X axis does not correct"
def test_all_translation():
t = Transform()
p = np.array([
[0, 0, 0, 1],
], dtype="float64")
expect_p = np.array([
[100, 12, -2, 1],
], dtype="float64")
t.set_translation(x=100, y=12, z=-2)
new_p = p @ t.view_matrix
assert np.all(
new_p == expect_p), "Translation for ALL axis does not correct"
def test_transform_raises_MissingDatSetError_when_one_dataset_given(self):
"""
When the transformer class receives only one dataset, exception should be raised
"""
transformer = Transform(
Extract.from_files(Constants._NYT_DATA_GOOD).get_datasets())
self.assertRaises(MissingDatasetError, transformer.transform_data)
def test_transform_raises_InvalidDatasetError_when_column_is_missing(self):
"""
If a requied column is missing from a dataset, exception should be raised
"""
transformer = Transform(
Extract.from_files(
Constants._NYT_DATA_MISSING_COLUMN).get_datasets())
self.assertRaises(InvalidDatasetError, transformer.transform_data)
def test_transform_raises_InvalidDatasetError_when_date_cannot_be_parsed(
self):
"""
if a date cannot be parsed, exception should be raised
"""
transformer = Transform(
Extract.from_files(Constants._NYT_DATA_BAD_DATE,
Constants._JH_DATA_GOOD).get_datasets())
self.assertRaises(InvalidDatasetError, transformer.transform_data)
def test_transform_raises_InvalidDatasetError_when_dataset_does_not_contain_dict_records(
self):
"""
if dataset is not dict records, exception should be raised
"""
transformer = Transform([
'string record',
])
self.assertRaises(InvalidDatasetError, transformer.transform_data)
def main(config_file, console=False, debug=False):
"""
数据清洗
"""
with open(config_file, encoding='utf-8') as f:
config_data = yaml.load(f)
if 'joinSource' in config_data:
reader = JoinSourceReader(config_data['source'],config_data['joinSource'])
else:
reader = Reader(config_data['source'])
target = None
if not console:
target = Target(config_data['target'])
else:
target = Target({'type': 'console'})
rowsLimit = None # 从源数据中读取的行数
if 'rowsLimit' in config_data['target']:
rowsLimit = config_data['target']['rowsLimit']
transform = None
if 'transform' in config_data:
transform = Transform(config_data['transform'])
count = 0
finish_run = False
for row in reader.next():
count += 1
if transform is not None:
for row_new in transform.do(row):
target.write(row_new)
else:
target.write(row)
if rowsLimit is not None and count >= rowsLimit:
finish_run = True
target.finish() # 通知已经完成了
break
if not finish_run and rowsLimit is not None:
target.finish() # 至少应该被运行一次
def test_transform_with_valid_data_returns_correct_date_range(self):
"""
When two valid dataasets are input, the merged output should only
have rows for the dates in common across both inputs
JH input ranges from 2020-01-22 to 2020-02-03
NYT input ranges from 2020-01-21 to 2020-02-03
Valid range therefore 2020-01-22 to 2020-02-03
"""
expected_min_date = date(2020, 1, 22)
expected_max_date = date(2020, 2, 3)
transformer = Transform(
Extract.from_files(Constants._NYT_DATA_GOOD,
Constants._JH_DATA_GOOD).get_datasets())
merged_data = transformer.transform_data()
min_date = min([d['date'] for d in merged_data])
max_date = max([d['date'] for d in merged_data])
assert min_date == expected_min_date and max_date == expected_max_date
def ingest(self, data):
transform = Transform()
# loop over all events inside data
for dic in data:
# for each item print out what type of event it is
event_type = (dic.get("type"))
# For every event type create new model
if event_type == "CUSTOMER":
transform.transform_before_loading_customer(dic)
elif event_type == "SITE_VISIT":
transform.transform_before_loading_site_visit(dic)
elif event_type == "IMAGE":
transform.transform_before_loading_image(dic)
elif event_type == "ORDER":
transform.transform_before_loading_order(dic)
from src.capture import Capture
from src.config import Config
from src.detect import Detect
from src.display import Display
from src.label import Label
from src.save import Save
from src.transform import Transform
from sys import argv
if len(argv) < 2:
print('You must specify a name argument.')
else:
config = Config().get()
capture = Capture(config)
trans = Transform(config)
display = Display()
detect = Detect()
label = Label()
with capture, display:
key = ''
while key != 'q':
success = False
image = capture.frame()
disk = trans.image_copy(image)
disk = trans.image_color_flip(disk)
small = trans.scale_image(image)
locs = detect.locations(small)
if len(locs) == 1:
success = True
from src.config import Config
from src.detect import Detect
from src.encode import Encode
from src.kisi import Kisi
from src.local_camera import LocalCamera
from src.repeat import Repeat
from src.transform import Transform
from datetime import datetime
config = Config().get()
camera = LocalCamera(config)
capture = Capture(config, camera)
encode = Encode(config)
kisi = Kisi(config)
repeat = Repeat(config)
trans = Transform(config)
detect = Detect()
refs, ref_map = encode.images()
print('\n')
with capture:
try:
while True:
image = capture.frame()
small = trans.scale_image(image)
_, encs = detect.all(small)
if len(encs) == 1:
lbl = False
cmps = detect.compare(refs, encs[0])
for i in range(len(cmps)):
def test_transform_raises_MissingDatSetError_when_no_datasets_given(self):
"""
When the transformer class receives no data, exception should be raised
"""
transformer = Transform([])
self.assertRaises(MissingDatasetError, transformer.transform_data)
from src.capture import Capture
from src.config import Config
from src.detect import Detect
from src.display import Display
from src.encode import Encode
from src.label import Label
from src.transform import Transform
config = Config().get()
capture = Capture(config)
encode = Encode(config)
trans = Transform(config)
detect = Detect()
display = Display()
label = Label()
refs, ref_map = encode.images()
with capture, display:
while display.key() != 'q':
image = capture.frame()
label.set_image(image)
small = trans.scale_image(image)
locs, encs = detect.all(small)
for i in range(len(encs)):
lbl = False
enc = encs[i]
loc = locs[i]
cmps = detect.compare(refs, enc)
for j in range(len(cmps)):