def test_parse_jsons_from_folder():
s3_client = boto3.client("s3")
log.info("Downloading zipped jsons.")
name = "threat_eg_jsons"
tmp_filepath = "/tmp/" + name + ".zip"
s3_client.download_file("cvapis-data", "jsons/" + name + ".zip",
tmp_filepath)
log.info("Zipped jsons downloaded.")
with open(tmp_filepath, "rb") as f:
log.info("Unzipping it.")
z = zipfile2.ZipFile(f)
for name_local in z.namelist():
print(" Extracting file", name_local)
z.extract(name_local, "/tmp/")
jsons_path = "/tmp/" + name
log.info("Unzipped to" + jsons_path)
log.info("Parrsing jsons in " + jsons_path)
all_jsons = os.listdir(jsons_path)
dets_per_media = {}
server_name = "ThreatClassifier"
for ind, single_json_rel in enumerate(all_jsons):
single_json = jsons_path + "/" + single_json_rel
log.info("Parsing: " + single_json)
avro_io = AvroIO()
x = AvroAPI(avro_io.read_json(single_json))
p = {}
p["server"] = server_name
props = [p]
dets = x.get_detections_from_props(props)
url = x.get_url()
dets_per_media[url] = dets
log.info("\n\n\n")
log.info("url: " + url)
log.info("dets: " + str(dets))
def test_create_avro_bin_w_registry():
avro_api = AvroAPI()
avro_api.append_image_ann(image_ann)
avro_api.set_url("blah")
avro_io = AvroIO()
avro_io.write(avro_api.get_response(), avro_bin)
assert True
def test_read_avro_json():
s3_client = boto3.client("s3")
log.info("Downloading json.")
tmp_filepath = "/tmp/Threatpipeline_response0_C.json"
s3_client.download_file("cvapis-data",
"jsons/Threatpipeline_response0_C.json",
tmp_filepath)
log.info("JSON downloaded.")
avro_io = AvroIO()
avro_str = avro_io.read_json(tmp_filepath)
# log.info("avro_str: " + str(avro_str))
assert len(avro_str) > 0
x = AvroAPI(avro_str)
def test_query_properties():
s3_client = boto3.client("s3")
log.info("Downloading json.")
tmp_filepath = "/tmp/Threatpipeline_response0_C.json"
s3_client.download_file("cvapis-data",
"jsons/Threatpipeline_response0_C.json",
tmp_filepath)
log.info("JSON downloaded.")
avro_io = AvroIO()
avro_str = avro_io.read_json(tmp_filepath)
# log.info("avro_str: " + str(avro_str))
assert len(avro_str) > 0
x = AvroAPI(avro_str)
p = {}
p["server"] = "ThreatClassifier"
props = [p]
dets = x.get_detections_from_props(props)
assert len(dets) == 1
log.info("dets: " + str(dets))
def test_get_detections_from_frame_anns():
s3_client = boto3.client("s3")
log.info("Downloading json.")
tmp_filepath = (
"/tmp/NHL_GAME_VIDEO_WPGOTT_M2_HOME_20180402_1520698435976.t.mp4.json")
s3_client.download_file(
"cvapis-data",
"jsons/NHL_GAME_VIDEO_WPGOTT_M2_HOME_20180402_1520698435976.t.mp4.json",
tmp_filepath,
)
log.info("JSON downloaded.")
x = AvroAPI(AvroIO.read_json(tmp_filepath))
print(json.dumps(x.get_detections_from_frame_anns(), indent=2))
def test_create_avro_bin():
avro_api = AvroAPI()
avro_api.append_image_ann(image_ann)
avro_api.set_url("blah")
avro_io = AvroIO()
response = avro_api.get_response()
avro_io.write(response, avro_bin)
flag = AvroIO.is_valid_avro_doc_static(response, avro_io.get_schema())
assert flag
def test_append_annotation_task():
avro_api = AvroAPI()
avro_api.append_image_ann(image_ann)
avro_api.set_url("blah")
avro_api.append_annotation_task(annotation_task_1)
avro_io = AvroIO()
response = avro_api.get_response()
# print(json.dumps(response))
# avro_io.encode_to_file(response, avro_bin)
flag = AvroIO.is_valid_avro_doc_static(response, avro_io.get_schema())
assert flag
AvroIO.write_json(response, annotation_task_1["id"] + ".json", 2)
def test_is_valid_avro_doc():
avroio = AvroIO()
doc = avroio.decode_file(temp_file)
assert avroio.is_valid_avro_doc(doc)
def test_remote_decode():
avroio = AvroIO(use_schema_registry=True)
doc = avroio.decode_file(temp_file)
print(json.dumps(doc, indent=2))
assert True
def test_remote_encode():
avroio = AvroIO(use_schema_registry=True)
assert avroio.write(AvroIO.read_json(local_json), temp_file)
def test_local_decode():
avroio = AvroIO()
res = avroio.decode_file(temp_file)
assert True
def test_local_encode():
avroio = AvroIO()
assert avroio.write(AvroIO.read_json(local_json), temp_file)
def test_create_avro_str():
avro_api = AvroAPI()
avro_api.append_image_ann(image_ann)
avro_api.set_url("blah")
AvroIO.write_json(avro_api.get_response(), avro_str, indent=2)
assert True
def test_append_trackssummary_from_csv():
jsons_folder = "../cvapis/data/jsons/"
# input_json_filepath=jsons_folder + 'Tampa-Bay-Lightning-vs-Montreal-Canadians_03102018_112_108.json'
# input_csv_filepath=jsons_folder + 'Tampa-Bay-Lightning-vs-Montreal-Canadians_03102018_112_108_GT.csv'
input_json_filepath = (
jsons_folder +
"NHL_GAME_VIDEO_ANACHI_M2_HOME_20182016_1518124767584.json")
input_csv_filepath = (
jsons_folder +
"NHL_GAME_VIDEO_ANACHI_M2_HOME_20182016_1518124767584.csv")
assert os.path.exists(input_json_filepath)
assert os.path.exists(input_csv_filepath)
filename = os.path.basename(input_json_filepath)
filename = os.path.splitext(filename)[0]
output_json_filepath = jsons_folder + filename + "_With_Human_Tracks.json"
# we load the input json
avro_api = AvroAPI(AvroIO.read_json(input_json_filepath))
# we parse the csv
cois = ["Sponsor", "Placement", "Start Timestamp", "Stop Timestamp"]
isponsor = -1
iplacement = -1
it1 = -1
it2 = -1
with open(input_csv_filepath) as csvfile:
readCSV = csv.reader(csvfile, delimiter=",")
titles = []
placements = []
sponsors = []
t1s = []
t2s = []
for i, row in enumerate(readCSV):
if i == 0:
titles = row
log.info("titles: " + str(titles))
log.info("Getting indexes of interest for columns named: " +
str(cois))
isponsor = titles.index(cois[0])
iplacement = titles.index(cois[1])
it1 = titles.index(cois[2])
it2 = titles.index(cois[3])
log.info("indexes: " + str(isponsor) + ", " + str(iplacement) +
", " + str(it1) + ", " + str(it2))
else:
placements.append(row[iplacement])
sponsors.append(row[isponsor])
t1aux = row[it1]
t2aux = row[it2]
if t1aux.find(":") == 1:
c1 = t1aux.split(":")
c2 = t2aux.split(":")
t1 = int(c1[0]) * 3600 + int(c1[1]) * 60 + int(c1[2])
t2 = int(c2[0]) * 3600 + int(c2[1]) * 60 + int(c2[2])
else: # we assume its seconds
t1 = int(t1aux)
t2 = int(t2aux)
t1s.append(t1)
t2s.append(t2)
log.info("placements[0:9]: " + str(placements[0:9]))
log.info("sponsors[0:9]: " + str(sponsors[0:9]))
log.info("t1s[0:9]: " + str(t1s[0:9]))
log.info("t2s[0:9]: " + str(t2s[0:9]))
log.info("placements[-9:]: " + str(placements[-9:]))
log.info("sponsors[-9:]: " + str(sponsors[-9:]))
log.info("t1s[-9:]: " + str(t1s[-9:]))
log.info("t2s[-9:]: " + str(t2s[-9:]))
for t1, t2, placement, sponsor in zip(t1s, t2s, placements, sponsors):
# we create the placecement property
p1 = create_prop(
confidence=1,
ver="1.0",
server="HAM",
property_type="placement",
value=placement,
)
# we create the sponsor property
p2 = create_prop(
confidence=1,
ver="1.0",
server="HAM",
property_type="sponsor",
value=sponsor,
)
ps = [p1, p2]
# We create the track
track = create_video_ann(t1=t1, t2=t2, props=ps)
avro_api.append_track_to_tracks_summary(track)
AvroIO.write_json(avro_api.get_response(), output_json_filepath, indent=2)
assert True
print("done")
def test_read_avro_bin():
assert os.path.exists(avro_bin)
avro_io = AvroIO()
avro_str = avro_io.decode_file(avro_bin)
assert avro_str