def main(automan_info, archive_info, storage_type, storage_info):
automan_info = json.loads(automan_info)
archive_info = json.loads(archive_info)
archive_dir = archive_info['archive_dir'].rstrip('/') + '/'
storage_client = StorageClientFactory.create(
storage_type,
json.loads(storage_info),
archive_info
)
AutomanArchiver.archive(automan_info, archive_info)
shutil.make_archive(
archive_dir + archive_info['archive_name'],
'gztar',
root_dir=TEMP_DIR)
if storage_type == 'AWS_S3':
storage_client.upload(automan_info, archive_dir)
# TODO post : ArchiviedLabelDataset
data = {
'file_path': archive_info['archive_dir'],
'file_name': archive_info['archive_name'] + '.tar.gz',
'annotation_id': archive_info['annotation_id'],
}
AutomanClient.send_result(automan_info, data)
def __get_frame_range(automan_info, project_id, annotation_id):
path = '/projects/' + str(project_id) + '/annotations/' + str(
annotation_id) + '/'
res = AutomanClient.send_get(automan_info, path).json()
dataset_path = '/projects/' + str(project_id) + '/datasets/' + str(
res['dataset_id']) + '/'
dataset = AutomanClient.send_get(automan_info, dataset_path).json()
return dataset['frame_count']
def __get_candidates(automan_info, project_id, original_id):
path = '/projects/' + str(project_id) + '/originals/' + str(original_id) + '/candidates/'
res = AutomanClient.send_get(automan_info, path).json()
candidates = []
for record in res['records']:
ext = '.jpg' if record['data_type'] == 'IMAGE' else '.pcd'
candidates.append({'id': record['candidate_id'], 'ext': ext})
return candidates
def __get_annotation_color(automan_info, project_id):
path = '/projects/' + str(project_id) + '/'
res = AutomanClient.send_get(automan_info, path).json()
colors = {}
for record in res['klassset']['records']:
config = json.loads(record['config'])
colors[record['name']] = config['color']
return colors
def __get_candidates(automan_info, project_id, original_id):
path = '/projects/' + str(project_id) + '/originals/' + str(
original_id) + '/candidates/'
res = AutomanClient.send_get(automan_info, path).json()
candidate_ids = []
for record in res['records']:
candidate_ids.append(record['candidate_id'])
return candidate_ids
def __get_annotation(automan_info, project_id, annotation_id, frame):
path = '/projects/' + str(project_id) + '/annotations/' + str(annotation_id) \
+ '/frames/' + str(frame) + '/objects/'
res = AutomanClient.send_get(automan_info, path).json()
# TODO format to "kitti format"
with open(TEMP_DIR + '/Annotations/' + str(frame).zfill(6) + '.json',
mode='w') as frame:
frame.write(json.dumps(res))
return res
def __get_annotation(automan_info, project_id, annotation_id, frame, annotations_dir):
path = '/projects/' + str(project_id) + '/annotations/' + str(annotation_id) \
+ '/frames/' + str(frame) + '/objects/'
res = AutomanClient.send_get(automan_info, path).json()
# TODO format to "kitti format"
# ensure directory
os.makedirs(annotations_dir, exist_ok=True)
with open(os.path.join( annotations_dir, str(frame).zfill(6) + '.json'), mode='w') as frame:
frame.write(json.dumps(res))
return res
def __get_candidates(automan_info, project_id, original_id):
path = '/projects/' + str(project_id) + '/originals/' + str(
original_id) + '/candidates/'
res = AutomanClient.send_get(automan_info, path).json()
candidates = []
topics = []
for c in res["records"]:
analyzed_info = json.loads(c['analyzed_info'])
candidate = {
'candidate_id': c["candidate_id"],
'msg_type': analyzed_info['msg_type'],
'topic_name': analyzed_info['topic_name']
}
candidates.append(candidate)
topics.append(analyzed_info['topic_name'])
return candidates, topics
def upload(self, automan_info):
jpg = glob.glob(self.extract_path+'*.jpg')
pcd = glob.glob(self.extract_path+'*.pcd')
for filepath in jpg + pcd:
name = os.path.split(filepath)[1]
data = {
'storage_id': str(self.storage_id),
'key': self.extract_path + name}
res = AutomanClient.send_result(
automan_info, data, automan_info['presigned']).text
presigned = json.loads(res)
headers = {'content-type': 'application/octet-stream'}
res = requests.put(
presigned['url'],
headers=headers,
data=open(filepath, 'rb')
)
if res.status_code != 204:
print('status_code=' + str(res.status_code) + ': ' + res.text)
def __get_annotation_image(automan_info, project_id, dataset_id,
candidate_id, frame):
path = '/projects/' + str(project_id) + '/datasets/' + str(dataset_id) \
+ '/candidates/' + str(candidate_id) + '/frames/' + str(frame) + '/'
img_url = AutomanClient.send_get(automan_info, path).json()
headers = {
'Authorization': 'JWT ' + automan_info['jwt'],
}
res = requests.get(img_url, headers=headers)
if res.status_code != 200:
return None
ext = '.jpg' if res.headers['Content-Type'] == 'image/jpeg' else '.pcd'
file_name = str(candidate_id) + '_' + str(frame).zfill(6) + ext
img_path = TEMP_DIR + '/Images/' + file_name
with open(img_path, mode='wb') as frame:
frame.write(res.content)
if ext == '.jpg':
return file_name
return None
def upload(self, automan_info, upload_dir=None, ext=''):
if upload_dir is None:
upload_dir = self.output_path
archive = glob.glob(self.output_path + '*' + ext)
for filepath in archive:
name = os.path.split(filepath)[1]
data = {
'storage_id': str(self.storage_id),
'key': self.output_path + name
}
res = AutomanClient.send_result(automan_info, data,
automan_info['presigned']).text
presigned = json.loads(res)
headers = {'content-type': 'application/octet-stream'}
res = requests.put(presigned['url'],
headers=headers,
data=open(filepath, 'rb'))
if 200 > res.status_code >= 300:
print(
f's3 upload status_code = {res.status_code}. body = {res.text}'
)
def __get_annotation_image(automan_info, project_id, dataset_id, candidate_id, frame, ext, images_dir):
path = '/projects/' + str(project_id) + '/datasets/' + str(dataset_id) \
+ '/candidates/' + str(candidate_id) + '/frames/' + str(frame) + '/'
img_url = AutomanClient.send_get(automan_info, path).text
if re.search(automan_info['host'], img_url):
headers = {
'Authorization': 'JWT ' + automan_info['jwt'],
}
else:
headers = {}
res = requests.get(img_url, headers=headers)
if 200 > res.status_code >= 300:
print(f'get annotation image status_code = {res.status_code}. body = {res.text}')
return None
# write images
os.makedirs(images_dir, exist_ok=True)
file_name = str(candidate_id) + '_' + str(frame).zfill(6) + ext
img_path = os.path.join(images_dir, file_name)
with open(img_path, mode='wb') as frame:
frame.write(res.content)
if ext == '.jpg':
return file_name
return None
cv2.imwrite(file_path + ".jpg", image,
[int(cv2.IMWRITE_JPEG_QUALITY), 100])
@staticmethod
def __parse_calib(calib_path):
fs = cv2.FileStorage(calib_path, cv2.FILE_STORAGE_READ)
camera_extrinsic_mat = fs.getNode("CameraExtrinsicMat").mat()
camera_mat = fs.getNode("CameraMat").mat()
dist_coeff = np.transpose(fs.getNode("DistCoeff").mat())
return camera_extrinsic_mat, camera_mat, dist_coeff
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--storage_type', required=True)
parser.add_argument('--storage_info', required=True)
parser.add_argument('--automan_info', required=True)
parser.add_argument('--raw_data_info', required=True)
args = parser.parse_args()
storage_client = StorageClientFactory.create(args.storage_type,
json.loads(args.storage_info))
storage_client.download()
path = storage_client.get_input_path()
output_dir = storage_client.get_output_dir()
os.makedirs(output_dir)
res = RosbagExtractor.extract(json.loads(args.automan_info), path, [],
output_dir, json.loads(args.raw_data_info))
AutomanClient.send_result(json.loads(args.automan_info), res)
if info.msg_type in MSG_DATA_TYPE_MAP.keys():
candidate = {
"analyzed_info": {
"topic_name": topic_name,
"msg_type": info.msg_type,
},
"data_type": MSG_DATA_TYPE_MAP[info.msg_type],
"frame_count": info.message_count
}
dataset_candidates.append(candidate)
return dataset_candidates
except Exception as e:
# FIXME
raise (e)
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--storage_type', required=True)
parser.add_argument('--storage_info', required=True)
parser.add_argument('--automan_info', required=True)
args = parser.parse_args()
storage_client = StorageClientFactory.create(args.storage_type,
json.loads(args.storage_info))
storage_client.download()
path = storage_client.get_local_path()
results = RosbagAnalyzer.analyze(path)
AutomanClient.send_analyzer_result(json.loads(args.automan_info), results)
camera_extrinsic_mat = fs.getNode("CameraExtrinsicMat").mat()
camera_mat = fs.getNode("CameraMat").mat()
dist_coeff = np.transpose(fs.getNode("DistCoeff").mat())
return camera_extrinsic_mat, camera_mat, dist_coeff
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--storage_type', required=True)
parser.add_argument('--storage_info', required=True)
parser.add_argument('--automan_info', required=True)
parser.add_argument('--raw_data_info', required=True)
args = parser.parse_args()
automan_info = json.loads(args.automan_info)
print('automan_info: ' + args.automan_info)
print('storage_info: ' + args.storage_info)
storage_client = StorageClientFactory.create(args.storage_type,
json.loads(args.storage_info))
storage_client.download()
path = storage_client.get_input_path()
output_dir = storage_client.get_output_dir()
if not os.path.exists(output_dir):
os.makedirs(output_dir)
res = RosbagExtractor.extract(automan_info, path, [], output_dir,
json.loads(args.raw_data_info))
if args.storage_type == 'AWS_S3':
storage_client.upload(automan_info)
res = AutomanClient.send_result(automan_info, res)
print(res)
colors = {}
for record in res['klassset']['records']:
config = json.loads(record['config'])
colors[record['name']] = config['color']
return colors
if __name__ == '__main__':
parser = argparse.ArgumentParser()
# FIXME parser.add_argument('--storage_type', required=True)
# FIXME parser.add_argument('--storage_info', required=True)
parser.add_argument('--automan_info', required=True)
parser.add_argument('--archive_info', required=True)
args = parser.parse_args()
automan_info = json.loads(args.automan_info)
archive_info = json.loads(args.archive_info)
AutomanArchiver.archive(automan_info, archive_info)
shutil.make_archive(archive_info['archive_dir'] + '/' +
archive_info['archive_name'],
'gztar',
root_dir=TEMP_DIR)
# TODO post : ArchiviedLabelDataset
data = {
'file_path': archive_info['archive_dir'],
'file_name': archive_info['archive_name'] + '.tar.gz',
'annotation_id': archive_info['annotation_id'],
}
AutomanClient.send_result(automan_info, data)
