def download(self): ensure_dir(self.target_dir) log.info('Downloading the train images from baidu...') log.warning('You might want to download using a better client') test_zip = os.path.join(self.target_dir, TRAIN_DATA[0]) self.download_file_from_google_drive(TRAIN_DATA[1], test_zip) self.extract_zip_file(test_zip, self.target_dir) log.info('downloading the train annotations from baidu...') test_zip = os.path.join(self.target_dir, TRAIN_ANNO[0]) self.download_file_from_google_drive(TRAIN_ANNO[1], test_zip) self.extract_zip_file(test_zip, self.target_dir) log.info('downloading the test images from google drive...') test_zip = os.path.join(self.target_dir, TEST_DATA[0]) self.download_file_from_google_drive(TEST_DATA[1], test_zip) self.extract_zip_file(test_zip, self.target_dir) log.info('downloading the test annotations from google drive...') test_zip = os.path.join(self.target_dir, TEST_ANNO[0]) self.download_file_from_google_drive(TEST_ANNO[1], test_zip) self.extract_zip_file(test_zip, self.target_dir) log.info('done')
def download(self): ensure_dir(self.target_dir) log.info('downloading the training images from google drive...') train_zip = os.path.join(self.target_dir, TRAIN_DATA[0]) self.download_file_from_google_drive(TRAIN_DATA[1], train_zip) self.extract_zip_file(train_zip, self.target_dir) log.info('downloading the validation images from google drive...') val_zip = os.path.join(self.target_dir, VAL_DATA[0]) self.download_file_from_google_drive(VAL_DATA[1], val_zip) self.extract_zip_file(val_zip, self.target_dir) log.info('downloading the bounding boxes annotations...') annotation_zip_file = self.download_file_from_web_server( ANNOTATIONS_URL, self.target_dir) self.extract_zip_file( os.path.join(self.target_dir, annotation_zip_file), self.target_dir) log.info('downloading eval tools...') tools_zip_file = self.download_file_from_web_server( EVAL_TOOLS_URL, self.target_dir) self.extract_zip_file(os.path.join(self.target_dir, tools_zip_file), self.target_dir) log.info('done')
def download(self): ensure_dir(self.target_dir) log.info('Downloading dataset...') zip_file = self.download_file_from_web_server(IMAGES_URL, self.target_dir) log.info('Finished download. Unzipping...') self.extract_zip_file(os.path.join(self.target_dir, zip_file), self.target_dir) log.info('done')
def export(self, target_dir): ensure_dir(target_dir) target_file = '{}_{}.json' output_filename = os.path.join( target_dir, target_file.format(self.dataset.name, 'train')) self._export(output_filename, self.dataset.train_set()) output_filename = os.path.join( target_dir, target_file.format(self.dataset.name, 'val')) self._export(output_filename, self.dataset.val_set())
def download(self): ensure_dir(self.target_dir) log.info('downloading the pascal faces from google drive...') train_zip = os.path.join(self.target_dir, TRAINVAL_DATA[0]) self.download_file_from_google_drive(TRAINVAL_DATA[1], train_zip) self.extract_tar_file(train_zip, self.target_dir) log.info('done')
def main(): parser = argparse.ArgumentParser() parser.add_argument('--input', dest='input', action='store', required=True, help='input csv containing the predictions') parser.add_argument('--data_dir', dest='data_dir', action='store', required=True, help='') parser.add_argument('--output_dir', dest='output_dir', action='store', required=True, help='') args = parser.parse_args() input_csv = args.input data_dir = args.data_dir output_dir = args.output_dir ensure_dir(output_dir) predictions = defaultdict(list) with open(input_csv, 'r') as csvfile: spamreader = csv.reader(csvfile, delimiter=' ') for row in spamreader: match = next(re.finditer(FILENAME_RE, row[0])) _id = match.group(1) score = float(row[1]) xmin = float(row[2]) ymin = float(row[3]) xmax = float(row[4]) ymax = float(row[5]) # left_x top_y width height detection_score predictions[_id].append( (xmin, ymin, xmax - xmin, ymax - ymin, score)) ds = FDDB(data_dir) for fold_id, fold_file in ds.folds(): target_file = os.path.join(output_dir, 'fold-{}-out.txt'.format(fold_id)) with open(fold_file, 'r') as src, open(target_file, 'w') as dest: for img_filename in src: img_filename = img_filename.strip() dest.write('{}\n'.format(img_filename)) if img_filename in predictions: pred = predictions[img_filename] dest.write('{}\n'.format(len(pred))) for p in pred: dest.write('{} {} {} {} {}\n'.format(*p)) else: dest.write('{}\n'.format(0))
def download(self): ensure_dir(self.target_dir) log.info('downloading the validation images from google drive...') val_zip = os.path.join(self.target_dir, VAL_DATA[0]) self.download_file_from_google_drive(VAL_DATA[1], val_zip) self.extract_zip_file(val_zip, self.target_dir) log.info('downloading the bounding boxes annotations...') anno_zip = os.path.join(self.target_dir, ANNOTATIONS_DATA[0]) self.download_file_from_google_drive(ANNOTATIONS_DATA[1], anno_zip) self.extract_zip_file(anno_zip, self.target_dir) log.info('done')
def _export(self, target_dir, dataset_name='train'): log.info('Converting %s data', dataset_name) images_root = os.path.join(target_dir, 'images/') annotations_root = os.path.join(target_dir, 'labels/') ensure_dir(annotations_root) with open(os.path.join(target_dir, '{}.txt'.format(dataset_name)), 'w') as full: for i in getattr(self.widerface, '{}_set'.format(dataset_name))(): if len(i.faces) > 0: path = i.copy_to(images_root, include_subdirs=True) full.write('{}\n'.format(path)) head, _ = os.path.splitext(path) head, tail = os.path.split(head) annotation_dir = os.path.join(annotations_root, i.subdir) if i.subdir else annotations_root ensure_dir(annotation_dir) annotation_file = os.path.join(annotation_dir, tail+'.txt') with open(annotation_file, 'w') as anno: for face in i.faces: bbox = self._convert(i.size, face) anno.write('0 ' + ' '.join([str(a) for a in bbox]) + '\n')
def download(self): ensure_dir(self.target_dir) log.info('Downloading images') annotation_zip_file = self.download_file_from_web_server( IMAGES_URL, self.target_dir) log.info('Finished download. Extracting.') self.extract_tar_file( os.path.join(self.target_dir, annotation_zip_file), os.path.join(self.target_dir, 'originalPics/')) log.info('Downloading annotations') annotation_zip_file = self.download_file_from_web_server( ANNOTATIONS_URL, self.target_dir) log.info('Finished download. Extracting.') self.extract_tar_file( os.path.join(self.target_dir, annotation_zip_file), self.target_dir) log.info('done')
def main(): parser = argparse.ArgumentParser() parser.add_argument('--input', dest='input', action='store', required=True, help='input tfrecord containing the predictions') parser.add_argument('--data_dir', dest='data_dir', action='store', required=True, help='') parser.add_argument('--output_dir', dest='output_dir', action='store', required=True, help='') args = parser.parse_args() input_file = args.input data_dir = args.data_dir output_dir = args.output_dir ensure_dir(output_dir) predictions = extract_predictions_from_tf_record(input_file) ds = FDDB(data_dir) for fold_id, fold_file in ds.folds(): target_file = os.path.join(output_dir, 'fold-{}-out.txt'.format(fold_id)) with open(fold_file, 'r') as src, open(target_file, 'w') as dest: for img_filename in src: img_filename = img_filename.strip() dest.write('{}\n'.format(img_filename)) if img_filename in predictions: pred = predictions[img_filename] dest.write('{}\n'.format(len(pred))) for p in pred: dest.write('{} {} {} {} {}\n'.format(*p)) else: dest.write('{}\n'.format(0))
def main(): parser = argparse.ArgumentParser() parser.add_argument('--input', dest='input', action='store', required=True, help='input tf record containing the predictions') parser.add_argument('--data_dir', dest='data_dir', action='store', required=True, help='') parser.add_argument('--output_dir', dest='output_dir', action='store', required=True, help='') args = parser.parse_args() tfrecord = args.input data_dir = args.data_dir output_dir = args.output_dir ds = Wider(data_dir) events = ds.events() for i in read_detections_from(tfrecord): event_id = filename.split('_')[0] event = events[event_id] item_name = os.path.splitext(os.path.basename(filename))[0] result_filename = '{}/{}.txt'.format(event, item_name) target_file = os.path.join(output_dir, result_filename) ensure_dir(target_file) with open(target_file, 'w') as dest: dest.write('{}\n'.format(item_name)) total = len(bboxes) dest.write('{}\n'.format(total)) for bbox in bboxes: dest.write('{} {} {} {} {}\n'.format(*bbox))
def _prepare(target_dir): log.info('Preparing target dir: %s', target_dir) ensure_dir(os.path.join(target_dir, 'images/')) ensure_dir(os.path.join(target_dir, 'labels/')) ensure_dir(os.path.join(target_dir, 'backup/')) ensure_dir(os.path.join(target_dir, 'results/')) log.info('Creating obj.names') with open(os.path.join(target_dir, 'obj.names'), 'w') as obj_names: obj_names.write('face\n') log.info('Creating obj.data') with open(os.path.join(target_dir, 'obj.data'), 'w') as obj_data: obj_data.write('classes = 1\n') obj_data.write('train = train.txt\n') obj_data.write('valid = val.txt\n') obj_data.write('names = obj.names\n') obj_data.write('backup = backup/\n')
import logging import sys import unittest import os from morghulis.afw import AFW from morghulis.os_utils import ensure_dir logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) log = logging.getLogger(__name__) AFW_DIR = os.path.dirname(__file__) + '/AFW_sample/' TMP_DIR = '/opt/project/.tmp/' ensure_dir(TMP_DIR) class AFWTests(unittest.TestCase): def setUp(self): self.afw = AFW(AFW_DIR) def test_train_set(self): for i in self.afw.images(): log.debug(i) for f in i.faces: log.debug(f) @unittest.skip("skipping because it takes too long") def test_download(self): self.afw = AFW(os.path.join(TMP_DIR, 'afw_download/')) self.afw.download()
def export(self, target_dir): ensure_dir(target_dir) self._prepare(target_dir) self._export(target_dir, 'train') self._export(target_dir, 'val')
def export(self, target_dir): ensure_dir(target_dir) self._prepare(target_dir) self._export(target_dir)
def download(self): ensure_dir(self.target_dir) tar_file = self.download_file_from_web_server(IMAGES_URL, self.target_dir) self.extract_tar_file(os.path.join(self.target_dir, tar_file), self.target_dir) log.info('done')
def export(self, target_dir): ensure_dir(target_dir) target_file = '{}.json' output_filename = os.path.join(target_dir, target_file.format(self.dataset.name)) self._export(output_filename, self.dataset.images())
def export(self, output_dir): ensure_dir(output_dir) self._export(output_dir, 'train') self._export(output_dir, 'val')
def export(self, output_dir): os_utils.ensure_dir(output_dir) self._export(output_dir, 'train')