Beispiel #1
0
    def extract(self, force_flag=False):
        """Checking the grid.xml file in the machine, if found use it.
        Else, download from the server.

        :param force_flag: force flag to extract.
        """
        final_grid_xml_file = os.path.join(self.extract_dir(), 'grid.xml')
        if not os.path.exists(self.extract_dir()):
            mk_dir(self.extract_dir())
        if force_flag or self.force_flag:
            self.remove_extracted_files()
        elif os.path.exists(final_grid_xml_file):
            return final_grid_xml_file

        # download data
        local_path = self.fetch_file()
        # move grid.xml to the correct directory
        expected_grid_xml_file = os.path.join(local_path, 'output', 'grid.xml')
        if not os.path.exists(expected_grid_xml_file):
            raise FileNotFoundError('The output does not contain an '
                                    '%s file.' % expected_grid_xml_file)

        # move the file we care about to the top of the extract dir
        shutil.copyfile(
            os.path.join(self.extract_dir(), expected_grid_xml_file),
            final_grid_xml_file)
        if not os.path.exists(final_grid_xml_file):
            raise CopyError('Error copying grid.xml')
        return final_grid_xml_file
    def fetch_file(self, retries=3):
        """Private helper to fetch a file from the sftp site.

        :param retries: int - number of reattempts that should be made in
                in case of network error etc.
          e.g. for event 20110413170148 this file would be fetched::
                20110413170148 directory

        .. note:: If a cached copy of the file exits, the path to the cache
           copy will simply be returned without invoking any network requests.

        :return: A string for the dataset path on the local storage system.
        :rtype: str

        :raises: EventUndefinedError, NetworkError
        """
        local_path = os.path.join(shakemap_cache_dir(), self.event_id)
        local_parent_path = os.path.join(local_path, "output")
        xml_file = os.path.join(local_parent_path, self.file_name())
        if os.path.exists(xml_file):
            return local_path

        # fetch from sftp
        trials = [i + 1 for i in xrange(retries)]
        remote_path = os.path.join(self.sftpclient.workdir_path, self.event_id)
        xml_remote_path = os.path.join(remote_path, "output", self.file_name())
        for counter in trials:
            last_error = None
            try:
                mk_dir(local_path)
                mk_dir(os.path.join(local_path, "output"))
                self.sftpclient.download_path(xml_remote_path, local_parent_path)
            except NetworkError, e:
                last_error = e
            except:
Beispiel #3
0
def fit_model(model, X_train, y_train, X_val, y_val, nb_epochs, batch_size,
              save_name, run, method):
    start_decay_epoch = [30, 60]
    decaylearningrate = train_callbacks.DecayLearningRate(start_decay_epoch)

    mk_dir(db_name + f"_checkpoints/{method}/{run}")

    callbacks = [
        ModelCheckpoint(db_name + f"_checkpoints/{method}/{run}" +
                        "/weights.{epoch:02d}-{val_loss:.2f}.hdf5",
                        monitor="val_loss",
                        verbose=1,
                        save_best_only=True,
                        mode="auto"), decaylearningrate
    ]

    hist = model.fit_generator(generator=data_generator_reg(
        X=X_train, Y=y_train, batch_size=batch_size),
                               steps_per_epoch=X_train.shape[0] // batch_size,
                               validation_data=(X_val, [y_val]),
                               epochs=nb_epochs,
                               verbose=1,
                               callbacks=callbacks)

    logging.debug("Saving weights...")
    model.save_weights(os.path.join(db_name + "_models/" + save_name,
                                    save_name + '.h5'),
                       overwrite=True)
    pd.DataFrame(hist.history).to_hdf(
        os.path.join(db_name + "_models/" + save_name,
                     'history_' + save_name + '.h5'), "history")
Beispiel #4
0
    def process_infolist(self, baseurl, infolist, fromdate, todate):
        newdls = []
        found = False
        finished = False
        infoWithDate = []
        for info in infolist:
            if info.has_key(self.DATE):
                infoWithDate.append(info)

        infoWithDate.sort(cmp=lambda x, y: cmp(x[self.DATE], y[self.DATE]))
        for info in infoWithDate:
            dateobj = info[self.DATE]
            if dateobj >= fromdate and dateobj <= todate:
                found = True
                datestr = dateobj.date().__str__()
                tmprel = os.path.join(self.name, datestr)

                rawdatedir = os.path.join(self.rawdir, tmprel)
                utils.mk_dir(rawdatedir)

                metadatedir = os.path.join(self.metadir, tmprel)
                utils.mk_dir(metadatedir)

                relurl = self.download_doc(baseurl, info, tmprel)
                if relurl:
                    newdls.append(relurl)
            elif found:
                finished = True
        if not finished or not found or not self.INFO_ORDERED:
            return False, newdls
        else:
            return True, newdls
Beispiel #5
0
    def process_infolist(self, baseurl, infolist, fromdate, todate):
        newdls = []
        found = False
        finished = False
        infoWithDate = []
        for info in infolist: 
            if info.has_key(self.DATE):
                infoWithDate.append(info)

        infoWithDate.sort(cmp = lambda x, y: cmp(x[self.DATE], y[self.DATE]))
        for info in infoWithDate:
                dateobj = info[self.DATE]
                if dateobj >= fromdate and dateobj <= todate:
                    found = True
                    datestr = dateobj.date().__str__()
                    tmprel = os.path.join (self.name, datestr)

                    rawdatedir   = os.path.join (self.rawdir, tmprel)
                    utils.mk_dir(rawdatedir)

                    metadatedir = os.path.join (self.metadir, tmprel)
                    utils.mk_dir(metadatedir)

                    relurl = self.download_doc(baseurl, info, tmprel)
                    if relurl:
                        newdls.append(relurl)
                elif found:
                    finished = True
        if not finished or not found or not self.INFO_ORDERED:
            return False, newdls
        else:
            return True, newdls
    def extract(self, force_flag=False):
        """Checking the grid.xml file in the machine, if found use it.
        Else, download from the server.

        :param force_flag: force flag to extract.
        """
        final_grid_xml_file = os.path.join(self.extract_dir(), "grid.xml")
        if not os.path.exists(self.extract_dir()):
            mk_dir(self.extract_dir())
        if force_flag or self.force_flag:
            self.remove_extracted_files()
        elif os.path.exists(final_grid_xml_file):
            return final_grid_xml_file

        # download data
        local_path = self.fetch_file()
        # move grid.xml to the correct directory
        expected_grid_xml_file = os.path.join(local_path, "output", "grid.xml")
        if not os.path.exists(expected_grid_xml_file):
            raise FileNotFoundError("The output does not contain an " "%s file." % expected_grid_xml_file)

        # move the file we care about to the top of the extract dir
        shutil.copyfile(os.path.join(self.extract_dir(), expected_grid_xml_file), final_grid_xml_file)
        if not os.path.exists(final_grid_xml_file):
            raise CopyError("Error copying grid.xml")
        return final_grid_xml_file
Beispiel #7
0
def main():
    args = get_args()
    input_path = args.input
    batch_size = args.batch_size
    nb_epochs = args.nb_epochs
    depth = args.depth
    k = args.width
    validation_split = args.validation_split

    logging.debug("Loading data...")
    image, gender, age, _, image_size, _ = load_data(input_path)
    X_data = image
    y_data_g = np_utils.to_categorical(gender, 2)

    #Quantize the age into 21 bins:
    age_bins = np.linspace(0, 100, 21)
    age_step = np.digitize(age, age_bins)
    y_data_a = np_utils.to_categorical(age_step, 21)
    #model = WideResNet(image_size, depth=depth, k=k)()
    model = TYY_1stream(image_size)()
    sgd = SGD(lr=0.1, momentum=0.9, nesterov=True)
    model.compile(
        optimizer=sgd,
        loss=["categorical_crossentropy", "categorical_crossentropy"],
        metrics=['accuracy'])

    logging.debug("Model summary...")
    model.count_params()
    model.summary()

    logging.debug("Saving model...")
    mk_dir("models")
    #with open(os.path.join("models", "WRN_{}_{}.json".format(depth, k)), "w") as f:
    with open(os.path.join("models", "TYY_1stream.json"), "w") as f:
        f.write(model.to_json())

    mk_dir("checkpoints")
    callbacks = [
        LearningRateScheduler(schedule=Schedule(nb_epochs)),
        ModelCheckpoint("checkpoints/weights.{epoch:02d}-{val_loss:.2f}.hdf5",
                        monitor="val_loss",
                        verbose=1,
                        save_best_only=True,
                        mode="auto")
    ]

    logging.debug("Running training...")
    hist = model.fit(X_data, [y_data_g, y_data_a],
                     batch_size=batch_size,
                     epochs=nb_epochs,
                     callbacks=callbacks,
                     validation_split=validation_split)

    logging.debug("Saving weights...")
    #model.save_weights(os.path.join("models", "WRN_{}_{}.h5".format(depth, k)), overwrite=True)
    model.save_weights(os.path.join("models", "TYY_1stream.h5"),
                       overwrite=True)
    #pd.DataFrame(hist.history).to_hdf(os.path.join("models", "history_{}_{}.h5".format(depth, k)), "history")
    pd.DataFrame(hist.history).to_hdf(
        os.path.join("models", "history_TYY_1steam.h5"), "history")
Beispiel #8
0
    def download_order(self, relpath, dateobj, metainfo, onclick):
        reobj = re.search('myfunViewDownLoad\s*\(\s*"(?P<ccin>\d+)"\s*,\s*"(?P<orderno>\d+)"\s*,\s*"(?P<flag>\w+)"\s*,\s*"(?P<casedetail>.+)"\s*,\s*"\w+"', onclick) 
        if not reobj:
            self.logger.warning(u'Could not get parameters in onclick: %s' % onclick)
            return None
 
        groupdict  = reobj.groupdict()
        ccin       = groupdict['ccin']
        orderno    = groupdict['orderno']
        flag       = groupdict['flag']
        casedetail = groupdict['casedetail']

        metainfo['caseno'] = casedetail
        filename   = self.get_filename(casedetail)

        if not filename:
            self.logger.warning(u'Could not get filename from %s' % casedetail)
            return None

        datestr = dateobj.__str__()

        utils.mk_dir(os.path.join(self.rawdir, self.name, datestr))
        utils.mk_dir(os.path.join(self.metadir, self.name, datestr))
        
        relurl   = os.path.join(relpath, datestr, filename)
        filepath = os.path.join(self.rawdir, relurl)
        metapath = os.path.join(self.metadir, relurl)

        if os.path.exists(filepath):
            self.logger.warning(u'Raw file already exists, skipping: %s ' % relurl)
        else:
            #ccin_no=001016200801769&order_no=2&flag=v&casedetail=MISC.CIVIL+APPLICATION%2F1769%2F2008&download_token_value_id=1367853726545
            self.logger.info(u'Downloading %s' % relurl)
            postdata = [('ccin_no', ccin), ('order_no', orderno), \
                        ('flag', flag), ('casedetail', casedetail), \
                        ('download_token_value_id', int(time.time())) ]

            webpage = self.download_url(self.orderurl, \
                                        referer=self.caseurl,\
                                        loadcookies = self.cookiefile.name,\
                                        postdata = postdata)

            if webpage:
                self.logger.info(u'Saving %s' % filepath)
                utils.save_file(filepath, webpage)
            else:
                self.logger.warning(u'Could not download ccin: %s number: %s ' % (ccin, orderno))
           
        if os.path.exists(filepath) and metainfo and \
                (self.updateMeta or not os.path.exists(metapath)):
            self.logger.info(u'Metainfo: %s' % metainfo)
            utils.print_tag_file(metapath, metainfo)

        if os.path.exists(filepath):
            return relurl

        return None
Beispiel #9
0
def attempt(request, id):
    context = {}

    context['queries'] = request.GET.copy()

    attempt = Attempt.objects.get(id=id)
    context['attempt'] = attempt

    dependencies = Dependency.objects.filter(
        attempt__id=id).order_by('package__name')
    context['dependencies'] = dependencies

    keyword_order = {'SELECT': 1, 'INSERT': 2, 'UPDATE': 3, 'DELETE': 4}

    context['statistics'] = {}
    for statistic in Statistic.objects.filter(attempt=attempt):
        context['statistics'][statistic.description] = max(
            statistic.count, context['statistics'].get(statistic.description,
                                                       0))

    actions = Action.objects.filter(attempt=attempt)
    context['actions'] = []
    for action in actions:
        fields = Field.objects.filter(action=action)
        counters = Counter.objects.filter(action=action)
        counters = sorted(counters,
                          key=lambda x: keyword_order.get(x.description, 10))
        context['actions'].append({
            'id': action.id,
            'url': action.url,
            'method': action.method,
            'fields': fields,
            'counters': counters
        })

    try:
        image = Image.objects.get(attempt=attempt)
        screenshot_filename = 'screenshot_{}.png'.format(attempt.id)
        utils.mk_dir(
            os.path.join(os.path.dirname(__file__), 'static', 'screenshots'))
        screenshot = open(
            os.path.join(os.path.dirname(__file__), 'static', 'screenshots',
                         screenshot_filename), 'wb')
        screenshot.write(image.data)
        screenshot.close()
        context['screenshot'] = '/static/screenshots/' + screenshot_filename
    except:
        traceback.print_exc()

    return render(request, 'attempt.html', context)
Beispiel #10
0
    def sync(self, fromdate, todate):
        dirname = os.path.join(self.rawdir, self.name)
        utils.mk_dir(dirname)

        dirname = os.path.join(self.metadir, self.name)
        utils.mk_dir(dirname)

        dls = []

        for mainurl in self.mainurls:
            while 1:
                infolist, nexturl = self.download_info_page(mainurl)
                finished, newdls = self.process_infolist(mainurl, infolist, \
                                                         fromdate, todate)
                dls.extend(newdls)
                if finished or nexturl == None:
                    break
                self.logger.info(u'Going to the next page: %s' % nexturl)
                mainurl = nexturl
        return dls
Beispiel #11
0
    def sync(self, fromdate, todate):
        dirname = os.path.join (self.rawdir, self.name)
        utils.mk_dir(dirname)

        dirname = os.path.join (self.metadir, self.name)
        utils.mk_dir(dirname)

        dls = []

        for mainurl in self.mainurls:
            while 1:
                infolist, nexturl = self.download_info_page(mainurl)
                finished, newdls = self.process_infolist(mainurl, infolist, \
                                                         fromdate, todate)
                dls.extend(newdls)
                if finished or nexturl == None:
                    break
                self.logger.info(u'Going to the next page: %s' % nexturl)
                mainurl = nexturl
        return dls
Beispiel #12
0
    def download_path(self, remote_path, local_path):
        """ Download remote_dir to local_dir.

        :param remote_path: The remote path that will be downloaded to local,
        :type remote_path: str

        :param local_path: The target path on local,
        :type local_path: str

        EXAMPLE : remote_path = '20130111133900' will be downloaded to
        local_dir/remote_path. It must be in the parent directory of remote
        dir.
        """
        # Check if remote_dir is exist
        if not self.is_path_exist(remote_path):
            print 'remote path is not exist %s' % remote_path
            return False
        if self.is_dir(remote_path):
            # get directory name
            dir_name = get_path_tail(remote_path)
            # create directory in local machine
            local_dir_path = os.path.join(local_path, dir_name)
            mk_dir(local_dir_path)
            # list all directory in remote path
            list_dir = self.sftp.listdir(remote_path)
            # iterate recursive
            for my_dir in list_dir:
                new_remote_path = os.path.join(remote_path, my_dir)
                self.download_path(new_remote_path, local_dir_path)
        else:
            # download file to local_path
            file_name = get_path_tail(remote_path)
            local_file_path = os.path.join(local_path, file_name)

            LOGGER.info('file %s will be downloaded to %s' %
                        (remote_path, local_file_path))
            self.sftp.get(remote_path, local_file_path)
Beispiel #13
0
def get_model(method, image_size, v):
    optMethod = Adam()
    N_densenet = 3
    depth_densenet = 3 * N_densenet + 4

    if method == 'piven':
        model = PI_Model(image_size, depth_densenet, method='piven')()
        save_name = 'densenet_eli_reg_%d_%d' % (depth_densenet, image_size)
        model.compile(optimizer=optMethod,
                      loss=piven_loss(eli=True),
                      metrics=[picp, mpiw, mae(method, v)])

    elif method == 'qd':
        model = PI_Model(image_size, depth_densenet, method='qd')()
        save_name = 'densenet_qd_reg_%d_%d' % (depth_densenet, image_size)
        model.compile(optimizer=optMethod,
                      loss=piven_loss(eli=False),
                      metrics=[picp, mpiw, mae(method, v)])

    elif method == 'only_point':
        model = DenseNet_reg(image_size, depth_densenet)()
        save_name = 'densenet_reg_%d_%d' % (depth_densenet, image_size)
        model.compile(optimizer=optMethod,
                      loss=["mae"],
                      metrics={'pred_a': 'mae'})

    else:
        raise ValueError(f"Invalid method name: {method}")

    logging.debug("Model summary...")
    model.count_params()
    model.summary()

    db_name = 'imdb'
    logging.debug("Saving model...")
    mk_dir(db_name + "_models")
    mk_dir(db_name + "_models/" + save_name)
    mk_dir(db_name + "_checkpoints")
    mk_dir(db_name + f"_checkpoints/{method}")

    with open(
            os.path.join(db_name + "_models/" + save_name,
                         save_name + '.json'), "w") as f:
        f.write(model.to_json())

    return model, save_name
Beispiel #14
0
    def fetch_file(self, retries=3):
        """Private helper to fetch a file from the sftp site.

        :param retries: int - number of reattempts that should be made in
                in case of network error etc.
          e.g. for event 20110413170148 this file would be fetched::
                20110413170148 directory

        .. note:: If a cached copy of the file exits, the path to the cache
           copy will simply be returned without invoking any network requests.

        :return: A string for the dataset path on the local storage system.
        :rtype: str

        :raises: EventUndefinedError, NetworkError
        """
        local_path = os.path.join(shakemap_cache_dir(), self.event_id)
        local_parent_path = os.path.join(local_path, 'output')
        xml_file = os.path.join(local_parent_path, self.file_name())
        if os.path.exists(xml_file):
            return local_path

        # fetch from sftp
        trials = [i + 1 for i in xrange(retries)]
        remote_path = os.path.join(self.sftpclient.workdir_path, self.event_id)
        xml_remote_path = os.path.join(remote_path, 'output', self.file_name())
        for counter in trials:
            last_error = None
            try:
                mk_dir(local_path)
                mk_dir(os.path.join(local_path, 'output'))
                self.sftpclient.download_path(xml_remote_path,
                                              local_parent_path)
            except NetworkError, e:
                last_error = e
            except:
Beispiel #15
0
def main():
    args = get_args()
    input_path = args.input
    images_path = args.dataset_path
    batch_size = args.batch_size
    nb_epochs = args.nb_epochs
    validation_split = args.validation_split

    logging.debug("Loading data...")

    dataset_name = 'imdb'
    data_loader = DataManager(dataset_name, dataset_path=input_path)
    ground_truth_data = data_loader.get_data()
    train_keys, val_keys = split_imdb_data(ground_truth_data, validation_split)

    print("Samples: Training - {}, Validation - {}".format(
        len(train_keys), len(val_keys)))
    input_shape = (IMG_SIZE, IMG_SIZE, 3)
    # images_path = 'data/imdb_crop/'

    image_generator = ImageGenerator(ground_truth_data,
                                     batch_size,
                                     input_shape[:2],
                                     train_keys,
                                     val_keys,
                                     path_prefix=images_path,
                                     vertical_flip_probability=0)

    n_age_bins = 21
    alpha = 1
    model = MobileNetDeepEstimator(input_shape[0],
                                   alpha,
                                   n_age_bins,
                                   weights='imagenet')()

    opt = SGD(lr=0.001)

    model.compile(
        optimizer=opt,
        loss=["binary_crossentropy", "categorical_crossentropy"],
        metrics={
            'gender': 'accuracy',
            'age': 'accuracy'
        },
    )

    logging.debug("Model summary...")
    model.count_params()
    model.summary()

    logging.debug("Saving model...")
    mk_dir("models")
    with open(os.path.join("models", "MobileNet.json"), "w") as f:
        f.write(model.to_json())

    mk_dir("checkpoints")

    run_id = "MobileNet - " + str(batch_size) + " " + '' \
        .join(random
              .SystemRandom()
              .choice(string.ascii_uppercase) for _ in
              range(10)
              )
    print(run_id)

    reduce_lr = ReduceLROnPlateau(verbose=1, epsilon=0.001, patience=4)

    callbacks = [
        LearningRateScheduler(schedule=Schedule(nb_epochs)), reduce_lr,
        ModelCheckpoint(os.path.join(
            'checkpoints', 'weights.{epoch:02d}-{val_loss:.2f}.hdf5'),
                        monitor="val_loss",
                        verbose=1,
                        save_best_only=True,
                        mode="auto"),
        TensorBoard(log_dir='logs/' + run_id)
    ]

    logging.debug("Running training...")

    logging.debug("steps : " + str(int((len(val_keys) / batch_size))))

    hist = model.fit_generator(
        image_generator.flow(mode='train'),
        steps_per_epoch=int(len(train_keys) / batch_size),
        epochs=nb_epochs,
        callbacks=callbacks,
        validation_data=image_generator.flow('val'),
        validation_steps=int(len(val_keys) / batch_size))

    logging.debug("Saving weights...")
    model.save(os.path.join("models", "MobileNet_model.h5"))
    model.save_weights(os.path.join("models", FINAL_WEIGHTS_PATH),
                       overwrite=True)
    pd.DataFrame(hist.history).to_hdf(os.path.join("models", "history.h5"),
                                      "history")
Beispiel #16
0
            sys.exit(0)

    leveldict = {'critical': logging.CRITICAL, 'error': logging.ERROR, \
                 'warning': logging.WARNING, 'info': logging.INFO, \
                 'debug': logging.DEBUG}

    logfmt  = '%(asctime)s: %(name)s: %(levelname)s %(message)s'
    datefmt = '%Y-%m-%d %H:%M:%S'

    if datadir == None:
        print >> sys.stderr, 'No data directory specified'
        print_usage(progname)
        sys.exit(0)

    statsdir = os.path.join(datadir, 'stats')
    utils.mk_dir(statsdir)

    if filename:
        filename = os.path.join(statsdir, filename)

    if filename:
        logging.basicConfig(\
            level   = leveldict[debuglevel], \
            format  = logfmt, \
            filename = filename, \
            datefmt = datefmt \
        )
    else:
        logging.basicConfig(\
            level   = leveldict[debuglevel], \
            format  = logfmt, \
Beispiel #17
0
        if todate == None:
            todate   = datetime.datetime.today() 
            fromdate = todate - datetime.timedelta(days = 7)
        else:
            fromdate = todate

    elif todate == None:
        todate = fromdate 


    basedir  = remlist[0]
    rawdir   = os.path.join(basedir, 'raw')
    metadir  = os.path.join(basedir, 'metatags')
    statsdir = os.path.join(basedir, 'stats')

    utils.mk_dir(rawdir)
    utils.mk_dir(metadir)
    utils.mk_dir(statsdir)

    courtobjs = []
    dldict = {'bombay'        : bombay.Bombay,               \
              'kolkata'       : kolkata.Kolkata,             \
              'kolkata_app'   : kolkata_app.KolkataApp,      \
              'punjab'        : punjab.Punjab,               \
              'uttaranchal'   : uttaranchal.Uttaranchal,     \
              'delhi'         : delhi.Delhi,                 \
              'jharkhand'     : jharkhand.Jharkhand,         \
              'gujarat'       : gujarat.Gujarat,             \
              'rajasthan'     : rajasthan.Rajasthan,         \
              'jodhpur'       : jodhpur.Jodhpur,             \
              'karnataka'     : karnataka.Karnataka,         \
def main():
    args = get_args()
    input_path = args.input
    batch_size = args.batch_size
    nb_epochs = args.nb_epochs
    depth = args.depth
    k = args.width
    validation_split = args.validation_split
    use_augmentation = args.aug

    logging.debug("Loading data...")
    image, gender, age, race, _, image_size, _ = load_data(input_path)
    X_data = image
    y_data_g = np_utils.to_categorical(gender, 2)
    y_data_a = np_utils.to_categorical(age, 101)
    y_data_r = np_utils.to_categorical(race, 5)

    model = WideResNet(image_size, depth=depth, k=k)()
    sgd = SGD(lr=0.1, momentum=0.9, nesterov=True)
    model.compile(optimizer=sgd,
                  loss=[
                      "categorical_crossentropy", "categorical_crossentropy",
                      "categorical_crossentropy"
                  ],
                  metrics=['accuracy'])

    logging.debug("Model summary...")
    model.count_params()
    model.summary()

    logging.debug("Saving model...")
    mk_dir("models")
    with open(os.path.join("models", "WRN_{}_{}.json".format(depth, k)),
              "w") as f:
        f.write(model.to_json())

    mk_dir("checkpoints")
    callbacks = [
        LearningRateScheduler(schedule=Schedule(nb_epochs)),
        ModelCheckpoint("checkpoints/weights.{epoch:02d}-{val_loss:.2f}.hdf5",
                        monitor="val_loss",
                        verbose=1,
                        save_best_only=True,
                        mode="auto")
    ]

    logging.debug("Running training...")

    data_num = len(X_data)
    indexes = np.arange(data_num)
    np.random.shuffle(indexes)
    X_data = X_data[indexes]
    y_data_g = y_data_g[indexes]
    y_data_a = y_data_a[indexes]
    y_data_r = y_data_r[indexes]
    train_num = int(data_num * (1 - validation_split))
    X_train = X_data[:train_num]
    X_test = X_data[train_num:]
    y_train_g = y_data_g[:train_num]
    y_test_g = y_data_g[train_num:]
    y_train_a = y_data_a[:train_num]
    y_test_a = y_data_a[train_num:]
    y_train_r = y_data_r[:train_num]
    y_test_r = y_data_r[train_num:]

    if use_augmentation:
        datagen = ImageDataGenerator(width_shift_range=0.1,
                                     height_shift_range=0.1,
                                     horizontal_flip=True,
                                     preprocessing_function=get_random_eraser(
                                         v_l=0, v_h=255))
        training_generator = MixupGenerator(X_train,
                                            [y_train_g, y_train_a, y_train_r],
                                            batch_size=batch_size,
                                            alpha=0.2,
                                            datagen=datagen)()
        hist = model.fit_generator(
            generator=training_generator,
            steps_per_epoch=train_num // batch_size,
            validation_data=(X_test, [y_test_g, y_test_a, y_test_r]),
            epochs=nb_epochs,
            verbose=1,
            callbacks=callbacks)
    else:
        hist = model.fit(X_train, [y_train_g, y_train_a, y_train_r],
                         batch_size=batch_size,
                         epochs=nb_epochs,
                         callbacks=callbacks,
                         validation_data=(X_test,
                                          [y_test_g, y_test_a, y_test_r]))

    logging.debug("Saving weights...")
    model.save_weights(os.path.join("models", "WRN_{}_{}.h5".format(depth, k)),
                       overwrite=True)
    pd.DataFrame(hist.history).to_hdf(
        os.path.join("models", "history_{}_{}.h5".format(depth, k)), "history")
Beispiel #19
0
    if fromdate == None:
        if todate == None:
            todate = datetime.datetime.today()
            fromdate = todate - datetime.timedelta(days=7)
        else:
            fromdate = todate

    elif todate == None:
        todate = fromdate

    basedir = remlist[0]
    rawdir = os.path.join(basedir, 'raw')
    metadir = os.path.join(basedir, 'metatags')
    statsdir = os.path.join(basedir, 'stats')

    utils.mk_dir(rawdir)
    utils.mk_dir(metadir)
    utils.mk_dir(statsdir)

    courtobjs = []
    dldict = {'bombay'        : bombay.Bombay,               \
              'kolkata'       : kolkata.Kolkata,             \
              'kolkata_app'   : kolkata_app.KolkataApp,      \
              'punjab'        : punjab.Punjab,               \
              'uttaranchal'   : uttaranchal.Uttaranchal,     \
              'delhi'         : delhi.Delhi,                 \
              'jharkhand'     : jharkhand.Jharkhand,         \
              'gujarat'       : gujarat.Gujarat,             \
              'rajasthan'     : rajasthan.Rajasthan,         \
              'jodhpur'       : jodhpur.Jodhpur,             \
              'karnataka'     : karnataka.Karnataka,         \
def main():
    args = get_args()
    input_path = args.input
    batch_size = args.batch_size
    nb_epochs = args.nb_epochs
    freeze_layers = args.freeze_layers
    depth = args.depth
    k = args.width
    validation_split = args.validation_split

    logging.debug("Loading data...")
    image, gender, age, _, image_size, _ = load_data(input_path)
    X_data = image
    y_data_g = np_utils.to_categorical(gender, 2)
    y_data_a = np_utils.to_categorical(age, 101)

    #Load weights
    weight_file = os.path.join("pretrained_models", "weights.18-4.06.hdf5")

    model = WideResNet(image_size, depth=depth, k=k)()
    model.load_weights(weight_file)

    # set the first 50 layers
    # to non-trainable (weights will not be updated)
    print(len(model.layers))
    if freeze_layers > 0:
        for layer in model.layers[:freeze_layers]:
            layer.trainable = False

    sgd = SGD(lr=0.1, momentum=0.9, nesterov=True)
    model.compile(
        optimizer=sgd,
        loss=["categorical_crossentropy", "categorical_crossentropy"],
        metrics=['accuracy'])

    logging.debug("Model summary...")
    model.count_params()
    model.summary()

    logging.debug("Saving model...")
    mk_dir("models")
    with open(os.path.join("models", "WRN_{}_{}.json".format(depth, k)),
              "w") as f:
        f.write(model.to_json())

    mk_dir("checkpoints")
    callbacks = [
        LearningRateScheduler(schedule=Schedule(nb_epochs)),
        ModelCheckpoint("checkpoints/weights.{epoch:02d}-{val_loss:.2f}.hdf5",
                        monitor="val_loss",
                        verbose=1,
                        save_best_only=True,
                        mode="auto")
    ]

    logging.debug("Running training...")
    # print('length of X', len(X_data))
    # print('length of y_data_g', y_data_g)
    # print('length of y_data_a', len(y_data_a))
    hist = model.fit(X_data, [y_data_g, y_data_a],
                     batch_size=batch_size,
                     epochs=nb_epochs,
                     callbacks=callbacks,
                     validation_split=validation_split)

    logging.debug("Saving weights...")
    model.save_weights(os.path.join("models", "WRN_{}_{}.h5".format(depth, k)),
                       overwrite=True)
    pd.DataFrame(hist.history).to_hdf(
        os.path.join("models", "history_{}_{}.h5".format(depth, k)), "history")
Beispiel #21
0
def main():
    args = get_args()
    input_path = args.input
    batch_size = args.batch_size
    nb_epochs = args.nb_epochs
    depth = args.depth
    k = args.width
    validation_split = args.validation_split
    use_augmentation = args.aug

    logging.debug("Loading data...")
    image, gender, age, _, image_size, _ = load_data(input_path)
    X_data = image
    y_data_a = np_utils.to_categorical(age, 101)

    #custom parameters
    nb_class = 2
    hidden_dim = 512

    vgg_model = VGGFace(include_top=False, input_shape=(224, 224, 3))
    last_layer = vgg_model.get_layer('pool5').output
    x = Flatten(name='flatten')(last_layer)
    x = Dense(hidden_dim, activation='relu', name='fc6')(x)
    x = Dense(hidden_dim, activation='relu', name='fc7')(x)
    out = Dense(nb_class, activation='softmax', name='fc8')(x)
    model = Model(vgg_model.input, out)
    sgd = SGD(lr=0.1, momentum=0.9, nesterov=True)
    model.compile(optimizer=sgd,
                  loss=["categorical_crossentropy"],
                  metrics=['accuracy'])

    logging.debug("Model summary...")
    model.count_params()
    model.summary()

    logging.debug("Saving model...")
    mk_dir("models")
    with open(os.path.join("models", "vgg_{}_{}.json".format(depth, k)),
              "w") as f:
        f.write(model.to_json())

    mk_dir("checkpoints")
    callbacks = [
        LearningRateScheduler(schedule=Schedule(nb_epochs)),
        ModelCheckpoint("checkpoints/weights.{epoch:02d}-{val_loss:.2f}.hdf5",
                        monitor="val_loss",
                        verbose=1,
                        save_best_only=True,
                        mode="auto")
    ]

    logging.debug("Running training...")

    data_num = len(X_data)
    indexes = np.arange(data_num)
    np.random.shuffle(indexes)
    X_data = X_data[indexes]
    y_data_g = y_data_g[indexes]
    y_data_a = y_data_a[indexes]
    train_num = int(data_num * (1 - validation_split))
    X_train = X_data[:train_num]
    X_test = X_data[train_num:]
    y_train_g = y_data_g[:train_num]
    y_test_g = y_data_g[train_num:]
    y_train_a = y_data_a[:train_num]
    y_test_a = y_data_a[train_num:]

    if use_augmentation:
        datagen = ImageDataGenerator(width_shift_range=0.1,
                                     height_shift_range=0.1,
                                     horizontal_flip=True,
                                     preprocessing_function=get_random_eraser(
                                         v_l=0, v_h=255))
        training_generator = MixupGenerator(X_train, [y_train_g, y_train_a],
                                            batch_size=batch_size,
                                            alpha=0.2,
                                            datagen=datagen)()
        hist = model.fit_generator(generator=training_generator,
                                   steps_per_epoch=train_num // batch_size,
                                   validation_data=(X_test,
                                                    [y_test_g, y_test_a]),
                                   epochs=nb_epochs,
                                   verbose=1,
                                   callbacks=callbacks)
    else:
        hist = model.fit(X_train, [y_train_g, y_train_a],
                         batch_size=batch_size,
                         epochs=nb_epochs,
                         callbacks=callbacks,
                         validation_data=(X_test, [y_test_g, y_test_a]))

    logging.debug("Saving weights...")
    model.save_weights(os.path.join("models", "vgg_{}_{}.h5".format(depth, k)),
                       overwrite=True)
    pd.DataFrame(hist.history).to_hdf(
        os.path.join("models", "history_{}_{}.h5".format(depth, k)), "history")
def main():
    args = get_args()
    validation_fold = args.validation_fold
    test_set = loadmat('./fold_{}.mat'.format(validation_fold))
    test_x = test_set['image']
    test_y = test_set['gender']
    fold_set = [0, 1, 2, 3, 4]
    fold_set.remove(validation_fold)
    l = []
    for i in fold_set:
        l.append(loadmat('./fold_{}.mat'.format(i)))
    train_x = l[0]['image']
    train_y = l[0]['gender']
    for i in range(1, len(l)):
        train_x = np.concatenate((train_x, l[i]['image']))
        train_y = np.concatenate((train_y, l[i]['gender']))

    #test_y = utils.to_categorical(test_y,num_classes=2)
    #train_y = utils.to_categorical(test_y,num_classes=2)
    print(test_y)
    for i, value in np.ndenumerate(test_y):
        if value == 'm':
            np.put(test_y, i, 0)
        elif value == 'f':
            np.put(test_y, i, 1)
        else:
            raise Exception
    print(test_y)

    for i, value in np.ndenumerate(train_y):
        if value == 'm':
            np.put(train_y, i, 0)
        elif value == 'f':
            np.put(train_y, i, 1)
        else:
            raise Exception

    cnn = CNN(input_size=IMG_SIZE)
    model = cnn.get_classifier()

    train_datagen = ImageDataGenerator(rescale=1. / 255,
                                       shear_range=0.2,
                                       zoom_range=0.2,
                                       horizontal_flip=True)
    test_datagen = ImageDataGenerator(rescale=1. / 255)
    training_set = train_datagen.flow(train_x, train_y, batch_size=BATCH_SIZE)
    test_set = test_datagen.flow(test_x, test_y, batch_size=BATCH_SIZE)

    mk_dir("checkpoints")
    callbacks = [
        ModelCheckpoint("checkpoints/weights.{epoch:02d}-{val_loss:.2f}.hdf5",
                        monitor="val_loss",
                        verbose=1,
                        save_best_only=True,
                        mode="auto")
    ]
    hist = model.fit_generator(training_set,
                               steps_per_epoch=len(train_x) // BATCH_SIZE,
                               epochs=NUM_EPOCHS,
                               validation_data=test_set,
                               validation_steps=len(test_set),
                               callbacks=callbacks)

    model.save('my_cnn.h5')
    pd.DataFrame(hist.history).to_hdf("history.h5", "history")
Beispiel #23
0
    model = Model(input=inputs, output=predictions)
    return model


if __name__ == '__main__':
    model = create_model()
    model.compile(optimizer=sgd, loss="categorical_crossentropy", metrics=['accuracy'])

    if print_model_summary:
        logging.debug("Model summary...")
        model.count_params()
        model.summary()

    if save_model_plot:
        logging.debug("Saving model plot...")
        mk_dir(MODEL_PATH)
        from keras.utils.visualize_util import plot
        plot(model, to_file=os.path.join(MODEL_PATH, 'WRN-{0}-{1}.png'.format(depth, k)), show_shapes=True)
        
    #Data Augmentation from page 6
    logging.debug("Creating ImageDataGenerators...")   
    train_datagen = ImageDataGenerator(
                                featurewise_center=True,
                                featurewise_std_normalization=True,
                                zca_whitening=True,
                                horizontal_flip=True)
    #NEED TO ADD (page 6): "random crops from image padded by 4 pixels on each side, filling missing pixels with reflections of original image."
    #SEE: https://github.com/szagoruyko/wide-residual-networks/blob/master/augmentation.lua
    train_datagen.fit(X_train, augment=True, rounds=2)

    test_datagen = ImageDataGenerator(
Beispiel #24
0
def main():
    args = get_args()
    input_path = args.input
    batch_size = args.batch_size
    nb_epochs = args.nb_epochs
    max_age = args.max_age + 1
    depth = args.depth
    k = args.width
    transfer_learning = args.transfer_learning
    validation_split = args.validation_split
    use_augmentation = args.aug
    initial_weights = '/home/paula/THINKSMARTER_/Model/demographics-model-prediction/pretrained_models/weights.18-4.06.hdf5'
    # weight_file = '/home/paula/THINKSMARTER_/Model/age-gender-estimation-adapted/checkpoints/weights.09-4.32.hdf5'

    _weight_decay = 0.0005
    _use_bias = False
    _weight_init = "he_normal"

    logging.debug("Loading data...")
    image, gender, age, _, image_size, _ = load_data(input_path)
    X_data = image
    y_data_g = np_utils.to_categorical(gender, 2)
    y_data_a = np_utils.to_categorical(age, max_age)

    if transfer_learning:

        model = WideResNet(image_size, depth=depth, k=k, units_age=101)()
        model.load_weights(initial_weights)

        inputs = model.input
        flatten = model.layers[-3].output  # capa flatten
        dense1 = Dense(units=2,
                       kernel_initializer=_weight_init,
                       use_bias=_use_bias,
                       kernel_regularizer=l2(_weight_decay),
                       activation="softmax")(flatten)
        dense2 = Dense(units=117,
                       kernel_initializer=_weight_init,
                       use_bias=_use_bias,
                       kernel_regularizer=l2(_weight_decay),
                       activation="softmax")(flatten)
        model = Model(inputs=inputs, outputs=[dense1, dense2])

        # ---------------------------------
        # IDEA: fine tuning (nomes entreno les dos ultimes capes)
        # for layer in model.layers[:-2]:
        #     layer.trainable = False

    else:
        model = WideResNet(image_size, depth=depth, k=k, units_age=max_age)()

    sgd = SGD(lr=0.1, momentum=0.9, nesterov=True)
    model.compile(
        optimizer=sgd,
        loss=["categorical_crossentropy", "categorical_crossentropy"],
        metrics=['accuracy'])

    logging.debug("Model summary...")
    model.count_params()
    model.summary()

    if args.plot_model:
        plot_model(model,
                   to_file='experiments_pictures/model_plot.png',
                   show_shapes=True,
                   show_layer_names=True)

    logging.debug("Saving model...")
    mk_dir("models")
    with open(os.path.join("models", "WRN_{}_{}.json".format(depth, k)),
              "w") as f:
        f.write(model.to_json())

    mk_dir("checkpoints")
    # tensorBoard = TensorBoard(log_dir='events', histogram_freq=0, batch_size=32, write_graph=True, write_grads=False, write_images=True, embeddings_freq=0, embeddings_layer_names=None, embeddings_metadata=None, embeddings_data=None)

    callbacks = [
        LearningRateScheduler(schedule=Schedule(nb_epochs)),
        ModelCheckpoint("checkpoints/weights.{epoch:02d}-{val_loss:.2f}.hdf5",
                        monitor="val_loss",
                        verbose=1,
                        save_best_only=True,
                        mode="auto")
    ]

    logging.debug("Running training...")

    data_num = len(X_data)
    indexes = np.arange(data_num)
    np.random.shuffle(indexes)
    X_data = X_data[indexes]
    y_data_g = y_data_g[indexes]
    y_data_a = y_data_a[indexes]
    train_num = int(data_num * (1 - validation_split))
    X_train = X_data[:train_num]
    X_test = X_data[train_num:]
    y_train_g = y_data_g[:train_num]
    y_test_g = y_data_g[train_num:]
    y_train_a = y_data_a[:train_num]
    y_test_a = y_data_a[train_num:]

    if use_augmentation:
        datagen = ImageDataGenerator(width_shift_range=0.1,
                                     height_shift_range=0.1,
                                     horizontal_flip=True,
                                     preprocessing_function=get_random_eraser(
                                         v_l=0, v_h=255))
        training_generator = MixupGenerator(X_train, [y_train_g, y_train_a],
                                            batch_size=batch_size,
                                            alpha=0.2,
                                            datagen=datagen)()

        hist = model.fit_generator(generator=training_generator,
                                   steps_per_epoch=train_num // batch_size,
                                   validation_data=(X_test,
                                                    [y_test_g, y_test_a]),
                                   epochs=nb_epochs,
                                   verbose=1,
                                   callbacks=callbacks)
    else:
        hist = model.fit(X_train, [y_train_g, y_train_a],
                         batch_size=batch_size,
                         epochs=nb_epochs,
                         callbacks=callbacks,
                         validation_data=(X_test, [y_test_g, y_test_a]))

    logging.debug("Saving weights...")
    model.save_weights(os.path.join("models", "WRN_{}_{}.h5".format(depth, k)),
                       overwrite=True)
    pd.DataFrame(hist.history).to_hdf(
        os.path.join("models", "history_{}_{}.h5".format(depth, k)), "history")

    with open('history_tmp.txt', 'w') as f:
        for key in hist.history:
            print(key, file=f)
        f.write('\n')
        json.dump(hist.history, f)
Beispiel #25
0
def main():
    args = get_args()
    input_path = args.input
    batch_size = args.batch_size
    nb_epochs = args.nb_epochs
    depth = args.depth
    k = args.width
    validation_split = args.validation_split
    use_augmentation = args.aug

    image_size = 224

    os.environ["CUDA_VISIBLE_DEVICES"] = "0,1"

    logging.debug("Loading data and Augmentor...")

    #sess = K.get_session()
    #sess = tf_debug.LocalCLIDebugWrapperSession(sess)
    #K.set_session(sess)

    #model = WideResNet(image_size, depth=22, k=k)()
    model = MyModel(image_size, trainable=False)()
    adam = Adam(lr=0.01, decay=0.001)
    sgd = SGD(lr=0.00001, momentum=0.9, nesterov=True, decay=0.0001)
    model.compile(optimizer=sgd,
                  loss=["categorical_crossentropy", "MSE"],
                  loss_weights=[0.5, 1.0],
                  metrics=['accuracy'])

    logging.debug("Model summary...")
    model.count_params()
    model.summary()

    #model.load_weights(os.path.join("checkpoints", "weights.03-4.78.hdf5"))

    logging.debug("Saving model...")
    mk_dir("models")
    with open(os.path.join("models", "WRN_{}_{}.json".format(depth, k)),
              "w") as f:
        f.write(model.to_json())

    mk_dir("checkpoints")

    train_datagen = ImageDataGenerator(rescale=1. / 255,
                                       shear_range=0.2,
                                       horizontal_flip=True,
                                       validation_split=0.2)

    train_generator = train_datagen.flow_from_directory(
        '../../dataset/imdb_crop/new_database/',
        target_size=(image_size, image_size),
        batch_size=batch_size,
        class_mode='categorical',
        subset='training',
        shuffle=True)
    print(next(train_generator)[1].shape)
    val_generator = train_datagen.flow_from_directory(
        '../../dataset/imdb_crop/new_database/',
        target_size=(image_size, image_size),
        batch_size=batch_size,
        class_mode='categorical',
        subset='validation',
        shuffle=True)

    callbacks = [
        LearningRateScheduler(schedule=Schedule(38138 // batch_size)),
        ModelCheckpoint("checkpoints/weights.{epoch:02d}-{val_loss:.2f}.hdf5",
                        monitor="val_loss",
                        verbose=1,
                        save_best_only=True,
                        mode="auto")
    ]

    #class_weight = get_class_weights(train_generator.classes)

    #print(class_weight)

    h = model.fit_generator(
        generate_data_generator(train_generator),
        use_multiprocessing=True,
        epochs=10,
        validation_data=generate_data_generator(val_generator),
        workers=12,
        steps_per_epoch=len(train_generator.classes) / batch_size,
        validation_steps=len(val_generator.classes) / batch_size,
        #class_weight=class_weight
    )

    logging.debug("Saving weights...")
    model.save_weights(os.path.join("models", "WRN_{}_{}.h5".format(depth, k)),
                       overwrite=True)
Beispiel #26
0
def main():
    args = get_args()
    input_path = args.input
    batch_size = args.batch_size
    nb_epochs = args.nb_epochs
    depth = args.depth
    k = args.width
    validation_split = args.validation_split
    use_augmentation = args.aug

    logging.debug("Loading data...")
    image, gender, age, _, image_size, _ = load_data(input_path)
    X_data = image
    y_data_g = np_utils.to_categorical(gender, 2)
    y_data_a = np_utils.to_categorical(age, 101)

    model = WideResNet(image_size, depth=depth, k=k)()
    sgd = SGD(lr=0.1, momentum=0.9, nesterov=True)
    model.compile(optimizer=sgd, loss=["categorical_crossentropy", "categorical_crossentropy"],
                  metrics=['accuracy'])

    logging.debug("Model summary...")
    model.count_params()
    model.summary()

    logging.debug("Saving model...")
    mk_dir("models")
    with open(os.path.join("models", "WRN_{}_{}.json".format(depth, k)), "w") as f:
        f.write(model.to_json())

    mk_dir("checkpoints")
    callbacks = [LearningRateScheduler(schedule=Schedule(nb_epochs)),
                 ModelCheckpoint("checkpoints/weights.{epoch:02d}-{val_loss:.2f}.hdf5",
                                 monitor="val_loss",
                                 verbose=1,
                                 save_best_only=True,
                                 mode="auto")
                 ]

    logging.debug("Running training...")

    data_num = len(X_data)
    indexes = np.arange(data_num)
    np.random.shuffle(indexes)
    X_data = X_data[indexes]
    y_data_g = y_data_g[indexes]
    y_data_a = y_data_a[indexes]
    train_num = int(data_num * (1 - validation_split))
    X_train = X_data[:train_num]
    X_test = X_data[train_num:]
    y_train_g = y_data_g[:train_num]
    y_test_g = y_data_g[train_num:]
    y_train_a = y_data_a[:train_num]
    y_test_a = y_data_a[train_num:]

    if use_augmentation:
        datagen = ImageDataGenerator(
            width_shift_range=0.1,
            height_shift_range=0.1,
            horizontal_flip=True,
            preprocessing_function=get_random_eraser(v_l=0, v_h=255))
        training_generator = MixupGenerator(X_train, [y_train_g, y_train_a], batch_size=batch_size, alpha=0.2,
                                            datagen=datagen)()
        hist = model.fit_generator(generator=training_generator,
                                   steps_per_epoch=train_num // batch_size,
                                   validation_data=(X_test, [y_test_g, y_test_a]),
                                   epochs=nb_epochs, verbose=1,
                                   callbacks=callbacks)
    else:
        hist = model.fit(X_train, [y_train_g, y_train_a], batch_size=batch_size, epochs=nb_epochs, callbacks=callbacks,
                         validation_data=(X_test, [y_test_g, y_test_a]))

    logging.debug("Saving weights...")
    model.save_weights(os.path.join("models", "WRN_{}_{}.h5".format(depth, k)), overwrite=True)
    pd.DataFrame(hist.history).to_hdf(os.path.join("models", "history_{}_{}.h5".format(depth, k)), "history")
Beispiel #27
0
#     else:
#         path_dir = os.listdir(file_dir)
#     sample = random.sample(path_dir, n)

#     for name in sample:
#         shutil.copyfile(os.path.join(file_dir, name),
#                         os.path.join(tar_dir, name))

if __name__ == '__main__':
    # FaceNet
    IMAGE_SIZE = 160
    PADDING = 0.2
    WEIGHTS = "../models/model.h5"
    base_path = "test"
    gender_list = ['W', 'M']
    mk_dir("results")

    tf.logging.set_verbosity(tf.logging.ERROR)
    faceAlign = Face_aligned(keras_model=WEIGHTS)

    for _, _, imgs in os.walk(base_path):
        '''
        Only support one face in the image
        '''
        for idx, im in enumerate(imgs):
            img_path = os.path.join(base_path, im)

            # Extract image suffix name
            (_, name) = os.path.split(img_path)
            image = cv2.imread(img_path)
Beispiel #28
0

if __name__ == '__main__':
    model = create_model()
    model.compile(optimizer=sgd,
                  loss="categorical_crossentropy",
                  metrics=['accuracy'])

    if print_model_summary:
        logging.debug("Model summary...")
        model.count_params()
        model.summary()

    if save_model_plot:
        logging.debug("Saving model plot...")
        mk_dir(MODEL_PATH)
        from keras.utils.visualize_util import plot
        plot(model,
             to_file=os.path.join(MODEL_PATH,
                                  'WRN-{0}-{1}.png'.format(depth, k)),
             show_shapes=True)

    # Data Augmentation based on page 6 (see README for full details)
    logging.debug("Creating ImageDataGenerators...")
    train_datagen = ImageDataGenerator(featurewise_center=True,
                                       featurewise_std_normalization=True,
                                       zca_whitening=True,
                                       horizontal_flip=True)
    train_datagen.fit(X_train, augment=True, rounds=2)

    test_datagen = ImageDataGenerator(featurewise_center=True,
def train_age():
    args = get_args()
    validation_fold = args.v
    test_set = loadmat('./age_fold_{}.mat'.format(validation_fold))
    test_x = test_set['image']
    test_y = test_set['age']
    test_y = [x.strip() for x in test_y] # since Matrices are of rectangular shapes in matlab
    test_y = np.array(test_y)

    fold_set = [0,1,2,3,4]
    fold_set.remove(validation_fold)
    l = []
    for i in fold_set:
        l.append(loadmat('./age_fold_{}.mat'.format(i)))
    train_x = l[0]['image']
    train_y = l[0]['age']
    for i in range(1,len(l)):
        train_x = np.concatenate((train_x,l[i]['image']))
        train_y = np.concatenate((train_y,l[i]['age']))
    train_y = [x.strip() for x in train_y]
    train_y = np.array(train_y)

    print(test_y[:50])
    for i, value in np.ndenumerate(test_y):
        if value == '(0, 2)':
            np.put(test_y,i,0)
        elif value == '(4, 6)':
            np.put(test_y,i,1)
        elif value == '(8, 12)':
            np.put(test_y,i,2)
        elif value == '(15, 20)':
            np.put(test_y,i,3)
        elif value == '(25, 32)':
            np.put(test_y,i,4)
        elif value == '(38, 43)':
            np.put(test_y,i,5)
        elif value == '(48, 53)':
            np.put(test_y,i,6)
        elif value == '(60, 100)':
            np.put(test_y,i,7)
        else:
            print(value)
            print(len(value))
            raise Exception
    print(test_y)

    for i, value in np.ndenumerate(train_y):
        if value == '(0, 2)':
            np.put(train_y,i,0)
        elif value == '(4, 6)':
            np.put(train_y,i,1)
        elif value == '(8, 12)':
            np.put(train_y,i,2)
        elif value == '(15, 20)':
            np.put(train_y,i,3)
        elif value == '(25, 32)':
            np.put(train_y,i,4)
        elif value == '(38, 43)':
            np.put(train_y,i,5)
        elif value == '(48, 53)':
            np.put(train_y,i,6)
        elif value == '(60, 100)':
            np.put(train_y,i,7)
        else:
            print(value)
            print(len(value))
            raise Exception
    test_y = utils.to_categorical(test_y,num_classes=8)
    train_y = utils.to_categorical(train_y,num_classes=8)

    #import pdb; pdb.set_trace()

    cnn = AgeCNN(input_size = IMG_SIZE)
    model = cnn.get_classifier()
    
    if IMG_AUG == 1:
        train_datagen = ImageDataGenerator(rescale = 1./255,
                                   shear_range = 0.2,
                                   zoom_range = 0.2,
                                   horizontal_flip = True)
    else:
        train_datagen = ImageDataGenerator(rescale = 1./255)
        
    test_datagen = ImageDataGenerator(rescale = 1./255)
    training_set = train_datagen.flow(train_x,train_y,batch_size=BATCH_SIZE)
    test_set = test_datagen.flow(test_x,test_y,batch_size=BATCH_SIZE)

    mk_dir("checkpoints")
    callbacks = [ModelCheckpoint("checkpoints/weights.{epoch:02d}-{val_loss:.2f}.hdf5",
                                 monitor="val_loss",
                                 verbose=1,
                                 save_best_only=True,
                                 mode="auto") 
                ]
    hist = model.fit_generator(training_set, steps_per_epoch = len(train_x)//BATCH_SIZE, epochs = NUM_EPOCHS, validation_data = test_set, validation_steps = len(test_set),callbacks=callbacks)
    
    model.save('age_cnn.h5')
    mk_dir("history")
    timestr = time.strftime("%Y%m%d-%H%M%S")
    pd.DataFrame(hist.history).to_hdf("./history/age_history-{}.h5".format(timestr), "history")
from utils import mk_dir, dice_coef, return_list
import torch
from torchvision import transforms
import os

DiscSeg_size = 640

# training data and label path
train_img_path = "REFUGE-Training400/Training400/Glaucoma/"
train_label_path = "Annotation-Training400/Annotation-Training400/Disc_Cup_Masks/Glaucoma/"

# validation data and label path
val_img_path = "REFUGE-Validation400/REFUGE-Validation400/"
val_label_path = "REFUGE-Validation400-GT/Disc_Cup_Masks/"

data_save_path = mk_dir("training_crop/data/")
label_save_path = mk_dir("training_crop/label/")


class MyTrainData(torch.utils.data.Dataset):
    def __init__(self, data_path, label_path, data_type, train=True):
        self.train = train
        self.data_path = data_path
        self.label_path = label_path
        if self.train:  # train the model
            self.train_list = return_list(
                data_path,
                data_type)  # the images name list of train and validation
        else:  # validate the model
            self.val_list = return_list(
                data_path,
Beispiel #31
0
def main():
    args = get_args()
    input_path = args.input
    batch_size = args.batch_size*args.gpus
    nb_epochs = args.nb_epochs
    depth = args.depth
    k = args.width
    validation_split = args.validation_split
    img_size = args.img_size
    val_path = args.val_path
    pretrained_fil = args.pretrained_fil
    input_shape = [img_size, img_size, 3]
    patience = 30
    gpu_num = args.gpus
    train_db = args.db

    logging.debug("Loading data...")
    '''
    image, gender, age, _, image_size, _ = load_data(input_path)
    X_data = image
    y_data_g = np_utils.to_categorical(gender, 2)
    y_data_a = np_utils.to_categorical(age, 101)
    '''
    batchdataload = BatchLoader(input_path,batch_size,img_size,train_db)
    valdataload = BatchLoader(val_path,batch_size,img_size)
    model = WideResNet(img_size, depth=depth, k=k)()
    #model = mini_XCEPTION(input_shape,101)
    with open(os.path.join("ag_models", "WRN_{}_{}.json".format(depth, k)), "w") as f:
        f.write(model.to_json())
    if pretrained_fil :
        model.load_weights(pretrained_fil)
    #sgd = SGD(lr=0.001, momentum=0.7, nesterov=True)
    adam = Adam(lr=0.0001,beta_1=0.9,beta_2=0.999,epsilon=1e-5)
    #model.compile(optimizer=sgd, loss=["categorical_crossentropy", "categorical_crossentropy"],
    #              metrics=['accuracy'])
    #if gpu_num >1:
        #model = multi_gpu_model(model,gpu_num)
    model.compile(optimizer=adam, loss=["categorical_crossentropy"],
                  metrics=['accuracy'])
    logging.debug("Model summary...")
    #model.count_params()
    model.summary()
    logging.debug("Saving model...")
    if not os.path.exists("./ag_models"):
        mk_dir("ag_models")

    reduce_lr = ReduceLROnPlateau(monitor="val_loss",factor=0.1,patience=patience*2,verbose=1,min_lr=0.0000001)
    early_stop = EarlyStopping('val_loss', patience=patience)
    modelcheckpoint = ModelCheckpoint("ag_models/weights.{epoch:02d}-{val_loss:.2f}.hdf5",\
                        monitor="val_loss",verbose=1,save_best_only=True,mode="auto",period=1000)
    #mk_dir("checkpoints")
    #reduce_lr = LearningRateScheduler(schedule=reduce_lr)
    #callbacks = [modelcheckpoint,early_stop,reduce_lr]
    callbacks = [modelcheckpoint,reduce_lr]
    logging.debug("Running training...")
    #whole training
    error_min = 0
    if whole_data :
        hist = model.fit_generator(data_geneter(batchdataload), steps_per_epoch=batchdataload.batch_num,
                              epochs=nb_epochs, verbose=1,
                              callbacks=callbacks,
                              validation_data=data_geneter(valdataload),
                              nb_val_samples=valdataload.batch_num,
                              nb_worker=1)
        logging.debug("Saving weights...")
        model.save_weights(os.path.join("ag_models", "WRN_{}_{}.h5".format(depth, k)),overwrite=True)
        #pd.DataFrame(hist.history).to_hdf(os.path.join("ag_models", "history_{}_{}.h5".format(depth, k)), "history")
    else:
        epoch_step = 0
        while epoch_step < nb_epochs:
            step = 0
            while step < batchdataload.batch_num:
                #X_data, y_data_g, y_data_a = batch_geneter(batchdataload)
                X_data, y_data_g = batch_geneter(batchdataload)
                #hist = model.fit(X_data, [y_data_g, y_data_a], batch_size=batch_size, epochs=1, verbose=2)
                hist = model.fit(X_data, y_data_g, batch_size=batch_size, epochs=1, verbose=2)
                step+=1
                if step % 100 ==0:
                    #val_data,val_g,val_a = batch_geneter(valdataload)
                    val_data,val_g = batch_geneter(valdataload)
                    #error_t = model.evaluate(val_data,[val_g,val_a],batch_size=batch_size,verbose=1)
                    error_t = model.evaluate(val_data,val_g,batch_size=batch_size,verbose=1)
                    print ("****** Epoch {} Step {}: ***********".format(str(epoch_step),str(step)) )
                    print (" loss: {}".format(error_t))
                    if epoch_step % 5 ==0:
                        #logging.debug("Saving weights...")
                        #val_data,val_g,val_a = batch_geneter(valdataload)
                        #error_t = model.evaluate(val_data,[val_g,val_a],batch_size=batch_size,verbose=1)
                        if error_t[4] >error_min:
                            logging.debug("Saving weights...")
                            model.save_weights(os.path.join("ag_models", "WRN_{}_{}_epoch{}_step{}.h5".format(depth, k,epoch_step,step)))
                            error_min = error_t[4]
            epoch_step+=1
            if epoch_step % 5 ==0:
                logging.debug("Saving weights...")
                #val_data,val_g,val_a = batch_geneter(valdataload)
                val_data,val_g = batch_geneter(valdataload)
                #error_t = model.evaluate(val_data,[val_g,val_a],batch_size=batch_size,verbose=1)
                error_t = model.evaluate(val_data,val_g,batch_size=batch_size,verbose=1)
                if error_t[1] >error_min:
                    model.save_weights(os.path.join("ag_models", "WRN_{}_{}_epoch{}.h5".format(depth, k,epoch_step)))
                    error_min = error_t[4]
                error_min =0
model = Model(inputs=x, outputs=[out1, out2, out3])
sgd = SGD(lr=1, momentum=0.9, nesterov=True)
model.compile(  # compile model
    optimizer=sgd,
    loss=[
        "categorical_crossentropy", "categorical_crossentropy",
        "categorical_crossentropy"
    ],
    metrics=['accuracy'])

# visualize, summarize and save model
plot_model(model, to_file='model.png')
model.summary()
print("Saving model...")
mk_dir("models")
with open(os.path.join("models", "kerasFunctionalModel.json"), "w") as f:
    f.write(model.to_json())

# load training data
print("Loading data...")
image, gender, age, ethnicity, image_size = load_data("./dage.mat")

# convert labels to truth vectors
X_data = image
y_data_g = np_utils.to_categorical(gender, 2)
y_data_a = np_utils.to_categorical(age, 117)
y_data_e = np_utils.to_categorical(ethnicity, 5)

# print shapes
print("Shapes of : image, gender, age, ethnicity...")
Beispiel #33
0
def main():
    # Set GPU memory usage
    config = tf.ConfigProto()
    config.gpu_options.allow_growth = True
    session = tf.Session(config=config)
    K.set_session(session)
    tf.logging.set_verbosity(tf.logging.ERROR)

    args = get_args()
    IMG_SIZE = args.image_size
    input_path = args.input
    batch_size = args.batch_size
    nb_epochs = args.nb_epochs
    validation_split = args.validation_split
    dataset_name = args.dataset_name
    n_age_bins = args.class_num
    embdding = args.embdding
    lr = args.lr

    logging.debug("[INFO] Loading data...")

    data_loader = DataManager(dataset_name)
    ground_truth_data = data_loader.get_data()
    train_keys, val_keys = split_data(ground_truth_data,
                                      validation_split=validation_split,
                                      do_shuffle=True)

    print("Samples: Training - {}, Validation - {}".format(
        len(train_keys), len(val_keys)))
    input_shape = (IMG_SIZE, IMG_SIZE, 3)

    image_generator = ImageGenerator(ground_truth_data,
                                     batch_size,
                                     input_shape[:2],
                                     train_keys,
                                     val_keys,
                                     path_prefix=input_path,
                                     vertical_flip_probability=0,
                                     eraser_probability=0,
                                     bins=n_age_bins)

    model = facenet_resnet(nb_class=n_age_bins,
                           embdding=embdding,
                           is_train=True,
                           weights="./models/facenet_keras_weights.h5")
    model.compile(optimizer=optimizers.SGD(lr=lr,
                                           momentum=0.9,
                                           decay=5e-4,
                                           nesterov=False),
                  loss={
                      'pred_g': focal_loss(alpha=.4, gamma=2),
                      'pred_a': mae,
                      "pred_e": "categorical_crossentropy"
                  },
                  loss_weights={
                      'pred_g': 0.2,
                      'pred_a': 1,
                      'pred_e': 0.4
                  },
                  metrics={
                      'pred_g': 'accuracy',
                      'pred_a': mae,
                      'pred_e': 'accuracy'
                  })

    logging.debug("[INFO] Saving model...")

    mk_dir("checkpoints")

    callbacks = [
        CSVLogger(os.path.join('checkpoints', 'train.csv'), append=False),
        ModelCheckpoint(os.path.join(
            'checkpoints',
            'weights.{epoch:02d}-{val_pred_g_acc:.3f}-{val_pred_a_mae:.3f}-{val_pred_e_acc:.3f}.h5'
        ),
                        monitor="val_loss",
                        verbose=1,
                        save_best_only=True,
                        mode="min"),
        # Use Stochastic Gradient Descent with Restart
        # https://github.com/emrul/Learning-Rate
        # Based on paper SGDR: STOCHASTIC GRADIENT DESCENT WITH WARM RESTARTS
        # SGDRScheduler(min_lr=lr*((0.1)**3), max_lr=lr, steps_per_epoch=np.ceil(len(train_keys) /
        #                                                                        batch_size), lr_decay=0.9, cycle_length=5, mult_factor=1.5)

        # Use Cyclical Learning Rate
        # CyclicLR(mode='triangular', step_size=np.ceil(
        #     len(train_keys)/batch_size), base_lr=lr*((0.1)**3), max_lr=lr)
        LearningRateScheduler(PolyDecay(lr, 0.9, nb_epochs).scheduler)
    ]

    logging.debug("[INFO] Running training...")

    history = model.fit_generator(
        image_generator.flow(mode='train'),
        steps_per_epoch=np.ceil(len(train_keys) / batch_size),
        epochs=nb_epochs,
        callbacks=callbacks,
        validation_data=image_generator.flow('val'),
        validation_steps=np.ceil(len(val_keys) / batch_size))

    logging.debug("[INFO] Saving weights...")

    K.clear_session()