def write_workingdir(source_id, workingDir):
source = Source.objects.get(pk = source_id)
allImages = Image.objects.filter(source = source)
# create the new Robot object
if Robot.objects.all().count() == 0:
version = 1
else:
version = Robot.objects.all().order_by('-version')[0].version + 1
newRobot = Robot(source = source, version = version, time_to_train = 1)
newRobot.path_to_model = os.path.join(MODEL_DIR, "robot" + str(newRobot.version))
# now, loop through the images and create some meta data files that MATLAB needs
pointInfoPath = os.path.join(workingDir + 'points')
fileNamesPath = os.path.join(workingDir + 'fileNames')
try:
os.mkdir(workingDir)
except OSError as e:
logging.info("Error creating workingDir: {error}".format(error=e.strerror))
raise
fItt = 0 #image iterator
pointFile = open(pointInfoPath, 'w')
fileNameFile = open(fileNamesPath, 'w')
for image in allImages:
if not image.status.featureFileHasHumanLabels:
continue
fItt = fItt + 1 #note that we start at 1, MATLAB style
featureFile = os.path.join(FEATURES_DIR, str(image.id) + "_" + image.get_process_date_short_str() + ".dat")
fileNameFile.write(featureFile + "\n") # write all file names in a file, so that MATLAB can find them
points = Point.objects.filter(image=image)
pItt = 0
print image.id
for point in points:
pItt = pItt + 1 #note that we start at 1, MATLAB style
Ann = Annotation.objects.filter(point=point, image=image)
pointFile.write(str(fItt) + ', ' + str(pItt) + ', ' + str(Ann[0].label.id) + '\n')
pointFile.close()
fileNameFile.close()
# write the label map function
labelMapPath = os.path.join(workingDir + 'labelmap')
mapFile = open(labelMapPath, 'w')
for labelgroup in LabelGroup.objects.filter():
mapFile.write(str(labelgroup.name) + ',' + str(labelgroup.id) + '\n')
mapFile.write('===\n')
for label in Label.objects.filter():
mapFile.write(str(label.id) + ',' + str(label.group_id) + '\n')
mapFile.close()
def trainRobot(source_id):
# first, see if we should train a new robot
hasNewImagesToTrainOn = False
nbrAnnotatedImages = 0
source = Source.objects.get(pk = source_id)
allImages = Image.objects.filter(source = source)
for image in allImages:
if (image.status.featureFileHasHumanLabels and not image.status.usedInCurrentModel):
hasNewImagesToTrainOn = True
if image.status.featureFileHasHumanLabels:
nbrAnnotatedImages = nbrAnnotatedImages + 1;
if ( not hasNewImagesToTrainOn or ( nbrAnnotatedImages < 5 ) ) : #TODO, add field to souce object that specify this threshold.
print 'Source ' + str(source_id) + ' has no new images to train on, aborting'
return 1
################### EVERYTHING OK, START TRAINING NEW MODEL ################
# create the new Robot object
newRobot = Robot(source=source, version = Robot.objects.all().order_by('-version')[0].version + 1, time_to_train = 1);
newRobot.path_to_model = join_project_root("images/models/robot" + str(newRobot.version))
newRobot.save();
print 'new robot version:' + str(newRobot.version)
# update the data base.
for image in allImages: # mark that these images are used in the current model.
if image.status.featureFileHasHumanLabels:
image.status.usedInCurrentModel = True;
image.status.save()
# grab the last robot
previousRobot = source.get_latest_robot()
if previousRobot == None:
oldModelPath = '';
else:
oldModelPath = previousRobot.path_to_model
print 'previous robot version:' + str(previousRobot.version)
# now, loop through the images and create some meta data files that MATLAB needs
workingDir = newRobot.path_to_model + '.workdir/'
pointInfoPath = os.path.join(workingDir + 'points')
fileNamesPath = os.path.join(workingDir, 'fileNames')
os.mkdir(workingDir)
fItt = 0 #image iterator
pointFile = open(pointInfoPath, 'w')
fileNameFile = open(fileNamesPath, 'w')
for image in allImages:
if not image.status.featureFileHasHumanLabels:
continue
fItt = fItt + 1 #note that we start at 1, MATLAB style
featureFile = os.path.join(FEATURES_DIR, str(image.id) + "_" + image.get_process_date_short_str() + ".dat")
fileNameFile.write(featureFile + "\n") # write all file names in a file, so that mablat can find them
points = Point.objects.filter(image=image)
pItt = 0
for point in points:
pItt = pItt + 1 #note that we start at 1, MATLAB style
Ann = Annotation.objects.filter(point=point, image=image)
pointFile.write(str(fItt) + ', ' + str(pItt) + ', ' + str(Ann[0].label.id) + '\n')
pointFile.close()
fileNameFile.close()
# call the matlab function
task_helpers.coralnet_trainRobot(
modelPath = newRobot.path_to_model,
oldModelPath = oldModelPath,
pointInfoPath = pointInfoPath,
fileNamesPath = fileNamesPath,
workDir = workingDir,
logFile = CV_LOG,
errorLogfile = TRAIN_ERROR_LOG,
)
# clean up
# rmtree(workingDir)
if os.path.isfile(TRAIN_ERROR_LOG):
for image in allImages: # roll back changes.
if image.status.featureFileHasHumanLabels:
image.status.usedInCurrentModel = False
image.status.save()
print("Sorry error detected in robot training!")
newRobot.delete()
else:
#if not (previousRobot == None):
#os.remove(oldModelPath) # remove old model, but keep the meta data files.
print 'Finished training new robot(' + str(newRobot.version) + ') for source id: ' + str(source_id)
def train_robot(source_id, force_train = False):
source = Source.objects.get(id = source_id)
if not source.need_new_robot() and not force_train:
return 1
allImages = Image.objects.filter(source = source, status__annotatedByHuman = True, status__featuresExtracted = True)
################### EVERYTHING OK, START TRAINING NEW MODEL ################
# create the new Robot object
if Robot.objects.all().count() == 0:
version = 1
else:
version = Robot.objects.all().order_by('-version')[0].version + 1
newRobot = Robot(source = source, version = version, time_to_train = 1)
newRobot.path_to_model = os.path.join(MODEL_DIR, "robot" + str(newRobot.version))
newRobot.save();
# update the data base.
for image in allImages: # mark that these images are used in the current model.
if not image.status.usedInCurrentModel:
image.status.usedInCurrentModel = True;
image.status.save()
# grab the last robot
previousRobot = source.get_latest_robot()
if previousRobot == None:
oldModelPath = '';
logging.info('Training first robot{id} for source{sid}: {sname}'.format(id = newRobot.version, sid = image.source_id, sname = image.source.name))
else:
oldModelPath = previousRobot.path_to_model
logging.info('Training robot{id} for source{sid}: {sname}. Previous was robot{pid}'.format(id = newRobot.version, sid = image.source_id, sname = image.source.name, pid = previousRobot.version))
# now, loop through the images and create some meta data files that MATLAB needs
workingDir = newRobot.path_to_model + '.workdir/'
pointInfoPath = os.path.join(workingDir + 'points')
fileNamesPath = os.path.join(workingDir + 'fileNames')
try:
os.mkdir(workingDir)
except OSError as e:
logging.info("Error creating workingDir: {error}".format(error=e.strerror))
raise
fItt = 0 #image iterator
pointFile = open(pointInfoPath, 'w')
fileNameFile = open(fileNamesPath, 'w')
for image in allImages:
fItt = fItt + 1 #note that we start at 1, MATLAB style
featureFile = os.path.join(FEATURES_DIR, str(image.id) + "_" + image.get_process_date_short_str() + ".dat")
fileNameFile.write(featureFile + "\n") # write all file names in a file, so that MATLAB can find them
points = Point.objects.filter(image=image)
pItt = 0
for point in points:
pItt = pItt + 1 #note that we start at 1, MATLAB style
Ann = Annotation.objects.filter(point=point, image=image)
pointFile.write(str(fItt) + ', ' + str(pItt) + ', ' + str(Ann[0].label.id) + '\n')
pointFile.close()
fileNameFile.close()
# write the label map function
labelMapPath = os.path.join(workingDir + 'labelmap')
mapFile = open(labelMapPath, 'w')
for labelgroup in LabelGroup.objects.filter():
mapFile.write(str(labelgroup.name) + ',' + str(labelgroup.id) + '\n')
mapFile.write('===\n')
for label in Label.objects.filter():
mapFile.write(str(label.id) + ',' + str(label.group_id) + '\n')
mapFile.close()
# call the matlab function
task_helpers.coralnet_trainRobot(
modelPath = newRobot.path_to_model,
oldModelPath = oldModelPath,
pointInfoPath = pointInfoPath,
fileNamesPath = fileNamesPath,
workDir = workingDir,
logFile = CV_LOG,
errorLogfile = TRAIN_ERROR_LOG,
)
if os.path.isfile(TRAIN_ERROR_LOG):
for image in allImages: # roll back changes.
image.status.usedInCurrentModel = False
image.status.save()
logging.info('ERROR training robot{id} for source{sid}: {sname}'.format(id = newRobot.version, sid = image.source_id, sname = image.source.name))
mail_admins('CoralNet Backend Error', 'in trainRobot')
os.rename(TRAIN_ERROR_LOG, TRAIN_ERROR_LOG + '.' + str(source_id)) # change the name so it won't interfere with other sources.
# newRobot.delete() Don't delete the model. It won't bother us anyays since it lacks the needed metadata.
return 0
else:
copyfile(newRobot.path_to_model + '.meta_all.png', os.path.join(ALLEVIATE_IMAGE_DIR, str(newRobot.version) + '.png')) #copy to the media folder where it can be viewed
if not (previousRobot == None):
os.remove(oldModelPath) # remove old model, but keep the meta data files.
shutil.rmtree(workingDir) # only remove temp files if everything worked. Else keep it for debugging purposes.
logging.info('Done training robot{id} for source{sid}: {sname}'.format(id = newRobot.version, sid = image.source_id, sname = image.source.name))
return 1