def __init__(self, model_name, root_dir, resume, k, loss, accs):
self.model_name = model_name
self.tb_dir = f"{root_dir}logs/tensorboard/"
self.tb_train_dir = f"{self.tb_dir}training/{self.model_name}/"
self.tb_finish_dir = f"{self.tb_dir}finished/{self.model_name}/"
existsdir = exists(self.tb_train_dir, type_="dir")
if not resume and existsdir:
self._tb_move(k)
elif not existsdir:
remake_dir(self.tb_train_dir)
self.tb_writer = SummaryWriter(log_dir=self.tb_train_dir)
if resume and not existsdir:
for epoch_1, epoch_loss in enumerate(loss):
epoch = epoch_1 + 1
epoch_acc = accs[epoch_1] if any(accs) else None
self.plot(epoch, epoch_loss, epoch_acc)
def createSdilesDict(self, basenc, dile, sdilesfolder):
variables = [v for v in basenc.variables.values() if v.name not in [d.name for d in basenc.dimensions.values()]]
sdiles = {}
for v in variables:
filename = ("sdile_"+
v.name+"_"+
str(dile.z)+"_"+
str(dile.y)+"_"+
str(dile.x)+".nc")
print filename
if exists(sdilesfolder, filename):
sdiles.update({v.name: Dataset(sdilesfolder+filename, mode='a',clobber = False)})
else:
d = [d for d in basenc.variables.values() if d.name in v.dimensions and d.name not in ['lat','lon']]
sdiles.update({v.name:self.initSdile(dile, v, d, sdilesfolder, filename)})
return sdiles
sdiles = kg.createSdilesDict(basenc, dile, sdilesfolder)
timer.stop()
print "super diles created in ", timer.formatted()
timer.reset('sdiles created')
# uris extraction
print "extracting uri list..."
timer.start()
urifolder = basefolder+"uris/"
name = "dile_"+str(dile.z)+"_"+str(dile.y)+"_"+str(dile.x)+"_"+"urilist.json"
if exists(urifolder,name):
urls = retrive(urifolder,name)
else:
urls = kg.getUriList(bboxidxs, baselink)
serialize(urls,urifolder,name)
timer.stop()
print len(urls)," extracted in ", timer.formatted()
timer.reset('uris extraction')
# index initialization
indexfolder = basefolder+"index/"
name = "dile_"+str(dile.z)+"_"+str(dile.y)+"_"+str(dile.x)+"_"+"index.json"
