def _convert_lccm_input(self, flt_directory_in, flt_directory_out):
gc.collect()
t1 = time()
lc = LandCoverDataset(in_storage=StorageFactory().get_storage(
'flt_storage', storage_location=flt_directory_in),
out_storage=StorageFactory().get_storage(
'flt_storage',
storage_location=flt_directory_out))
# lc.get_header() # added 23 june 2009 by mm
mask = lc.get_mask()
idx = where(mask == 0)[0]
lcsubset = DatasetSubset(lc, idx)
print "Converting:"
lcsubset.write_dataset(attributes=["relative_x"],
out_table_name="land_covers")
lc.delete_one_attribute("relative_x")
lcsubset.write_dataset(attributes=["relative_y"],
out_table_name="land_covers")
lc.delete_one_attribute("relative_y")
lc.flush_dataset()
gc.collect()
# lc_names = lc.get_primary_attribute_names()
for attr in lc.get_primary_attribute_names():
print " ", attr
lcsubset.write_dataset(attributes=[attr],
out_table_name="land_covers")
lc.delete_one_attribute(attr)
logger.log_status("Data conversion done. " + str(time() - t1) + " s")
def _convert_lccm_input(self, flt_directory_in, flt_directory_out):
gc.collect()
t1 = time()
lc = LandCoverDataset(in_storage = StorageFactory().get_storage('flt_storage', storage_location = flt_directory_in),
out_storage = StorageFactory().get_storage('flt_storage', storage_location = flt_directory_out))
# lc.get_header() # added 23 june 2009 by mm
mask = lc.get_mask()
idx = where(mask==0)[0]
lcsubset = DatasetSubset(lc, idx)
print "Converting:"
lcsubset.write_dataset(attributes=["relative_x"], out_table_name="land_covers")
lc.delete_one_attribute("relative_x")
lcsubset.write_dataset(attributes=["relative_y"], out_table_name="land_covers")
lc.delete_one_attribute("relative_y")
lc.flush_dataset()
gc.collect()
# lc_names = lc.get_primary_attribute_names()
for attr in lc.get_primary_attribute_names():
print " ", attr
lcsubset.write_dataset(attributes=[attr], out_table_name="land_covers")
lc.delete_one_attribute(attr)
logger.log_status("Data conversion done. " + str(time()-t1) + " s")
input_year = sys.argv[1]
flt_directory_in = options.input
flt_directory_out = options.output
print flt_directory_out
test_flag = options.test_flag
shutil.rmtree(flt_directory_out)
os.mkdir(flt_directory_out)
logger.log_status("Convert input data from ", str(input_year))
lc = LandCoverDataset(in_storage = StorageFactory().get_storage('flt_storage', storage_location = flt_directory_in),
out_storage = StorageFactory().get_storage('flt_storage', storage_location = flt_directory_out), debuglevel=4)
lc.get_header()
# mask = lc.get_mask()
# idx = where(mask==0)[0]
# lcsubset = DatasetSubset(lc, idx)
print "Creating and writing relative_x and relative_y:"
lc.write_dataset(attributes=["relative_x"], out_table_name="land_covers",
valuetypes=valuetypes)
lc.delete_one_attribute("relative_x")
lc.write_dataset(attributes=["relative_y"], out_table_name="land_covers",
valuetypes=valuetypes)
lc.delete_one_attribute("relative_y")
print "done."
#years = [1995, 1999]
#years = [2002]
#years = sys.argv[3]
years = [2007, 2007]
lc1 = LandCoverDataset(in_storage = StorageFactory().get_storage('flt_storage',
storage_location = os.path.join(flt_directory_in, str(years[0]))),
out_storage = StorageFactory().get_storage('flt_storage',
storage_location = os.path.join(flt_directory_out, str(years[0]))))
agents_index = where(lc1.get_attribute(index_attribute))[0]
lc1subset = DatasetSubset(lc1, agents_index)
print "Writing set 1:"
for attr in lc1.get_primary_attribute_names():
print " ", attr
lc1subset.write_dataset(attributes=[attr], out_table_name="land_covers")
lc1.delete_one_attribute(attr) # leaving this line in causes the processing of every other input data file; commenting it causes memory error
lc2 = LandCoverDataset(in_storage = StorageFactory().get_storage('flt_storage',
storage_location = os.path.join(flt_directory_in, str(years[1]))),
out_storage = StorageFactory().get_storage('flt_storage',
storage_location = os.path.join(flt_directory_out, str(years[1]))))
lc2subset = DatasetSubset(lc2, agents_index)
print "Writing set 2:"
for attr in lc2.get_primary_attribute_names():
print " ", attr
lc2subset.write_dataset(attributes=[attr], out_table_name="land_covers")
lc2.delete_one_attribute(attr) # leaving this line in causes the processing of every other input data file ; commenting it causes memory error
logger.log_status("Data storage done. " + str(time()-t1) + " s")
lc1 = LandCoverDataset(in_storage=StorageFactory().get_storage(
'flt_storage',
storage_location=os.path.join(flt_directory_in, str(years[0]))),
out_storage=StorageFactory().get_storage(
'flt_storage',
storage_location=os.path.join(
flt_directory_out, str(years[0]))))
agents_index = where(lc1.get_attribute(index_attribute))[0]
lc1subset = DatasetSubset(lc1, agents_index)
print "Writing set 1:"
for attr in lc1.get_primary_attribute_names():
print " ", attr
lc1subset.write_dataset(attributes=[attr], out_table_name="land_covers")
lc1.delete_one_attribute(
attr
) # leaving this line in causes the processing of every other input data file; commenting it causes memory error
lc2 = LandCoverDataset(in_storage=StorageFactory().get_storage(
'flt_storage',
storage_location=os.path.join(flt_directory_in, str(years[1]))),
out_storage=StorageFactory().get_storage(
'flt_storage',
storage_location=os.path.join(
flt_directory_out, str(years[1]))))
lc2subset = DatasetSubset(lc2, agents_index)
print "Writing set 2:"
for attr in lc2.get_primary_attribute_names():
print " ", attr
lc2subset.write_dataset(attributes=[attr], out_table_name="land_covers")
