def __singleRun(self):
path_provider = PathProvider()
ps = ProcessService(path_provider)
if self.use_profile:
path_providers = [
GroupExecutor.singleExecute(self.groups_dir, 1, self.input_dir,
self.zoom, self.use_profile),
GroupExecutor.singleExecute(self.groups_dir, 2, self.input_dir,
self.zoom, self.use_profile),
GroupExecutor.singleExecute(self.groups_dir, 3, self.input_dir,
self.zoom, self.use_profile)
]
self.mergeTiles(path_providers)
# ps.profileMergeSingleRun(self.input_dir, self.use_profile, self.zoom)
# path_provider1 = PathProvider("1\\")
# path_provider2 = PathProvider("2\\")
# path_provider3 = PathProvider("3\\")
# path_provider4 = PathProvider("4\\")
# path_providers = [path_provider1, path_provider2, path_provider3, path_provider4]
# self.mergeTiles(path_providers)
else:
ps.basicMerge(self.input_dir)
ps.basicTile(not self.use_profile, self.zoom)
Io.deleteDirectory(path_provider.output_data_path)
Io.deleteDirectory(path_provider.output_merged_path)
def mergeTiles(self, path_providers):
print("Merging tiles...")
src_paths = [provider.output_tiles_path for provider in path_providers]
Io.mergeTiles(src_paths, PathProvider().output_tiles_path, self.zoom)
for provider in path_providers:
Io.deleteDirectory(provider.output_path)
def profileMerge(self, input_dir):
self.__profilePrep(input_dir)
print('Merging...')
Io.deleteFile(self.path_provider.merged_file)
Io.makeDirectories(self.path_provider.output_merged_path)
Tr.gdalMerge(input_data=self.path_provider.output_data_path,
out_file=self.path_provider.merged_file)
def basicMerge(self, input_dir):
Io.deleteDirectory(self.path_provider.output_data_path)
Io.makeDirectories(self.path_provider.output_data_path)
Io.copyFiles(src=input_dir,
dest=self.path_provider.output_data_path,
file_names=self.file_names)
print('Merging...')
Io.deleteFile(self.path_provider.merged_file)
Io.makeDirectories(self.path_provider.output_merged_path)
Tr.gdalMerge(input_data=self.path_provider.output_data_path,
out_file=self.path_provider.merged_file,
is_pct=True)
def main(argv=None):
print(
'Usage: python src\\Main.py [-k -z <zoom_levels> -m <method> -g <groups>]'
)
input_dir = PathProvider.input_250k
zoom = '13'
use_profile = False
method = 'single'
groups = '3'
opts, args = getopt.getopt(argv, 'kz:m:g:')
for opt, arg in opts:
if opt == '-k':
use_profile = True
input_dir = PathProvider.input_50k
if opt == '-z':
zoom = arg
if opt == '-m':
method = arg
if opt == '-g':
groups = arg
if not use_profile:
groups = groups + '-250'
# --- Temp TODO: delete
# if not use_profile:
# zoom = '11'
# input_dir = PathProvider.input_250k
# else:
# zoom = '13'
# input_dir = PathProvider.input_50k
# ---
Io.makeDirectories(PathProvider.log_dir)
logger_service = LoggerService(groups,
method,
interval=10,
backup_count=15)
logger_service.start()
es = ExecutorService(groups, input_dir, zoom, use_profile)
es.execute(method)
logger_service.close()
print("EXIT")
def __groupRunRay(self):
ray.init(num_cpus=mp.cpu_count())
count = Io.getFilesCount(self.groups_dir)
path_providers = ray.get([
GroupExecutor.rayExecute.remote(self.groups_dir, i, self.input_dir,
self.zoom, self.use_profile)
for i in range(1, count + 1)
])
self.mergeTiles(path_providers)
def __groupRunMultiprocessing(self):
count = Io.getFilesCount(self.groups_dir)
params = [(self.groups_dir, i, self.input_dir, self.zoom,
self.use_profile) for i in range(1, count + 1)]
with NoDaemonProcessPool(mp.cpu_count()) as p:
path_providers = p.starmap(GroupExecutor.singleExecute, params)
p.close()
p.join()
self.mergeTiles(path_providers)
def __groupRunMultithreading(self):
count = Io.getFilesCount(self.groups_dir)
with concurrent.futures.ThreadPoolExecutor() as executor:
futures = [
executor.submit(GroupExecutor.singleExecute, self.groups_dir,
i, self.input_dir, self.zoom, self.use_profile)
for i in range(1, count + 1)
]
path_providers = [future.result() for future in futures]
self.mergeTiles(path_providers)
def gdalMerge(input_data, out_file, is_pct=False):
files_to_merge = Io.getFilesList(input_data)
if is_pct:
additional_options = ['-a_nodata', '255', '-pct']
else:
additional_options = ['-a_nodata', '0', '-co', 'PHOTOMETRIC=RGB']
params = additional_options + [
'-ot', 'Byte', '-of', 'GTiff', '-co', 'COMPRESS=DEFLATE', '-co',
'BIGTIFF=YES', '-o', out_file
]
params = params + files_to_merge
subprocess.call(["python", PathProvider.scripts_path + "gdal_merge.py"] +
params)
def translateIntoOneFile(input_data, out_path):
files = Io.getFilesList(input_data)
for file in files:
file_name = file.split('\\')
file_name_len = len(file_name)
file_name = file_name[file_name_len - 1]
params = [
'-b', '1', '-b', '2', '-b', '3', '-a_srs', 'EPSG:27700', '-of',
'GTiff', '-co', 'PHOTOMETRIC=RGB', '-co', 'COMPRESS=DEFLATE',
'-co', 'BIGTIFF=YES', file, out_path + file_name
]
subprocess.call(
[PathProvider.env_path + 'Library\\bin\\gdal_translate.exe'] +
params)
def profileToProfile(input_data, out_path):
files = Io.getFilesList(input_data)
for image in files:
in_image = Image.open(image)
out_im = ImageCms.profileToProfile(
im=in_image,
inputProfile=PathProvider.input_icc,
outputProfile=PathProvider.output_icc,
outputMode='RGB')
file_name = image.split('\\')
file_name_len = len(file_name)
file_name = file_name[file_name_len - 1]
out_im.save(out_path + file_name)
def basicTile(self, is_pct, zoom):
print('Translating')
Io.deleteFile(self.path_provider.translated_file)
Tr.gdalTranslate(input_file=self.path_provider.merged_file,
out_file=self.path_provider.translated_file,
is_pct=is_pct)
print('Warping')
Io.deleteFile(self.path_provider.warped_file)
Tr.gdalWarp(in_file=self.path_provider.translated_file,
out_file=self.path_provider.warped_file)
print('Tiling')
Io.deleteDirectory(self.path_provider.output_tiles_path)
Io.makeDirectories(self.path_provider.output_tiles_path)
Tr.gdal2Tiles(in_file=self.path_provider.warped_file,
out_dir=self.path_provider.output_tiles_path,
zoom=zoom)
# additional_options = ['-a_nodata', '255',
# '-pct']
# else:
# additional_options = ['-a_nodata', '0',
# '-co', 'PHOTOMETRIC=RGB']
# params = additional_options + [
# '-of', 'GTiff',
# '-co', 'COMPRESS=DEFLATE',
# '-co', 'BIGTIFF=YES',
# '-o', merged_file
# ]
# params = params + files_to_merge
# subprocess.call(["python", scripts_path + "gdal_merge.py"] + params)
print('Translating...')
Io.deleteFile(translated_file)
if is_pct:
additional_options = ['-expand', 'rgb']
else:
additional_options = ['-b', '1', '-b', '2', '-b', '3']
params = additional_options + [
'-ot', 'Byte', '-of', 'GTiff', '-a_srs', 'EPSG:27700', '-a_nodata',
'0', '-co', 'COMPRESS=DEFLATE', '-co', 'BIGTIFF=YES', merged_file,
translated_file
]
subprocess.call([env_path + 'Library\\bin\\gdal_translate.exe'] + params)
warp_in_file = translated_file
def profileMergeSingleRun(self, input_dir, use_profile, zoom):
# self.__profilePrep(input_dir)
Io.deleteDirectory(self.path_provider.output_tmp2_path + '0\\')
Io.deleteDirectory(self.path_provider.output_tmp2_path + '1\\')
Io.deleteDirectory(self.path_provider.output_tmp2_path + '2\\')
Io.deleteDirectory(self.path_provider.output_tmp2_path + '3\\')
Io.deleteDirectory(self.path_provider.output_tmp2_path + '4\\')
Io.makeDirectories(self.path_provider.output_tmp2_path + '0\\')
Io.makeDirectories(self.path_provider.output_tmp2_path + '1\\')
Io.makeDirectories(self.path_provider.output_tmp2_path + '2\\')
Io.makeDirectories(self.path_provider.output_tmp2_path + '3\\')
Io.makeDirectories(self.path_provider.output_tmp2_path + '4\\')
# Io.deleteFile(self.path_provider.merged_file)
# Io.makeDirectories(self.path_provider.output_merged_path)
# Io.deleteFile(self.path_provider.output_merged_path + 'merged1.tif')
# Io.deleteFile(self.path_provider.output_merged_path + 'merged2.tif')
# Io.deleteFile(self.path_provider.output_merged_path + 'merged3.tif')
# Io.deleteFile(self.path_provider.output_merged_path + 'merged4.tif')
# print('Merge 1')
# Tr.gdalMerge(input_data=self.path_provider.output_data_path + '1\\',
# out_file=self.path_provider.output_merged_path + 'merged1.tif')
# print('Merge 2')
# Tr.gdalMerge(input_data=self.path_provider.output_data_path + '2\\',
# out_file=self.path_provider.output_merged_path + 'merged2.tif')
# print('Merge 3')
# Tr.gdalMerge(input_data=self.path_provider.output_data_path + '3\\',
# out_file=self.path_provider.output_merged_path + 'merged3.tif')
# print('Merge 4')
# Tr.gdalMerge(input_data=self.path_provider.output_data_path + '4\\',
# out_file=self.path_provider.output_merged_path + 'merged4.tif')
# print('Merge all')
# Tr.gdalMerge(input_data=self.path_provider.output_merged_path,
# out_file=self.path_provider.merged_file)
Io.copyFiles(input_dir, self.path_provider.output_tmp2_path + '0\\')
reg = ('HP', 'HT', 'HU', 'HW', 'HX', 'HY', 'HZ', 'NA', 'NB', 'NC',
'ND', 'NF', 'NG', 'NH', 'NJ', 'NK', 'NL', 'NM', 'NN', 'NO')
Io.moveFiles(self.path_provider.output_tmp2_path + '0\\',
self.path_provider.output_tmp2_path + '1\\',
regex=reg)
reg = ('NR', 'NS', 'NT', 'NU', 'NW', 'NX', 'NY', 'NZ', 'OV', 'SC',
'SD', 'SE', 'TA')
Io.moveFiles(self.path_provider.output_tmp2_path + '0\\',
self.path_provider.output_tmp2_path + '2\\',
regex=reg)
reg = ('SH', 'SJ', 'SK', 'TF', 'TG', 'SM', 'SN', 'SO', 'SP', 'TL',
'TM')
Io.moveFiles(self.path_provider.output_tmp2_path + '0\\',
self.path_provider.output_tmp2_path + '3\\',
regex=reg)
reg = ('SR', 'SS', 'ST', 'SU', 'TQ', 'TR', 'SV', 'SW', 'SX', 'SY',
'SZ', 'TV')
Io.moveFiles(self.path_provider.output_tmp2_path + '0\\',
self.path_provider.output_tmp2_path + '4\\',
regex=reg)
Io.deleteDirectory(self.path_provider.output_tmp2_path + '0\\')
path_provider = PathProvider("1\\")
ps = ProcessService(path_provider)
ps.profileMerge(self.path_provider.output_tmp2_path + '1\\')
ps.basicTile(not use_profile, zoom)
Io.deleteDirectory(self.path_provider.output_tmp2_path + '1\\')
path_provider = PathProvider("2\\")
ps = ProcessService(path_provider)
ps.profileMerge(self.path_provider.output_tmp2_path + '2\\')
ps.basicTile(not use_profile, zoom)
Io.deleteDirectory(self.path_provider.output_tmp2_path + '2\\')
path_provider = PathProvider("3\\")
ps = ProcessService(path_provider)
ps.profileMerge(self.path_provider.output_tmp2_path + '3\\')
ps.basicTile(not use_profile, zoom)
Io.deleteDirectory(self.path_provider.output_tmp2_path + '3\\')
path_provider = PathProvider("4\\")
ps = ProcessService(path_provider)
ps.profileMerge(self.path_provider.output_tmp2_path + '4\\')
ps.basicTile(not use_profile, zoom)
Io.deleteDirectory(self.path_provider.output_tmp2_path + '4\\')
def __profilePrep(self, input_dir):
Io.deleteDirectory(self.path_provider.output_tmp_path)
Io.makeDirectories(self.path_provider.output_tmp_path)
Io.copyFiles(src=input_dir,
dest=self.path_provider.output_tmp_path,
file_names=self.file_names,
file_name_regex="*.tif")
print('Profiling')
Io.deleteDirectory(self.path_provider.output_data_path)
Io.makeDirectories(self.path_provider.output_data_path)
Tr.profileToProfile(input_data=self.path_provider.output_tmp_path,
out_path=self.path_provider.output_data_path)
Io.copyFiles(src=input_dir,
dest=self.path_provider.output_data_path,
file_names=self.file_names,
file_name_regex="*.TFW")
# print('Translating into one file...')
# Io.makeDirectories(self.path_provider.output_data_path)
# Tr.translateIntoOneFile(input_data=self.path_provider.output_tmp2_path,
# out_path=self.path_provider.output_data_path)
print('Cleaning temp dirs...')
Io.deleteDirectory(self.path_provider.output_tmp_path)