def open_file(self, filename, force_date = None):
self._file_list = list()
ds = None
if os.linesep in filename:
# already list of files (date range)
return filename.split(os.linesep)
mtype = mimetypes.guess_type(filename)[0]
if mtype is None or 'xml' not in mtype:
# assuming text file containing list of VFR files
try:
f = open(filename)
i = 0
lines = f.read().splitlines()
for line in lines:
if len(line) < 1 or line.startswith('#'):
continue # skip empty or commented lines
if not line.startswith('http://') and \
not line.startswith('20'):
# determine date if missing
if not force_date:
if line.startswith('ST_Z'):
date = yesterday()
else:
date = last_day_of_month()
else:
date = force_date
line = date + '_' + line
if not line.endswith('.xml.gz'):
# add extension if missing
line += '.xml.gz'
if not os.path.exists(line):
if not line.startswith('http://'):
line = 'http://vdp.cuzk.cz/vymenny_format/soucasna/' + line
line = download_vfr(line)
self._file_list.append(line)
i += 1
VfrLogger.msg("%d VFR files will be processed..." % len(self._file_list))
except IOError:
raise VfrError("Unable to read '%s'" % filename)
f.close()
else:
# single VFR file
self._file_list.append(filename)
return self._file_list
def _check_epsg(self):
if not self._conn:
return
cursor = self._conn.cursor()
try:
cursor.execute("SELECT srid FROM spatial_ref_sys WHERE srid = 5514")
except StandardError as e:
raise VfrError("PostGIS doesn't seems to be activated. %s" % e)
epsg_exists = bool(cursor.fetchall())
if not epsg_exists:
stmt = """INSERT INTO spatial_ref_sys (srid, auth_name, auth_srid, proj4text, srtext) VALUES ( 5514, 'EPSG', 5514, '+proj=krovak +lat_0=49.5 +lon_0=24.83333333333333 +alpha=30.28813972222222 +k=0.9999 +x_0=0 +y_0=0 +ellps=bessel +towgs84=570.8,85.7,462.8,4.998,1.587,5.261,3.56 +units=m +no_defs ', 'PROJCS["S-JTSK / Krovak East North",GEOGCS["S-JTSK",DATUM["System_Jednotne_Trigonometricke_Site_Katastralni",SPHEROID["Bessel 1841",6377397.155,299.1528128,AUTHORITY["EPSG","7004"]],TOWGS84[570.8,85.7,462.8,4.998,1.587,5.261,3.56],AUTHORITY["EPSG","6156"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4156"]],PROJECTION["Krovak"],PARAMETER["latitude_of_center",49.5],PARAMETER["longitude_of_center",24.83333333333333],PARAMETER["azimuth",30.28813972222222],PARAMETER["pseudo_standard_parallel_1",78.5],PARAMETER["scale_factor",0.9999],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["metre",1,AUTHORITY["EPSG","9001"]],AXIS["X",EAST],AXIS["Y",NORTH],AUTHORITY["EPSG","5514"]]')"""
cursor.execute(stmt)
self._conn.commit()
VfrLogger.msg("EPSG 5514 defined in DB")
cursor.close()
def print_summary(self):
stime = time.time()
layer_list = copy.deepcopy(self._layer_list)
if not layer_list:
for idx in range(self._ods.GetLayerCount()):
layer_list.append(self._ods.GetLayer(idx).GetName())
VfrLogger.msg("Summary")
for layer_name in layer_list:
layer = self._ods.GetLayerByName(layer_name)
if not layer:
continue
sys.stdout.write("Layer %-20s ... %10d features\n" % \
(layer_name, layer.GetFeatureCount()))
nsec = time.time() - stime
etime = str(datetime.timedelta(seconds=nsec))
VfrLogger.msg("Time elapsed: %s" % str(etime))
def run(self, append=False, extended=False):
ipass = 0
stime = time.time()
layer_list = copy.deepcopy(self._layer_list)
pg = hasattr(self, "_conn")
if pg:
self.schema_list = []
epsg_checked = False
for fname in self._file_list:
VfrLogger.msg("Processing %s (%d out of %d)..." % \
(fname, ipass+1, len(self._file_list)))
# open OGR datasource
ids = self._open_ds(fname)
if ids is None:
ipass += 1
continue # unable to open - skip
if not self.odsn:
# no output datasource given -> list available layers and exit
layer_list = self._list_layers(extended, sys.stdout)
if extended and os.path.exists(filename):
compare_list(layer_list, parse_xml_gz(filename))
else:
if self.odsn is None:
self.odsn = '.' # current directory
if pg and not epsg_checked:
# check if EPSG 5514 exists in output DB (only first pass)
self._check_epsg()
epsg_checked = True
if not layer_list:
for l in self._list_layers(fd=None):
if l not in layer_list:
layer_list.append(l)
schema_name = None
if pg:
# build datasource string per file
odsn_reset = self.odsn
if self._schema_per_file or self._schema:
if self._schema_per_file:
# set schema per file
schema_name = os.path.basename(fname).rstrip('.xml.gz').lower()
if schema_name[0].isdigit():
schema_name = 'vfr_' + schema_name
else:
schema_name = self._schema.lower()
# create schema in output DB if needed
self._create_schema(schema_name)
self.odsn += ' active_schema=%s' % schema_name
if schema_name not in self.schema_list:
self.schema_list.append(schema_name)
self._ods.Destroy() # TODO: do it better
self._ods = self._odrv.Open(self.odsn, True)
if self._ods is None:
raise VfrError("Unable to open or create new datasource '%s'" % self.odsn)
# check mode - process changes or append
mode = Mode.write
if fname.split('_')[-1][0] == 'Z':
mode = Mode.change
if pg:
# insert required
os.environ['PG_USE_COPY'] = 'NO'
elif append:
mode = Mode.append
if pg:
# force copy over insert
os.environ['PG_USE_COPY'] = 'YES'
# do the conversion
try:
nfeat = self._convert_vfr(mode, schema_name)
except RuntimeError as e:
raise VfrError("Unable to read %s: %s" % (fname, e))
if pg:
# reset datasource string per file
if self._schema_per_file or self._schema:
self.odsn = odsn_reset
self._ods.Destroy()
self._ods = self._odrv.Open(self.odsn, True)
if self._ods is None:
raise VfrError("Unable to open or create new datasource '%s'" % self.odsn)
if nfeat > 0:
append = True # append on next passes
ids.Destroy()
self._ids = None
ipass += 1
def _convert_vfr(self, mode = Mode.write, schema=None):
if self._overwrite and mode == Mode.write:
# delete also layers which are not part of ST_UKSH
for layer in ("ulice", "parcely", "stavebniobjekty", "adresnimista"):
if self._ods.GetLayerByName(layer) is not None:
self._ods.DeleteLayer(layer)
# process features marked for deletion first
dlist = None # statistics
if mode == Mode.change:
dlayer = self._ids.GetLayerByName('ZaniklePrvky')
if dlayer:
dlist = self._process_deleted_features(dlayer)
# process layers
start = time.time()
nlayers = self._ids.GetLayerCount()
nfeat = 0
for iLayer in range(nlayers):
layer = self._ids.GetLayer(iLayer)
layer_name = layer.GetName()
### force lower case for output layers, some drivers are doing
### that automatically anyway
layer_name_lower = layer_name.lower()
if self._layer_list and layer_name not in self._layer_list:
# process only selected layers
continue
if layer_name == 'ZaniklePrvky':
# skip deleted features (already done)
continue
olayer = self._ods.GetLayerByName('%s' % layer_name_lower)
sys.stdout.write("Processing layer %-20s ..." % layer_name)
if not self._overwrite and (olayer and mode == Mode.write):
sys.stdout.write(" already exists (use --overwrite or --append to modify existing data)\n")
continue
### TODO: fix output drivers not to use default geometry
### names
if self.frmt in ('PostgreSQL', 'OCI') and not self._geom_name:
if layer_name_lower == 'ulice':
self._remove_option('GEOMETRY_NAME')
self._lco_options.append('GEOMETRY_NAME=definicnicara')
else:
self._remove_option('GEOMETRY_NAME')
self._lco_options.append('GEOMETRY_NAME=definicnibod')
# delete layer if exists and append is not True
if olayer and mode == Mode.write:
if self._delete_layer(layer_name_lower):
olayer = None
# create new output layer if not exists
if not olayer:
olayer = self._create_layer(layer_name_lower, layer)
if olayer is None:
raise VfrError("Unable to export layer '%s'. Exiting..." % layer_name)
# pre-process changes
if mode == Mode.change:
change_list = self._process_changes(layer, olayer)
if dlist and layer_name in dlist: # add features to be deleted
change_list.update(dlist[layer_name])
ifeat = n_nogeom = 0
geom_idx = -1
# make sure that PG sequence is up-to-date (import for fid == -1)
fid = -1
if hasattr(self, "_conn"): # TODO (do it better)
if schema:
table_name = '%s.%s' % (schema, layer_name_lower)
else:
table_name = layer_name_lower
fid = self._get_fid_max(table_name)
if fid > 0:
self._update_fid_seq(table_name, fid)
if fid is None or fid == -1:
fid = olayer.GetFeatureCount()
# start transaction in output layer
if olayer.TestCapability(ogr.OLCTransactions):
olayer.StartTransaction()
# delete marked features first (changes only)
if mode == Mode.change and dlist and layer_name in dlist:
for fid in dlist[layer_name].keys():
olayer.DeleteFeature(fid)
# do mapping for fields (needed for Esri Shapefile when
# field names are truncated)
field_map = [i for i in range(0, layer.GetLayerDefn().GetFieldCount())]
# copy features from source to destination layer
layer.ResetReading()
feature = layer.GetNextFeature()
while feature:
# check for changes first (delete/update/add)
if mode == Mode.change:
c_fid = feature.GetFID()
action, o_fid = change_list.get(c_fid, (None, None))
if action is None:
raise VfrError("Layer %s: unable to find feature %d" % (layer_name, c_fid))
# feature marked to be changed (delete first)
if action in (Action.delete, Action.update):
olayer.DeleteFeature(o_fid)
# determine fid for new feature
if action == Action.add:
fid = -1
else:
fid = o_fid
if action == Action.delete:
# do nothing and continue
feature = layer.GetNextFeature()
ifeat += 1
continue
else:
fid += 1
# clone feature
### ofeature = feature.Clone() # replace by SetFrom()
ofeature = ogr.Feature(olayer.GetLayerDefn())
ofeature.SetFromWithMap(feature, True, field_map)
# modify geometry columns if requested
if self._geom_name:
if geom_idx < 0:
geom_idx = feature.GetGeomFieldIndex(self._geom_name)
# delete remaining geometry columns
### not needed - see SetFrom()
### odefn = ofeature.GetDefnRef()
### for i in range(odefn.GetGeomFieldCount()):
### if i == geom_idx:
### continue
### odefn.DeleteGeomFieldDefn(i)
self._modify_feature(feature, geom_idx, ofeature)
if ofeature.GetGeometryRef() is None:
n_nogeom += 1
if self._nogeomskip:
# skip feature without geometry
feature = layer.GetNextFeature()
ofeature.Destroy()
continue
# set feature id
if fid >= -1:
# fid == -1 -> unknown fid
ofeature.SetFID(fid)
# add new feature to output layer
olayer.CreateFeature(ofeature)
feature = layer.GetNextFeature()
ifeat += 1
# commit transaction in output layer
if olayer.TestCapability(ogr.OLCTransactions):
olayer.CommitTransaction()
# print statistics per layer
sys.stdout.write(" %10d features" % ifeat)
if mode == Mode.change:
n_added = n_updated = n_deleted = 0
for action, unused in change_list.itervalues():
if action == Action.update:
n_updated += 1
elif action == Action.add:
n_added += 1
else: # Action.delete:
n_deleted += 1
sys.stdout.write(" (%5d added, %5d updated, %5d deleted)" % \
(n_added, n_updated, n_deleted))
else:
sys.stdout.write(" added")
if n_nogeom > 0:
if self._nogeomskip:
sys.stdout.write(" (%d without geometry skipped)" % n_nogeom)
else:
sys.stdout.write(" (%d without geometry)" % n_nogeom)
sys.stdout.write("\n")
nfeat += ifeat
# update sequence for PG
if hasattr(self, "_conn"):
### fid = get_fid_max(userdata['pgconn'], layer_name_lower)
if fid > 0:
if schema:
table_name = '%s.%s' % (schema, layer_name_lower)
else:
table_name = layer_name_lower
self._update_fid_seq(table_name, fid)
# final statistics (time elapsed)
VfrLogger.msg("Time elapsed: %d sec" % (time.time() - start))
return nfeat
