def load(self):
self._columns = {}
self._data = []
path = as_path(self.url)
if path is None:
return False
with open(path, "r", newline="") as f:
reader = csv.reader(f, dialect=csv.excel, quoting=csv.QUOTE_ALL)
idx_lookup = {} # {i: idx, ...}
for row in reader:
if not self._columns:
idx = 0
for i, value in enumerate(row):
value = value.strip()
if value:
self._columns[idx] = value
idx_lookup[i] = idx
idx += 1
if not self._columns:
return True
else:
self._data.append(
dict([(idx_lookup[i], DString(value.strip()))
for i, value in enumerate(row)
if i in idx_lookup]))
return True
def export_data(self, query): def get_value(item): value = item.descriptor if value is None: return "" value = value.label.value if value is None: return "" return value path = as_path(self.url, check_if_exists = False) if path is None: return wb = Workbook() wb.guess_types = True ws = wb.active ws.append(query.columns) for i in range(len(query.columns)): ws.cell(row = 1, column = i + 1).font = Font(bold = True) for row in query: ws.append([get_value(row[column]) for column in query.columns]) wb.save(path)
def time_modified(self):
path = self._path
if path is None:
path = as_path(self._value)
if not path is None:
return os.stat(path).st_mtime
return -1
def load(self):
def _get_points(shape):
points = shape.points
if hasattr(shape, "z"):
x, y = zip(*points)
return list(zip(x, y, shape.z))
return points
self._columns = {}
self._data = []
path = as_path(self.url)
if path is None:
return False
# get SRID
srid = -1
srid_vertical = -1
# TODO try to discover SRID from .prj file
# get geometries & data
geo_column = "Geometry"
sf = Reader(path)
names = []
for idx, name in enumerate(sf.fields[1:]):
self._columns[idx + 1] = name[0]
names.append(name[0])
while geo_column in names:
geo_column = geo_column + "_"
shp_type = sf.shape(0).shapeType
shapes_wkt = { # WKT formatting for geometric shapes
1: "POINT(%s)",
3: "LINESTRING(%s)",
5: "POLYGON((%s))",
8: "MULTIPOINT(%s)",
11: "POINTZ(%s)",
13: "LINESTRINGZ(%s)",
15: "POLYGONZ((%s))",
18: "MULTIPOINTZ(%s)",
21: "POINTM(%s)",
23: "LINESTRINGM(%s)",
25: "POLYGONM((%s))",
28: "MULTIPOINTM(%s)",
}
if not shp_type in shapes_wkt:
raise Exception("Unrecognized shapefile type")
geometries = [shapes_wkt[shp_type] % ", ".join([" ".join([str(p) for p in point]) for point in _get_points(shape)]) for shape in sf.shapes()]
# geometries = [wkt definition, ...] in order of records
for i, record in enumerate(sf.records()):
self._data.append(dict([(idx, DString(str(record[idx - 1]).strip())) for idx in self._columns]))
self._data[-1][0] = DGeometry(geometries[i], srid = srid, srid_vertical = srid_vertical)
self._columns[0] = geo_column
return True
def on_import(self, *args): url, format = QtWidgets.QFileDialog.getOpenFileUrl(self.view, caption = "Import User Tool", filter = "(*.txt)") url = str(url.toString()) if not url: return path = as_path(url) if path is None: return self.view.usertools.import_tool(path)
def on_export(self, *args): url, format = QtWidgets.QFileDialog.getSaveFileUrl(self.view, caption = "Export User Tool As", filter = "Text file (*.txt)") url = str(url.toString()) if not url: return path = as_path(url, check_if_exists = False) if path is None: return self.view.usertools.export_tool(self.get_selected()[0], path)
def on_load_descriptors(self):
url, format = QtWidgets.QFileDialog.getOpenFileUrl(
self.view, caption="Import Descriptors", filter="(*.txt)")
url = str(url.toString())
if not url:
return
path = as_path(url)
if path is None:
return
with open(path, "r") as f:
_, descriptors = json.load(f)
self.load_descriptors(descriptors)
self.update()
def on_connect(self):
item = self.recent_list.currentItem()
if not item:
return
row = item.data(QtCore.Qt.UserRole)
if len(row) == 1:
url = row[0]
path = as_path(url)
if path is not None:
self.view.registry.set("recent_dir", os.path.split(path)[0])
self.parent.on_connect(url, None)
elif len(row) == 2:
identifier, connstr = row
self.parent.on_connect(identifier, connstr)
def export_data(self, query):
def get_value(item):
value = item.descriptor
if value is None:
return ""
value = value.label.value
if value is None:
return ""
return value
path = as_path(self.url, check_if_exists=False)
if path is None:
return
with open(path, "w", newline="") as f:
writer = csv.writer(f, dialect=csv.excel, quoting=csv.QUOTE_ALL)
writer.writerow(query.columns)
for row in query:
writer.writerow(
[get_value(row[column]) for column in query.columns])
def load(self):
self._columns.clear()
self._data.clear()
path = as_path(self.url)
if path is None:
return False
wb = load_workbook(filename = path, read_only = True)
for sheet in wb.sheetnames:
self._columns[sheet] = {}
self._data[sheet] = []
ws = wb[sheet]
idx = 0
idx_lookup = {} # {i: idx, ...}
for i, cell in enumerate(list(ws.iter_rows(max_row = 1))[0]):
value = cell.value
if value is not None:
value = str(value).strip()
if value:
self._columns[sheet][idx] = value
idx_lookup[i] = idx
idx += 1
if not self._columns[sheet]:
continue
for row in ws.iter_rows(min_row = 2):
self._data[sheet].append({})
for i, cell in enumerate(row):
value = cell.value
if value is not None:
value = str(value).strip()
if value and (i in idx_lookup):
self._data[sheet][-1][idx_lookup[i]] = DString(value)
return True
def save_clusters_as_pdf(tgt_path, samples):
clusters = defaultdict(list) # {label: [sample, ...], ...}
for sample in samples:
if sample.cluster is None:
continue
clusters[sample.cluster].append(sample)
if not clusters:
return
collect = {}
for cluster in natsorted(clusters.keys()):
collect[cluster] = natsorted(clusters[cluster],
key=lambda sample: sample.id)
clusters = collect
renderer = QtSvg.QSvgRenderer()
w_src_max = 0
h_src_max = 0
cmax = 0
for cluster in clusters:
for sample in clusters[cluster]:
cmax += 1
src_path = as_path(sample.resource.value, check_if_exists=False)
renderer.load(src_path)
rnd_size = renderer.defaultSize()
w_src_max = max(w_src_max, rnd_size.width())
h_src_max = max(h_src_max, rnd_size.height())
printer = QtPrintSupport.QPrinter()
printer.setWinPageSize(QtGui.QPageSize.A4)
printer.setResolution(300)
printer.setOrientation(QtPrintSupport.QPrinter.Portrait)
printer.setOutputFormat(QtPrintSupport.QPrinter.PdfFormat)
printer.setOutputFileName(tgt_path)
w_max = printer.width()
h_max = printer.height()
scale = min(1, w_max / w_src_max, h_max / h_src_max)
painter = QtGui.QPainter(printer)
td = QtGui.QTextDocument()
font = td.defaultFont()
font.setPointSize(36)
td.setDefaultFont(font)
def init_page(printer, td, painter, cluster, page, new_page):
if new_page:
printer.newPage()
td.setHtml("Cluster: %s, Page: %s" % (cluster, page))
td.drawContents(painter)
return td.size().height()
cnt = 0
for cluster in clusters:
x = 0
y = 0
h_max_row = 0
page = 1
y = init_page(printer, td, painter, cluster, page, cnt > 0)
for sample in clusters[cluster]:
print("\rgen. pdf %d/%d " % (cnt + 1, cmax), end="")
cnt += 1
src_path = as_path(sample.resource.value, check_if_exists=False)
renderer.load(src_path)
rnd_size = renderer.defaultSize()
w = rnd_size.width() * scale
h = rnd_size.height() * scale
h_max_row = max(h_max_row, h)
if x + w > w_max:
x = 0
y += h_max_row
h_max_row = h
if y + h_max_row > h_max:
x = 0
page += 1
y = init_page(printer, td, painter, cluster, page, True)
renderer.render(painter, QtCore.QRectF(x, y, w, h))
x += w
painter.end()
def export_data(self, query):
def get_label(item):
label = item.descriptor
if label is None:
return None
return label.label
def abbrev_to(name, chars, columns_abbrev):
if len(name) > chars:
n = 1
while True:
name_new = name[:chars - len(str(n))] + str(n)
if not name_new in columns_abbrev.values():
return name_new
n += 1
return name
path = as_path(self.url, check_if_exists = False)
if path is None:
return
geometries = [] # [[coords, geometry_type], ...]
row_idxs = [] # [row_idx, ...]
for row_idx, row in enumerate(query):
for column in row:
label = get_label(row[column])
if label.__class__.__name__ == "DGeometry":
geometries.append(label.coords)
row_idxs.append(row_idx)
break
if not row_idxs:
return
columns_abbrev = {} # {column: column_abbrev, ...}; abbreviated column names
for column in query.columns:
column_abbrev = column
if len(column_abbrev) > 10:
if "." in column_abbrev:
column_abbrev = column_abbrev.split(".")
column_abbrev = "_".join([abbrev_to(column_abbrev[0], 4, columns_abbrev), abbrev_to(column_abbrev[1], 5, columns_abbrev)])
else:
column_abbrev = abbrev_to(column_abbrev, 10, columns_abbrev)
column_abbrev = column_abbrev.replace(".", "_")
columns_abbrev[column] = column_abbrev
shapeType = -1
shape_types = {
"POINT": 1,
"LINESTRING": 3,
"POLYGON": 5,
"MULTIPOINT": 8,
"POINTZ": 11,
"LINESTRINGZ": 13,
"POLYGONZ": 15,
"MULTIPOINTZ": 18,
"POINTM": 21,
"LINESTRINGM": 23,
"POLYGONM": 25,
"MULTIPOINTM": 28,
}
for _, geometry_type in geometries:
if geometry_type not in shape_types:
raise Exception("Unknown geometry type")
if shapeType > -1:
if shape_types[geometry_type] != shapeType:
raise Exception("Geometries must be of the same type")
else:
shapeType = shape_types[geometry_type]
sf = Writer(shapeType = shapeType)
types = {} # {column: type, ...}
shp_types = {bool: "C", int: "N", float: "N", str: "C"}
conv_order = ["N", "C"]
for row in query:
for column in row:
label = get_label(row[column])
if label.__class__.__name__ != "DString":
continue
value = label.try_numeric
typ = type(value)
typ = shp_types[typ] if typ in shp_types else "C"
if (not column in types) or ((typ != types[column]) and (conv_order.index(typ) > conv_order.index(types[column]))):
types[column] = typ
for column in types:
sf.field(columns_abbrev[column], fieldType = types[column], size = "128")
for i in range(len(geometries)):
row = query[row_idxs[i]]
coords = geometries[i][0]
if shapeType in [1, 11, 21]: # point types
sf.point(*coords[0], shapeType = shapeType)
else:
sf.poly(shapeType = shapeType, parts = [coords])
if types:
record = []
for column in types:
label = get_label(row[column])
if label is not None:
label = label.value
record.append(label)
sf.record(*record)
sf.save(path)