def read_csv(files):
"""
Read a group of CSV properly formated CSV files and create the program data structures
:param files: dict {file#: ("root", "map_path", "hash_path")}
:return: (mappings dict, file_hash dict)
mappings dict: {(file#,line#): ("old_path", "int_path", "ext_path",...)}
file_hash dict: {(file#, "path"): hash}
"""
mappings = {}
file_hash = {}
for file_num in files:
root, map_path, hash_path = files[file_num]
if map_path is not None:
with csv23.open(map_path, "r") as csv_file:
csv_reader = csv.reader(csv_file)
next(csv_reader) # ignore the header
line_num = 1
for row in csv_reader:
row = csv23.fix(row)
line_num += 1
mappings[(file_num, line_num)] = tuple(row)
if hash_path is not None:
with csv23.open(hash_path, "r") as csv_file:
csv_reader = csv.reader(csv_file)
next(csv_reader) # ignore the header
for row in csv_reader:
row = csv23.fix(row)
path = os.path.join(row[0], row[1])
file_hash[(file_num, path)] = row[2]
if len(mappings) == 0:
mappings = None
if len(file_hash) == 0:
file_hash = None
return mappings, file_hash
def print_errors(errors, files=None, csv_path=None):
"""
Writes a list of errors to a CSV file, or prints to the console
If files is provided, then the root of the file data is printed, instead of the file number.
:param errors: list [(file#, line#, "Issue")]
:param files: dict {file#: ("root", "map_path", "hash_path")}
:param file_path: string path to location to create a CSV file. If None, print to console
:return: None
"""
errors.sort()
if csv_path is None:
for file_num, line_num, issue in errors:
if files is not None:
root = files[file_num][0]
else:
root = file_num
print("{0}, {1}, {2}".format(root, line_num, issue))
else:
with csv23.open(csv_path, "w") as csv_file:
csv_writer = csv.writer(csv_file)
header = ["file", "line", "issue"]
csv23.write(csv_writer, header)
for file_num, line_num, issue in errors:
if files is not None:
root = files[file_num]
else:
root = file_num
row = [root, line_num, issue]
csv23.write(csv_writer, row)
def unzip():
""" Just do it """
with csv23.open(Config.csv_path, "r") as handle:
handle.readline() # remove header
csvreader = csv.reader(handle)
for row in csvreader:
row = csv23.fix(row)
park = row[0]
plot = row[1]
date = row[5]
suffix = row[6]
source = row[13]
if source:
local = source.replace(Config.remote, Config.local_zip)
dest = os.path.join(Config.local_img, park, plot, date)
if suffix:
dest += suffix
print("unzip", local, dest)
try:
os.makedirs(dest)
except OSError:
print("Failed to create dir", dest)
continue
with zipfile.ZipFile(local, "r") as zip_ref:
zip_ref.extractall(dest)
def check_paths(csv_path):
"""Check that the paths in csv_path exist in the filesystem."""
line = 0
missing_paths = []
with csv23.open(csv_path, "r") as csv_file:
csv_reader = csv.reader(csv_file)
for row in csv_reader:
row = csv23.fix(row)
line += 1
if line == 1:
# skipping the header
continue
name = row[Config.name_index]
folder = row[Config.folder_index]
if not name or not folder:
print("Bad record at line {0}".format(line))
continue
if Config.path_root is None:
path = os.path.join(folder, name)
else:
path = os.path.join(Config.path_root, folder, name)
if not os.path.exists(path):
missing_paths.append((path, line))
for path, line in sorted(missing_paths):
print("Path `{0}` not found at line {1}".format(path, line))
def readfile(filename):
"""Read the input CSV data in filename (formatted as described above)."""
data = {}
with csv23.open(filename, "r") as in_file:
reader = csv.reader(in_file)
next(reader) # ignore the header
for row in reader:
row = csv23.fix(row)
# print(row[2:7])
site = row[0]
team = row[2]
quad = row[3]
name = "{0}|{1}{2}".format(site, team, quad)
corner = int(row[4][-1:])
if corner not in (1, 2):
print(site, team, name, name, corner)
continue
# pylint: disable=invalid-name
# I think x,y,z, while too short, are the best variables names here.
x = float(row[5])
y = float(row[6])
z = float(row[7])
if name not in data:
data[name] = {corner: (x, y, z)}
else:
data[name][corner] = (x, y, z)
return data
def main():
csv_path = r"data/reorg.csv"
with csv23.open(csv_path, "r") as csv_file:
csv_reader = csv.reader(csv_file)
next(csv_reader) # ignore the header
# check_unique_sources(csv_reader)
check_unusual(csv_reader)
def write_folder_to_file(folder, csv_path):
hashlist = get_file_hashes(folder)
with csv23.open(csv_path, "w") as csv_file:
csv_writer = csv.writer(csv_file)
header = ["path", "name", "hash"]
csv23.write(csv_writer, header)
for row in hashlist:
csv23.write(csv_writer, row)
def read_csv(csv_path):
"""Read the list of photos from csv_path."""
rows = []
with csv23.open(csv_path, "r") as csv_file:
csv_reader = csv.reader(csv_file)
next(csv_reader) # skip the header
for row in csv_reader:
row = csv23.fix(row)
rows.append(row)
return rows
def make_csv(kind, fields):
"""Write a CSV file for all the fields found for kind."""
data = get_all_items(kind)
with csv23.open(kind + ".csv", "w") as out_file:
writer = csv.writer(out_file)
csv23.write(writer, fields)
for item in data:
values = simplify(item, fields)
csv23.write(writer, values)
def writedata(filename, data):
"""Write the data as CSV in filename."""
# pylint: disable=too-many-locals
with csv23.open(filename, "w") as out_file:
writer = csv.writer(out_file)
header = [
"Site",
"Team",
"Quad",
"Corner",
"Easting",
"Northing",
"SideLength",
"Slope(%)",
]
csv23.write(writer, header)
for name in sorted(data.keys()):
site, teamquad = name.split("|")
team = teamquad[:1]
quad = teamquad[1:]
if 1 in data[name]:
p_1 = data[name][1]
else:
print("Error Corner 1 not found in " + name)
continue
if 2 in data[name]:
p_3 = data[name][2]
else:
print("Error Corner 2 not found in " + name)
continue
p_2, p_4, side_len, slope = corners3d(p_1, p_3)
row = [
site,
team,
quad,
3,
p_2[0],
p_2[1],
round(side_len, 3),
round(slope, 3),
]
csv23.write(writer, row)
row = [
site,
team,
quad,
4,
p_4[0],
p_4[1],
round(side_len, 3),
round(slope, 3),
]
csv23.write(writer, row)
def ashowone(park, csv_path=None):
if csv_path:
with csv23.open(csv_path, "w") as csv_file:
csv_writer = csv.writer(csv_file)
csv23.write(csv_writer, asset_header)
for row in assets(park):
csv23.write(csv_writer, row)
else:
print(",".join(asset_header))
for item in assets(park):
print(",".join([my_str(x) for x in item]))
def load_csv_file(csv_path):
"""Return a list of the rows in the CSV."""
records = []
with csv23.open(csv_path, "r") as csv_file:
csv_reader = csv.reader(csv_file)
next(csv_reader) # ignore the header
for row in csv_reader:
row = csv23.fix(row)
records.append(row)
return records
def read_csv_map(csv_path):
mappings = {}
with csv23.open(csv_path, "r") as csv_file:
csv_reader = csv.reader(csv_file)
next(csv_reader) # ignore the header
for row in csv_reader:
row = csv23.fix(row)
old_path = row[0]
new_path = row[2] if row[1] is None else row[1]
mappings[old_path] = new_path
return mappings
def build_csv(csv_path):
with csv23.open(csv_path, "w") as csv_file:
csv_writer = csv.writer(csv_file)
csv23.write(csv_writer, table_column_names)
for site in sites:
site_id = sites[site]
for asset_code in asset_types:
data = [
site, "{0}".format(asset_code), asset_types[asset_code]
]
print(data)
response = location_query(site_id, "{0}".format(asset_code))
convert_xml_to_csv(data, response, csv_writer)
def lshowall(csv_path=None):
if csv_path:
with csv23.open(csv_path, "w") as csv_file:
csv_writer = csv.writer(csv_file)
csv23.write(csv_writer, loc_header)
for park in sites:
for row in locations(park):
csv23.write(csv_writer, row)
else:
print(",".join(loc_header))
for park in sites:
for item in locations(park):
print(",".join([my_str(x) for x in item]))
def main(folder, ext="tif", csv_path=None):
"""Create an outputter (CSV of print) and output files below folder."""
if csv_path is None:
print(",".join(["folder", "filename", "ext", "size"]))
check_folder(folder, ext)
else:
with csv23.open(csv_path, "w") as csv_file:
csv_writer = csv.writer(csv_file)
def put_in_csv(row):
csv23.write(csv_writer, row)
put_in_csv(["folder", "filename", "ext", "size"])
check_folder(folder, ext, put_in_csv)
def read_csv_map(csv_path):
mappings = {}
with csv23.open(csv_path, "r") as csv_file:
csv_reader = csv.reader(csv_file)
next(csv_reader) # ignore the header
for row in csv_reader:
row = csv23.fix(row)
old_fgdb = row[0]
mosaic = row[1]
old_path = row[2]
new_fgdb = row[3]
new_path = row[4]
new_fgdb_mosaic = os.path.join(new_fgdb, mosaic)
if new_fgdb_mosaic not in mappings:
mappings[new_fgdb_mosaic] = []
mappings[new_fgdb_mosaic].append((old_path, new_path))
return mappings
def create_csv(csv_path, folder):
"""Create a CSV file describing all the photos in folder."""
# pylint: disable=too-many-locals
header = "folder,photo,id,namedate,exifdate,lat,lon,gpsdate,size,filedate"
with csv23.open(csv_path, "w") as csv_file:
csv_writer = csv.writer(csv_file)
csv23.write(csv_writer, header.split(","))
for root, dirs, files in os.walk(root):
folder = os.path.basename(root)
# Remove the skip dirs from the current sub directories
for skipdir in Config.skip_dirs:
if skipdir in dirs:
dirs.remove(skipdir)
for filename in files:
base, extension = os.path.splitext(filename)
if extension.lower() == ".jpg":
code, name_date = get_name_parts(base)
path = os.path.join(root, filename)
size = os.path.getsize(path)
with open(path, "rb") as in_file:
# exifread wants binary data
tags = exifread.process_file(in_file, details=False)
lat = get_latitude(tags)
lon = get_longitude(tags)
exif_date = get_exif_date(tags)
gps_date = get_gps_date(tags)
file_date = datetime.datetime.fromtimestamp(
os.path.getmtime(path)).isoformat()
# convert date to a microsoft excel acceptable ISO format
file_date = file_date.replace("T", " ")
row = [
folder,
filename,
code,
name_date,
exif_date,
lat,
lon,
gps_date,
size,
file_date,
]
csv23.write(csv_writer, row)
def files_in_csv(csv_path):
"""Return a set of standardized relative paths to photos in the file at csv_path."""
paths = set()
with csv23.open(csv_path, "r") as csv_file:
csv_reader = csv.reader(csv_file)
next(csv_reader) # skip the header
for row in csv_reader:
row = csv23.fix(row)
path = row[Config.name_index]
if Config.folder_index is not None:
folder = row[Config.folder_index]
path = os.path.join(folder, path)
if Config.unit_index is not None:
folder = row[Config.unit_index]
path = os.path.join(folder, path)
path = standardize(path)
paths.add(path)
return paths
def main():
""" Just do it """
with csv23.open(Config.csv_path, "r") as handle:
csvreader = csv.reader(handle)
next(csvreader) # remove header
for row in csvreader:
row = csv23.fix(row)
source = row[13]
if source:
local = source.replace(Config.remote, Config.local_zip)
# print("copy", source, local)
if not os.path.exists(source):
print("File not found", source)
else:
print("copy {0}".format(os.path.basename(local)))
try:
shutil.copy(source, local)
except IOError:
os.makedirs(os.path.dirname(local))
shutil.copy(source, local)
1) Themes (layer files) in TM database that are not in the TM filesystem
2) Themes (layer files) in TM filesystem that are not in the TM database
"""
from __future__ import absolute_import, division, print_function, unicode_literals
import csv
import os
import csv23
tm_filesystem = r"X:\GIS\ThemeMgr"
tm_database = r"data\TM_20171206.csv"
unique_themes = set([])
with csv23.open(tm_database, "r") as csv_file:
csv_reader = csv.reader(csv_file)
next(csv_reader) # ignore the header
for row in csv_reader:
row = csv23.fix(row)
theme = row[3]
unique_themes.add(theme)
print("Missing Themes:")
for theme in sorted(unique_themes):
if theme and not os.path.exists(theme):
print(" " + theme)
print("Extra Themes:")
for root, dirs, files in os.walk(tm_filesystem):
if ".git" in dirs:
def test_csv(csv_path):
with csv23.open(csv_path, "w") as csv_file:
csv_writer = csv.writer(csv_file)
csv23.write(csv_writer, table_column_names)
response = test_service()
convert_xml_to_csv(["ANIA", "4100", "Building"], response, csv_writer)
Checks the file used to fix the PDS
"""
from __future__ import absolute_import, division, print_function, unicode_literals
import csv
import os
import csv23
old_x = r"\\INPAKROVMDIST\gisdata"
new_x = r"\\INPAKROVMDIST\gisdata2"
csv_path = r"data\PDS Fixes - X.csv"
unique_themes = set([])
with csv23.open(csv_path, "r") as csv_file:
csv_reader = csv.reader(csv_file)
next(csv_reader) # ignore the header
line = 1
for row in csv_reader:
row = csv23.fix(row)
line += 1
# check that time stamp is valid and increasing
old_workspace = None if len(row[1].strip()) == 0 else row[1]
old_datasource = None if len(row[3].strip()) == 0 else row[3]
new_workspace = None if len(row[5].strip()) == 0 else row[5]
if old_workspace is None:
print("old workspace is missing from line {}".format(line))
if new_workspace is None:
print("new workspace is missing from line {}".format(line))
if old_workspace is None or new_workspace is None:
def organize():
"""Add additional attributes to the ifsar file list."""
# pylint: disable=too-many-locals,too-many-branches,too-many-statements
with csv23.open(Config.out_path, "w") as out_file:
csv_writer = csv.writer(out_file)
header = [
"folder",
"filename",
"ext",
"size",
"legacy",
"nga",
"kind",
"edge",
"cell",
"lat",
"lon",
"tfw",
"xml",
"html",
"txt",
"tif_xml",
"ovr",
"aux",
"rrd",
"aux_old",
"crc",
"extras",
"skip",
]
csv23.write(csv_writer, header)
with csv23.open(Config.csv_path, "r") as csv_file:
csv_reader = csv.reader(csv_file)
next(csv_reader) # ignore the header
for row in csv_reader:
row = csv23.fix(row)
path_in, name, ext, size = row
name = name.lower()
path = path_in.lower()
legacy = "N"
if "legacy" in path:
legacy = "Y"
nga = "N"
if "nga_30" in path:
nga = "Y"
kind = ""
if "ori" in name or ("ori" in path and "priority" not in path):
kind = "ori"
if kind == "ori" and "_sup" in name:
kind = "ori_sup"
if "dsm" in name or "dsm" in path:
kind = "dsm"
if "dtm" in name or "dtm" in path:
kind = "dtm"
edge = "N"
if "edge" in path:
edge = "Y"
cell = ""
lat, lon = 0, 0
match = re.search(r"\\cell_(\d*)\\", path)
if match:
cell = match.group(1)
match = re.search(r"\\cell_([def])\\",
path) # path -> 196, e -> 197, f -> 198
if match:
cell = ord(match.group(1)) - ord("d") + 196
match = re.search(r"_n(\d*)w(\d*)", name)
if match:
lat = int(match.group(1)) / 100
lon = int(match.group(2)) / 100
#
# Check for supplemental *.html, *.aux.xml, etc files
#
file_path = os.path.join(path_in, name)
exts_found = [
sup.replace(file_path, "").lower()
for sup in glob.glob(file_path + ".*")
]
# exts_possible = ['.tif', '.tfw','.xml','.html','.txt','.tif.xml',
# '.tif.ovr','.tif.aux.xml', '.rrd', '.aux', '.tif.crc']
tfw, xml, html, txt, tif_xml = 0, 0, 0, 0, 0
ovr, aux, rrd, aux_old, crc = 0, 0, 0, 0, 0
if ".tfw" in exts_found:
tfw = 1
if ".xml" in exts_found:
xml = 1
if ".html" in exts_found:
html = 1
if ".txt" in exts_found:
txt = 1
if ".tif.xml" in exts_found:
tif_xml = 1
if ".tif.ovr" in exts_found:
ovr = 1
if ".tif.aux.xml" in exts_found:
aux = 1
if ".rrd" in exts_found:
rrd = 1
if ".aux" in exts_found:
aux_old = 1
if ".tif.crc" in exts_found:
crc = 1
extras = (len(exts_found) - 1 - tfw - xml - html - txt -
tif_xml - ovr - aux - rrd - aux_old - crc
) # 1 for the tif that must exist
out_row = [
path_in,
name,
ext,
size,
legacy,
nga,
kind,
edge,
cell,
lat,
lon,
tfw,
xml,
html,
txt,
tif_xml,
ovr,
aux,
rrd,
aux_old,
crc,
extras,
"N",
]
csv23.write(csv_writer, out_row)
def load_csv(csv_path, col=0):
with csv23.open(csv_path, "r") as csv_file:
csv_reader = csv.reader(csv_file)
next(csv_reader) # ignore the header
data = [csv23.fix(row)[col] for row in csv_reader]
return data
import csv23
class Config(object):
"""Namespace for configuration parameters. Edit as needed."""
# pylint: disable=useless-object-inheritance,too-few-public-methods
# Write the results to this file path.
results = r"c:\tmp\sr.csv"
# The folder to search.
start = r"c:\tmp\Changing Tides"
with csv23.open(Config.results, "w") as f:
csv_writer = csv.writer(f)
header = ["mxd", "data_frame", "spatial_reference"]
csv23.write(csv_writer, header)
csv_writer.writerow()
for root, dirs, files in os.walk(Config.start):
for file in files:
if os.path.splitext(file)[1].lower() == ".mxd":
suspect = os.path.join(root, file)
print("Checking {}".format(suspect))
try:
mxd = arcpy.mapping.MapDocument(suspect)
for df in arcpy.mapping.ListDataFrames(mxd):
print(" data frame {0} has spatial reference: {1}".
format(df.name, df.spatialReference.name))
row = [suspect, df.name, df.spatialReference.name]
