def read_csv(files): """ Read a group of CSV properly formated CSV files and create the program data structures :param files: dict {file#: ("root", "map_path", "hash_path")} :return: (mappings dict, file_hash dict) mappings dict: {(file#,line#): ("old_path", "int_path", "ext_path",...)} file_hash dict: {(file#, "path"): hash} """ mappings = {} file_hash = {} for file_num in files: root, map_path, hash_path = files[file_num] if map_path is not None: with csv23.open(map_path, "r") as csv_file: csv_reader = csv.reader(csv_file) next(csv_reader) # ignore the header line_num = 1 for row in csv_reader: row = csv23.fix(row) line_num += 1 mappings[(file_num, line_num)] = tuple(row) if hash_path is not None: with csv23.open(hash_path, "r") as csv_file: csv_reader = csv.reader(csv_file) next(csv_reader) # ignore the header for row in csv_reader: row = csv23.fix(row) path = os.path.join(row[0], row[1]) file_hash[(file_num, path)] = row[2] if len(mappings) == 0: mappings = None if len(file_hash) == 0: file_hash = None return mappings, file_hash
def print_errors(errors, files=None, csv_path=None): """ Writes a list of errors to a CSV file, or prints to the console If files is provided, then the root of the file data is printed, instead of the file number. :param errors: list [(file#, line#, "Issue")] :param files: dict {file#: ("root", "map_path", "hash_path")} :param file_path: string path to location to create a CSV file. If None, print to console :return: None """ errors.sort() if csv_path is None: for file_num, line_num, issue in errors: if files is not None: root = files[file_num][0] else: root = file_num print("{0}, {1}, {2}".format(root, line_num, issue)) else: with csv23.open(csv_path, "w") as csv_file: csv_writer = csv.writer(csv_file) header = ["file", "line", "issue"] csv23.write(csv_writer, header) for file_num, line_num, issue in errors: if files is not None: root = files[file_num] else: root = file_num row = [root, line_num, issue] csv23.write(csv_writer, row)
def unzip(): """ Just do it """ with csv23.open(Config.csv_path, "r") as handle: handle.readline() # remove header csvreader = csv.reader(handle) for row in csvreader: row = csv23.fix(row) park = row[0] plot = row[1] date = row[5] suffix = row[6] source = row[13] if source: local = source.replace(Config.remote, Config.local_zip) dest = os.path.join(Config.local_img, park, plot, date) if suffix: dest += suffix print("unzip", local, dest) try: os.makedirs(dest) except OSError: print("Failed to create dir", dest) continue with zipfile.ZipFile(local, "r") as zip_ref: zip_ref.extractall(dest)
def check_paths(csv_path): """Check that the paths in csv_path exist in the filesystem.""" line = 0 missing_paths = [] with csv23.open(csv_path, "r") as csv_file: csv_reader = csv.reader(csv_file) for row in csv_reader: row = csv23.fix(row) line += 1 if line == 1: # skipping the header continue name = row[Config.name_index] folder = row[Config.folder_index] if not name or not folder: print("Bad record at line {0}".format(line)) continue if Config.path_root is None: path = os.path.join(folder, name) else: path = os.path.join(Config.path_root, folder, name) if not os.path.exists(path): missing_paths.append((path, line)) for path, line in sorted(missing_paths): print("Path `{0}` not found at line {1}".format(path, line))
def readfile(filename): """Read the input CSV data in filename (formatted as described above).""" data = {} with csv23.open(filename, "r") as in_file: reader = csv.reader(in_file) next(reader) # ignore the header for row in reader: row = csv23.fix(row) # print(row[2:7]) site = row[0] team = row[2] quad = row[3] name = "{0}|{1}{2}".format(site, team, quad) corner = int(row[4][-1:]) if corner not in (1, 2): print(site, team, name, name, corner) continue # pylint: disable=invalid-name # I think x,y,z, while too short, are the best variables names here. x = float(row[5]) y = float(row[6]) z = float(row[7]) if name not in data: data[name] = {corner: (x, y, z)} else: data[name][corner] = (x, y, z) return data
def main(): csv_path = r"data/reorg.csv" with csv23.open(csv_path, "r") as csv_file: csv_reader = csv.reader(csv_file) next(csv_reader) # ignore the header # check_unique_sources(csv_reader) check_unusual(csv_reader)
def write_folder_to_file(folder, csv_path): hashlist = get_file_hashes(folder) with csv23.open(csv_path, "w") as csv_file: csv_writer = csv.writer(csv_file) header = ["path", "name", "hash"] csv23.write(csv_writer, header) for row in hashlist: csv23.write(csv_writer, row)
def read_csv(csv_path): """Read the list of photos from csv_path.""" rows = [] with csv23.open(csv_path, "r") as csv_file: csv_reader = csv.reader(csv_file) next(csv_reader) # skip the header for row in csv_reader: row = csv23.fix(row) rows.append(row) return rows
def make_csv(kind, fields): """Write a CSV file for all the fields found for kind.""" data = get_all_items(kind) with csv23.open(kind + ".csv", "w") as out_file: writer = csv.writer(out_file) csv23.write(writer, fields) for item in data: values = simplify(item, fields) csv23.write(writer, values)
def writedata(filename, data): """Write the data as CSV in filename.""" # pylint: disable=too-many-locals with csv23.open(filename, "w") as out_file: writer = csv.writer(out_file) header = [ "Site", "Team", "Quad", "Corner", "Easting", "Northing", "SideLength", "Slope(%)", ] csv23.write(writer, header) for name in sorted(data.keys()): site, teamquad = name.split("|") team = teamquad[:1] quad = teamquad[1:] if 1 in data[name]: p_1 = data[name][1] else: print("Error Corner 1 not found in " + name) continue if 2 in data[name]: p_3 = data[name][2] else: print("Error Corner 2 not found in " + name) continue p_2, p_4, side_len, slope = corners3d(p_1, p_3) row = [ site, team, quad, 3, p_2[0], p_2[1], round(side_len, 3), round(slope, 3), ] csv23.write(writer, row) row = [ site, team, quad, 4, p_4[0], p_4[1], round(side_len, 3), round(slope, 3), ] csv23.write(writer, row)
def ashowone(park, csv_path=None): if csv_path: with csv23.open(csv_path, "w") as csv_file: csv_writer = csv.writer(csv_file) csv23.write(csv_writer, asset_header) for row in assets(park): csv23.write(csv_writer, row) else: print(",".join(asset_header)) for item in assets(park): print(",".join([my_str(x) for x in item]))
def load_csv_file(csv_path): """Return a list of the rows in the CSV.""" records = [] with csv23.open(csv_path, "r") as csv_file: csv_reader = csv.reader(csv_file) next(csv_reader) # ignore the header for row in csv_reader: row = csv23.fix(row) records.append(row) return records
def read_csv_map(csv_path): mappings = {} with csv23.open(csv_path, "r") as csv_file: csv_reader = csv.reader(csv_file) next(csv_reader) # ignore the header for row in csv_reader: row = csv23.fix(row) old_path = row[0] new_path = row[2] if row[1] is None else row[1] mappings[old_path] = new_path return mappings
def build_csv(csv_path): with csv23.open(csv_path, "w") as csv_file: csv_writer = csv.writer(csv_file) csv23.write(csv_writer, table_column_names) for site in sites: site_id = sites[site] for asset_code in asset_types: data = [ site, "{0}".format(asset_code), asset_types[asset_code] ] print(data) response = location_query(site_id, "{0}".format(asset_code)) convert_xml_to_csv(data, response, csv_writer)
def lshowall(csv_path=None): if csv_path: with csv23.open(csv_path, "w") as csv_file: csv_writer = csv.writer(csv_file) csv23.write(csv_writer, loc_header) for park in sites: for row in locations(park): csv23.write(csv_writer, row) else: print(",".join(loc_header)) for park in sites: for item in locations(park): print(",".join([my_str(x) for x in item]))
def main(folder, ext="tif", csv_path=None): """Create an outputter (CSV of print) and output files below folder.""" if csv_path is None: print(",".join(["folder", "filename", "ext", "size"])) check_folder(folder, ext) else: with csv23.open(csv_path, "w") as csv_file: csv_writer = csv.writer(csv_file) def put_in_csv(row): csv23.write(csv_writer, row) put_in_csv(["folder", "filename", "ext", "size"]) check_folder(folder, ext, put_in_csv)
def read_csv_map(csv_path): mappings = {} with csv23.open(csv_path, "r") as csv_file: csv_reader = csv.reader(csv_file) next(csv_reader) # ignore the header for row in csv_reader: row = csv23.fix(row) old_fgdb = row[0] mosaic = row[1] old_path = row[2] new_fgdb = row[3] new_path = row[4] new_fgdb_mosaic = os.path.join(new_fgdb, mosaic) if new_fgdb_mosaic not in mappings: mappings[new_fgdb_mosaic] = [] mappings[new_fgdb_mosaic].append((old_path, new_path)) return mappings
def create_csv(csv_path, folder): """Create a CSV file describing all the photos in folder.""" # pylint: disable=too-many-locals header = "folder,photo,id,namedate,exifdate,lat,lon,gpsdate,size,filedate" with csv23.open(csv_path, "w") as csv_file: csv_writer = csv.writer(csv_file) csv23.write(csv_writer, header.split(",")) for root, dirs, files in os.walk(root): folder = os.path.basename(root) # Remove the skip dirs from the current sub directories for skipdir in Config.skip_dirs: if skipdir in dirs: dirs.remove(skipdir) for filename in files: base, extension = os.path.splitext(filename) if extension.lower() == ".jpg": code, name_date = get_name_parts(base) path = os.path.join(root, filename) size = os.path.getsize(path) with open(path, "rb") as in_file: # exifread wants binary data tags = exifread.process_file(in_file, details=False) lat = get_latitude(tags) lon = get_longitude(tags) exif_date = get_exif_date(tags) gps_date = get_gps_date(tags) file_date = datetime.datetime.fromtimestamp( os.path.getmtime(path)).isoformat() # convert date to a microsoft excel acceptable ISO format file_date = file_date.replace("T", " ") row = [ folder, filename, code, name_date, exif_date, lat, lon, gps_date, size, file_date, ] csv23.write(csv_writer, row)
def files_in_csv(csv_path): """Return a set of standardized relative paths to photos in the file at csv_path.""" paths = set() with csv23.open(csv_path, "r") as csv_file: csv_reader = csv.reader(csv_file) next(csv_reader) # skip the header for row in csv_reader: row = csv23.fix(row) path = row[Config.name_index] if Config.folder_index is not None: folder = row[Config.folder_index] path = os.path.join(folder, path) if Config.unit_index is not None: folder = row[Config.unit_index] path = os.path.join(folder, path) path = standardize(path) paths.add(path) return paths
def main(): """ Just do it """ with csv23.open(Config.csv_path, "r") as handle: csvreader = csv.reader(handle) next(csvreader) # remove header for row in csvreader: row = csv23.fix(row) source = row[13] if source: local = source.replace(Config.remote, Config.local_zip) # print("copy", source, local) if not os.path.exists(source): print("File not found", source) else: print("copy {0}".format(os.path.basename(local))) try: shutil.copy(source, local) except IOError: os.makedirs(os.path.dirname(local)) shutil.copy(source, local)
1) Themes (layer files) in TM database that are not in the TM filesystem 2) Themes (layer files) in TM filesystem that are not in the TM database """ from __future__ import absolute_import, division, print_function, unicode_literals import csv import os import csv23 tm_filesystem = r"X:\GIS\ThemeMgr" tm_database = r"data\TM_20171206.csv" unique_themes = set([]) with csv23.open(tm_database, "r") as csv_file: csv_reader = csv.reader(csv_file) next(csv_reader) # ignore the header for row in csv_reader: row = csv23.fix(row) theme = row[3] unique_themes.add(theme) print("Missing Themes:") for theme in sorted(unique_themes): if theme and not os.path.exists(theme): print(" " + theme) print("Extra Themes:") for root, dirs, files in os.walk(tm_filesystem): if ".git" in dirs:
def test_csv(csv_path): with csv23.open(csv_path, "w") as csv_file: csv_writer = csv.writer(csv_file) csv23.write(csv_writer, table_column_names) response = test_service() convert_xml_to_csv(["ANIA", "4100", "Building"], response, csv_writer)
Checks the file used to fix the PDS """ from __future__ import absolute_import, division, print_function, unicode_literals import csv import os import csv23 old_x = r"\\INPAKROVMDIST\gisdata" new_x = r"\\INPAKROVMDIST\gisdata2" csv_path = r"data\PDS Fixes - X.csv" unique_themes = set([]) with csv23.open(csv_path, "r") as csv_file: csv_reader = csv.reader(csv_file) next(csv_reader) # ignore the header line = 1 for row in csv_reader: row = csv23.fix(row) line += 1 # check that time stamp is valid and increasing old_workspace = None if len(row[1].strip()) == 0 else row[1] old_datasource = None if len(row[3].strip()) == 0 else row[3] new_workspace = None if len(row[5].strip()) == 0 else row[5] if old_workspace is None: print("old workspace is missing from line {}".format(line)) if new_workspace is None: print("new workspace is missing from line {}".format(line)) if old_workspace is None or new_workspace is None:
def organize(): """Add additional attributes to the ifsar file list.""" # pylint: disable=too-many-locals,too-many-branches,too-many-statements with csv23.open(Config.out_path, "w") as out_file: csv_writer = csv.writer(out_file) header = [ "folder", "filename", "ext", "size", "legacy", "nga", "kind", "edge", "cell", "lat", "lon", "tfw", "xml", "html", "txt", "tif_xml", "ovr", "aux", "rrd", "aux_old", "crc", "extras", "skip", ] csv23.write(csv_writer, header) with csv23.open(Config.csv_path, "r") as csv_file: csv_reader = csv.reader(csv_file) next(csv_reader) # ignore the header for row in csv_reader: row = csv23.fix(row) path_in, name, ext, size = row name = name.lower() path = path_in.lower() legacy = "N" if "legacy" in path: legacy = "Y" nga = "N" if "nga_30" in path: nga = "Y" kind = "" if "ori" in name or ("ori" in path and "priority" not in path): kind = "ori" if kind == "ori" and "_sup" in name: kind = "ori_sup" if "dsm" in name or "dsm" in path: kind = "dsm" if "dtm" in name or "dtm" in path: kind = "dtm" edge = "N" if "edge" in path: edge = "Y" cell = "" lat, lon = 0, 0 match = re.search(r"\\cell_(\d*)\\", path) if match: cell = match.group(1) match = re.search(r"\\cell_([def])\\", path) # path -> 196, e -> 197, f -> 198 if match: cell = ord(match.group(1)) - ord("d") + 196 match = re.search(r"_n(\d*)w(\d*)", name) if match: lat = int(match.group(1)) / 100 lon = int(match.group(2)) / 100 # # Check for supplemental *.html, *.aux.xml, etc files # file_path = os.path.join(path_in, name) exts_found = [ sup.replace(file_path, "").lower() for sup in glob.glob(file_path + ".*") ] # exts_possible = ['.tif', '.tfw','.xml','.html','.txt','.tif.xml', # '.tif.ovr','.tif.aux.xml', '.rrd', '.aux', '.tif.crc'] tfw, xml, html, txt, tif_xml = 0, 0, 0, 0, 0 ovr, aux, rrd, aux_old, crc = 0, 0, 0, 0, 0 if ".tfw" in exts_found: tfw = 1 if ".xml" in exts_found: xml = 1 if ".html" in exts_found: html = 1 if ".txt" in exts_found: txt = 1 if ".tif.xml" in exts_found: tif_xml = 1 if ".tif.ovr" in exts_found: ovr = 1 if ".tif.aux.xml" in exts_found: aux = 1 if ".rrd" in exts_found: rrd = 1 if ".aux" in exts_found: aux_old = 1 if ".tif.crc" in exts_found: crc = 1 extras = (len(exts_found) - 1 - tfw - xml - html - txt - tif_xml - ovr - aux - rrd - aux_old - crc ) # 1 for the tif that must exist out_row = [ path_in, name, ext, size, legacy, nga, kind, edge, cell, lat, lon, tfw, xml, html, txt, tif_xml, ovr, aux, rrd, aux_old, crc, extras, "N", ] csv23.write(csv_writer, out_row)
def load_csv(csv_path, col=0): with csv23.open(csv_path, "r") as csv_file: csv_reader = csv.reader(csv_file) next(csv_reader) # ignore the header data = [csv23.fix(row)[col] for row in csv_reader] return data
import csv23 class Config(object): """Namespace for configuration parameters. Edit as needed.""" # pylint: disable=useless-object-inheritance,too-few-public-methods # Write the results to this file path. results = r"c:\tmp\sr.csv" # The folder to search. start = r"c:\tmp\Changing Tides" with csv23.open(Config.results, "w") as f: csv_writer = csv.writer(f) header = ["mxd", "data_frame", "spatial_reference"] csv23.write(csv_writer, header) csv_writer.writerow() for root, dirs, files in os.walk(Config.start): for file in files: if os.path.splitext(file)[1].lower() == ".mxd": suspect = os.path.join(root, file) print("Checking {}".format(suspect)) try: mxd = arcpy.mapping.MapDocument(suspect) for df in arcpy.mapping.ListDataFrames(mxd): print(" data frame {0} has spatial reference: {1}". format(df.name, df.spatialReference.name)) row = [suspect, df.name, df.spatialReference.name]