def dbf_to_csv(unzipped_dir, csv_dir, years=None, overwrite=False):
if years == None:
# diff the subdirectories of unzipped_dir and csv_dir
# to see which years haven't been converted yet.
years = [
year for year in listdir(unzipped_dir)
if year not in listdir(csv_dir)
]
for year in years:
year = str(year) # in case it's passed as integer
destination_dir = join(csv_dir, year)
origin_dir = join(unzipped_dir, year)
if not exists(destination_dir):
mkdir(destination_dir)
dbfs = [f for f in listdir(origin_dir) if f.lower().endswith('.dbf')]
for dbf_file in dbfs:
destination_file = join(destination_dir, dbf_file[:-3] + 'csv')
origin_file = join(origin_dir, dbf_file)
if exists(destination_file) and overwrite == False:
continue
try:
print("Converting %s from year %s" % (dbf_file, year))
with dbf.Table(origin_file).open() as table:
dbf.export(table, destination_file) # as csv
except:
print("Error converting %s from year %s." % (dbf_file, year))
continue
def handle(self, *args, **options):
for dbf_filename in options['file_dbf']:
# Convert DBF into CSV:
# https://pythonhosted.org/dbf/
# https://stackoverflow.com/questions/45948295/how-to-convert-dbf-to-csv
# https://stackoverflow.com/questions/13944427/error-when-trying-to-convert-dbf-to-csv-in-python#13944926
try:
name_extension = dbf_filename.split(".")
if name_extension[-1] != 'dbf':
self.stdout.write(
self.style.ERROR(
'Il file {} non ha estensione .dbf e non è stato importato.'
.format(dbf_filename)))
continue
tablename = ".".join(name_extension[:-1])
table = dbf.Table(dbf_filename)
table.open()
dbf.export(table, "{}.csv".format(tablename), header=True)
self.stdout.write(
self.style.SUCCESS(
"File '{}' convertito con successo.".format(
dbf_filename)))
except:
self.stdout.write(
self.style.ERROR(
"Si è verificato un errore nel convertire il file '{}'. Il file non è stato convertito."
.format(dbf_filename)))
def dbf_to_csv(unzipped_dir, csv_dir, years=None, overwrite=False):
if years == None:
# diff the subdirectories of unzipped_dir and csv_dir
# to see which years haven't been converted yet.
years = [year for year in listdir(unzipped_dir) if year not in listdir(csv_dir)]
for year in years:
year = str(year) # in case it's passed as integer
destination_dir = join(csv_dir, year)
origin_dir = join(unzipped_dir, year)
if not exists(destination_dir):
mkdir(destination_dir)
dbfs = [f for f in listdir(origin_dir) if f.lower().endswith('.dbf')]
for dbf_file in dbfs:
destination_file = join(destination_dir, dbf_file[:-3]+'csv')
origin_file = join(origin_dir, dbf_file)
if exists(destination_file) and overwrite==False:
continue
try:
print("Converting %s from year %s"%(dbf_file, year))
with dbf.Table(origin_file).open() as table:
dbf.export(table, destination_file) # as csv
except:
print("Error converting %s from year %s."%(dbf_file, year))
continue
def import_records(self, csvfile):
import datetime
filetype = os.path.splitext(csvfile)[-1][1:].lower()
if filetype == 'dbf':
table = dbf.Table(csvfile)
table.open()
dbf.export(table, r'temp.csv', header =False)
csvfile = 'temp.csv'
elif filetype == 'csv':
lines = open(csvfile, 'r').readlines()
lines.pop(0)
files = open(csvfile, 'w')
for line in lines:
files.write(line)
files.close()
csvData = csv.reader(open(csvfile, 'rb'))
new_csv = []
for line in csvData:
L = line[:5] + [line[5].strip()] + line[6:]
new_csv.append(L)
self.cur.executemany('INSERT OR IGNORE INTO registros VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', tuple(new_csv))
self.conn.commit()
def dbf2csv(filename):
'''
tem que ter a package dbf (sudo pip install dbf)
entre com o caminho+nome da tabela, a saida sera escrita
na mesma pasta com o .csv
'''
import dbf, os
a = dbf.Table(filename).open()
output = filename[:-4] + '.csv'
dbf.export(a, filename=output, encoding='utf-8')
f1 = open(output, 'r')
f2 = open(output[:-4] + '_.csv', 'w')
for line in f1:
f2.write(line.replace(' ', ''))
f1.close()
f2.close()
os.remove(output)
os.rename(output[:-4] + '_.csv', output)
del a
def main():
import os
import glob
import dbf
folder = "/Users/kprovost/Dropbox/BirdlifeRangeMaps/"
dbfFolder = "/Users/kprovost/Downloads/Promeropidae/"
count = 0
with open(folder+"/compileDBF.temp","a") as compDbf:
for filename in glob.glob(dbfFolder+"*.dbf"):
#print(filename)
print(filename)
dBase = dbf.Table(filename)
dbf.export(dBase,folder+"/compileDBF.temp",header=True)
with open(folder+"/compileDBF.temp","w") as tempDbf:
output = tempDbf.read()
compDbf.write(output)
print("done")
def dbf_reader(dbf_path, csv_path):
table = dbf.Table(dbf_path, codepage='cp936')
table.open()
dbf.export(table, csv_path, encoding='utf-8')
import requests, zipfile, io, dbf, csv
from osgeo import ogr
from xml.dom import minidom
# parcely katastralniho uzemi obce Stoky
URL = "http://services.cuzk.cz/shp/ku/epsg-5514/764051.zip"
# download, save and unzip file
r = requests.get(URL)
z = zipfile.ZipFile(io.BytesIO(r.content))
z.extractall()
# count of unique polygons
with dbf.Table("764051/PARCELY_KN_P.dbf") as dbs:
dbf.export(dbs, "PARCELY_KN_P.csv")
with open("PARCELY_KN_P.csv", "r") as table:
count = len(table.readlines())
kn = count - 1 # header of table
# convert layer of shp to geometry and extract the area
shp = ogr.Open("764051/PARCELY_KN_P.shp")
layer = shp.GetLayer(0)
areas = []
for feature in layer:
geom = feature.GetGeometryRef()
area = geom.GetArea()
areas.append(area)
average_area = sum(areas) / kn
# creat new csv file for statistics >> total count of unique polygons; average area of polygons
import dbf
with dbf.Table("result.dbf") as db:
dbf.export(db, "test.csv", header=True)
import dbf
with dbf.Table('allforms.dbf') as table:
dbf.export(table, 'output.csv')
# DBF - encoding cp1250
import dbf
with dbf.Table('test.dbf') as table:
dbf.export(table, 'junk.csv')
