def main(args):
with DwCAReader(args.source_file) as dwca:
if valid_dwca(dwca):
ordered_fields = list(dwca.core_terms)
# Only last part as Shapefiles field names are limited to 10 chars
ordered_fields_truncated = [
f.rsplit('/')[-1] for f in ordered_fields
]
import pdb
pdb.set_trace()
with ShapefileOutput(args.destination, args.crs,
ordered_fields_truncated) as out:
for line in dwca.each_line():
try:
gis_data = dwcaline_to_epsg4326(line)
additional_values = [
unicode_to_ascii(line.data[f])
for f in ordered_fields
]
out.insert_line(gis_data['lat'], gis_data['lon'],
additional_values)
sys.stdout.write('.')
except CannotConvertException:
pass
def __init__(self, config, dwca):
"""Initialize Meuh engine for a given config object and DwC-A
config: object
dwca: open file
"""
self.dwca = DwCAReader(dwca)
# Parse config
self.assessments = AssessmentList(config['assessments'], self.dwca)
def handle(self, *args, **options):
if len(args) != 1:
raise CommandError('Please give the path to the source DwcA.')
else:
source_filename = args[0]
try:
source = DwCAReader(source_filename)
if options['truncate']:
self.stdout.write("Truncating existing records...")
truncate_all_tables()
self.stdout.write("Done.\n")
lines = source.each_line()
for l in lines:
self.stdout.write('.')
create_occurrence_from_dwcaline(l)
except IOError as e:
raise CommandError('Cannot open source DwcA: %s' % source_filename)
def main():
# TODO: Better CLI Help
# TODO: split into parse_args() and main(args) (see dwca2shp)
parser = argparse.ArgumentParser(
description="Import a DarwinCore Archive file into CartDB.")
parser.add_argument('--domain',
help='Your CartoDB domain (without .cartodb.com).',
required=True)
parser.add_argument('--api-key',
dest='apikey',
help='Your CartoDB API key.',
required=True)
parser.add_argument('--table',
help="CartoDB destination table name",
required=True)
parser.add_argument('--truncate-table',
action='store_true',
dest='truncate',
help="Truncate destination table prior to import.")
parser.add_argument('source_file',
help="Source DwC-A file",
type=argparse.FileType('r'))
args = parser.parse_args()
target_table = args.table
out = CartoDBOutput(args.apikey, args.domain, args.table)
if args.truncate:
if query_yes_no("Are you sure you want to truncate the database ? Data will be LOST !", default="no"):
out.truncate_table()
with DwCAReader(args.source_file) as dwca:
if valid_dwca(dwca):
for line in dwca.each_line():
try:
out.insert_line(**dwcaline_to_epsg4326(line))
sys.stdout.write('.')
except CartoDBException as e:
print ("CartoDB error: ", e)
else:
# TODO: more detailed message
print "Invalid source DwC-A file."
class MeuhEngine(object):
# Preferably, use 'with' to ensure proper cleanup at the end
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
self.cleanup()
def __init__(self, config, dwca):
"""Initialize Meuh engine for a given config object and DwC-A
config: object
dwca: open file
"""
self.dwca = DwCAReader(dwca)
# Parse config
self.assessments = AssessmentList(config['assessments'], self.dwca)
def run(self, log_stream):
"""Executes all the tests on the target DwC-A, according to the configuration.
Progress is logged on log_stream so the user isn't left in the dark in case it's long.
Report is returned upon completion.
"""
r = MeuhReport()
log_stream.write('Engine running...\n')
r.log_start_time()
log_stream.write('Applicable assessments:\n')
applicable = self.assessments.applicable
r.log_applicable_assessments(applicable)
log_stream.write("Applicable: {ass}\n".format(ass=applicable))
non_applicable = self.assessments.non_applicable
log_stream.write("Non-applicable: {ass}\n".format(ass=non_applicable))
r.log_unapplicable_assessments(non_applicable)
log_stream.write('1. Testing metadata\n')
# TODO: Implement
log_stream.write('2. Testing each line of archive...\n')
lines_assessments = self.assessments.applicable_for_each_line
log_stream.write("The following assessments will be executed for each line: {ass}\n".format(ass=lines_assessments))
for line in self.dwca.each_line():
log_stream.write('.')
for a in lines_assessments:
a.assess_line(line)
# Assemble results
for a in applicable:
r.store_assessment_log(a, a.logger)
log_stream.write('\nEngine finished execution.\n')
r.log_end_time()
return r
def cleanup(self):
self.dwca.close()
