def main(args): with DwCAReader(args.source_file) as dwca: if valid_dwca(dwca): ordered_fields = list(dwca.core_terms) # Only last part as Shapefiles field names are limited to 10 chars ordered_fields_truncated = [ f.rsplit('/')[-1] for f in ordered_fields ] import pdb pdb.set_trace() with ShapefileOutput(args.destination, args.crs, ordered_fields_truncated) as out: for line in dwca.each_line(): try: gis_data = dwcaline_to_epsg4326(line) additional_values = [ unicode_to_ascii(line.data[f]) for f in ordered_fields ] out.insert_line(gis_data['lat'], gis_data['lon'], additional_values) sys.stdout.write('.') except CannotConvertException: pass
def __init__(self, config, dwca): """Initialize Meuh engine for a given config object and DwC-A config: object dwca: open file """ self.dwca = DwCAReader(dwca) # Parse config self.assessments = AssessmentList(config['assessments'], self.dwca)
def handle(self, *args, **options): if len(args) != 1: raise CommandError('Please give the path to the source DwcA.') else: source_filename = args[0] try: source = DwCAReader(source_filename) if options['truncate']: self.stdout.write("Truncating existing records...") truncate_all_tables() self.stdout.write("Done.\n") lines = source.each_line() for l in lines: self.stdout.write('.') create_occurrence_from_dwcaline(l) except IOError as e: raise CommandError('Cannot open source DwcA: %s' % source_filename)
def main(): # TODO: Better CLI Help # TODO: split into parse_args() and main(args) (see dwca2shp) parser = argparse.ArgumentParser( description="Import a DarwinCore Archive file into CartDB.") parser.add_argument('--domain', help='Your CartoDB domain (without .cartodb.com).', required=True) parser.add_argument('--api-key', dest='apikey', help='Your CartoDB API key.', required=True) parser.add_argument('--table', help="CartoDB destination table name", required=True) parser.add_argument('--truncate-table', action='store_true', dest='truncate', help="Truncate destination table prior to import.") parser.add_argument('source_file', help="Source DwC-A file", type=argparse.FileType('r')) args = parser.parse_args() target_table = args.table out = CartoDBOutput(args.apikey, args.domain, args.table) if args.truncate: if query_yes_no("Are you sure you want to truncate the database ? Data will be LOST !", default="no"): out.truncate_table() with DwCAReader(args.source_file) as dwca: if valid_dwca(dwca): for line in dwca.each_line(): try: out.insert_line(**dwcaline_to_epsg4326(line)) sys.stdout.write('.') except CartoDBException as e: print ("CartoDB error: ", e) else: # TODO: more detailed message print "Invalid source DwC-A file."
class MeuhEngine(object): # Preferably, use 'with' to ensure proper cleanup at the end def __enter__(self): return self def __exit__(self, type, value, traceback): self.cleanup() def __init__(self, config, dwca): """Initialize Meuh engine for a given config object and DwC-A config: object dwca: open file """ self.dwca = DwCAReader(dwca) # Parse config self.assessments = AssessmentList(config['assessments'], self.dwca) def run(self, log_stream): """Executes all the tests on the target DwC-A, according to the configuration. Progress is logged on log_stream so the user isn't left in the dark in case it's long. Report is returned upon completion. """ r = MeuhReport() log_stream.write('Engine running...\n') r.log_start_time() log_stream.write('Applicable assessments:\n') applicable = self.assessments.applicable r.log_applicable_assessments(applicable) log_stream.write("Applicable: {ass}\n".format(ass=applicable)) non_applicable = self.assessments.non_applicable log_stream.write("Non-applicable: {ass}\n".format(ass=non_applicable)) r.log_unapplicable_assessments(non_applicable) log_stream.write('1. Testing metadata\n') # TODO: Implement log_stream.write('2. Testing each line of archive...\n') lines_assessments = self.assessments.applicable_for_each_line log_stream.write("The following assessments will be executed for each line: {ass}\n".format(ass=lines_assessments)) for line in self.dwca.each_line(): log_stream.write('.') for a in lines_assessments: a.assess_line(line) # Assemble results for a in applicable: r.store_assessment_log(a, a.logger) log_stream.write('\nEngine finished execution.\n') r.log_end_time() return r def cleanup(self): self.dwca.close()