def handle(self, *args, **options): path = options['path'] carrier_slug = options['carrier'] phone_type = options['type'] skip = options['skip'] delimiter = options['delimiter'] quotechar = options['quotechar'] batchsize = options['batchsize'] verbosity = options['verbosity'] if verbosity > 2: logger.setLevel(logging.DEBUG) connection.force_debug_cursor = True carrier = Carrier.objects.get(slug=carrier_slug) try: self.last_inserted_person = Person.objects.latest('id') except Person.DoesNotExist: pass try: self.last_inserted_address = Address.objects.latest('id') except Address.DoesNotExist: pass try: self.last_inserted_phone = Phone.objects.latest('id') except Phone.DoesNotExist: pass start = datetime.datetime.now() for row in self.read(path, skip, delimiter, quotechar): try: record = ZoomRecord.parse_gvt(row, carrier, phone_type) self.handle_valid(record) except (CNPJValidationError, CPFValidationError): self.handle_document_error(record) except ZipCodeValidationError: self.handle_zipcode_error(record) except PhoneValidationError: self.handle_phone_error(record) except AreaCodeValidationError: self.handle_areacode_error(record) except UnicodeEncodeError as e: import pdb pdb.set_trace() sys.stdout.write('\r%s' % (self.counter['record'] + 1)) sys.stdout.flush() if (self.counter['batch'] + 1) == batchsize: self.persist(self.counter['batch']) self.counter['batch'] = 0 else: self.counter['batch'] += 1 self.counter['record'] += 1 self.persist(self.counter['batch']) sys.stdout.write(self.get_finish(start, datetime.datetime.now())) sys.stdout.flush()
def handle(self, *args, **options): path = options['path'] areacode = options['areacode'] carrier_slug = options['carrier'] phone_type = options['type'] skip = options['skip'] delimiter = options['delimiter'] quotechar = options['quotechar'] batchsize = options['batchsize'] verbosity = options['verbosity'] if verbosity > 2: logger.setLevel(logging.DEBUG) connection.force_debug_cursor = True carrier = Carrier.objects.get(slug=carrier_slug) try: self.last_inserted_person = Person.objects.latest('id') except Person.DoesNotExist: pass try: self.last_inserted_address = Address.objects.latest('id') except Address.DoesNotExist: pass try: self.last_inserted_phone = Phone.objects.latest('id') except Phone.DoesNotExist: pass start = datetime.datetime.now() paths = [path,] if os.path.isdir(path): paths = [] for (dirpath, dirnames, filenames) in os.walk(path): paths.extend(['%s/%s' % (path, filename) for filename in filenames if '.csv' in filename]) break for path in paths: print 'File to process: %s' % path for row in self.read(path, skip, delimiter, quotechar): sys.stdout.write('\r%s' % (self.counter['record'] + 1)) sys.stdout.flush() try: record = ZoomRecord.parse(row, carrier, areacode, phone_type) self.handle_valid(record) except (CNPJValidationError, CPFValidationError): self.handle_document_error(record) except ZipCodeValidationError: self.handle_zipcode_error(record) except PhoneValidationError: self.handle_phone_error(record) except AreaCodeValidationError: self.handle_areacode_error(record) except UnicodeEncodeError as e: import pdb pdb.set_trace() if (self.counter['batch'] + 1) == batchsize: self.persist(self.counter['batch']) self.counter['batch'] = 0 else: self.counter['batch'] += 1 self.counter['record'] += 1 self.persist(self.counter['batch']) self.counter['batch'] = 0 sys.stdout.write(self.get_finish(start, datetime.datetime.now())) sys.stdout.flush()