def __init__(self, *args, **kwargs): self.savepoint = Savepoint('peps_nacp_ids') self.host = settings.PEP_SOURCE_HOST self.port = settings.PEP_SOURCE_PORT self.database = settings.PEP_SOURCE_DATABASE self.user = settings.PEP_SOURCE_USER self.password = settings.PEP_SOURCE_PASSWORD self.PEP_QUERY = (""" SELECT p.id, d.declaration_id FROM core_person p LEFT JOIN core_declaration d ON p.id = d.person_id WHERE is_pep = TRUE AND d.nacp_declaration = TRUE AND d.confirmed='a' GROUP BY p.id, d.declaration_id ORDER BY p.id """) self.all_peps = {getattr(pep, 'source_id'): pep for pep in Pep.objects.filter( is_pep=True )} self.all_nacp_id = [getattr(pep, 'nacp_id') for pep in Pep.objects.filter(is_pep=True)] self.check_peps = [] super().__init__(*args, **kwargs)
class Command(BaseCommand): help = '---' def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.savepoint = Savepoint('declarations_savepoint') self.converter = DeclarationConverter() def add_arguments(self, parser): parser.add_argument('--pep_source_id', nargs='?', type=int) parser.add_argument('--pep_id', nargs='?', type=int) parser.add_argument('--declaration_id', nargs='?', type=str) parser.add_argument('--resave', action='store_true') def load_one(self, pep: Pep): for nacp_id in pep.nacp_id: self.converter.save_declarations_for_pep(nacp_declarant_id=nacp_id, resave=self.resave) def load_declaration(self, declaration_id: str): self.converter.save_one_declaration(declaration_id=declaration_id, resave=self.resave) def load_all(self): i = 0 count = len(self.converter.only_peps) for nacp_declarant_id in self.converter.only_peps: i += 1 self.stdout.write(f'\rProgress: {i} of {count}', ending='') self.stdout.flush() if not self.savepoint.has(nacp_declarant_id): self.converter.save_declarations_for_pep(nacp_declarant_id=nacp_declarant_id, resave=self.resave) self.savepoint.add(nacp_declarant_id) self.stdout.write() def handle(self, *args, **options): pep_source_id = options['pep_source_id'] pep_id = options['pep_id'] declaration_id = options['declaration_id'] self.resave = options['resave'] if pep_source_id: self.load_one(Pep.objects.get(source_id=pep_source_id)) elif pep_id: self.load_one(Pep.objects.get(id=pep_id)) elif declaration_id: self.load_declaration(declaration_id) else: self.load_all() self.stdout.write('Load done. Start running all fixes') DeclarationsFixSet().run_all_fixes() self.stdout.write('All fixes applied')
def handle(self, *args, **options): source_type = options['source_type'][0] queryset = Person.objects.all() savepoint = Savepoint(f'collect_person_data-{source_type}') progress = CommandProgress(self, queryset.count()) for person in queryset.iterator(): if savepoint.has(person.pk): progress.next(silent=True) continue progress.next() controller = ProvidersController(person) if source_type == 'all': controller.put_json_to_fields() else: controller.put_json_to_fields(source_type) savepoint.add(person.pk) progress.end() savepoint.close() self.stdout.write('Done.')
def handle(self, *args, **options): source = options['source'][0] controller = ConnectorsController(source) savepoint = Savepoint(f'run_person_connector-{source}') progress = CommandProgress(self, controller.get_count()) for obj in controller.iter_objects(): if savepoint.has(obj.pk): progress.next(silent=True) continue progress.next() controller.migrate_object(obj) savepoint.add(obj.pk) progress.end() savepoint.close() self.stdout.write('Done.')
class Command(BaseCommand): help = 'fetch and store Peps id from the National agency on corruption prevention' def __init__(self, *args, **kwargs): self.savepoint = Savepoint('peps_nacp_ids') self.host = settings.PEP_SOURCE_HOST self.port = settings.PEP_SOURCE_PORT self.database = settings.PEP_SOURCE_DATABASE self.user = settings.PEP_SOURCE_USER self.password = settings.PEP_SOURCE_PASSWORD self.PEP_QUERY = (""" SELECT p.id, d.declaration_id FROM core_person p LEFT JOIN core_declaration d ON p.id = d.person_id WHERE is_pep = TRUE AND d.nacp_declaration = TRUE AND d.confirmed='a' GROUP BY p.id, d.declaration_id ORDER BY p.id """) self.all_peps = {getattr(pep, 'source_id'): pep for pep in Pep.objects.filter( is_pep=True )} self.all_nacp_id = [getattr(pep, 'nacp_id') for pep in Pep.objects.filter(is_pep=True)] self.check_peps = [] super().__init__(*args, **kwargs) def add_arguments(self, parser): pass def process(self, host=None, port=None): host = host or self.host port = port or self.port self.stdout.write(f'business_pep: psycopg2 connect to {host}:{port}...') connection = psycopg2.connect( host=host, port=port, database=self.database, user=self.user, password=self.password ) cursor = connection.cursor() cursor.execute(self.PEP_QUERY) i = 0 rows = cursor.fetchall() count = len(rows) for pep_data in rows: i += 1 self.stdout.write(f'\rProgress: {i} of {count}', ending='') self.stdout.flush() pep_source_id = pep_data[0] if self.savepoint.has(pep_source_id): continue pep = self.all_peps.get(pep_source_id) if not pep: self.stdout.write(f'No PEP from ANTAC`s DB with id={pep_data[0]} in our database') self.savepoint.add(pep_source_id) continue declaration_id = pep_data[1].replace('nacp_', '') response = requests.get(settings.NACP_DECLARATION_RETRIEVE + declaration_id) if response.status_code != 200: self.stdout.write(f'cannot find the declaration with id: {declaration_id}') self.savepoint.add(pep_source_id) continue declaration_data = json.loads(response.text) # storing PEP nacp_id from declarations list pep_nacp_id = declaration_data.get('user_declarant_id') if not isinstance(pep_nacp_id, int) or not pep_nacp_id: self.stdout.write(f'Check invalid declarant NACP id ({pep_nacp_id}) from declaration ' f'with NACP id {declaration_id}') self.savepoint.add(pep_source_id) continue elif pep_nacp_id not in pep.nacp_id: pep.nacp_id.append(pep_nacp_id) pep.save(update_fields=['nacp_id']) self.savepoint.add(pep_source_id) # additional check of matching PEP`s last_name and first_name # last_name = declaration_data['data']['step_1']['data'].get('lastname') # first_name = declaration_data['data']['step_1']['data'].get('firstname') # if ( # pep.last_name != last_name.lower() # or pep.first_name != first_name.lower() # ): # self.stdout.write( # f'PEP data from our DB with id {pep.id}: {pep.last_name} {pep.first_name}, ' # f'from declaration: {last_name} {first_name}') # self.check_peps.append( # f'PEP data from our DB with id {pep.id}: {pep.last_name} {pep.first_name}, ' # f'from declaration: {last_name} {first_name}') # continue self.savepoint.close() self.stdout.write() self.stdout.write('Done!') def handle(self, *args, **options): self.stdout.write(f'business_pep: use SSH tunnel: {settings.PEP_SOURCE_USE_SSH}') if not settings.PEP_SOURCE_USE_SSH: self.process() return sshtunnel.TUNNEL_TIMEOUT = settings.PEP_TUNNEL_TIMEOUT sshtunnel.SSH_TIMEOUT = settings.PEP_SSH_TIMEOUT with sshtunnel.SSHTunnelForwarder( (settings.PEP_SSH_SERVER_IP, settings.PEP_SSH_SERVER_PORT), ssh_username=settings.PEP_SSH_USERNAME, ssh_pkey=settings.PEP_SSH_PKEY, remote_bind_address=(self.host, self.port), # local_bind_address=(settings.PEP_LOCAL_SOURCE_HOST, settings.PEP_LOCAL_SOURCE_PORT) ) as tunnel: self.stdout.write( f'business_pep: tunnel is active: {tunnel.is_active} on ' f'{tunnel.local_bind_host}:{tunnel.local_bind_port}' ) self.process( host=tunnel.local_bind_host, port=tunnel.local_bind_port, )
class Command(BaseCommand): help = 'Saves Companies data into DB' def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.converter = DeclarationConverter() self.savepoint = Savepoint('declarations_steps', type=str) def add_arguments(self, parser): pass def resave_transactions(self, data, declaration): Transaction.objects.filter(declaration=declaration).delete() self.converter.save_transaction(data, declaration) def resave_liability(self, data, declaration): Liability.objects.filter(declaration=declaration).delete() self.converter.save_liability(data, declaration) def resave_property(self, data, declaration): Property.objects.filter(declaration=declaration).exclude( type=Property.UNFINISHED_CONSTRUCTION, ).delete() self.converter.save_property(data, declaration) def resave_unfinished_construction(self, data, declaration): Property.objects.filter( declaration=declaration, type=Property.UNFINISHED_CONSTRUCTION, ).delete() self.converter.save_unfinished_construction(data, declaration) def resave_all_steps_with_city(self, data, declaration): self.converter.save_declarant_data(data, declaration) self.resave_liability(data, declaration) self.resave_property(data, declaration) self.resave_unfinished_construction(data, declaration) def process_declaration(self, data, declaration): # self.resave_transactions(data, declaration) self.resave_all_steps_with_city(data, declaration) def handle(self, *args, **options): i = 0 qs = Declaration.objects.all() count = qs.count() for declaration in qs: i += 1 self.stdout.write(f'\rProgress: {i} of {count}', ending='') self.stdout.flush() if self.savepoint.has(declaration.nacp_declaration_id): continue self.converter.current_declaration = declaration data = self.converter.download_declaration( str(declaration.nacp_declaration_id)) if not data: raise CommandError( f'No data for declaration {declaration.nacp_declaration_id}' ) self.converter.save_relatives_data(data['data'], declaration) self.process_declaration(data=data['data'], declaration=declaration) self.savepoint.add(declaration.nacp_declaration_id) self.savepoint.close() self.stdout.write() self.stdout.write('Resave done. Start running all fixes') DeclarationsFixSet().run_all_fixes() self.stdout.write('All fixes applied')
def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.converter = DeclarationConverter() self.savepoint = Savepoint('declarations_steps', type=str)