def __init__(self, *args, **kwargs):
self.savepoint = Savepoint('peps_nacp_ids')
self.host = settings.PEP_SOURCE_HOST
self.port = settings.PEP_SOURCE_PORT
self.database = settings.PEP_SOURCE_DATABASE
self.user = settings.PEP_SOURCE_USER
self.password = settings.PEP_SOURCE_PASSWORD
self.PEP_QUERY = ("""
SELECT p.id,
d.declaration_id
FROM core_person p
LEFT JOIN core_declaration d ON p.id = d.person_id
WHERE is_pep = TRUE
AND d.nacp_declaration = TRUE
AND d.confirmed='a'
GROUP BY p.id,
d.declaration_id
ORDER BY p.id
""")
self.all_peps = {getattr(pep, 'source_id'): pep for pep in Pep.objects.filter(
is_pep=True
)}
self.all_nacp_id = [getattr(pep, 'nacp_id') for pep in Pep.objects.filter(is_pep=True)]
self.check_peps = []
super().__init__(*args, **kwargs)
class Command(BaseCommand):
help = '---'
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.savepoint = Savepoint('declarations_savepoint')
self.converter = DeclarationConverter()
def add_arguments(self, parser):
parser.add_argument('--pep_source_id', nargs='?', type=int)
parser.add_argument('--pep_id', nargs='?', type=int)
parser.add_argument('--declaration_id', nargs='?', type=str)
parser.add_argument('--resave', action='store_true')
def load_one(self, pep: Pep):
for nacp_id in pep.nacp_id:
self.converter.save_declarations_for_pep(nacp_declarant_id=nacp_id, resave=self.resave)
def load_declaration(self, declaration_id: str):
self.converter.save_one_declaration(declaration_id=declaration_id, resave=self.resave)
def load_all(self):
i = 0
count = len(self.converter.only_peps)
for nacp_declarant_id in self.converter.only_peps:
i += 1
self.stdout.write(f'\rProgress: {i} of {count}', ending='')
self.stdout.flush()
if not self.savepoint.has(nacp_declarant_id):
self.converter.save_declarations_for_pep(nacp_declarant_id=nacp_declarant_id, resave=self.resave)
self.savepoint.add(nacp_declarant_id)
self.stdout.write()
def handle(self, *args, **options):
pep_source_id = options['pep_source_id']
pep_id = options['pep_id']
declaration_id = options['declaration_id']
self.resave = options['resave']
if pep_source_id:
self.load_one(Pep.objects.get(source_id=pep_source_id))
elif pep_id:
self.load_one(Pep.objects.get(id=pep_id))
elif declaration_id:
self.load_declaration(declaration_id)
else:
self.load_all()
self.stdout.write('Load done. Start running all fixes')
DeclarationsFixSet().run_all_fixes()
self.stdout.write('All fixes applied')
def handle(self, *args, **options):
source_type = options['source_type'][0]
queryset = Person.objects.all()
savepoint = Savepoint(f'collect_person_data-{source_type}')
progress = CommandProgress(self, queryset.count())
for person in queryset.iterator():
if savepoint.has(person.pk):
progress.next(silent=True)
continue
progress.next()
controller = ProvidersController(person)
if source_type == 'all':
controller.put_json_to_fields()
else:
controller.put_json_to_fields(source_type)
savepoint.add(person.pk)
progress.end()
savepoint.close()
self.stdout.write('Done.')
def handle(self, *args, **options):
source = options['source'][0]
controller = ConnectorsController(source)
savepoint = Savepoint(f'run_person_connector-{source}')
progress = CommandProgress(self, controller.get_count())
for obj in controller.iter_objects():
if savepoint.has(obj.pk):
progress.next(silent=True)
continue
progress.next()
controller.migrate_object(obj)
savepoint.add(obj.pk)
progress.end()
savepoint.close()
self.stdout.write('Done.')
class Command(BaseCommand):
help = 'fetch and store Peps id from the National agency on corruption prevention'
def __init__(self, *args, **kwargs):
self.savepoint = Savepoint('peps_nacp_ids')
self.host = settings.PEP_SOURCE_HOST
self.port = settings.PEP_SOURCE_PORT
self.database = settings.PEP_SOURCE_DATABASE
self.user = settings.PEP_SOURCE_USER
self.password = settings.PEP_SOURCE_PASSWORD
self.PEP_QUERY = ("""
SELECT p.id,
d.declaration_id
FROM core_person p
LEFT JOIN core_declaration d ON p.id = d.person_id
WHERE is_pep = TRUE
AND d.nacp_declaration = TRUE
AND d.confirmed='a'
GROUP BY p.id,
d.declaration_id
ORDER BY p.id
""")
self.all_peps = {getattr(pep, 'source_id'): pep for pep in Pep.objects.filter(
is_pep=True
)}
self.all_nacp_id = [getattr(pep, 'nacp_id') for pep in Pep.objects.filter(is_pep=True)]
self.check_peps = []
super().__init__(*args, **kwargs)
def add_arguments(self, parser):
pass
def process(self, host=None, port=None):
host = host or self.host
port = port or self.port
self.stdout.write(f'business_pep: psycopg2 connect to {host}:{port}...')
connection = psycopg2.connect(
host=host,
port=port,
database=self.database,
user=self.user,
password=self.password
)
cursor = connection.cursor()
cursor.execute(self.PEP_QUERY)
i = 0
rows = cursor.fetchall()
count = len(rows)
for pep_data in rows:
i += 1
self.stdout.write(f'\rProgress: {i} of {count}', ending='')
self.stdout.flush()
pep_source_id = pep_data[0]
if self.savepoint.has(pep_source_id):
continue
pep = self.all_peps.get(pep_source_id)
if not pep:
self.stdout.write(f'No PEP from ANTAC`s DB with id={pep_data[0]} in our database')
self.savepoint.add(pep_source_id)
continue
declaration_id = pep_data[1].replace('nacp_', '')
response = requests.get(settings.NACP_DECLARATION_RETRIEVE + declaration_id)
if response.status_code != 200:
self.stdout.write(f'cannot find the declaration with id: {declaration_id}')
self.savepoint.add(pep_source_id)
continue
declaration_data = json.loads(response.text)
# storing PEP nacp_id from declarations list
pep_nacp_id = declaration_data.get('user_declarant_id')
if not isinstance(pep_nacp_id, int) or not pep_nacp_id:
self.stdout.write(f'Check invalid declarant NACP id ({pep_nacp_id}) from declaration '
f'with NACP id {declaration_id}')
self.savepoint.add(pep_source_id)
continue
elif pep_nacp_id not in pep.nacp_id:
pep.nacp_id.append(pep_nacp_id)
pep.save(update_fields=['nacp_id'])
self.savepoint.add(pep_source_id)
# additional check of matching PEP`s last_name and first_name
# last_name = declaration_data['data']['step_1']['data'].get('lastname')
# first_name = declaration_data['data']['step_1']['data'].get('firstname')
# if (
# pep.last_name != last_name.lower()
# or pep.first_name != first_name.lower()
# ):
# self.stdout.write(
# f'PEP data from our DB with id {pep.id}: {pep.last_name} {pep.first_name}, '
# f'from declaration: {last_name} {first_name}')
# self.check_peps.append(
# f'PEP data from our DB with id {pep.id}: {pep.last_name} {pep.first_name}, '
# f'from declaration: {last_name} {first_name}')
# continue
self.savepoint.close()
self.stdout.write()
self.stdout.write('Done!')
def handle(self, *args, **options):
self.stdout.write(f'business_pep: use SSH tunnel: {settings.PEP_SOURCE_USE_SSH}')
if not settings.PEP_SOURCE_USE_SSH:
self.process()
return
sshtunnel.TUNNEL_TIMEOUT = settings.PEP_TUNNEL_TIMEOUT
sshtunnel.SSH_TIMEOUT = settings.PEP_SSH_TIMEOUT
with sshtunnel.SSHTunnelForwarder(
(settings.PEP_SSH_SERVER_IP, settings.PEP_SSH_SERVER_PORT),
ssh_username=settings.PEP_SSH_USERNAME,
ssh_pkey=settings.PEP_SSH_PKEY,
remote_bind_address=(self.host, self.port),
# local_bind_address=(settings.PEP_LOCAL_SOURCE_HOST, settings.PEP_LOCAL_SOURCE_PORT)
) as tunnel:
self.stdout.write(
f'business_pep: tunnel is active: {tunnel.is_active} on '
f'{tunnel.local_bind_host}:{tunnel.local_bind_port}'
)
self.process(
host=tunnel.local_bind_host,
port=tunnel.local_bind_port,
)
class Command(BaseCommand):
help = 'Saves Companies data into DB'
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.converter = DeclarationConverter()
self.savepoint = Savepoint('declarations_steps', type=str)
def add_arguments(self, parser):
pass
def resave_transactions(self, data, declaration):
Transaction.objects.filter(declaration=declaration).delete()
self.converter.save_transaction(data, declaration)
def resave_liability(self, data, declaration):
Liability.objects.filter(declaration=declaration).delete()
self.converter.save_liability(data, declaration)
def resave_property(self, data, declaration):
Property.objects.filter(declaration=declaration).exclude(
type=Property.UNFINISHED_CONSTRUCTION, ).delete()
self.converter.save_property(data, declaration)
def resave_unfinished_construction(self, data, declaration):
Property.objects.filter(
declaration=declaration,
type=Property.UNFINISHED_CONSTRUCTION,
).delete()
self.converter.save_unfinished_construction(data, declaration)
def resave_all_steps_with_city(self, data, declaration):
self.converter.save_declarant_data(data, declaration)
self.resave_liability(data, declaration)
self.resave_property(data, declaration)
self.resave_unfinished_construction(data, declaration)
def process_declaration(self, data, declaration):
# self.resave_transactions(data, declaration)
self.resave_all_steps_with_city(data, declaration)
def handle(self, *args, **options):
i = 0
qs = Declaration.objects.all()
count = qs.count()
for declaration in qs:
i += 1
self.stdout.write(f'\rProgress: {i} of {count}', ending='')
self.stdout.flush()
if self.savepoint.has(declaration.nacp_declaration_id):
continue
self.converter.current_declaration = declaration
data = self.converter.download_declaration(
str(declaration.nacp_declaration_id))
if not data:
raise CommandError(
f'No data for declaration {declaration.nacp_declaration_id}'
)
self.converter.save_relatives_data(data['data'], declaration)
self.process_declaration(data=data['data'],
declaration=declaration)
self.savepoint.add(declaration.nacp_declaration_id)
self.savepoint.close()
self.stdout.write()
self.stdout.write('Resave done. Start running all fixes')
DeclarationsFixSet().run_all_fixes()
self.stdout.write('All fixes applied')
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.converter = DeclarationConverter()
self.savepoint = Savepoint('declarations_steps', type=str)