Пример #1
0
    def save_to_db(self, file):
        with open(file, newline='') as csvfile:
            for row in DictReader(csvfile):
                name = row['CompanyName'].lower()
                number = row[' CompanyNumber']
                code = name + number
                country = AddressConverter().save_or_get_country(
                    row['CountryOfOrigin'])
                address = (
                    f"{row['RegAddress.Country']} {row['RegAddress.PostCode']} "
                    f"{row['RegAddress.County']} {row['RegAddress.PostTown']} "
                    f"{row[' RegAddress.AddressLine2']} {row['RegAddress.AddressLine1']} "
                    f"{row['RegAddress.POBox']} {row['RegAddress.CareOf']}")
                company_type = self.save_or_get_company_type(
                    row['CompanyCategory'], 'en')
                status = self.save_or_get_status(row['CompanyStatus'])
                registration_date = format_date_to_yymmdd(
                    row['IncorporationDate'])
                company = Company.objects.filter(edrpou=number).first()
                if not company:
                    company = Company(name=name,
                                      company_type=company_type,
                                      edrpou=number,
                                      address=address,
                                      country=country,
                                      status=status,
                                      registration_date=registration_date,
                                      code=code)
                    company.save()
                else:
                    update_fields = []
                    if company.name != name:
                        company.name = name
                        update_fields.append('name')
                    if company.company_type != company_type:
                        company.company_type = company_type
                        update_fields.append('company_type')
                    if company.address != address:
                        company.address = address
                        update_fields.append('address')
                    if company.country != country:
                        company.country = country
                        update_fields.append('country')
                    if company.status != status:
                        company.status = status
                        update_fields.append('status')
                    if to_lower_string_if_exists(
                            company.registration_date) != registration_date:
                        company.registration_date = registration_date
                        update_fields.append('registration_date')
                    if company.code != code:
                        company.code = code
                        update_fields.append('code')

                    if update_fields:
                        company.save(update_fields=update_fields)

            print('All companies from UK register were saved')
Пример #2
0
 def save_detail_company_to_db(self, records):
     for record in records:
         name = record.xpath('NAME')[0].text.lower()
         short_name = record.xpath('SHORT_NAME')[0].text
         if short_name:
             short_name = short_name.lower()
         company_type = record.xpath('OPF')[0].text
         if company_type:
             company_type = self.save_or_get_company_type(
                 company_type, 'uk')
         edrpou = record.xpath('EDRPOU')[0].text
         if not edrpou:
             self.report.invalid_data += 1
             continue
         code = name + edrpou
         address = record.xpath('ADDRESS')[0].text
         status = self.save_or_get_status(record.xpath('STAN')[0].text)
         founding_document_number = record.xpath(
             'FOUNDING_DOCUMENT_NUM')[0].text
         executive_power = record.xpath('EXECUTIVE_POWER')[0].text
         if executive_power:
             executive_power = executive_power.lower()
         # if len(record.xpath('ACTIVITY_KINDS')[0]):
         #     self.add_company_to_kved(record.xpath('ACTIVITY_KINDS')[0], code)
         superior_management = record.xpath('SUPERIOR_MANAGEMENT')[0].text
         if superior_management:
             superior_management = superior_management.lower()
         # if len(record.xpath('SIGNERS')[0]):
         #     self.add_signers(record.xpath('SIGNERS')[0], code)
         authorized_capital = record.xpath('AUTHORIZED_CAPITAL')[0].text
         if authorized_capital:
             authorized_capital = authorized_capital.replace(',', '.')
             authorized_capital = float(authorized_capital)
         bylaw = self.save_or_get_bylaw(record.xpath('STATUTE')[0].text)
         registration_date = None
         registration_info = None
         registration = record.xpath('REGISTRATION')[0].text
         if registration:
             registration_date = format_date_to_yymmdd(
                 get_first_word(registration))
             registration_info = cut_first_word(registration)
         managing_paper = record.xpath('MANAGING_PAPER')[0].text
         if managing_paper:
             managing_paper = managing_paper.lower()
         # TODO: refactor branches storing
         # if len(record.xpath('BRANCHES')[0]):
         #     self.add_branches(record.xpath('BRANCHES')[0], code)
         # if record.xpath('TERMINATION_STARTED_INFO/OP_DATE'):
         #     self.add_termination_started(record, code)
         # if record.xpath('BANKRUPTCY_READJUSTMENT_INFO/OP_DATE'):
         #     self.add_bancruptcy_readjustment(record, code)
         # if len(record.xpath('PREDECESSORS')[0]):
         #     self.add_company_to_predecessors(record.xpath('PREDECESSORS')[0], code)
         # if len(record.xpath('ASSIGNEES')[0]):
         #     self.add_assignees(record.xpath('ASSIGNEES')[0], code)
         terminated_info = record.xpath('TERMINATED_INFO')[0].text
         if terminated_info:
             terminated_info = terminated_info.lower()
         termination_cancel_info = record.xpath(
             'TERMINATION_CANCEL_INFO')[0].text
         if termination_cancel_info:
             termination_cancel_info = termination_cancel_info.lower()
         contact_info = record.xpath('CONTACTS')[0].text
         # if record.xpath('EXCHANGE_DATA')[0]:
         #     self.add_exchange_data(record.xpath('EXCHANGE_DATA')[0], code)
         vp_dates = record.xpath('VP_DATES')[0].text
         authority = self.save_or_get_authority(
             record.xpath('CURRENT_AUTHORITY')[0].text)
         # self.add_company_detail(founding_document_number, executive_power, superior_management, managing_paper,
         #                         terminated_info, termination_cancel_info, vp_dates, code)
         # ToDo: resolve the problem of having records with the same company name amd edrpou
         # that results in the same code
         company = Company.objects.filter(code=code).first()
         if not company:
             company = Company(name=name,
                               short_name=short_name,
                               company_type=company_type,
                               edrpou=edrpou,
                               authorized_capital=authorized_capital,
                               status=status,
                               bylaw=bylaw,
                               registration_date=registration_date,
                               registration_info=registration_info,
                               contact_info=contact_info,
                               authority=authority,
                               code=code)
             company.save()
             # self.bulk_manager.add_create(company)
         else:
             update_fields = []
             if company.name != name:
                 company.name = name
                 update_fields.append('name')
             if company.short_name != short_name:
                 company.short_name = short_name
                 update_fields.append('short_name')
             if company.company_type != company_type:
                 company.company_type = company_type
                 update_fields.append('company_type')
             if company.authorized_capital != authorized_capital:
                 company.authorized_capital = authorized_capital
                 update_fields.append('authorized_capital')
             if company.address != address:
                 company.address = address
                 update_fields.append('address')
             if company.status != status:
                 company.status = status
                 update_fields.append('status')
             if company.bylaw != bylaw:
                 company.bylaw = bylaw
                 update_fields.append('bylaw')
             if to_lower_string_if_exists(
                     company.registration_date) != registration_date:
                 company.registration_date = registration_date
                 update_fields.append('registration_date')
             if company.registration_info != registration_info:
                 company.registration_info = registration_info
                 update_fields.append('registration_info')
             if company.contact_info != contact_info:
                 company.contact_info = contact_info
                 update_fields.append('contact_info')
             if company.authority != authority:
                 company.authority = authority
                 update_fields.append('authority')
             if update_fields:
                 update_fields.append('updated_at')
                 company.save(update_fields=update_fields)
                 # self.bulk_manager.add_update(company)
         if len(record.xpath('FOUNDERS')[0]):
             self.save_or_update_founders(
                 record.xpath('FOUNDERS')[0], company)
     # if len(self.bulk_manager.update_queues['business_register.Company']):
     #     self.bulk_manager.commit_update(Company, ['name', 'short_name', 'company_type',
     #                                               'authorized_capital', 'address', 'status',
     #                                               'bylaw', 'registration_date',
     #                                               'registration_info', 'contact_info',
     #                                               'authority'])
     # if len(self.bulk_manager.create_queues['business_register.Company']):
     #     self.bulk_manager.commit_create(Company)
     if len(self.bulk_manager.queues['business_register.Founder']):
         self.bulk_manager.commit(Founder)
     self.bulk_manager.queues['business_register.Founder'] = []
Пример #3
0
    def save_to_db(self, file):
        with open(file, newline='') as csvfile:
            index = 0

            for row in DictReader(csvfile):
                name = row['CompanyName'].lower()
                # number is unique identifier in Company House
                number = row[' CompanyNumber']
                code = name + number
                country = self.save_or_get_country(row['CountryOfOrigin'])
                address = (
                    f"{row['RegAddress.Country']} {row['RegAddress.PostCode']} "
                    f"{row['RegAddress.County']} {row['RegAddress.PostTown']} "
                    f"{row[' RegAddress.AddressLine2']} {row['RegAddress.AddressLine1']} "
                    f"{row['RegAddress.POBox']} {row['RegAddress.CareOf']}")
                company_type = self.save_or_get_company_type(
                    row['CompanyCategory'], 'en')
                status = self.save_or_get_status(row['CompanyStatus'])
                if len(row['IncorporationDate']) == 10:
                    registration_date = format_date_to_yymmdd(
                        row['IncorporationDate'])
                else:
                    registration_date = None
                source = Company.GREAT_BRITAIN_REGISTER
                company = Company.objects.filter(
                    edrpou=number,
                    source=Company.GREAT_BRITAIN_REGISTER).first()
                if not company:
                    company = Company(name=name,
                                      company_type=company_type,
                                      edrpou=number,
                                      address=address,
                                      country=country,
                                      status=status,
                                      registration_date=registration_date,
                                      code=code,
                                      source=source)
                    company.save()
                else:
                    update_fields = []
                    if company.name != name:
                        company.name = name
                        update_fields.append('name')
                    if company.company_type_id != company_type.id:
                        company.company_type = company_type
                        update_fields.append('company_type')
                    if company.address != address:
                        company.address = address
                        update_fields.append('address')
                    if company.country_id != country.id:
                        company.country = country
                        update_fields.append('country')
                    if company.status_id != status.id:
                        company.status = status
                        update_fields.append('status')
                    if to_lower_string_if_exists(
                            company.registration_date) != registration_date:
                        company.registration_date = registration_date
                        update_fields.append('registration_date')
                    if company.code != code:
                        company.code = code
                        update_fields.append('code')
                    if company.source != source:
                        company.source = source
                        update_fields.append('source')
                    if update_fields:
                        update_fields.append('updated_at')
                        company.save(update_fields=update_fields)

            print('All companies from UK register were saved')
Пример #4
0
 def save_to_db(self, records):
     for record in records:
         fullname = record.xpath('NAME')[0].text
         if not fullname:
             logger.warning(f'ФОП без прізвища: {record}')
             # self.report.invalid_data += 1
             continue
         if len(fullname) > 100:
             logger.warning(f'ФОП із задовгим прізвищем: {record}')
             continue
         if fullname:
             fullname = fullname.lower()
         address = record.xpath('ADDRESS')[0].text
         if not address:
             address = 'EMPTY'
         code = fullname + address
         status = self.save_or_get_status(record.xpath('STAN')[0].text)
         registration_text = record.xpath('REGISTRATION')[0].text
         # first getting date, then registration info if REGISTRATION.text exists
         registration_date = None
         registration_date_second = None
         registration_number = None
         registration_info = None
         if registration_text:
             registration_info = registration_text
             registration_text = registration_text.split()
             registration_date = format_date_to_yymmdd(registration_text[0])
             registration_date_second = format_date_to_yymmdd(
                 registration_text[1])
             if 3 <= len(registration_text):
                 registration_number = registration_text[2]
         estate_manager = record.xpath('ESTATE_MANAGER')[0].text
         termination_text = record.xpath('TERMINATED_INFO')[0].text
         termination_date = None
         terminated_info = None
         if termination_text:
             termination_date = format_date_to_yymmdd(
                 get_first_word(termination_text))
             terminated_info = cut_first_word(termination_text)
         termination_cancel_info = record.xpath(
             'TERMINATION_CANCEL_INFO')[0].text
         contact_info = record.xpath('CONTACTS')[0].text
         vp_dates = record.xpath('VP_DATES')[0].text
         if record.xpath('CURRENT_AUTHORITY')[0].text:
             authority = self.save_or_get_authority(
                 record.xpath('CURRENT_AUTHORITY')[0].text)
         else:
             authority = None
         fop_kveds = record.xpath('ACTIVITY_KINDS')[0]
         exchange_data = record.xpath('EXCHANGE_DATA')[0]
         self.time_it('getting data from record')
         fop = Fop.objects.filter(code=code).first()
         self.time_it('trying get fops\t\t')
         if not fop:
             fop = Fop(fullname=fullname,
                       address=address,
                       status=status,
                       registration_date=registration_date,
                       registration_date_second=registration_date_second,
                       registration_number=registration_number,
                       registration_info=registration_info,
                       estate_manager=estate_manager,
                       termination_date=termination_date,
                       terminated_info=terminated_info,
                       termination_cancel_info=termination_cancel_info,
                       contact_info=contact_info,
                       vp_dates=vp_dates,
                       authority=authority,
                       code=code)
             self.bulk_manager.add(fop)
             if len(fop_kveds):
                 self.add_fop_kveds_to_dict(fop_kveds, code)
             if len(exchange_data):
                 self.add_fop_exchange_data_to_dict(exchange_data, code)
             self.time_it('save fops\t\t')
         else:
             # TODO: make a decision: our algorithm when Fop changes fullname or address?
             update_fields = []
             if fop.status_id != status.id:
                 fop.status_id = status.id
                 update_fields.append('status_id')
             if to_lower_string_if_exists(
                     fop.registration_date) != registration_date:
                 fop.registration_date = registration_date
                 update_fields.append('registration_date')
             if to_lower_string_if_exists(fop.registration_date_second
                                          ) != registration_date_second:
                 fop.registration_date_second = registration_date_second
                 update_fields.append('registration_date_second')
             if fop.registration_number != registration_number:
                 fop.registration_number = registration_number
                 update_fields.append('registration_number')
             if fop.registration_info != registration_info:
                 fop.registration_info = registration_info
                 update_fields.append('registration_info')
             if fop.estate_manager != estate_manager:
                 fop.estate_manager = estate_manager
                 update_fields.append('estate_manager')
             if fop.termination_date and str(
                     fop.termination_date) != termination_date:
                 fop.termination_date = termination_date
                 update_fields.append('termination_date')
             if fop.terminated_info != terminated_info:
                 fop.terminated_info = terminated_info
                 update_fields.append('terminated_info')
             if fop.termination_cancel_info != termination_cancel_info:
                 fop.termination_cancel_info = termination_cancel_info
                 update_fields.append('termination_cancel_info')
             if fop.contact_info != contact_info:
                 fop.contact_info = contact_info
                 update_fields.append('contact_info')
             if fop.vp_dates != vp_dates:
                 fop.vp_dates = vp_dates
                 update_fields.append('vp_dates')
             if fop.authority_id != authority.id:
                 fop.authority_id = authority.id
                 update_fields.append('authority_id')
             self.time_it('compare fops\t\t')
             if len(update_fields):
                 update_fields.append('updated_at')
                 fop.save(update_fields=update_fields)
             self.time_it('update fops\t\t')
             if len(fop_kveds):
                 self.update_fop_kveds(fop_kveds, fop)
             self.time_it('delete outdated kveds\t')
             if len(exchange_data):
                 self.update_fop_exchange_data(exchange_data, fop)
             self.time_it('update exchange_data\t')
     if len(self.bulk_manager.queues['business_register.Fop']):
         self.bulk_manager.commit(Fop)
     for fop in self.bulk_manager.queues['business_register.Fop']:
         if fop.code not in self.new_fops_foptokveds:
             continue
         foptokveds = self.new_fops_foptokveds[fop.code]
         for foptokved in foptokveds:
             foptokved.fop = fop
             self.bulk_manager.add(foptokved)
     self.new_fops_foptokveds = {}
     for fop in self.bulk_manager.queues['business_register.Fop']:
         if fop.code not in self.new_fops_exchange_data:
             continue
         fop_exchangedata = self.new_fops_exchange_data[fop.code]
         for exchangedata in fop_exchangedata:
             exchangedata.fop = fop
             self.bulk_manager.add(exchangedata)
     self.new_fops_exchange_data = {}
     self.bulk_manager.queues['business_register.Fop'] = []
     if len(self.bulk_manager.queues['business_register.FopToKved']):
         self.bulk_manager.commit(FopToKved)
     if len(self.bulk_manager.queues['business_register.ExchangeDataFop']):
         self.bulk_manager.commit(ExchangeDataFop)
     self.bulk_manager.queues['business_register.FopToKved'] = []
     self.bulk_manager.queues['business_register.ExchangeDataFop'] = []
     self.time_it('save others\t\t')
Пример #5
0
 def save_or_update_peps(self, peps_data):
     for pep_data in peps_data:
         source_id = pep_data[0]
         code = str(source_id)
         last_name = pep_data[1].lower()
         first_name = pep_data[2].lower()
         middle_name = pep_data[3].lower()
         fullname = f'{last_name} {first_name} {middle_name}'
         fullname_eng = f'{pep_data[4]} {pep_data[5]} {pep_data[6]}'.lower()
         fullname_transcriptions_eng = pep_data[7].lower()
         is_pep = pep_data[8]
         date_of_birth = to_lower_string_if_exists(pep_data[9])
         place_of_birth = to_lower_string_if_exists(pep_data[10])
         place_of_birth_eng = to_lower_string_if_exists(pep_data[11])
         sanctions = pep_data[12]
         sanctions_eng = pep_data[13]
         criminal_record = pep_data[14]
         criminal_record_eng = pep_data[15]
         assets_info = pep_data[16]
         assets_info_eng = pep_data[17]
         criminal_proceedings = pep_data[18]
         criminal_proceedings_eng = pep_data[19]
         wanted = pep_data[20]
         wanted_eng = pep_data[21]
         info = pep_data[22]
         info_eng = pep_data[23]
         pep_type_number = pep_data[24]
         pep_type = self.PEP_TYPES.get(
             pep_type_number) if pep_type_number else None
         reason_of_termination_number = to_lower_string_if_exists(
             pep_data[25])
         reason_of_termination = (
             self.PEP_TYPES.get(reason_of_termination_number)
             if reason_of_termination_number else None)
         is_dead = (reason_of_termination_number == 1)
         termination_date = to_lower_string_if_exists(pep_data[26])
         pep = self.all_peps_dict.get(source_id)
         if not pep:
             pep = Pep.objects.create(
                 code=code,
                 first_name=first_name,
                 middle_name=middle_name,
                 last_name=last_name,
                 fullname=fullname,
                 fullname_eng=fullname_eng,
                 fullname_transcriptions_eng=fullname_transcriptions_eng,
                 info=info,
                 info_eng=info_eng,
                 sanctions=sanctions,
                 sanctions_eng=sanctions_eng,
                 criminal_record=criminal_record,
                 criminal_record_eng=criminal_record_eng,
                 assets_info=assets_info,
                 assets_info_eng=assets_info_eng,
                 criminal_proceedings=criminal_proceedings,
                 criminal_proceedings_eng=criminal_proceedings_eng,
                 wanted=wanted,
                 wanted_eng=wanted_eng,
                 date_of_birth=date_of_birth,
                 place_of_birth=place_of_birth,
                 place_of_birth_eng=place_of_birth_eng,
                 is_pep=is_pep,
                 pep_type=pep_type,
                 is_dead=is_dead,
                 termination_date=termination_date,
                 reason_of_termination=reason_of_termination,
                 source_id=source_id)
             self.all_peps_dict[source_id] = pep
         else:
             update_fields = []
             if pep.first_name != first_name:
                 pep.first_name = first_name
                 update_fields.append('first_name')
             if pep.middle_name != middle_name:
                 pep.middle_name = middle_name
                 update_fields.append('middle_name')
             if pep.last_name != last_name:
                 pep.last_name = last_name
                 update_fields.append('last_name')
             if pep.fullname_transcriptions_eng != fullname_transcriptions_eng:
                 pep.fullname_transcriptions_eng = fullname_transcriptions_eng
                 update_fields.append('fullname_transcriptions_eng')
             if pep.info != info:
                 pep.info = info
                 update_fields.append('info')
             if pep.info_eng != info_eng:
                 pep.info_eng = info_eng
                 update_fields.append('info_eng')
             if pep.sanctions != sanctions:
                 pep.sanctions = sanctions
                 update_fields.append('sanctions')
             if pep.sanctions_eng != sanctions_eng:
                 pep.sanctions_eng = sanctions_eng
                 update_fields.append('sanctions_eng')
             if pep.criminal_record != criminal_record:
                 pep.criminal_record = criminal_record
                 update_fields.append('criminal_record')
             if pep.criminal_record_eng != criminal_record_eng:
                 pep.criminal_record_eng = criminal_record_eng
                 update_fields.append('criminal_record_eng')
             if pep.assets_info != assets_info:
                 pep.assets_info = assets_info
                 update_fields.append('assets_info')
             if pep.assets_info_eng != assets_info_eng:
                 pep.assets_info_eng = assets_info_eng
                 update_fields.append('assets_info_eng')
             if pep.criminal_proceedings != criminal_proceedings:
                 pep.criminal_proceedings = criminal_proceedings
                 update_fields.append('criminal_proceedings')
             if pep.criminal_proceedings_eng != criminal_proceedings_eng:
                 pep.criminal_proceedings_eng = criminal_proceedings_eng
                 update_fields.append('criminal_proceedings_eng')
             if pep.wanted != wanted:
                 pep.wanted = wanted
                 update_fields.append('wanted')
             if pep.wanted_eng != wanted_eng:
                 pep.wanted_eng = wanted_eng
                 update_fields.append('wanted_eng')
             if pep.date_of_birth != date_of_birth:
                 pep.date_of_birth = date_of_birth
                 update_fields.append('date_of_birth')
             if pep.place_of_birth != place_of_birth:
                 pep.place_of_birth = place_of_birth
                 update_fields.append('place_of_birth')
             if pep.place_of_birth_eng != place_of_birth_eng:
                 pep.place_of_birth_eng = place_of_birth_eng
                 update_fields.append('place_of_birth_eng')
             if pep.is_pep != is_pep:
                 pep.is_pep = is_pep
                 update_fields.append('is_pep')
             if pep.pep_type != pep_type:
                 pep.pep_type = pep_type
                 update_fields.append('pep_type')
             if pep.is_dead != is_dead:
                 pep.is_dead = is_dead
                 update_fields.append('is_dead')
             if pep.termination_date != termination_date:
                 pep.termination_date = termination_date
                 update_fields.append('termination_date')
             if pep.reason_of_termination != reason_of_termination:
                 pep.reason_of_termination = reason_of_termination
                 update_fields.append('reason_of_termination')
             if pep.source_id != source_id:
                 pep.source_id = source_id
                 update_fields.append('source_id')
             if len(update_fields):
                 pep.save(update_fields=update_fields)
Пример #6
0
 def save_or_update_peps_companies(self, peps_companies_data):
     for link in peps_companies_data:
         pep_source_id = link[0]
         pep = self.all_peps_dict.get(pep_source_id)
         if not pep:
             logger.info(
                 f'No such pep in our DB. '
                 f'Check records in the source DB with id {pep_source_id}')
             continue
         company_antac_id = link[1]
         start_date = to_lower_string_if_exists(link[2])
         confirmation_date = to_lower_string_if_exists(link[3])
         end_date = to_lower_string_if_exists(link[4])
         category = link[5]
         edrpou = link[6]
         is_state_company = link[7]
         company_short_name_eng = link[8]
         company_name = link[9]
         company = Company.objects.filter(antac_id=company_antac_id).first()
         if not company and edrpou:
             company = Company.objects.filter(edrpou=edrpou).first()
             if company:
                 company.antac_id = company_antac_id
                 company.save(update_fields=['antac_id'])
         if not company:
             company = Company.objects.create(name=company_name,
                                              edrpou=edrpou,
                                              code=company_name + edrpou,
                                              antac_id=company_antac_id,
                                              from_antac_only=True)
             self.create_company_link_with_pep(company, pep,
                                               company_short_name_eng,
                                               category, start_date,
                                               confirmation_date, end_date,
                                               is_state_company)
         else:
             already_stored_link = CompanyLinkWithPep.objects.filter(
                 pep=pep, company=company).first()
             if already_stored_link:
                 update_fields = []
                 if already_stored_link.company_short_name_eng != company_short_name_eng:
                     already_stored_link.company_short_name_eng = company_short_name_eng
                     update_fields.append('company_short_name_eng')
                 if already_stored_link.category != category:
                     already_stored_link.category = category
                     update_fields.append('category')
                 if already_stored_link.start_date != start_date:
                     already_stored_link.start_date = start_date
                     update_fields.append('start_date')
                 if already_stored_link.confirmation_date != confirmation_date:
                     already_stored_link.confirmation_date = confirmation_date
                     update_fields.append('confirmation_date')
                 if already_stored_link.end_date != end_date:
                     already_stored_link.end_date = end_date
                     update_fields.append('end_date')
                 if already_stored_link.is_state_company != is_state_company:
                     already_stored_link.is_state_company = is_state_company
                     update_fields.append('is_state_company')
                 if update_fields:
                     already_stored_link.save(update_fields=update_fields)
             else:
                 self.create_company_link_with_pep(company, pep,
                                                   company_short_name_eng,
                                                   category, start_date,
                                                   confirmation_date,
                                                   end_date,
                                                   is_state_company)
Пример #7
0
 def save_or_update_peps(self, peps_data):
     for pep_data in peps_data:
         source_id = pep_data[0]
         code = str(source_id)
         last_name = pep_data[1].lower()
         first_name = pep_data[2].lower()
         middle_name = pep_data[3].lower()
         fullname = f'{last_name} {first_name} {middle_name}'
         fullname_transcriptions_eng = pep_data[7].lower()
         is_pep = pep_data[8]
         date_of_birth = self.parse_date_of_birth(pep_data[9])
         place_of_birth = to_lower_string_if_exists(pep_data[10])
         sanctions = pep_data[12]
         criminal_record = pep_data[14]
         assets_info = pep_data[16]
         criminal_proceedings = pep_data[18]
         wanted = pep_data[20]
         info = pep_data[22]
         pep_type_number = pep_data[24]
         pep_type = self.PEP_TYPES.get(
             pep_type_number) if pep_type_number else None
         reason_of_termination_number = pep_data[25]
         reason_of_termination = (
             self.REASONS_OF_TERMINATION.get(reason_of_termination_number)
             if reason_of_termination_number else None)
         is_dead = (reason_of_termination_number == 1)
         termination_date = pep_data[26]
         pep = self.peps_dict.get(code)
         if not pep:
             pep = Pep.objects.create(
                 source_id=source_id,
                 code=code,
                 first_name=first_name,
                 middle_name=middle_name,
                 last_name=last_name,
                 fullname=fullname,
                 fullname_transcriptions_eng=fullname_transcriptions_eng,
                 is_pep=is_pep,
                 date_of_birth=date_of_birth,
                 place_of_birth=place_of_birth,
                 sanctions=sanctions,
                 criminal_record=criminal_record,
                 assets_info=assets_info,
                 criminal_proceedings=criminal_proceedings,
                 wanted=wanted,
                 info=info,
                 pep_type=pep_type,
                 reason_of_termination=reason_of_termination,
                 is_dead=is_dead,
                 termination_date=termination_date,
             )
             self.peps_dict[code] = pep
         else:
             update_fields = []
             if pep.source_id != source_id:
                 pep.source_id = source_id
                 update_fields.append('source_id')
             if pep.first_name != first_name:
                 pep.first_name = first_name
                 update_fields.append('first_name')
             if pep.middle_name != middle_name:
                 pep.middle_name = middle_name
                 update_fields.append('middle_name')
             if pep.last_name != last_name:
                 pep.last_name = last_name
                 update_fields.append('last_name')
             if pep.fullname != fullname:
                 pep.fullname = fullname
                 update_fields.append('fullname')
             if pep.fullname_transcriptions_eng != fullname_transcriptions_eng:
                 pep.fullname_transcriptions_eng = fullname_transcriptions_eng
                 update_fields.append('fullname_transcriptions_eng')
             if pep.is_pep != is_pep:
                 pep.is_pep = is_pep
                 update_fields.append('is_pep')
             if pep.date_of_birth != date_of_birth:
                 pep.date_of_birth = date_of_birth
                 update_fields.append('date_of_birth')
             if pep.place_of_birth != place_of_birth:
                 pep.place_of_birth = place_of_birth
                 update_fields.append('place_of_birth')
             if pep.sanctions != sanctions:
                 pep.sanctions = sanctions
                 update_fields.append('sanctions')
             if pep.criminal_record != criminal_record:
                 pep.criminal_record = criminal_record
                 update_fields.append('criminal_record')
             if pep.assets_info != assets_info:
                 pep.assets_info = assets_info
                 update_fields.append('assets_info')
             if pep.criminal_proceedings != criminal_proceedings:
                 pep.criminal_proceedings = criminal_proceedings
                 update_fields.append('criminal_proceedings')
             if pep.wanted != wanted:
                 pep.wanted = wanted
                 update_fields.append('wanted')
             if pep.info != info:
                 pep.info = info
                 update_fields.append('info')
             if pep.pep_type != pep_type:
                 pep.pep_type = pep_type
                 update_fields.append('pep_type')
             if pep.reason_of_termination != reason_of_termination:
                 pep.reason_of_termination = reason_of_termination
                 update_fields.append('reason_of_termination')
             if pep.is_dead != is_dead:
                 pep.is_dead = is_dead
                 update_fields.append('is_dead')
             if pep.termination_date != termination_date:
                 pep.termination_date = termination_date
                 update_fields.append('termination_date')
             if len(update_fields):
                 update_fields.append('updated_at')
                 pep.save(update_fields=update_fields)
             del self.outdated_peps_dict[code]
     if self.outdated_peps_dict:
         for pep in self.outdated_peps_dict.values():
             pep.soft_delete()