def update_data( new_data_filename, diff_date, skip_fields, verbose=False ):

    from entity_record import entity_record
    print "updating with new data from %s..." % (new_data_filename)
    
    for data_i, record in enumerate( base_record.iter_json_records( new_data_filename ) ):

        if verbose:
            if data_i > 0 and data_i % 100 == 0:
                sys.stdout.write( '\r%d' % (data_i) )
                sys.stdout.flush()

        record = exemption_record(record)

        if record.exists():
            record.save_diff( diff_date, skip_fields=skip_fields )
        else:
            record.add_to_db()
            
        record.update_entity()

        # if record['publication_id'] == 12664:
        #     print 
        #     print record.normalize_data( record['supplier']  )
        #     print record
        #     print record.get_entity()

    if verbose:
        print '\r%d' % (data_i)

    exemption_record.write_records()
    entity_record.write_records()
def update_entities():

    from entity_record import entity_record
    for record in exemption_record.iter_records():

        record = exemption_record(record)
        record.update_entity()

        if record['publication_id'] == 12664:
            print record.get_entity()

    entity_record.write_records()