Example #1
0
def variable_data(request):
    sectors = []
    for sector_table in FacilityTable.objects.all():
        sectors.append(sector_table.display_dict)
    overview_csv = CsvReader(os.path.join(settings.PROJECT_ROOT, "data", "table_definitions", "overview.csv"))
    overview_data = []
    for z in overview_csv.iter_dicts():
        overview_data.append(z)
    return HttpResponse(json.dumps({"sectors": sectors, "overview": overview_data}))
Example #2
0
 def load_lga_data_from_csv(self, path, row_contains_variable_slug=False):
     csv_reader = CsvReader(path)
     for d in csv_reader.iter_dicts():
         if '_lga_id' not in d:
             print "MISSING LGA ID:", d
             continue
         if d['_lga_id'] not in self.lga_ids:
             continue
         lga = LGA.objects.get(id=d['_lga_id'])
         if row_contains_variable_slug:
             if 'slug' in d and 'value' in d:
                 lga.add_data_from_dict({d['slug']: d['value']})
             else:
                 print "MISSING SLUG OR VALUE:", d
         else:
             lga.add_data_from_dict(d, and_calculate=True)
Example #3
0
 def mark_available_lgas(self):
     lga_ids = []
     facility_csv_files = [ff['data_source'] for ff in self._config['facility_csvs']]
     #this process takes about 6 seconds...
     for csv_file in facility_csv_files:
         data_dir = os.path.join(self._data_dir, 'facility_csvs')
         path = os.path.join(data_dir, csv_file)
         csv_reader = CsvReader(path)
         for d in csv_reader.iter_dicts():
             lga_id = d.get('_lga_id')
             if lga_id is not None and lga_id not in lga_ids:
                 lga_ids.append(lga_id)
     for lga_id in lga_ids:
         try:
             lga = LGA.objects.get(id=lga_id)
             lga.data_available=True
             lga.save()
         except LGA.DoesNotExist, e:
             print "lga not found: %s" % str(lga_id)
def initialisation_repartition_dataset(chemin_csv):
    ligne = CsvReader.recuperer_derniere_ligne_csv(chemin_csv)
    nbre_images = int(ligne[1])

    nbre_eval = int(nbre_images / 4)
    nbre_train = nbre_images - nbre_eval

    print("Nombre d'images pour l'entrainement : {}".format(nbre_train))
    print("Nombre d'images pour l'évaluation : {}".format(nbre_eval))
    SPLITS_TO_SIZES['train'] = nbre_train
    SPLITS_TO_SIZES['test'] = nbre_eval
Example #5
0
 def load_table_types(table_types):
     for tt_data in table_types:
         name = tt_data['name']
         slug = tt_data['slug']
         data_source = tt_data['data_source']
         curtable = FacilityTable.objects.create(name=name, slug=slug)
         csv_reader = CsvReader(os.path.join(self._data_dir,"table_definitions", data_source))
         display_order = 0
         for input_d in csv_reader.iter_dicts():
             subs = []
             if 'subgroups' in input_d:
                 for sg in input_d['subgroups'].split(" "):
                     if sg in subgroups:
                         subs.append({'name': subgroups[sg], 'slug': sg})
             for sub in subs:
                 curtable.add_column(sub)
             try:
                 input_d['display_order'] = display_order
                 d = TableColumn.load_row_from_csv(input_d)
                 display_order += 1
                 curtable.add_variable(d)
             except:
                 print "Error importing table definition for data: %s" % input_d
    def create(self, chemin_csv, classes):
        """Permet la création d'un fichier TFRecords grâces aux associations
            image - label du dataset, inscrit dans un fichier CSV."""

        writer = tf.python_io.TFRecordWriter(self.chemin_tfrecords)
        lignes_csv = CsvReader.recuperer_lignes_csv(chemin_csv)

        print("Enregistrement du fichier TFRecords en cours...")

        classes_map = visages.labels_to_class_name(classes)

        for ligne in lignes_csv:
            chemin_image = os.path.join(self.chemin_dataset, "{}.{}".format(ligne[0], ligne[1]))

            image = cv2.imread(chemin_image)

            height, width, channels = image.shape

            # encodage de l'image pour créer le record
            encoded_image = cv2.imencode('.{}'.format(ligne[1]), image)[1].tostring()

            if int(ligne[4]) in classes_map:

                print('Adding image {}x{} at path {} for label {}'.format(width, height, chemin_image, ligne[4].encode()))

                record = tf.train.Example(features=tf.train.Features(
                    feature={
                        'image/height': _int64_feature(height),
                        'image/width': _int64_feature(width),
                        'image/encoded': _bytes_feature(encoded_image),
                        'image/format': _bytes_feature(ligne[1].encode('utf8')),
                        'image/class/label': _int64_feature(int(ligne[4]))
                    }
                ))

                writer.write(record.SerializeToString())

        writer.close()
Example #7
0
 def __init__(self):
     path = 'data/cities'
     self.csv_reader = CsvReader(path)
     self.cached_ip_address = base.Client('memcached:11211',
                                          serde=JsonSerde())
Example #8
0
 def create_objects_from_csv(self, model, path):
     csv_reader = CsvReader(path)
     for d in csv_reader.iter_dicts():
         model.objects.get_or_create(**d)