def variable_data(request): sectors = [] for sector_table in FacilityTable.objects.all(): sectors.append(sector_table.display_dict) overview_csv = CsvReader(os.path.join(settings.PROJECT_ROOT, "data", "table_definitions", "overview.csv")) overview_data = [] for z in overview_csv.iter_dicts(): overview_data.append(z) return HttpResponse(json.dumps({"sectors": sectors, "overview": overview_data}))
def load_lga_data_from_csv(self, path, row_contains_variable_slug=False): csv_reader = CsvReader(path) for d in csv_reader.iter_dicts(): if '_lga_id' not in d: print "MISSING LGA ID:", d continue if d['_lga_id'] not in self.lga_ids: continue lga = LGA.objects.get(id=d['_lga_id']) if row_contains_variable_slug: if 'slug' in d and 'value' in d: lga.add_data_from_dict({d['slug']: d['value']}) else: print "MISSING SLUG OR VALUE:", d else: lga.add_data_from_dict(d, and_calculate=True)
def mark_available_lgas(self): lga_ids = [] facility_csv_files = [ff['data_source'] for ff in self._config['facility_csvs']] #this process takes about 6 seconds... for csv_file in facility_csv_files: data_dir = os.path.join(self._data_dir, 'facility_csvs') path = os.path.join(data_dir, csv_file) csv_reader = CsvReader(path) for d in csv_reader.iter_dicts(): lga_id = d.get('_lga_id') if lga_id is not None and lga_id not in lga_ids: lga_ids.append(lga_id) for lga_id in lga_ids: try: lga = LGA.objects.get(id=lga_id) lga.data_available=True lga.save() except LGA.DoesNotExist, e: print "lga not found: %s" % str(lga_id)
def initialisation_repartition_dataset(chemin_csv): ligne = CsvReader.recuperer_derniere_ligne_csv(chemin_csv) nbre_images = int(ligne[1]) nbre_eval = int(nbre_images / 4) nbre_train = nbre_images - nbre_eval print("Nombre d'images pour l'entrainement : {}".format(nbre_train)) print("Nombre d'images pour l'évaluation : {}".format(nbre_eval)) SPLITS_TO_SIZES['train'] = nbre_train SPLITS_TO_SIZES['test'] = nbre_eval
def load_table_types(table_types): for tt_data in table_types: name = tt_data['name'] slug = tt_data['slug'] data_source = tt_data['data_source'] curtable = FacilityTable.objects.create(name=name, slug=slug) csv_reader = CsvReader(os.path.join(self._data_dir,"table_definitions", data_source)) display_order = 0 for input_d in csv_reader.iter_dicts(): subs = [] if 'subgroups' in input_d: for sg in input_d['subgroups'].split(" "): if sg in subgroups: subs.append({'name': subgroups[sg], 'slug': sg}) for sub in subs: curtable.add_column(sub) try: input_d['display_order'] = display_order d = TableColumn.load_row_from_csv(input_d) display_order += 1 curtable.add_variable(d) except: print "Error importing table definition for data: %s" % input_d
def create(self, chemin_csv, classes): """Permet la création d'un fichier TFRecords grâces aux associations image - label du dataset, inscrit dans un fichier CSV.""" writer = tf.python_io.TFRecordWriter(self.chemin_tfrecords) lignes_csv = CsvReader.recuperer_lignes_csv(chemin_csv) print("Enregistrement du fichier TFRecords en cours...") classes_map = visages.labels_to_class_name(classes) for ligne in lignes_csv: chemin_image = os.path.join(self.chemin_dataset, "{}.{}".format(ligne[0], ligne[1])) image = cv2.imread(chemin_image) height, width, channels = image.shape # encodage de l'image pour créer le record encoded_image = cv2.imencode('.{}'.format(ligne[1]), image)[1].tostring() if int(ligne[4]) in classes_map: print('Adding image {}x{} at path {} for label {}'.format(width, height, chemin_image, ligne[4].encode())) record = tf.train.Example(features=tf.train.Features( feature={ 'image/height': _int64_feature(height), 'image/width': _int64_feature(width), 'image/encoded': _bytes_feature(encoded_image), 'image/format': _bytes_feature(ligne[1].encode('utf8')), 'image/class/label': _int64_feature(int(ligne[4])) } )) writer.write(record.SerializeToString()) writer.close()
def __init__(self): path = 'data/cities' self.csv_reader = CsvReader(path) self.cached_ip_address = base.Client('memcached:11211', serde=JsonSerde())
def create_objects_from_csv(self, model, path): csv_reader = CsvReader(path) for d in csv_reader.iter_dicts(): model.objects.get_or_create(**d)