def variable_data(request):
sectors = []
for sector_table in FacilityTable.objects.all():
sectors.append(sector_table.display_dict)
overview_csv = CsvReader(os.path.join(settings.PROJECT_ROOT, "data", "table_definitions", "overview.csv"))
overview_data = []
for z in overview_csv.iter_dicts():
overview_data.append(z)
return HttpResponse(json.dumps({"sectors": sectors, "overview": overview_data}))
def load_lga_data_from_csv(self, path, row_contains_variable_slug=False):
csv_reader = CsvReader(path)
for d in csv_reader.iter_dicts():
if '_lga_id' not in d:
print "MISSING LGA ID:", d
continue
if d['_lga_id'] not in self.lga_ids:
continue
lga = LGA.objects.get(id=d['_lga_id'])
if row_contains_variable_slug:
if 'slug' in d and 'value' in d:
lga.add_data_from_dict({d['slug']: d['value']})
else:
print "MISSING SLUG OR VALUE:", d
else:
lga.add_data_from_dict(d, and_calculate=True)
def mark_available_lgas(self):
lga_ids = []
facility_csv_files = [ff['data_source'] for ff in self._config['facility_csvs']]
#this process takes about 6 seconds...
for csv_file in facility_csv_files:
data_dir = os.path.join(self._data_dir, 'facility_csvs')
path = os.path.join(data_dir, csv_file)
csv_reader = CsvReader(path)
for d in csv_reader.iter_dicts():
lga_id = d.get('_lga_id')
if lga_id is not None and lga_id not in lga_ids:
lga_ids.append(lga_id)
for lga_id in lga_ids:
try:
lga = LGA.objects.get(id=lga_id)
lga.data_available=True
lga.save()
except LGA.DoesNotExist, e:
print "lga not found: %s" % str(lga_id)
def initialisation_repartition_dataset(chemin_csv):
ligne = CsvReader.recuperer_derniere_ligne_csv(chemin_csv)
nbre_images = int(ligne[1])
nbre_eval = int(nbre_images / 4)
nbre_train = nbre_images - nbre_eval
print("Nombre d'images pour l'entrainement : {}".format(nbre_train))
print("Nombre d'images pour l'évaluation : {}".format(nbre_eval))
SPLITS_TO_SIZES['train'] = nbre_train
SPLITS_TO_SIZES['test'] = nbre_eval
def load_table_types(table_types):
for tt_data in table_types:
name = tt_data['name']
slug = tt_data['slug']
data_source = tt_data['data_source']
curtable = FacilityTable.objects.create(name=name, slug=slug)
csv_reader = CsvReader(os.path.join(self._data_dir,"table_definitions", data_source))
display_order = 0
for input_d in csv_reader.iter_dicts():
subs = []
if 'subgroups' in input_d:
for sg in input_d['subgroups'].split(" "):
if sg in subgroups:
subs.append({'name': subgroups[sg], 'slug': sg})
for sub in subs:
curtable.add_column(sub)
try:
input_d['display_order'] = display_order
d = TableColumn.load_row_from_csv(input_d)
display_order += 1
curtable.add_variable(d)
except:
print "Error importing table definition for data: %s" % input_d
def create(self, chemin_csv, classes):
"""Permet la création d'un fichier TFRecords grâces aux associations
image - label du dataset, inscrit dans un fichier CSV."""
writer = tf.python_io.TFRecordWriter(self.chemin_tfrecords)
lignes_csv = CsvReader.recuperer_lignes_csv(chemin_csv)
print("Enregistrement du fichier TFRecords en cours...")
classes_map = visages.labels_to_class_name(classes)
for ligne in lignes_csv:
chemin_image = os.path.join(self.chemin_dataset, "{}.{}".format(ligne[0], ligne[1]))
image = cv2.imread(chemin_image)
height, width, channels = image.shape
# encodage de l'image pour créer le record
encoded_image = cv2.imencode('.{}'.format(ligne[1]), image)[1].tostring()
if int(ligne[4]) in classes_map:
print('Adding image {}x{} at path {} for label {}'.format(width, height, chemin_image, ligne[4].encode()))
record = tf.train.Example(features=tf.train.Features(
feature={
'image/height': _int64_feature(height),
'image/width': _int64_feature(width),
'image/encoded': _bytes_feature(encoded_image),
'image/format': _bytes_feature(ligne[1].encode('utf8')),
'image/class/label': _int64_feature(int(ligne[4]))
}
))
writer.write(record.SerializeToString())
writer.close()
def __init__(self):
path = 'data/cities'
self.csv_reader = CsvReader(path)
self.cached_ip_address = base.Client('memcached:11211',
serde=JsonSerde())
def create_objects_from_csv(self, model, path):
csv_reader = CsvReader(path)
for d in csv_reader.iter_dicts():
model.objects.get_or_create(**d)