def load_societies(items):
delete_all(Society)
societies = []
for item in items:
if item['dataset'] == 'EA':
source = get_source("EA")
elif item['dataset'] == 'LRB':
source = get_source("Binford")
else:
logging.warn(
"Could not determine source for row %s, skipping" % item
)
continue
societies.append(Society(
ext_id=item['soc_id'],
xd_id=item['xd_id'],
name=item['soc_name'],
source=source,
alternate_names=item['alternate_names'],
focal_year=item['main_focal_year'],
))
logging.info("Saving society %s" % item)
Society.objects.bulk_create(societies)
return len(societies)
def load_regions(geojson):
delete_all(GeographicRegion)
with open(geojson) as fp:
regions = json.load(fp)['features']
GeographicRegion.objects.bulk_create([
GeographicRegion(**{k.lower(): v for k, v in region['properties'].items()})
for region in regions])
return len(regions)
def load_regions(geojson):
delete_all(GeographicRegion)
with open(geojson) as fp:
regions = json.load(fp)['features']
GeographicRegion.objects.bulk_create([
GeographicRegion(
**{k.lower(): v
for k, v in region['properties'].items()}) for region in regions
])
return len(regions)
def load_societies(items):
society_links = [
'SCCS_society_equivalent',
'WNAI_society_equivalent1',
'WNAI_society_equivalent2',
'WNAI_society_equivalent3',
'WNAI_society_equivalent4',
'WNAI_society_equivalent5',
]
delete_all(Society)
societies = []
for item in items:
societies.append(Society(
ext_id=item['soc_id'],
xd_id=item['xd_id'],
original_name=item['ORIG_name_and_ID_in_this_dataset'],
name=item['pref_name_for_society'],
source=get_source(item['dataset']),
alternate_names=item['alt_names_by_society'],
focal_year=item['main_focal_year'],
hraf_link=item['HRAF_name_ID'],
chirila_link=item['CHIRILA_society_equivalent']
))
for key in society_links:
value = item.get(key)
if value:
ext_id = value.split('(')[len(value.split('(')) - 1]
society = Society(
ext_id=ext_id[0:len(ext_id) - 1],
xd_id=item['xd_id'],
original_name=value,
name=item['pref_name_for_society'],
alternate_names=item['alt_names_by_society'],
focal_year=item['main_focal_year'],
source=get_source(key[0:key.find('_')])
)
if society.ext_id not in [x.ext_id for x in societies]:
societies.append(society)
logging.info("Saving society %s" % society.ext_id)
logging.info("Saving society %s" % item)
Society.objects.bulk_create(societies)
return len(societies)
def xd_to_language(items, languoids):
delete_all(Language)
delete_all(LanguageFamily)
delete_all(ISOCode)
glottolog = {l['id']: l for l in languoids}
societies_ = defaultdict(list)
for s in Society.objects.all():
societies_[s.xd_id].append(s)
families = {}
languages = {}
isocodes = {}
count = 0
for item in items:
societies = societies_.get(item['xd_id'])
if not societies: # pragma: no cover
logging.warning("No societies found with xd_id %(xd_id)s" % item)
continue
ldata = glottolog.get(item['DialectLanguageGlottocode'])
if not ldata: # pragma: no cover
logging.warning("No language found for %s, skipping" % item)
continue
_xd_to_language(item, societies, ldata, languages, families, isocodes)
count += 1
return count
