def person(self, lib_object: JSON, person: Person) -> Person:
name = lib_object.get("name")
given_name = lib_object.get("givenName")
family_name = lib_object.get("familyName")
if not name:
if given_name and family_name:
name = given_name + " " + family_name
else:
logger.warning("Person without name: {}".format(lib_object["id"]))
name = _("Unknown")
if not given_name and not family_name and " " in name:
given_name = name.split(" ")[-2]
family_name = name.split(" ")[-1]
logger.warning("Inferring given and family name from compound name")
if not given_name:
logger.warning("Person without given name: {}".format(lib_object["id"]))
given_name = _("Unknown")
if not family_name:
logger.warning("Person without family name: {}".format(lib_object["id"]))
family_name = _("Unknown")
person.name = name
person.given_name = given_name
person.family_name = family_name
person.location = self.retrieve(
Location, lib_object.get("location"), person.oparl_id
)
return person
def import_inmate_file(csvid):
#if r.get("inmate_cache_init"):
#initiate_inmates()
csvfile = CsvBulkUpload.objects.all().filter(id = csvid)
new_data = csv.DictReader(open( csvfile.csv_file.url , 'r'))
#r = Redis()
camp_obj = False
for datum in new_data:
camp_obj = RescueCamp.objects.get(id = int(datum.get("camped_at", "")))
if #r.get(md5(datum.get("phone", "") + datum.get("name","") + datum.get("age",0))):
continue
else:
#r.set(md5(datum.get("phone", "") + datum.get("name","") + datum.get("age",0)), 1)
gender = 2
if( len(datum.get("gender", "")) > 0 ):
if(datum.get("gender", "")[0] == "m" or datum.get("gender", "")[0] == "M"):
gender = 0
elif(datum.get("gender", "")[0] == "f" or datum.get("gender", "")[0] == "F"):
gender = 1
Person(name = datum.get("name", "") ,phone = datum.get("phone", "") , age = datum.get("age", "") ,
gender = gender , address = datum.get("address", "") , notes = datum.get("notes", "") , camped_at = camp_obj , district = datum.get("district", ""),
status = "new" , checkin_date = parsedate(datum.get("checkin_date", None)) , checkout_date = parsedate(datum.get("checkout_date", None)) ).save()
url = CsvBulkUpload.objects.all().filter(id = csvid)
#For Shell Testing
#exec(open('mainapp/csvimporter.py').read())
def dashboard_register(request):
user = request.POST.get('user', False)
passs = request.POST.get('pass', False)
if user != False and passs != False:
user = User.objects.create_user(user, user + '@vehiclesystem.com',
passs)
user.save()
person = Person(person_user=user, person_city='Pune')
person.save()
return HttpResponseRedirect('/dashboard/login/')
context = {}
template = loader.get_template('register.html')
return HttpResponse(template.render(context, request))
def test_person_no_name(self):
data = {
"id": "https://oparl.example.org/person/no-name",
"type": "https://schema.oparl.org/1.1/Person",
"created": "2011-11-11T11:11:00+01:00",
"modified": "2012-08-16T14:05:27+02:00",
}
converter = JsonToDb(MockLoader())
with self.assertLogs(json_to_db.__name__, level="WARNING") as cm:
person = Person()
converter.person(data, person)
self.assertEqual(
cm.output,
[
"WARNING:"
+ json_to_db.__name__
+ ":Person without name: https://oparl.example.org/person/no-name",
"WARNING:"
+ json_to_db.__name__
+ ":Person without given name: https://oparl.example.org/person/no-name",
"WARNING:"
+ json_to_db.__name__
+ ":Person without family name: https://oparl.example.org/person/no-name",
],
)
def test_person_only_name(self):
data = {
"id": "https://oparl.example.org/person/only-name",
"type": "https://schema.oparl.org/1.1/Person",
"name": "Max Mustermann",
"created": "2011-11-11T11:11:00+01:00",
"modified": "2012-08-16T14:05:27+02:00",
}
converter = JsonToDb(MockLoader())
with self.assertLogs(json_to_db.__name__, level="WARNING") as cm:
person = Person()
converter.person(data, person)
self.assertEqual(person.name, "Max Mustermann")
self.assertEqual(person.given_name, "Max")
self.assertEqual(person.family_name, "Mustermann")
self.assertEqual(
cm.output,
[
"WARNING:"
+ json_to_db.__name__
+ ":Inferring given and family name from compound name"
],
)
def test_person_string_representation(self): person = Person( name="Kavit", phone="123456789", age="24", gender="male", address="near railway crossing", district="pkd") self.assertEqual(str(person), person.name)
def import_inmate_file(csvid):
import django
django.setup()
from mainapp.models import Person, RescueCamp, CsvBulkUpload
upload = CsvBulkUpload.objects.get(id = csvid)
upload.csv_file.open(mode="rb")
new_data = csv.DictReader(codecs.iterdecode(upload.csv_file.file, 'utf-8'))
for datum in new_data:
'''
try:
identifier_str = (datum.get("phone", "") + datum.get("name","") + datum.get("age",0)).encode('utf-8')
identifier = md5(identifier_str).hexdigest()
#this will fail. we should deal with the removed unique_identifier
p = Person.objects.get(unique_identifier=identifier)
except ValueError as e:
print("Invalid camp ID. row = "+ str(datum))
except RescueCamp.DoesNotExist as e:
print("Camp does not exist. row = "+ str(datum))
except Person.DoesNotExist:
'''
empty = 0
header = ["name" , "phone" , "address" , "notes" , "district" , "checkin_date" , "checkout_date" , "gender" , "age" ]
for i in header:
if(datum.get(i, "").strip() == ""):empty+=1
if(empty == len(header)):
continue
gender = 2
if( len(datum.get("gender", "")) > 0 ):
if(datum.get("gender", "")[0] == "m" or datum.get("gender", "")[0] == "M"):
gender = 0
elif(datum.get("gender", "")[0] == "f" or datum.get("gender", "")[0] == "F"):
gender = 1
age = '-1'
if(datum.get("age", "").strip() != ""):
age = datum.get("age", "").strip()
Person(
name = datum.get("name", ""),
phone = datum.get("phone", ""),
age = age,
gender = gender,
address = datum.get("address", ""),
notes = datum.get("notes", ""),
camped_at = upload.camp ,
district = datum.get("district", "").lower(),
status = "new",
checkin_date = parsedate(datum.get("checkin_date", None)),
checkout_date = parsedate(datum.get("checkout_date", None))
).save()
CsvBulkUpload.objects.filter(id = csvid).update(is_completed = True)
def test_person(self):
# Body is mandatory for organization
self.converter.import_anything("https://oparl.example.org/body/1")
self.converter.ensure_organization_type()
person = Person()
data = self.api_data["https://oparl.example.org/person/29"]
self.converter.person(data, person)
self.assertEqual(person.name, "Prof. Dr. Max Mustermann")
self.assertEqual(person.given_name, "Max")
self.assertEqual(person.family_name, "Mustermann")
self.assertEqual(person.location, None)
def import_inmate_file(csvid):
import django
django.setup()
from mainapp.models import Person, RescueCamp, CsvBulkUpload
upload = CsvBulkUpload.objects.get(id = csvid)
upload.csv_file.open(mode="rb")
new_data = csv.DictReader(codecs.iterdecode(upload.csv_file.file, 'utf-8'))
camp_obj = False
for datum in new_data:
try:
camp_id = int(datum.get("camped_at", ""))
camp_obj = RescueCamp.objects.get(id = camp_id)
identifier_str = (datum.get("phone", "") + datum.get("name","") + datum.get("age",0)).encode('utf-8')
identifier = md5(identifier_str).hexdigest()
#this will fail. we should deal with the removed unique_identifier
p = Person.objects.get(unique_identifier=identifier)
except ValueError as e:
print("Invalid camp ID. row = "+ str(datum))
except RescueCamp.DoesNotExist as e:
print("Camp does not exist. row = "+ str(datum))
except Person.DoesNotExist:
gender = 2
if( len(datum.get("gender", "")) > 0 ):
if(datum.get("gender", "")[0] == "m" or datum.get("gender", "")[0] == "M"):
gender = 0
elif(datum.get("gender", "")[0] == "f" or datum.get("gender", "")[0] == "F"):
gender = 1
Person(
unique_identifier = identifier,
name = datum.get("name", ""),
phone = datum.get("phone", ""),
age = datum.get("age", ""),
gender = gender,
address = datum.get("address", ""),
notes = datum.get("notes", ""),
camped_at = camp_obj,
district = datum.get("district", ""),
status = "new",
checkin_date = parsedate(datum.get("checkin_date", None)),
checkout_date = parsedate(datum.get("checkout_date", None))
).save()
def meeting(self, libobject: OParl.Meeting):
meeting = self.check_existing(libobject, Meeting)
if not meeting:
return
logging.info("Processing Meeting {}".format(libobject.get_id()))
meeting.start = self.glib_datetime_to_python(libobject.get_start())
meeting.end = self.glib_datetime_to_python(libobject.get_end())
meeting.location = self.location(libobject.get_location())
meeting.invitation = self.file(libobject.get_invitation())
meeting.verbatim_protocol = self.file(
libobject.get_verbatim_protocol())
meeting.results_protocol = self.file(libobject.get_results_protocol())
meeting.cancelled = libobject.get_cancelled() or False
meeting.save()
auxiliary_files = []
for oparlfile in libobject.get_auxiliary_file():
djangofile = self.file(oparlfile)
if djangofile:
auxiliary_files.append(djangofile)
meeting.auxiliary_files = auxiliary_files
persons = []
for oparlperson in libobject.get_participant():
djangoperson = Person.by_oparl_id(oparlperson.get_id())
if djangoperson:
persons.append(djangoperson)
else:
self.meeting_person_queue[libobject.get_id()].append(
oparlperson.get_id())
meeting.persons = persons
for index, oparlitem in enumerate(libobject.get_agenda_item()):
self.agendaitem(oparlitem, index, meeting)
meeting.save()
return meeting
def meeting_embedded(self, libobject, meeting):
changed = False
auxiliary_files = []
for oparlfile in libobject.get_auxiliary_file():
djangofile = self.file(oparlfile)
if djangofile:
auxiliary_files.append(djangofile)
changed = changed or not self.is_queryset_equal_list(
meeting.auxiliary_files, auxiliary_files)
meeting.auxiliary_files = auxiliary_files
persons = []
for oparlperson in libobject.get_participant():
djangoperson = Person.by_oparl_id(oparlperson.get_id())
if djangoperson:
persons.append(djangoperson)
else:
self.meeting_person_queue[libobject.get_id()].append(
oparlperson.get_id())
changed = changed or not self.is_queryset_equal_list(
meeting.persons, persons)
meeting.persons = persons
for index, oparlitem in enumerate(libobject.get_agenda_item()):
self.agendaitem(oparlitem, index, meeting)
organizations = []
for organization_url in libobject.get_organization_url():
djangoorganization = Organization.objects.filter(
oparl_id=organization_url).first()
if djangoorganization:
organizations.append(djangoorganization)
else:
self.meeting_organization_queue[meeting].append(
organization_url)
changed = changed or not self.is_queryset_equal_list(
meeting.organizations, organizations)
meeting.organizations = organizations
return changed
def add_missing_associations(self):
logging.info("Adding missing meeting <-> persons associations")
for meeting_id, person_ids in self.meeting_person_queue.items():
meeting = Meeting.by_oparl_id(meeting_id)
meeting.persons = [
Person.by_oparl_id(person_id) for person_id in person_ids
]
meeting.save()
logging.info("Adding missing agenda item <-> paper associations")
for item_id, paper_id in self.agenda_item_paper_queue.items():
item = AgendaItem.objects_with_deleted.get(oparl_id=item_id)
item.paper = Paper.objects_with_deleted.filter(
oparl_id=paper_id).first()
if not item.paper:
message = "Missing Paper: {}, ({})".format(paper_id, item_id)
self.errorlist.append(message)
item.save()
logging.info("Adding missing memberships")
for organization, libobject in self.membership_queue:
self.membership(organization, libobject)
logging.info("Adding missing papper to consultations")
for consultation, paper in self.consultation_paper_queue:
consultation.paper = Paper.by_oparl_id(paper)
consultation.save()
logging.info("Adding missing meetings to consultations")
for consultation, meeting in self.consultation_meeting_queue:
consultation.meeting = Meeting.by_oparl_id(meeting)
consultation.save()
logging.info("Adding missing organizations to papers")
for paper, organization_url in self.paper_organization_queue:
paper.organizations.add(Organization.by_oparl_id(organization_url))
def add_missing_associations(self):
self.logger.info(
"Adding {} missing meeting <-> persons associations".format(
len(self.meeting_person_queue.items())))
for meeting_id, person_ids in self.meeting_person_queue.items():
meeting = Meeting.by_oparl_id(meeting_id)
meeting.persons = [
Person.by_oparl_id(person_id) for person_id in person_ids
]
meeting.save()
self.logger.info(
"Adding {} missing agenda item <-> paper associations".format(
len(self.agenda_item_paper_queue.items())))
for item_id, paper_id in self.agenda_item_paper_queue.items():
item = AgendaItem.objects_with_deleted.get(oparl_id=item_id)
item.paper = Paper.objects_with_deleted.filter(
oparl_id=paper_id).first()
if not item.paper:
message = "Missing Paper: {}, ({})".format(paper_id, item_id)
self.errorlist.append(message)
item.save()
self.logger.info("Adding {} missing memberships".format(
len(self.membership_queue)))
for organization, libobject in self.membership_queue:
person = Person.objects_with_deleted.filter(
oparl_id=libobject.get_person().get_id()).first()
if not person:
self.logger.warn("The person {} is missing".format(
libobject.get_id()))
self.person(libobject.get_person())
self.membership(organization, libobject)
self.logger.info("Adding {} missing paper to consultations".format(
len(self.consultation_paper_queue)))
for consultation, paper in self.consultation_paper_queue:
consultation.paper = Paper.objects_with_deleted.filter(
oparl_id=paper).first()
consultation.save()
self.logger.info("Adding {} missing meetings to consultations".format(
len(self.consultation_meeting_queue)))
for consultation, meeting in self.consultation_meeting_queue:
consultation.meeting = Meeting.objects_with_deleted.filter(
oparl_id=meeting).first()
consultation.save()
self.logger.info("Adding {} missing organizations to papers".format(
len(self.paper_organization_queue)))
for paper, organization_url in self.paper_organization_queue:
org = Organization.objects_with_deleted.filter(
oparl_id=organization_url).first()
if not org:
org = self.organization_without_embedded(
self.client.parse_url(organization_url))
paper.organizations.add(org)
self._add_organizations(self.consultation_organization_queue,
Consultation)
self._add_organizations(self.meeting_organization_queue, Meeting)
def import_inmate_file(csvid, is_recovery=False):
import django
django.setup()
from mainapp.models import Person, CsvBulkUpload
upload = CsvBulkUpload.objects.get(id=csvid)
try:
upload.csv_file.open(mode="rb")
new_data = csv.DictReader(codecs.iterdecode(upload.csv_file.file, "utf-8"))
for datum in new_data:
"""
try:
identifier_str = (datum.get("phone", "") + datum.get("name","") + datum.get("age",0)).encode('utf-8')
identifier = md5(identifier_str).hexdigest()
#this will fail. we should deal with the removed unique_identifier
p = Person.objects.get(unique_identifier=identifier)
except ValueError as e:
print("Invalid camp ID. row = "+ str(datum))
except RescueCamp.DoesNotExist as e:
print("Camp does not exist. row = "+ str(datum))
except Person.DoesNotExist:
"""
empty = 0
header = [
"name",
"phone",
"address",
"notes",
"district",
"checkin_date",
"checkout_date",
"gender",
"age",
]
for i in header:
if not datum.get(i, ""):
empty += 1
continue
if datum.get(i, "").strip() == "":
empty += 1
if empty == len(header):
continue
gender = 2
if len(datum.get("gender", "")) > 0:
if datum.get("gender", "")[0] == "m" or datum.get("gender", "")[0] == "M":
gender = 0
elif datum.get("gender", "")[0] == "f" or datum.get("gender", "")[0] == "F":
gender = 1
age = "-1"
if datum.get("age", ""):
if datum.get("age", "").strip() != "":
age = datum.get("age", "").strip()
district = ""
if datum.get("district", ""):
district = district.lower()
Person(
name=datum.get("name", "")[:50],
phone=datum.get("phone", ""),
age=int(float(age)),
gender=gender,
address=datum.get("address", ""),
notes=datum.get("notes", ""),
camped_at=upload.camp,
district=datum.get("district", "").lower(),
status="new",
checkin_date=parsedate(datum.get("checkin_date", None)),
checkout_date=parsedate(datum.get("checkout_date", None)),
).save()
if is_recovery:
csv_name = CsvBulkUpload.objects.get(id=csvid).name
CsvBulkUpload.objects.filter(id=csvid).update(
is_completed=True, failure_reason="", name="rec-" + csv_name[:15]
)
else:
CsvBulkUpload.objects.filter(id=csvid).update(is_completed=True, failure_reason="")
except Exception as e:
CsvBulkUpload.objects.filter(id=csvid).update(failure_reason=(getattr(e, "message", repr(e))))