Exemplo n.º 1
0
def check_and_scale_pdfs(dbo):
    """
    Goes through all PDFs in the database to see if they have been
    scaled (have a suffix of _scaled.pdf) and scales down any unscaled
    ones.
    """
    if not SCALE_PDF_DURING_BATCH:
        al.warn("SCALE_PDF_DURING_BATCH is disabled, not scaling pdfs",
                "media.check_and_scale_pdfs", dbo)
    if not configuration.scale_pdfs(dbo):
        al.warn(
            "ScalePDFs config option disabled in this database, not scaling pdfs",
            "media.check_and_scale_pdfs", dbo)
    mp = db.query(dbo, \
        "SELECT MediaName FROM media WHERE LOWER(MediaName) LIKE '%.pdf' AND " \
        "LOWER(MediaName) NOT LIKE '%_scaled.pdf'")
    for m in mp:
        filepath = db.query_string(
            dbo, "SELECT Path FROM dbfs WHERE Name='%s'" % m["MEDIANAME"])
        original_name = str(m["MEDIANAME"])
        new_name = str(m["MEDIANAME"])
        new_name = new_name[0:len(new_name) - 4] + "_scaled.pdf"
        odata = dbfs.get_string(dbo, original_name)
        data = scale_pdf(odata)
        # Update the media entry with the new name
        db.execute(dbo, "UPDATE media SET MediaName = '%s' WHERE MediaName = '%s'" % \
            ( new_name, original_name))
        # Update the dbfs entry with the new name
        dbfs.rename_file(dbo, filepath, original_name, new_name)
        # Update the PDF file data
        dbfs.put_string(dbo, new_name, filepath, data)
    al.debug("found and scaled %d pdfs" % len(mp),
             "media.check_and_scale_pdfs", dbo)
Exemplo n.º 2
0
Arquivo: media.py Projeto: magul/asm3
def check_and_scale_pdfs(dbo, force = False):
    """
    Goes through all PDFs in the database to see if they have been
    scaled (have a suffix of _scaled.pdf) and scales down any unscaled
    ones.
    If force is set, then all PDFs are checked and scaled again even
    if they've been scaled before.
    """
    if not configuration.scale_pdfs(dbo):
        al.warn("ScalePDFs config option disabled in this database, not scaling pdfs", "media.check_and_scale_pdfs", dbo)
        return
    if force:
        mp = db.query(dbo, \
            "SELECT ID, MediaName FROM media WHERE LOWER(MediaName) LIKE '%.pdf' ORDER BY ID DESC")
    else:
        mp = db.query(dbo, \
            "SELECT ID, MediaName FROM media WHERE LOWER(MediaName) LIKE '%.pdf' AND " \
            "LOWER(MediaName) NOT LIKE '%_scaled.pdf' ORDER BY ID DESC")
    for i, m in enumerate(mp):
        filepath = db.query_string(dbo, "SELECT Path FROM dbfs WHERE Name='%s'" % m["MEDIANAME"])
        original_name = str(m["MEDIANAME"])
        new_name = str(m["ID"]) + "_scaled.pdf"
        odata = dbfs.get_string(dbo, original_name)
        data = scale_pdf(odata)
        al.debug("scaling %s (%d of %d): old size %d, new size %d" % (new_name, i, len(mp), len(odata), len(data)), "check_and_scale_pdfs", dbo)
        # Update the media entry with the new name
        db.execute(dbo, "UPDATE media SET MediaName = '%s' WHERE ID = %d" % ( new_name, m["ID"]))
        # Update the dbfs entry from old name to new name (will be overwritten in a minute but safer than delete)
        dbfs.rename_file(dbo, filepath, original_name, new_name)
        # Store the PDF file data with the new name - if there was a need to change it
        if len(data) < len(odata):
            dbfs.put_string(dbo, new_name, filepath, data)
    al.debug("found and scaled %d pdfs" % len(mp), "media.check_and_scale_pdfs", dbo)
Exemplo n.º 3
0
Arquivo: cron.py Projeto: rutaq/asm3
def ttask(fn, dbo):
    """ Runs a function and times how long it takes """
    x = time.time()
    fn(dbo)
    elapsed = time.time() - x
    if elapsed > 10:
        al.warn("complete in %0.2f sec" % elapsed, fn.__name__, dbo)
    else:
        al.debug("complete in %0.2f sec" % elapsed, fn.__name__, dbo)
Exemplo n.º 4
0
def check_and_scale_pdfs(dbo):
    """
    Goes through all PDFs in the database to see if they have been
    scaled (have a suffix of _scaled.pdf) and scales down any unscaled
    ones.
    """
    if not gs_installed():
        al.warn("ghostscript is not installed, can't scale pdfs",
                "media.check_and_scale_pdfs", dbo)
        return
    mp = db.query(dbo, \
        "SELECT MediaName FROM media WHERE LOWER(MediaName) LIKE '%.pdf' AND " \
        "LOWER(MediaName) NOT LIKE '%_scaled.pdf'")
    for m in mp:
        filepath = db.query_string(
            dbo, "SELECT Path FROM dbfs WHERE Name='%s'" % m["MEDIANAME"])
        original_name = str(m["MEDIANAME"])
        new_name = str(m["MEDIANAME"])
        new_name = new_name[0:len(new_name) - 4] + "_scaled.pdf"
        inputfile = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False)
        outputfile = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False)
        odata = dbfs.get_string(dbo, original_name)
        inputfile.write(odata)
        inputfile.flush()
        inputfile.close()
        outputfile.close()
        scale_pdf_file(inputfile.name, outputfile.name)
        f = open(outputfile.name, "r")
        data = f.read()
        f.close()
        os.unlink(inputfile.name)
        os.unlink(outputfile.name)
        # Update the media entry with the new name
        db.execute(dbo, "UPDATE media SET MediaName = '%s' WHERE MediaName = '%s'" % \
            ( new_name, original_name))
        # Update the dbfs entry with the new name
        dbfs.rename_file(dbo, filepath, original_name, new_name)
        # Update the PDF file data
        dbfs.put_string(dbo, new_name, filepath, data)
    al.debug("found and scaled %d pdfs" % len(mp),
             "media.check_and_scale_pdfs", dbo)
Exemplo n.º 5
0
def check_and_scale_pdfs(dbo):
    """
    Goes through all PDFs in the database to see if they have been
    scaled (have a suffix of _scaled.pdf) and scales down any unscaled
    ones.
    """
    if not gs_installed(): 
        al.warn("ghostscript is not installed, can't scale pdfs", "media.check_and_scale_pdfs", dbo)
        return
    mp = db.query(dbo, \
        "SELECT MediaName FROM media WHERE LOWER(MediaName) LIKE '%.pdf' AND " \
        "LOWER(MediaName) NOT LIKE '%_scaled.pdf'")
    for m in mp:
        filepath = db.query_string(dbo, "SELECT Path FROM dbfs WHERE Name='%s'" % m["MEDIANAME"])
        original_name = str(m["MEDIANAME"])
        new_name = str(m["MEDIANAME"])
        new_name = new_name[0:len(new_name)-4] + "_scaled.pdf"
        inputfile = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False)
        outputfile = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False)
        odata = dbfs.get_string(dbo, original_name)
        inputfile.write(odata)
        inputfile.flush()
        inputfile.close()
        outputfile.close()
        scale_pdf_file(inputfile.name, outputfile.name)
        f = open(outputfile.name, "r")
        data = f.read()
        f.close()
        os.unlink(inputfile.name)
        os.unlink(outputfile.name)
        # Update the media entry with the new name
        db.execute(dbo, "UPDATE media SET MediaName = '%s' WHERE MediaName = '%s'" % \
            ( new_name, original_name))
        # Update the dbfs entry with the new name
        dbfs.rename_file(dbo, filepath, original_name, new_name)
        # Update the PDF file data
        dbfs.put_string(dbo, new_name, filepath, data)
    al.debug("found and scaled %d pdfs" % len(mp), "media.check_and_scale_pdfs", dbo)
Exemplo n.º 6
0
def csvimport(dbo, csvdata, encoding = "utf8", user = "", createmissinglookups = False, cleartables = False, checkduplicates = False):
    """
    Imports the csvdata.
    createmissinglookups: If a lookup value is given that's not in our data, add it
    cleartables: Clear down the animal, owner and adoption tables before import
    """

    # Convert line endings to standard unix lf to prevent
    # the Python CSV importer barfing.
    csvdata = csvdata.replace("\r\n", "\n")
    csvdata = csvdata.replace("\r", "\n")

    if user == "":
        user = "******"
    else:
        user = "******" % user

    reader = None
    if encoding == "utf8":
        reader = utils.UnicodeCSVReader(StringIO(csvdata))
    else:
        reader = utils.UnicodeCSVReader(StringIO(csvdata), encoding=encoding)

    # Make sure we have a valid header
    cols = None
    for row in reader:
        cols = row
        break
    if cols is None:
        asynctask.set_last_error(dbo, "Your CSV file is empty")
        return

    onevalid = False
    hasanimal = False
    hasanimalname = False
    hasmed = False
    hasvacc = False
    hasperson = False
    haspersonlastname = False
    haspersonname = False
    haslicence = False
    haslicencenumber = False
    hasmovement = False
    hasmovementdate = False
    hasdonation = False
    hasdonationamount = False
    hasoriginalowner = False
    hasoriginalownerlastname = False
    for col in cols:
        if col in VALID_FIELDS: onevalid = True
        if col.startswith("ANIMAL"): hasanimal = True
        if col == "ANIMALNAME": hasanimalname = True
        if col.startswith("ORIGINALOWNER"): hasoriginalowner = True
        if col.startswith("VACCINATION"): hasvacc = True
        if col.startswith("MEDICAL"): hasmed = True
        if col.startswith("LICENSE"): haslicence = True
        if col == "LICENSENUMBER": haslicencenumber = True
        if col == "ORIGINALOWNERLASTNAME": hasoriginalownerlastname = True
        if col.startswith("PERSON"): hasperson = True
        if col == "PERSONLASTNAME": haspersonlastname = True
        if col == "PERSONNAME": haspersonname = True
        if col.startswith("MOVEMENT"): hasmovement = True
        if col == "MOVEMENTDATE": hasmovementdate = True
        if col.startswith("DONATION"): hasdonation = True
        if col == "DONATIONAMOUNT": hasdonationamount = True

    # Any valid fields?
    if not onevalid:
        asynctask.set_last_error(dbo, "Your CSV file did not contain any fields that ASM recognises")
        return

    # If we have any animal fields, make sure at least ANIMALNAME is supplied
    if hasanimal and not hasanimalname:
        asynctask.set_last_error(dbo, "Your CSV file has animal fields, but no ANIMALNAME column")
        return

    # If we have any person fields, make sure at least PERSONLASTNAME or PERSONNAME is supplied
    if hasperson and not haspersonlastname and not haspersonname:
        asynctask.set_last_error(dbo, "Your CSV file has person fields, but no PERSONNAME or PERSONLASTNAME column")
        return

    # If we have any original owner fields, make sure at least ORIGINALOWNERLASTNAME is supplied
    if hasoriginalowner and not hasoriginalownerlastname:
        asynctask.set_last_error(dbo, "Your CSV file has original owner fields, but no ORIGINALOWNERLASTNAME column")
        return

    # If we have any movement fields, make sure MOVEMENTDATE is supplied
    if hasmovement and not hasmovementdate:
        asynctask.set_last_error(dbo, "Your CSV file has movement fields, but no MOVEMENTDATE column")
        return

    # If we have any donation fields, we need an amount
    if hasdonation and not hasdonationamount:
        asynctask.set_last_error(dbo, "Your CSV file has donation fields, but no DONATIONAMOUNT column")
        return

    # We also need a valid person
    if hasdonation and not (haspersonlastname or haspersonname):
        asynctask.set_last_error(dbo, "Your CSV file has donation fields, but no person to apply the donation to")
        return

    # If we have any med fields, we need an animal
    if hasmed and not hasanimal:
        asynctask.set_last_error(dbo, "Your CSV file has medical fields, but no animal to apply them to")
        return

    # If we have any vacc fields, we need an animal
    if hasvacc and not hasanimal:
        asynctask.set_last_error(dbo, "Your CSV file has vaccination fields, but no animal to apply them to")
        return

    # If we have licence fields, we need a number
    if haslicence and not haslicencenumber:
        asynctask.set_last_error(dbo, "Your CSV file has license fields, but no LICENSENUMBER column")
        return

    # We also need a valid person
    if haslicence and not (haspersonlastname or haspersonname):
        asynctask.set_last_error(dbo, "Your CSV file has license fields, but no person to apply the license to")

    # Read the whole CSV file into a list of maps. Note, the
    # reader has a cursor at the second row already because
    # we read the header in the first row above
    data = []
    for row in reader:
        currow = {}
        for i, col in enumerate(row):
            if i >= len(cols): continue # skip if we run out of cols
            currow[cols[i]] = col
        data.append(currow)

    al.debug("reading CSV data, found %d rows" % len(data), "csvimport.csvimport", dbo)

    # If we're clearing down tables first, do it now
    if cleartables:
        al.warn("Resetting the database by removing all non-lookup data", "csvimport.csvimport", dbo)
        dbupdate.reset_db(dbo)

    # Now that we've read them in, go through all the rows
    # and start importing.
    errors = []
    rowno = 1
    asynctask.set_progress_max(dbo, len(data))
    for row in data:

        al.debug("import csv: row %d of %d" % (rowno, len(data)), "csvimport.csvimport", dbo)
        asynctask.increment_progress_value(dbo)

        # Should we stop?
        if asynctask.get_cancel(dbo): break

        # Do we have animal data to read?
        animalid = 0
        if hasanimal and gks(row, "ANIMALNAME") != "":
            a = {}
            a["animalname"] = gks(row, "ANIMALNAME")
            a["sheltercode"] = gks(row, "ANIMALCODE")
            a["shortcode"] = gks(row, "ANIMALCODE")
            if gks(row, "ANIMALSEX") == "": 
                a["sex"] = "2" # Default unknown if not set
            else:
                a["sex"] = gks(row, "ANIMALSEX").lower().startswith("m") and "1" or "0"
            a["basecolour"] = gkl(dbo, row, "ANIMALCOLOR", "basecolour", "BaseColour", createmissinglookups)
            if a["basecolour"] == "0":
                a["basecolour"] = str(configuration.default_colour(dbo))
            a["species"] = gkl(dbo, row, "ANIMALSPECIES", "species", "SpeciesName", createmissinglookups)
            if a["species"] == "0":
                a["species"] = str(configuration.default_species(dbo))
            a["animaltype"] = gkl(dbo, row, "ANIMALTYPE", "animaltype", "AnimalType", createmissinglookups)
            if a["animaltype"] == "0":
                a["animaltype"] = str(configuration.default_type(dbo))
            a["breed1"] = gkbr(dbo, row, "ANIMALBREED1", a["species"], createmissinglookups)
            if a["breed1"] == "0":
                a["breed1"] = str(configuration.default_breed(dbo))
            a["breed2"] = gkbr(dbo, row, "ANIMALBREED2", a["species"], createmissinglookups)
            if a["breed2"] != "0" and a["breed2"] != a["breed1"]:
                a["crossbreed"] = "on"
            a["size"] = gkl(dbo, row, "ANIMALSIZE", "lksize", "Size", False)
            if gks(row, "ANIMALSIZE") == "": 
                a["size"] = str(configuration.default_size(dbo))
            a["internallocation"] = gkl(dbo, row, "ANIMALLOCATION", "internallocation", "LocationName", createmissinglookups)
            if a["internallocation"] == "0":
                a["internallocation"] = str(configuration.default_location(dbo))
            a["unit"] = gks(row, "ANIMALUNIT")
            a["comments"] = gks(row, "ANIMALCOMMENTS")
            a["markings"] = gks(row, "ANIMALMARKINGS")
            a["hiddenanimaldetails"] = gks(row, "ANIMALHIDDENDETAILS")
            a["healthproblems"] = gks(row, "ANIMALHEALTHPROBLEMS")
            a["notforadoption"] = gkbi(row, "ANIMALNOTFORADOPTION")
            a["nonshelter"] = gkbi(row, "ANIMALNONSHELTER")
            a["housetrained"] = gkynu(row, "ANIMALHOUSETRAINED")
            a["goodwithcats"] = gkynu(row, "ANIMALGOODWITHCATS")
            a["goodwithdogs"] = gkynu(row, "ANIMALGOODWITHDOGS")
            a["goodwithkids"] = gkynu(row, "ANIMALGOODWITHKIDS")
            a["reasonforentry"] = gks(row, "ANIMALREASONFORENTRY")
            a["estimatedage"] = gks(row, "ANIMALAGE")
            a["dateofbirth"] = gkd(dbo, row, "ANIMALDOB", True)
            if gks(row, "ANIMALDOB") == "" and a["estimatedage"] != "":
                a["dateofbirth"] = "" # if we had an age and dob was blank, prefer the age
            a["datebroughtin"] = gkd(dbo, row, "ANIMALENTRYDATE", True)
            a["deceaseddate"] = gkd(dbo, row, "ANIMALDECEASEDDATE")
            a["neutered"] = gkbc(row, "ANIMALNEUTERED")
            a["neutereddate"] = gkd(dbo, row, "ANIMALNEUTEREDDATE")
            if a["neutereddate"] != "": a["neutered"] = "on"
            a["microchipnumber"] = gks(row, "ANIMALMICROCHIP")
            if a["microchipnumber"] != "": a["microchipped"] = "on"
            a["microchipdate"] = gkd(dbo, row, "ANIMALMICROCHIPDATE")
            # image data if any was supplied
            imagedata = gks(row, "ANIMALIMAGE")
            if imagedata.startswith("http"):
                # It's a URL, get the image from the remote server
                r = utils.get_image_url(imagedata, timeout=5000)
                if r["status"] == 200:
                    al.debug("retrieved image from %s (%s bytes)" % (imagedata, len(r["response"])), "csvimport.csvimport", dbo)
                    imagedata = "data:image/jpeg;base64,%s" % base64.b64encode(r["response"])
                else:
                    row_error(errors, "animal", rowno, row, "error reading image from '%s': %s" % (imagedata, r), dbo, sys.exc_info())
                    continue
            elif imagedata.startswith("data:image"):
                # It's a base64 encoded data URI - do nothing as attach_file requires it
                pass
            else:
                # We don't know what it is, don't try and do anything with it
                imagedata = ""
            # If an original owner is specified, create a person record
            # for them and attach it to the animal as original owner
            if gks(row, "ORIGINALOWNERLASTNAME") != "":
                p = {}
                p["title"] = gks(row, "ORIGINALOWNERTITLE")
                p["initials"] = gks(row, "ORIGINALOWNERINITIALS")
                p["forenames"] = gks(row, "ORIGINALOWNERFIRSTNAME")
                p["surname"] = gks(row, "ORIGINALOWNERLASTNAME")
                p["address"] = gks(row, "ORIGINALOWNERADDRESS")
                p["town"] = gks(row, "ORIGINALOWNERCITY")
                p["county"] = gks(row, "ORIGINALOWNERSTATE")
                p["postcode"] = gks(row, "ORIGINALOWNERZIPCODE")
                p["jurisdiction"] = gkl(dbo, row, "ORIGINALOWNERJURISDICTION", "jurisdiction", "JurisdictionName", createmissinglookups)
                p["hometelephone"] = gks(row, "ORIGINALOWNERHOMEPHONE")
                p["worktelephone"] = gks(row, "ORIGINALOWNERWORKPHONE")
                p["mobiletelephone"] = gks(row, "ORIGINALOWNERCELLPHONE")
                p["emailaddress"] = gks(row, "ORIGINALOWNEREMAIL")
                try:
                    if checkduplicates:
                        dups = person.get_person_similar(dbo, p["emailaddress"], p["surname"], p["forenames"], p["address"])
                        if len(dups) > 0:
                            a["originalowner"] = str(dups[0]["ID"])
                    if "originalowner" not in a:
                        ooid = person.insert_person_from_form(dbo, utils.PostedData(p, dbo.locale), user, geocode=False)
                        a["originalowner"] = str(ooid)
                        # Identify an ORIGINALOWNERADDITIONAL additional fields and create them
                        create_additional_fields(dbo, row, errors, rowno, "ORIGINALOWNERADDITIONAL", "person", ooid)
                except Exception as e:
                    row_error(errors, "originalowner", rowno, row, e, dbo, sys.exc_info())
            try:
                if checkduplicates:
                    dup = animal.get_animal_sheltercode(dbo, a["sheltercode"])
                    if dup is not None:
                        animalid = dup["ID"]
                if animalid == 0:
                    animalid, newcode = animal.insert_animal_from_form(dbo, utils.PostedData(a, dbo.locale), user)
                    # Identify any ANIMALADDITIONAL additional fields and create them
                    create_additional_fields(dbo, row, errors, rowno, "ANIMALADDITIONAL", "animal", animalid)
                # If we have some image data, add it to the animal
                if len(imagedata) > 0:
                    imagepost = utils.PostedData({ "filename": "image.jpg", "filetype": "image/jpeg", "filedata": imagedata }, dbo.locale)
                    media.attach_file_from_form(dbo, user, media.ANIMAL, animalid, imagepost)
            except Exception as e:
                row_error(errors, "animal", rowno, row, e, dbo, sys.exc_info())

        # Person data?
        personid = 0
        if hasperson and (gks(row, "PERSONLASTNAME") != "" or gks(row, "PERSONNAME") != ""):
            p = {}
            p["ownertype"] = gks(row, "PERSONCLASS")
            if p["ownertype"] != "1" and p["ownertype"] != "2": 
                p["ownertype"] = "1"
            p["title"] = gks(row, "PERSONTITLE")
            p["initials"] = gks(row, "PERSONINITIALS")
            p["forenames"] = gks(row, "PERSONFIRSTNAME")
            p["surname"] = gks(row, "PERSONLASTNAME")
            # If we have a person name, all upto the last space is first names,
            # everything after the last name
            if gks(row, "PERSONNAME") != "":
                pname = gks(row, "PERSONNAME")
                if pname.find(" ") != -1:
                    p["forenames"] = pname[0:pname.rfind(" ")]
                    p["surname"] = pname[pname.rfind(" ")+1:]
                else:
                    p["surname"] = pname
            p["address"] = gks(row, "PERSONADDRESS")
            p["town"] = gks(row, "PERSONCITY")
            p["county"] = gks(row, "PERSONSTATE")
            p["postcode"] = gks(row, "PERSONZIPCODE")
            p["jurisdiction"] = gkl(dbo, row, "PERSONJURISDICTION", "jurisdiction", "JurisdictionName", createmissinglookups)
            p["hometelephone"] = gks(row, "PERSONHOMEPHONE")
            p["worktelephone"] = gks(row, "PERSONWORKPHONE")
            p["mobiletelephone"] = gks(row, "PERSONCELLPHONE")
            p["emailaddress"] = gks(row, "PERSONEMAIL")
            p["gdprcontactoptin"] = gks(row, "PERSONGDPRCONTACTOPTIN")
            flags = gks(row, "PERSONFLAGS")
            if gkb(row, "PERSONFOSTERER"): flags += ",fosterer"
            if gkb(row, "PERSONMEMBER"): flags += ",member"
            if gkb(row, "PERSONDONOR"): flags += ",donor"
            p["flags"] = flags
            p["comments"] = gks(row, "PERSONCOMMENTS")
            p["membershipnumber"] = gks(row, "PERSONMEMBERSHIPNUMBER")
            p["membershipexpires"] = gkd(dbo, row, "PERSONMEMBERSHIPEXPIRY")
            p["matchactive"] = gkbi(row, "PERSONMATCHACTIVE")
            if p["matchactive"] == "1":
                if "PERSONMATCHADDED" in cols: p["matchadded"] = gkd(dbo, row, "PERSONMATCHADDED")
                if "PERSONMATCHEXPIRES" in cols: p["matchexpires"] = gkd(dbo, row, "PERSONMATCHEXPIRES")
                if "PERSONMATCHSEX" in cols: p["matchsex"] = gks(row, "PERSONMATCHSEX").lower().startswith("m") and "1" or "0"
                if "PERSONMATCHSIZE" in cols: p["matchsize"] = gkl(dbo, row, "PERSONMATCHSIZE", "lksize", "Size", False)
                if "PERSONMATCHCOLOR" in cols: p["matchcolour"] = gkl(dbo, row, "PERSONMATCHCOLOR", "basecolour", "BaseColour", createmissinglookups)
                if "PERSONMATCHAGEFROM" in cols: p["matchagefrom"] = gks(row, "PERSONMATCHAGEFROM")
                if "PERSONMATCHAGETO" in cols: p["matchageto"] = gks(row, "PERSONMATCHAGETO")
                if "PERSONMATCHTYPE" in cols: p["matchanimaltype"] = gkl(dbo, row, "PERSONMATCHTYPE", "animaltype", "AnimalType", createmissinglookups)
                if "PERSONMATCHSPECIES" in cols: p["matchspecies"] = gkl(dbo, row, "PERSONMATCHSPECIES", "species", "SpeciesName", createmissinglookups)
                if "PERSONMATCHBREED1" in cols: p["matchbreed"] = gkbr(dbo, row, "PERSONMATCHBREED1", p["matchspecies"], createmissinglookups)
                if "PERSONMATCHBREED2" in cols: p["matchbreed2"] = gkbr(dbo, row, "PERSONMATCHBREED2", p["matchspecies"], createmissinglookups)
                if "PERSONMATCHGOODWITHCATS" in cols: p["matchgoodwithcats"] = gkynu(row, "PERSONMATCHGOODWITHCATS")
                if "PERSONMATCHGOODWITHDOGS" in cols: p["matchgoodwithdogs"] = gkynu(row, "PERSONMATCHGOODWITHDOGS")
                if "PERSONMATCHGOODWITHCHILDREN" in cols: p["matchgoodwithchildren"] = gkynu(row, "PERSONMATCHGOODWITHCHILDREN")
                if "PERSONMATCHHOUSETRAINED" in cols: p["matchhousetrained"] = gkynu(row, "PERSONMATCHHOUSETRAINED")
                if "PERSONMATCHCOMMENTSCONTAIN" in cols: p["matchcommentscontain"] = gks(row, "PERSONMATCHCOMMENTSCONTAIN")
            try:
                if checkduplicates:
                    dups = person.get_person_similar(dbo, p["emailaddress"], p["surname"], p["forenames"], p["address"])
                    if len(dups) > 0:
                        personid = dups[0].ID
                        # Merge flags and any extra details
                        person.merge_flags(dbo, user, personid, flags)
                        person.merge_gdpr_flags(dbo, user, personid, p["gdprcontactoptin"])
                        # If we deduplicated on the email address, and address details are
                        # present, assume that they are newer than the ones we had and update them
                        # (we do this by setting force=True parameter to merge_person_details,
                        # otherwise we do a regular merge which only fills in any blanks)
                        person.merge_person_details(dbo, user, personid, p, force=dups[0].EMAILADDRESS == p["emailaddress"])
                if personid == 0:
                    personid = person.insert_person_from_form(dbo, utils.PostedData(p, dbo.locale), user, geocode=False)
                    # Identify any PERSONADDITIONAL additional fields and create them
                    create_additional_fields(dbo, row, errors, rowno, "PERSONADDITIONAL", "person", personid)
            except Exception as e:
                row_error(errors, "person", rowno, row, e, dbo, sys.exc_info())

        # Movement to tie animal/person together?
        movementid = 0
        if hasmovement and personid != 0 and animalid != 0 and gks(row, "MOVEMENTDATE") != "":
            m = {}
            m["person"] = str(personid)
            m["animal"] = str(animalid)
            movetype = gks(row, "MOVEMENTTYPE")
            if movetype == "": movetype = "1" # Default to adoption if not supplied
            m["type"] = str(movetype)
            m["movementdate"] = gkd(dbo, row, "MOVEMENTDATE", True)
            m["returndate"] = gkd(dbo, row, "MOVEMENTRETURNDATE")
            m["comments"] = gks(row, "MOVEMENTCOMMENTS")
            m["returncategory"] = str(configuration.default_entry_reason(dbo))
            try:
                movementid = movement.insert_movement_from_form(dbo, user, utils.PostedData(m, dbo.locale))
            except Exception as e:
                row_error(errors, "movement", rowno, row, e, dbo, sys.exc_info())

        # Donation?
        if hasdonation and personid != 0 and gkc(row, "DONATIONAMOUNT") != 0:
            d = {}
            d["person"] = str(personid)
            d["animal"] = str(animalid)
            d["movement"] = str(movementid)
            d["amount"] = str(gkc(row, "DONATIONAMOUNT"))
            d["fee"] = str(gkc(row, "DONATIONFEE"))
            d["comments"] = gks(row, "DONATIONCOMMENTS")
            d["received"] = gkd(dbo, row, "DONATIONDATE", True)
            d["chequenumber"] = gks(row, "DONATIONCHECKNUMBER")
            d["type"] = gkl(dbo, row, "DONATIONTYPE", "donationtype", "DonationName", createmissinglookups)
            if d["type"] == "0":
                d["type"] = str(configuration.default_donation_type(dbo))
            d["payment"] = gkl(dbo, row, "DONATIONPAYMENT", "donationpayment", "PaymentName", createmissinglookups)
            if d["payment"] == "0":
                d["payment"] = "1"
            try:
                financial.insert_donation_from_form(dbo, user, utils.PostedData(d, dbo.locale))
            except Exception as e:
                row_error(errors, "payment", rowno, row, e, dbo, sys.exc_info())
            if movementid != 0: movement.update_movement_donation(dbo, movementid)

        # Vaccination?
        if hasvacc and animalid != 0 and gks(row, "VACCINATIONDUEDATE") != "":
            v = {}
            v["animal"] = str(animalid)
            v["type"] = gkl(dbo, row, "VACCINATIONTYPE", "vaccinationtype", "VaccinationType", createmissinglookups)
            if v["type"] == "0":
                v["type"] = str(configuration.default_vaccination_type(dbo))
            v["required"] = gkd(dbo, row, "VACCINATIONDUEDATE", True)
            v["given"] = gkd(dbo, row, "VACCINATIONGIVENDATE")
            v["expires"] = gkd(dbo, row, "VACCINATIONEXPIRESDATE")
            v["batchnumber"] = gks(row, "VACCINATIONBATCHNUMBER")
            v["manufacturer"] = gks(row, "VACCINATIONMANUFACTURER")
            v["comments"] = gks(row, "VACCINATIONCOMMENTS")
            try:
                medical.insert_vaccination_from_form(dbo, user, utils.PostedData(v, dbo.locale))
            except Exception as e:
                row_error(errors, "vaccination", rowno, row, e, dbo, sys.exc_info())

        # Medical?
        if hasmed and animalid != 0 and gks(row, "MEDICALGIVENDATE") != "" and gks(row, "MEDICALNAME") != "":
            m = {}
            m["animal"] = str(animalid)
            m["treatmentname"] = gks(row, "MEDICALNAME")
            m["dosage"] = gks(row, "MEDICALDOSAGE")
            m["startdate"] = gkd(dbo, row, "MEDICALGIVENDATE")
            m["comments"] = gks(row, "MEDICALCOMMENTS")
            m["singlemulti"] = "0" # single treatment
            m["status"] = "2" # completed
            try:
                medical.insert_regimen_from_form(dbo, user, utils.PostedData(m, dbo.locale))
            except Exception as e:
                row_error(errors, "medical", rowno, row, e, dbo, sys.exc_info())

        # License?
        if haslicence and personid != 0 and gks(row, "LICENSENUMBER") != "":
            l = {}
            l["person"] = str(personid)
            l["animal"] = str(animalid)
            l["type"] = gkl(dbo, row, "LICENSETYPE", "licencetype", "LicenceTypeName", createmissinglookups)
            if l["type"] == "0": l["type"] = 1
            l["number"] = gks(row, "LICENSENUMBER")
            l["fee"] = str(gkc(row, "LICENSEFEE"))
            l["issuedate"] = gkd(dbo, row, "LICENSEISSUEDATE")
            l["expirydate"] = gkd(dbo, row, "LICENSEEXPIRESDATE")
            l["comments"] = gks(row, "LICENSECOMMENTS")
            try:
                financial.insert_licence_from_form(dbo, user, utils.PostedData(l, dbo.locale))
            except Exception as e:
                row_error(errors, "license", rowno, row, e, dbo, sys.exc_info())

        rowno += 1

    h = [ "<p>%d success, %d errors</p><table>" % (len(data) - len(errors), len(errors)) ]
    for rowno, row, err in errors:
        h.append("<tr><td>%s</td><td>%s</td><td>%s</td></tr>" % (rowno, row, err))
    h.append("</table>")
    return "".join(h)