def migrate_meta():
    from app.models import User, Role, Permission, \
    IUCNStatus, ESAStatus, OrganismType, GrowthFormRaunkiaer, ReproductiveRepetition, \
    DicotMonoc, AngioGymno, SpandExGrowthType, SourceType, Database, Purpose, MissingData, ContentEmail, Ecoregion, Continent, InvasiveStatusStudy, InvasiveStatusElsewhere, StageTypeClass, \
    TransitionType, MatrixComposition, StartSeason, StudiedSex, Captivity, Species, Taxonomy, Trait, \
    Publication, Study, AuthorContact, AdditionalSource, Population, Stage, StageType, Treatment, \
    MatrixStage, MatrixValue, Matrix, Interval, Fixed, Small, CensusTiming, PurposeEndangered, PurposeWeed, Institute

    print "Migrating Meta Tables..."
    try:
        Role.insert_roles()
        Species.migrate()
        Taxonomy.migrate()
        Trait.migrate()
        Publication.migrate()
        AuthorContact.migrate()
        Population.migrate()
        StageType.migrate()
        MatrixValue.migrate()
        Matrix.migrate()
        Fixed.migrate()
        Study.migrate()
        User.migrate()
        Version.migrate()
        Institute.migrate()
    except:
        "Error migrating metadata"
    finally:
        "Done + good"
   
    return
示例#2
0
def migrate_meta():
    from app.models import User, Role, Permission, \
    IUCNStatus, OrganismType, GrowthFormRaunkiaer, ReproductiveRepetition, \
    DicotMonoc, AngioGymno, SpandExGrowthType, SourceType, Database, Purpose, MissingData, ContentEmail, Ecoregion, Continent, InvasiveStatusStudy, InvasiveStatusElsewhere, StageTypeClass, \
    TransitionType, MatrixComposition, StartSeason, StudiedSex, Captivity, Species, Taxonomy, Trait, \
    Publication, AuthorContact, AdditionalSource, Population, Stage, StageType, Treatment, \
    MatrixStage, MatrixValue, Matrix, Interval, Fixed, Small, CensusTiming, PurposeEndangered, PurposeWeed, Institute, Version, \
    PublicationsProtocol, DigitizationProtocol, Protocol, CommonTerm

    print "Migrating Meta Tables..."
    Role.insert_roles()
    Species.migrate()
    Taxonomy.migrate()
    Trait.migrate()
    Publication.migrate()
    AuthorContact.migrate()
    Population.migrate()
    StageType.migrate()
    MatrixValue.migrate()
    Matrix.migrate()
    Fixed.migrate()
    Version.migrate()
    Institute.migrate()
    User.migrate()
    Database.migrate()
    Status.migrate()
    PublicationsProtocol.migrate()
    DigitizationProtocol.migrate()
    CommonTerm.migrate()

    return
示例#3
0
def version_current():
    models = [
        Species(),
        Taxonomy(),
        Trait(),
        Publication(),
        AuthorContact(),
        Population(),
        StageType(),
        MatrixValue(),
        Matrix(),
        Fixed(),
        Institute(),
        Protocol()
    ]

    for model in models:
        model_version(model)
示例#4
0
def submit_new(data):
    import datetime

    version = {
        'checked': True,
        'checked_count': 1,
        'statuses': Status.query.filter_by(status_name="Green").first()
    }

    if data["population_database_id"] == "XXX" or data[
            "population_database_id"] == "X.X.X":
        version = {
            'checked': False,
            'checked_count': 0,
            'statuses': Status.query.filter_by(status_name="Pending").first()
        }

    # When checking for null data later, these need to be excluded, as they will always have a value
    ignore_keys = ['version_ok', 'version_latest', 'version_original']
    ''' DigitizationProtocol '''

    #    digitization_protocol = DigitizationProtocol.query.filter_by(field_name=data["digitization_protocol"]).first()

    #   if digitization_protocol == None:
    #           ac_dict = {'protocol_id' : protocol.id,
    #           'field_name' : data['field_name'],
    #           'name_in_csv' : data["name_in_csv"],
    #           'database_model' : data["database_model"],
    #           'field_description' : data["field_description"],
    #           'field_short_description' : data["field_short_description"]
    #           }

    #           ac_cleaned = data_clean(ac_dict)
    #           digitization_protocol = Protocol(**ac_cleaned["kwargs"])

    #           db.session.add(digitization_protocol)
    #           db.session.commit()
    ''' Publication '''

    publications_protocol = PublicationsProtocol.query.filter_by(
        protocol_number=data["publications_protocol_id"]).first()

    if data["publication_DOI_ISBN"] == None:
        publication = Publication.query.filter_by(
            authors=data["publication_authors"]
        ).filter_by(year=data["publication_year"]).filter_by(
            journal_name=data["publication_journal_name"]).filter_by(
                additional_source_string=data[
                    "publication_additional_source_string"]).filter_by(
                        study_notes=data["publication_study_notes"]).first()
    else:
        publication = Publication.query.filter_by(
            DOI_ISBN=data["publication_DOI_ISBN"]).first()

    if publication == None:
        purposes = {
            "Comparative Demography":
            data["publication_purpose_comparative_demography"],
            "Spatial Demography":
            data["publication_purpose_spatial_demography"],
            "Abiotic Impacts":
            data["publication_purpose_abiotic"],
            "PVA":
            data["publication_purpose_pva"],
            "Species Dynamics Description":
            data["publication_purpose_species_dynamics_description"],
            "Interspecific Interactions":
            data["publication_purpose_interspecific_interactions"],
            "Management Evaluation":
            data["publication_purpose_management_evaluation"],
            "Methodological Advancement":
            data["publication_purpose_methodological_advancement"]
        }

        queryset = [
            Purpose.query.filter(Purpose.purpose_name == key).first()
            for key, val in purposes.items() if val == '1'
        ]

        if data['publication_missing_data'] != 'NDY' and data[
                'publication_missing_data']:
            missing_data_unicode = data['publication_missing_data'].replace(
                " ", "").split(';')
            missing_data = [
                MissingData.query.filter_by(missing_code=key).first()
                for key in missing_data_unicode
                if MissingData.query.filter_by(missing_code=key).first()
            ]
        else:
            missing_data = 'NDY'

        pub_dict = {
            'authors':
            data["publication_authors"],
            'year':
            data["publication_year"],
            'publications_protocol':
            publications_protocol,
            'DOI_ISBN':
            data["publication_DOI_ISBN"],
            'additional_source_string':
            data["publication_additional_source_string"],
            'journal_name':
            data["publication_journal_name"],
            'date_digitised':
            datetime.datetime.strptime(data['publication_date_digitization'],
                                       "%d/%m/%Y").strftime("%Y-%m-%d")
            if data['publication_date_digitization'] else None,
            'purposes':
            queryset,
            'study_notes':
            data["publication_study_notes"]
        }

        pub_cleaned = data_clean(pub_dict)

        # if not all(value == None for key, value in pub_cleaned["kwargs"].items() if key not in ignore_keys) and study_present:
        publication = Publication(**pub_cleaned["kwargs"])
        db.session.add(publication)
        db.session.commit()

        publication.missing_data = missing_data if type(
            missing_data) == list else []

        db.session.add(publication)
        db.session.commit()
        ''' Publication Version '''
        #version = version_data(pub_cleaned)
        publication_version = Version(**version)
        publication_version.publication = publication
        publication.colour = gen_hex_code()

        possible_user = User.query.filter_by(
            name=data["publication_student"]).first()
        na_user = User.query.filter_by(name="N/A").first()
        if possible_user == None:
            possible_user = na_user

        publication_version.entered_by_id = possible_user.id if possible_user else None,
        publication_version.checked_by_id = na_user.id if na_user else None,

        db.session.add(publication_version)
        db.session.commit()
        publication_version.original_version_id = publication_version.id
        db.session.add(publication_version)
        db.session.commit()
        ''' Author contact '''

        author_contacts = AuthorContact.query.filter_by(
            corresponding_author=data["publication_corresponding_author"]
        ).filter_by(
            corresponding_author_email=data["publication_corresponding_email"]
        ).first()

        if author_contacts == None:
            ac_dict = {
                'publication_id':
                publication.id,
                'date_contacted':
                datetime.datetime.strptime(data['date_author_contacted'],
                                           "%d/%m/%Y").strftime("%Y-%m-%d")
                if data['date_author_contacted'] else None,
                'date_contacted_again':
                datetime.datetime.strptime(data['date_author_contacted_again'],
                                           "%d/%m/%Y").strftime("%Y-%m-%d")
                if data['date_author_contacted_again'] else None,
                'extra_content_email':
                data["correspondence_email_content"],
                'author_reply':
                data["correspondence_author_reply"],
                'corresponding_author':
                data["publication_corresponding_author"],
                'corresponding_author_email':
                data["publication_corresponding_email"],
                'correspondence_email_content':
                data["correspondence_email_content"],
                'extra_content_email':
                data["extra_content_email"],
                'contacting_user_id':
                possible_user.id if possible_user else None
            }

            ac_cleaned = data_clean(ac_dict)
            author_contact = AuthorContact(**ac_cleaned["kwargs"])

            db.session.add(author_contact)
            db.session.commit()
            ''' Author Contact Version '''
            #version = version_data(ac_cleaned)
            author_contact_version = Version(**version)
            author_contact_version.author_contact = author_contact
            db.session.add(author_contact_version)
            db.session.commit()
            author_contact_version.original_version_id = author_contact_version.id
            db.session.add(author_contact_version)
            db.session.commit()
    ''' Species '''
    species = Species.query.filter_by(
        species_accepted=data["species_accepted"]).first()
    iucn = IUCNStatus.query.filter_by(
        status_code=data["species_iucn_status_id"]).first()

    if species == None:
        species_dict = {
            'gbif_taxon_key': data["species_gbif_taxon_key"],
            'species_iucn_taxonid': data["species_iucn_taxonid"],
            'species_accepted': data["species_accepted"],
            'species_common': data["species_common"],
            'iucn_status_id': iucn.id if iucn else None,
            'image_path': data["image_path"],
            'image_path2': data["image_path2"]
        }

        species_cleaned = data_clean(species_dict)
        species = Species(**species_cleaned["kwargs"])

        db.session.add(species)
        db.session.commit()
        ''' Species Version '''
        #version = version_data(species_cleaned)
        species_version = Version(**version)
        species_version.species = species
        db.session.add(species_version)
        db.session.commit()
        species_version.original_version_id = species_version.id
        db.session.add(species_version)
        db.session.commit()
    ''' Trait '''
    spand_ex_growth_type = SpandExGrowthType.query.filter_by(
        type_name=data["trait_spand_ex_growth_type_id"]).first()
    dicot_monoc = DicotMonoc.query.filter_by(
        dicot_monoc_name=data["trait_dicot_monoc_id"]).first()
    growth_form_raunkiaer = GrowthFormRaunkiaer.query.filter_by(
        form_name=data["trait_growth_form_raunkiaer_id"]).first()
    organism_type = OrganismType.query.filter_by(
        type_name=data["trait_organism_type_id"]).first()
    angio_gymno = AngioGymno.query.filter_by(
        angio_gymno_name=data["trait_angio_gymno_id"]).first()

    trait = Trait.query.filter_by(species_id=species.id).first()

    if trait == None:
        trait_dict = {
            'species_id':
            species.id,
            'organism_type':
            organism_type,
            'dicot_monoc':
            dicot_monoc,
            'angio_gymno':
            angio_gymno,
            'species_seedbank':
            coerce_boolean(data["species_seedbank"]),
            'species_gisd_status':
            coerce_boolean(data["species_gisd_status"]),
            'species_clonality':
            coerce_boolean(data["species_clonality"]),
            'spand_ex_growth_type_id':
            spand_ex_growth_type.id if spand_ex_growth_type else None,
            'growth_form_raunkiaer_id':
            growth_form_raunkiaer.id if growth_form_raunkiaer else None
        }

        trait_cleaned = data_clean(trait_dict)
        trait = Trait(**trait_cleaned["kwargs"])

        db.session.add(trait)
        db.session.commit()
        ''' Trait Version '''
        #version = version_data(trait_cleaned)
        trait_version = Version(**version)
        trait_version.trait = trait
        db.session.add(trait_version)
        db.session.commit()
        trait_version.original_version_id = trait_version.id
        db.session.add(trait_version)
        db.session.commit()
    ''' Taxonomy '''
    tax = Taxonomy.query.filter_by(species_id=species.id).first()
    if tax == None:
        tax_dict = {
            'authority':
            None,
            'tpl_version':
            None,
            'infraspecies_accepted':
            None,
            'species_epithet_accepted':
            None,
            'genus_accepted':
            data["taxonomy_genus_accepted"],
            'genus':
            data["taxonomy_genus"],
            'family':
            data["taxonomy_family"],
            'tax_order':
            data["taxonomy_order"],
            'tax_class':
            data["taxonomy_class"],
            'phylum':
            data["taxonomy_phylum"],
            'kingdom':
            data["taxonomy_kingdom"],
            'col_check_date':
            datetime.datetime.strptime(data["taxonomy_col_check_date"],
                                       "%d/%m/%Y").strftime("%Y-%m-%d")
            if data['taxonomy_col_check_date'] else None,
            'col_check_ok':
            coerce_boolean(data["taxonomy_col_check_ok"])
        }

        tax_cleaned = data_clean(tax_dict)

        # if not all(value == None for key, value in tax_cleaned["kwargs"].items() if key not in ignore_keys):
        tax = Taxonomy(**tax_cleaned["kwargs"])
        db.session.add(tax)
        db.session.commit()
        tax.species = species
        db.session.add(tax)
        db.session.commit()
        ''' Taxonomy Version '''
        #version = version_data(tax_cleaned)
        taxonomy_version = Version(**version)
        taxonomy_version.version_number = 1
        taxonomy_version.taxonomy = tax
        db.session.add(taxonomy_version)
        db.session.commit()
        taxonomy_version.original_version_id = taxonomy_version.id
        db.session.add(taxonomy_version)
        db.session.commit()
    ''' Study '''
    # What if all none? Will they be grouped together?
    #    study = Study.query.filter_by(publication_id=publication.id, study_start=data["study_start"], study_end=data["study_end"]).first()
    #    if study == None:
    #        purpose_endangered = PurposeEndangered.query.filter_by(purpose_name=data["study_purpose_endangered_id"]).first() if data["study_purpose_endangered_id"] else data["study_purpose_endangered_id"]
    #
    #        purpose_weed = PurposeWeed.query.filter_by(purpose_name="study_purpose_weed_id").first() if data["study_purpose_weed_id"] else data["study_purpose_weed_id"]
    #        database_source = Institute.query.filter_by(institution_name=data["study_database_source"]).first()# if data["study_purpose_weed_id"] else data["study_purpose_endangered_id"]
    #
    #        study_dict = {'study_duration' : data["study_duration"],
    #        'study_start' : data["study_start"],
    #        'study_end' :  data["study_end"],
    #        'number_populations' : data["study_number_populations"],
    #        'purpose_endangered_id' : purpose_endangered.id if purpose_endangered else None,
    #        'purpose_weed_id' : purpose_weed.id if purpose_weed else None,
    #        'database_source' : database_source}
    #
    #        study_cleaned = data_clean(study_dict)
    #
    #        # if not all(value == None for key, value in study_cleaned["kwargs"].items() if key not in ignore_keys) and population_present:
    #        study = Study(**study_cleaned["kwargs"])
    #        db.session.add(study)
    #        db.session.commit()
    #
    #        study.publication_id = publication.id
    #        study.species_id = species.id
    #        db.session.add(study)
    #        db.session.commit()
    #
    #
    #        ''' Study Version '''
    #        version = version_data(study_cleaned)
    #        study_version = Version(**version)
    #        study_version.version_number = 1
    #        study_version.study = study
    #        db.session.add(study_version)
    #        db.session.commit()
    #        study_version.original_version_id = study_version.id
    #        db.session.add(study_version)
    #        db.session.commit()
    ''' Protocol '''

    #    digitization_protocol = DigitizationProtocol.query.filter_by(field_name=data["digitization_protocol_id"]).first()
    #    commonterm = CommonTerm.query.filter_by(common_value_name=data["commonterm_id"]).first()

    #    protocol = Protocol.query.filter_by(protocol_id=protocol.id).first()

    #    if protocol == None:
    #        protocol_dict = {'protocol_id' : protocol.id,
    #        'digitization_protocol_id' : digitization_protocol.id if digitization_protocol else None,
    #        'commonterm_id' : commonterm.id if commonterm else None}

    #        protocol_cleaned = data_clean(protocol_dict)
    #        protocol = Protocol(**protocol_cleaned["kwargs"])

    #        db.session.add(protocol)
    #        db.session.commit()
    ''' Population '''
    '''            '''
    invasive_status_study = InvasiveStatusStudy.query.filter_by(
        status_name=data["population_invasive_status_study_id"]).first()
    invasive_status_elsewhere = InvasiveStatusStudy.query.filter_by(
        status_name=data["population_invasive_status_elsewhere_id"]).first()
    ecoregion = Ecoregion.query.filter_by(
        ecoregion_code=data["population_ecoregion_id"]).first()
    continent = Continent.query.filter_by(
        continent_name=data["population_continent_id"]).first()
    ###Danny trying add database meta-table in correct location
    database = Database.query.filter_by(
        database_master_version=data["population_database_id"]).first()

    purpose_endangered = PurposeEndangered.query.filter_by(
        purpose_name=data["study_purpose_endangered_id"]).first(
        ) if data["study_purpose_endangered_id"] else data[
            "study_purpose_endangered_id"]

    purpose_weed = PurposeWeed.query.filter_by(
        purpose_name="study_purpose_weed_id").first(
        ) if data["study_purpose_weed_id"] else data["study_purpose_weed_id"]
    database_source = Institute.query.filter_by(
        institution_name=data["study_database_source_id"]).first()

    pop = Population.query.filter_by(population_name=data["population_name"],
                                     publication_id=publication.id,
                                     species_id=species.id).first()

    if pop == None:
        pop_dict = {
            'population_name':
            data["population_name"],
            'latitude':
            data["population_latitude"],
            'lat_ns':
            data["lat_ns"],
            'lat_deg':
            data["lat_deg"],
            'lat_min':
            data["lat_min"],
            'lat_sec':
            data["lat_sec"],
            'longitude':
            data["population_longitude"],
            'lon_ew':
            data["lon_ew"],
            'lon_deg':
            data["lon_deg"],
            'lon_min':
            data["lon_min"],
            'lon_sec':
            data["lon_sec"],
            'altitude':
            data["population_altitude"],
            #'pop_size' : data["population_pop_size"],
            'country':
            data["population_country"],
            'invasive_status_study_id':
            invasive_status_study.id if invasive_status_study else None,
            'invasive_status_elsewhere_id':
            invasive_status_elsewhere.id
            if invasive_status_elsewhere else None,
            'ecoregion':
            ecoregion,
            'continent':
            continent,
            'database':
            database,
            'within_site_replication':
            data['population_within_site_replication'],
            'study_duration':
            data["study_duration"],
            'study_start':
            data["study_start"],
            'study_end':
            data["study_end"],
            'number_populations':
            data["study_number_populations"],
            'purpose_endangered_id':
            purpose_endangered.id if purpose_endangered else None,
            'purpose_weed_id':
            purpose_weed.id if purpose_weed else None,
            'database_source':
            database_source
        }

        pop_cleaned = data_clean(pop_dict)

        # if not all(value == None for key, value in pop_cleaned["kwargs"].items() if key not in ignore_keys) and matrix_present:
        pop = Population(**pop_cleaned["kwargs"])

        db.session.add(pop)
        db.session.commit()

        pop.species_author = data["species_author"]
        pop.publication_id = publication.id
        pop.species_id = species.id

        db.session.add(pop)
        db.session.commit()
        ''' Population Version '''
        #version = version_data(pop_cleaned)
        population_version = Version(**version)
        population_version.version_number = 1
        population_version.population = pop
        db.session.add(population_version)
        db.session.commit()
        population_version.original_version_id = population_version.id
        db.session.add(population_version)
        db.session.commit()
    ''' Matrix '''
    treatment_string = data["matrix_treatment_id"]

    if treatment_string == 'NDY':
        treatment = 'NDY'
    elif treatment_string == None:
        treatment = None
    else:
        treatment = Treatment.query.filter_by(
            treatment_name=data["matrix_treatment_id"]
        ).first() if Treatment.query.filter_by(
            treatment_name=data["matrix_treatment_id"]).first() else Treatment(
                treatment_name=data["matrix_treatment_id"])
        db.session.add(treatment)
        db.session.commit()

    matrix_dict = {
        'treatment':
        treatment,
        'matrix_split':
        coerce_boolean(data["matrix_split"]),
        'matrix_composition':
        MatrixComposition.query.filter_by(
            comp_name=data["matrix_composition_id"]).first(),
        'matrix_criteria_size':
        data["matrix_criteria_size"],
        'matrix_criteria_ontogeny':
        coerce_boolean(data["matrix_criteria_ontogeny"]),
        'matrix_criteria_age':
        coerce_boolean(data["matrix_criteria_age"]),
        'matrix_start_month':
        data["matrix_start_month"],
        'matrix_end_month':
        data["matrix_end_month"],
        'matrix_start_year':
        data["matrix_start_year"],
        'matrix_end_year':
        data["matrix_end_year"],
        'studied_sex':
        StudiedSex.query.filter_by(
            sex_code=data["matrix_studied_sex_id"]).first(),
        'start_season':
        StartSeason.query.filter_by(
            season_id=data["matrix_start_season_id"]).first()
        if data["matrix_start_season_id"] else None,
        'end_season':
        EndSeason.query.filter_by(
            season_id=data["matrix_end_season_id"]).first()
        if data["matrix_end_season_id"] else None,
        'matrix_fec':
        coerce_boolean(data["matrix_fec"]),
        'matrix_a_string':
        data["matrix_a_string"],
        'matrix_f_string':
        data["matrix_f_string"],
        'matrix_u_string':
        data["matrix_u_string"],
        'matrix_c_string':
        data["matrix_c_string"],
        'non_independence':
        data["matrix_non_independence"],
        'matrix_dimension':
        data["matrix_dimension"],
        'non_independence_author':
        data["matrix_non_independence_author"],
        'matrix_complete':
        coerce_boolean(data["matrix_complete"]),
        'class_number':
        data["matrix_class_number"],
        'observations':
        data["matrix_observations"],
        'captivities':
        Captivity.query.filter_by(
            cap_code=data["matrix_captivity_id"]).first(),
        'class_author':
        data["matrix_class_author"],
        'class_organized':
        data["matrix_class_organized"],
        'matrix_difficulty':
        data["matrix_difficulty"],
        'seasonal':
        coerce_boolean(data["matrix_seasonal"]),
        'survival_issue':
        calc_surv_issue(data["matrix_u_string"]),
        'periodicity':
        data["matrix_periodicity"],
        'matrix_irreducible':
        is_matrix_irreducible(data["matrix_a_string"]),
        'matrix_primitive':
        is_matrix_primitive(data["matrix_a_string"]),
        'matrix_ergodic':
        is_matrix_ergodic(data["matrix_a_string"]),
        'matrix_lambda':
        calc_lambda(data["matrix_a_string"])
    }

    matrix_cleaned = data_clean(matrix_dict)

    # if not all(value == None for key, value in matrix_cleaned["kwargs"].items() if key not in ignore_keys):
    matrix = Matrix(**matrix_cleaned["kwargs"])

    db.session.add(matrix)
    db.session.commit()

    matrix.population_id = pop.id

    db.session.add(matrix)
    db.session.commit()
    ''' matrix Version '''
    #version = version_data(matrix_cleaned)
    matrix_version = Version(**version)
    matrix_version.version_number = 1
    matrix_version.matrix = matrix
    db.session.add(matrix_version)
    db.session.commit()
    matrix_version.original_version_id = matrix_version.id
    db.session.add(matrix_version)
    db.session.commit()
    ''' Fixed '''

    fixed = Fixed.query.filter_by(matrix=matrix).first()

    if fixed == None:
        fixed_dict = {
            'matrix':
            matrix,
            'census_timings':
            CensusTiming.query.filter_by(
                census_name=data["fixed_census_timing_id"]).first(),
            'seed_stage_error':
            coerce_boolean(data["fixed_seed_stage_error"]),
            'smalls':
            Small.query.filter_by(small_name=data["fixed_small_id"]).first(),
            'vector_str':
            data["matrix_vectors_includes_na"]
        }

        fixed_cleaned = data_clean(fixed_dict)
        fixed = Fixed(**fixed_cleaned["kwargs"])

        db.session.add(fixed)
        db.session.commit()
        ''' fixed Version '''
        #version = version_data(fixed_cleaned)
        fixed_version = Version(**version)
        fixed_version.version_number = 1
        fixed_version.fixed = fixed
        db.session.add(fixed_version)
        db.session.commit()
        fixed_version.original_version_id = fixed_version.id
        db.session.add(fixed_version)
        db.session.commit()
def submit_new(data):
    species = Species.query.filter_by(species_accepted=data["species_accepted"]).first()

    if species == None:
        species = Species(species_accepted=data["species_accepted"])        
        species.gbif_taxon_key = data["species_gbif_taxon_key"]
        species.species_author = data["species_author"]
        species.species_accepted = data["species_accepted"]        
        species.species_esa_status = ESAStatus.query.filter_by(status_code=data["species_esa_status_id"]).first()        
        species.species_common = data["species_common"]
        species.species_iucn_status = IUCNStatus.query.filter_by(status_code=data["species_iucn_status_id"]).first()

        db.session.add(species)
        db.session.commit()

        ''' Species Version '''
        species_version = Version()
        species_version.version_number = 0
        species_version.species = species
        db.session.add(species_version) 
        db.session.commit()  
        species_version.version_of_id = species_version.id
        species_version.checked = True
        species_version.checked_count = 1
        species_version.statuses = Status.query.filter_by(status_name="Green").first()
        species_version.user = User.query.filter_by(username="******").first()
        species_version.database = Database.query.filter_by(database_name="COMPADRE 4").first()

        db.session.add(species_version)
        db.session.commit()

    ''' Publication '''    
    if data["publication_DOI_ISBN"] == None:
        publication = Publication.query.filter_by(authors=data["publication_authors"]).filter_by(year=data["publication_year"]).filter_by(name=data["publication_journal_name"]).first()
    else: 
        publication = Publication.query.filter_by(DOI_ISBN=data["publication_DOI_ISBN"]).first()    
    if publication == None:
        publication = Publication()
        publication.authors = data["publication_authors"]
        publication.year = data["publication_year"]
        publication.DOI_ISBN = data["publication_DOI_ISBN"]
        publication.additional_source_string = data["publication_additional_source_string"]
        publication.name = data["publication_journal_name"]

        db.session.add(publication)
        db.session.commit()

        ''' Publication Version '''
        publication_version = Version()
        publication_version.version_number = 0
        publication_version.publication = publication
        db.session.add(publication_version) 
        db.session.commit()  
        publication_version.version_of_id = publication_version.id
        publication_version.checked = True
        publication_version.checked_count = 1
        publication_version.statuses = Status.query.filter_by(status_name="Green").first()
        publication_version.user = User.query.filter_by(username="******").first()
        publication_version.database = Database.query.filter_by(database_name="COMPADRE 4").first()

        db.session.add(publication_version)
        db.session.commit()

    ''' Trait '''
    spand_ex_growth_type = SpandExGrowthType.query.filter_by(type_name=data["trait_spand_ex_growth_type_id"]).first()
    dicot_monoc = DicotMonoc.query.filter_by(dicot_monoc_name=data["trait_dicot_monoc_id"]).first()
    growth_form_raunkiaer = GrowthFormRaunkiaer.query.filter_by(form_name=data["trait_growth_form_raunkiaer_id"]).first()
    organism_type = OrganismType.query.filter_by(type_name=data["trait_organism_type_id"]).first()
    angio_gymno = AngioGymno.query.filter_by(angio_gymno_name=data["trait_angio_gymno_id"]).first()

    trait = Trait.query.filter_by(species_id=species.id).first()

    if trait == None:
        trait = Trait()
        trait.species_id = species.id
        trait.organism_type = organism_type
        trait.dicot_monoc = dicot_monoc
        trait.angio_gymno = angio_gymno
        trait.spand_ex_growth_type = spand_ex_growth_type
        trait.growth_form_raunkiaer = growth_form_raunkiaer

        db.session.add(trait)
        db.session.commit()

        ''' Trait Version '''
        trait_version = Version()
        trait_version.version_number = 0
        trait_version.trait = trait
        db.session.add(trait_version) 
        db.session.commit()  
        trait_version.version_of_id = trait_version.id
        trait_version.checked = True
        trait_version.checked_count = 1
        trait_version.statuses = Status.query.filter_by(status_name="Green").first()
        trait_version.user = User.query.filter_by(username="******").first()
        trait_version.database = Database.query.filter_by(database_name="COMPADRE 4").first()

        db.session.add(trait_version)
        db.session.commit()

    ''' Study '''
    # What if all none? Will they be grouped together?
    purpose_endangered = PurposeEndangered.query.filter_by(purpose_name=data["study_purpose_endangered_id"]).first()
    purpose_weed = PurposeWeed.query.filter_by(purpose_name="study_purpose_weed_id").first()

    study = Study.query.filter_by(publication_id=publication.id, study_start=data["study_start"], study_end=data["study_end"]).first()
    if study == None:
        study = Study()
        study.study_duration = data["study_duration"]
        study.study_start = data["study_start"]
        study.study_end = data["study_end"]
        study.publication_id = publication.id
        study.number_populations = data["study_number_populations"]

        db.session.add(study)
        db.session.commit()

        ''' Study Version '''
        study_version = Version()
        study_version.version_number = 0
        study_version.study = study
        db.session.add(study_version) 
        db.session.commit()  
        study_version.version_of_id = study_version.id
        study_version.checked = True
        study_version.checked_count = 1
        study_version.statuses = Status.query.filter_by(status_name="Green").first()
        study_version.user = User.query.filter_by(username="******").first()
        study_version.database = Database.query.filter_by(database_name="COMPADRE 4").first()

        db.session.add(study_version)
        db.session.commit()


    ''' Population '''
    invasive_status_study = InvasiveStatusStudy.query.filter_by(status_name=data["population_invasive_status_study_id"]).first()
    invasive_status_elsewhere = InvasiveStatusStudy.query.filter_by(status_name=data["population_invasive_status_elsewhere_id"]).first()
    ecoregion = Ecoregion.query.filter_by(ecoregion_code=data["population_ecoregion_id"]).first()
    continent = Continent.query.filter_by(continent_name=data["population_continent_id"]).first()

    pop = Population.query.filter_by(name=data["population_name"], species_id=species.id, publication_id=publication.id).first()



    if pop == None:
        pop = Population()
        pop.species_author = data["species_author"]
        pop.name = data["population_name"]
        pop.species_id = species.id
        pop.publication_id = publication.id
        pop.study_id = study.id

        pop.longitude = data["population_longitude"]
        pop.latitude = data["population_latitude"]
        pop.altitude = data["population_altitude"]
        pop.pop_size = data["population_pop_size"]
        pop.country = data["population_country"]

        pop.invasive_status_study = invasive_status_study
        pop.invasive_status_elsewhere = invasive_status_elsewhere
        pop.ecoregion = ecoregion
        pop.continent = continent

        db.session.add(pop)
        db.session.commit()

        ''' Population Version '''
        population_version = Version()
        population_version.version_number = 0
        population_version.population = pop
        db.session.add(population_version) 
        db.session.commit()  
        population_version.version_of_id = population_version.id
        population_version.checked = True
        population_version.checked_count = 1
        population_version.statuses = Status.query.filter_by(status_name="Green").first()
        population_version.user = User.query.filter_by(username="******").first()
        population_version.database = Database.query.filter_by(database_name="COMPADRE 4").first()

        db.session.add(population_version)
        db.session.commit()

    ''' Taxonomy '''
    tax = Taxonomy.query.filter_by(species_id=species.id).first()
    if tax == None:
        tax = Taxonomy()
        tax.species_author = species.species_author
        tax.species_accepted = species.species_accepted
        tax.authority = None
        tax.tpl_version = None
        tax.infraspecies_accepted = None
        tax.species_epithet_accepted = None
        tax.genus_accepted = data["taxonomy_genus_accepted"]
        tax.genus = data["taxonomy_genus"]
        tax.family = data["taxonomy_family"]
        tax.tax_order = data["taxonomy_order"]
        tax.tax_class = data["taxonomy_class"]
        tax.phylum = data["taxonomy_phylum"]
        tax.kingdom = data["taxonomy_kingdom"]
        tax.species = species
        tax.publication = publication
        tax.col_check_date = data["taxonomy_col_check_date"]
        tax.col_check_ok = data["taxonomy_col_check_ok"]

        db.session.add(tax)
        db.session.commit()

        ''' Taxonomy Version '''
        taxonomy_version = Version()
        taxonomy_version.version_number = 0
        taxonomy_version.taxonomy = tax
        db.session.add(taxonomy_version) 
        db.session.commit()  
        taxonomy_version.version_of_id = taxonomy_version.id
        taxonomy_version.checked = True
        taxonomy_version.checked_count = 1
        taxonomy_version.statuses = Status.query.filter_by(status_name="Green").first()
        taxonomy_version.user = User.query.filter_by(username="******").first()
        taxonomy_version.database = Database.query.filter_by(database_name="COMPADRE 4").first()

        db.session.add(taxonomy_version)
        db.session.commit()

    ''' Matrix '''
    matrix = Matrix()
    treatment = Treatment.query.filter_by(treatment_name=data["matrix_treatment_id"]).first()
    
    if treatment == None:
        treatment = Treatment(treatment_name=data["matrix_treatment_id"])
    
    matrix.treatment = treatment
    matrix.matrix_split = data["matrix_split"]
    
    composition = MatrixComposition.query.filter_by(comp_name=data["matrix_composition_id"]).first()
    

    matrix.matrix_composition = composition

    
    matrix.survival_issue = data["matrix_survival_issue"]    
    matrix.periodicity = data["matrix_periodicity"]
    matrix.matrix_criteria_size = data["matrix_criteria_size"]
    matrix.matrix_criteria_ontogeny = coerce_boolean(data["matrix_criteria_ontogeny"])
    matrix.matrix_criteria_age = coerce_boolean(data["matrix_criteria_age"]) 
    
    matrix.matrix_start_month = data["matrix_start_month"]
    matrix.matrix_end_month = data["matrix_end_month"]
    matrix.matrix_start_year = data["matrix_start_year"]
    matrix.matrix_end_year = data["matrix_end_year"]

    matrix.studied_sex = StudiedSex.query.filter_by(sex_code=data["matrix_studied_sex_id"]).first()


    start_season = StartSeason.query.filter_by(season_id=data["matrix_start_season_id"]).first()
    end_season = EndSeason.query.filter_by(season_id=data["matrix_end_season_id"]).first()

    if start_season != None:
        matrix.start_season = start_season

    if end_season != None:
        matrix.end_season = end_season
        
    matrix.matrix_fec = coerce_boolean(data["matrix_fec"])

    matrix.matrix_a_string = data["matrix_a_string"]
    matrix.matrix_f_string = data["matrix_f_string"]
    matrix.matrix_u_string = data["matrix_u_string"]
    matrix.matrix_c_string = data["matrix_c_string"]

    matrix.non_independence = data["matrix_non_independence"]
    matrix.dimension = data["matrix_dimension"]
    matrix.non_independence_author = data["matrix_non_independence_author"]
    matrix.matrix_complete = data["matrix_complete"]
    matrix.vectors_includes_na = data["matrix_vectors_includes_na"]
    matrix.class_number = data["matrix_class_number"]
    matrix.observations = data["matrix_observations"]

    captivities = Captivity.query.filter_by(cap_code=data["matrix_captivity_id"]).first()

    matrix.captivities = captivities
    matrix.class_author = data["matrix_class_author"]
    matrix.matrix_difficulty = data["matrix_difficulty"]
    matrix.independent = data["matrix_independent"]
    matrix.seasonal = data["matrix_seasonal"]

    matrix.uid = generate_uid(species, publication, pop, matrix)

    matrix.population = pop
    matrix.study = study

    db.session.add(matrix)
    db.session.commit()

    ''' matrix Version '''
    matrix_version = Version()
    matrix_version.version_number = 0
    matrix_version.matrix = matrix
    db.session.add(matrix_version) 
    db.session.commit()
    matrix_version.version_of_id = matrix_version.id
    matrix_version.checked = True
    matrix_version.checked_count = 1
    matrix_version.statuses = Status.query.filter_by(status_name="Green").first()
    matrix_version.user = User.query.filter_by(username="******").first()
    matrix_version.database = Database.query.filter_by(database_name="COMPADRE 4").first()

    db.session.add(matrix_version)
    db.session.commit()

    ''' Fixed '''

    fixed = Fixed.query.filter_by(matrix=matrix).first()

    if fixed == None:
        fixed = Fixed()
        fixed.matrix = matrix
        fixed.census_timings = CensusTiming.query.filter_by(census_name=data["fixed_census_timing_id"]).first()
        fixed.seed_stage_error = data["fixed_seed_stage_error"]
        fixed.smalls = Small.query.filter_by(small_name=data["fixed_small_id"]).first()

        db.session.add(fixed)
        db.session.commit()

        ''' fixed Version '''
        fixed_version = Version()
        fixed_version.version_number = 0
        fixed_version.fixed = fixed
        db.session.add(fixed_version) 
        db.session.commit()
        fixed_version.version_of_id = fixed_version.id
        fixed_version.checked = True
        fixed_version.checked_count = 1
        fixed_version.statuses = Status.query.filter_by(status_name="Green").first()
        fixed_version.user = User.query.filter_by(username="******").first()
        fixed_version.database = Database.query.filter_by(database_name="COMPADRE 4").first()

        db.session.add(fixed_version)
        db.session.commit()