def migrate_meta():
from app.models import User, Role, Permission, \
IUCNStatus, ESAStatus, OrganismType, GrowthFormRaunkiaer, ReproductiveRepetition, \
DicotMonoc, AngioGymno, SpandExGrowthType, SourceType, Database, Purpose, MissingData, ContentEmail, Ecoregion, Continent, InvasiveStatusStudy, InvasiveStatusElsewhere, StageTypeClass, \
TransitionType, MatrixComposition, StartSeason, StudiedSex, Captivity, Species, Taxonomy, Trait, \
Publication, Study, AuthorContact, AdditionalSource, Population, Stage, StageType, Treatment, \
MatrixStage, MatrixValue, Matrix, Interval, Fixed, Small, CensusTiming, PurposeEndangered, PurposeWeed, Institute
print "Migrating Meta Tables..."
try:
Role.insert_roles()
Species.migrate()
Taxonomy.migrate()
Trait.migrate()
Publication.migrate()
AuthorContact.migrate()
Population.migrate()
StageType.migrate()
MatrixValue.migrate()
Matrix.migrate()
Fixed.migrate()
Study.migrate()
User.migrate()
Version.migrate()
Institute.migrate()
except:
"Error migrating metadata"
finally:
"Done + good"
return
def migrate_meta():
from app.models import User, Role, Permission, \
IUCNStatus, OrganismType, GrowthFormRaunkiaer, ReproductiveRepetition, \
DicotMonoc, AngioGymno, SpandExGrowthType, SourceType, Database, Purpose, MissingData, ContentEmail, Ecoregion, Continent, InvasiveStatusStudy, InvasiveStatusElsewhere, StageTypeClass, \
TransitionType, MatrixComposition, StartSeason, StudiedSex, Captivity, Species, Taxonomy, Trait, \
Publication, AuthorContact, AdditionalSource, Population, Stage, StageType, Treatment, \
MatrixStage, MatrixValue, Matrix, Interval, Fixed, Small, CensusTiming, PurposeEndangered, PurposeWeed, Institute, Version, \
PublicationsProtocol, DigitizationProtocol, Protocol, CommonTerm
print "Migrating Meta Tables..."
Role.insert_roles()
Species.migrate()
Taxonomy.migrate()
Trait.migrate()
Publication.migrate()
AuthorContact.migrate()
Population.migrate()
StageType.migrate()
MatrixValue.migrate()
Matrix.migrate()
Fixed.migrate()
Version.migrate()
Institute.migrate()
User.migrate()
Database.migrate()
Status.migrate()
PublicationsProtocol.migrate()
DigitizationProtocol.migrate()
CommonTerm.migrate()
return
def entity_add(self, json):
en_type = json['type']
if en_type == "":
json['type'] = "info"
entity = Info()
schema = InfoSchema()
else:
if type == "me":
entity = Metabolism()
schema = MetabolismSchema()
elif type == 'gn':
entity = General()
schema = GeneralSchema()
elif type == 'ge':
entity = Genus()
schema = GenusSchema()
elif type == 'sp':
entity = Species()
schema = SpeciesSchema()
elif type == 'di':
entity = Disease()
schema = DiseaseSchema()
entity.import_data(json)
refs = json['refs']
for ref in refs:
new_ref = Ref()
new_ref.import_data(ref)
entity.refs.append(new_ref)
entity.add(entity)
return schema.dump(entity).data
def test_species_model_3(self):
data = speciesdata
d = data['24']
d2 = data['25']
id = 24
id2 = 25
species = Species(
id, d['name'], d['classification'], d['language'],
d['average_height'], d['eye_colors'],
"https://s3.amazonaws.com/tf.images/reduced-ban16554.jpg")
species2 = Species(
id2, d2['name'], d2['classification'], d2['language'],
d2['average_height'], d2['eye_colors'],
"https://s3.amazonaws.com/tf.images/reduced-ban16554.jpg")
test_db.session.add(species)
test_db.session.add(species2)
s = test_db.session.query(Species).first()
sclass = s.classification
self.assertEqual(d['classification'], sclass)
test_db.session.delete(species)
test_db.session.delete(species2)
test_db.session.commit()
def test_species_1(self):
"""Test querying the database by attribute using simple keywords"""
data = speciesdata
d = data['24']
id = 24
species = Species(
id, d['name'], d['classification'], d['language'],
d['average_height'], d['eye_colors'],
"https://s3.amazonaws.com/tf.images/reduced-ban16554.jpg")
test_db.session.add(species)
s = test_db.session.query(Species).all()
count = len(s)
self.assertEqual(1, count)
test_db.session.delete(species)
test_db.session.commit()
def version_current():
models = [
Species(),
Taxonomy(),
Trait(),
Publication(),
AuthorContact(),
Population(),
StageType(),
MatrixValue(),
Matrix(),
Fixed(),
Institute(),
Protocol()
]
for model in models:
model_version(model)
def entity_update(self, type, info_id, json):
if type == 'info':
entity = Info()
elif type == 'gn':
entity = General()
elif type == 'me':
entity = Metabolism()
elif type == 'ge':
entity = Genus()
elif type == 'sp':
entity = Species()
elif type == 'di':
entity = Disease()
else:
abort(401)
val = entity.query.get_or_404(info_id)
for key, value in json.items():
setattr(val, key, value)
val.update()
def submit_new(data):
import datetime
version = {
'checked': True,
'checked_count': 1,
'statuses': Status.query.filter_by(status_name="Green").first()
}
if data["population_database_id"] == "XXX" or data[
"population_database_id"] == "X.X.X":
version = {
'checked': False,
'checked_count': 0,
'statuses': Status.query.filter_by(status_name="Pending").first()
}
# When checking for null data later, these need to be excluded, as they will always have a value
ignore_keys = ['version_ok', 'version_latest', 'version_original']
''' DigitizationProtocol '''
# digitization_protocol = DigitizationProtocol.query.filter_by(field_name=data["digitization_protocol"]).first()
# if digitization_protocol == None:
# ac_dict = {'protocol_id' : protocol.id,
# 'field_name' : data['field_name'],
# 'name_in_csv' : data["name_in_csv"],
# 'database_model' : data["database_model"],
# 'field_description' : data["field_description"],
# 'field_short_description' : data["field_short_description"]
# }
# ac_cleaned = data_clean(ac_dict)
# digitization_protocol = Protocol(**ac_cleaned["kwargs"])
# db.session.add(digitization_protocol)
# db.session.commit()
''' Publication '''
publications_protocol = PublicationsProtocol.query.filter_by(
protocol_number=data["publications_protocol_id"]).first()
if data["publication_DOI_ISBN"] == None:
publication = Publication.query.filter_by(
authors=data["publication_authors"]
).filter_by(year=data["publication_year"]).filter_by(
journal_name=data["publication_journal_name"]).filter_by(
additional_source_string=data[
"publication_additional_source_string"]).filter_by(
study_notes=data["publication_study_notes"]).first()
else:
publication = Publication.query.filter_by(
DOI_ISBN=data["publication_DOI_ISBN"]).first()
if publication == None:
purposes = {
"Comparative Demography":
data["publication_purpose_comparative_demography"],
"Spatial Demography":
data["publication_purpose_spatial_demography"],
"Abiotic Impacts":
data["publication_purpose_abiotic"],
"PVA":
data["publication_purpose_pva"],
"Species Dynamics Description":
data["publication_purpose_species_dynamics_description"],
"Interspecific Interactions":
data["publication_purpose_interspecific_interactions"],
"Management Evaluation":
data["publication_purpose_management_evaluation"],
"Methodological Advancement":
data["publication_purpose_methodological_advancement"]
}
queryset = [
Purpose.query.filter(Purpose.purpose_name == key).first()
for key, val in purposes.items() if val == '1'
]
if data['publication_missing_data'] != 'NDY' and data[
'publication_missing_data']:
missing_data_unicode = data['publication_missing_data'].replace(
" ", "").split(';')
missing_data = [
MissingData.query.filter_by(missing_code=key).first()
for key in missing_data_unicode
if MissingData.query.filter_by(missing_code=key).first()
]
else:
missing_data = 'NDY'
pub_dict = {
'authors':
data["publication_authors"],
'year':
data["publication_year"],
'publications_protocol':
publications_protocol,
'DOI_ISBN':
data["publication_DOI_ISBN"],
'additional_source_string':
data["publication_additional_source_string"],
'journal_name':
data["publication_journal_name"],
'date_digitised':
datetime.datetime.strptime(data['publication_date_digitization'],
"%d/%m/%Y").strftime("%Y-%m-%d")
if data['publication_date_digitization'] else None,
'purposes':
queryset,
'study_notes':
data["publication_study_notes"]
}
pub_cleaned = data_clean(pub_dict)
# if not all(value == None for key, value in pub_cleaned["kwargs"].items() if key not in ignore_keys) and study_present:
publication = Publication(**pub_cleaned["kwargs"])
db.session.add(publication)
db.session.commit()
publication.missing_data = missing_data if type(
missing_data) == list else []
db.session.add(publication)
db.session.commit()
''' Publication Version '''
#version = version_data(pub_cleaned)
publication_version = Version(**version)
publication_version.publication = publication
publication.colour = gen_hex_code()
possible_user = User.query.filter_by(
name=data["publication_student"]).first()
na_user = User.query.filter_by(name="N/A").first()
if possible_user == None:
possible_user = na_user
publication_version.entered_by_id = possible_user.id if possible_user else None,
publication_version.checked_by_id = na_user.id if na_user else None,
db.session.add(publication_version)
db.session.commit()
publication_version.original_version_id = publication_version.id
db.session.add(publication_version)
db.session.commit()
''' Author contact '''
author_contacts = AuthorContact.query.filter_by(
corresponding_author=data["publication_corresponding_author"]
).filter_by(
corresponding_author_email=data["publication_corresponding_email"]
).first()
if author_contacts == None:
ac_dict = {
'publication_id':
publication.id,
'date_contacted':
datetime.datetime.strptime(data['date_author_contacted'],
"%d/%m/%Y").strftime("%Y-%m-%d")
if data['date_author_contacted'] else None,
'date_contacted_again':
datetime.datetime.strptime(data['date_author_contacted_again'],
"%d/%m/%Y").strftime("%Y-%m-%d")
if data['date_author_contacted_again'] else None,
'extra_content_email':
data["correspondence_email_content"],
'author_reply':
data["correspondence_author_reply"],
'corresponding_author':
data["publication_corresponding_author"],
'corresponding_author_email':
data["publication_corresponding_email"],
'correspondence_email_content':
data["correspondence_email_content"],
'extra_content_email':
data["extra_content_email"],
'contacting_user_id':
possible_user.id if possible_user else None
}
ac_cleaned = data_clean(ac_dict)
author_contact = AuthorContact(**ac_cleaned["kwargs"])
db.session.add(author_contact)
db.session.commit()
''' Author Contact Version '''
#version = version_data(ac_cleaned)
author_contact_version = Version(**version)
author_contact_version.author_contact = author_contact
db.session.add(author_contact_version)
db.session.commit()
author_contact_version.original_version_id = author_contact_version.id
db.session.add(author_contact_version)
db.session.commit()
''' Species '''
species = Species.query.filter_by(
species_accepted=data["species_accepted"]).first()
iucn = IUCNStatus.query.filter_by(
status_code=data["species_iucn_status_id"]).first()
if species == None:
species_dict = {
'gbif_taxon_key': data["species_gbif_taxon_key"],
'species_iucn_taxonid': data["species_iucn_taxonid"],
'species_accepted': data["species_accepted"],
'species_common': data["species_common"],
'iucn_status_id': iucn.id if iucn else None,
'image_path': data["image_path"],
'image_path2': data["image_path2"]
}
species_cleaned = data_clean(species_dict)
species = Species(**species_cleaned["kwargs"])
db.session.add(species)
db.session.commit()
''' Species Version '''
#version = version_data(species_cleaned)
species_version = Version(**version)
species_version.species = species
db.session.add(species_version)
db.session.commit()
species_version.original_version_id = species_version.id
db.session.add(species_version)
db.session.commit()
''' Trait '''
spand_ex_growth_type = SpandExGrowthType.query.filter_by(
type_name=data["trait_spand_ex_growth_type_id"]).first()
dicot_monoc = DicotMonoc.query.filter_by(
dicot_monoc_name=data["trait_dicot_monoc_id"]).first()
growth_form_raunkiaer = GrowthFormRaunkiaer.query.filter_by(
form_name=data["trait_growth_form_raunkiaer_id"]).first()
organism_type = OrganismType.query.filter_by(
type_name=data["trait_organism_type_id"]).first()
angio_gymno = AngioGymno.query.filter_by(
angio_gymno_name=data["trait_angio_gymno_id"]).first()
trait = Trait.query.filter_by(species_id=species.id).first()
if trait == None:
trait_dict = {
'species_id':
species.id,
'organism_type':
organism_type,
'dicot_monoc':
dicot_monoc,
'angio_gymno':
angio_gymno,
'species_seedbank':
coerce_boolean(data["species_seedbank"]),
'species_gisd_status':
coerce_boolean(data["species_gisd_status"]),
'species_clonality':
coerce_boolean(data["species_clonality"]),
'spand_ex_growth_type_id':
spand_ex_growth_type.id if spand_ex_growth_type else None,
'growth_form_raunkiaer_id':
growth_form_raunkiaer.id if growth_form_raunkiaer else None
}
trait_cleaned = data_clean(trait_dict)
trait = Trait(**trait_cleaned["kwargs"])
db.session.add(trait)
db.session.commit()
''' Trait Version '''
#version = version_data(trait_cleaned)
trait_version = Version(**version)
trait_version.trait = trait
db.session.add(trait_version)
db.session.commit()
trait_version.original_version_id = trait_version.id
db.session.add(trait_version)
db.session.commit()
''' Taxonomy '''
tax = Taxonomy.query.filter_by(species_id=species.id).first()
if tax == None:
tax_dict = {
'authority':
None,
'tpl_version':
None,
'infraspecies_accepted':
None,
'species_epithet_accepted':
None,
'genus_accepted':
data["taxonomy_genus_accepted"],
'genus':
data["taxonomy_genus"],
'family':
data["taxonomy_family"],
'tax_order':
data["taxonomy_order"],
'tax_class':
data["taxonomy_class"],
'phylum':
data["taxonomy_phylum"],
'kingdom':
data["taxonomy_kingdom"],
'col_check_date':
datetime.datetime.strptime(data["taxonomy_col_check_date"],
"%d/%m/%Y").strftime("%Y-%m-%d")
if data['taxonomy_col_check_date'] else None,
'col_check_ok':
coerce_boolean(data["taxonomy_col_check_ok"])
}
tax_cleaned = data_clean(tax_dict)
# if not all(value == None for key, value in tax_cleaned["kwargs"].items() if key not in ignore_keys):
tax = Taxonomy(**tax_cleaned["kwargs"])
db.session.add(tax)
db.session.commit()
tax.species = species
db.session.add(tax)
db.session.commit()
''' Taxonomy Version '''
#version = version_data(tax_cleaned)
taxonomy_version = Version(**version)
taxonomy_version.version_number = 1
taxonomy_version.taxonomy = tax
db.session.add(taxonomy_version)
db.session.commit()
taxonomy_version.original_version_id = taxonomy_version.id
db.session.add(taxonomy_version)
db.session.commit()
''' Study '''
# What if all none? Will they be grouped together?
# study = Study.query.filter_by(publication_id=publication.id, study_start=data["study_start"], study_end=data["study_end"]).first()
# if study == None:
# purpose_endangered = PurposeEndangered.query.filter_by(purpose_name=data["study_purpose_endangered_id"]).first() if data["study_purpose_endangered_id"] else data["study_purpose_endangered_id"]
#
# purpose_weed = PurposeWeed.query.filter_by(purpose_name="study_purpose_weed_id").first() if data["study_purpose_weed_id"] else data["study_purpose_weed_id"]
# database_source = Institute.query.filter_by(institution_name=data["study_database_source"]).first()# if data["study_purpose_weed_id"] else data["study_purpose_endangered_id"]
#
# study_dict = {'study_duration' : data["study_duration"],
# 'study_start' : data["study_start"],
# 'study_end' : data["study_end"],
# 'number_populations' : data["study_number_populations"],
# 'purpose_endangered_id' : purpose_endangered.id if purpose_endangered else None,
# 'purpose_weed_id' : purpose_weed.id if purpose_weed else None,
# 'database_source' : database_source}
#
# study_cleaned = data_clean(study_dict)
#
# # if not all(value == None for key, value in study_cleaned["kwargs"].items() if key not in ignore_keys) and population_present:
# study = Study(**study_cleaned["kwargs"])
# db.session.add(study)
# db.session.commit()
#
# study.publication_id = publication.id
# study.species_id = species.id
# db.session.add(study)
# db.session.commit()
#
#
# ''' Study Version '''
# version = version_data(study_cleaned)
# study_version = Version(**version)
# study_version.version_number = 1
# study_version.study = study
# db.session.add(study_version)
# db.session.commit()
# study_version.original_version_id = study_version.id
# db.session.add(study_version)
# db.session.commit()
''' Protocol '''
# digitization_protocol = DigitizationProtocol.query.filter_by(field_name=data["digitization_protocol_id"]).first()
# commonterm = CommonTerm.query.filter_by(common_value_name=data["commonterm_id"]).first()
# protocol = Protocol.query.filter_by(protocol_id=protocol.id).first()
# if protocol == None:
# protocol_dict = {'protocol_id' : protocol.id,
# 'digitization_protocol_id' : digitization_protocol.id if digitization_protocol else None,
# 'commonterm_id' : commonterm.id if commonterm else None}
# protocol_cleaned = data_clean(protocol_dict)
# protocol = Protocol(**protocol_cleaned["kwargs"])
# db.session.add(protocol)
# db.session.commit()
''' Population '''
''' '''
invasive_status_study = InvasiveStatusStudy.query.filter_by(
status_name=data["population_invasive_status_study_id"]).first()
invasive_status_elsewhere = InvasiveStatusStudy.query.filter_by(
status_name=data["population_invasive_status_elsewhere_id"]).first()
ecoregion = Ecoregion.query.filter_by(
ecoregion_code=data["population_ecoregion_id"]).first()
continent = Continent.query.filter_by(
continent_name=data["population_continent_id"]).first()
###Danny trying add database meta-table in correct location
database = Database.query.filter_by(
database_master_version=data["population_database_id"]).first()
purpose_endangered = PurposeEndangered.query.filter_by(
purpose_name=data["study_purpose_endangered_id"]).first(
) if data["study_purpose_endangered_id"] else data[
"study_purpose_endangered_id"]
purpose_weed = PurposeWeed.query.filter_by(
purpose_name="study_purpose_weed_id").first(
) if data["study_purpose_weed_id"] else data["study_purpose_weed_id"]
database_source = Institute.query.filter_by(
institution_name=data["study_database_source_id"]).first()
pop = Population.query.filter_by(population_name=data["population_name"],
publication_id=publication.id,
species_id=species.id).first()
if pop == None:
pop_dict = {
'population_name':
data["population_name"],
'latitude':
data["population_latitude"],
'lat_ns':
data["lat_ns"],
'lat_deg':
data["lat_deg"],
'lat_min':
data["lat_min"],
'lat_sec':
data["lat_sec"],
'longitude':
data["population_longitude"],
'lon_ew':
data["lon_ew"],
'lon_deg':
data["lon_deg"],
'lon_min':
data["lon_min"],
'lon_sec':
data["lon_sec"],
'altitude':
data["population_altitude"],
#'pop_size' : data["population_pop_size"],
'country':
data["population_country"],
'invasive_status_study_id':
invasive_status_study.id if invasive_status_study else None,
'invasive_status_elsewhere_id':
invasive_status_elsewhere.id
if invasive_status_elsewhere else None,
'ecoregion':
ecoregion,
'continent':
continent,
'database':
database,
'within_site_replication':
data['population_within_site_replication'],
'study_duration':
data["study_duration"],
'study_start':
data["study_start"],
'study_end':
data["study_end"],
'number_populations':
data["study_number_populations"],
'purpose_endangered_id':
purpose_endangered.id if purpose_endangered else None,
'purpose_weed_id':
purpose_weed.id if purpose_weed else None,
'database_source':
database_source
}
pop_cleaned = data_clean(pop_dict)
# if not all(value == None for key, value in pop_cleaned["kwargs"].items() if key not in ignore_keys) and matrix_present:
pop = Population(**pop_cleaned["kwargs"])
db.session.add(pop)
db.session.commit()
pop.species_author = data["species_author"]
pop.publication_id = publication.id
pop.species_id = species.id
db.session.add(pop)
db.session.commit()
''' Population Version '''
#version = version_data(pop_cleaned)
population_version = Version(**version)
population_version.version_number = 1
population_version.population = pop
db.session.add(population_version)
db.session.commit()
population_version.original_version_id = population_version.id
db.session.add(population_version)
db.session.commit()
''' Matrix '''
treatment_string = data["matrix_treatment_id"]
if treatment_string == 'NDY':
treatment = 'NDY'
elif treatment_string == None:
treatment = None
else:
treatment = Treatment.query.filter_by(
treatment_name=data["matrix_treatment_id"]
).first() if Treatment.query.filter_by(
treatment_name=data["matrix_treatment_id"]).first() else Treatment(
treatment_name=data["matrix_treatment_id"])
db.session.add(treatment)
db.session.commit()
matrix_dict = {
'treatment':
treatment,
'matrix_split':
coerce_boolean(data["matrix_split"]),
'matrix_composition':
MatrixComposition.query.filter_by(
comp_name=data["matrix_composition_id"]).first(),
'matrix_criteria_size':
data["matrix_criteria_size"],
'matrix_criteria_ontogeny':
coerce_boolean(data["matrix_criteria_ontogeny"]),
'matrix_criteria_age':
coerce_boolean(data["matrix_criteria_age"]),
'matrix_start_month':
data["matrix_start_month"],
'matrix_end_month':
data["matrix_end_month"],
'matrix_start_year':
data["matrix_start_year"],
'matrix_end_year':
data["matrix_end_year"],
'studied_sex':
StudiedSex.query.filter_by(
sex_code=data["matrix_studied_sex_id"]).first(),
'start_season':
StartSeason.query.filter_by(
season_id=data["matrix_start_season_id"]).first()
if data["matrix_start_season_id"] else None,
'end_season':
EndSeason.query.filter_by(
season_id=data["matrix_end_season_id"]).first()
if data["matrix_end_season_id"] else None,
'matrix_fec':
coerce_boolean(data["matrix_fec"]),
'matrix_a_string':
data["matrix_a_string"],
'matrix_f_string':
data["matrix_f_string"],
'matrix_u_string':
data["matrix_u_string"],
'matrix_c_string':
data["matrix_c_string"],
'non_independence':
data["matrix_non_independence"],
'matrix_dimension':
data["matrix_dimension"],
'non_independence_author':
data["matrix_non_independence_author"],
'matrix_complete':
coerce_boolean(data["matrix_complete"]),
'class_number':
data["matrix_class_number"],
'observations':
data["matrix_observations"],
'captivities':
Captivity.query.filter_by(
cap_code=data["matrix_captivity_id"]).first(),
'class_author':
data["matrix_class_author"],
'class_organized':
data["matrix_class_organized"],
'matrix_difficulty':
data["matrix_difficulty"],
'seasonal':
coerce_boolean(data["matrix_seasonal"]),
'survival_issue':
calc_surv_issue(data["matrix_u_string"]),
'periodicity':
data["matrix_periodicity"],
'matrix_irreducible':
is_matrix_irreducible(data["matrix_a_string"]),
'matrix_primitive':
is_matrix_primitive(data["matrix_a_string"]),
'matrix_ergodic':
is_matrix_ergodic(data["matrix_a_string"]),
'matrix_lambda':
calc_lambda(data["matrix_a_string"])
}
matrix_cleaned = data_clean(matrix_dict)
# if not all(value == None for key, value in matrix_cleaned["kwargs"].items() if key not in ignore_keys):
matrix = Matrix(**matrix_cleaned["kwargs"])
db.session.add(matrix)
db.session.commit()
matrix.population_id = pop.id
db.session.add(matrix)
db.session.commit()
''' matrix Version '''
#version = version_data(matrix_cleaned)
matrix_version = Version(**version)
matrix_version.version_number = 1
matrix_version.matrix = matrix
db.session.add(matrix_version)
db.session.commit()
matrix_version.original_version_id = matrix_version.id
db.session.add(matrix_version)
db.session.commit()
''' Fixed '''
fixed = Fixed.query.filter_by(matrix=matrix).first()
if fixed == None:
fixed_dict = {
'matrix':
matrix,
'census_timings':
CensusTiming.query.filter_by(
census_name=data["fixed_census_timing_id"]).first(),
'seed_stage_error':
coerce_boolean(data["fixed_seed_stage_error"]),
'smalls':
Small.query.filter_by(small_name=data["fixed_small_id"]).first(),
'vector_str':
data["matrix_vectors_includes_na"]
}
fixed_cleaned = data_clean(fixed_dict)
fixed = Fixed(**fixed_cleaned["kwargs"])
db.session.add(fixed)
db.session.commit()
''' fixed Version '''
#version = version_data(fixed_cleaned)
fixed_version = Version(**version)
fixed_version.version_number = 1
fixed_version.fixed = fixed
db.session.add(fixed_version)
db.session.commit()
fixed_version.original_version_id = fixed_version.id
db.session.add(fixed_version)
db.session.commit()
def submit_new(data):
species = Species.query.filter_by(species_accepted=data["species_accepted"]).first()
if species == None:
species = Species(species_accepted=data["species_accepted"])
species.gbif_taxon_key = data["species_gbif_taxon_key"]
species.species_author = data["species_author"]
species.species_accepted = data["species_accepted"]
species.species_esa_status = ESAStatus.query.filter_by(status_code=data["species_esa_status_id"]).first()
species.species_common = data["species_common"]
species.species_iucn_status = IUCNStatus.query.filter_by(status_code=data["species_iucn_status_id"]).first()
db.session.add(species)
db.session.commit()
''' Species Version '''
species_version = Version()
species_version.version_number = 0
species_version.species = species
db.session.add(species_version)
db.session.commit()
species_version.version_of_id = species_version.id
species_version.checked = True
species_version.checked_count = 1
species_version.statuses = Status.query.filter_by(status_name="Green").first()
species_version.user = User.query.filter_by(username="******").first()
species_version.database = Database.query.filter_by(database_name="COMPADRE 4").first()
db.session.add(species_version)
db.session.commit()
''' Publication '''
if data["publication_DOI_ISBN"] == None:
publication = Publication.query.filter_by(authors=data["publication_authors"]).filter_by(year=data["publication_year"]).filter_by(name=data["publication_journal_name"]).first()
else:
publication = Publication.query.filter_by(DOI_ISBN=data["publication_DOI_ISBN"]).first()
if publication == None:
publication = Publication()
publication.authors = data["publication_authors"]
publication.year = data["publication_year"]
publication.DOI_ISBN = data["publication_DOI_ISBN"]
publication.additional_source_string = data["publication_additional_source_string"]
publication.name = data["publication_journal_name"]
db.session.add(publication)
db.session.commit()
''' Publication Version '''
publication_version = Version()
publication_version.version_number = 0
publication_version.publication = publication
db.session.add(publication_version)
db.session.commit()
publication_version.version_of_id = publication_version.id
publication_version.checked = True
publication_version.checked_count = 1
publication_version.statuses = Status.query.filter_by(status_name="Green").first()
publication_version.user = User.query.filter_by(username="******").first()
publication_version.database = Database.query.filter_by(database_name="COMPADRE 4").first()
db.session.add(publication_version)
db.session.commit()
''' Trait '''
spand_ex_growth_type = SpandExGrowthType.query.filter_by(type_name=data["trait_spand_ex_growth_type_id"]).first()
dicot_monoc = DicotMonoc.query.filter_by(dicot_monoc_name=data["trait_dicot_monoc_id"]).first()
growth_form_raunkiaer = GrowthFormRaunkiaer.query.filter_by(form_name=data["trait_growth_form_raunkiaer_id"]).first()
organism_type = OrganismType.query.filter_by(type_name=data["trait_organism_type_id"]).first()
angio_gymno = AngioGymno.query.filter_by(angio_gymno_name=data["trait_angio_gymno_id"]).first()
trait = Trait.query.filter_by(species_id=species.id).first()
if trait == None:
trait = Trait()
trait.species_id = species.id
trait.organism_type = organism_type
trait.dicot_monoc = dicot_monoc
trait.angio_gymno = angio_gymno
trait.spand_ex_growth_type = spand_ex_growth_type
trait.growth_form_raunkiaer = growth_form_raunkiaer
db.session.add(trait)
db.session.commit()
''' Trait Version '''
trait_version = Version()
trait_version.version_number = 0
trait_version.trait = trait
db.session.add(trait_version)
db.session.commit()
trait_version.version_of_id = trait_version.id
trait_version.checked = True
trait_version.checked_count = 1
trait_version.statuses = Status.query.filter_by(status_name="Green").first()
trait_version.user = User.query.filter_by(username="******").first()
trait_version.database = Database.query.filter_by(database_name="COMPADRE 4").first()
db.session.add(trait_version)
db.session.commit()
''' Study '''
# What if all none? Will they be grouped together?
purpose_endangered = PurposeEndangered.query.filter_by(purpose_name=data["study_purpose_endangered_id"]).first()
purpose_weed = PurposeWeed.query.filter_by(purpose_name="study_purpose_weed_id").first()
study = Study.query.filter_by(publication_id=publication.id, study_start=data["study_start"], study_end=data["study_end"]).first()
if study == None:
study = Study()
study.study_duration = data["study_duration"]
study.study_start = data["study_start"]
study.study_end = data["study_end"]
study.publication_id = publication.id
study.number_populations = data["study_number_populations"]
db.session.add(study)
db.session.commit()
''' Study Version '''
study_version = Version()
study_version.version_number = 0
study_version.study = study
db.session.add(study_version)
db.session.commit()
study_version.version_of_id = study_version.id
study_version.checked = True
study_version.checked_count = 1
study_version.statuses = Status.query.filter_by(status_name="Green").first()
study_version.user = User.query.filter_by(username="******").first()
study_version.database = Database.query.filter_by(database_name="COMPADRE 4").first()
db.session.add(study_version)
db.session.commit()
''' Population '''
invasive_status_study = InvasiveStatusStudy.query.filter_by(status_name=data["population_invasive_status_study_id"]).first()
invasive_status_elsewhere = InvasiveStatusStudy.query.filter_by(status_name=data["population_invasive_status_elsewhere_id"]).first()
ecoregion = Ecoregion.query.filter_by(ecoregion_code=data["population_ecoregion_id"]).first()
continent = Continent.query.filter_by(continent_name=data["population_continent_id"]).first()
pop = Population.query.filter_by(name=data["population_name"], species_id=species.id, publication_id=publication.id).first()
if pop == None:
pop = Population()
pop.species_author = data["species_author"]
pop.name = data["population_name"]
pop.species_id = species.id
pop.publication_id = publication.id
pop.study_id = study.id
pop.longitude = data["population_longitude"]
pop.latitude = data["population_latitude"]
pop.altitude = data["population_altitude"]
pop.pop_size = data["population_pop_size"]
pop.country = data["population_country"]
pop.invasive_status_study = invasive_status_study
pop.invasive_status_elsewhere = invasive_status_elsewhere
pop.ecoregion = ecoregion
pop.continent = continent
db.session.add(pop)
db.session.commit()
''' Population Version '''
population_version = Version()
population_version.version_number = 0
population_version.population = pop
db.session.add(population_version)
db.session.commit()
population_version.version_of_id = population_version.id
population_version.checked = True
population_version.checked_count = 1
population_version.statuses = Status.query.filter_by(status_name="Green").first()
population_version.user = User.query.filter_by(username="******").first()
population_version.database = Database.query.filter_by(database_name="COMPADRE 4").first()
db.session.add(population_version)
db.session.commit()
''' Taxonomy '''
tax = Taxonomy.query.filter_by(species_id=species.id).first()
if tax == None:
tax = Taxonomy()
tax.species_author = species.species_author
tax.species_accepted = species.species_accepted
tax.authority = None
tax.tpl_version = None
tax.infraspecies_accepted = None
tax.species_epithet_accepted = None
tax.genus_accepted = data["taxonomy_genus_accepted"]
tax.genus = data["taxonomy_genus"]
tax.family = data["taxonomy_family"]
tax.tax_order = data["taxonomy_order"]
tax.tax_class = data["taxonomy_class"]
tax.phylum = data["taxonomy_phylum"]
tax.kingdom = data["taxonomy_kingdom"]
tax.species = species
tax.publication = publication
tax.col_check_date = data["taxonomy_col_check_date"]
tax.col_check_ok = data["taxonomy_col_check_ok"]
db.session.add(tax)
db.session.commit()
''' Taxonomy Version '''
taxonomy_version = Version()
taxonomy_version.version_number = 0
taxonomy_version.taxonomy = tax
db.session.add(taxonomy_version)
db.session.commit()
taxonomy_version.version_of_id = taxonomy_version.id
taxonomy_version.checked = True
taxonomy_version.checked_count = 1
taxonomy_version.statuses = Status.query.filter_by(status_name="Green").first()
taxonomy_version.user = User.query.filter_by(username="******").first()
taxonomy_version.database = Database.query.filter_by(database_name="COMPADRE 4").first()
db.session.add(taxonomy_version)
db.session.commit()
''' Matrix '''
matrix = Matrix()
treatment = Treatment.query.filter_by(treatment_name=data["matrix_treatment_id"]).first()
if treatment == None:
treatment = Treatment(treatment_name=data["matrix_treatment_id"])
matrix.treatment = treatment
matrix.matrix_split = data["matrix_split"]
composition = MatrixComposition.query.filter_by(comp_name=data["matrix_composition_id"]).first()
matrix.matrix_composition = composition
matrix.survival_issue = data["matrix_survival_issue"]
matrix.periodicity = data["matrix_periodicity"]
matrix.matrix_criteria_size = data["matrix_criteria_size"]
matrix.matrix_criteria_ontogeny = coerce_boolean(data["matrix_criteria_ontogeny"])
matrix.matrix_criteria_age = coerce_boolean(data["matrix_criteria_age"])
matrix.matrix_start_month = data["matrix_start_month"]
matrix.matrix_end_month = data["matrix_end_month"]
matrix.matrix_start_year = data["matrix_start_year"]
matrix.matrix_end_year = data["matrix_end_year"]
matrix.studied_sex = StudiedSex.query.filter_by(sex_code=data["matrix_studied_sex_id"]).first()
start_season = StartSeason.query.filter_by(season_id=data["matrix_start_season_id"]).first()
end_season = EndSeason.query.filter_by(season_id=data["matrix_end_season_id"]).first()
if start_season != None:
matrix.start_season = start_season
if end_season != None:
matrix.end_season = end_season
matrix.matrix_fec = coerce_boolean(data["matrix_fec"])
matrix.matrix_a_string = data["matrix_a_string"]
matrix.matrix_f_string = data["matrix_f_string"]
matrix.matrix_u_string = data["matrix_u_string"]
matrix.matrix_c_string = data["matrix_c_string"]
matrix.non_independence = data["matrix_non_independence"]
matrix.dimension = data["matrix_dimension"]
matrix.non_independence_author = data["matrix_non_independence_author"]
matrix.matrix_complete = data["matrix_complete"]
matrix.vectors_includes_na = data["matrix_vectors_includes_na"]
matrix.class_number = data["matrix_class_number"]
matrix.observations = data["matrix_observations"]
captivities = Captivity.query.filter_by(cap_code=data["matrix_captivity_id"]).first()
matrix.captivities = captivities
matrix.class_author = data["matrix_class_author"]
matrix.matrix_difficulty = data["matrix_difficulty"]
matrix.independent = data["matrix_independent"]
matrix.seasonal = data["matrix_seasonal"]
matrix.uid = generate_uid(species, publication, pop, matrix)
matrix.population = pop
matrix.study = study
db.session.add(matrix)
db.session.commit()
''' matrix Version '''
matrix_version = Version()
matrix_version.version_number = 0
matrix_version.matrix = matrix
db.session.add(matrix_version)
db.session.commit()
matrix_version.version_of_id = matrix_version.id
matrix_version.checked = True
matrix_version.checked_count = 1
matrix_version.statuses = Status.query.filter_by(status_name="Green").first()
matrix_version.user = User.query.filter_by(username="******").first()
matrix_version.database = Database.query.filter_by(database_name="COMPADRE 4").first()
db.session.add(matrix_version)
db.session.commit()
''' Fixed '''
fixed = Fixed.query.filter_by(matrix=matrix).first()
if fixed == None:
fixed = Fixed()
fixed.matrix = matrix
fixed.census_timings = CensusTiming.query.filter_by(census_name=data["fixed_census_timing_id"]).first()
fixed.seed_stage_error = data["fixed_seed_stage_error"]
fixed.smalls = Small.query.filter_by(small_name=data["fixed_small_id"]).first()
db.session.add(fixed)
db.session.commit()
''' fixed Version '''
fixed_version = Version()
fixed_version.version_number = 0
fixed_version.fixed = fixed
db.session.add(fixed_version)
db.session.commit()
fixed_version.version_of_id = fixed_version.id
fixed_version.checked = True
fixed_version.checked_count = 1
fixed_version.statuses = Status.query.filter_by(status_name="Green").first()
fixed_version.user = User.query.filter_by(username="******").first()
fixed_version.database = Database.query.filter_by(database_name="COMPADRE 4").first()
db.session.add(fixed_version)
db.session.commit()
import csv
from app.models import Species, Traits
dataReader = csv.reader(open("C:/LifeHistory/Lifehistory/Seabirds-edited.csv"), delimiter=',', quotechar='"')
count = 1
for row in dataReader:
if row[0] != 'order':
print row
species = Species()
species.order=row[0]
species.family=row[1]
species.genus=row[2]
species.species=row[3]
species.species_id=row[4]
species.synonyms=row[5]
species.common_name_1=row[6]
species.common_name_2=row[7]
species.common_name_3=row[8]
species.common_name_4=row[9]
species.common_name_5=row[10]
species.common_name_6=row[11]
species.iucn_status=row[12]
species.red_list_criteria=row[13]
species.year_assessed=row[14]
species.population_trend=row[15]
species.breeding_dist=row[24]
species.nest_locations=row[25]
species.hatchling_type=row[31]
species.foraging_distance=row[52]
species.coloniality=row[65]
import csv
from app.models import Species, Traits
dataReader = csv.reader(open("C:/LifeHistory/Lifehistory/Seabirds-edited.csv"),
delimiter=',',
quotechar='"')
count = 1
for row in dataReader:
if row[0] != 'order':
print row
species = Species()
species.order = row[0]
species.family = row[1]
species.genus = row[2]
species.species = row[3]
species.species_id = row[4]
species.synonyms = row[5]
species.common_name_1 = row[6]
species.common_name_2 = row[7]
species.common_name_3 = row[8]
species.common_name_4 = row[9]
species.common_name_5 = row[10]
species.common_name_6 = row[11]
species.iucn_status = row[12]
species.red_list_criteria = row[13]
species.year_assessed = row[14]
species.population_trend = row[15]
species.breeding_dist = row[24]
species.nest_locations = row[25]
species.hatchling_type = row[31]