示例#1
0
    def download(self, engine=None, debug=False):
        Script.download(self, engine, debug)

        # structure_plot_year table
        self.engine.auto_create_table(Table("structure_plot_year"), url=self.urls["structure_plot_year"])
        self.engine.insert_data_from_url(self.urls["structure_plot_year"])

        # structure_plot_year table
        self.engine.auto_create_table(Table("plots"), url=self.urls["plots"])
        self.engine.insert_data_from_url(self.urls["plots"])

        # species table
        self.engine.download_file(self.urls["species"], "original_MSH_SPECIES_DESCRIPTORS.csv")
        data_path = self.engine.format_filename("MSH_SPECIES_DESCRIPTORS.csv")

        old_data = os.path.normpath(self.engine.find_file("original_MSH_SPECIES_DESCRIPTORS.csv"))

        with open(old_data, 'rU') as infile, open(data_path, 'w')as new_data:
            for line in infile:
                line = str(line).encode('utf-8')
                new_data.write(line)
        infile.close()
        new_data.close()

        self.engine.auto_create_table(Table("species"),
                                      filename="MSH_SPECIES_DESCRIPTORS.csv")
        self.engine.insert_data_from_file(data_path)

        # species_plot_year tables
        table = Table("species_plot_year")
        table.delimiter = ','
        table.columns = [
            ('record_id', ('pk-auto',)),
            ('plot_id_year', ('char',)),
            ('plot_name', ('char',)),
            ('plot_number', ('int',)),
            ('year', ('int',)),
            ('species', ('ct_column',)),
            ('count', ('ct-double',))
        ]

        table.ct_column = 'species'
        table.ct_names = ['Abilas', 'Abipro', 'Achmil', 'Achocc', 'Agoaur', 'Agrexa', 'Agrpal', 'Agrsca', 'Alnvir',
                          'Anamar', 'Antmic', 'Antros', 'Aqifor', 'Arcnev', 'Arnlat', 'Astled', 'Athdis', 'Blespi',
                          'Brocar', 'Brosit', 'Carmer', 'Carmic', 'Carpac', 'Carpay', 'Carpha', 'Carros', 'Carspe',
                          'Casmin', 'Chaang', 'Cirarv', 'Cisumb', 'Crycas', 'Danint', 'Descae', 'Elyely', 'Epiana',
                          'Eriova', 'Eripyr', 'Fesocc', 'Fravir', 'Gencal', 'Hiealb', 'Hiegra', 'Hyprad', 'Junmer',
                          'Junpar', 'Juncom', 'Leppun', 'Lommar', 'Luepec', 'Luihyp', 'Luplat', 'Luplep', 'Luzpar',
                          'Maiste', 'Pencar', 'Pencon', 'Penser', 'Phahas', 'Phlalp', 'Phldif', 'Phyemp', 'Pincon',
                          'Poasec', 'Poldav', 'Polmin', 'Pollon', 'Poljun', 'Popbal', 'Potarg', 'Psemen', 'Raccan',
                          'Rumace', 'Salsit', 'Saxfer', 'Senspp', 'Sibpro', 'Sorsit', 'Spiden', 'Trispi', 'Tsumer',
                          'Vacmem', 'Vervir', 'Vioadu', 'Xerten']

        self.engine.table = table
        self.engine.create_table()
        self.engine.insert_data_from_url(self.urls["species_plot_year"])
    def download(self, engine=None, debug=False):
        Script.download(self, engine, debug)
        engine = self.engine
        files = ["Macroplot_data_Rev.txt", "Microplot_data.txt", "Site_variables.txt", "Species_list.txt"]
        engine.download_files_from_archive(self.urls["data"], files, filetype="zip")

        # Create table species
        engine.auto_create_table(Table('species', cleanup=Cleanup(correct_invalid_value, nulls=['NA'])),
                                 filename="Species_list.txt")
        engine.insert_data_from_file(engine.format_filename("Species_list.txt"))

        # Create table sites
        engine.auto_create_table(Table('sites', cleanup=Cleanup(correct_invalid_value, nulls=['NA'])),
                                 filename="Site_variables.txt")
        engine.insert_data_from_file(engine.format_filename("Site_variables.txt"))

        # Create table microplots
        table = Table('microplots')
        table.columns = [('record_id', ('pk-auto',)), ('SpCode', ('char', '30')), ('Count', ('ct-int',))]
        table.ct_names = ['BSP1', 'BSP2', 'BSP3', 'BSP4', 'BSP5', 'BSP6', 'BSP7', 'BSP8', 'BSP9',
                          'BSP10', 'BSP11', 'BSP12', 'BSP13', 'BSP14', 'BSP15', 'BSP16', 'BSP17',
                          'BSP18', 'BSP20', 'BSP21', 'BSP22', 'BSP23', 'BSP24', 'BSP25', 'BSP26',
                          'BSP27', 'BSP28', 'BSP29', 'BSP30', 'BSP31', 'BSP33', 'BSP34', 'BSP35',
                          'BSP36', 'BSP37', 'BSP41', 'BSP42', 'BSP43', 'BSP44', 'BSP45', 'BSP46',
                          'BSP47', 'BSP48', 'BSP49', 'BSP50', 'BSP51', 'BSP52', 'BSP53', 'BSP54',
                          'BSP55', 'BSP56', 'BSP57', 'BSP58', 'BSP59', 'BSP60', 'BSP61', 'BSP62',
                          'BSP63', 'BSP64', 'BSP65', 'BSP66', 'BSP67', 'BSP68', 'BSP69', 'BSP70',
                          'BSP71', 'BSP72', 'BSP73', 'BSP74', 'BSP75', 'BSP76', 'BSP78', 'BSP79',
                          'BSP80', 'BSP82', 'BSP83', 'BSP84', 'BSP85', 'BSP86', 'BSP87', 'BSP88',
                          'BSP89', 'BSP90', 'BSP91', 'BSP92', 'BSP93', 'BSP94', 'BSP95', 'BSP96',
                          'BSP97', 'BSP98', 'BSP99', 'BSP100', 'BSP101', 'BSP102', 'BSP104']
        table.ct_column = 'PlotID'
        engine.auto_create_table(table, filename="Microplot_data.txt")
        engine.insert_data_from_file(engine.format_filename("Microplot_data.txt"))

        # Create table microplots
        table = Table('macroplots')
        table.ct_names = ['TreeGirth1', 'TreeGirth2', 'TreeGirth3', 'TreeGirth4', 'TreeGirth5']
        table.ct_column = 'Tree'
        table.columns = [('record_id', ('pk-auto',)), ('PlotID', ('char', '20')), ('SpCode', ('char', '30')),
                         ('Girth', ('ct-int',))]
        engine.auto_create_table(table, filename="Macroplot_data_Rev.txt")
        engine.insert_data_from_file(engine.format_filename("Macroplot_data_Rev.txt"))
示例#3
0
    def download(self, engine=None, debug=False):
        Script.download(self, engine, debug)
        engine = self.engine
        files = ["Macroplot_data_Rev.txt", "Microplot_data.txt", "Site_variables.txt", "Species_list.txt"]
        engine.download_files_from_archive(self.urls["data"], files, filetype="zip")

        # Create table species
        engine.auto_create_table(Table('species', cleanup=self.cleanup_func_table),
                                 filename="Species_list.txt")
        engine.insert_data_from_file(engine.format_filename("Species_list.txt"))

        # Create table sites
        engine.auto_create_table(Table('sites', cleanup=self.cleanup_func_table),
                                 filename="Site_variables.txt")
        engine.insert_data_from_file(engine.format_filename("Site_variables.txt"))

        # Create table microplots
        table = Table('microplots')
        table.columns = [('record_id', ('pk-auto',)), ('SpCode', ('char', '30')), ('Count', ('ct-int',))]
        table.ct_names = ['BSP1', 'BSP2', 'BSP3', 'BSP4', 'BSP5', 'BSP6', 'BSP7', 'BSP8', 'BSP9',
                          'BSP10', 'BSP11', 'BSP12', 'BSP13', 'BSP14', 'BSP15', 'BSP16', 'BSP17',
                          'BSP18', 'BSP20', 'BSP21', 'BSP22', 'BSP23', 'BSP24', 'BSP25', 'BSP26',
                          'BSP27', 'BSP28', 'BSP29', 'BSP30', 'BSP31', 'BSP33', 'BSP34', 'BSP35',
                          'BSP36', 'BSP37', 'BSP41', 'BSP42', 'BSP43', 'BSP44', 'BSP45', 'BSP46',
                          'BSP47', 'BSP48', 'BSP49', 'BSP50', 'BSP51', 'BSP52', 'BSP53', 'BSP54',
                          'BSP55', 'BSP56', 'BSP57', 'BSP58', 'BSP59', 'BSP60', 'BSP61', 'BSP62',
                          'BSP63', 'BSP64', 'BSP65', 'BSP66', 'BSP67', 'BSP68', 'BSP69', 'BSP70',
                          'BSP71', 'BSP72', 'BSP73', 'BSP74', 'BSP75', 'BSP76', 'BSP78', 'BSP79',
                          'BSP80', 'BSP82', 'BSP83', 'BSP84', 'BSP85', 'BSP86', 'BSP87', 'BSP88',
                          'BSP89', 'BSP90', 'BSP91', 'BSP92', 'BSP93', 'BSP94', 'BSP95', 'BSP96',
                          'BSP97', 'BSP98', 'BSP99', 'BSP100', 'BSP101', 'BSP102', 'BSP104']
        table.ct_column = 'PlotID'
        engine.auto_create_table(table, filename="Microplot_data.txt")
        engine.insert_data_from_file(engine.format_filename("Microplot_data.txt"))

        # Create table microplots
        table = Table('macroplots')
        table.ct_names = ['TreeGirth1', 'TreeGirth2', 'TreeGirth3', 'TreeGirth4', 'TreeGirth5']
        table.ct_column = 'Tree'
        table.columns = [('record_id', ('pk-auto',)), ('PlotID', ('char', '20')), ('SpCode', ('char', '30')),
                         ('Girth', ('ct-int',))]
        engine.auto_create_table(table, filename="Macroplot_data_Rev.txt")
        engine.insert_data_from_file(engine.format_filename("Macroplot_data_Rev.txt"))
示例#4
0
    def download(self, engine=None, debug=False):
        Script.download(self, engine, debug)
        engine = self.engine
        engine.download_files_from_archive(self.urls["data"], ["Data_Files/Amniote_Database_Aug_2015.csv",
                                                               "Data_Files/Amniote_Database_References_Aug_2015.csv",
                                                               "Data_Files/Amniote_Range_Count_Aug_2015.csv"],
                                           filetype="zip")

        ct_column = 'trait'  # all tables use the same ct_column name

        # Create tables from Amniote_Database_Aug.csv and Amniote_Database_References_Aug_2015.csv
        # Both reference and main have the same headers

        ct_names = ['female_maturity_d', 'litter_or_clutch_size_n', 'litters_or_clutches_per_y', 'adult_body_mass_g',
                    'maximum_longevity_y', 'gestation_d', 'weaning_d', 'birth_or_hatching_weight_g', 'weaning_weight_g',
                    'egg_mass_g', 'incubation_d', 'fledging_age_d', 'longevity_y', 'male_maturity_d',
                    'inter_litter_or_interbirth_interval_y', 'female_body_mass_g', 'male_body_mass_g',
                    'no_sex_body_mass_g', 'egg_width_mm', 'egg_length_mm', 'fledging_mass_g', 'adult_svl_cm',
                    'male_svl_cm', 'female_svl_cm', 'birth_or_hatching_svl_cm', 'female_svl_at_maturity_cm',
                    'female_body_mass_at_maturity_g', 'no_sex_svl_cm', 'no_sex_maturity_d']

        # Create table main from Amniote_Database_Aug_2015.csv

        columns = [
            ('record_id', ('pk-auto',)), ('class', ('char', '20')), ('order', ('char', '20')),
            ('family', ('char', '20')), ('genus', ('char', '20')), ('species', ('char', '50')),
            ('subspecies', ('char', '20')), ('common_name', ('char', '400')), ('trait_value', ('ct-double',))]
        table_main = Table('main', delimiter=',', cleanup=self.cleanup_func_table)
        table_main.ct_column = ct_column
        table_main.ct_names = ct_names
        table_main.columns = columns
        engine.auto_create_table(table_main,
                                 filename="Amniote_Database_Aug_2015.csv")
        engine.insert_data_from_file(engine.format_filename("Amniote_Database_Aug_2015.csv"))

        # Create table reference from Amniote_Database_References_Aug_2015.csv
        reference_columns = [
            ('record_id', ('pk-auto',)), ('class', ('char', '20')), ('order', ('char', '20')),
            ('family', ('char', '20')), ('genus', ('char', '20')), ('species', ('char', '50')),
            ('subspecies', ('char', '20')), ('common_name', ('char', '400')), ('reference', ('ct-char',))]

        table_references = Table('references', delimiter=',', cleanup=self.cleanup_func_table)
        table_references.ct_column = ct_column
        table_references.ct_names = ct_names
        table_references.columns = reference_columns
        engine.auto_create_table(table_references,
                                 filename="Amniote_Database_References_Aug_2015.csv")
        engine.insert_data_from_file(engine.format_filename("Amniote_Database_References_Aug_2015.csv"))

        # Create table Range
        # This table has different values for headers from the above tables.
        range_ct_names = ["min_female_maturity", "max_female_maturity", "count_female_maturity", "min_litter_clutch_size",
                    "max_litter_clutch_size", "count_litter_clutch_size", "min_litters_clutches",
                    "max_litters_clutches", "count_litters_clutches", "min_adult_body_mass", "max_adult_body_mass",
                    "count_adult_body_mass", "min_maximum_longevity", "max_maximum_longevity",
                    "count_maximum_longevity", "min_gestation", "max_gestation", "count_gestation", "min_weaning",
                    "max_weaning", "count_weaning", "min_birth_hatching_weight", "max_birth_hatching_weight",
                    "count_birth_hatching_weight", "min_weaning_weight", "max_weaning_weight", "count_weaning_weight",
                    "min_egg_mass", "max_egg_mass", "count_egg_mass", "min_incubation", "max_incubation",
                    "count_incubation", "min_fledging_age", "max_fledging_age", "count_fledging_age",
                    "min_male_maturity", "max_male_maturity", "count_male_maturity",
                    "min_inter_litter_interbirth_interval", "max_inter_litter_interbirth_interval",
                    "count_inter_litter_interbirth_interval", "min_female_body_mass", "max_female_body_mass",
                    "count_female_body_mass", "min_male_body_mass", "max_male_body_mass", "count_male_body_mass",
                    "min_no_sex_body_mass", "max_no_sex_body_mass", "count_no_sex_body_mass", "min_egg_width",
                    "max_egg_width", "count_egg_width", "min_egg_length", "max_egg_length", "count_egg_length",
                    "min_fledging_mass", "max_fledging_mass", "count_fledging_mass", "min_adult_svl", "max_adult_svl",
                    "count_adult_svl", "min_male_svl", "max_male_svl", "count_male_svl", "min_female_svl",
                    "max_female_svl", "count_female_svl", "min_hatching_svl", "max_hatching_svl", "count_hatching_svl",
                    "min_female_svl_at_maturity", "max_female_svl_at_maturity", "count_female_svl_at_maturity",
                    "min_female_body_mass_at_maturity", "max_female_body_mass_at_maturity",
                    "count_female_body_mass_at_maturity", "min_no_sex_svl", "max_no_sex_svl", "count_no_sex_svl",
                    "min_no_sex_maturity", "max_no_sex_maturity", "count_no_sex_maturity"]
        range_columns = [
            ('record_id', ('pk-auto',)), ('classx', ('char', '20')), ('orderx', ('char', '20')),
            ('familyx', ('char', '20')), ('genus', ('char', '20')), ('species', ('char', '50')),
            ('subspecies', ('char', '20')), ('common_name', ('char', '400')), ('trait_value', ('ct-double',))]

        table_range = Table('range', delimiter=',', cleanup=self.cleanup_func_table)
        table_range.ct_column = ct_column
        table_range.ct_names = range_ct_names
        table_range.columns = range_columns
        engine.auto_create_table(table_range,
                                 filename="Amniote_Range_Count_Aug_2015.csv")
        engine.insert_data_from_file(engine.format_filename("Amniote_Range_Count_Aug_2015.csv"))