def __init__(self, args = None): self.args = args self.in_marker = "vamps2" self.out_marker = "vampsdev" self.local_marker = "all_local" self.table_names = const.table_names_dict[self.in_marker] self.db_info_dict = self.get_db_info_dict() self.mysql_utils_in = util.Mysql_util(host = self.db_info_dict["host_in"], db = self.db_info_dict["db_in"], read_default_group = self.db_info_dict["read_default_group"]) self.mysql_utils_out = util.Mysql_util(host = self.db_info_dict["host_out"], db = self.db_info_dict["db_out"], read_default_group = self.db_info_dict["read_default_group"])
def __init__(self, metadata=None): """ 1) upload simple tables (table_names_simple) 2) upload combine tables no foreign keys (content, person, place, season, source) 3) get ids 4) upload tables with ids """ self.utils = util.Utils() self.metadata = metadata if self.utils.is_local(): self.db_schema = 'mcm_history' self.mysql_utils = util.Mysql_util(host='localhost', db=self.db_schema, read_default_group='clienthome') self.utils.print_both("host = 'localhost', db = {}".format( self.db_schema)) else: self.db_schema = 'mcmurdohistory_metadata' host = '127.0.0.1' self.mysql_utils = util.Mysql_util(host=host, db=self.db_schema, read_default_group='client') # self.mysql_utils = util.Mysql_util(host = 'taylor.unm.edu', db = self.db_schema, read_default_group = 'client') self.utils.print_both("host = {}, db {}".format( host, self.db_schema)) self.entry_table_name = self.table_names_w_ids[0] all_tables_sql_res = self.mysql_utils.get_table_names(self.db_schema) self.all_tables_set = set(self.utils.extract(all_tables_sql_res[0])) self.many_values_to_one_field_column_names = self.utils.flatten_2d_list( self.many_values_to_one_field.values()) self.special_tables = self.get_special_tables() self.simple_tables = list(self.all_tables_set - set(self.special_tables)) self.upload_empty() self.utils.print_both("End of Upload superclass")
def benchmark_w_return_2(self, t0): t1 = time.time() total = float(t1 - t0) / 60 print('time: %.2f m' % total) # # print("time_res = %s s" % total) if __name__ == '__main__': utils = util.Utils() if (utils.is_local() == True): # mysql_utils = util.Mysql_util(host = "vampsdev", db = "vamps", read_default_group = "clientservers") mysql_utils = util.Mysql_util(host="localhost", db="test_vamps", read_default_group="clienthome") else: mysql_utils = util.Mysql_util(host="vampsdb", db="vamps", read_default_group="client") csv_dir = "/usr/local/tmp" in_filename = "rep_id_refhvr_ids" file_extension = ".csv" out_file_extension = ".separated" # query = "show tables" # a = mysql_utils.execute_fetch_select(query) update_refhvr_ids = Update_refhvr_ids()
# print id_taxon_list # {'domain': [(1L, 'Bacteria')], 'family': [(1L, 'empty_family'), (2L, 'Acidimicrobiaceae')], 'species': [(1L, 'empty_species')], 'phylum': [(1L, 'empty_phylum'), (2L, 'Actinobacteria')], 'klass': [(1L, 'empty_klass'), (2L, 'Actinobacteria')], 'genus': [(1L, 'empty_genus'), (2L, 'Acidimicrobium'), (3L, 'Ilumatobacter')], 'order': [(1L, 'Acidimicrobiales'), (2L, 'empty_order')]} # (), # see insert_tax(self) # ... # for locus, tax_arr in self.taxonomy_sorted.items(): # tax_arr: [('domain', 'Bacteria'), ('phylum', '"Actinobacteria"'), ('klass', 'Actinobacteria'), ('order', 'Acidimicrobiales'), ('family', 'Acidimicrobiaceae'), ('genus', 'Acidimicrobium')] if __name__ == '__main__': utils = util.Utils() if (utils.is_local() == True): mysql_utils = util.Mysql_util(host="localhost", db="spingo_rdp", read_default_group="clienthome") else: mysql_utils = util.Mysql_util(host="bpcweb7.bpcservers.private", db="test", read_default_group="client") # query = "show tables" # a = mysql_utils.execute_fetch_select(query) # print a # ((('spingo_rdp',), ('spingo_rdp_sequence',)), ['Tables_in_spingo_rdp']) parser = Parse_RDP() # todo: ARGS if (utils.is_local() == True): in_fa_gz_file_name = "/Users/ashipunova/Dropbox/mix/today_ch/spingo_assign/small_current_bact.fa"
print "total seq originally: " print self.all_cnt_orig for counter in range(1, int(self.all_cnt_orig[0][0][0]), chunk_size): my_sql = """insert ignore into sequence (sequence) select distinct uncompress(sequence_comp) as sequence from sequence_ill limit %s, %s ON DUPLICATE KEY UPDATE sequence = VALUES(sequence) """ % (counter, chunk_size) print my_sql print mysql_utils.execute_no_fetch_w_info(my_sql) if __name__ == '__main__': utils = util.Utils() if (utils.is_local() == True): host = 'localhost' db = 'test_env454' read_default_group = 'clienthome' else: host = "bpcdb1" db = 'env454' read_default_group = 'client' mysql_utils = util.Mysql_util(host=host, db=db, read_default_group=read_default_group) # print "host = %s, db = %s" % (host, db) seq = Sequence() seq.uncompress_all()
# INSERT INTO custom_metadata_fields (project_id, field_name, field_units, example) VALUES () ## UPDATE custom_metadata_fields SET field_name = "temperature" WHERE field_name = "temp" AND project_id = "$1"; # custom 3 # UPDATE custom_metadata_110 # SET env_biome_sec = "subseafloor aquatic biome"; if __name__ == '__main__': # ~/BPC/python-scripts$ python upload_dco_metadata_csv_to_vamps.py -f /Users/ashipunova/BPC/vamps-node.js/user_data/vamps2/AnnaSh/metadata-project_DCO_GAI_Bv3v5_AnnaSh_1500930353039.csv utils = util.Utils() if utils.is_local() == True: mysql_utils = util.Mysql_util(host='localhost', db='vamps2', read_default_group='clienthome') print("host = 'localhost', db = 'vamps2'") else: mysql_utils = util.Mysql_util(host='vampsdb', db='vamps2', read_default_group='client') # mysql_utils = util.Mysql_util(host = 'vampsdev', db = 'vamps2', read_default_group = 'client') # print("host = 'vampsdev', db = 'vamps2'") parser = argparse.ArgumentParser() parser.add_argument('-f', '--file_name', required=True, action='store',
""" print query vampsdev_vamps_mysql_util.execute_no_fetch(query) def insert_into_rep_id_refhvr_id(self): separate_refids_string = str(self.separate_refids_arr).strip('[]') query = """ INSERT IGNORE INTO rep_id_refhvr_id (rep_id, refhvr_id) VALUES %s; """ % separate_refids_string print query vampsdev_vamps_mysql_util.execute_no_fetch(query) if __name__ == '__main__': vampsdev_vamps_mysql_util = util.Mysql_util(host = "vampsdev", db = "vamps", read_default_group = "clientservers") query = "show tables" a = vampsdev_vamps_mysql_util.execute_fetch_select(query) update_refhvr_ids = Update_refhvr_ids() # print "AAA" # !!! Uncomment !!! # update_refhvr_ids.drop_table("refids_per_dataset_temp") # update_refhvr_ids.create_table_refids_per_dataset_temp() # update_refhvr_ids.insert_refids_per_dataset_temp() # update_refhvr_ids.get_dataset_id() # update_refhvr_ids.get_project_id() # update_refhvr_ids.insert_into_refids_per_dataset() res, field_names = update_refhvr_ids.get_rep_id_refhvr_ids() # print field_names update_refhvr_ids.separate_refids(update_refhvr_ids.make_dictionary_from_res(res))