def __init__(self, args = None):
self.args = args
self.in_marker = "vamps2"
self.out_marker = "vampsdev"
self.local_marker = "all_local"
self.table_names = const.table_names_dict[self.in_marker]
self.db_info_dict = self.get_db_info_dict()
self.mysql_utils_in = util.Mysql_util(host = self.db_info_dict["host_in"],
db = self.db_info_dict["db_in"],
read_default_group = self.db_info_dict["read_default_group"])
self.mysql_utils_out = util.Mysql_util(host = self.db_info_dict["host_out"],
db = self.db_info_dict["db_out"],
read_default_group = self.db_info_dict["read_default_group"])
def __init__(self, metadata=None):
"""
1) upload simple tables (table_names_simple)
2) upload combine tables no foreign keys (content, person, place, season, source)
3) get ids
4) upload tables with ids
"""
self.utils = util.Utils()
self.metadata = metadata
if self.utils.is_local():
self.db_schema = 'mcm_history'
self.mysql_utils = util.Mysql_util(host='localhost',
db=self.db_schema,
read_default_group='clienthome')
self.utils.print_both("host = 'localhost', db = {}".format(
self.db_schema))
else:
self.db_schema = 'mcmurdohistory_metadata'
host = '127.0.0.1'
self.mysql_utils = util.Mysql_util(host=host,
db=self.db_schema,
read_default_group='client')
# self.mysql_utils = util.Mysql_util(host = 'taylor.unm.edu', db = self.db_schema, read_default_group = 'client')
self.utils.print_both("host = {}, db {}".format(
host, self.db_schema))
self.entry_table_name = self.table_names_w_ids[0]
all_tables_sql_res = self.mysql_utils.get_table_names(self.db_schema)
self.all_tables_set = set(self.utils.extract(all_tables_sql_res[0]))
self.many_values_to_one_field_column_names = self.utils.flatten_2d_list(
self.many_values_to_one_field.values())
self.special_tables = self.get_special_tables()
self.simple_tables = list(self.all_tables_set -
set(self.special_tables))
self.upload_empty()
self.utils.print_both("End of Upload superclass")
def benchmark_w_return_2(self, t0):
t1 = time.time()
total = float(t1 - t0) / 60
print('time: %.2f m' % total)
#
# print("time_res = %s s" % total)
if __name__ == '__main__':
utils = util.Utils()
if (utils.is_local() == True):
# mysql_utils = util.Mysql_util(host = "vampsdev", db = "vamps", read_default_group = "clientservers")
mysql_utils = util.Mysql_util(host="localhost",
db="test_vamps",
read_default_group="clienthome")
else:
mysql_utils = util.Mysql_util(host="vampsdb",
db="vamps",
read_default_group="client")
csv_dir = "/usr/local/tmp"
in_filename = "rep_id_refhvr_ids"
file_extension = ".csv"
out_file_extension = ".separated"
# query = "show tables"
# a = mysql_utils.execute_fetch_select(query)
update_refhvr_ids = Update_refhvr_ids()
# print id_taxon_list
# {'domain': [(1L, 'Bacteria')], 'family': [(1L, 'empty_family'), (2L, 'Acidimicrobiaceae')], 'species': [(1L, 'empty_species')], 'phylum': [(1L, 'empty_phylum'), (2L, 'Actinobacteria')], 'klass': [(1L, 'empty_klass'), (2L, 'Actinobacteria')], 'genus': [(1L, 'empty_genus'), (2L, 'Acidimicrobium'), (3L, 'Ilumatobacter')], 'order': [(1L, 'Acidimicrobiales'), (2L, 'empty_order')]}
# (),
# see insert_tax(self)
# ...
# for locus, tax_arr in self.taxonomy_sorted.items():
# tax_arr: [('domain', 'Bacteria'), ('phylum', '"Actinobacteria"'), ('klass', 'Actinobacteria'), ('order', 'Acidimicrobiales'), ('family', 'Acidimicrobiaceae'), ('genus', 'Acidimicrobium')]
if __name__ == '__main__':
utils = util.Utils()
if (utils.is_local() == True):
mysql_utils = util.Mysql_util(host="localhost",
db="spingo_rdp",
read_default_group="clienthome")
else:
mysql_utils = util.Mysql_util(host="bpcweb7.bpcservers.private",
db="test",
read_default_group="client")
# query = "show tables"
# a = mysql_utils.execute_fetch_select(query)
# print a
# ((('spingo_rdp',), ('spingo_rdp_sequence',)), ['Tables_in_spingo_rdp'])
parser = Parse_RDP()
# todo: ARGS
if (utils.is_local() == True):
in_fa_gz_file_name = "/Users/ashipunova/Dropbox/mix/today_ch/spingo_assign/small_current_bact.fa"
print "total seq originally: "
print self.all_cnt_orig
for counter in range(1, int(self.all_cnt_orig[0][0][0]), chunk_size):
my_sql = """insert ignore into sequence (sequence)
select distinct uncompress(sequence_comp) as sequence
from sequence_ill limit %s, %s
ON DUPLICATE KEY UPDATE sequence = VALUES(sequence)
""" % (counter, chunk_size)
print my_sql
print mysql_utils.execute_no_fetch_w_info(my_sql)
if __name__ == '__main__':
utils = util.Utils()
if (utils.is_local() == True):
host = 'localhost'
db = 'test_env454'
read_default_group = 'clienthome'
else:
host = "bpcdb1"
db = 'env454'
read_default_group = 'client'
mysql_utils = util.Mysql_util(host=host,
db=db,
read_default_group=read_default_group)
# print "host = %s, db = %s" % (host, db)
seq = Sequence()
seq.uncompress_all()
# INSERT INTO custom_metadata_fields (project_id, field_name, field_units, example) VALUES ()
## UPDATE custom_metadata_fields SET field_name = "temperature" WHERE field_name = "temp" AND project_id = "$1";
# custom 3
# UPDATE custom_metadata_110
# SET env_biome_sec = "subseafloor aquatic biome";
if __name__ == '__main__':
# ~/BPC/python-scripts$ python upload_dco_metadata_csv_to_vamps.py -f /Users/ashipunova/BPC/vamps-node.js/user_data/vamps2/AnnaSh/metadata-project_DCO_GAI_Bv3v5_AnnaSh_1500930353039.csv
utils = util.Utils()
if utils.is_local() == True:
mysql_utils = util.Mysql_util(host='localhost',
db='vamps2',
read_default_group='clienthome')
print("host = 'localhost', db = 'vamps2'")
else:
mysql_utils = util.Mysql_util(host='vampsdb',
db='vamps2',
read_default_group='client')
# mysql_utils = util.Mysql_util(host = 'vampsdev', db = 'vamps2', read_default_group = 'client')
# print("host = 'vampsdev', db = 'vamps2'")
parser = argparse.ArgumentParser()
parser.add_argument('-f',
'--file_name',
required=True,
action='store',
"""
print query
vampsdev_vamps_mysql_util.execute_no_fetch(query)
def insert_into_rep_id_refhvr_id(self):
separate_refids_string = str(self.separate_refids_arr).strip('[]')
query = """
INSERT IGNORE INTO rep_id_refhvr_id (rep_id, refhvr_id)
VALUES %s;
""" % separate_refids_string
print query
vampsdev_vamps_mysql_util.execute_no_fetch(query)
if __name__ == '__main__':
vampsdev_vamps_mysql_util = util.Mysql_util(host = "vampsdev", db = "vamps", read_default_group = "clientservers")
query = "show tables"
a = vampsdev_vamps_mysql_util.execute_fetch_select(query)
update_refhvr_ids = Update_refhvr_ids()
# print "AAA"
# !!! Uncomment !!!
# update_refhvr_ids.drop_table("refids_per_dataset_temp")
# update_refhvr_ids.create_table_refids_per_dataset_temp()
# update_refhvr_ids.insert_refids_per_dataset_temp()
# update_refhvr_ids.get_dataset_id()
# update_refhvr_ids.get_project_id()
# update_refhvr_ids.insert_into_refids_per_dataset()
res, field_names = update_refhvr_ids.get_rep_id_refhvr_ids()
# print field_names
update_refhvr_ids.separate_refids(update_refhvr_ids.make_dictionary_from_res(res))
