def __init__(self, pars):
import obj_id_sockets
#import numpy # to get a random number, but also used in feature code
#f = open('testsuite.transx_production_configs.par.py')
f = open('testsuite.par.py') # Import params from testsuite, which
# allows 127.0.0.1 test configs.
exec f
f.close()
self.pars = parameters['ingest_tools_pars']
self.pars.update(pars)
htm_tools = ingest_tools.HTM_Tools(self.pars)
rcd = ingest_tools.RDB_Column_Defs(\
rdb_table_names=self.pars['rdb_table_names'], \
rdb_db_name=self.pars['rdb_name_2'], \
col_definitions=ingest_tools.new_rdb_col_defs)
rcd.init_generate_mysql_strings(self.pars)
rdbt = ingest_tools.Rdb_Tools(self.pars, rcd, htm_tools, \
rdb_host_ip=self.pars['rdb_host_ip_2'], \
rdb_user=self.pars['rdb_user'], \
rdb_name=self.pars['rdb_name_2'])
sdss_fcr_iso = ingest_tools.SDSS_FCR_Ingest_Status_Object(\
rdb_host_ip=self.pars['rdb_host_ip_3'], \
rdb_user=self.pars['rdb_user'], \
rdb_name=self.pars['rdb_name_3'], \
table_name=self.pars['sdss_fields_table_name'], \
sdss_fields_doc_fpath_list=self.pars['sdss_fields_doc_fpath_list'],\
hostname=self.pars['hostname'])
srcdbt = ingest_tools.Source_Database_Tools(self.pars, rcd, htm_tools, \
rdb_host_ip=self.pars['rdb_host_ip_4'], \
rdb_user=self.pars['rdb_user_4'],\
rdb_name=self.pars['rdb_name_4'])
slfits_repo = ingest_tools.SDSS_Local_Fits_Repository(self.pars)
cur_pid = str(os.getpid())
tcp_runs = ingest_tools.TCP_Runtime_Methods(cur_pid=cur_pid)
self.xrsio = ingest_tools.XRPC_RDB_Server_Interface_Object(
self.pars, tcp_runs, rdbt, htm_tools, srcdbt, sdss_fcr_iso,
slfits_repo)
#arffify_path = os.environ.get("TCP_DIR") + \
# '/Software/feature_extract/MLData'
#sys.path.append(arffify_path)
#import arffify
# TODO: this file should only be stored in memory-disk-partition
# TODO: this file should be deleted after read/use.
out_fpath = "/tmp/temp_%s_%d.arff" % (cur_pid, \
random.randint(0,100000000))
self.arffmaker = arffify.Maker(search=[], outfile=out_fpath, \
skip_class=True, local_xmls=True, dorun=False)
self.socket_client = obj_id_sockets.socket_client({},\
server_type='feat_class_srcid_groups')
def populate_ptel_object_tables_using_list(self, mosfits_fpath_list):
""" Using a list of Pairitel mos*fits files, extract astrometry,
photometry and add to MySQL DB.
"""
footprint_index_server_pars = self.generate_index_server_pars(\
'footprint_id')
ptel_obj_index_server_pars = self.generate_index_server_pars(\
'ptel_obj_id')
ptf_obj_index_server_pars = self.generate_index_server_pars(\
'ptf_obj_id')
sdss_obj_index_server_pars = self.generate_index_server_pars(\
'obj_id')
pars = ingest_tools.pars
pars.update(self.pars['ingest_tools_pars']) # Use TestSuite params
# These objects are also used in: populate_ptel_related_sources():
self.htm_tools = ingest_tools.HTM_Tools(pars) #TODO: this needed?
self.rcd = ingest_tools.RDB_Column_Defs(\
rdb_table_names=pars['rdb_table_names'], \
rdb_db_name=pars['rdb_name_2'], \
col_definitions=ingest_tools.new_rdb_col_defs)
self.rcd.init_generate_mysql_strings(pars)
self.rdbt = ingest_tools.Rdb_Tools(pars, self.rcd, self.htm_tools, \
rdb_host_ip=pars['rdb_host_ip_2'], \
rdb_user=pars['rdb_user'], \
rdb_name=pars['rdb_name_2'], \
footprint_index_server_pars=footprint_index_server_pars,\
ptel_obj_index_server_pars=ptel_obj_index_server_pars,\
sdss_obj_index_server_pars=sdss_obj_index_server_pars,\
ptf_obj_index_server_pars=ptf_obj_index_server_pars)
for mos_globpath in mosfits_fpath_list:
mos_fname = mos_globpath[mos_globpath.rfind('/')+1:]
mos_fname_root_trunc = mos_fname[:30]
try:
has_been_ingested = self.rdbt.check_obsid_has_been_ingested(\
mos_fname_root_trunc)
except:
print "SKIP INSERT of existing ptel objects:", \
mos_fname_root_trunc
continue
if (has_been_ingested == 0):
a = ptel_astrophot.PTEL_data_block(mos_globpath, \
req_filts=['j','h','k'])
a.runit()
astrophot_dict = a.out_results
self.rdbt.insert_pairitel_astrometry_into_rdb(\
phot_dict=astrophot_dict, \
mosfits_fpath=mos_fname)
#self.rdbt.add_obsid_to_ingested_table(mos_fname_root_trunc)
self.rdbt.add_obsid_to_ingested_table(mos_fname)
ingest_tools.pars['source_region_lock_dbname'] = 'source_test_db'
ingest_tools.pars['footprint_host_ip'] = server_ip
ingest_tools.pars['footprint_user'] = server_user
ingest_tools.pars['footprint_dbname'] = "object_test_db"
#if (len(sys.argv) != 2):
# print "invalid input"
# sys.exit()
#try:
# src_id = int(sys.argv[1])
#except:
# print "invalid src_id"
# sys.exit()
rdbt = ingest_tools.Rdb_Tools(ingest_tools.pars, None, None, \
rdb_host_ip=ingest_tools.pars['rdb_host_ip_2'], \
rdb_user=ingest_tools.pars['rdb_user'], \
rdb_name=ingest_tools.pars['rdb_name_2'])
srcdbt = ingest_tools.Source_Database_Tools(\
ingest_tools.pars, None, None, \
rdb_host_ip=ingest_tools.pars['rdb_host_ip_4'], \
rdb_user=ingest_tools.pars['rdb_user_4'],\
rdb_name=ingest_tools.pars['rdb_name_4'])
feat_db = feature_extraction_interface.Feature_database()
feat_db.initialize_mysql_connection(\
rdb_host_ip=ingest_tools.pars['rdb_features_host_ip'],\
rdb_user=ingest_tools.pars['rdb_features_user'], \
rdb_name=ingest_tools.pars['rdb_features_db_name'], \
feat_lookup_tablename=ingest_tools.pars['feat_lookup_tablename'], \
feat_values_tablename=ingest_tools.pars['feat_values_tablename'])
def import_modules(self):
""" Import all module dependencies during client "initialization"
"""
# store specific function as something local?
import ingest_tools
ingest_tools_pars = ingest_tools.pars
import feature_extraction_interface
ingest_tools_pars.update({\
'rdb_name_2':self.pars['object_dbname'],
'rdb_name_3':self.pars['object_dbname'],
'footprint_dbname':self.pars['object_dbname'],
'classdb_database':self.pars['source_dbname'],
'rdb_name_4':self.pars['source_dbname'],
'rdb_features_db_name':self.pars['source_dbname'],
'source_region_lock_dbname':self.pars['source_dbname'],
})
self.htm_tools = ingest_tools.HTM_Tools(ingest_tools_pars)
self.rcd = ingest_tools.RDB_Column_Defs(\
rdb_table_names=ingest_tools_pars['rdb_table_names'], \
rdb_db_name=ingest_tools_pars['rdb_name_2'], \
col_definitions=ingest_tools.new_rdb_col_defs)
self.rcd.generate_internal_structures('ptf')
self.rdbt = ingest_tools.Rdb_Tools(ingest_tools_pars, self.rcd, \
self.htm_tools, \
rdb_host_ip=ingest_tools_pars['rdb_host_ip_2'], \
rdb_user=ingest_tools_pars['rdb_user'], \
rdb_name=ingest_tools_pars['rdb_name_2'])
self.sdss_fcr_iso = ingest_tools.SDSS_FCR_Ingest_Status_Object(\
rdb_host_ip=ingest_tools_pars['rdb_host_ip_3'], \
rdb_user=ingest_tools_pars['rdb_user'], \
rdb_name=ingest_tools_pars['rdb_name_3'], \
table_name=ingest_tools_pars['sdss_fields_table_name'], \
sdss_fields_doc_fpath_list=\
ingest_tools_pars['sdss_fields_doc_fpath_list'],\
hostname=ingest_tools_pars['hostname'])
self.srcdbt = ingest_tools.Source_Database_Tools(ingest_tools_pars, \
self.rcd, self.htm_tools, \
rdb_host_ip=ingest_tools_pars['rdb_host_ip_4'], \
rdb_user=ingest_tools_pars['rdb_user_4'],\
rdb_name=ingest_tools_pars['rdb_name_4'])
self.slfits_repo = ingest_tools.SDSS_Local_Fits_Repository(\
ingest_tools_pars)
self.tcp_runs = ingest_tools.TCP_Runtime_Methods(\
cur_pid=str(os.getpid()))
self.xrsio = ingest_tools.XRPC_RDB_Server_Interface_Object(\
ingest_tools_pars, self.tcp_runs, self.rdbt, self.htm_tools,\
self.srcdbt, self.sdss_fcr_iso, self.slfits_repo)
self.feat_db = feature_extraction_interface.Feature_database()
self.feat_db.initialize_mysql_connection(\
rdb_host_ip=ingest_tools_pars['rdb_features_host_ip'],\
rdb_user=ingest_tools_pars['rdb_features_user'], \
rdb_name=ingest_tools_pars['rdb_features_db_name'], \
feat_lookup_tablename=ingest_tools_pars['feat_lookup_tablename'],\
feat_values_tablename=ingest_tools_pars['feat_values_tablename'])
self.feat_db.create_feature_lookup_dict()
# Make this accessible for later use:
self.get_features_using_srcid_xml_tuple_list = \
ingest_tools.get_features_using_srcid_xml_tuple_list
diff_source['dec_rms']
])
return tup_list
if __name__ == '__main__':
htm_tools = ingest_tools.HTM_Tools(ingest_tools_pars)
rcd = ingest_tools.RDB_Column_Defs(\
rdb_table_names=ingest_tools_pars['rdb_table_names'], \
rdb_db_name=ingest_tools_pars['rdb_name_2'], \
col_definitions=ingest_tools.new_rdb_col_defs)
rcd.generate_internal_structures('ptf')
rdbt = ingest_tools.Rdb_Tools(ingest_tools_pars, rcd, htm_tools, \
rdb_host_ip=ingest_tools_pars['rdb_host_ip_2'], \
rdb_user=ingest_tools_pars['rdb_user'], \
rdb_name=ingest_tools_pars['rdb_name_2'])
sdss_fcr_iso = ingest_tools.SDSS_FCR_Ingest_Status_Object(\
rdb_host_ip=ingest_tools_pars['rdb_host_ip_3'], \
rdb_user=ingest_tools_pars['rdb_user'], \
rdb_name=ingest_tools_pars['rdb_name_3'], \
table_name=ingest_tools_pars['sdss_fields_table_name'], \
sdss_fields_doc_fpath_list=\
ingest_tools_pars['sdss_fields_doc_fpath_list'],\
hostname=ingest_tools_pars['hostname'])
srcdbt = ingest_tools.Source_Database_Tools(ingest_tools_pars, rcd, \
htm_tools, \
rdb_host_ip=ingest_tools_pars['rdb_host_ip_4'], \
rdb_user=ingest_tools_pars['rdb_user_4'],\