def populate_ptel_related_sources(self):
""" Using Pairitel objects existing in test ptel-object database,
here we construct sources test source values.
"""
pars = ingest_tools.pars
pars.update(self.pars['ingest_tools_pars']) # Use TestSuite params
srcid_index_server_pars = self.generate_index_server_pars('src_id')
self.srcdbt = ingest_tools.Source_Database_Tools(pars, self.rcd, \
self.htm_tools, \
rdb_host_ip=pars['rdb_host_ip_4'], \
rdb_user=pars['rdb_user_4'],\
rdb_name=pars['rdb_name_4'],\
srcid_index_server_pars=srcid_index_server_pars)
ra_range = self.pars['testvals_ptel_objects_fov_props']['ra_max'] - \
self.pars['testvals_ptel_objects_fov_props']['ra_min']
dec_range = self.pars['testvals_ptel_objects_fov_props']['dec_max'] - \
self.pars['testvals_ptel_objects_fov_props']['dec_min']
box_range = self.pars['testvals_ptel_source_boxrange_table_count'][0]
ra_midpt = ra_range/2.0 + \
self.pars['testvals_ptel_objects_fov_props']['ra_min']
dec_midpt = dec_range/2.0 + \
self.pars['testvals_ptel_objects_fov_props']['dec_min']
self.tcp_runs.populate_srcid_table_loop(pars, self.srcdbt, self.rdbt, \
ra=ra_midpt, dec=dec_midpt, \
box_degree_range=box_range, \
box_overlap_factor=0.05)
def __init__(self, pars):
import obj_id_sockets
#import numpy # to get a random number, but also used in feature code
#f = open('testsuite.transx_production_configs.par.py')
f = open('testsuite.par.py') # Import params from testsuite, which
# allows 127.0.0.1 test configs.
exec f
f.close()
self.pars = parameters['ingest_tools_pars']
self.pars.update(pars)
htm_tools = ingest_tools.HTM_Tools(self.pars)
rcd = ingest_tools.RDB_Column_Defs(\
rdb_table_names=self.pars['rdb_table_names'], \
rdb_db_name=self.pars['rdb_name_2'], \
col_definitions=ingest_tools.new_rdb_col_defs)
rcd.init_generate_mysql_strings(self.pars)
rdbt = ingest_tools.Rdb_Tools(self.pars, rcd, htm_tools, \
rdb_host_ip=self.pars['rdb_host_ip_2'], \
rdb_user=self.pars['rdb_user'], \
rdb_name=self.pars['rdb_name_2'])
sdss_fcr_iso = ingest_tools.SDSS_FCR_Ingest_Status_Object(\
rdb_host_ip=self.pars['rdb_host_ip_3'], \
rdb_user=self.pars['rdb_user'], \
rdb_name=self.pars['rdb_name_3'], \
table_name=self.pars['sdss_fields_table_name'], \
sdss_fields_doc_fpath_list=self.pars['sdss_fields_doc_fpath_list'],\
hostname=self.pars['hostname'])
srcdbt = ingest_tools.Source_Database_Tools(self.pars, rcd, htm_tools, \
rdb_host_ip=self.pars['rdb_host_ip_4'], \
rdb_user=self.pars['rdb_user_4'],\
rdb_name=self.pars['rdb_name_4'])
slfits_repo = ingest_tools.SDSS_Local_Fits_Repository(self.pars)
cur_pid = str(os.getpid())
tcp_runs = ingest_tools.TCP_Runtime_Methods(cur_pid=cur_pid)
self.xrsio = ingest_tools.XRPC_RDB_Server_Interface_Object(
self.pars, tcp_runs, rdbt, htm_tools, srcdbt, sdss_fcr_iso,
slfits_repo)
#arffify_path = os.environ.get("TCP_DIR") + \
# '/Software/feature_extract/MLData'
#sys.path.append(arffify_path)
#import arffify
# TODO: this file should only be stored in memory-disk-partition
# TODO: this file should be deleted after read/use.
out_fpath = "/tmp/temp_%s_%d.arff" % (cur_pid, \
random.randint(0,100000000))
self.arffmaker = arffify.Maker(search=[], outfile=out_fpath, \
skip_class=True, local_xmls=True, dorun=False)
self.socket_client = obj_id_sockets.socket_client({},\
server_type='feat_class_srcid_groups')
#if (len(sys.argv) != 2):
# print "invalid input"
# sys.exit()
#try:
# src_id = int(sys.argv[1])
#except:
# print "invalid src_id"
# sys.exit()
rdbt = ingest_tools.Rdb_Tools(ingest_tools.pars, None, None, \
rdb_host_ip=ingest_tools.pars['rdb_host_ip_2'], \
rdb_user=ingest_tools.pars['rdb_user'], \
rdb_name=ingest_tools.pars['rdb_name_2'])
srcdbt = ingest_tools.Source_Database_Tools(\
ingest_tools.pars, None, None, \
rdb_host_ip=ingest_tools.pars['rdb_host_ip_4'], \
rdb_user=ingest_tools.pars['rdb_user_4'],\
rdb_name=ingest_tools.pars['rdb_name_4'])
feat_db = feature_extraction_interface.Feature_database()
feat_db.initialize_mysql_connection(\
rdb_host_ip=ingest_tools.pars['rdb_features_host_ip'],\
rdb_user=ingest_tools.pars['rdb_features_user'], \
rdb_name=ingest_tools.pars['rdb_features_db_name'], \
feat_lookup_tablename=ingest_tools.pars['feat_lookup_tablename'], \
feat_values_tablename=ingest_tools.pars['feat_values_tablename'])
dbi_src = db_importer.Source(make_dict_if_given_xml=False)
#rfv = Rdb_Form_VOsource(ingest_tools.pars, rdbt, srcdbt, feat_db, dbi_src)
#rfv.get_vosource_url_for_srcid(src_id)
def import_modules(self):
""" Import all module dependencies during client "initialization"
"""
# store specific function as something local?
import ingest_tools
ingest_tools_pars = ingest_tools.pars
import feature_extraction_interface
ingest_tools_pars.update({\
'rdb_name_2':self.pars['object_dbname'],
'rdb_name_3':self.pars['object_dbname'],
'footprint_dbname':self.pars['object_dbname'],
'classdb_database':self.pars['source_dbname'],
'rdb_name_4':self.pars['source_dbname'],
'rdb_features_db_name':self.pars['source_dbname'],
'source_region_lock_dbname':self.pars['source_dbname'],
})
self.htm_tools = ingest_tools.HTM_Tools(ingest_tools_pars)
self.rcd = ingest_tools.RDB_Column_Defs(\
rdb_table_names=ingest_tools_pars['rdb_table_names'], \
rdb_db_name=ingest_tools_pars['rdb_name_2'], \
col_definitions=ingest_tools.new_rdb_col_defs)
self.rcd.generate_internal_structures('ptf')
self.rdbt = ingest_tools.Rdb_Tools(ingest_tools_pars, self.rcd, \
self.htm_tools, \
rdb_host_ip=ingest_tools_pars['rdb_host_ip_2'], \
rdb_user=ingest_tools_pars['rdb_user'], \
rdb_name=ingest_tools_pars['rdb_name_2'])
self.sdss_fcr_iso = ingest_tools.SDSS_FCR_Ingest_Status_Object(\
rdb_host_ip=ingest_tools_pars['rdb_host_ip_3'], \
rdb_user=ingest_tools_pars['rdb_user'], \
rdb_name=ingest_tools_pars['rdb_name_3'], \
table_name=ingest_tools_pars['sdss_fields_table_name'], \
sdss_fields_doc_fpath_list=\
ingest_tools_pars['sdss_fields_doc_fpath_list'],\
hostname=ingest_tools_pars['hostname'])
self.srcdbt = ingest_tools.Source_Database_Tools(ingest_tools_pars, \
self.rcd, self.htm_tools, \
rdb_host_ip=ingest_tools_pars['rdb_host_ip_4'], \
rdb_user=ingest_tools_pars['rdb_user_4'],\
rdb_name=ingest_tools_pars['rdb_name_4'])
self.slfits_repo = ingest_tools.SDSS_Local_Fits_Repository(\
ingest_tools_pars)
self.tcp_runs = ingest_tools.TCP_Runtime_Methods(\
cur_pid=str(os.getpid()))
self.xrsio = ingest_tools.XRPC_RDB_Server_Interface_Object(\
ingest_tools_pars, self.tcp_runs, self.rdbt, self.htm_tools,\
self.srcdbt, self.sdss_fcr_iso, self.slfits_repo)
self.feat_db = feature_extraction_interface.Feature_database()
self.feat_db.initialize_mysql_connection(\
rdb_host_ip=ingest_tools_pars['rdb_features_host_ip'],\
rdb_user=ingest_tools_pars['rdb_features_user'], \
rdb_name=ingest_tools_pars['rdb_features_db_name'], \
feat_lookup_tablename=ingest_tools_pars['feat_lookup_tablename'],\
feat_values_tablename=ingest_tools_pars['feat_values_tablename'])
self.feat_db.create_feature_lookup_dict()
# Make this accessible for later use:
self.get_features_using_srcid_xml_tuple_list = \
ingest_tools.get_features_using_srcid_xml_tuple_list
rdb_host_ip=ingest_tools_pars['rdb_host_ip_2'], \
rdb_user=ingest_tools_pars['rdb_user'], \
rdb_name=ingest_tools_pars['rdb_name_2'])
sdss_fcr_iso = ingest_tools.SDSS_FCR_Ingest_Status_Object(\
rdb_host_ip=ingest_tools_pars['rdb_host_ip_3'], \
rdb_user=ingest_tools_pars['rdb_user'], \
rdb_name=ingest_tools_pars['rdb_name_3'], \
table_name=ingest_tools_pars['sdss_fields_table_name'], \
sdss_fields_doc_fpath_list=\
ingest_tools_pars['sdss_fields_doc_fpath_list'],\
hostname=ingest_tools_pars['hostname'])
srcdbt = ingest_tools.Source_Database_Tools(ingest_tools_pars, rcd, \
htm_tools, \
rdb_host_ip=ingest_tools_pars['rdb_host_ip_4'], \
rdb_user=ingest_tools_pars['rdb_user_4'],\
rdb_name=ingest_tools_pars['rdb_name_4'])
slfits_repo = ingest_tools.SDSS_Local_Fits_Repository(ingest_tools_pars)
tcp_runs = ingest_tools.TCP_Runtime_Methods(cur_pid=str(os.getpid()))
xrsio = ingest_tools.XRPC_RDB_Server_Interface_Object(ingest_tools_pars, \
tcp_runs, rdbt, htm_tools, srcdbt, sdss_fcr_iso, slfits_repo)
feat_db = feature_extraction_interface.Feature_database()
feat_db.initialize_mysql_connection(\
rdb_host_ip=ingest_tools_pars['rdb_features_host_ip'],\
rdb_user=ingest_tools_pars['rdb_features_user'], \
rdb_name=ingest_tools_pars['rdb_features_db_name'], \
feat_lookup_tablename=ingest_tools_pars['feat_lookup_tablename'], \
feat_values_tablename=ingest_tools_pars['feat_values_tablename'])