def test_database_function(self): # SETUP ALL DATABASE CONNECTIONS # SETUP ALL DATABASE CONNECTIONS from sherlock import database db = database( log=log, settings=settings ) dbConns, dbVersions = db.connect() self.transientsDbConn = dbConns["transients"] self.cataloguesDbConn = dbConns["catalogues"] from fundamentals.mysql import readquery sqlQuery = u""" SELECT VERSION(); """ % locals() rows = readquery( log=log, sqlQuery=sqlQuery, dbConn=self.transientsDbConn, quiet=False ) print(rows) rows = readquery( log=log, sqlQuery=sqlQuery, dbConn=self.cataloguesDbConn, quiet=False ) print(rows)
def __init__( self, log, settings=False, ): self.log = log log.debug("instansiating a new 'update_wiki_pages' object") self.settings = settings # INITIAL ACTIONS # SETUP ALL DATABASE CONNECTIONS from sherlock import database db = database(log=self.log, settings=self.settings) dbConns, dbVersions = db.connect() self.transientsDbConn = dbConns["transients"] self.cataloguesDbConn = dbConns["catalogues"] self.basicColumns = [ "view_name", "master table", "description", "version_number", "object_types", "object_type", "number_of_rows", "url", "object_type_accuracy", "last_updated", ] return None
def __init__( self, log, settings=False, update=False, transientIdList=[] ): self.log = log log.debug("instansiating a new 'classifier' object") self.settings = settings self.update = update self.transientIdList = transientIdList # xt-self-arg-tmpx # VARIABLE DATA ATRRIBUTES self.transientsMetadataList = [] # INITIAL ACTIONS # SETUP DATABASE CONNECTIONS from sherlock import database db = database( log=self.log, settings=self.settings ) self.transientsDbConn, self.cataloguesDbConn, self.pmDbConn = db.get() return None
def __init__( self, log, settings=False, ): self.log = log log.debug("instansiating a new 'cleanup_database_tables' object") self.settings = settings # xt-self-arg-tmpx # 2. @flagged: what are the default attrributes each object could have? Add them to variable attribute set here # Variable Data Atrributes # 3. @flagged: what variable attrributes need overriden in any baseclass(es) used # Override Variable Data Atrributes # Initial Actions from sherlock import database db = database( log=self.log, settings=self.settings ) self.transientsDbConn, self.cataloguesDbConn, self.pmDbConn = db.get() return None
def test_crossmatching(self): # SETUP ALL DATABASE CONNECTIONS from sherlock import database db = database(log=log, settings=settings) dbConns, dbVersions = db.connect() transientsDbConn = dbConns["transients"] cataloguesDbConn = dbConns["catalogues"] pmDbConn = dbConns["marshall"] from sherlock.commonutils import get_crossmatch_catalogues_column_map colMaps = get_crossmatch_catalogues_column_map(log=log, dbConn=cataloguesDbConn) from sherlock import transient_classifier classifier = transient_classifier(log=log, settings=settings) transientsMetadataList = classifier._get_transient_metadata_from_database_list( ) crossmatches = classifier._crossmatch_transients_against_catalogues( colMaps=colMaps, transientsMetadataList=transientsMetadataList) classifications, crossmatches = classifier._rank_classifications( colMaps=colMaps, crossmatches=crossmatches) classifier._update_transient_database( classifications=classifications, transientsMetadataList=transientsMetadataList, colMaps=colMaps, crossmatches=crossmatches)
def test_tunnel(self): from sherlock import database db = database(log=log, settings=settings) sshPort = db._setup_tunnel( tunnelParameters=settings["database settings"]["static catalogues"] ["tunnel"]) return
def test_database_function_exception(self): from sherlock import database try: this = database(log=log, settings=settings, fakeKey="break the code") this.get() assert False except Exception, e: assert True print str(e)
def test_classification_annotations(self): from sherlock import database db = database(log=log, settings=settings) dbConns, dbVersions = db.connect() transientsDbConn = dbConns["transients"] cataloguesDbConn = dbConns["catalogues"] from sherlock.commonutils import get_crossmatch_catalogues_column_map colMaps = get_crossmatch_catalogues_column_map(log=log, dbConn=cataloguesDbConn) from sherlock import transient_classifier classifier = transient_classifier(log=log, settings=settings) classifier.classification_annotations()
def __init__(self, log, settings=False, pathToDataFile=False, version=False, catalogueName="", coordinateList=[], radiusArcsec=False): self.log = log log.debug("instansiating a new '_base_importer' object") self.settings = settings self.pathToDataFile = pathToDataFile self.version = version self.catalogueName = catalogueName self.coordinateList = coordinateList self.radiusArcsec = radiusArcsec self.myPid = str(os.getpid()) # xt-self-arg-tmpx # INITIAL ACTIONS # SETUP DATABASE CONNECTIONS # SETUP ALL DATABASE CONNECTIONS from sherlock import database db = database(log=self.log, settings=self.settings) dbConns, dbVersions = db.connect() self.transientsDbConn = dbConns["transients"] self.cataloguesDbConn = dbConns["catalogues"] self.pmDbConn = dbConns["marshall"] # OPEN THE FILE TO IMPORT THE DATA FROM if pathToDataFile: pathToReadFile = pathToDataFile try: self.log.debug("attempting to open the file %s" % (pathToReadFile, )) readFile = codecs.open(pathToReadFile, mode='r') self.catData = readFile.read() readFile.close() except IOError, e: message = 'could not open the file %s' % (pathToReadFile, ) self.log.critical(message) raise IOError(message) readFile.close()
def __init__( self, log, settings=False, ): self.log = log log.debug("instansiating a new 'database_cleaner' object") self.settings = settings # xt-self-arg-tmpx # INITIAL ACTIONS# Initial Actions # SETUP ALL DATABASE CONNECTIONS from sherlock import database db = database(log=self.log, settings=self.settings) dbConns, dbVersions = db.connect() self.transientsDbConn = dbConns["transients"] self.cataloguesDbConn = dbConns["catalogues"] self.pmDbConn = dbConns["marshall"] return None
def __init__( self, log, settings=False, pathToDataFile=False, version=False, catalogueName="", coordinateList=[] ): self.log = log log.debug("instansiating a new '_base_importer' object") self.settings = settings self.pathToDataFile = pathToDataFile self.version = version self.catalogueName = catalogueName self.coordinateList = coordinateList # xt-self-arg-tmpx # INITIAL ACTIONS # SETUP DATABASE CONNECTIONS from sherlock import database db = database( log=self.log, settings=self.settings ) self.transientsDbConn, self.cataloguesDbConn, self.pmDbConn = db.get() if pathToDataFile: pathToReadFile = pathToDataFile try: self.log.debug("attempting to open the file %s" % (pathToReadFile,)) readFile = codecs.open(pathToReadFile, mode='r') self.catData = readFile.read() readFile.close() except IOError, e: message = 'could not open the file %s' % (pathToReadFile,) self.log.critical(message) raise IOError(message) readFile.close()
try: shutil.rmtree(pathToOutputDir) except: pass # COPY INPUT TO OUTPUT DIR shutil.copytree(pathToInputDir, pathToOutputDir) # Recursively create missing directories if not os.path.exists(pathToOutputDir): os.makedirs(pathToOutputDir) # SETUP ALL DATABASE CONNECTIONS from sherlock import database db = database( log=log, settings=settings ) dbConns, dbVersions = db.connect() transientsDbConn = dbConns["transients"] cataloguesDbConn = dbConns["catalogues"] class test_get_crossmatch_catalogues_column_map(unittest.TestCase): def test_get_crossmatch_catalogues_column_map_function(self): from sherlock.commonutils import get_crossmatch_catalogues_column_map colMaps = get_crossmatch_catalogues_column_map( log=log, dbConn=cataloguesDbConn )