def init(self, server): try: config = configparser.ConfigParser() config.read(f'./config_{server}.ini', encoding='utf-8') appconfig.dbhost = config['MYSQL']['host'] appconfig.dbname = config['MYSQL']['dbname'] appconfig.dbuser = config['MYSQL']['user'] appconfig.dbpassword = config['MYSQL']['password'] appconfig.dbport = config['MYSQL']['port'] appconfig.redis_host_port = config['REDIS']['host_port'] appconfig.redis_master = config['REDIS']['master'] appconfig.redis_password = config['REDIS']['password'] appconfig.kafka_bootstrap_servers = config['KAFKA'][ 'bootstrap_servers'] appconfig.kafka_topics = config['KAFKA']['topics'] appconfig.kafka_group_id = config['KAFKA']['group_id'] appconfig.kafka_consumer_timeout_ms = config['KAFKA'][ 'consumer_timeout_ms'] appconfig.kafka_auto_offset_reset = config['KAFKA'][ 'auto_offset_reset'] appconfig.upload_path = config['FILEPATH']['upload_path'] appconfig.scrap_path = config['FILEPATH']['scrap_path'] appconfig.crop_path = config['FILEPATH']['crop_path'] appconfig.thumb_path = config['FILEPATH']['thumb_path'] except Exception as ex: exUtil.printException() raise Exception(f'check your parameters: {server}')
def _main(argv=None): try: #argv = ['aa', 'maple', 'dev'] if len(argv) < 3: print( 'MapleApp needs two parameters. \nFirst: "maple" or ... \nSecond: "dev: or "prod"\n' ) return serviceType = argv[1] buildType = argv[2] if buildType != 'dev' and buildType != 'prod': print( f'You have invalid type parameter({buildType}).\nYou have to set "dev" or "prod" for second param.' ) return InitLogger.initLogging() Config().init(buildType) appExecutor = AppExecutor() if appExecutor.start(serviceType, buildType) == False: print( f'You have wrong service type param({serviceType}). Service type is "maple".' ) sys.exit() except Exception as ex: exutil.printException()
def initMySqlWithDatabase(self, host, port, user, passwd, dbname, cbAndEvt, poolCount=4): try: self.host = host self.port = port self.user = user self.passwd = passwd self.dbname = dbname self.cbAndEvt = cbAndEvt logging.info("{0}:{1} {2}, DB Name: {3}".format( host, port, user, dbname)) dbconfig = self.__getDefaultConfig(host, port, user, passwd) dbconfig['database'] = dbname self.pool = mysqlconn.pooling.MySQLConnectionPool( pool_size=poolCount, pool_name=self.DB_POOLNAME, **dbconfig) self.cbAndEvt.onEventDbConnect.fire(dbname) return True except Exception as ex: exutil.printException() self.cbAndEvt.onEventDbDisconnect.fire(dbname) return False
def nonSelect(self, query): try: conn = self.pool.get_connection() cursor = conn.cursor() cursor.execute(query) conn.commit() return True except Exception as ex: exutil.printException() return False finally: if conn.is_connected() == True: cursor.close() conn.close()
def multiQueries(self, queries): try: conn = self.pool.get_connection() conn.autocommit = False cursor = conn.cursor() for query in queries: cursor.execute(query) conn.commit() return True except Exception as ex: exutil.printException() return False finally: if conn.is_connected() == True: cursor.close() conn.close()
def select(self, query): try: conn = self.pool.get_connection() cursor = conn.cursor(dictionary=True) cursor.execute(query) records = cursor.fetchall() return records if len(records) > 0 else None #return [ dict(line) for line in [zip([ column[0] for column in cursor.description], row) for row in cursor.fetchall()] ] except Exception as ex: exutil.printException() #self.cbAndEvt.onDbError.fire(str(ex)) #if(self.cbAndEvt != None): # self.cbAndEvt.on_disconnected("disconnected") return None finally: if conn.is_connected() == True: cursor.close() conn.close()
def initMySql(self, host, port, user, passwd, cbAndEvt, poolCount=1): try: self.host = host self.port = port self.user = user self.passwd = passwd self.cbAndEvt = cbAndEvt self.dbname = 'init' logging.info("{0}:{1} {2}, DB Name: 'None'".format( host, port, user)) dbconfig = self.__getDefaultConfig(host, port, user, passwd) self.pool = mysqlconn.pooling.MySQLConnectionPool( pool_size=poolCount, pool_name=self.DB_POOLNAME, **dbconfig) self.cbAndEvt.onEventDbConnect.fire('init') return True except Exception as ex: exutil.printException() self.cbAndEvt.onEventDbDisconnect.fire('init') return False
def createDatabase(self, scode, host, port, user, passwd): #scode means database name try: logging.info("{0}:{1} {2}, DB Name: {3}".format( host, port, user, scode)) dbconfig = { "user": user, "password": passwd, "host": host, "port": port, "connect_timeout": 3000 } pool = mysqlconn.pooling.MySQLConnectionPool( pool_size=2, pool_name='DBMAKER_POOL', **dbconfig) sql = f"CREATE DATABASE IF NOT EXISTS {scode} " conn = pool.get_connection() cursor = conn.cursor() cursor.execute(sql) conn.commit() pool._remove_connections() return True except Exception as ex: exutil.printException() return False
def _insertAirKoreaQuality(self, sido, body): try: jobj = json.loads(body) items = jobj['list'] #json array for item in items: #sido, gugun, sensorAt, so2, co, o3, no2, pm10, pm25, khai, khaiGrade, so2Grade, coGrade, o3Grade, no2Grade, pm10Grade, pm25Grade self.airKoreaQualityRepository.insertAirQuality( sido, item['stationName'], item['dataTime'] + ':00', item['so2Value'], item['coValue'], item['o3Value'], item['no2Value'], item['pm10Value'], item['pm25Value'], item['khaiValue'], item['khaiGrade'], item['so2Grade'], item['coGrade'], item['o3Grade'], item['no2Grade'], item['pm10Grade'], item['pm25Grade']) except: exutil.PrintException()
def doJobLoadAirKoreaList(self): logging.info('AirKoreaCollector Start a Job') sensorAt = datetime.now().strftime("%Y-%m-%d %H:00:00") if self.airKoreaQualityRepository.isExistSensorAt( sensorAt) == True: #sensorAt 데이터가 있으면 실행하지 않음 return for sido in self.sidoList: try: url = f'http://openapi.airkorea.or.kr/openapi/services/rest/ArpltnInforInqireSvc/getCtprvnRltmMesureDnsty?serviceKey={appconfig.airkorea_apikey}&numOfRows={self.numOfRows}&pageNo=1&sidoName={sido}&ver={self.ver}&_returnType=json' resp = requests.get(url) body = resp.content.decode('utf-8') self._insertAirKoreaQuality(sido, body) except Exception as ex: exutil.PrintException() pass
def scrap(self, url): try: page = requests.get(url) logging.info(page.text) if 'location.href' in page.text: content = page.text url = content[content.find('location.href=') + len('location.href=') + 1:content.find(';')] url = url.replace('"', '').replace("'", '') page = requests.get(url) soup = BeautifulSoup(page.content, 'html.parser') title = None img_url = None body = None for tag in soup.find_all("meta"): if tag.get("property", None) == "og:title": title = tag.get("content", None) elif tag.get("property", None) == "og:image": img_url = tag.get("content", None) if img_url.startswith("//") == True: img_url = img_url[:2] elif tag.get("property", None) == "og:description": body = tag.get("content", None) if title == None: title = soup.title if img_url == None: img_url = self.getConditionalImage(url, soup) if img_url == None: img_url = self.getFirstImage(url, soup) if body == None: body = self.getBodyText(soup) title = pymysql.escape_string(title) body = pymysql.escape_string(body) if len(body) > 100: body = body[:100] img_url = pymysql.escape_string(img_url) title, subtitle = self.__extractTitle(title) return title, subtitle, body, img_url except Exception as ex: exutil.printException()
def closeMySql(self): try: self.pool._remove_connections() self.cbAndEvt.onEventDbDisconnect.fire(self.dbname) except Exception as ex: exutil.printException()