def init(self, server):
try:
config = configparser.ConfigParser()
config.read(f'./config_{server}.ini', encoding='utf-8')
appconfig.dbhost = config['MYSQL']['host']
appconfig.dbname = config['MYSQL']['dbname']
appconfig.dbuser = config['MYSQL']['user']
appconfig.dbpassword = config['MYSQL']['password']
appconfig.dbport = config['MYSQL']['port']
appconfig.redis_host_port = config['REDIS']['host_port']
appconfig.redis_master = config['REDIS']['master']
appconfig.redis_password = config['REDIS']['password']
appconfig.kafka_bootstrap_servers = config['KAFKA'][
'bootstrap_servers']
appconfig.kafka_topics = config['KAFKA']['topics']
appconfig.kafka_group_id = config['KAFKA']['group_id']
appconfig.kafka_consumer_timeout_ms = config['KAFKA'][
'consumer_timeout_ms']
appconfig.kafka_auto_offset_reset = config['KAFKA'][
'auto_offset_reset']
appconfig.upload_path = config['FILEPATH']['upload_path']
appconfig.scrap_path = config['FILEPATH']['scrap_path']
appconfig.crop_path = config['FILEPATH']['crop_path']
appconfig.thumb_path = config['FILEPATH']['thumb_path']
except Exception as ex:
exUtil.printException()
raise Exception(f'check your parameters: {server}')
def _main(argv=None):
try:
#argv = ['aa', 'maple', 'dev']
if len(argv) < 3:
print(
'MapleApp needs two parameters. \nFirst: "maple" or ... \nSecond: "dev: or "prod"\n'
)
return
serviceType = argv[1]
buildType = argv[2]
if buildType != 'dev' and buildType != 'prod':
print(
f'You have invalid type parameter({buildType}).\nYou have to set "dev" or "prod" for second param.'
)
return
InitLogger.initLogging()
Config().init(buildType)
appExecutor = AppExecutor()
if appExecutor.start(serviceType, buildType) == False:
print(
f'You have wrong service type param({serviceType}). Service type is "maple".'
)
sys.exit()
except Exception as ex:
exutil.printException()
def initMySqlWithDatabase(self,
host,
port,
user,
passwd,
dbname,
cbAndEvt,
poolCount=4):
try:
self.host = host
self.port = port
self.user = user
self.passwd = passwd
self.dbname = dbname
self.cbAndEvt = cbAndEvt
logging.info("{0}:{1} {2}, DB Name: {3}".format(
host, port, user, dbname))
dbconfig = self.__getDefaultConfig(host, port, user, passwd)
dbconfig['database'] = dbname
self.pool = mysqlconn.pooling.MySQLConnectionPool(
pool_size=poolCount, pool_name=self.DB_POOLNAME, **dbconfig)
self.cbAndEvt.onEventDbConnect.fire(dbname)
return True
except Exception as ex:
exutil.printException()
self.cbAndEvt.onEventDbDisconnect.fire(dbname)
return False
def nonSelect(self, query):
try:
conn = self.pool.get_connection()
cursor = conn.cursor()
cursor.execute(query)
conn.commit()
return True
except Exception as ex:
exutil.printException()
return False
finally:
if conn.is_connected() == True:
cursor.close()
conn.close()
def multiQueries(self, queries):
try:
conn = self.pool.get_connection()
conn.autocommit = False
cursor = conn.cursor()
for query in queries:
cursor.execute(query)
conn.commit()
return True
except Exception as ex:
exutil.printException()
return False
finally:
if conn.is_connected() == True:
cursor.close()
conn.close()
def select(self, query):
try:
conn = self.pool.get_connection()
cursor = conn.cursor(dictionary=True)
cursor.execute(query)
records = cursor.fetchall()
return records if len(records) > 0 else None
#return [ dict(line) for line in [zip([ column[0] for column in cursor.description], row) for row in cursor.fetchall()] ]
except Exception as ex:
exutil.printException()
#self.cbAndEvt.onDbError.fire(str(ex))
#if(self.cbAndEvt != None):
# self.cbAndEvt.on_disconnected("disconnected")
return None
finally:
if conn.is_connected() == True:
cursor.close()
conn.close()
def initMySql(self, host, port, user, passwd, cbAndEvt, poolCount=1):
try:
self.host = host
self.port = port
self.user = user
self.passwd = passwd
self.cbAndEvt = cbAndEvt
self.dbname = 'init'
logging.info("{0}:{1} {2}, DB Name: 'None'".format(
host, port, user))
dbconfig = self.__getDefaultConfig(host, port, user, passwd)
self.pool = mysqlconn.pooling.MySQLConnectionPool(
pool_size=poolCount, pool_name=self.DB_POOLNAME, **dbconfig)
self.cbAndEvt.onEventDbConnect.fire('init')
return True
except Exception as ex:
exutil.printException()
self.cbAndEvt.onEventDbDisconnect.fire('init')
return False
def createDatabase(self, scode, host, port, user,
passwd): #scode means database name
try:
logging.info("{0}:{1} {2}, DB Name: {3}".format(
host, port, user, scode))
dbconfig = {
"user": user,
"password": passwd,
"host": host,
"port": port,
"connect_timeout": 3000
}
pool = mysqlconn.pooling.MySQLConnectionPool(
pool_size=2, pool_name='DBMAKER_POOL', **dbconfig)
sql = f"CREATE DATABASE IF NOT EXISTS {scode} "
conn = pool.get_connection()
cursor = conn.cursor()
cursor.execute(sql)
conn.commit()
pool._remove_connections()
return True
except Exception as ex:
exutil.printException()
return False
def _insertAirKoreaQuality(self, sido, body):
try:
jobj = json.loads(body)
items = jobj['list'] #json array
for item in items:
#sido, gugun, sensorAt, so2, co, o3, no2, pm10, pm25, khai, khaiGrade, so2Grade, coGrade, o3Grade, no2Grade, pm10Grade, pm25Grade
self.airKoreaQualityRepository.insertAirQuality(
sido, item['stationName'], item['dataTime'] + ':00',
item['so2Value'], item['coValue'], item['o3Value'],
item['no2Value'], item['pm10Value'], item['pm25Value'],
item['khaiValue'], item['khaiGrade'], item['so2Grade'],
item['coGrade'], item['o3Grade'], item['no2Grade'],
item['pm10Grade'], item['pm25Grade'])
except:
exutil.PrintException()
def doJobLoadAirKoreaList(self):
logging.info('AirKoreaCollector Start a Job')
sensorAt = datetime.now().strftime("%Y-%m-%d %H:00:00")
if self.airKoreaQualityRepository.isExistSensorAt(
sensorAt) == True: #sensorAt 데이터가 있으면 실행하지 않음
return
for sido in self.sidoList:
try:
url = f'http://openapi.airkorea.or.kr/openapi/services/rest/ArpltnInforInqireSvc/getCtprvnRltmMesureDnsty?serviceKey={appconfig.airkorea_apikey}&numOfRows={self.numOfRows}&pageNo=1&sidoName={sido}&ver={self.ver}&_returnType=json'
resp = requests.get(url)
body = resp.content.decode('utf-8')
self._insertAirKoreaQuality(sido, body)
except Exception as ex:
exutil.PrintException()
pass
def scrap(self, url):
try:
page = requests.get(url)
logging.info(page.text)
if 'location.href' in page.text:
content = page.text
url = content[content.find('location.href=') +
len('location.href=') + 1:content.find(';')]
url = url.replace('"', '').replace("'", '')
page = requests.get(url)
soup = BeautifulSoup(page.content, 'html.parser')
title = None
img_url = None
body = None
for tag in soup.find_all("meta"):
if tag.get("property", None) == "og:title":
title = tag.get("content", None)
elif tag.get("property", None) == "og:image":
img_url = tag.get("content", None)
if img_url.startswith("//") == True:
img_url = img_url[:2]
elif tag.get("property", None) == "og:description":
body = tag.get("content", None)
if title == None: title = soup.title
if img_url == None: img_url = self.getConditionalImage(url, soup)
if img_url == None: img_url = self.getFirstImage(url, soup)
if body == None: body = self.getBodyText(soup)
title = pymysql.escape_string(title)
body = pymysql.escape_string(body)
if len(body) > 100:
body = body[:100]
img_url = pymysql.escape_string(img_url)
title, subtitle = self.__extractTitle(title)
return title, subtitle, body, img_url
except Exception as ex:
exutil.printException()
def closeMySql(self):
try:
self.pool._remove_connections()
self.cbAndEvt.onEventDbDisconnect.fire(self.dbname)
except Exception as ex:
exutil.printException()