def init(self, server):
try:
config = configparser.ConfigParser()
config.read(f'./config_{server}.ini', encoding='utf-8')
appconfig.dbhost = config['MYSQL']['host']
appconfig.dbname = config['MYSQL']['dbname']
appconfig.dbuser = config['MYSQL']['user']
appconfig.dbpassword = config['MYSQL']['password']
appconfig.dbport = config['MYSQL']['port']
appconfig.redis_host_port = config['REDIS']['host_port']
appconfig.redis_master = config['REDIS']['master']
appconfig.redis_password = config['REDIS']['password']
appconfig.kafka_bootstrap_servers = config['KAFKA'][
'bootstrap_servers']
appconfig.kafka_topics = config['KAFKA']['topics']
appconfig.kafka_group_id = config['KAFKA']['group_id']
appconfig.kafka_consumer_timeout_ms = config['KAFKA'][
'consumer_timeout_ms']
appconfig.kafka_auto_offset_reset = config['KAFKA'][
'auto_offset_reset']
appconfig.upload_path = config['FILEPATH']['upload_path']
appconfig.scrap_path = config['FILEPATH']['scrap_path']
appconfig.crop_path = config['FILEPATH']['crop_path']
appconfig.thumb_path = config['FILEPATH']['thumb_path']
except Exception as ex:
exUtil.printException()
raise Exception(f'check your parameters: {server}')
def _main(argv=None):
try:
#argv = ['aa', 'maple', 'dev']
if len(argv) < 3:
print(
'MapleApp needs two parameters. \nFirst: "maple" or ... \nSecond: "dev: or "prod"\n'
)
return
serviceType = argv[1]
buildType = argv[2]
if buildType != 'dev' and buildType != 'prod':
print(
f'You have invalid type parameter({buildType}).\nYou have to set "dev" or "prod" for second param.'
)
return
InitLogger.initLogging()
Config().init(buildType)
appExecutor = AppExecutor()
if appExecutor.start(serviceType, buildType) == False:
print(
f'You have wrong service type param({serviceType}). Service type is "maple".'
)
sys.exit()
except Exception as ex:
exutil.printException()
def initMySqlWithDatabase(self,
host,
port,
user,
passwd,
dbname,
cbAndEvt,
poolCount=4):
try:
self.host = host
self.port = port
self.user = user
self.passwd = passwd
self.dbname = dbname
self.cbAndEvt = cbAndEvt
logging.info("{0}:{1} {2}, DB Name: {3}".format(
host, port, user, dbname))
dbconfig = self.__getDefaultConfig(host, port, user, passwd)
dbconfig['database'] = dbname
self.pool = mysqlconn.pooling.MySQLConnectionPool(
pool_size=poolCount, pool_name=self.DB_POOLNAME, **dbconfig)
self.cbAndEvt.onEventDbConnect.fire(dbname)
return True
except Exception as ex:
exutil.printException()
self.cbAndEvt.onEventDbDisconnect.fire(dbname)
return False
def nonSelect(self, query):
try:
conn = self.pool.get_connection()
cursor = conn.cursor()
cursor.execute(query)
conn.commit()
return True
except Exception as ex:
exutil.printException()
return False
finally:
if conn.is_connected() == True:
cursor.close()
conn.close()
def multiQueries(self, queries):
try:
conn = self.pool.get_connection()
conn.autocommit = False
cursor = conn.cursor()
for query in queries:
cursor.execute(query)
conn.commit()
return True
except Exception as ex:
exutil.printException()
return False
finally:
if conn.is_connected() == True:
cursor.close()
conn.close()
def select(self, query):
try:
conn = self.pool.get_connection()
cursor = conn.cursor(dictionary=True)
cursor.execute(query)
records = cursor.fetchall()
return records if len(records) > 0 else None
#return [ dict(line) for line in [zip([ column[0] for column in cursor.description], row) for row in cursor.fetchall()] ]
except Exception as ex:
exutil.printException()
#self.cbAndEvt.onDbError.fire(str(ex))
#if(self.cbAndEvt != None):
# self.cbAndEvt.on_disconnected("disconnected")
return None
finally:
if conn.is_connected() == True:
cursor.close()
conn.close()
def initMySql(self, host, port, user, passwd, cbAndEvt, poolCount=1):
try:
self.host = host
self.port = port
self.user = user
self.passwd = passwd
self.cbAndEvt = cbAndEvt
self.dbname = 'init'
logging.info("{0}:{1} {2}, DB Name: 'None'".format(
host, port, user))
dbconfig = self.__getDefaultConfig(host, port, user, passwd)
self.pool = mysqlconn.pooling.MySQLConnectionPool(
pool_size=poolCount, pool_name=self.DB_POOLNAME, **dbconfig)
self.cbAndEvt.onEventDbConnect.fire('init')
return True
except Exception as ex:
exutil.printException()
self.cbAndEvt.onEventDbDisconnect.fire('init')
return False
def createDatabase(self, scode, host, port, user,
passwd): #scode means database name
try:
logging.info("{0}:{1} {2}, DB Name: {3}".format(
host, port, user, scode))
dbconfig = {
"user": user,
"password": passwd,
"host": host,
"port": port,
"connect_timeout": 3000
}
pool = mysqlconn.pooling.MySQLConnectionPool(
pool_size=2, pool_name='DBMAKER_POOL', **dbconfig)
sql = f"CREATE DATABASE IF NOT EXISTS {scode} "
conn = pool.get_connection()
cursor = conn.cursor()
cursor.execute(sql)
conn.commit()
pool._remove_connections()
return True
except Exception as ex:
exutil.printException()
return False
def scrap(self, url):
try:
page = requests.get(url)
logging.info(page.text)
if 'location.href' in page.text:
content = page.text
url = content[content.find('location.href=') +
len('location.href=') + 1:content.find(';')]
url = url.replace('"', '').replace("'", '')
page = requests.get(url)
soup = BeautifulSoup(page.content, 'html.parser')
title = None
img_url = None
body = None
for tag in soup.find_all("meta"):
if tag.get("property", None) == "og:title":
title = tag.get("content", None)
elif tag.get("property", None) == "og:image":
img_url = tag.get("content", None)
if img_url.startswith("//") == True:
img_url = img_url[:2]
elif tag.get("property", None) == "og:description":
body = tag.get("content", None)
if title == None: title = soup.title
if img_url == None: img_url = self.getConditionalImage(url, soup)
if img_url == None: img_url = self.getFirstImage(url, soup)
if body == None: body = self.getBodyText(soup)
title = pymysql.escape_string(title)
body = pymysql.escape_string(body)
if len(body) > 100:
body = body[:100]
img_url = pymysql.escape_string(img_url)
title, subtitle = self.__extractTitle(title)
return title, subtitle, body, img_url
except Exception as ex:
exutil.printException()
def closeMySql(self):
try:
self.pool._remove_connections()
self.cbAndEvt.onEventDbDisconnect.fire(self.dbname)
except Exception as ex:
exutil.printException()