def __init__(self, prefix="http://www.lyrics.com/lyric/"):
self.__file_system = FileSystem()
self.__logger = getLogger(Logger.ANALYSER)
self.__linguistics = Linguistics()
self.__seed_prefix = prefix
self.__model_writer = ModelWriter()
self.__database = LyricDatabase()
def fish_scheduler(frequency, name=0):
logger = getLogger(Logger.FISHER)
logger.info('fish_scheduler %s start...' % (name))
queue = MessageQueue()
fisher = Fisher()
while frequency:
frequency = frequency - 1
seed = None
html = None
while seed is None:
seed = queue.seed
if seed == '':
html = ''
else:
seed = ('http://www.lyrics.com' + seed).encode('utf-8')
logger.info('Fisher %s start dealing the %dth seed : %s.' %
(name, frequency, seed))
html = fisher.fish(seed)
if html is None:
html = ''
queue.html = html
logger.info('Fisher %s stop dealing the %dth seed : %s.' %
(name, frequency, seed))
logger.info('fish_scheduler %s end...' % (name))
def analyse_scheduler(frequency, name=0):
try:
logger = getLogger(Logger.ANALYSER)
logger.info('analyse_scheduler %s start...' % (name))
queue = MessageQueue()
parser = LyricParser()
database = IRModel()
analyser = Analyser(parser, database)
while frequency:
frequency = frequency - 1
html = None
# 超时起点计时器
start_time = time.time()
while html is None:
html = queue.html
if html is None:
end_time = time.time()
if (end_time - start_time) > TIME_OUT:
logger.error(
"analyse_scheduler %s exited. Caused by : the time waitting for fish is too long."
% (name))
return
logger.info('Analyser %s start parsing the %dth text.' %
(name, frequency))
analyser.resolve(html)
logger.info('Analyser %s stop parsing the %dth text.' %
(name, frequency))
logger.info('analyse_scheduler %s end...' % (name))
except Exception as e:
logger.error('analyse_scheduler %s exited. Caused by: %s' %
(name, str(e)))
traceback.print_exc()
def scheduler(amount, frequency=1, analyser=True):
logger = getLogger(Logger.SCHEDULER)
logger.info('Scheduler start.')
queue = MessageQueue()
queue.clear()
analyser_pool = Pool()
fish_pool = Pool()
seed_pool = Pool()
for i in range(amount):
fish_pool.apply_async(fish_scheduler, args=(frequency, i))
seed_pool.apply_async(seed_scheduler, args=(frequency, i))
'''
网络 IO 时间远远大于本地解析时间
因此将解析进程始终设置为一个
'''
fish_pool.close()
seed_pool.close()
if analyser:
analyser_pool.apply_async(analyse_scheduler,
args=(frequency * amount, 0))
analyser_pool.close()
analyser_pool.join()
fish_pool.join()
seed_pool.join()
logger.info('Scheduler end.')
queue.clear()
def __init__(self):
self.__proxy_pool = ProxyPool()
self.__user_agent_pool = UserAgentPool()
self.__max_mask_count = 40
self.__mask_count = self.__max_mask_count
self.__config()
self.__logger = getLogger(Logger.DOWNLOADER)
def __init__(self, parser, database):
if not isinstance (database, Database):
raise ValueError('The database must be a subclass of Database')
if not isinstance (parser, Parser):
raise ValueError('The parser must be a subclass of Parser')
self.__logger = getLogger(Logger.ANALYSER)
self.__parser = parser
self.__database = database
def __init__(self):
self.__logger = getLogger(Logger.STORAGE)
self.__gen_path = GEN
# self.database_path = DATABASE
self.model_path = MODEL
self.word_doc_tf = self.model_path + "word_doc_tf/"
self.word_prefix_word = self.model_path + "word_prefix_word/"
self.prefix_doc_tf = self.model_path + "prefix_doc_tf/"
self.raw_lyrics_json = self.model_path + "lyrics.json"
self.detect()
def seed_scheduler(frequency, name=0):
logger = getLogger(Logger.SEEDER)
logger.info('seed_scheduler %s start.' % (name))
queue = MessageQueue()
seeder = Seeder()
while frequency:
seed = None
while seed is None or queue.exists(seed):
seed = seeder.seed()
queue.seed = seed
queue.repeat(seed)
logger.info('Seeder %s get the %dth seed : %s.' %
(name, frequency, seed))
frequency = frequency - 1
logger.info('seed_scheduler %s end.' % (name))
def proxy(self):
logger = getLogger(Logger.PROXY)
url = self.__proxy_pool_host + '/get'
try:
time.sleep(self.__sleep_time)
proxy = requests.get(url, proxies=self.__local_proxies, \
headers=self.__local_headers).content
except requests.exceptions.ConnectionError as e:
return self.__default_proxy
re_proxy = r'^\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}:\d{1,5}$'
if not re.match(re_proxy, proxy):
proxy = self.__default_proxy
else:
proxy = 'http://' + proxy
logger.info("Get a new proxy : %s." % proxy)
return proxy
def analyse_scheduler(frequency, name=0):
logger = getLogger(Logger.ANALYSER)
logger.info('analyse_scheduler %s start.' % (name))
queue = MessageQueue()
parser = LyricParser()
database = JsonFile()
analyser = Analyser(parser, database)
while frequency:
frequency = frequency - 1
html = None
while html is None:
html = queue.html
logger.info('Analyser %s start parsing the %dth text.' %
(name, frequency))
analyser.resolve(html)
logger.info('Analyser %s stop parsing the %dth text.' %
(name, frequency))
logger.info('analyse_scheduler %s end.' % (name))
def fish_scheduler(frequency, name=0):
try:
logger = getLogger(Logger.FISHER)
logger.info('fish_scheduler %s start..' % (name))
queue = MessageQueue()
fisher = Fisher()
while frequency:
frequency = frequency - 1
seed = None
html = None
# 超时起点计时器
start_time = time.time()
while seed is None:
seed = queue.seed
if seed is None:
end_time = time.time()
if (end_time - start_time) > TIME_OUT:
logger.error(
"fish_scheduler %s exited. Caused by : the time waitting for seed is too long."
% (name))
return
if seed == '':
html = ''
else:
seed = ('http://www.lyrics.com' + seed).encode('utf-8')
logger.info('Fisher %s start dealing the %dth seed : %s.' %
(name, frequency, seed))
html = fisher.fish(seed)
if html is None:
html = ''
queue.html = seed + "$S$E$E$D$" + html
logger.info('Fisher %s stop dealing the %dth seed : %s.' %
(name, frequency, seed))
logger.info('fish_scheduler %s end..' % (name))
except Exception as e:
logger.error('fish_scheduler %s exited. Caused by: %s' %
(name, str(e)))
traceback.print_exc()
def seed_scheduler(frequency, name=0):
try:
logger = getLogger(Logger.SEEDER)
logger.info('seed_scheduler %s start.' % (name))
queue = MessageQueue()
seeder = Seeder()
while frequency:
seed = None
while seed is None or queue.exists(seed):
seed = seeder.seed()
queue.seed = seed
queue.repeat(seed)
logger.info('Seeder %s get the %dth seed : %s.' %
(name, frequency, seed))
frequency = frequency - 1
logger.info('seed_scheduler %s end.' % (name))
except Exception as e:
logger.error('seed_scheduler %s exited. Caused by: %s' %
(name, str(e)))
traceback.print_exc()
def scheduler(amount, frequency=1):
logger = getLogger(Logger.SCHEDULER)
logger.info('Scheduler start.')
queue = MessageQueue()
queue.clear()
analyser_pool = Pool()
fish_pool = Pool()
seed_pool = Pool()
for i in range(amount):
analyser_pool.apply_async(analyse_scheduler, args=(frequency, i))
fish_pool.apply_async(fish_scheduler, args=(frequency, i))
seed_pool.apply_async(seed_scheduler, args=(frequency, i))
analyser_pool.close()
fish_pool.close()
seed_pool.close()
analyser_pool.join()
fish_pool.join()
seed_pool.join()
logger.info('Scheduler end.')
queue.clear()
from db import db_conn
from log.logger import getLogger
from data import tables_data
logger = getLogger()
def create_table():
for table_name in tables_data.TABLES:
table_description = tables_data.TABLES[table_name]
logger.debug("Creating table {}: ".format(table_name))
result, code = db_conn.execute_query(table_description)
if result is None and code != 200:
logger.error("Failed creating database " + str(code))
def get_emp_data():
get_emp_query = ("SELECT * FROM `employees`")
result, code = db_conn.execute_query(get_emp_query)
print(result)
print(code)
def __init__(self, database=DEFAULT_DATABASE_NAME):
self.database = DATABASE + DEFAULT_DATABASE_NAME
self.__logger = getLogger(Logger.STORAGE)
self.conn = None
self.cursor = None
self.detect()
import subprocess
from log import logger
log = logger.getLogger(__name__)
def run_cmd(*vargs, **dargs):
proc = subprocess.run(vargs,
**dargs,
stdout=subprocess.PIPE,
universal_newlines=True)
try:
proc.check_returncode()
except subprocess.CalledProcessError:
log.error(proc.stdout)
exit(1)
return proc
def checkout(branch, new=False):
cmd = ['git', 'checkout', branch]
if new:
cmd = cmd[:-1] + ['-b'] + cmd[-1:]
return run_cmd(*cmd)
def add_all():
return run_cmd('git', 'add', '-A')
def commit(msg, add_all=False):
def __init__(self):
super(LyricParser, self).__init__()
self.__logger = getLogger(Logger.ANALYSER)
import os
from flask import Flask
from flask.ext.restplus import Api, RestException
from flask_cors import CORS
from urlparse import urlsplit
from flask.ext.mongoalchemy import MongoAlchemy
from docs.app import UserDocument, EventDocument, AssistanceDocument, WatherDocument, VenueDocument
from datetime import timedelta
from log.logger import getLogger
from services.MailService import MailService
log = getLogger()
app = Flask(__name__)
app.debug = True
# Configuracion de app.
app.config['SECRET_KEY'] = 'super-secret'
app.config['JWT_REQUIRED_CLAIMS'] = []
app.config['JWT_EXPIRATION_DELTA'] = timedelta(hours=12)
app.config['BUNDLE_ERRORS'] = True
app.config['SEND_EMAILS'] = True
# Configuracion de MongoDB.
url = os.environ.get('MONGOLAB_URI', 'mongodb://localhost/enjoy-events')
app.config['MONGOALCHEMY_CONNECTION_STRING'] = url
parsed = urlsplit(url)
app.config['MONGOALCHEMY_DATABASE'] = parsed.path[1:]
db = MongoAlchemy(app)
CORS(app)
errors = {
import mysql.connector
from mysql.connector import errorcode
from db import db_const
from config import config
from log import logger
logObj = logger.getLogger()
class MySQL(object):
_instance = None
def __init__(self):
self.connection = self._instance.connection
self.cursor = self._instance.cursor
def __new__(cls):
if cls._instance is None:
cls._instance = object.__new__(cls)
# TODO: take this value from config.json
config_obj = config.load_config()
# db_config = {'database': config_obj.database_name, 'host': config_obj.db_host,
# 'password': config_obj.db_password, 'port': config_obj.db_port, 'user': config_obj.db_username}
db_config = {
'database': 'rbac',
'host': '127.0.0.1',
'password': '******',
'port': 3306,
'user': '******'
}
try:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
core struct graph, containing vertices and edges
"""
from log import logger
l = logger.getLogger("keepindoors/spark/spark.py")
from mongodb.mongo_cli import *
class graph(object):
"""
graph struct
"""
# nodes set V, containing all nodeId [nodeId1,nodeId2...]
vertices = set()
# edges set E, containing all edgeID [(v1,v2),(v3,v4)]. v is the nodeId ,and v1 < v2
edges = set()
# graphId, using uuid, for example 'da0d7ceb-4b97-11e8-aaec-8c8590861163'
id = ''
# graph info
info = {}
def loadEdgesFromMongo(self,mongo_cli,databbase_name,collection_name):
"""
load doc_pairs as Edges from mongo db
:param mongo_cli:
:param databbase_name:
:param collection_name:
:return:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
hive cli
"""
from log import logger
l = logger.getLogger("keepindoors/hive/hive_cli.py")
def loadRowsFromHive(self, spark, table_name):
"""
load row list from hive ,specifying the table_name
:param spark:
pyspark.sql.session.SparkSession
:param table_name:
string
:return:
list of pyspark.sql.Row
if table is not exist, return None
"""
try:
return spark.sql("select * from " + table_name).limit()
except:
l.error("Table':" + table_name + "' is not exist!")
return None
import os
from flask import Flask
from flask.ext.restplus import Api, RestException
from flask_cors import CORS
from urlparse import urlsplit
from flask.ext.mongoalchemy import MongoAlchemy
from docs.app import UserDocument, EventDocument, AssistanceDocument, WatherDocument, VenueDocument
from datetime import timedelta
from log.logger import getLogger
from services.MailService import MailService
log = getLogger()
app = Flask(__name__)
app.debug = True
# Configuracion de app.
app.config['SECRET_KEY'] = 'super-secret'
app.config['JWT_REQUIRED_CLAIMS'] = []
app.config['JWT_EXPIRATION_DELTA'] = timedelta(hours=12)
app.config['BUNDLE_ERRORS'] = True
app.config['SEND_EMAILS'] = True
# Configuracion de MongoDB.
url = os.environ.get('MONGOLAB_URI', 'mongodb://localhost/enjoy-events')
app.config['MONGOALCHEMY_CONNECTION_STRING'] = url
parsed = urlsplit(url)
app.config['MONGOALCHEMY_DATABASE'] = parsed.path[1:]
db = MongoAlchemy(app)
CORS(app)
global LOGOBJ
userIp = request.remote_addr
uUid = utils.getuUid()
LOGOBJ.debug("accessing /login by :" + str(userIp) + str(uUid))
userName = ""
passWord = ""
authSuccess = False
#if already logged in or if auth is not required we need to proceed to next page
if CONFIGOBJ.auth_enabled:
userName = str(request.form.get('username'))
passWord = str(request.form.get('password'))
#TODO: call post sql query with given username and password
#like
#authSuccess = sqlQuery()
authSuccess = True
if authSuccess == True:
#TODO: call the rbac server next layer i.e resource page
LOGOBJ.debug("Successful!")
return {'username': userName, 'password': passWord}
if __name__ == '__main__':
global CONFIGOBJ
global LOGOBJ
LOGOBJ = logger.getLogger()
CONFIGOBJ = config.load_config()
db_utils.create_table()
app.run(host='0.0.0.0',
port=CONFIGOBJ.rbac_port,
debug=True,
threaded=True)
def __init__(self, path='json/'):
self.__path = path
self.__logger = getLogger(Logger.ANALYSER)
if not os.path.exists(self.__path):
os.mkdir(self.__path)
