def __init__(self, prefix="http://www.lyrics.com/lyric/"): self.__file_system = FileSystem() self.__logger = getLogger(Logger.ANALYSER) self.__linguistics = Linguistics() self.__seed_prefix = prefix self.__model_writer = ModelWriter() self.__database = LyricDatabase()
def fish_scheduler(frequency, name=0): logger = getLogger(Logger.FISHER) logger.info('fish_scheduler %s start...' % (name)) queue = MessageQueue() fisher = Fisher() while frequency: frequency = frequency - 1 seed = None html = None while seed is None: seed = queue.seed if seed == '': html = '' else: seed = ('http://www.lyrics.com' + seed).encode('utf-8') logger.info('Fisher %s start dealing the %dth seed : %s.' % (name, frequency, seed)) html = fisher.fish(seed) if html is None: html = '' queue.html = html logger.info('Fisher %s stop dealing the %dth seed : %s.' % (name, frequency, seed)) logger.info('fish_scheduler %s end...' % (name))
def analyse_scheduler(frequency, name=0): try: logger = getLogger(Logger.ANALYSER) logger.info('analyse_scheduler %s start...' % (name)) queue = MessageQueue() parser = LyricParser() database = IRModel() analyser = Analyser(parser, database) while frequency: frequency = frequency - 1 html = None # 超时起点计时器 start_time = time.time() while html is None: html = queue.html if html is None: end_time = time.time() if (end_time - start_time) > TIME_OUT: logger.error( "analyse_scheduler %s exited. Caused by : the time waitting for fish is too long." % (name)) return logger.info('Analyser %s start parsing the %dth text.' % (name, frequency)) analyser.resolve(html) logger.info('Analyser %s stop parsing the %dth text.' % (name, frequency)) logger.info('analyse_scheduler %s end...' % (name)) except Exception as e: logger.error('analyse_scheduler %s exited. Caused by: %s' % (name, str(e))) traceback.print_exc()
def scheduler(amount, frequency=1, analyser=True): logger = getLogger(Logger.SCHEDULER) logger.info('Scheduler start.') queue = MessageQueue() queue.clear() analyser_pool = Pool() fish_pool = Pool() seed_pool = Pool() for i in range(amount): fish_pool.apply_async(fish_scheduler, args=(frequency, i)) seed_pool.apply_async(seed_scheduler, args=(frequency, i)) ''' 网络 IO 时间远远大于本地解析时间 因此将解析进程始终设置为一个 ''' fish_pool.close() seed_pool.close() if analyser: analyser_pool.apply_async(analyse_scheduler, args=(frequency * amount, 0)) analyser_pool.close() analyser_pool.join() fish_pool.join() seed_pool.join() logger.info('Scheduler end.') queue.clear()
def __init__(self): self.__proxy_pool = ProxyPool() self.__user_agent_pool = UserAgentPool() self.__max_mask_count = 40 self.__mask_count = self.__max_mask_count self.__config() self.__logger = getLogger(Logger.DOWNLOADER)
def __init__(self, parser, database): if not isinstance (database, Database): raise ValueError('The database must be a subclass of Database') if not isinstance (parser, Parser): raise ValueError('The parser must be a subclass of Parser') self.__logger = getLogger(Logger.ANALYSER) self.__parser = parser self.__database = database
def __init__(self): self.__logger = getLogger(Logger.STORAGE) self.__gen_path = GEN # self.database_path = DATABASE self.model_path = MODEL self.word_doc_tf = self.model_path + "word_doc_tf/" self.word_prefix_word = self.model_path + "word_prefix_word/" self.prefix_doc_tf = self.model_path + "prefix_doc_tf/" self.raw_lyrics_json = self.model_path + "lyrics.json" self.detect()
def seed_scheduler(frequency, name=0): logger = getLogger(Logger.SEEDER) logger.info('seed_scheduler %s start.' % (name)) queue = MessageQueue() seeder = Seeder() while frequency: seed = None while seed is None or queue.exists(seed): seed = seeder.seed() queue.seed = seed queue.repeat(seed) logger.info('Seeder %s get the %dth seed : %s.' % (name, frequency, seed)) frequency = frequency - 1 logger.info('seed_scheduler %s end.' % (name))
def proxy(self): logger = getLogger(Logger.PROXY) url = self.__proxy_pool_host + '/get' try: time.sleep(self.__sleep_time) proxy = requests.get(url, proxies=self.__local_proxies, \ headers=self.__local_headers).content except requests.exceptions.ConnectionError as e: return self.__default_proxy re_proxy = r'^\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}:\d{1,5}$' if not re.match(re_proxy, proxy): proxy = self.__default_proxy else: proxy = 'http://' + proxy logger.info("Get a new proxy : %s." % proxy) return proxy
def analyse_scheduler(frequency, name=0): logger = getLogger(Logger.ANALYSER) logger.info('analyse_scheduler %s start.' % (name)) queue = MessageQueue() parser = LyricParser() database = JsonFile() analyser = Analyser(parser, database) while frequency: frequency = frequency - 1 html = None while html is None: html = queue.html logger.info('Analyser %s start parsing the %dth text.' % (name, frequency)) analyser.resolve(html) logger.info('Analyser %s stop parsing the %dth text.' % (name, frequency)) logger.info('analyse_scheduler %s end.' % (name))
def fish_scheduler(frequency, name=0): try: logger = getLogger(Logger.FISHER) logger.info('fish_scheduler %s start..' % (name)) queue = MessageQueue() fisher = Fisher() while frequency: frequency = frequency - 1 seed = None html = None # 超时起点计时器 start_time = time.time() while seed is None: seed = queue.seed if seed is None: end_time = time.time() if (end_time - start_time) > TIME_OUT: logger.error( "fish_scheduler %s exited. Caused by : the time waitting for seed is too long." % (name)) return if seed == '': html = '' else: seed = ('http://www.lyrics.com' + seed).encode('utf-8') logger.info('Fisher %s start dealing the %dth seed : %s.' % (name, frequency, seed)) html = fisher.fish(seed) if html is None: html = '' queue.html = seed + "$S$E$E$D$" + html logger.info('Fisher %s stop dealing the %dth seed : %s.' % (name, frequency, seed)) logger.info('fish_scheduler %s end..' % (name)) except Exception as e: logger.error('fish_scheduler %s exited. Caused by: %s' % (name, str(e))) traceback.print_exc()
def seed_scheduler(frequency, name=0): try: logger = getLogger(Logger.SEEDER) logger.info('seed_scheduler %s start.' % (name)) queue = MessageQueue() seeder = Seeder() while frequency: seed = None while seed is None or queue.exists(seed): seed = seeder.seed() queue.seed = seed queue.repeat(seed) logger.info('Seeder %s get the %dth seed : %s.' % (name, frequency, seed)) frequency = frequency - 1 logger.info('seed_scheduler %s end.' % (name)) except Exception as e: logger.error('seed_scheduler %s exited. Caused by: %s' % (name, str(e))) traceback.print_exc()
def scheduler(amount, frequency=1): logger = getLogger(Logger.SCHEDULER) logger.info('Scheduler start.') queue = MessageQueue() queue.clear() analyser_pool = Pool() fish_pool = Pool() seed_pool = Pool() for i in range(amount): analyser_pool.apply_async(analyse_scheduler, args=(frequency, i)) fish_pool.apply_async(fish_scheduler, args=(frequency, i)) seed_pool.apply_async(seed_scheduler, args=(frequency, i)) analyser_pool.close() fish_pool.close() seed_pool.close() analyser_pool.join() fish_pool.join() seed_pool.join() logger.info('Scheduler end.') queue.clear()
from db import db_conn from log.logger import getLogger from data import tables_data logger = getLogger() def create_table(): for table_name in tables_data.TABLES: table_description = tables_data.TABLES[table_name] logger.debug("Creating table {}: ".format(table_name)) result, code = db_conn.execute_query(table_description) if result is None and code != 200: logger.error("Failed creating database " + str(code)) def get_emp_data(): get_emp_query = ("SELECT * FROM `employees`") result, code = db_conn.execute_query(get_emp_query) print(result) print(code)
def __init__(self, database=DEFAULT_DATABASE_NAME): self.database = DATABASE + DEFAULT_DATABASE_NAME self.__logger = getLogger(Logger.STORAGE) self.conn = None self.cursor = None self.detect()
import subprocess from log import logger log = logger.getLogger(__name__) def run_cmd(*vargs, **dargs): proc = subprocess.run(vargs, **dargs, stdout=subprocess.PIPE, universal_newlines=True) try: proc.check_returncode() except subprocess.CalledProcessError: log.error(proc.stdout) exit(1) return proc def checkout(branch, new=False): cmd = ['git', 'checkout', branch] if new: cmd = cmd[:-1] + ['-b'] + cmd[-1:] return run_cmd(*cmd) def add_all(): return run_cmd('git', 'add', '-A') def commit(msg, add_all=False):
def __init__(self): super(LyricParser, self).__init__() self.__logger = getLogger(Logger.ANALYSER)
import os from flask import Flask from flask.ext.restplus import Api, RestException from flask_cors import CORS from urlparse import urlsplit from flask.ext.mongoalchemy import MongoAlchemy from docs.app import UserDocument, EventDocument, AssistanceDocument, WatherDocument, VenueDocument from datetime import timedelta from log.logger import getLogger from services.MailService import MailService log = getLogger() app = Flask(__name__) app.debug = True # Configuracion de app. app.config['SECRET_KEY'] = 'super-secret' app.config['JWT_REQUIRED_CLAIMS'] = [] app.config['JWT_EXPIRATION_DELTA'] = timedelta(hours=12) app.config['BUNDLE_ERRORS'] = True app.config['SEND_EMAILS'] = True # Configuracion de MongoDB. url = os.environ.get('MONGOLAB_URI', 'mongodb://localhost/enjoy-events') app.config['MONGOALCHEMY_CONNECTION_STRING'] = url parsed = urlsplit(url) app.config['MONGOALCHEMY_DATABASE'] = parsed.path[1:] db = MongoAlchemy(app) CORS(app) errors = {
import mysql.connector from mysql.connector import errorcode from db import db_const from config import config from log import logger logObj = logger.getLogger() class MySQL(object): _instance = None def __init__(self): self.connection = self._instance.connection self.cursor = self._instance.cursor def __new__(cls): if cls._instance is None: cls._instance = object.__new__(cls) # TODO: take this value from config.json config_obj = config.load_config() # db_config = {'database': config_obj.database_name, 'host': config_obj.db_host, # 'password': config_obj.db_password, 'port': config_obj.db_port, 'user': config_obj.db_username} db_config = { 'database': 'rbac', 'host': '127.0.0.1', 'password': '******', 'port': 3306, 'user': '******' } try:
#!/usr/bin/env python # -*- coding: utf-8 -*- """ core struct graph, containing vertices and edges """ from log import logger l = logger.getLogger("keepindoors/spark/spark.py") from mongodb.mongo_cli import * class graph(object): """ graph struct """ # nodes set V, containing all nodeId [nodeId1,nodeId2...] vertices = set() # edges set E, containing all edgeID [(v1,v2),(v3,v4)]. v is the nodeId ,and v1 < v2 edges = set() # graphId, using uuid, for example 'da0d7ceb-4b97-11e8-aaec-8c8590861163' id = '' # graph info info = {} def loadEdgesFromMongo(self,mongo_cli,databbase_name,collection_name): """ load doc_pairs as Edges from mongo db :param mongo_cli: :param databbase_name: :param collection_name: :return:
#!/usr/bin/env python # -*- coding: utf-8 -*- """ hive cli """ from log import logger l = logger.getLogger("keepindoors/hive/hive_cli.py") def loadRowsFromHive(self, spark, table_name): """ load row list from hive ,specifying the table_name :param spark: pyspark.sql.session.SparkSession :param table_name: string :return: list of pyspark.sql.Row if table is not exist, return None """ try: return spark.sql("select * from " + table_name).limit() except: l.error("Table':" + table_name + "' is not exist!") return None
import os from flask import Flask from flask.ext.restplus import Api, RestException from flask_cors import CORS from urlparse import urlsplit from flask.ext.mongoalchemy import MongoAlchemy from docs.app import UserDocument, EventDocument, AssistanceDocument, WatherDocument, VenueDocument from datetime import timedelta from log.logger import getLogger from services.MailService import MailService log = getLogger() app = Flask(__name__) app.debug = True # Configuracion de app. app.config['SECRET_KEY'] = 'super-secret' app.config['JWT_REQUIRED_CLAIMS'] = [] app.config['JWT_EXPIRATION_DELTA'] = timedelta(hours=12) app.config['BUNDLE_ERRORS'] = True app.config['SEND_EMAILS'] = True # Configuracion de MongoDB. url = os.environ.get('MONGOLAB_URI', 'mongodb://localhost/enjoy-events') app.config['MONGOALCHEMY_CONNECTION_STRING'] = url parsed = urlsplit(url) app.config['MONGOALCHEMY_DATABASE'] = parsed.path[1:] db = MongoAlchemy(app) CORS(app)
global LOGOBJ userIp = request.remote_addr uUid = utils.getuUid() LOGOBJ.debug("accessing /login by :" + str(userIp) + str(uUid)) userName = "" passWord = "" authSuccess = False #if already logged in or if auth is not required we need to proceed to next page if CONFIGOBJ.auth_enabled: userName = str(request.form.get('username')) passWord = str(request.form.get('password')) #TODO: call post sql query with given username and password #like #authSuccess = sqlQuery() authSuccess = True if authSuccess == True: #TODO: call the rbac server next layer i.e resource page LOGOBJ.debug("Successful!") return {'username': userName, 'password': passWord} if __name__ == '__main__': global CONFIGOBJ global LOGOBJ LOGOBJ = logger.getLogger() CONFIGOBJ = config.load_config() db_utils.create_table() app.run(host='0.0.0.0', port=CONFIGOBJ.rbac_port, debug=True, threaded=True)
def __init__(self, path='json/'): self.__path = path self.__logger = getLogger(Logger.ANALYSER) if not os.path.exists(self.__path): os.mkdir(self.__path)