def main(): cfg = ConfigManager(config_parms) parser = argparse.ArgumentParser(description="Load Trillium Bridge map into RF2 ComplexMap file") parser.add_argument('configfile', help="Configuration file location") parser.add_argument('infile', help="Input TSV file") parser.add_argument('-f', '--firstrow', metavar="first row", help="First row with data (1 based)", type=int, default=2) parser.add_argument('-m', '--moduleId', type=int, help="Module Identifier", required=True) parser.add_argument('-r', '--refsetId', type=int, help="Refset Identifier", required=True) opts = parser.parse_args() cfg.set_configfile(opts.configfile) # Throws assertionError if the id isn't valid mid = sctid(opts.moduleId) rid = sctid(opts.refsetId) with open(opts.infile, 'rU') as csvfile: reader = csv.reader(csvfile, delimiter='\t') zip(range(1, opts.firstrow), reader) # Skip nrows rows [procrow(row, opts) for row in reader] if entries: print("Writing records to table") db = ComplexMapDB() for s in range(0, len(entries), 1000): writeblock(entries[s:s+chunksize], db.connect(), db._tname(True))
class SendMail(object): config = ConfigManager() def __init__(self, title, content, file=None): self.username = self.config.yaml_setting['email']['username'] self.passwd = self.config.yaml_setting['email']['passwd'] self.recv = self.config.yaml_setting['email']['recv'] self.email_host = self.config.yaml_setting['email']['email_host'] self.port = self.config.yaml_setting['email']['port'] self.title = title self.content = content self.file = file def send_mail(self): msg = MIMEMultipart() #发送内容的对象 if self.file: #处理附件的 att = MIMEText(open(self.file).read()) att["Content-Type"] = 'application/octet-stream' att["Content-Disposition"] = 'attachment; filename="%s"' % self.file msg.attach(att) msg.attach(MIMEText(self.content)) #邮件正文的内容 msg['Subject'] = self.title # 邮件主题 msg['From'] = self.username # 发送者账号 msg['To'] = ";".join(self.recv) # 接收者账号列表 self.smtp = smtplib.SMTP(self.email_host, port=self.port) #发送邮件服务器的对象 self.smtp.login(self.username, self.passwd) try: self.smtp.sendmail(self.username, self.recv, msg.as_string()) except Exception as e: logging.exception(e) def __del__(self): self.smtp.quit()
class MongoDBClient(object): config = ConfigManager() mongo_account = config.yaml_setting['datasouce']['mongoBD']['url'] mongo_database = config.yaml_setting['datasouce']['mongoBD']['database'] def __init__(self): """初始化连接""" try: client = MongoClient(self.mongo_account) self._db = client[self.mongo_database] except Exception as e: logging.exception(e) def get_conn(self): return self._db def insert(self, table, result): db = self.get_conn() collection = db[table] collection.insert_one(result) def __getattr__(self, command): """ 方法重载 :param command: :return: """ def _(*args): return getattr(self.get_conn(), command)(*args) # 重新组装方法调用 return _
class RedisClient(object): config = ConfigManager() redis_account = config.yaml_setting['datasouce']['redis']['url'] def __init__(self): """初始化连接""" pool = ConnectionPool.from_url(self.redis_account) self._redis = StrictRedis(connection_pool=pool) @property def get_conn(self): """ :return: 获取连接 """ return self._redis
def __init__(self): # the path of this sensor self.__path = os.getcwd() # configuration manager for sensor properties self.__config_manager = ConfigManager() self.__config_id = "" # the ID of this sensor self.__sensor_id = "" # the configuration watcher callback self.__config_watcher = None # flag for stopping the sensor self.__stop_flag = False # when stopping, the flag is set to True, then the callback is called self.__stop_callback = None self.__defaults = {}
class DataBaseExport(object): config = ConfigManager() base_table = config.yaml_setting['base_name'] def __init__(self): """ 获取各数据库连接池 :param name: :param result: """ self.con_pg = PostgreClient() if DATAFLAG['postgre'] else None self.con_mysql = MysqlClient() if DATAFLAG['mysql'] else None self.con_mongo = MongoDBClient() if DATAFLAG['mongo'] else None def output_base(self, name, result): """ 不同数据库写入操作 :param name: :param result: :return: """ if self.con_pg: self.insert_postgre(name, result) if self.con_mysql: self.insert_mysql(name, result) if self.con_mongo: self.insert_mongo(name, result) def insert_postgre(self, table, datas={}): flag = False try: db = self.con_pg.get_conn() cursor = db.cursor() keys = ','.join(datas.keys()) values = ','.join(['%s'] * len(datas)) table = self.base_table + table sql = 'insert into {table}({keys}) values ({values})'.format( table=table, keys=keys, values=values) cursor.execute(sql, tuple(datas.values())) db.commit() cursor.close() db.close() flag = True except Exception as e: print("查询失败->%s" % e) db.rollback() finally: return flag def insert_mongo(self, table, result): db = self.con_mongo.get_conn() collection = db[table] collection.insert_one(result) def insert_mysql(self, table, datas={}): flag = False try: db = self.con_mysql.get_conn() cursor = db.cursor() keys = ','.join(datas.keys()) values = ','.join(['%s'] * len(datas)) sql = 'insert into {table}({keys}) values ({values})'.format( table=table, keys=keys, values=values) cursor.execute(sql, tuple(datas.values())) db.commit() cursor.close() db.close() flag = True except Exception as e: db.rollback() print("执行失败->%s" % e) finally: return flag
For more information on this file, see https://docs.djangoproject.com/en/3.1/topics/settings/ For the full list of settings and their values, see https://docs.djangoproject.com/en/3.1/ref/settings/ """ from pathlib import Path from config.ConfigManager import ConfigManager # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve(strict=True).parent.parent # ConfigManager provides configuration settings for e.g. database CONFIG_MANAGER = ConfigManager() # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/3.1/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = 'a102w%tdryj2xib)_@d6^lye$2w3+icu^o*e3b3vx7uf#-*r9n' # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True ALLOWED_HOSTS = [] # Application definition INSTALLED_APPS = [
class Sensor(Observable): OBS_OUTPUT = 0 OBS_CMD_CONFIGURE = 1 OBS_CMD_REMOVE = 2 OBS_CMD_RESTART = 3 OBS_CMD_DUPLICATE = 4 OBS_CMD_MENU = 5 def __init__(self): # the path of this sensor self.__path = os.getcwd() # configuration manager for sensor properties self.__config_manager = ConfigManager() self.__config_id = "" # the ID of this sensor self.__sensor_id = "" # the configuration watcher callback self.__config_watcher = None # flag for stopping the sensor self.__stop_flag = False # when stopping, the flag is set to True, then the callback is called self.__stop_callback = None self.__defaults = {} # # Sets the ID of this sensor. # def set_id(self, ident): self.__id = ident # #:function get_id | | # Returns the ID of this sensor. It is not yet valid in the # constructor. #: return string #:/function # def get_id(self): assert self.__id, "The ID is invalid in the constructor." return self.__id # # Sets the configuration ID for this sensor. # If the user has set a callback function for watching the config, # the config will be watched. # def set_config_id(self, config_id): self.__config_id = str(config_id) if (self.__config_watcher): Sensor.watch_config(self, self.__config_watcher) # #:function get_config_id | | # Returns the unique config ID of the sensor. You can use the ID # if you need to store data in other places. #: return string #:/function # def get_config_id(self): return self.__config_id # #:function set_config | key, value | # Stores the given value in the configuration base. #: param key | string | The key name. #: param value | string | The value to set. #:/function # def set_config(self, key, value): assert(self.__config_manager) if (not self.__stop_flag): run_in_main_thread(self.__config_manager.set, self.__config_id, key, value) # #:function get_config | key | # Returns the configuration value for the given key. #: param key | string | The key name. #: return string #:/function # def get_config(self, key): assert(self.__config_manager) value = run_in_main_thread(self.__config_manager.get, self.__config_id, key) if (value == self.__config_manager.UNDEF): value = self.__defaults.get(key, "") return value # #:function watch_config | callback | # Registers a watcher for config changes. #: param callback | function | The callback function for configuration # changes. #:/function # def watch_config(self, callback): self.__config_watcher = callback if (self.get_config_id()): self.__config_manager.watch(self.get_config_id(), self.__on_watch_config) # #:function _set_config_type | key, type, default | # Sets the data types to be used for the configuration values. # Use this method in the constructor of your sensor. #: param key | string | The name of the configuration key. #: param type | enum | The data type of the configuration key. #: param default | string | The default value or unset keys. #:/function # def _set_config_type(self, key, dtype, default): self.__defaults[key] = default # #:function get_path | | Returns the filesystem path of the sensor. Use this # method if you want to load resource files that come # with the sensor. #: return string #:/function # def get_path(self): return self.__path # #:function set_path_to_purge | paths | # Sets the paths which are to be removed when the desklet that # uses the sensor gets removed. # If your sensor creates files, you need to specify the paths here # in order to clean up. #: param paths | string list | The paths to purge. #:/function # def set_paths_to_purge(self, paths): key = PURGE_KEY value = ",".join(paths) self.__config_manager.set(self.__config_id, key, value) # #:function new_output || Returns a new empty [cmd:TargetSettings] object for sending data to # the display. #: return TargetSettings #:/function # def new_output(self): return TargetSettings() # #:function send_output | output | Sends the given [cmd:TargetSettings] object to the display. # [emph:Never call this method from within a thread!] #: param output | TargetSettings | The object for sending to the display. #:/function # def send_output(self, output): run_in_main_thread(self.update_observer, self.OBS_OUTPUT, output) # #:function add_timer | interval, callback, *args | Adds a timeout function with the given # interval in ms. #: param interval | int | The timeout interval between each invokation of the callback. #: param callback | function | The callback function. #:/function # def add_timer(self, interval, callback, *args): def __wrap(self, callback, args): if (not self.__stop_flag): try: return callback(*args) except: from utils.error import Error Error().handle("unknown") return False else: return False return gobject.timeout_add(interval, __wrap, self, callback, args) # #:function add_thread | threadfunction, *args | Adds and runs a new thread. # Use this to start new threads. It's recommended to put blocking actions into threads # in order to not block [app:gDesklets]. #: param threadfunction | function | The thread function. #:/function # def add_thread(self, threadfunction, *args): # the thread should not start before setup is complete, therefore # we are using the GTK idle handler def run_thread(threadfunction, args): thread.start_new_thread(threadfunction, args) return False gobject.idle_add(run_thread, threadfunction, args) # # Sends an action to this sensor. # def send_action(self, call, path, args = []): if (not self.__stop_flag): try: self.call_action(call, path, args) except: from utils.error import Error Error().handle("unknown") # #:function call_function | call, path, *args | Method for handling action calls from the # display. Sensors have to override this method. [emph:This method may soon be # deprecated!] #: param call | string | The function to call. #: param path | int list | The path of the target on which the action occurred. #: param args | any list | The list of arguments for the function call. #:/function # def call_action(self, call, path, args = []): raise NotImplementedError def watch_stop(self, callback): """ Registers a callback that will be called when sensor will be stopped. @param callback: @type callback: callable(callback) """ self.__stop_callback = callback # # Stops this sensor and cleans up. # def stop(self): self.__stop_flag = True if (self.__stop_callback): self.__stop_callback() del self.__config_watcher self.__config_manager.remove_watcher(self.get_config_id(), self.__on_watch_config) # shut down sensor self._shutdown() # #:function is_stopped | | Returns whether this sensor has been stopped. Use this method to # check if your threads have to terminate. #: return bool #:/function # def is_stopped(self): return self.__stop_flag # #:function _shutdown | | Executes tasks for sensor shutdown. Override this method if your # sensor has to clean up things after it has been terminated. #:/function # def _shutdown(self): pass # #:function _new_configurator | | Creates and returns an empty new sensor configurator. # [emph:This method may soon be deprecated!] #: return SensorConfigurator. #:/function # def _new_configurator(self): return SensorConfigurator(self.set_config, self.get_config) # # Returns the configurator of this sensor. # def get_configurator(self): pass # # Method for duplicating the display using this sensor # def _duplicate_display(self): self.update_observer(self.OBS_CMD_DUPLICATE) # #:function open_menu | menu | Opens the given popup menu. The menu is a list of # [cmd:(label:str, sensitive:bool, submenu:list, callback:function, args: list)] # tuples. #: param menu | list | The menu to open. #:/function # def open_menu(self, menu): menu += [()] run_in_main_thread(self.update_observer, self.OBS_CMD_MENU, menu) # # Reacts on changes in the configuration and delegates the call to the # user's callback. # def __on_watch_config(self, path, value): assert(self.__config_watcher) key = path[-1] self.__config_watcher(key, value) def __on_configure(self, src): self.update_observer(self.OBS_CMD_CONFIGURE, None) def __on_remove(self, src): self.update_observer(self.OBS_CMD_REMOVE, None) def __on_restart(self, src): self.update_observer(self.OBS_CMD_RESTART, None) # some required aliases for older sensors to work _get_id = get_id _get_path = get_path _new_output = new_output _send_output = send_output _add_timer = add_timer _add_thread = add_thread _is_stopped = is_stopped _get_config_id = get_config_id _watch_config = watch_config _set_config = set_config _get_config = get_config _set_paths_to_purge = set_paths_to_purge _open_menu = open_menu
class PostgreClient(object): config = ConfigManager() postgre_account = config.yaml_setting['datasouce']['postgresql'] def __init__(self): """ 初始化连接 """ try: self._pool = PersistentDB(creator=pg8000, maxusage=None, threadlocal=None, setsession=[], ping=0, closeable=False, **self.postgre_account) except Exception as e: logging.exception(e) def get_conn(self): """取数据库连接""" return self._pool.connection() def insert(self, table, datas={}): flag = False try: db = self.get_conn() cursor = db.cursor() keys = ','.join(datas.keys()) values = ','.join(['%s'] * len(datas)) sql = 'insert into {table}({keys}) values ({values})'.format(table=table, keys=keys, values=values) cursor.execute(sql, tuple(datas.values())) db.commit() cursor.close() db.close() flag = True except Exception as e: logging.exception(e) db.rollback() finally: return flag def query(self, sql): """ 数据库查询操作 :param sql:需要查询的语句 :return:返回元组 """ results = () try: db = self.get_conn() cursor = db.cursor() cursor.execute(sql) results = cursor.fetchall() db.commit() cursor.close() db.close() except Exception as e: logging.exception(e) db.rollback() finally: return results def split_sql(self,sql): sqllist = sql.split(';') return sqllist[0:-1] def noquery(self, sqls): """ 数据库非查询操作,删除,建表,赋权,更新等操作 :param sql:需要执行的语句,可支持多条以";"为分隔的事务操作,每一句sql必须加";" !!! :return:返回逻辑值 """ flag = False try: db = self.get_conn() cursor = db.cursor() #cursor.execute(sql) for sql in self.split_sql(sqls): cursor.execute(sql) db.commit() cursor.close() db.close() flag = True except Exception as e: logging.exception(e) db.rollback() finally: return flag
def __init__(self): self.configManager = ConfigManager() self.API = Auth.getAPI(self.configManager.CONFIG_PATH)
class Querier: def __init__(self): self.configManager = ConfigManager() self.API = Auth.getAPI(self.configManager.CONFIG_PATH) def queryByHashtag(self, HT, since, until, resultType="recent", items=100, lang="es", savedEnabled=True): query_term = "#" + HT tweets = tweepy.Cursor(self.API.search, q=query_term, result_type=resultType, since=since, until=until, lang=lang, show_user=True).items(items) query_dir = None if savedEnabled: query_dir = self.configManager.newQueryPath(query_term) query = FM.saveTweetQuery(tweets, os.path.join(query_dir, 'tweets.json'), verbose=True) else: query = FM.parseTweetQuery(tweets) return [query, query_dir] def queryByUser(self, username, since, until, resultType="recent", items=100, savedEnabled=True): tweets = tweepy.Cursor(self.API.user_timeline, id=username, result_type=resultType, since=since, until=until).items(items) if savedEnabled: query_dir = self.configManager.newQueryPath(username) query = FM.saveTweetQuery(tweets, os.path.join(query_dir, 'tweets.json'), verbose=True) else: query = FM.parseTweetQuery(tweets) return [query, query_dir] def loadSavedQuery(self, term, time): path = os.path.join(self.configManager.DATA_PATH, (time + " " + term).replace(" ", "_")) return [FM.loadTweetQuery(path), path] def getGeneratedQueryPath(self, term, time): return os.path.join(self.configManager.DATA_PATH, (time + " " + term).replace(" ", "_"), 'generated') def getRateLimits(): json.dumps(self.API.rate_limit_status(), sort_keys=True, indent=2, separators=(',', ': '))
class MysqlClient(object): config = ConfigManager() mysql_account = config.yaml_setting['datasouce']['mysql'] def __init__(self): """ 初始化连接 """ try: self._pool = PersistentDB(pymysql, maxusage=None, threadlocal=None, closeable=False, **self.mysql_account) except Exception as e: logging.exception(e) def get_conn(self): """取数据库连接""" return self._pool.connection() def insert(self, table, datas={}): flag = False try: db = self.get_conn() cursor = db.cursor() keys = ','.join(datas.keys()) values = ','.join(['%s'] * len(datas)) sql = 'insert into {table}({keys}) values ({values})'.format( table=table, keys=keys, values=values) #print(sql) cursor.execute(sql, tuple(datas.values())) db.commit() cursor.close() db.close() flag = True except Exception as e: db.rollback() logging.exception(e) finally: return flag def insertmany(self, table, datas=None): flag = False try: db = self.get_conn() cursor = db.cursor() keys = ','.join(datas[0].keys()) values = ','.join(['%s'] * len(datas[0])) sql = 'insert into {table}({keys}) values ({values})'.format( table=table, keys=keys, values=values) # print(sql) # print([tuple(data.values()) for data in datas]) cursor.executemany(sql, [tuple(data.values()) for data in datas]) db.commit() cursor.close() db.close() flag = True except Exception as e: db.rollback() logging.exception(e) finally: return flag def query(self, sql): """ 数据库查询操作 :param sql:需要查询的语句 :return:返回元组 """ results = () try: db = self.get_conn() cursor = db.cursor() cursor.execute(sql) results = cursor.fetchall() db.commit() cursor.close() db.close() except Exception as e: db.rollback() logging.exception(e) finally: return results def noquery(self, sql): """ 数据库非查询操作,删除,建表,赋权,更新等操作 :param sql:需要执行的语句 :return:返回元组 """ flag = False try: db = self.get_conn() cursor = db.cursor() cursor.execute(sql) db.commit() cursor.close() db.close() flag = True except Exception as e: db.rollback() logging.exception(e) finally: return flag