def main():
cfg = ConfigManager(config_parms)
parser = argparse.ArgumentParser(description="Load Trillium Bridge map into RF2 ComplexMap file")
parser.add_argument('configfile', help="Configuration file location")
parser.add_argument('infile', help="Input TSV file")
parser.add_argument('-f', '--firstrow', metavar="first row", help="First row with data (1 based)", type=int, default=2)
parser.add_argument('-m', '--moduleId', type=int, help="Module Identifier", required=True)
parser.add_argument('-r', '--refsetId', type=int, help="Refset Identifier", required=True)
opts = parser.parse_args()
cfg.set_configfile(opts.configfile)
# Throws assertionError if the id isn't valid
mid = sctid(opts.moduleId)
rid = sctid(opts.refsetId)
with open(opts.infile, 'rU') as csvfile:
reader = csv.reader(csvfile, delimiter='\t')
zip(range(1, opts.firstrow), reader) # Skip nrows rows
[procrow(row, opts) for row in reader]
if entries:
print("Writing records to table")
db = ComplexMapDB()
for s in range(0, len(entries), 1000):
writeblock(entries[s:s+chunksize], db.connect(), db._tname(True))
class SendMail(object):
config = ConfigManager()
def __init__(self, title, content, file=None):
self.username = self.config.yaml_setting['email']['username']
self.passwd = self.config.yaml_setting['email']['passwd']
self.recv = self.config.yaml_setting['email']['recv']
self.email_host = self.config.yaml_setting['email']['email_host']
self.port = self.config.yaml_setting['email']['port']
self.title = title
self.content = content
self.file = file
def send_mail(self):
msg = MIMEMultipart()
#发送内容的对象
if self.file: #处理附件的
att = MIMEText(open(self.file).read())
att["Content-Type"] = 'application/octet-stream'
att["Content-Disposition"] = 'attachment; filename="%s"' % self.file
msg.attach(att)
msg.attach(MIMEText(self.content)) #邮件正文的内容
msg['Subject'] = self.title # 邮件主题
msg['From'] = self.username # 发送者账号
msg['To'] = ";".join(self.recv) # 接收者账号列表
self.smtp = smtplib.SMTP(self.email_host, port=self.port)
#发送邮件服务器的对象
self.smtp.login(self.username, self.passwd)
try:
self.smtp.sendmail(self.username, self.recv, msg.as_string())
except Exception as e:
logging.exception(e)
def __del__(self):
self.smtp.quit()
class MongoDBClient(object):
config = ConfigManager()
mongo_account = config.yaml_setting['datasouce']['mongoBD']['url']
mongo_database = config.yaml_setting['datasouce']['mongoBD']['database']
def __init__(self):
"""初始化连接"""
try:
client = MongoClient(self.mongo_account)
self._db = client[self.mongo_database]
except Exception as e:
logging.exception(e)
def get_conn(self):
return self._db
def insert(self, table, result):
db = self.get_conn()
collection = db[table]
collection.insert_one(result)
def __getattr__(self, command):
"""
方法重载
:param command:
:return:
"""
def _(*args):
return getattr(self.get_conn(), command)(*args) # 重新组装方法调用
return _
class RedisClient(object):
config = ConfigManager()
redis_account = config.yaml_setting['datasouce']['redis']['url']
def __init__(self):
"""初始化连接"""
pool = ConnectionPool.from_url(self.redis_account)
self._redis = StrictRedis(connection_pool=pool)
@property
def get_conn(self):
"""
:return: 获取连接
"""
return self._redis
def __init__(self):
# the path of this sensor
self.__path = os.getcwd()
# configuration manager for sensor properties
self.__config_manager = ConfigManager()
self.__config_id = ""
# the ID of this sensor
self.__sensor_id = ""
# the configuration watcher callback
self.__config_watcher = None
# flag for stopping the sensor
self.__stop_flag = False
# when stopping, the flag is set to True, then the callback is called
self.__stop_callback = None
self.__defaults = {}
class DataBaseExport(object):
config = ConfigManager()
base_table = config.yaml_setting['base_name']
def __init__(self):
"""
获取各数据库连接池
:param name:
:param result:
"""
self.con_pg = PostgreClient() if DATAFLAG['postgre'] else None
self.con_mysql = MysqlClient() if DATAFLAG['mysql'] else None
self.con_mongo = MongoDBClient() if DATAFLAG['mongo'] else None
def output_base(self, name, result):
"""
不同数据库写入操作
:param name:
:param result:
:return:
"""
if self.con_pg:
self.insert_postgre(name, result)
if self.con_mysql:
self.insert_mysql(name, result)
if self.con_mongo:
self.insert_mongo(name, result)
def insert_postgre(self, table, datas={}):
flag = False
try:
db = self.con_pg.get_conn()
cursor = db.cursor()
keys = ','.join(datas.keys())
values = ','.join(['%s'] * len(datas))
table = self.base_table + table
sql = 'insert into {table}({keys}) values ({values})'.format(
table=table, keys=keys, values=values)
cursor.execute(sql, tuple(datas.values()))
db.commit()
cursor.close()
db.close()
flag = True
except Exception as e:
print("查询失败->%s" % e)
db.rollback()
finally:
return flag
def insert_mongo(self, table, result):
db = self.con_mongo.get_conn()
collection = db[table]
collection.insert_one(result)
def insert_mysql(self, table, datas={}):
flag = False
try:
db = self.con_mysql.get_conn()
cursor = db.cursor()
keys = ','.join(datas.keys())
values = ','.join(['%s'] * len(datas))
sql = 'insert into {table}({keys}) values ({values})'.format(
table=table, keys=keys, values=values)
cursor.execute(sql, tuple(datas.values()))
db.commit()
cursor.close()
db.close()
flag = True
except Exception as e:
db.rollback()
print("执行失败->%s" % e)
finally:
return flag
For more information on this file, see https://docs.djangoproject.com/en/3.1/topics/settings/ For the full list of settings and their values, see https://docs.djangoproject.com/en/3.1/ref/settings/ """ from pathlib import Path from config.ConfigManager import ConfigManager # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve(strict=True).parent.parent # ConfigManager provides configuration settings for e.g. database CONFIG_MANAGER = ConfigManager() # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/3.1/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = 'a102w%tdryj2xib)_@d6^lye$2w3+icu^o*e3b3vx7uf#-*r9n' # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True ALLOWED_HOSTS = [] # Application definition INSTALLED_APPS = [
class Sensor(Observable):
OBS_OUTPUT = 0
OBS_CMD_CONFIGURE = 1
OBS_CMD_REMOVE = 2
OBS_CMD_RESTART = 3
OBS_CMD_DUPLICATE = 4
OBS_CMD_MENU = 5
def __init__(self):
# the path of this sensor
self.__path = os.getcwd()
# configuration manager for sensor properties
self.__config_manager = ConfigManager()
self.__config_id = ""
# the ID of this sensor
self.__sensor_id = ""
# the configuration watcher callback
self.__config_watcher = None
# flag for stopping the sensor
self.__stop_flag = False
# when stopping, the flag is set to True, then the callback is called
self.__stop_callback = None
self.__defaults = {}
#
# Sets the ID of this sensor.
#
def set_id(self, ident):
self.__id = ident
#
#:function get_id | |
# Returns the ID of this sensor. It is not yet valid in the
# constructor.
#: return string
#:/function
#
def get_id(self):
assert self.__id, "The ID is invalid in the constructor."
return self.__id
#
# Sets the configuration ID for this sensor.
# If the user has set a callback function for watching the config,
# the config will be watched.
#
def set_config_id(self, config_id):
self.__config_id = str(config_id)
if (self.__config_watcher):
Sensor.watch_config(self, self.__config_watcher)
#
#:function get_config_id | |
# Returns the unique config ID of the sensor. You can use the ID
# if you need to store data in other places.
#: return string
#:/function
#
def get_config_id(self):
return self.__config_id
#
#:function set_config | key, value |
# Stores the given value in the configuration base.
#: param key | string | The key name.
#: param value | string | The value to set.
#:/function
#
def set_config(self, key, value):
assert(self.__config_manager)
if (not self.__stop_flag):
run_in_main_thread(self.__config_manager.set,
self.__config_id, key, value)
#
#:function get_config | key |
# Returns the configuration value for the given key.
#: param key | string | The key name.
#: return string
#:/function
#
def get_config(self, key):
assert(self.__config_manager)
value = run_in_main_thread(self.__config_manager.get,
self.__config_id, key)
if (value == self.__config_manager.UNDEF):
value = self.__defaults.get(key, "")
return value
#
#:function watch_config | callback |
# Registers a watcher for config changes.
#: param callback | function | The callback function for configuration
# changes.
#:/function
#
def watch_config(self, callback):
self.__config_watcher = callback
if (self.get_config_id()):
self.__config_manager.watch(self.get_config_id(),
self.__on_watch_config)
#
#:function _set_config_type | key, type, default |
# Sets the data types to be used for the configuration values.
# Use this method in the constructor of your sensor.
#: param key | string | The name of the configuration key.
#: param type | enum | The data type of the configuration key.
#: param default | string | The default value or unset keys.
#:/function
#
def _set_config_type(self, key, dtype, default):
self.__defaults[key] = default
#
#:function get_path | | Returns the filesystem path of the sensor. Use this
# method if you want to load resource files that come
# with the sensor.
#: return string
#:/function
#
def get_path(self): return self.__path
#
#:function set_path_to_purge | paths |
# Sets the paths which are to be removed when the desklet that
# uses the sensor gets removed.
# If your sensor creates files, you need to specify the paths here
# in order to clean up.
#: param paths | string list | The paths to purge.
#:/function
#
def set_paths_to_purge(self, paths):
key = PURGE_KEY
value = ",".join(paths)
self.__config_manager.set(self.__config_id,
key, value)
#
#:function new_output || Returns a new empty [cmd:TargetSettings] object for sending data to
# the display.
#: return TargetSettings
#:/function
#
def new_output(self): return TargetSettings()
#
#:function send_output | output | Sends the given [cmd:TargetSettings] object to the display.
# [emph:Never call this method from within a thread!]
#: param output | TargetSettings | The object for sending to the display.
#:/function
#
def send_output(self, output):
run_in_main_thread(self.update_observer, self.OBS_OUTPUT, output)
#
#:function add_timer | interval, callback, *args | Adds a timeout function with the given
# interval in ms.
#: param interval | int | The timeout interval between each invokation of the callback.
#: param callback | function | The callback function.
#:/function
#
def add_timer(self, interval, callback, *args):
def __wrap(self, callback, args):
if (not self.__stop_flag):
try:
return callback(*args)
except:
from utils.error import Error
Error().handle("unknown")
return False
else:
return False
return gobject.timeout_add(interval, __wrap, self, callback, args)
#
#:function add_thread | threadfunction, *args | Adds and runs a new thread.
# Use this to start new threads. It's recommended to put blocking actions into threads
# in order to not block [app:gDesklets].
#: param threadfunction | function | The thread function.
#:/function
#
def add_thread(self, threadfunction, *args):
# the thread should not start before setup is complete, therefore
# we are using the GTK idle handler
def run_thread(threadfunction, args):
thread.start_new_thread(threadfunction, args)
return False
gobject.idle_add(run_thread, threadfunction, args)
#
# Sends an action to this sensor.
#
def send_action(self, call, path, args = []):
if (not self.__stop_flag):
try:
self.call_action(call, path, args)
except:
from utils.error import Error
Error().handle("unknown")
#
#:function call_function | call, path, *args | Method for handling action calls from the
# display. Sensors have to override this method. [emph:This method may soon be
# deprecated!]
#: param call | string | The function to call.
#: param path | int list | The path of the target on which the action occurred.
#: param args | any list | The list of arguments for the function call.
#:/function
#
def call_action(self, call, path, args = []): raise NotImplementedError
def watch_stop(self, callback):
"""
Registers a callback that will be called when sensor will be stopped.
@param callback:
@type callback: callable(callback)
"""
self.__stop_callback = callback
#
# Stops this sensor and cleans up.
#
def stop(self):
self.__stop_flag = True
if (self.__stop_callback): self.__stop_callback()
del self.__config_watcher
self.__config_manager.remove_watcher(self.get_config_id(),
self.__on_watch_config)
# shut down sensor
self._shutdown()
#
#:function is_stopped | | Returns whether this sensor has been stopped. Use this method to
# check if your threads have to terminate.
#: return bool
#:/function
#
def is_stopped(self): return self.__stop_flag
#
#:function _shutdown | | Executes tasks for sensor shutdown. Override this method if your
# sensor has to clean up things after it has been terminated.
#:/function
#
def _shutdown(self): pass
#
#:function _new_configurator | | Creates and returns an empty new sensor configurator.
# [emph:This method may soon be deprecated!]
#: return SensorConfigurator.
#:/function
#
def _new_configurator(self): return SensorConfigurator(self.set_config,
self.get_config)
#
# Returns the configurator of this sensor.
#
def get_configurator(self): pass
#
# Method for duplicating the display using this sensor
#
def _duplicate_display(self):
self.update_observer(self.OBS_CMD_DUPLICATE)
#
#:function open_menu | menu | Opens the given popup menu. The menu is a list of
# [cmd:(label:str, sensitive:bool, submenu:list, callback:function, args: list)]
# tuples.
#: param menu | list | The menu to open.
#:/function
#
def open_menu(self, menu):
menu += [()]
run_in_main_thread(self.update_observer, self.OBS_CMD_MENU, menu)
#
# Reacts on changes in the configuration and delegates the call to the
# user's callback.
#
def __on_watch_config(self, path, value):
assert(self.__config_watcher)
key = path[-1]
self.__config_watcher(key, value)
def __on_configure(self, src):
self.update_observer(self.OBS_CMD_CONFIGURE, None)
def __on_remove(self, src):
self.update_observer(self.OBS_CMD_REMOVE, None)
def __on_restart(self, src):
self.update_observer(self.OBS_CMD_RESTART, None)
# some required aliases for older sensors to work
_get_id = get_id
_get_path = get_path
_new_output = new_output
_send_output = send_output
_add_timer = add_timer
_add_thread = add_thread
_is_stopped = is_stopped
_get_config_id = get_config_id
_watch_config = watch_config
_set_config = set_config
_get_config = get_config
_set_paths_to_purge = set_paths_to_purge
_open_menu = open_menu
class PostgreClient(object):
config = ConfigManager()
postgre_account = config.yaml_setting['datasouce']['postgresql']
def __init__(self):
"""
初始化连接
"""
try:
self._pool = PersistentDB(creator=pg8000, maxusage=None, threadlocal=None,
setsession=[], ping=0, closeable=False, **self.postgre_account)
except Exception as e:
logging.exception(e)
def get_conn(self):
"""取数据库连接"""
return self._pool.connection()
def insert(self, table, datas={}):
flag = False
try:
db = self.get_conn()
cursor = db.cursor()
keys = ','.join(datas.keys())
values = ','.join(['%s'] * len(datas))
sql = 'insert into {table}({keys}) values ({values})'.format(table=table, keys=keys, values=values)
cursor.execute(sql, tuple(datas.values()))
db.commit()
cursor.close()
db.close()
flag = True
except Exception as e:
logging.exception(e)
db.rollback()
finally:
return flag
def query(self, sql):
"""
数据库查询操作
:param sql:需要查询的语句
:return:返回元组
"""
results = ()
try:
db = self.get_conn()
cursor = db.cursor()
cursor.execute(sql)
results = cursor.fetchall()
db.commit()
cursor.close()
db.close()
except Exception as e:
logging.exception(e)
db.rollback()
finally:
return results
def split_sql(self,sql):
sqllist = sql.split(';')
return sqllist[0:-1]
def noquery(self, sqls):
"""
数据库非查询操作,删除，建表，赋权，更新等操作
:param sql:需要执行的语句,可支持多条以";"为分隔的事务操作，每一句sql必须加";" ！！！
:return:返回逻辑值
"""
flag = False
try:
db = self.get_conn()
cursor = db.cursor()
#cursor.execute(sql)
for sql in self.split_sql(sqls):
cursor.execute(sql)
db.commit()
cursor.close()
db.close()
flag = True
except Exception as e:
logging.exception(e)
db.rollback()
finally:
return flag
def __init__(self):
self.configManager = ConfigManager()
self.API = Auth.getAPI(self.configManager.CONFIG_PATH)
class Querier:
def __init__(self):
self.configManager = ConfigManager()
self.API = Auth.getAPI(self.configManager.CONFIG_PATH)
def queryByHashtag(self,
HT,
since,
until,
resultType="recent",
items=100,
lang="es",
savedEnabled=True):
query_term = "#" + HT
tweets = tweepy.Cursor(self.API.search,
q=query_term,
result_type=resultType,
since=since,
until=until,
lang=lang,
show_user=True).items(items)
query_dir = None
if savedEnabled:
query_dir = self.configManager.newQueryPath(query_term)
query = FM.saveTweetQuery(tweets,
os.path.join(query_dir, 'tweets.json'),
verbose=True)
else:
query = FM.parseTweetQuery(tweets)
return [query, query_dir]
def queryByUser(self,
username,
since,
until,
resultType="recent",
items=100,
savedEnabled=True):
tweets = tweepy.Cursor(self.API.user_timeline,
id=username,
result_type=resultType,
since=since,
until=until).items(items)
if savedEnabled:
query_dir = self.configManager.newQueryPath(username)
query = FM.saveTweetQuery(tweets,
os.path.join(query_dir, 'tweets.json'),
verbose=True)
else:
query = FM.parseTweetQuery(tweets)
return [query, query_dir]
def loadSavedQuery(self, term, time):
path = os.path.join(self.configManager.DATA_PATH,
(time + " " + term).replace(" ", "_"))
return [FM.loadTweetQuery(path), path]
def getGeneratedQueryPath(self, term, time):
return os.path.join(self.configManager.DATA_PATH,
(time + " " + term).replace(" ", "_"), 'generated')
def getRateLimits():
json.dumps(self.API.rate_limit_status(),
sort_keys=True,
indent=2,
separators=(',', ': '))
class MysqlClient(object):
config = ConfigManager()
mysql_account = config.yaml_setting['datasouce']['mysql']
def __init__(self):
"""
初始化连接
"""
try:
self._pool = PersistentDB(pymysql,
maxusage=None,
threadlocal=None,
closeable=False,
**self.mysql_account)
except Exception as e:
logging.exception(e)
def get_conn(self):
"""取数据库连接"""
return self._pool.connection()
def insert(self, table, datas={}):
flag = False
try:
db = self.get_conn()
cursor = db.cursor()
keys = ','.join(datas.keys())
values = ','.join(['%s'] * len(datas))
sql = 'insert into {table}({keys}) values ({values})'.format(
table=table, keys=keys, values=values)
#print(sql)
cursor.execute(sql, tuple(datas.values()))
db.commit()
cursor.close()
db.close()
flag = True
except Exception as e:
db.rollback()
logging.exception(e)
finally:
return flag
def insertmany(self, table, datas=None):
flag = False
try:
db = self.get_conn()
cursor = db.cursor()
keys = ','.join(datas[0].keys())
values = ','.join(['%s'] * len(datas[0]))
sql = 'insert into {table}({keys}) values ({values})'.format(
table=table, keys=keys, values=values)
# print(sql)
# print([tuple(data.values()) for data in datas])
cursor.executemany(sql, [tuple(data.values()) for data in datas])
db.commit()
cursor.close()
db.close()
flag = True
except Exception as e:
db.rollback()
logging.exception(e)
finally:
return flag
def query(self, sql):
"""
数据库查询操作
:param sql:需要查询的语句
:return:返回元组
"""
results = ()
try:
db = self.get_conn()
cursor = db.cursor()
cursor.execute(sql)
results = cursor.fetchall()
db.commit()
cursor.close()
db.close()
except Exception as e:
db.rollback()
logging.exception(e)
finally:
return results
def noquery(self, sql):
"""
数据库非查询操作,删除，建表，赋权，更新等操作
:param sql:需要执行的语句
:return:返回元组
"""
flag = False
try:
db = self.get_conn()
cursor = db.cursor()
cursor.execute(sql)
db.commit()
cursor.close()
db.close()
flag = True
except Exception as e:
db.rollback()
logging.exception(e)
finally:
return flag
