def home():
""" Display the home page """
columns_to_hide = ('vautoid', 'carfaxid', 'body_type', 'created_at')
db_handler = DBHandler()
records = db_handler.get_all_potential_records()
table_headers = list(records[0].keys())
make_model_records = db_handler.get_all_make_models()
make_model = defaultdict(list)
for record in make_model_records:
make_model[record["make"]].append(record["model"])
make_model_year_col = [record["make_model_year"] for record in records]
years = defaultdict(int)
year_regex = re.compile(r"^\s*(\d+)")
for value in make_model_year_col:
match_obj = year_regex.search(value)
if match_obj is not None:
years[match_obj.group(1)] += 1
years = sorted(
years.items(),
reverse=True) # This will return a list of tuple (key, value)
return render_template("home.html",
table_headers=table_headers,
records=records,
make_model=make_model,
columns_to_hide=columns_to_hide,
years=years)
def showUrl(local=False, token=None):
if local:
local_make_response = data_to_object
if token is None:
return local_make_response(False, {'message': 'Invalid data'})
else:
local_make_response = data_to_response
if not ('token' in request.form):
return local_make_response(False, {'message': 'Invalid data'})
token = request.form['token']
db = DBHandler()
try:
username = db.tokenLookup(token)
urls = db.userOwn(username)
except DBException as err:
return local_make_response(False, {'message': err.message})
else:
urls = list(urls)
for i in range(len(urls)):
urls[i] = list(urls[i])
urls[i][0] = baseurl + urls[i][0]
return local_make_response(True, {'url': urls})
def start(bot, update):
database = DBHandler()
database.add_id(update.message.chat_id)
logger.info("Connected to bot: %d" % update.message.chat_id)
bot.send_message(chat_id=update.message.chat_id, text="Hello")
bot.send_message(chat_id=update.message.chat_id,
text=database.get_latest_post()[0])
def __init__(self):
self.config = util.load_config()
util.register_gcp_credential(self.config)
self.dbhandler = DBHandler(self.config)
self.gcphandler = GCPHandler(self.config)
target_ids = util.load_target_ids('target.txt')
self.user_sids = self.dbhandler.select_user_sids(target_ids)
def get_user_data(self, userid):
db = DBHandler()
db.connect()
query = "SELECT email, signup_date FROM Users where userid = %s;"
cursor = db.executeQuery(query, (str(userid)))
tupl = cursor.fetchone()
return tupl
def register(local=False, username=None, password=None):
if local:
local_make_response = data_to_object
if (username is None) | (password is None):
return local_make_response(False, {'message': 'Invalid data'})
else:
local_make_response = data_to_response
if (not ('username' in request.form)) | (not ('password'
in request.form)):
return local_make_response(False, {'message': 'Invalid data'})
username = request.form['username']
password = request.form['password']
if (len(username) < 2):
return local_make_response(
False, {'message': 'Username\'s length should be more than 2'})
if (len(password) < 8):
return local_make_response(
False, {'message': 'Password\'s length should be more than 7'})
db = DBHandler()
try:
db.userCreate(username, password)
except DBException as err:
return local_make_response(False, {'message': err.message})
else:
return local_make_response(True)
def changePassword(local=False,
token=None,
oldPassword=None,
newPassword=None):
if local:
local_make_response = data_to_object
if (token is None) | (oldPassword is None) | (newPassword is None):
return local_make_response(False, {'message': 'Invalid data'})
else:
local_make_response = data_to_response
if (not ('token' in request.form)) | (
not ('oldPassword' in request.form)) | (not ('newPassword'
in request.form)):
return local_make_response(False, {'message': 'Invalid data'})
token = request.form['token']
oldPassword = request.form['oldPassword']
newPassword = request.form['newPassword']
db = DBHandler()
try:
username = db.tokenLookup(token)
db.userChangePassword(username, oldPassword, newPassword)
except DBException as err:
return local_make_response(False, {'message': err.message})
else:
return local_make_response(True)
def unlock(self, username, password, create_new):
user_data_dir = os.path.join(self.data_directory, username)
if create_new:
if os.path.exists(user_data_dir):
raise AuthenticationError('User {} already exists'.format(username))
else:
os.mkdir(user_data_dir)
else:
if not os.path.exists(user_data_dir):
raise AuthenticationError('User {} does not exist'.format(username))
if not os.path.exists(os.path.join(user_data_dir, 'rotkehlchen.db')):
# This is bad. User directory exists but database is missing.
# Make a backup of the directory that user should probably remove
# on his own. At the same time delete the directory so that a new
# user account can be created
shutil.move(
user_data_dir,
os.path.join(self.data_directory, 'backup_%s' % username)
)
raise AuthenticationError(
'User {} exists but DB is missing. Somehow must have been manually '
'deleted or is corrupt. Please recreate the user account.'.format(username))
self.db = DBHandler(user_data_dir, username, password)
self.user_data_dir = user_data_dir
return user_data_dir
class FileHandler(object):
"""
This class takes care of appending chunks to the specified file that
is being uploaded. When the upload is complete, the file is then moved
to its specified path.
"""
def __init__(self, body):
"""
Constructor that needs a file path
"""
self.body = body
self.db = DBHandler(current_app.config["DBFILE"])
self.cursor = self.db.getCursor()
def write_chunk(self):
"""
Find the file hash in SQLite, append the chunk to the returned file.
Each chunk will be checksummed to ensure file/data integrity.
"""
pass
def create_file_metadata(self):
"""
Create the file metadata.
"""
if all(k in self.body for k in ('name', 'path', 'size', 'hash')):
c = self.db.getCursor()
if self.db.exists("file", "hash", self.body['hash']):
return Response(
'File exists\n',
status=200, mimetype='text/plain')
else:
try:
sql = """
INSERT INTO file(name, path, size, file_hash)
VALUES('{name}','{path}', {size}, '{hash}')
""".format(
name = self.body['name'],
path = self.body['path'],
hash = self.body['hash'],
size = self.body['size'])
print sql
c.execute(sql)
self.db.commit()
except Exception, e:
errormsg = \
u"Unsuccessful database insert transaction:" \
+ str(e)
print errormsg
#log.exception(errormsg, self.__class__.__name__)
return Response(
'Unsuccessful database insert transaction\n',
status=500, mimetype='text/plain')
return Response('', status=201, mimetype='text/plain')
return Response(
'Missing data in JSON\n',
status=500, mimetype='text/plain')
def run(bot):
database = DBHandler()
while True:
for user in database.get_items():
latest_posts = get_updates(database)
send_posts_to(bot, user, latest_posts)
sleep(1 * config.HOURS)
def __init__(self, host, port, dbhost="localhost", dbport=27017):
threading.Thread.__init__(self)
self.ssap = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.ssap.bind((host, port))
self.ssap.listen(1)
self.client_handlers = dict()
self.dbhandler = DBHandler(mongohost=dbhost, mongoport=dbport)
print('listening on ', port)
def __init__(self,
timeout=None,
db_config=("127.0.0.1", "my_database", "my_user")):
assert timeout == None or isinstance(
timeout, int), "timeout must be an integer in seconds"
self.channels = {} # channel_name = [channel_callback]
self.mustExit = threading.Event()
self.timeout = timeout
self.db = DBHandler(db_config)
def resolve(short):
db = DBHandler()
try:
redir = db.urlLookup(short)
db.urlVisit(short)
except DBException as err:
return err.message
else:
return redirect(redir)
class Uploader:
def __init__(self):
self.config = util.load_config()
util.register_gcp_credential(self.config)
self.dbhandler = DBHandler(self.config)
self.gcphandler = GCPHandler(self.config)
target_ids = util.load_target_ids('target.txt')
self.user_sids = self.dbhandler.select_user_sids(target_ids)
def upload_per_user(self):
tmpdir = os.path.abspath('./tmp')
for user_sid in self.user_sids:
user_dir = os.path.join(tmpdir, str(user_sid))
if not os.path.exists(user_dir):
continue
track_ids = [
x for x in os.listdir(user_dir) if '.DS_Store' not in x
]
for track_id in track_ids:
self.upload_hls(user_sid, track_id)
shutil.rmtree(f'./tmp/{user_sid}')
def upload_nohls(self):
tmpdir = os.path.abspath('./tmp')
user_sids = [
x for x in os.listdir(tmpdir)
if '.DS_Store' not in x and 'readme' not in x
]
for user_sid in user_sids:
user_dir = os.path.join(tmpdir, str(user_sid))
track_ids = [
x for x in os.listdir(user_dir) if '.DS_Store' not in x
]
for track_id in track_ids:
self.upload_hls(user_sid, track_id)
shutil.rmtree(f'./tmp/{user_sid}')
def upload_hls(self, user_sid, track_id):
upload_url = self.config['UPLOADURL']
hashstr = str(user_sid) + track_id
hashkey = hashlib.sha1()
hashkey.update(hashstr.encode('utf-8'))
hlshash = hashkey.hexdigest()
filedir = f'./tmp/{user_sid}/{track_id}'
files = os.listdir(filedir)
for file in files:
if '.mp3' in file:
continue
response = requests.post(
url=upload_url,
files={'file': open(os.path.join(filedir, file), 'rb')},
data={'dir': f'/hls/{hlshash}'})
print(response.status_code, response.text)
self.dbhandler.update_track_hls(track_id, hlshash)
def save_comments():
""" Save comments to potential deals table """
potential_deal_id = int(request.form.get("id"))
action = request.form.get("action")
if action.lower() == "none":
action = None
comments = request.form.get("comments")
db_handler = DBHandler()
db_handler.update_by_id(potential_deal_id, action, comments)
# return redirect(url_for("home"))
return jsonify({"success": True}), 200
def __init__(self, parent=None):
super().__init__(parent)
conn = sqlite3.connect('cetizen.db')
self.db = DBHandler(conn)
self.ui = loadUi('cetizen.ui', self)
self.logger.setText('')
self.table.setAlternatingRowColors(True)
#self.table.setRootIsDecorated(False)
self.setWindowTitle('Cetizen Crawler')
self.ui.show()
def logout(local=False, token=None):
if local:
local_make_response = data_to_object
else:
local_make_response = data_to_response
if 'token' in request.form:
token = request.form['token']
if not (token is None):
db = DBHandler()
db.userLogout(request.form['token'])
return local_make_response(True)
def __init__(self, test=False, remote=False):
"""
Initialize a new CrossValidation instance.
:param test: if running in test mode or not
:type test: boolean
:param remote: whether to use the remote database that is configured in DBHandler
:type remote: boolean
"""
self._test = test
self.dbh = DBHandler(league=LEAGUES[0], test=self._test, remote=remote)
self.data = {_l: {_y: (None, None) for _y in YEARS} for _l in LEAGUES}
self._indces = {_l: 0 for _l in LEAGUES}
def _startDataRead(self):
"""
Método utilizado para configurar a conexão socket e inicializar uma thread para a leitura dos dados e atualização da interface grafica
:param ip: ip da conexão socket
:param port: porta para a conexao socket
"""
try:
self._apogeu = int(self._apogeu)
self._port = int(self._port)
self._serverIP = str(ip_address(self._serverIP))
if self._login == 'supernova' and self._senha == 'astra':
Window.set_system_cursor("wait")
self._connect = Cliente(self._serverIP, self._port)
self._connect.start()
Window.set_system_cursor("arrow")
self._updateThread = Thread(target = self.updater)
self._updateThread.daemon = False
self.ids.imagem_conexao.background_normal = 'imgs/conectado.png'
self.ids.latitude.font_size = self.ids.altitude.font_size/2
self.ids.longitude.font_size = self.ids.altitude.font_size/2
self.ids.graphAcelerometro.clearLabel()
self.ids.graphGiroscopio.clearLabel()
self.ids.graphAltitude.clearLabel()
self._dataBase = DBHandler(self._missao)
self._disableNewConnections()
self._limitesGraficos()
self.enableSwitchesAndButtons()
self._updateThread.start()
self._conn.dismiss()
else:
self._connError.ids.erroConnect.text = "Senha incorreta!"
self._connError.open()
raise Exception('Senha incorreta!')
except ValueError:
if (type(self._apogeu) != int):
self._connError.ids.erroConnect.text = "Selecione o apogeu!"
self._connError.open()
else:
self._connError.ids.erroConnect.text = "Erro: server/port mal definidos!"
self._connError.open()
raise ValueError
except ConnectionRefusedError:
Window.set_system_cursor("arrow")
self._connError.ids.erroConnect.text = "Falha ao conectar!"
self._connError.open()
raise ConnectionRefusedError
def __init__(self):
self.continue_schedule = False
self.local_db = DBHandler()
self.all_synonyms = set()
self.trustpilot = TrustPilotCrawler()
self.reddit = RedditScraper()
self.scrapers = {
'trustpilot': TrustPilotCrawler(),
'reddit': RedditScraper()
}
# TODO: Make Environment Variables for API info
self.kwe_api = f'http://{os.environ["KWE_API_HOST"]}/'
self.kwe_api_key = {'Authorization': os.environ['KWE_API_KEY']}
self.sa_api = f'http://{os.environ["SA_API_HOST"]}/prediction/'
self.sa_api_key = {'Authorization': os.environ['SA_API_KEY']}
self.synonym_api = f'http://{os.environ["GATEWAY_API_HOST"]}/api/synonyms'
self.synonym_api_key = {'Authorization': os.environ['GATEWAY_API_KEY']}
self.sentiment_categories = [{
'category': 'positive',
'upper_limit': 1,
'lower_limit': 0.55
}, {
'category': 'negative',
'upper_limit': 0.45,
'lower_limit': 0
}, {
'category': 'neutral',
'upper_limit': 0.55,
'lower_limit': 0.45
}]
self.kwe_interval = timedelta(hours=1)
self.kwe_latest = self._read_kwe_date()
self.continue_schedule = True
self.schedule_thread = Thread()
self.crawler_schedule_thread = Thread()
self.begin_schedule()
logging.info(
f'Initiated scheduler, will create snapshots from {self.kwe_latest}'
)
def __init__(self, body):
"""
Constructor that needs a file path
"""
self.body = body
self.db = DBHandler(current_app.config["DBFILE"])
self.cursor = self.db.getCursor()
def authenticate_user(self):
username = self.username_line_edit.text()
password = self.password_line_edit.text()
with DBHandler(self.context.get_database) as cursor:
find_user_query = """
SELECT priviledge FROM users WHERE
username = ? AND password = ?
"""
cursor.execute(find_user_query, [username, password])
results = cursor.fetchone()
try:
if results[0] == 0:
from saleswindow import SalesWindow
# Create session and all activites will be recorded with `username`
self.saleswindow = SalesWindow(username=username,
context=self.context)
self.saleswindow.show()
self.hide()
if results[0] == 1:
from adminwindow import AdminWindow
self.adminwindow = AdminWindow(context=self.context)
self.adminwindow.show()
self.hide()
except TypeError as e:
self.error_label.setText(str(e))
def main():
# Read in data and extract data arrays
#logging.info("Reading input files.")
dbh = DBHandler(options.db_config_filename, options.db_config_name)
ssh = SmartMetHandler(options.smartmet_config_filename,
options.smartmet_config_name,
sleep_time=options.requests_throttle_time)
fh = FileHandler(s3_bucket='fmi-sasse-classification-dataset')
tracker = Tracker(dbh, ssh)
starttime = dt.datetime.strptime(options.starttime, "%Y-%m-%dT%H:%M:%S")
endtime = dt.datetime.strptime(options.endtime, "%Y-%m-%dT%H:%M:%S")
d = starttime
ymd = '{}{}{}'.format(d.year, d.month, d.day)
while d <= endtime:
logging.info('Processing time: {}'.format(d))
tracker.run(d)
d += timedelta(hours=1)
if ymd != '{}{}{}'.format(d.year, d.month, d.day):
save_dataset(tracker, d, fh=fh, db=dbh)
ymd = '{}{}{}'.format(d.year, d.month, d.day)
#if tracker.dataset is not None and len(tracker.dataset) > 0:
save_dataset(tracker, d, fh=fh, db=dbh)
def get_results(self):
from_ = self.from_datetime_edit
to = self.to_datetime_edit
# from_.setDateTime(now)
# to = to.dateTimeFromText(now)
from_date = from_.textFromDateTime(from_.dateTime()) # Starting date
to_date = to.textFromDateTime(to.dateTime()) # Ending date
with DBHandler(self.context.get_database) as cursor:
cursor.execute(
"""SELECT product_name, sum(quantity_sold), sum(price)
FROM `orders` WHERE username= ? AND
`date` BETWEEN ? AND ?
GROUP BY product_name;""",
[self.username, from_date, to_date])
products_result = cursor.fetchall()
cursor.execute(
"""SELECT sum(price)
FROM `orders` WHERE username = ?
AND `date` BETWEEN ? AND ?""",
[self.username, from_date, to_date])
total_result = cursor.fetchone()
# print(products_result)
return products_result, total_result
def create(local=False, url=None, short=None, token=None):
if local:
local_make_response = data_to_object
if url is None:
return local_make_response(False, {'message': 'Invalid data'})
else:
local_make_response = data_to_response
if (not ('url' in request.form)):
return local_make_response(False, {'message': 'Invalid data'})
url = request.form['url']
if 'short' in request.form:
short = request.form['short']
else:
short = None
if 'token' in request.form:
token = request.form['token']
if not (token is None):
try:
username = db.tokenLookup(request.form['token'])
except DBException as err:
return local_make_response(False, {'message': err.message})
else:
username = None
if not (short is None):
if not (short.isalnum()):
return local_make_response(
False, {'message': 'Slug must be alphanumeric.'})
db = DBHandler()
if (not validators.url(url)) | ('turang.ga' in url):
return local_make_response(False, {'message': 'URL is not valid.'})
newUrl = baseurl
try:
newUrl += db.urlNew(url, short, username)
except DBException as err:
return local_make_response(False, {'message': err.message})
else:
return local_make_response(True, {'url': newUrl})
def __init__(self,
id=None,
create=None,
db_config=("127.0.0.1", "my_database", "my_user"),
related_task_id=None,
mustExit=None):
assert id or create, "Bad arguments, no id nor create"
assert id == None or isinstance(
id, int), "Bad arguments id, must be an int"
assert mustExit, "Bad arguments, mustExit not provided"
assert create == None or (
isinstance(create, tuple) and 3 <= len(create) <= 4
), "Bad argument create, must be (user_id,type,request in json[,is_visible_to_user true/false])"
self.db = DBHandler(db_config)
if create:
self.status, self.progress_percent, self.user_id, self.type, self.request, self.result = "created", 0, create[
0], create[1], create[2], []
if isinstance(self.request, str):
self.request = json.loads(self.request)
r = self.db.execute(
"INSERT INTO tasks (status, progress_percent, type, request, user_id) VALUES (%s, %s, %s, %s::jsonb, %s) RETURNING id, start_date, end_date ;",
[
self.status, self.progress_percent, self.type,
json.dumps(self.request), self.user_id
])
r = self.db.fetchall(r)
self.id, self.start_date, self.end_date = r[0]
if related_task_id:
self.db.execute(
"UPDATE tasks SET related_task_id = %s WHERE id = %s",
[related_task_id, self.id])
self.related_task_id = related_task_id
with Notifier() as notifier:
notifier.send_notification(
"task", "id={},status=created".format(self.id))
else:
r = self.db.execute(
"SELECT status, progress_percent, type, request, result, user_id, start_date, end_date, related_task_id FROM tasks WHERE id = %s;",
[id])
r = self.db.fetchall(r)
self.id = id
self.status, self.progress_percent, self.type, self.request, self.result, self.user_id, self.start_date, self.end_date, self.related_task_id = r[
0]
if self.result == None:
self.result = []
def main():
#if options.dataset_file is None or options.output_file is None:
dbh = DBHandler(options.db_config_file, options.db_config_name)
dbh.return_df = False
fh = FileHandler(s3_bucket=options.bucket)
with open(options.example_config_file) as f:
setups = yaml.load(f.read(), Loader=yaml.FullLoader)
for setup in setups['examples']:
output_file = '{}/{}-{}.csv'.format(
setup['output_dir'], setup['starttime'].strftime('%Y%m%dT%H%M%S'),
setup['endtime'].strftime('%Y%m%dT%H%M%S'))
# Read in data and extract data arrays
logging.info("Reading input data for {} - {}...".format(
setup['starttime'], setup['endtime']))
features, meta_params, labels, all_params = get_param_names(
options.param_config_file)
data = fh.read_data(
[setup['dataset_file']],
options,
return_meta=True,
starttime=setup['starttime'], #.strftime('%Y-%m-%dT%H:%M:%S'),
endtime=setup['endtime'])[0] #.strftime('%Y-%m-%dT%H:%M:%S'))[0]
X, y, meta = data
model = fh.load_model(setup['model_file'])
scaler = fh.load_model(setup['scaler_file'])
logging.info('Predicting with {} samples...'.format(len(X)))
y_pred = model.predict(X)
df = pd.DataFrame(meta, columns=options.meta_params)
X_inv = pd.DataFrame(scaler.inverse_transform(X), columns=X.columns)
df = pd.concat(
[df.reset_index(drop=True),
X_inv.reset_index(drop=True)], axis=1)
df = dbh.get_geom_for_dataset_rows(df)
df['y_pred'] = y_pred
df['y'] = y
fh.df_to_csv(df, output_file)
def __init__(self, wetoken):
self.db = DBHandler(wetoken)
self.handlers = {
CODE_OK: self.handle_ok,
CODE_FOP: self.handle_failed_to_open,
CODE_WRONG: self.handle_wrong_pwd,
CODE_UNBIND: self.handle_unbind,
}
self.wetoken = wetoken
def get_product_list(self):
with DBHandler(self.context.get_database) as cursor:
cursor.execute("SELECT * FROM products")
results = cursor.fetchall()
# Genexpr to get all items from database
self.product_list = [
self._Product(*value) for _, value in enumerate(results)
]
def roll(self, crit_type, database, weapon_type):
if database == '--Select database--' or weapon_type == '--Select weapon type--':
self.textlabel.setText('Invalid input.\nCheck the dropdown menus.')
return
db_filename = database + '.sqlite'
global dbobj
dbobj = DBHandler(db_filename)
if self.checkbox.isChecked():
crit = dbobj.select_random(crit_type, 1, weapon_type)
if crit == None:
self.textlabel.setText('No crits in the database!')
return
else:
crit = dbobj.select_random(crit_type, 0, weapon_type)
if crit == None:
self.textlabel.setText(
'No more unused crits.\nUntick \'used\' box or reset')
return
dbobj.set_used(crit[2], crit_type)
crittext = 'Author: {}\nName: {}\n\n{}'.format(crit[1], crit[2],
crit[3])
self.textlabel.setText(crittext)
def __init__(self,test=False,remote=False):
"""
Initialize a new CrossValidation instance.
:param test: if running in test mode or not
:type test: boolean
:param remote: whether to use the remote database that is configured in DBHandler
:type remote: boolean
"""
self._test = test
self.dbh = DBHandler(league=LEAGUES[0],test=self._test,remote=remote)
self.data = {_l:{_y:(None,None) for _y in YEARS} for _l in LEAGUES}
self._indces = {_l:0 for _l in LEAGUES}
def delete(local=False, short=None, token=None):
if local:
local_make_response = data_to_object
if (short is None) | (token is None):
return local_make_response(False, {'message': 'Invalid data'})
else:
local_make_response = data_to_response
if ((not ('short' in request.form)) | (not ('token' in request.form))):
return local_make_response(False, {'message': 'Invalid data'})
short = request.form['short']
token = request.form['token']
db = DBHandler()
try:
username = db.tokenLookup(token)
db.urlDelete(username, short)
except DBException as err:
return local_make_response(False, {'message': err.message})
else:
return local_make_response(True)
def countWords(self, dbName):
query = "Select * from wort order by count desc"
dbh = DBHandler()
dbh.reconnect(dbName)
cursor = dbh.execute(query)
dbh.commit()
if cursor == None: print "No cursor returned: ", cursor
return [[entry[1], entry[2]] for entry in cursor.fetchall()]
def login(local=False, username=None, password=None):
if local:
local_make_response = data_to_object
if (username is None) | (password is None):
return local_make_response(False, {'message': 'Invalid data'})
else:
local_make_response = data_to_response
if (not ('username' in request.form)) | (not ('password'
in request.form)):
return local_make_response(False, {'message': 'Invalid data'})
username = request.form['username']
password = request.form['password']
db = DBHandler()
try:
db.userLogin(username, password)
except DBException as err:
return local_make_response(False, {'message': err.message})
else:
return local_make_response(True, {'token': db.tokenNew(username)})
class Notifier:
def __init__(self,
timeout=None,
db_config=("127.0.0.1", "my_database", "my_user")):
assert timeout == None or isinstance(
timeout, int), "timeout must be an integer in seconds"
self.channels = {} # channel_name = [channel_callback]
self.mustExit = threading.Event()
self.timeout = timeout
self.db = DBHandler(db_config)
def __enter__(self):
return self
def __exit__(self):
self.stop()
def __del__(self):
for channel in self.channels.keys():
self.db.execute("UNLISTEN {}".format(channel))
def __repr__(self):
return "subscribed channels: {}".format(self.channels.keys())
def stop(self):
self.mustExit.set()
def subscribeChannelCallback(self, channel, cb):
""" Subscribe a callback to a channel. If events are received on this channel, the callback will be triggered """
if channel not in self.channels:
self.channels[channel] = []
self.db.execute("LISTEN {};".format(channel))
self.channels[channel].append(cb)
def sendNotification(self, channel, msg):
self.db.execute("NOTIFY task,'{}';".format(msg))
def run(self, timeout=None):
assert timeout == None or is_instance(
timeout, int), "timeout must be integer in seconds"
if not timeout:
timeout = self.timeout
sel = selectors.DefaultSelector()
sel.register(self.db.db_conn, selectors.EVENT_READ)
while not self.mustExit.is_set():
events = sel.select(timeout)
if len(events) == 0:
continue
for key, mask in events:
if key.fileobj == self.db.db_conn:
self.db.db_conn.poll()
for notification in self.db.db_conn.notifies:
if notification.channel in self.channels:
for action in self.channels[notification.channel]:
if action:
action(notification)
self.db.db_conn.notifies.clear()
class Handle:
cnweeks = (u"星期一", u"星期二", u"星期三", u"星期四", u"星期五", u"星期六", u"星期日")
def __init__(self, wetoken):
self.db = DBHandler(wetoken)
self.handlers = {
CODE_OK: self.handle_ok,
CODE_FOP: self.handle_failed_to_open,
CODE_WRONG: self.handle_wrong_pwd,
CODE_UNBIND: self.handle_unbind,
}
self.wetoken = wetoken
def parse_data(self, data):
self.weekday_c = self.weekend_c = 0
(self.times, date) = data
first = True
self.last = datetime.datetime(1994, 10, 2)
for day in date:
d = datetime.datetime(int(day[0]), int(day[1]), int(day[2]))
if first:
first = False
self.last = d
if d.weekday() in range(0, 5):
self.weekday_c = self.weekday_c + 1;
else:
self.weekend_c = self.weekend_c + 1;
self.date = "%s %s"% (str(self.last.date()), self.cnweeks[self.last.weekday()])
def consult(self):
status = Crawler(self.wetoken, self.db).work()
return self.handlers[status.code](status)
def parse_bind_message(self, msg):
temp = msg.split(u' ')
if len(temp) < 3:
raise Exception("Wrong msg while split")
if len(temp[2]) != 9 or re.search(RE_SQL,temp[2]) != None:
raise Exception("Wrong stu_id format")
if re.search(RE_SQL, temp[1]) != None:
raise Exception("Wrong name format")
return (temp[1], temp[2])
def bind(self, msg):
try:
(name, stu_id) = self.parse_bind_message(msg)
self.db.bind(name, stu_id)
except Exception as var:
print(var)
return u"绑定失败，请按照格式重新绑定"
return u"绑定成功！可以开始查询\n直接回复 查询 或者语音回复查询皆可"
def handle_ok(self, status):
self.parse_data(status.data)
return u"%s 本学期已跑操 %s 次，其中平时 %s 次，周末 %s 次。\n最后一次跑操时间为：\n%s" % (status.name, str(self.times), str(self.weekday_c), str(self.weekend_c), self.date)
def handle_failed_to_open(self, status):
return u"由于网络原因，体育部网站暂时无法访问，请稍后再试"
def handle_wrong_pwd(self, status):
return u"登陆失败，请检查你的学号和姓名：\n%s %s" % (status.stu_id, status.name)
def handle_unbind(self, status):
return u"你还没有绑定，请先绑定"
class Parser(object):
game_start_time = 0
game_start_offset_time = "0:00"
game_init_found = False
def __init__(self):
self.db = DBHandler()
@property
def get_server_start(self):
return self.mod.server_start
def renew_db_connection(self):
return self.db.renew_db_connection()
def init_game(self, data):
timestamp_sec, subsec = str(time.time()).split('.')
self.game_start_time = timestamp_sec
self.game_start_offset_time = data['time']
# enable writing analyzis
self.game_init_found = True
# create Session for collecting data and close maybe other Sessions
self.db.close_session()
self.db.create_session()
# reset runtime to zero,
# otherwise init_game will have runtime of game before
# and only first action in this round will have actuall runtime
data['runtime'] = 0
# overwrite the timestamp with the round starting timestamp
data['timestamp'] = timestamp_sec
self.db.init_game(data)
def exit(self, data):
self.db.exit(data)
def shutdown_game(self, data):
self.db.shutdown_game(data)
# disable writing analyzis
self.game_init_found = False
#commit now write data to Database
self.db.commit_session()
#and close the session
self.db.close_session()
def red(self, data):
self.db.red(data)
def blue(self, data):
self.db.blue(data)
def score(self, data):
self.db.score(data)
def client_connect(self, data):
self.db.client_connect(data)
def client_userinfo_changed(self, data):
self.db.client_userinfo_changed(data)
def client_begin(self, data):
self.db.client_begin(data)
def client_disconnect(self, data):
self.db.client_disconnect(data)
def item(self, data):
self.db.item(data)
def kill(self, data):
self.db.kill(data)
def say(self, data):
self.db.say(data)
def tell(self, data):
self.db.tell(data)
def sayteam(self, data):
self.db.sayteam(data)
def empty(self, data):
self.db.empty(data)
def set_engine(self, engine):
self.engine = engine
self.db.set_engine(engine)
def load_mod(self, mod):
module = importlib.import_module('mods.' + mod)
if not hasattr(module, 'init'):
log.error("cannot find 'init' in possible mod")
return False
try:
_pre_init = module.init()
# loading module for handling
self.mod =_pre_init()
return True
except:
log.error("loading mod failed")
return False
@property
def get_log_structure(self):
return self.mod.log_structure
@property
def translater(self):
return self.mod.translate
def build_key_value(self, data):
return self.mod.key_value(data)
def call_action_db_handler(self, action, data):
"""
forwards the data-dictionary to the method
which is mapped to the action Paramater
"""
if action == 'init_game':
self.init_game(data)
elif not self.game_init_found:
# don't start logging of games in the middle
# otherwise map, game_type and other things will be unknown
log.debug("waiting for new game to start analyzing: " + action + " -> " + str(data))
elif action == 'exit':
self.exit(data)
elif action == 'shutdown_game':
self.shutdown_game(data)
elif action == 'score':
self.score(data)
elif action == 'red':
self.red(data)
elif action == 'blue':
self.blue(data)
elif action == 'client_connect':
self.client_connect(data)
elif action == 'client_userinfo_changed':
self.client_userinfo_changed(data)
elif action == 'client_begin':
self.client_begin(data)
elif action == 'client_disconnect':
self.client_disconnect(data)
elif action == 'item':
self.item(data)
elif action == 'Kill':
self.kill(data)
elif action == 'say':
self.say(data)
elif action == 'tell':
self.tell(data)
elif action == 'sayteam':
self.sayteam(data)
elif action == 'empty':
self.empty(data)
else:
log.debug("unknown Action: '" + action + "'")
# needed because of foreign keys and searches ...
if self.game_init_found:
self.db.commit_session()
def calc_real_timestamp(self, happening_offset_time):
start_minute, start_second = str(self.game_start_offset_time).split(':')
if len(str(start_second)) == 1:
start_second = int(start_second) * 10
game_start_offset = int(start_minute) * 60 + int(start_second)
happening_minute, happening_second = str(happening_offset_time).split(':')
if len(str(start_second)) == 1:
happening_second = int(happening_second) * 10
game_happening_offset = int(happening_minute) * 60 + int(happening_second)
game_runtime = game_happening_offset - game_start_offset
real_timestamp = int(self.game_start_time) + game_runtime
return real_timestamp, game_runtime
def analyze(self, line):
"""
analyzes the log line with the
log_entries dictionary-list from the loaded mod
and redirect the parsed values to
the methods which handle them in the correct way
into the database
"""
for structure in self.get_log_structure:
if line.endswith(self.mod.server_start):
# don't handle serverstart lines,
# especially if it contains in partially written loglines
break
elif re.search(structure['search'], line):
try:
data = parse(structure['parser'], line).named
except:
log.error("PARSING ERROR")
log.debug("Parser Rule: " + structure['parser'])
log.debug("Line to parse: " + line)
break
try:
if structure['custom']:
for k, v in self.build_key_value(data['arguments']).items():
# translate keys
if self.translater.has_key(k):
key = self.translater[k]
else:
key = k
# concat dictionaries
data[key] = v
del data['arguments']
except:
pass
if not data:
log.warning("\n" +
"Mod: " + self.mod.mod_name + "\n" + \
"Action '" + structure['search'] + "' found, but Parser not matching!" + "\n" + \
"Parser Rule: " + structure['parser'] + "\n" + \
"Log Entry: " + line + "\n"
)
break
else:
log.debug("\n" +
"Mod: " + self.mod.mod_name + "\n" + \
"Action: '" + structure['search'] + "'" + "\n" + \
"Data: '" + str(data) + "\n"
)
#clean up time
data['time'] = data['time'].strip()
#add timestamp and runtime to dictionary
time_format_check = parse("{:d}:{:d}",data['time'])
if not time_format_check:
log.error("found strange time format")
break
data['timestamp'], data['runtime'] = self.calc_real_timestamp(data['time'])
try:
self.call_action_db_handler(structure['call'], data)
except:
log.debug("call_action_db_handler crashed on: '%s' for '%s'" % (str(structure['call']), str(data)))
pass
break
else:
log.debug("NO MATCH:\n" + line)
def __init__(self):
self.db = DBHandler()
class CrossValidation(object):
"""
A class that implements our unique way to cross validate - 4 leagues for training, 1 for test -> 5 folds.
Since our need to avoid a situation where we have tested a classifier on examples that are older from some of the examples used to fit the classifier,
we've implemented our own Cross Validation that always learn from 4 leagues (all years defined) and test against the fifth league.
"""
def __init__(self,test=False,remote=False):
"""
Initialize a new CrossValidation instance.
:param test: if running in test mode or not
:type test: boolean
:param remote: whether to use the remote database that is configured in DBHandler
:type remote: boolean
"""
self._test = test
self.dbh = DBHandler(league=LEAGUES[0],test=self._test,remote=remote)
self.data = {_l:{_y:(None,None) for _y in YEARS} for _l in LEAGUES}
self._indces = {_l:0 for _l in LEAGUES}
def load_data(self,lookback=2):
"""
Creates all the examples from the database based on the lookback parameter.
Sets the complete_examples, complete_tags and cv_list attributes for later use.
:param lookback: how many previous games do we wish to include in the aggregation that creates the examples
:type lookback: integer
:rtype: None
"""
for league in LEAGUES:
for year in YEARS:
self.dbh.league = league
self.data[league][year] = self.dbh.create_examples(year, lookback)
self._indces[league] += len(self.data[league][year][0])
else:
if league != LEAGUES[0]:
self._indces[league] += self._indces[LEAGUES[LEAGUES.index(league)-1]]
self.complete_examples = []
self.complete_tags = []
for _l in LEAGUES:
for _y in YEARS:
self.complete_examples.extend(self.data[_l][_y][0])
self.complete_tags.extend(self.data[_l][_y][1])
res = []
for league in LEAGUES:
train_leagues = list(set(LEAGUES) - set([league]))
train_leagues.sort(key=LEAGUES.index)
test_league = [league]
train_data , test_data = self.create_indices_leagues(train_leagues,test_league)
res.append((numpy.array(train_data) , numpy.array(test_data)))
self.cv_list = res
@property
def _leagues_indeces(self):
"""
Return a mapping of {league:examples and tags indices}
:rtype: {league:range()}
"""
pass
d={}
for league in LEAGUES:
if league == LEAGUES[0]:
d[league] = range(0,self._indces[league])
else:
d[league] = range(self._indces[LEAGUES[LEAGUES.index(league)-1]],self._indces[league])
return d
def create_indices_leagues(self,train,test):
"""
Given a train set of examples and test set of examples, return a tuple of lists that holds the examples indices (for sklearn classes).
:param train: training set of examples (with tags) (4 leagues)
:param test: testing set of examples (with tags) (1 league)
:rtype: tuple(list,list)
"""
_train , _test = [] , []
for _l in train:
if _l == LEAGUES[0]:
_train.extend(range(0,self._indces[_l]))
else:
_train.extend(range(self._indces[LEAGUES[LEAGUES.index(_l)-1]],self._indces[_l]))
for _t in test:
if _t == LEAGUES[0]:
_test.extend(range(0,self._indces[_t]))
else:
_test.extend(range(self._indces[LEAGUES[LEAGUES.index(_t)-1]],self._indces[_t]))
return _train , _test
@property
def leagues_cross_validation(self):
"""
Returns a list of 5 folds for usage as a cross validation instance from sklearn.
:rtype: list
"""
return self.cv_list
def _leagues_cross_validation(self):
"""
Generator that yields tuples of ((train_examples,train_tags),(test_examples,test_tags)).
:rtype: tuple
"""
for league in LEAGUES:
train_leagues = list(set(LEAGUES) - set([league]))
train_leagues.sort(key=LEAGUES.index)
training_examples , training_tags = [] , []
test_examples , test_tags = [] , []
for _league in train_leagues:
for year in YEARS:
training_examples.extend(self.data[_league][year][0])
training_tags.extend(self.data[_league][year][1])
for year in YEARS:
test_examples.extend(self.data[league][year][0])
test_tags.extend(self.data[league][year][1])
yield (training_examples,training_tags) , (test_examples,test_tags)
from flask import Flask
from dbhandler import DBHandler
import config
app = Flask(__name__)
dbhandler = DBHandler(config.DATABASES['default'])
db = dbhandler.initialize(app, ['user_account'])
@app.route('/test')
def test_request():
User = dbhandler.user_account
usr = db.session.query(User).filter_by(username='******').first()
return usr.username
if __name__ == '__main__':
app.run(debug=True)
for w in newRES.values():
RES = addWord(RES, w)
if TO_FILE:
print "Writing to File"
cap_out = open(outFile, "w")
cap_out.write("Anzahl;Wort;kommt vor in;\n")
for w in RES.values():
cnt = w.cnt
word = w.word
loc = w.locationsAsStr().replace(";", ".")
cont = ("%s;%s;%s;\n" % (cnt, word, loc)).encode("latin-1")
cap_out.write(cont)
cap_out.close()
else:
dbh = DBHandler(True)
for w in RES.values():
cnt = w.cnt
word = w.word
locs = w.locations
if " " in word:
continue
wID = dbh.addWord(word, cnt)
for loc in locs:
dbh.addLocation(wID, loc.paragraph, int(loc.rdNr))
# readDocFile('../tests/doc/P_089-104.doc')
