def start(bot, update):
database = DBHandler()
database.add_id(update.message.chat_id)
logger.info("Connected to bot: %d" % update.message.chat_id)
bot.send_message(chat_id=update.message.chat_id, text="Hello")
bot.send_message(chat_id=update.message.chat_id,
text=database.get_latest_post()[0])
def home():
""" Display the home page """
columns_to_hide = ('vautoid', 'carfaxid', 'body_type', 'created_at')
db_handler = DBHandler()
records = db_handler.get_all_potential_records()
table_headers = list(records[0].keys())
make_model_records = db_handler.get_all_make_models()
make_model = defaultdict(list)
for record in make_model_records:
make_model[record["make"]].append(record["model"])
make_model_year_col = [record["make_model_year"] for record in records]
years = defaultdict(int)
year_regex = re.compile(r"^\s*(\d+)")
for value in make_model_year_col:
match_obj = year_regex.search(value)
if match_obj is not None:
years[match_obj.group(1)] += 1
years = sorted(
years.items(),
reverse=True) # This will return a list of tuple (key, value)
return render_template("home.html",
table_headers=table_headers,
records=records,
make_model=make_model,
columns_to_hide=columns_to_hide,
years=years)
def roll(self, crit_type, database, weapon_type):
if database == '--Select database--' or weapon_type == '--Select weapon type--':
self.textlabel.setText('Invalid input.\nCheck the dropdown menus.')
return
db_filename = database + '.sqlite'
global dbobj
dbobj = DBHandler(db_filename)
if self.checkbox.isChecked():
crit = dbobj.select_random(crit_type, 1, weapon_type)
if crit == None:
self.textlabel.setText('No crits in the database!')
return
else:
crit = dbobj.select_random(crit_type, 0, weapon_type)
if crit == None:
self.textlabel.setText(
'No more unused crits.\nUntick \'used\' box or reset')
return
dbobj.set_used(crit[2], crit_type)
crittext = 'Author: {}\nName: {}\n\n{}'.format(crit[1], crit[2],
crit[3])
self.textlabel.setText(crittext)
def showUrl(local=False, token=None):
if local:
local_make_response = data_to_object
if token is None:
return local_make_response(False, {'message': 'Invalid data'})
else:
local_make_response = data_to_response
if not ('token' in request.form):
return local_make_response(False, {'message': 'Invalid data'})
token = request.form['token']
db = DBHandler()
try:
username = db.tokenLookup(token)
urls = db.userOwn(username)
except DBException as err:
return local_make_response(False, {'message': err.message})
else:
urls = list(urls)
for i in range(len(urls)):
urls[i] = list(urls[i])
urls[i][0] = baseurl + urls[i][0]
return local_make_response(True, {'url': urls})
def unlock(self, username, password, create_new):
user_data_dir = os.path.join(self.data_directory, username)
if create_new:
if os.path.exists(user_data_dir):
raise AuthenticationError('User {} already exists'.format(username))
else:
os.mkdir(user_data_dir)
else:
if not os.path.exists(user_data_dir):
raise AuthenticationError('User {} does not exist'.format(username))
if not os.path.exists(os.path.join(user_data_dir, 'rotkehlchen.db')):
# This is bad. User directory exists but database is missing.
# Make a backup of the directory that user should probably remove
# on his own. At the same time delete the directory so that a new
# user account can be created
shutil.move(
user_data_dir,
os.path.join(self.data_directory, 'backup_%s' % username)
)
raise AuthenticationError(
'User {} exists but DB is missing. Somehow must have been manually '
'deleted or is corrupt. Please recreate the user account.'.format(username))
self.db = DBHandler(user_data_dir, username, password)
self.user_data_dir = user_data_dir
return user_data_dir
def changePassword(local=False,
token=None,
oldPassword=None,
newPassword=None):
if local:
local_make_response = data_to_object
if (token is None) | (oldPassword is None) | (newPassword is None):
return local_make_response(False, {'message': 'Invalid data'})
else:
local_make_response = data_to_response
if (not ('token' in request.form)) | (
not ('oldPassword' in request.form)) | (not ('newPassword'
in request.form)):
return local_make_response(False, {'message': 'Invalid data'})
token = request.form['token']
oldPassword = request.form['oldPassword']
newPassword = request.form['newPassword']
db = DBHandler()
try:
username = db.tokenLookup(token)
db.userChangePassword(username, oldPassword, newPassword)
except DBException as err:
return local_make_response(False, {'message': err.message})
else:
return local_make_response(True)
def register(local=False, username=None, password=None):
if local:
local_make_response = data_to_object
if (username is None) | (password is None):
return local_make_response(False, {'message': 'Invalid data'})
else:
local_make_response = data_to_response
if (not ('username' in request.form)) | (not ('password'
in request.form)):
return local_make_response(False, {'message': 'Invalid data'})
username = request.form['username']
password = request.form['password']
if (len(username) < 2):
return local_make_response(
False, {'message': 'Username\'s length should be more than 2'})
if (len(password) < 8):
return local_make_response(
False, {'message': 'Password\'s length should be more than 7'})
db = DBHandler()
try:
db.userCreate(username, password)
except DBException as err:
return local_make_response(False, {'message': err.message})
else:
return local_make_response(True)
def get_user_data(self, userid):
db = DBHandler()
db.connect()
query = "SELECT email, signup_date FROM Users where userid = %s;"
cursor = db.executeQuery(query, (str(userid)))
tupl = cursor.fetchone()
return tupl
def get_results(self):
from_ = self.from_datetime_edit
to = self.to_datetime_edit
# from_.setDateTime(now)
# to = to.dateTimeFromText(now)
from_date = from_.textFromDateTime(from_.dateTime()) # Starting date
to_date = to.textFromDateTime(to.dateTime()) # Ending date
with DBHandler(self.context.get_database) as cursor:
cursor.execute(
"""SELECT product_name, sum(quantity_sold), sum(price)
FROM `orders` WHERE username= ? AND
`date` BETWEEN ? AND ?
GROUP BY product_name;""",
[self.username, from_date, to_date])
products_result = cursor.fetchall()
cursor.execute(
"""SELECT sum(price)
FROM `orders` WHERE username = ?
AND `date` BETWEEN ? AND ?""",
[self.username, from_date, to_date])
total_result = cursor.fetchone()
# print(products_result)
return products_result, total_result
def main():
# Read in data and extract data arrays
#logging.info("Reading input files.")
dbh = DBHandler(options.db_config_filename, options.db_config_name)
ssh = SmartMetHandler(options.smartmet_config_filename,
options.smartmet_config_name,
sleep_time=options.requests_throttle_time)
fh = FileHandler(s3_bucket='fmi-sasse-classification-dataset')
tracker = Tracker(dbh, ssh)
starttime = dt.datetime.strptime(options.starttime, "%Y-%m-%dT%H:%M:%S")
endtime = dt.datetime.strptime(options.endtime, "%Y-%m-%dT%H:%M:%S")
d = starttime
ymd = '{}{}{}'.format(d.year, d.month, d.day)
while d <= endtime:
logging.info('Processing time: {}'.format(d))
tracker.run(d)
d += timedelta(hours=1)
if ymd != '{}{}{}'.format(d.year, d.month, d.day):
save_dataset(tracker, d, fh=fh, db=dbh)
ymd = '{}{}{}'.format(d.year, d.month, d.day)
#if tracker.dataset is not None and len(tracker.dataset) > 0:
save_dataset(tracker, d, fh=fh, db=dbh)
def __init__(self):
self.config = util.load_config()
util.register_gcp_credential(self.config)
self.dbhandler = DBHandler(self.config)
self.gcphandler = GCPHandler(self.config)
target_ids = util.load_target_ids('target.txt')
self.user_sids = self.dbhandler.select_user_sids(target_ids)
def authenticate_user(self):
username = self.username_line_edit.text()
password = self.password_line_edit.text()
with DBHandler(self.context.get_database) as cursor:
find_user_query = """
SELECT priviledge FROM users WHERE
username = ? AND password = ?
"""
cursor.execute(find_user_query, [username, password])
results = cursor.fetchone()
try:
if results[0] == 0:
from saleswindow import SalesWindow
# Create session and all activites will be recorded with `username`
self.saleswindow = SalesWindow(username=username,
context=self.context)
self.saleswindow.show()
self.hide()
if results[0] == 1:
from adminwindow import AdminWindow
self.adminwindow = AdminWindow(context=self.context)
self.adminwindow.show()
self.hide()
except TypeError as e:
self.error_label.setText(str(e))
def run(bot):
database = DBHandler()
while True:
for user in database.get_items():
latest_posts = get_updates(database)
send_posts_to(bot, user, latest_posts)
sleep(1 * config.HOURS)
def __init__(self,
timeout=None,
db_config=("127.0.0.1", "my_database", "my_user")):
assert timeout == None or isinstance(
timeout, int), "timeout must be an integer in seconds"
self.channels = {} # channel_name = [channel_callback]
self.mustExit = threading.Event()
self.timeout = timeout
self.db = DBHandler(db_config)
def get_product_list(self):
with DBHandler(self.context.get_database) as cursor:
cursor.execute("SELECT * FROM products")
results = cursor.fetchall()
# Genexpr to get all items from database
self.product_list = [
self._Product(*value) for _, value in enumerate(results)
]
def __init__(self, host, port, dbhost="localhost", dbport=27017):
threading.Thread.__init__(self)
self.ssap = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.ssap.bind((host, port))
self.ssap.listen(1)
self.client_handlers = dict()
self.dbhandler = DBHandler(mongohost=dbhost, mongoport=dbport)
print('listening on ', port)
def resolve(short):
db = DBHandler()
try:
redir = db.urlLookup(short)
db.urlVisit(short)
except DBException as err:
return err.message
else:
return redirect(redir)
def test_2(self):
h = DBHandler(inmemory=True)
records = [Record(1, datetime(2020,12,24,0,0,0), 88, 99),
Record(1, datetime(2020,12,24,0,0,0), 88, 99),
Record(1, datetime(2020,12,24,0,0,0), 88, 99),]
[h.insert_record(r) for r in records]
h.update_uploaded_row_by_less_then_uid(2)
rows = h.read_unuploaded_rows()
uids = [r[0] for r in rows]
self.assertEqual(uids, [3])
def save_comments():
""" Save comments to potential deals table """
potential_deal_id = int(request.form.get("id"))
action = request.form.get("action")
if action.lower() == "none":
action = None
comments = request.form.get("comments")
db_handler = DBHandler()
db_handler.update_by_id(potential_deal_id, action, comments)
# return redirect(url_for("home"))
return jsonify({"success": True}), 200
def __init__(self, parent=None):
super().__init__(parent)
conn = sqlite3.connect('cetizen.db')
self.db = DBHandler(conn)
self.ui = loadUi('cetizen.ui', self)
self.logger.setText('')
self.table.setAlternatingRowColors(True)
#self.table.setRootIsDecorated(False)
self.setWindowTitle('Cetizen Crawler')
self.ui.show()
def __init__(self, test=False, remote=False):
"""
Initialize a new CrossValidation instance.
:param test: if running in test mode or not
:type test: boolean
:param remote: whether to use the remote database that is configured in DBHandler
:type remote: boolean
"""
self._test = test
self.dbh = DBHandler(league=LEAGUES[0], test=self._test, remote=remote)
self.data = {_l: {_y: (None, None) for _y in YEARS} for _l in LEAGUES}
self._indces = {_l: 0 for _l in LEAGUES}
def logout(local=False, token=None):
if local:
local_make_response = data_to_object
else:
local_make_response = data_to_response
if 'token' in request.form:
token = request.form['token']
if not (token is None):
db = DBHandler()
db.userLogout(request.form['token'])
return local_make_response(True)
def _startDataRead(self):
"""
Método utilizado para configurar a conexão socket e inicializar uma thread para a leitura dos dados e atualização da interface grafica
:param ip: ip da conexão socket
:param port: porta para a conexao socket
"""
try:
self._apogeu = int(self._apogeu)
self._port = int(self._port)
self._serverIP = str(ip_address(self._serverIP))
if self._login == 'supernova' and self._senha == 'astra':
Window.set_system_cursor("wait")
self._connect = Cliente(self._serverIP, self._port)
self._connect.start()
Window.set_system_cursor("arrow")
self._updateThread = Thread(target = self.updater)
self._updateThread.daemon = False
self.ids.imagem_conexao.background_normal = 'imgs/conectado.png'
self.ids.latitude.font_size = self.ids.altitude.font_size/2
self.ids.longitude.font_size = self.ids.altitude.font_size/2
self.ids.graphAcelerometro.clearLabel()
self.ids.graphGiroscopio.clearLabel()
self.ids.graphAltitude.clearLabel()
self._dataBase = DBHandler(self._missao)
self._disableNewConnections()
self._limitesGraficos()
self.enableSwitchesAndButtons()
self._updateThread.start()
self._conn.dismiss()
else:
self._connError.ids.erroConnect.text = "Senha incorreta!"
self._connError.open()
raise Exception('Senha incorreta!')
except ValueError:
if (type(self._apogeu) != int):
self._connError.ids.erroConnect.text = "Selecione o apogeu!"
self._connError.open()
else:
self._connError.ids.erroConnect.text = "Erro: server/port mal definidos!"
self._connError.open()
raise ValueError
except ConnectionRefusedError:
Window.set_system_cursor("arrow")
self._connError.ids.erroConnect.text = "Falha ao conectar!"
self._connError.open()
raise ConnectionRefusedError
def __init__(self):
self.continue_schedule = False
self.local_db = DBHandler()
self.all_synonyms = set()
self.trustpilot = TrustPilotCrawler()
self.reddit = RedditScraper()
self.scrapers = {
'trustpilot': TrustPilotCrawler(),
'reddit': RedditScraper()
}
# TODO: Make Environment Variables for API info
self.kwe_api = f'http://{os.environ["KWE_API_HOST"]}/'
self.kwe_api_key = {'Authorization': os.environ['KWE_API_KEY']}
self.sa_api = f'http://{os.environ["SA_API_HOST"]}/prediction/'
self.sa_api_key = {'Authorization': os.environ['SA_API_KEY']}
self.synonym_api = f'http://{os.environ["GATEWAY_API_HOST"]}/api/synonyms'
self.synonym_api_key = {'Authorization': os.environ['GATEWAY_API_KEY']}
self.sentiment_categories = [{
'category': 'positive',
'upper_limit': 1,
'lower_limit': 0.55
}, {
'category': 'negative',
'upper_limit': 0.45,
'lower_limit': 0
}, {
'category': 'neutral',
'upper_limit': 0.55,
'lower_limit': 0.45
}]
self.kwe_interval = timedelta(hours=1)
self.kwe_latest = self._read_kwe_date()
self.continue_schedule = True
self.schedule_thread = Thread()
self.crawler_schedule_thread = Thread()
self.begin_schedule()
logging.info(
f'Initiated scheduler, will create snapshots from {self.kwe_latest}'
)
def __init__(self,
id=None,
create=None,
db_config=("127.0.0.1", "my_database", "my_user"),
related_task_id=None,
mustExit=None):
assert id or create, "Bad arguments, no id nor create"
assert id == None or isinstance(
id, int), "Bad arguments id, must be an int"
assert mustExit, "Bad arguments, mustExit not provided"
assert create == None or (
isinstance(create, tuple) and 3 <= len(create) <= 4
), "Bad argument create, must be (user_id,type,request in json[,is_visible_to_user true/false])"
self.db = DBHandler(db_config)
if create:
self.status, self.progress_percent, self.user_id, self.type, self.request, self.result = "created", 0, create[
0], create[1], create[2], []
if isinstance(self.request, str):
self.request = json.loads(self.request)
r = self.db.execute(
"INSERT INTO tasks (status, progress_percent, type, request, user_id) VALUES (%s, %s, %s, %s::jsonb, %s) RETURNING id, start_date, end_date ;",
[
self.status, self.progress_percent, self.type,
json.dumps(self.request), self.user_id
])
r = self.db.fetchall(r)
self.id, self.start_date, self.end_date = r[0]
if related_task_id:
self.db.execute(
"UPDATE tasks SET related_task_id = %s WHERE id = %s",
[related_task_id, self.id])
self.related_task_id = related_task_id
with Notifier() as notifier:
notifier.send_notification(
"task", "id={},status=created".format(self.id))
else:
r = self.db.execute(
"SELECT status, progress_percent, type, request, result, user_id, start_date, end_date, related_task_id FROM tasks WHERE id = %s;",
[id])
r = self.db.fetchall(r)
self.id = id
self.status, self.progress_percent, self.type, self.request, self.result, self.user_id, self.start_date, self.end_date, self.related_task_id = r[
0]
if self.result == None:
self.result = []
def create(local=False, url=None, short=None, token=None):
if local:
local_make_response = data_to_object
if url is None:
return local_make_response(False, {'message': 'Invalid data'})
else:
local_make_response = data_to_response
if (not ('url' in request.form)):
return local_make_response(False, {'message': 'Invalid data'})
url = request.form['url']
if 'short' in request.form:
short = request.form['short']
else:
short = None
if 'token' in request.form:
token = request.form['token']
if not (token is None):
try:
username = db.tokenLookup(request.form['token'])
except DBException as err:
return local_make_response(False, {'message': err.message})
else:
username = None
if not (short is None):
if not (short.isalnum()):
return local_make_response(
False, {'message': 'Slug must be alphanumeric.'})
db = DBHandler()
if (not validators.url(url)) | ('turang.ga' in url):
return local_make_response(False, {'message': 'URL is not valid.'})
newUrl = baseurl
try:
newUrl += db.urlNew(url, short, username)
except DBException as err:
return local_make_response(False, {'message': err.message})
else:
return local_make_response(True, {'url': newUrl})
def main():
#if options.dataset_file is None or options.output_file is None:
dbh = DBHandler(options.db_config_file, options.db_config_name)
dbh.return_df = False
fh = FileHandler(s3_bucket=options.bucket)
with open(options.example_config_file) as f:
setups = yaml.load(f.read(), Loader=yaml.FullLoader)
for setup in setups['examples']:
output_file = '{}/{}-{}.csv'.format(
setup['output_dir'], setup['starttime'].strftime('%Y%m%dT%H%M%S'),
setup['endtime'].strftime('%Y%m%dT%H%M%S'))
# Read in data and extract data arrays
logging.info("Reading input data for {} - {}...".format(
setup['starttime'], setup['endtime']))
features, meta_params, labels, all_params = get_param_names(
options.param_config_file)
data = fh.read_data(
[setup['dataset_file']],
options,
return_meta=True,
starttime=setup['starttime'], #.strftime('%Y-%m-%dT%H:%M:%S'),
endtime=setup['endtime'])[0] #.strftime('%Y-%m-%dT%H:%M:%S'))[0]
X, y, meta = data
model = fh.load_model(setup['model_file'])
scaler = fh.load_model(setup['scaler_file'])
logging.info('Predicting with {} samples...'.format(len(X)))
y_pred = model.predict(X)
df = pd.DataFrame(meta, columns=options.meta_params)
X_inv = pd.DataFrame(scaler.inverse_transform(X), columns=X.columns)
df = pd.concat(
[df.reset_index(drop=True),
X_inv.reset_index(drop=True)], axis=1)
df = dbh.get_geom_for_dataset_rows(df)
df['y_pred'] = y_pred
df['y'] = y
fh.df_to_csv(df, output_file)
def main():
# Read in data and extract data arrays
logging.info("Reading input data for {} - {}...".format(
options.starttime, options.endtime))
#if options.dataset_file is None or options.output_file is None:
dbh = DBHandler(options.db_config_filename, options.db_config_name)
dbh.return_df = False
fh = FileHandler(s3_bucket=options.bucket)
starttime = dt.datetime.strptime(options.starttime, "%Y-%m-%dT%H:%M:%S")
endtime = dt.datetime.strptime(options.endtime, "%Y-%m-%dT%H:%M:%S")
features, meta_params, labels, all_params = get_param_names(
options.param_config_filename)
scaler = fh.load_model(options.scaler_file)
# TODO change to read from operational data
data = pd.DataFrame(dbh.get_dataset(all_params), columns=all_params)
# TODO use original id stored in db. The id is used to assign predicted classes to a storm object (while saving to db)
# As far as we do not have operational data, dummy ids are used
data.loc[:, 'id'] = 0
data = data.loc[data['weather_parameter'] == 'WindGust']
# Add week
data['point_in_time'] = pd.to_datetime(data['point_in_time'], utc=True)
data['week'] = data['point_in_time'].dt.week
X = data.loc[:, features]
X = scaler.transform(X)
model = fh.load_model(options.model_file)
logging.info('Predicting with {} {} samples...'.format(
options.model, len(X)))
y_pred = model.predict(X)
# Save to db
logging.info('Saving...')
dbh.save_classes(data.loc[:, 'id'], y_pred)
logging.info('done.')
def update_database(self, product):
now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M")
with DBHandler(self.context.get_database) as cursor:
update_SQL = """
UPDATE products SET
quantity_in_stock = quantity_in_stock - ?
"""
insert_SQL = """
INSERT INTO orders VALUES (?, ?, ?, ?, ?, ?)
"""
cursor.execute(update_SQL, [product.quantity])
cursor.execute(insert_SQL, [
None, self.username, now, product.name, product.quantity,
product.subtotal
])
print('Success')
def delete(local=False, short=None, token=None):
if local:
local_make_response = data_to_object
if (short is None) | (token is None):
return local_make_response(False, {'message': 'Invalid data'})
else:
local_make_response = data_to_response
if ((not ('short' in request.form)) | (not ('token' in request.form))):
return local_make_response(False, {'message': 'Invalid data'})
short = request.form['short']
token = request.form['token']
db = DBHandler()
try:
username = db.tokenLookup(token)
db.urlDelete(username, short)
except DBException as err:
return local_make_response(False, {'message': err.message})
else:
return local_make_response(True)
