def parse(self, response):
self.driver.get(response.url)
self.scroll_down_n_times(1)
questions = self.get_questions()
time.sleep(self.SLEEP_TIME)
# best_answers = self.get_best_answers()
# time.sleep(self.SLEEP_TIME)
dates = self.get_question_dates()
time.sleep(self.SLEEP_TIME)
# assert len(questions) == len(best_answers) == len(dates)
if not self.CATEGORY:
categories = self.get_categories()
time.sleep(self.SLEEP_TIME)
assert len(questions) == len(categories)
for i in range(len(questions)):
date = DbManager.convert_date_string_to_datetime(dates[i])
category = self.CATEGORY if self.CATEGORY else categories[i]
question = { 'content': questions[i], 'upvotes': -1, 'category': category, 'timestamp': date }
DbManager.add_question(question)
# answer = { 'content': best_answers[i], 'upvotes': -1, 'is_best_answer': True, 'timestamp': datetime.now()}
# DbManager.add_answer(answer)
self.driver.close()
def build_index_from_pickles(self):
dbManager = DbManager()
pickleManager = PickleManager()
print("merging pickles: %s" % (datetime.datetime.now()))
full_index = pickleManager.merge_index_structures()
print("saving full index: %s" % (datetime.datetime.now()))
dbManager.save_full_index(full_index)
print("full index saved: %s" % (datetime.datetime.now()))
def get_movies(self, term):
dbManager = DbManager()
db_list = dbManager.get_movies(term)
#for index, row in db_list.iterrows():
return [{
'id': m[1]['id'],
'label': m[1]['title'],
'value': m[1]['title']
} for m in db_list.iterrows()]
def handle_whois_queue():
"""
read visitors from database table and look for whois info
:return: nothing
"""
if is_whois_locked():
return False
lock_whois(True)
try:
visitor_lst = read_queue()
for visitor in visitor_lst:
ws = WhoisTracker()
dbman = DbManager()
dbman.connect()
try:
handle_whois(ws,
dbman,
visitor_ip=visitor['visitor_ip'],
referer_url=visitor['referer_url'],
page_url=visitor['page_url'],
datetime_of_visit=visitor['datetime_of_visit']
)
del_visitor_from_queue(visitor['id'], dbman.conn)
dbman.commit_changes(True)
except:
dbman.commit_changes(False)
raise
finally:
dbman.close(commit=None)
finally:
lock_whois(False)
return True
def main_task(max_days, stations_list, mail_list):
properties = []
property_urls = []
db_manager = DbManager()
domain, initial_search_urls = get_station_based_initial_search_urls(
max_days, stations_list)
for initial_search_url in initial_search_urls:
try:
results_pages = get_results_pages(domain, initial_search_url)
for results_page in results_pages:
domain, individual_house_links = get_individual_house_links(
domain, results_page)
for house_link in individual_house_links:
property_data = scrape_individual_house(domain, house_link)
db_manager.add_new_property(property_data)
if property_data['url'] not in property_urls:
properties.append(property_data)
property_urls.append(property_data['url'])
except Exception as e:
send_email(
'error@error_email.com',
f'An error occurred whilst scraping {initial_search_url}: {str(e)}'
)
pass
help_to_buy_filters = ['new', 'luxury', 'help']
help_to_buy_properties = []
for item in properties:
if any(word in item['full_description_text'] for word in help_to_buy_filters) \
or any(word in item['key_features_text'] for word in help_to_buy_filters):
help_to_buy_properties.append(item)
properties.remove(item)
data = [{
'type':
'Possibly Help to Buy',
'properties':
help_to_buy_properties,
'number_of_properties':
len(help_to_buy_properties),
'average_price':
np.average([item['price'] for item in help_to_buy_properties]),
}, {
'type':
'Other Properties',
'properties':
properties,
'number_of_properties':
len(properties),
'average_price':
np.average([item['price'] for item in properties]),
}]
send_results(data, mail_list)
def test_run_sql():
db = DbManager()
with pytest.raises(DbManagerError) as error_info:
db.run_sql("blablala1")
db.run_sql("INSERT INTO data VALUES (?,?,?,?)", [(1, 2, 3, 4)])
db.run_sql("INSERT INTO cities VALUES (?,?,?,?,?)",
[(3, "4", "5", "6", "&")])
def addCarhabLayer(self, carhabLayer):
db = DbManager(carhabLayer.dbPath)
if not db.version() == last_db_version():
versions = [os.path.basename(os.path.join(pluginDirectory,'update', f)).split('upd_')[1].split('.sql')[0] for f in listdir(os.path.join(pluginDirectory,'update')) if os.path.basename(os.path.join(pluginDirectory,'update', f)).startswith('upd_')]
versions = [v for v in versions if v > db.version()]
versions.sort()
for v in versions:
res = db.executeScript(os.path.join(pluginDirectory,'update', 'upd_'+v+'.sql'))
if res:
error_update_db()
else:
db.set_version(last_db_version())
db.commit()
self.layerMap[carhabLayer.id] = carhabLayer
root = QgsProject.instance().layerTreeRoot()
group = root.addGroup('carhab_'+carhabLayer.getName())
for tableToLoad in ('point', 'polyline', 'polygon'):
group.addLayer(self.loadLayerTable(carhabLayer, tableToLoad))
#self.loadLayerTable(carhabLayer.dbPath, tableToLoad)
# Create row corresponding to carhab layer into carhab layer list.
uiPath = os.path.join(pluginDirectory, 'carhab_layer_item.ui')
carhabLayerWdgt = loadUi(uiPath)
jobNameLbl = carhabLayerWdgt.findChild(QLabel,'job_name_label')
jobNameLbl.setText(carhabLayer.getName())
def save_index_and_max_freq(self, index_structures, file_name):
dbManager = DbManager()
index = {}
max_freqs = {}
for index_structure in index_structures:
dbManager.insert_max_freq_doc(index_structure.doc_id,
index_structure.get_max_freq())
for i in range(len(index_structure.Terms)):
term = index_structure.Terms[i]
frequency = index_structure.Frequencies[i]
if term in index:
index[term] = index[term] + "," + str(frequency)
else:
index[term] = str(frequency)
self.storeData(index, "data_" + file_name)
def rank_cosine_sim(self, doc_ids, q_terms):
dbManager = DbManager()
builder = StructureBuilder()
docs_relevant_scores = {}
q_freqs = dict()
# set the query terms frequencies
for q_term in q_terms:
if q_term in q_freqs:
q_freqs[q_term] += 1
else:
q_freqs[q_term] = 1
# set max frequency
sorted_q_freqs = sorted(q_freqs.items(),
key=operator.itemgetter(1),
reverse=True)
max_q_freq = sorted_q_freqs[0][1]
for doc_id in doc_ids:
tf_idf_sum = 0
denom_di_sum = 0
denom_qi_sum = 0
for q_term in q_terms:
q_doc_freq = self.get_q_doc_freq(q_term, doc_id)
if q_doc_freq == None: continue # not found on index
max_freq_doc = dbManager.get_max_freq_doc(doc_id)
if max_freq_doc != None:
self.max_freq_docs[doc_id] = max_freq_doc
# number of documents in DC in which q_term appears at least once.
n_docs_q_term = len(self.q_terms_freqs[q_term])
tf_idf_doc = self.calc_tf_idf(q_doc_freq, max_freq_doc,
self.docs_count,
n_docs_q_term)
tf_idf_q = self.calc_tf_idf(q_freqs[q_term], max_q_freq,
self.docs_count, n_docs_q_term)
tf_idf = tf_idf_doc * tf_idf_q
tf_idf_sum += tf_idf
denom_di_sum += tf_idf_doc**2
denom_qi_sum += tf_idf_q**2
#tf_idf_sum += tf_idf_doc
denom = math.sqrt(denom_di_sum) * math.sqrt(denom_qi_sum)
score = tf_idf_sum / denom
docs_relevant_scores[doc_id] = round(score, 3)
sorted_docs_total_freqs = sorted(docs_relevant_scores.items(),
key=operator.itemgetter(1),
reverse=True)
return sorted_docs_total_freqs
def setupUi(self, MainWindow):
MainWindow.setObjectName("MainWindow")
MainWindow.resize(1440, 900)
MainWindow.setWindowTitle("TextBlock Application")
self.centralwidget = QtWidgets.QWidget(MainWindow)
self.centralwidget.setObjectName("centralwidget")
self.db = DbManager('text_block')
self.setStyle(MainWindow)
self.setMenubar(MainWindow)
self.setHeaderText()
self.setButtons()
self.setIOLayout()
self.retranslateUi(MainWindow)
QtCore.QMetaObject.connectSlotsByName(MainWindow)
def test_get_stations():
data_manager = DataManager(DbManager())
result = data_manager.get_stations()
assert type(result) == list
assert len(result) > 0
result = data_manager.get_stations(531)
assert type(result) == tuple
assert len(result) == 7
def run(self):
try:
data = DataManager(DbManager()).prepare_necessary_data()
self.signals.result.emit(data)
except DataManagerError as e:
self.signals.error.emit(e)
get_logger("main_program").exception(e)
finally:
self.signals.finished.emit()
def fetch_resource(resource_type):
args = Args(resource_type)
db_manager = DbManager(args)
scraper = get_scraper(args)
if args.refetch:
db_manager.delete_resource(args.db_key)
if not db_manager.resource_exists(args.db_key):
resource_data = scraper.get_resource(args.query_params)
if scraper.driver:
scraper.driver.quit()
db_manager.save_resource(args.db_key, resource_data)
return db_manager.fetch_resource(args.db_key)
def run(self ):
print 'run worker'
try:
print 'begin try'
if CarhabLayerRegistry.instance().getCurrentCarhabLayer():
# Connect to db.
self.db = DbManager(CarhabLayerRegistry.instance().getCurrentCarhabLayer().dbPath)
# To let import geos invalid geometries.
self.layer.setValid(True)
layerFeatCount = self.layer.featureCount()
# Initialisation of useful variables to calculate progression.
lastDecimal = 0
i = 0
for feature in self.layer.getFeatures():
if not self.stop:
featGeom = feature.geometry()
if featGeom.type() == 2: # Polygons case
if featGeom.isMultipart(): # Split multipolygons
for part in featGeom.asGeometryCollection():
if not part.isGeosValid(): # May be a problem...
print 'part not valid'
self.insertPolygon(part)
else:
self.insertPolygon(feature.geometry())
# Calculate and emit new progression value (each percent only).
newDecimal = int(100*i/layerFeatCount)
if lastDecimal != newDecimal:
self.progress.emit(newDecimal)
lastDecimal = newDecimal
i = i + 1
else: # Thread has been aborted
print 'abort'
# Cancel inserts already done
self.db.conn.rollback()
self.finished.emit(False, 2)
break
self.db.commit()
self.db.close()
self.finished.emit(True, 0)
else: # None current carhab layer (error code 1)
self.finished.emit(False, 1)
except:
print 'exception'
import traceback
print traceback.format_exc()
self.error.emit(traceback.format_exc())
self.finished.emit(False, 0)
def test_get_sensors_by_station_id(stat_id, expected):
db = DbManager()
result = db.get_sensors_by_station_id(stat_id)
assert type(result) == list
assert len(result) == expected
with pytest.raises(DbManagerError):
db.get_sensors_by_station_id("")
db.get_sensors_by_station_id(None)
def rank(self, doc_ids, q_terms):
"""
docs => array, every
doc => should have an array of terms
[term] => must have frequency
Calculate Relevance Score
1) Calculate TF
I need for each document the list of words (non-stop, stemmed/lemmatized) and their frequencies
I need the frequency of the most frequently-occurred term of each document (constant per document)
2) Calculate IDF
I need the number of documents in DC (constant)
I need the number of documents in DC in which w appears at least once.
"""
dbManager = DbManager()
builder = StructureBuilder()
docs_relevant_scores = {}
for doc_id in doc_ids:
tf_idf_sum = 0
for q_term in q_terms:
q_doc_freq = self.get_q_doc_freq(q_term, doc_id)
if q_doc_freq == None: continue # not found on index
max_freq_doc = dbManager.get_max_freq_doc(doc_id)
if max_freq_doc != None:
self.max_freq_docs[doc_id] = max_freq_doc
# number of documents in DC in which q_term appears at least once.
n_docs_q_term = len(self.q_terms_freqs[q_term])
tf_idf_doc = self.calc_tf_idf(q_doc_freq, max_freq_doc,
self.docs_count,
n_docs_q_term)
tf_idf_sum += tf_idf_doc
docs_relevant_scores[doc_id] = round(tf_idf_sum, 3)
sorted_docs_total_freqs = sorted(docs_relevant_scores.items(),
key=operator.itemgetter(1),
reverse=True)
return sorted_docs_total_freqs
def insertPolygon(self, geometry):
# Convert geometry
wktGeom = geometry.exportToWkt()
# To force 2D geometry
if len(wktGeom.split('Z')) > 1:
wktGeom = wktGeom.split('Z')[0] + wktGeom.split('Z')[1]
wktGeom = wktGeom.replace(" 0,", ",")
wktGeom = wktGeom.replace(" 0)", ")")
geom = "GeomFromText('"
geom += wktGeom
geom += "', 2154)"
geomObj = {}
geomObj['the_geom'] = geom
db = DbManager(CarhabLayerRegistry.instance().getCurrentCarhabLayer().dbPath)
r = Recorder(db, 'polygon')
r.input(geomObj)
db.commit()
db.close()
def __init__(self):
wx.Frame.__init__(self, None, wx.ID_ANY, u'艦ZUME', size=(450, 500))
self.isCaputre = False
# アイコン
icon = wx.Icon('./data/img/kanzume.ico', wx.BITMAP_TYPE_ICO)
self.SetIcon(icon)
# パケットキャプチャ準備
self.pcap = PacketCapture()
self.pcap.set_text(self.set_text)
self.pcap.set_progress_value(self.set_progress_value)
self.army_mng = None
# データベース準備
self.dbm = DbManager()
# ステータスバーの初期化
self.CreateStatusBar()
self.SetStatusText(u'Let\'s capture!')
self.GetStatusBar().SetBackgroundColour(None)
# メニューバーの初期化
selected_nwif_index = self.dbm.get_selected_nwif_index()
self.pcap.nif_mng.set_selected_devId(selected_nwif_index)
menuBar = CaptureMenuBar(self.pcap.nif_mng.get_networkif_listf())
menuBar.set_selected_nwif(selected_nwif_index)
self.SetMenuBar(menuBar)
self.Bind(wx.EVT_MENU, self.select_networkif)
# パネル
panel_base = wx.Panel(self, wx.ID_ANY)
self.panel_capture = CapturePanel(panel_base)
self.panel_result = ResultPanel(panel_base, self.dbm)
# クリックリスナ登録
self.panel_capture.set_on_click_listener(self.cap_btn_click_listener)
layout = wx.BoxSizer(wx.VERTICAL)
layout.Add(self.panel_capture, flag=wx.EXPAND | wx.ALL, border=10)
layout.Add(self.panel_result, proportion=1, flag=wx.EXPAND)
panel_base.SetSizer(layout)
layout.Fit(panel_base)
def test_get_from_api():
data_manager = DataManager(DbManager())
with pytest.raises(ApiError) as e_info:
data_manager.call_api("")
data_manager.call_api("asdef")
data_manager.call_api("http://google.com")
with pytest.raises(ApiError) as e_info:
data_manager.call_api(DataManager.URL_DATA)
result = data_manager.call_api(data_manager.URL_STATIONS)
assert type(result) == list
assert len(result) > 0
result = data_manager.call_api(f"{DataManager.URL_DATA}/92")
assert type(result) == dict
assert result["key"] == "PM10"
assert type(result["values"]) == list
assert len(result["values"]) > 0
assert "date" in result["values"][0]
def get_qprecs(self, algo, movies, nr_recs, items, user_id):
dbManager = DbManager()
#user_id = dbManager.insert_and_get_min_user_id()
rating_value = 5
# remove all ratings from this user
dbManager.remove_ratings(user_id)
# insert ratings (value: 10) for user 0 for each movie
for m in movies:
dbManager.insert_rating(user_id, m, rating_value)
# get recs
all_recs = self.get_recs_from_recserver([user_id], nr_recs, algo,
items)
print(all_recs)
for user_recs in all_recs:
if user_recs['recs'] != None:
for rec in user_recs['recs']:
rec['title'] = dbManager.get_movie(
rec['item']).loc[0, 'title']
rec['image_url'] = self.get_image_url(rec['item'])
return all_recs
grid_layout.addWidget(signup_btn, 4, 1)
signup_btn.clicked.connect(self.on_signup_click)
self.password_field.setEchoMode(QLineEdit.PasswordEchoOnEdit)
self.show()
def on_signup_click(self):
try:
self.db.signup(self.login_field.text(), self.purse_field.text(),
self.password_field.text(), self.email_field.text())
except:
self.error_dialog = QtWidgets.QErrorMessage()
self.error_dialog.showMessage('Such user already is exist!')
return
#придумать как вернуться назад к ауфу
QMessageBox.information(
self, "User created.",
str(self.login_field.text() + " has been added! Please, Log In!"),
QMessageBox.Ok)
lg = self.wm.get_window("login")
lg.setVisible(True)
self.setVisible(False)
if __name__ == '__main__':
app = QApplication(sys.argv)
wm = WindowsManager()
wm.add("db", DbManager())
ex = Signup(wm)
sys.exit(app.exec_())
def get_document(self, doc_id):
dbManager = DbManager()
return dbManager.get_document(doc_id)
def get_candidate_documents_ids(self, q_terms):
"""
1) look for each stemmed query term in redis to get their documents e.g. => 1:5,2:10,4:22
2) create a dictionary with doc:count
3) only return the docs that have all query terms, if there are less than 50,
then reduce the amount of minimum query terms until you get 50 or more.
--for every doc that already exists add the current freq
--3) order the dictionary by the most frequent and return the id docs list.
"""
MAX_DOCUMENTS_TO_RETRIEVE = 50
candidate_documents = []
dbManager = DbManager()
docs_total_freqs = {
} # used to track every candidate document => doc:totalFrequency
q_terms_count_per_doc = {
} # only tracks the q_terms_count_per_doc => doc:count
docs_count_all_terms = []
count = 1
for q_t in q_terms:
doc_freqs = dbManager.get_index_term(q_t)
if doc_freqs != None:
docs_with_frequency = doc_freqs.split(',')
self.q_terms_freqs[q_t] = docs_with_frequency
for doc_with_freq in docs_with_frequency:
if len(doc_with_freq) > 1:
doc_freq = doc_with_freq.split(
':'
) # 0-index is docId | 1-index is the doc frequency
docId = doc_freq[0]
docFreq = doc_freq[1]
if docId in docs_total_freqs:
docs_total_freqs[docId] += docFreq
q_terms_count_per_doc[docId] += 1
if count == len(q_terms) and q_terms_count_per_doc[
docId] == len(q_terms):
docs_count_all_terms.append(q_terms)
else:
docs_total_freqs[docId] = docFreq
q_terms_count_per_doc[docId] = 1
count += 1
q_terms_min_size = len(q_terms) if len(q_terms) == 1 else int(
len(q_terms) / 2)
# get the most frequent terms
sorted_docs_total_freqs = sorted(docs_total_freqs.items(),
key=operator.itemgetter(1),
reverse=True)
for k, v in sorted_docs_total_freqs:
# get the docs which have all query terms at least
if q_terms_count_per_doc[k] == len(q_terms):
candidate_documents.append(k)
elif len(docs_count_all_terms) < MAX_DOCUMENTS_TO_RETRIEVE:
candidate_documents.append(k)
if len(candidate_documents) >= MAX_DOCUMENTS_TO_RETRIEVE:
break
if len(candidate_documents) > MAX_DOCUMENTS_TO_RETRIEVE:
candidate_documents = candidate_documents[
0:MAX_DOCUMENTS_TO_RETRIEVE]
return candidate_documents
def test_get_data_by_sensors_ids():
db = DbManager()
result = db.get_data_by_sensors_ids([3575, 3576])
assert len(result) >= 96
assert type(result) == list
def last_db_version():
db = DbManager(path.join(pluginDirectory, 'empty.sqlite'))
return db.version()
cond_others = "pref_cd not in (12, 13, 14)"
sql = """ select
sum( case when {cond_tokyo} then 1 else 0 end ) as cnt_tokyo
,sum( case when {cond_chiba} then 1 else 0 end ) as cnt_chiba
,sum( case when {cond_kanagawa} then 1 else 0 end ) as cnt_kanagawa
,sum( case when {cond_others} then 1 else 0 end ) as cnt_others
from users
""".format(cond_tokyo=cond_tokyo,
cond_kanagawa=cond_kanagawa,
cond_chiba=cond_chiba,
cond_others=cond_others)
print(sql)
dbm = DbManager()
result = dbm.fetch_one(sql)
print(result)
(cnt_tokyo, cnt_chiba, cnt_kanagawa, cnt_others) = result
text = """現在の会員数レポート {year}年{month}月
東京: {cnt_tokyo}
千葉：{cnt_chiba}
神奈川：{cnt_kanagawa}
その他：{cnt_others}
""".format(year=year,
month=month,
cnt_tokyo=cnt_tokyo,
cnt_chiba=cnt_chiba,
cnt_kanagawa=cnt_kanagawa,
cnt_others=cnt_others)
def main():
# See configuration.ini for details
fetch_regobs, load_excel_data, api_fetch_list, api_delete_list, api_initialize_list = load_configuration()
# Handle command line arguments
force_update = parse_command_line_arguments()
logging.info(
'Application started with force_update={}'.format(force_update))
# Create engine and db_inserter
try:
engine = create_db_connection()
except Exception as e:
logging.exception('Cannot connect to the database')
raise e
db_manager = DbManager(engine)
if load_excel_data:
logging.info('Adding excel data to database table excel_data..')
excel_data = process_excel_data()
ExcelData.metadata.create_all(engine)
db_manager.insert_dataframe('excel_data', excel_data, if_exists='replace')
logging.info('Fetching RegObs data..')
# Fetch regobs data from api
if fetch_regobs:
try:
api_data = Regobs().get_data()
except Exception as e:
logging.exception('Cannot fetch RegObs data')
raise e
to_csv(api_data, 'csv_files/regobs.csv')
# Load regobs data from csv file (can be useful for debugging or testing incremental update)
else:
api_data = read_csv('csv_files/regobs.csv')
# Incremental update. Only update added, updated or deleted records in database tables.
if not force_update:
# Specify that the dataframe should be appended to the existing data in the database tables
if_table_exists_in_database = 'append'
# Query current data in database
logging.info('Querying regobs table from database..')
try:
db_data = db_manager.query_all_data_from_table(
'regobs_data', 'reg_id')
except exc.NoSuchTableError as e:
logging.exception(
'The table regobs_data does not exist in the database. Run the application with --force-update command line parameter to initialize all tables and fetch all data.')
raise e
except Exception as e:
logging.exception('Cannot query RegObs data from database')
raise e
# Compare current database data with new api data
logging.info(
'Comparing dataframes to determine which rows are added or removed..')
# Rows to delete from all tables
deleted_rows = dataframe_difference(
db_data, api_data, ['reg_id', 'dt_change_time'])
# Rows to add to all tables
new_rows = dataframe_difference(
api_data, db_data, ['reg_id', 'dt_change_time'])
deleted_reg_ids = list(deleted_rows['reg_id'])
deleted_reg_ids = [int(x) for x in deleted_reg_ids]
logging.info('Records with the following reg_ids will be deleted from the database: {}'.format(
deleted_reg_ids))
if deleted_reg_ids:
# Delete removed rows from api's
try:
for data_class in api_delete_list:
logging.info(
'Deleting removed records for: {}'.format(data_class.__name__))
db_manager.delete_rows_with_reg_id(
deleted_reg_ids, data_class)
except Exception as e:
logging.exception(
'Cannot delete removed records from database table')
raise e
else:
logging.info(
'There are no deleted records to remove from the database')
if not new_rows.empty:
logging.info(
'Number of new records to add: {}'.format(len(new_rows)))
try:
avalanche_incident_list = AvalancheIncident.from_dataframe(
new_rows
)
except Exception as e:
logging.exception(
'Cannot create avalanche_incident_list from regobs data')
raise e
# Append new rows to regobs table
try:
insert_regobs_data_to_database(new_rows, db_manager, 'append')
except Exception as e:
logging.exception(
'Cannot append RegObs data to database table')
raise e
else:
avalanche_incident_list = []
# Initialize database and load all data
elif force_update:
# Specify that the dataframe should replace existing data in the database table
if_table_exists_in_database = 'replace'
try:
avalanche_incident_list = AvalancheIncident.from_dataframe(
api_data
)
except Exception as e:
logging.exception(
'Cannot create avalanche_incident_list from regobs data')
raise e
logging.info('Initializing database tables..')
try:
initialize_tables(api_initialize_list, engine)
except Exception as e:
logging.exception(
'Cannot initialize tables in database')
raise e
try:
insert_regobs_data_to_database(api_data, db_manager, 'replace')
except Exception as e:
logging.exception(
'Cannot add RegObs data to database table')
raise e
if not avalanche_incident_list:
logging.info('There is no new records to add to the database')
logging.info('The application terminated successfully')
return
try:
api_table_dict = get_table_dict_for_apis_in_list(
api_fetch_list, avalanche_incident_list)
except Exception as e:
logging.exception(
'Error fetching API data')
raise e
# Set new database connection
db_manager.engine = create_db_connection()
try:
insert_data_for_table_dict(
api_table_dict, db_manager, if_table_exists_in_database)
except Exception as e:
logging.exception(
'Cannot add API data to database table')
raise e
logging.info('The application terminated successfully')
class Import(QObject):
finished = pyqtSignal(bool, int)
error = pyqtSignal(str)
progress = pyqtSignal(float)
def __init__(self, layer):
super(Import, self).__init__()
self.layer = layer
# To let aborting thread from outside
self.stop = False
def insertPolygon(self, geometry):
# Convert geometry
wktGeom = geometry.exportToWkt()
# To force 2D geometry
if len(wktGeom.split('Z')) > 1:
wktGeom = wktGeom.split('Z')[0] + wktGeom.split('Z')[1]
wktGeom = wktGeom.replace(" 0,", ",")
wktGeom = wktGeom.replace(" 0)", ")")
geom = "GeomFromText('"
geom += wktGeom
geom += "', 2154)"
geomObj = {}
geomObj['the_geom'] = geom
db = DbManager(CarhabLayerRegistry.instance().getCurrentCarhabLayer().dbPath)
r = Recorder(db, 'polygon')
r.input(geomObj)
db.commit()
db.close()
def run(self ):
print 'run worker'
try:
print 'begin try'
if CarhabLayerRegistry.instance().getCurrentCarhabLayer():
# Connect to db.
self.db = DbManager(CarhabLayerRegistry.instance().getCurrentCarhabLayer().dbPath)
# To let import geos invalid geometries.
self.layer.setValid(True)
layerFeatCount = self.layer.featureCount()
# Initialisation of useful variables to calculate progression.
lastDecimal = 0
i = 0
for feature in self.layer.getFeatures():
if not self.stop:
featGeom = feature.geometry()
if featGeom.type() == 2: # Polygons case
if featGeom.isMultipart(): # Split multipolygons
for part in featGeom.asGeometryCollection():
if not part.isGeosValid(): # May be a problem...
print 'part not valid'
self.insertPolygon(part)
else:
self.insertPolygon(feature.geometry())
# Calculate and emit new progression value (each percent only).
newDecimal = int(100*i/layerFeatCount)
if lastDecimal != newDecimal:
self.progress.emit(newDecimal)
lastDecimal = newDecimal
i = i + 1
else: # Thread has been aborted
print 'abort'
# Cancel inserts already done
self.db.conn.rollback()
self.finished.emit(False, 2)
break
self.db.commit()
self.db.close()
self.finished.emit(True, 0)
else: # None current carhab layer (error code 1)
self.finished.emit(False, 1)
except:
print 'exception'
import traceback
print traceback.format_exc()
self.error.emit(traceback.format_exc())
self.finished.emit(False, 0)
def test_get_all_stations_by_param():
db = DbManager()
result = db.get_all_stations_by_param("CO")
assert len(result) > 1
assert type(result) == list
def track_whois(visitor_id, ip):
"""
:param visitor_id: id of the visitor table
:param ip: visitor ip_address
:return: nothing
"""
dbman = DbManager()
dbman.connect()
org_id = dbman.find_org_by_ip(ip)
if org_id is None:
ws = WhoisScraper()
whois_tuple = ws.scrape_whois(ip)
if whois_tuple is not None:
org_id = dbman.append_new_org(whois_tuple)
dbman.update_visitor(visitor_id, org_id)
else:
dbman.update_visitor(visitor_id, org_id)
dbman.close()
def test__get_data_by_stations_ids():
db = DbManager()
result = db.get_data_by_stations_ids([530, 531], "NO2")
assert len(result) >= 47
from main_window import UiMainWindow
from PyQt5 import QtCore, QtGui, QtWidgets
from data_manager import DataManager
from db_manager import DbManager, DbManagerError
import logging_setup
import sys
if __name__ == "__main__":
"""
Runs main application.
"""
app = QtWidgets.QApplication(sys.argv)
main_window = QtWidgets.QMainWindow()
main_logger = logging_setup.get_logger("main_program")
try:
data_base = DbManager()
data_manager = DataManager(data_base)
ui = UiMainWindow(data_manager, main_logger)
ui.setup_ui(main_window)
main_window.show()
except DbManagerError as e:
main_logger.exception(e)
sys.exit(app.exec_())
# coding=utf8
from db_manager import DbManager
databases = DbManager()
# ----------------------新添加的数据库在此处初始化--------------------------------
# 若有新添加的数据库，先在db_config中添加配置信息后，在下面初始化
# 如：cps_conn = databases.get_one_coon('cps')
# 之后哪里需要用，直接 'from db_manage.db_init import XXXX_conn' 即可
# 使用方法：XXXX_conn.query(<your_sql>)
cps_conn = databases.get_one_coon('cps')
# --------------------需要engine的时候才需要在下面添加------------------------------
# 若是需要直接操作engine，可以在下面进行初始化
# 如：cps_engine = databases.get_one_engine('cps')
# 功能：pd.read_sql(<your_sql>, XXXX_engine)
cps_engine = databases.get_one_engine('cps')
if __name__ == '__main__':
sql = 'SELECT * from biz.user_mmv_status LIMIT 2'
results = cps_conn.query(sql) # 返回包含元组的list
def setup_db(self):
dbManager = DbManager()
dbManager.setup_db()
def add_snippets(self, ranked_docs, query):
dbManager = DbManager()
builder = StructureBuilder()
docs_with_snippets = []
tf_idf_q_terms = {}
q_terms = builder.get_stemmed_tems(query)
for q_term in q_terms:
# number of documents in DC in which q_term appears at least once.
n_docs_q_term = len(self.q_terms_freqs[q_term]
) if q_term in self.q_terms_freqs else 0
if n_docs_q_term != 0:
freq_d = len([q for q in q_terms if q == q_term])
max_q_freq = self.get_local_max_freq(q_terms)
tf_idf_q_terms[q_term] = self.calc_tf_idf(
freq_d, max_q_freq, self.docs_count, n_docs_q_term)
else:
tf_idf_q_terms[q_term] = 0
for ranked_doc in ranked_docs:
doc_id = ranked_doc[0]
docs_relevant_scores = {}
doc = dbManager.get_document(doc_id)
if doc == None: continue
sentences = self.get_doc_sentences(doc)
title = sentences.pop(0)['content']
for sentence in sentences:
senetence_content = sentence['content']
# if the sentence has less than 2 character then it is probabily not an actual sentence.
if len(senetence_content) <= 2: continue
sentence_id = sentence['id']
tf_idf_sum = 0
denom_di_sum = 0
denom_qi_sum = 0
index_sentence = builder.get_stemmed_terms_frequencies_from_doc(
sentence)
for q_term in q_terms:
# check the not stemmed words
if q_term in index_sentence.Terms:
q_sentence_freq = index_sentence.get_term_freq(q_term)
max_freq = index_sentence.get_max_freq()
# if the query term doesn't have frequency on the sentence and there is no max freq. then disregard this q_term
if (q_sentence_freq == 0 and max_freq == 0):
continue
tf_idf_doc = self.calc_tf_idf(
q_sentence_freq, max_freq, self.docs_count,
len(self.q_terms_freqs[q_term]))
tf_idf_q = tf_idf_q_terms[q_term]
# The two sentences in d that have the highest cosine similarity with respect to q; with TF-IDF as the term weighting scheme.
tf_idf_sum += tf_idf_doc * tf_idf_q
denom_di_sum += tf_idf_doc**2
denom_qi_sum += tf_idf_q**2
denom = math.sqrt(denom_di_sum) * math.sqrt(denom_qi_sum)
score = tf_idf_sum / denom if denom != 0 else 0
docs_relevant_scores[sentence_id] = round(score, 3)
sorted_docs_total_freqs = sorted(docs_relevant_scores.items(),
key=operator.itemgetter(1),
reverse=True)
top_sentences = sorted_docs_total_freqs[0:2]
top_snippets = [
s['content'] for s in sentences
if s['id'] == top_sentences[0][0]
or s['id'] == top_sentences[1][0]
]
docs_with_snippets.append({
"docId": doc_id,
"score": ranked_doc[1],
"title": title,
"snippets": top_snippets
})
return docs_with_snippets
def get_current_user_id(self):
dbManager = DbManager()
user_id = dbManager.insert_and_get_min_user_id()
return user_id
def save_to_database_task(property_data):
db_manager = DbManager()
db_manager.add_new_property(property_data)
API_KEY = '<bot_api_key>'
URL = 'https://api.telegram.org/bot{BOT_API_KEY}/sendMessage?chat_id={CHANNEL_NAME}&text={MESSAGE_TEXT}'
CHANNEL_NAME = '<channel_name>'
MSG_TEMPLATE = '''На форуме новый вопрос: "{title}"
Опубликуйте ответ тут: {link}'''
if __name__ == '__main__':
print('Start')
posts = get_content()
if len(posts) == 0:
exit(0)
db = DbManager('test.db')
db.connect()
for post in posts:
title = post['title']
link = post['link']
result = db.find_post(title)
if len(result) == 0:
db.add_new_post(title, link)
msg = MSG_TEMPLATE.format(title=title, link=link)
url = URL.format(
BOT_API_KEY=API_KEY,
CHANNEL_NAME=CHANNEL_NAME,
MESSAGE_TEXT=msg
)
from flask import Blueprint
from flask import Flask, abort, request, jsonify
from flask import jsonify
from flask import redirect
import json
import logging
from db_manager import DbManager
# print("URL Service")
logging.basicConfig(level=logging.DEBUG)
app = Flask(__name__)
dbmgr = DbManager()
dbmgr.connect_db()
# url_service = Blueprint('url_api', __name__) # flask blueprints: https://flask.palletsprojects.com/en/1.1.x/blueprints/
#
# geturlshrtn: returns the shortened url for a given url (internal)
#
@app.route('/geturlshrtn/<url_shrt_code>', methods=['GET'])
def getUrlShrtn(url_shrt_code):
print("\n\n/geturlshrtn/ for: " + str(url_shrt_code))
url_shrtn = dbmgr.get_url_shrtn(url_shrt_code)
return jsonify(url_shrtn)
class KanzumeFrame(wx.Frame):
"""
メインパネル
"""
def cap_btn_click_listener(self, event):
startWorker(self.done_capture, self.capture)
def capture(self):
logging.info('パケットキャプチャ開始')
self.panel_capture.cap_btn.Disable()
self.set_progress_value(0)
self.panel_capture.text.Hide()
self.panel_capture.progress.Show()
self.SetStatusText("capture......")
return self.pcap.startCapture()
def done_capture(self, result):
self.army_mng = result.get()
self.panel_capture.text.Show()
self.panel_capture.progress.Hide()
if self.army_mng is not None:
self.panel_result.updateResult(self.army_mng)
self.SetStatusText("capture success!")
else:
self.SetStatusText("capture failed...")
self.panel_capture.cap_btn.Enable()
logging.info('パケットキャプチャ終了')
def select_networkif(self, event):
self.pcap.nif_mng.set_selected_devId(event.GetId())
self.dbm.update_nwif(event.GetId())
def set_text(self, text):
self.panel_capture.set_text(text)
def set_progress_value(self, value):
self.panel_capture.set_progress_value(value)
def __init__(self):
wx.Frame.__init__(self, None, wx.ID_ANY, u'艦ZUME', size=(450, 500))
self.isCaputre = False
# アイコン
icon = wx.Icon('./data/img/kanzume.ico', wx.BITMAP_TYPE_ICO)
self.SetIcon(icon)
# パケットキャプチャ準備
self.pcap = PacketCapture()
self.pcap.set_text(self.set_text)
self.pcap.set_progress_value(self.set_progress_value)
self.army_mng = None
# データベース準備
self.dbm = DbManager()
# ステータスバーの初期化
self.CreateStatusBar()
self.SetStatusText(u'Let\'s capture!')
self.GetStatusBar().SetBackgroundColour(None)
# メニューバーの初期化
selected_nwif_index = self.dbm.get_selected_nwif_index()
self.pcap.nif_mng.set_selected_devId(selected_nwif_index)
menuBar = CaptureMenuBar(self.pcap.nif_mng.get_networkif_listf())
menuBar.set_selected_nwif(selected_nwif_index)
self.SetMenuBar(menuBar)
self.Bind(wx.EVT_MENU, self.select_networkif)
# パネル
panel_base = wx.Panel(self, wx.ID_ANY)
self.panel_capture = CapturePanel(panel_base)
self.panel_result = ResultPanel(panel_base, self.dbm)
# クリックリスナ登録
self.panel_capture.set_on_click_listener(self.cap_btn_click_listener)
layout = wx.BoxSizer(wx.VERTICAL)
layout.Add(self.panel_capture, flag=wx.EXPAND | wx.ALL, border=10)
layout.Add(self.panel_result, proportion=1, flag=wx.EXPAND)
panel_base.SetSizer(layout)
layout.Fit(panel_base)
