def test_connection_to_db(self):
db_con = DbConnection()
# if connect() function call raises exception unit test fails
cursor = db_con.connect()
# is right instance and not null
self.assertIsInstance(cursor, pymssql.Cursor)
def __init__(self):
super().__init__()
self.setupUi(self)
# Изображение
self.pixmap = QPixmap('./uploads/tutorial3.jpg')
# Если картинки нет, то QPixmap будет пустым,
# а исключения не будет
# self.image = QLabel(self)
self.image.setPixmap(self.pixmap)
self.gridLayout_2.addWidget(self.image, 4, 0, 1, 1)
self.good_education = QLabel(self.info)
self.good_education.setMaximumSize(QSize(16777215, 50))
self.good_education.setStyleSheet(
"font-size: 25px; font-weight: 900;\n")
self.good_education.setAlignment(Qt.AlignCenter)
self.good_education.setObjectName("good_education")
self.gridLayout_2.addWidget(self.good_education, 5, 0, 1, 1)
self.good_education.setText("Удачного обучения!")
self.setWindowTitle("Ansheapable")
self.load_mp3('uploads/super_angry.mp3')
# making constants
make_constants(self)
self.json = TableWithJson('./parsing/user_info.json')
self.json.read_file()
self.create_piechart()
self.reset_table()
# widgets which are common for the choose panel
self.choose_area = [
self.choose_hard_russian, self.choose_long_english,
self.choose_basic_english, self.choose_basic_russian,
self.choose_title, self.choose_long_russian, self.choose_numbers
]
# widgets which are common for the writing field
self.tr_area = [
self.tr_back_label, self.tr_english_layout, self.tr_info_panel,
self.tr_russian_layout, self.tr_back_label, self.tr_text_field,
self.tr_writing_area, self.tr_title, self.tr_first_letter,
self.tr_come_back
]
self.rez_area = [
self.rez_title, self.rez_mistakes, self.rez_main_text,
self.rez_new_game, self.rez_empty_filler
]
self.tr_come_back.clicked.connect(self.come_back)
self.rez_new_game.clicked.connect(self.to_main_window)
self.hide_elements(self.tr_area)
self.hide_elements(self.rez_area)
# add click listener to our buttons
for item in self.choose_area:
obj_type = type(item)
if obj_type == QPushButton:
item.clicked.connect(self.open_writin_panel)
self.db_eventor = DbConnection('./parsing/db.sqlite')
def test_config_parser(self):
db_con = DbConnection()
db_con.read_config()
config = db_con.get_config()
#config is of the right type
self.assertIsInstance(config.get('AzureDB', 'server_URI'), str)
self.assertIsInstance(config.get('AzureDB', 'username'), str)
self.assertIsInstance(config.get('AzureDB', 'password'), str)
self.assertIsInstance(config.get('AzureDB', 'database_1'), str)
# config is not empty
self.assertTrue(len(config.get('AzureDB', 'server_URI')) > 0)
self.assertTrue(len(config.get('AzureDB', 'username')) > 0)
self.assertTrue(len(config.get('AzureDB', 'password')) > 0)
self.assertTrue(len(config.get('AzureDB', 'database_1')) > 0)
def main():
logger = Logger('coin_application', 'coins.log',
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
db = DbConnection('localhost', 6379, logger.logger)
BitcoinCrawler(60, logger.logger,
'https://api.coinmarketcap.com/v1/ticker/bitcoin/', 10, db)
def __init__(self, csvFilePath, clearTableFunction, insertRowFunction):
super(FillTempTable, self).__init__()
self.csvFilePath = csvFilePath
self.clearTableFunction = clearTableFunction
self.insertRowFunction = insertRowFunction
conn = DbConnection().connection
self.dbCore = Levy_Db(conn, None)
self.lock = threading.Lock()
def sendStatusEmail(venueUid, eventUid, manual=False):
"""
Sends an email for the given event to everyone on the email list of the given
venue.
1. Builds the email html body
2. gathers the email addresses
3. sends emails
Parameters
__________
venueUid : int the venue_uid to send status emails for
evenutUid : int the event_uid to send status emails for
Reutrns
_______
out : boolean True if successful
"""
from db_connection import DbConnection
conn = DbConnection().connection
statusRows = getStatusRows(conn, venueUid)
body = '<font face="verdana">'
table = getTable(statusRows)
body += getIntro(conn, venueUid, eventUid)
summary, totalTablets, badTablets = getSummary(statusRows)
body += summary
body += table
if (manual):
body += "<h1><b>*</b></h1>"
subject = getSubject(totalTablets, badTablets)
emailList = getEmailList(conn, venueUid)
for email in emailList:
gmail.sendGmail("*****@*****.**", "fkTUfbmv2YVy",
"*****@*****.**", email[0], subject,
body, 'Please, enable html to view this report')
return True
def __init__(self, db=None, style="numbered", sortRefs=False):
self.citekeys = []
self.sortRefs = sortRefs
self.style = style
self.references = bibArray(style)
self.uniqueCitekeys = []
self.missingRefs = []
self.inTextRefs = {}
self.opening = "["
self.closing = "]"
if style == "parenthetical":
self.opening = "("
self.closing = ")"
if db:
self.db = db
else:
self.db = DbConnection()
self.db.guiPrompt()
class MyWidget(QMainWindow, Ui_MainWindow):
def __init__(self):
super().__init__()
self.setupUi(self)
# Изображение
self.pixmap = QPixmap('./uploads/tutorial3.jpg')
# Если картинки нет, то QPixmap будет пустым,
# а исключения не будет
# self.image = QLabel(self)
self.image.setPixmap(self.pixmap)
self.gridLayout_2.addWidget(self.image, 4, 0, 1, 1)
self.good_education = QLabel(self.info)
self.good_education.setMaximumSize(QSize(16777215, 50))
self.good_education.setStyleSheet(
"font-size: 25px; font-weight: 900;\n")
self.good_education.setAlignment(Qt.AlignCenter)
self.good_education.setObjectName("good_education")
self.gridLayout_2.addWidget(self.good_education, 5, 0, 1, 1)
self.good_education.setText("Удачного обучения!")
self.setWindowTitle("Ansheapable")
self.load_mp3('uploads/super_angry.mp3')
# making constants
make_constants(self)
self.json = TableWithJson('./parsing/user_info.json')
self.json.read_file()
self.create_piechart()
self.reset_table()
# widgets which are common for the choose panel
self.choose_area = [
self.choose_hard_russian, self.choose_long_english,
self.choose_basic_english, self.choose_basic_russian,
self.choose_title, self.choose_long_russian, self.choose_numbers
]
# widgets which are common for the writing field
self.tr_area = [
self.tr_back_label, self.tr_english_layout, self.tr_info_panel,
self.tr_russian_layout, self.tr_back_label, self.tr_text_field,
self.tr_writing_area, self.tr_title, self.tr_first_letter,
self.tr_come_back
]
self.rez_area = [
self.rez_title, self.rez_mistakes, self.rez_main_text,
self.rez_new_game, self.rez_empty_filler
]
self.tr_come_back.clicked.connect(self.come_back)
self.rez_new_game.clicked.connect(self.to_main_window)
self.hide_elements(self.tr_area)
self.hide_elements(self.rez_area)
# add click listener to our buttons
for item in self.choose_area:
obj_type = type(item)
if obj_type == QPushButton:
item.clicked.connect(self.open_writin_panel)
self.db_eventor = DbConnection('./parsing/db.sqlite')
# self.db_eventor.random_text('hard russian')
def hide_elements(self, elements):
''' hide group of elements '''
for element in elements:
element.hide()
def show_elements(self, elements):
''' show group of elements '''
for element in elements:
element.show()
def open_writin_panel(self):
self.hide_elements(self.choose_area)
self.hide_elements(self.rez_area)
self.show_elements(self.tr_area)
name = self.sender().objectName()
row_name = ' '.join(name.split('_')[1:])
if 'russian' in name:
self.tr_english_layout.hide()
self.cur_language = self.RUSSIAN
else:
self.cur_language = self.ENGLISH
self.tr_russian_layout.hide()
text = self.db_eventor.random_text(row_name)
self.tr_title.setText(self.NAMES[row_name])
self.game_trainer = PendingGame(self, text, row_name)
self.tr_text_field.setText(text)
def to_main_window(self):
self.hide_elements(self.tr_area)
self.hide_elements(self.rez_area)
self.show_elements(self.choose_area)
def come_back(self):
self.hide_elements(self.tr_area)
self.hide_elements(self.rez_area)
self.show_elements(self.choose_area)
self.game_trainer.ending_without_result()
def create_piechart(self):
''' creating piechart '''
# adding objects into the series
meta = []
data = self.json.get_data()
for key in self.NAMES.keys():
meta.append([self.NAMES[key], data[key]['counter']])
self.field = self.horizontalLayout_12
self.series = QPieSeries()
# adding data in series
for name, value in meta:
self.series.append(name, value)
# creating qchart
self.chart = QChart()
self.chart.addSeries(self.series)
self.chart.setAnimationOptions(QChart.SeriesAnimations)
self.chart.setTitle("Статистика по количеству")
# and adding animations and view
self.chart_view = QChartView(self.chart)
self.chart_view.setRenderHint(QPainter.Antialiasing)
# at all we have a widget which are going to be pushed in your field
self.field.addWidget(self.chart_view)
def reset_table(self):
''' update a info table '''
dat = self.json.get_data()
self.basic_russian_max_2.setText(str(
dat['basic russian']['max_speed']))
self.basic_russian_avg_2.setText(str(
dat['basic russian']['avg_value']))
self.basic_russian_counter_2.setText(
str(dat['basic russian']['counter']))
self.basic_english_max_2.setText(str(
dat['basic english']['max_speed']))
self.basic_english_avg_2.setText(str(
dat['basic english']['avg_value']))
self.basic_english_counter_2.setText(
str(dat['basic english']['counter']))
self.long_russian_max_2.setText(str(dat['long russian']['max_speed']))
self.long_russian_avg_2.setText(str(dat['long russian']['avg_value']))
self.long_russian_counter_2.setText(str(
dat['long russian']['counter']))
self.long_english_max_2.setText(str(dat['long english']['max_speed']))
self.long_english_avg_2.setText(str(dat['long english']['avg_value']))
self.long_english_counter_2.setText(str(
dat['long english']['counter']))
self.hard_russian_max_2.setText(str(dat['hard russian']['max_speed']))
self.hard_russian_avg_2.setText(str(dat['hard russian']['avg_value']))
self.hard_russian_counter_2.setText(str(
dat['hard russian']['counter']))
self.numbers_max_2.setText(str(dat['numbers']['max_speed']))
self.numbers_avg_2.setText(str(dat['numbers']['avg_value']))
self.numbers_counter_2.setText(str(dat['numbers']['counter']))
def load_mp3(self, filename):
''' loading shep '''
fn = inspect.getframeinfo(inspect.currentframe()).filename
path = os.path.dirname(os.path.abspath(fn))
cur_dir = path
# print(sys.argv)
media = QUrl.fromLocalFile(cur_dir + '/' + filename)
content = QtMultimedia.QMediaContent(media)
self.player = QtMultimedia.QMediaPlayer()
self.player.setMedia(content)
class documentFormatter:
""" base class for formatting bibliographies """
def __init__(self, db=None, style="numbered", sortRefs=False):
self.citekeys = []
self.sortRefs = sortRefs
self.style = style
self.references = bibArray(style)
self.uniqueCitekeys = []
self.missingRefs = []
self.inTextRefs = {}
self.opening = "["
self.closing = "]"
if style == "parenthetical":
self.opening = "("
self.closing = ")"
if db:
self.db = db
else:
self.db = DbConnection()
self.db.guiPrompt()
def getUniqueCitekeys(self):
""" return just the unique citekeys """
if len(self.citekeys) > 1:
u = {}
for key in self.citekeys: u[key] = key
#self.uniqueCitekeys = u.keys()
for k in u.keys():
print "Key: %s" % k
# remove duplicates here
if (self.citekeys.count(k) > 1): # if ref occurs more than once
indices = []
print "Before: %i" % len(self.citekeys)
for i in range(0, len(self.citekeys)):
if self.citekeys[i] == k: # find where this ref occurs
indices.append(i)
counter = 0
for i in range(1, len(indices)): # remove all but first
print "indices: %s" % indices
self.citekeys.pop(indices[i] - counter)
counter += 1 # counter needed since size of array is changing
print "After: %i" % len(self.citekeys)
#else:
# self.uniqueCitekeys = self.citekeys
def getReferences(self):
""" attempts to retrieve all the references from the DB """
for key in self.citekeys:
try:
b = bibItem(db=self.db, key=key)
except invalidDBKeyException:
self.missingRefs.append(key)
else:
if key:
print "Inserting key %s" % key
self.references.append(b)
print "CITATION: %s" % self.references.inTextCitation(b["citekey"])
if self.sortRefs == True:
self.references.sort(refSorter)
self.references.changeInTextCounts()
print "Citations: %s" % self.references.getInTextCitations()
self.formatReferences()
for i in self.references: print i
print "IN TEXT REFS %s" % self.inTextRefs
def printMissingRefs(self):
""" prints a list of missing references to the screen """
print "Missing keys: "
for r in self.missingRefs:
print " %s" % r
def formatReferences(self):
""" creates dict of formatted in text references """
if self.style == "parenthetical":
self.references.modifyDuplicates()
for k in self.inTextRefs.keys():
formatted = ""
formattedRefs = []
refs = self.refsInCitation(k)
for r in refs:
try:
c = self.references.inTextCitation(r)
except KeyError:
formattedRefs.append("MISSING: %s" % r)
else:
print "THIS WAS C: %s" % c
formattedRefs.append("%s" % self.references.inTextCitation(r))
if len(refs) > 1:
print "FR: %s" % formattedRefs
s = ", ".join(formattedRefs)
formatted = "%s%s%s" % (self.opening, s, self.closing)
else:
formatted = "%s%s%s" % (self.opening, formattedRefs[0], self.closing)
self.inTextRefs[k] = formatted
def removeColons(self, reference):
"""quick hack to replace Doe:2001 with Doe2001"""
p = re.compile(r':')
return p.sub('', reference)
def refsInCitation(self, citation):
""" get the citekeys inside an unformatted inText citation """
refs = []
spaceRemover = re.compile(r'^\s+')
squiqqlyRemover = re.compile(r'[\{\}]')
citation = squiqqlyRemover.sub('', citation)
print "List: %s" % citation
for reference in citation.split(","):
reference = spaceRemover.sub('', reference)
reference = self.removeColons(reference)
refs.append(reference)
return(refs)
def openFile(self, fileName):
""" an empty method that needs to be subclassed """
pass
def getCitekeys(self):
""" an empty method that needs to be subclassed """
pass
def replaceCitekeys(self):
""" an empty method that needs to be subclassed """
pass
def formatBibliography(self):
""" an empty method that needs to be subclassed """
pass
def writeFile(self, output):
""" an empty method that needs to be subclassed """
pass
def formatDocument(self, input=None, output=None):
""" formats a document - replacing references and creating a ref list"""
if input:
self.filename = input
# open the file - this method needs to be subclassed.
self.openFile(self.filename)
# extract the citekeys from the document (handled by the subclass)
self.getCitekeys()
# get just the unique citekeys
self.getUniqueCitekeys()
print "Unique: %s" % self.uniqueCitekeys
# get the references from the database
self.getReferences()
# replace the citekeys inside the document (handled by subclass)
if self.style != "unformatted":
self.replaceCitekeys()
# format the bibliography (handled by subclass)
self.formatBibliography()
# write to file (handled by subclass)
if output:
self.writeFile(output)
self.printMissingRefs()
#Avnet Data Export
import sys
from db_connection import DbConnection
import HipChat
import csv
import traceback
import paramiko
try:
EXPORT_FILE_PATH = "/data/Avnet_Export/"
conn = DbConnection().connection
venueUid = sys.argv[1]
cursor = conn.cursor()
cursor.execute("SELECT DATE(CONVERT_TZ(NOW(), 'UTC', (SELECT local_timezone_long FROM setup.venues WHERE id = %s)))", (venueUid))
date = cursor.fetchone()[0]
formattedDate = date.strftime("%Y%m%d")
##########
# ORDERS #
##########
orderFilename = "order_" + formattedDate + ".csv"
orderCursor = conn.cursor()
orderCursor.execute('''SELECT
orders.event_uid,
events_x_venues.event_name,
orders.id AS 'order_uid',
orders.unit_uid,
units.name AS 'suite',
from db_connection import DbConnection
import TabletStatusMailer
import requests
import traceback
from config import CheckMateConfig
try:
conn = DbConnection().connection
cursor = conn.cursor()
# get all of the venues that have tablets
cursor.execute("SELECT DISTINCT venue_uid FROM tablets.venues_x_tablet")
for venueRow in cursor.fetchall():
venueUid = venueRow[0]
#for each venue find the events between where the current time is within the 'seconds_before_event' interval.
#I.E. are we within 12 hours of the event
cursor.execute(
"SELECT events.id AS event_id FROM setup.events \
JOIN setup.tablet_report_controls ON events.venue_uid = tablet_report_controls.venue_uid \
WHERE events.venue_uid = %s \
AND event_date BETWEEN NOW() AND DATE_ADD(NOW(), INTERVAL seconds_before_event SECOND) \
AND tablet_alert_sent = 0 \
ORDER BY event_date ASC", (str(venueUid)))
#for each of these events send a tablet status alert email
for eventRow in cursor.fetchall():
eventUid = eventRow[0]
if TabletStatusMailer.sendStatusEmail(venueUid, eventUid) == True:
from db_connection import DbConnection
import os.path
import hashlib
MENU_ITEM_IMAGE_PATH = "/data/media/202/images/menu_items/"
db = DbConnection().connection
menuItemCursor = db.cursor()
menuItemCursor.execute("SELECT * FROM menus.menu_items WHERE show_image =1");
menuItems = menuItemCursor.fetchall()
for menuItem in menuItems:
image = MENU_ITEM_IMAGE_PATH + str(menuItem[0]) + ".png";
if os.path.isfile(image):
fileHash = hashlib.md5(open(image).read()).hexdigest()
cursor = db.cursor()
cursor.execute("INSERT INTO media.images(pointer_uid, image_type, image_hash, created_at)\
VALUES ({0}, 'menu_items', '{1}', NOW())".format(menuItem[0], fileHash))
db.commit()
else:
print "Can't find it: " + image
import sqlalchemy as db
from sqlalchemy import Sequence
from db_connection import DbConnection
dbConnection = DbConnection()
class Kronikarz(dbConnection.Model):
__tablename__ = 'kronikarz'
nr_indeksu = db.Column('nr_indeksu', db.String(6), primary_key=True)
imie = db.Column('imie', db.String(15))
nazwisko = db.Column('nazwisko', db.String(20))
email = db.Column('email', db.String(35))
haslo = db.Column('haslo', db.String(15))
uczelnia = db.Column('uczelnia', db.String(40))
wydzial = db.Column('wydzial', db.String(40))
kierunek = db.Column('kierunek', db.String(40))
rodzaj = db.Column('rodzaj', db.String(40))
def __init__(self, nr_indeksu, imie, nazwisko, email, haslo, uczlenia,
wydzial, kierunek, rodzaj):
self.nr_indeksu = nr_indeksu
self.imie = imie
self.nazwisko = nazwisko
self.email = email
self.haslo = haslo
self.uczlenia = uczlenia
self.wydzial = wydzial
self.kierunek = kierunek
self.rodzaj = rodzaj
class GenerateDataset:
__dataset_folder = "db_generated_datasets"
__dataset_name = "ner_dataset.txt"
def __concatenate_sql_queries_and_select(self, doc_nr, query_nr):
# nr 2 is ordered randomly
self.sql_query_list = [
(
"select TOP " + str(doc_nr) +
" cn_fname, cn_lname, cn_resume " # is cn_res not random
"from tblCandidate "
"where cn_fname IS NOT NULL "
"AND DATALENGTH(cn_fname)>2 "
"AND cn_lname IS NOT NULL "
"AND DATALENGTH(cn_lname)>2 "
"AND cn_resume LIKE '%[a-z0-9]%' "
"AND DATALENGTH(cn_resume)>14000 "
"AND cn_res=0;"),
(
"SELECT TOP " + str(doc_nr) +
" cn_fname, cn_lname, cn_resume " # is random
"FROM tblCandidate "
"WHERE cn_fname IS NOT NULL "
"AND DATALENGTH(cn_fname)>2 "
"AND cn_lname IS NOT NULL "
"AND DATALENGTH(cn_lname)>2 "
"AND cn_resume LIKE '%[a-z0-9]%' "
"AND DATALENGTH(cn_resume)>17000 "
"AND cn_res=0 ORDER BY NEWID();"),
(
"select TOP " + str(doc_nr) +
" cn_fname, cn_lname, cn_resume, cn_present_position " # not cn_res & not random
"FROM tblCandidate "
"WHERE cn_fname IS NOT NULL "
"AND DATALENGTH(cn_fname)>2 "
"AND cn_lname IS NOT NULL "
"AND DATALENGTH(cn_lname)>2 "
"AND cn_resume LIKE '%[a-z0-9]%' "
"AND DATALENGTH(cn_resume)>10000 "
"AND cn_present_position IS NOT NULL "
"AND cn_present_position LIKE '%[a-z0-9]%' "
"AND cn_res=0 ORDER BY NEWID();")
]
return self.sql_query_list[query_nr]
def __get_db_cursor(self):
self.__db_cursor = DbConnection().connect()
def __set_sql_query(self, query):
self.__query_to_execute = query
def __execute_query(self):
self.__db_cursor.execute(self.__query_to_execute)
def pull_db_records(self, query_nr, doc_nr):
self.__get_db_cursor()
self.__set_sql_query(
self.__concatenate_sql_queries_and_select(doc_nr, query_nr))
self.__execute_query()
self.raw_db_table = []
print("Pulling " + str(doc_nr) + " records")
self.raw_db_table = self.__db_cursor.fetchall()
print("Pulled " + str(len(self.raw_db_table)) + " records")
def tokenize_text(self):
self.tokenized_docs_by_lines = []
for doc in self.raw_db_table:
#rtokenizer = RegexpTokenizer(r'\w+')
#tokens = rtokenizer.tokenize(doc[2])
doc_lines = doc[2].splitlines()
tokenized_doc_lines = []
rtokenizer = RegexpTokenizer(r'\w+')
for line in doc_lines:
line = rtokenizer.tokenize(line)
#line = word_tokenize(line)
if line != []:
tokenized_doc_lines.append(line)
#optional remove stop words
#filtered_words = [w for w in tokens if not w in stopwords.words('english')]
# append doc to global list
self.tokenized_docs_by_lines.append(tokenized_doc_lines)
print("Split lines and tokenized text")
def pos_tag_tokens(self):
self.pos_doc_tokens = []
for doc in self.tokenized_docs_by_lines:
tagged_doc_lines = []
for line in doc:
tagged_line = nltk.pos_tag(line)
tagged_doc_lines.append(tagged_line)
self.pos_doc_tokens.append(tagged_doc_lines)
print("POS tagged tokens")
def ner_tag_tokens(self):
self.name_tag_tokens()
self.current_position_tag_tokens()
def name_tag_tokens(self):
self.ner_doc_tokens = []
for doc_idx, doc in enumerate(self.tokenized_docs_by_lines):
tagged_doc = []
for line in doc:
single_doc_line = []
for token_idx, token in enumerate(line):
rtokenizer = RegexpTokenizer(r'\w+')
matching_names = rtokenizer.tokenize(
(str(self.raw_db_table[doc_idx][0]) + " " +
str(self.raw_db_table[doc_idx][1])).lower())
if any(token.lower() == s for s in matching_names):
#replace word with tagged tuple
single_doc_line.append((token, "PERS"))
else:
single_doc_line.append((token, "O"))
tagged_doc.append(single_doc_line)
self.ner_doc_tokens.append(tagged_doc)
print("NER name tagged tokens")
def current_position_tag_tokens(self):
for doc_idx, doc in enumerate(self.tokenized_docs_by_lines):
for line_idx, line in enumerate(doc):
matching_curpos_window = word_tokenize(
(str(self.raw_db_table[doc_idx][3])).lower())
last_index_of_line = len(line) - 1
last_index_of_window = len(matching_curpos_window) - 1
for current_tkn_idx in range(
0, (last_index_of_line - last_index_of_window) + 1):
current_window = line[current_tkn_idx:current_tkn_idx +
len(matching_curpos_window)]
current_window = [x.lower() for x in current_window]
if current_window == matching_curpos_window:
# change ner tag to current position
for found_idx in range(
current_tkn_idx,
(current_tkn_idx + last_index_of_window) + 1):
self.ner_doc_tokens[doc_idx][line_idx][
found_idx] = (line[found_idx],
"EMPHIST-CURPOS")
print("NER current position tagged tokens")
def nonlocal_ner_tag_tokens(self):
home = expanduser("~")
os.environ['CLASSPATH'] = home + '/stanford-ner-2015-12-09'
os.environ[
'STANFORD_MODELS'] = home + '/stanford-ner-2015-12-09/classifiers'
st = StanfordNERTagger("english.all.3class.distsim.crf.ser.gz",
java_options='-mx4000m')
stanford_dir = st._stanford_jar[0].rpartition('/')[0]
stanford_jars = find_jars_within_path(stanford_dir)
st._stanford_jar = ':'.join(stanford_jars)
# do not tokenise text
nltk.internals.config_java(
options=
'-tokenizerFactory edu.stanford.nlp.process.WhitespaceTokenizer -tokenizerOptions "tokenizeNLs=true"'
)
self.nonlocal_ner_doc_tokens = []
temp_nonlocal_bulk_process = []
length_of_docs = [len(doc) for doc in self.tokenized_docs_by_lines]
for doc_idx, doc in enumerate(self.tokenized_docs_by_lines):
for line_idx, line in enumerate(doc):
temp_nonlocal_bulk_process.append(line)
temp_nonlocal_bulk_process = st.tag_sents(temp_nonlocal_bulk_process)
current_idx = 0
for doc_len_idx, doc_len in enumerate(length_of_docs):
self.nonlocal_ner_doc_tokens.append(
temp_nonlocal_bulk_process[current_idx:current_idx + doc_len])
current_idx += doc_len
print("NER nonlocal tagged tokens")
def save_tagged_tokens(self):
directory = self.__dataset_folder
files = glob.glob('*.txt')
for filename in files:
os.unlink(filename)
path = self.__dataset_folder + "/"
for doc_idx, doc in enumerate(self.ner_doc_tokens):
doc_file = open(path + str(doc_idx) + '.txt',
'w',
encoding='utf-8')
for line_idx, line in enumerate(doc):
for token_idx, token in enumerate(line):
# token, pos_tag, ner_tag
#print("length ner doc: " + str(len(doc)) + " length pos doc: " + str(len(self.pos_doc_tokens[doc_idx])) + " length nonlocal doc: " + str(len(self.nonlocal_ner_doc_tokens[doc_idx])))
doc_file.write("{}\t{}\t{}\t{}\n".format(
token[0],
self.pos_doc_tokens[doc_idx][line_idx][token_idx][1],
self.nonlocal_ner_doc_tokens[doc_idx][line_idx]
[token_idx][1], token[1]))
doc_file.write("\n")
doc_file.close()
print("Saved tagged tokens to: " + path)
def read_tagged_tokens(self):
dataset_docs = []
for filename in os.listdir(self.__dataset_folder):
current_file_path = self.__dataset_folder + "/" + filename
if current_file_path.endswith(".txt"):
with io.open(current_file_path, 'r',
encoding='utf-8') as tsvin:
single_doc = []
single_line = []
tsvin = csv.reader(tsvin, delimiter='\t')
for row in tsvin:
if not row:
single_doc.append(single_line)
single_line = []
else:
single_line.append(
(row[0], row[1], row[2], row[3]))
dataset_docs.append(single_doc)
return dataset_docs
def __get_db_cursor(self):
self.__db_cursor = DbConnection().connect()
def sendEmail(allAppliedActions):
print "Preparing Integration Email"
#convert the uuid keyed dict of actions into a list of actions (we don't care about the uuids anymore
# tempActions = []
# for action in allAppliedActions:
# tempActions.append(allAppliedActions[action])
#
# allAppliedActions = tempActions
appliedActions = allAppliedActions
conn = DbConnection().connection
emailerDb = EmailerDb(conn)
venueUids = emailerDb.getVenueUids()
print "VENUE UIDS: " + str(venueUids)
for venueUid in venueUids:
venueUid = venueUid[
0] #DON'T FORGET python will return single values from mysql as an array with only one item
appliedActions = getVenueAppliedActions(allAppliedActions, venueUid)
print "Starting email for venue " + str(venueUid)
print str(appliedActions)
venueName = emailerDb.getVenueName(venueUid)
sendEmail = False
if len(appliedActions) > 0:
sendEmail = True
emailBody = "<html>Daily Integration has been completed for, " + venueName + ". The following changes have been applied:"
#EMPLOYEES:
employeeActions = getAppliedActionsByType(appliedActions, "setup",
"employees")
if len(employeeActions) > 0:
emailBody += "<h3>EMPLOYEES:</h4>"
addEmployeeActions = getInsertActions(employeeActions)
if len(addEmployeeActions) > 0:
emailBody += " New Employees:<ul>"
for addAction in addEmployeeActions:
levyTempPointerUid = addAction[0][13]
emailBody += getNewEmployeeLine(
emailerDb, levyTempPointerUid)
emailBody += "</ul>"
editEmployeeActions = getUpdateActions(employeeActions)
if len(editEmployeeActions) > 0:
emailBody += "Updated Employees:<ul>"
for editAction in editEmployeeActions:
emailBody += getUpdatedEmployeeLine(
emailerDb, editAction)
emailBody += "</ul>"
#CUSTOMERS
customerActions = getAppliedActionsByType(appliedActions,
"patrons", "patrons")
suiteAssignmentActions = getAppliedActionsByType(
appliedActions, "info", "unit_x_patrons")
if len(customerActions) + len(suiteAssignmentActions) > 0:
sendEmail = True
emailBody += "<h3>CUSTOMERS:</h3>"
addCustomerActions = getInsertActions(customerActions)
if len(addCustomerActions) > 0:
emailBody += "New Customers:<ul>"
for addAction in addCustomerActions:
levyTempPointerUid = addAction[0][13]
emailBody += getNewCustomerLine(
emailerDb, levyTempPointerUid)
emailBody += "</ul>"
editCustomerActions = getUpdateActions(customerActions)
if len(editCustomerActions) > 0:
emailBody += "Updated Customers:<ul>"
for editAction in editCustomerActions:
emailBody += getUpdatedCustomerLine(
emailerDb, editAction)
emailBody += "</ul>"
addSuiteAssignmentActions = getInsertActions(
suiteAssignmentActions)
if len(addSuiteAssignmentActions) > 0:
emailBody += "Suite Assignments:<ul>"
for addAction in addSuiteAssignmentActions:
emailBody += getNewSuiteAssignmentLine(
emailerDb, addAction)
emailBody += "</ul>"
deactivateSuiteAssignments = getDeactivateActions(
suiteAssignmentActions)
print "Deactivate: " + str(deactivateSuiteAssignments)
if len(deactivateSuiteAssignments) > 0:
emailBody += "Suite Deassignments:<ul>"
for deactivateAction in deactivateSuiteAssignments:
emailBody += getDeactivateSuiteAssignmentLine(
emailerDb, deactivateAction)
emailBody += "</ul>"
#EVENTS
eventActions = getAppliedActionsByType(appliedActions, "setup",
"events")
if len(eventActions) > 0:
sendEmail = True
emailBody += "<h3>EVENTS:</h3>"
addEventActions = getInsertActions(eventActions)
if len(addEventActions) > 0:
emailBody += "New Events: <ul>"
for addAction in addEventActions:
levyTempPointerUid = addAction[0][13]
emailBody += getNewEventLine(emailerDb,
levyTempPointerUid)
emailBody += "</ul>"
eventXVenuesActions = getAppliedActionsByType(
appliedActions, "setup", "events_x_venues")
updateEventXVenuesActions = getUpdateActions(eventXVenuesActions)
updateEventActions = getUpdateActions(eventActions)
updateAllEventActions = eventXVenuesActions + updateEventActions
if len(updateAllEventActions) > 0:
sendEmail = True
emailBody += "UpdatedEvents:<ul>"
for updateAction in updateAllEventActions:
emailBody += getUpdatedEventLine(emailerDb, updateAction)
emailBody += "</ul>"
#ITEMS
updateItems = False
itemActions = getAppliedActionsByType(appliedActions, "menus",
"menu_items")
if len(itemActions) > 0:
sendEmail = True
emailBody += "<h3>ITEMS:</h3>"
addItemActions = getInsertActions(itemActions)
if len(addItemActions) > 0:
emailBody += "New Items:<ul>"
for addAction in addItemActions:
levyTempPointerUid = addAction[0][13]
emailBody += getNewItemLine(emailerDb,
levyTempPointerUid)
emailBody += "</ul>"
updateMenuItemActions = getUpdateActions(itemActions)
if len(updateMenuItemActions) > 0:
updateItems = True
emailBody += "Updated Items:<ul>"
for updateAction in updateMenuItemActions:
emailBody += getUpdatedMenuItemLine(
emailerDb, updateAction)
menuXMenuActions = getAppliedActionsByType(appliedActions, "menus",
"menu_x_menu_items")
updateMenuXMenuItemActions = getUpdateActions(menuXMenuActions)
if len(updateMenuXMenuItemActions) > 0:
sendEmail = True
if updateItems == False:
updateItems = True
emailBody += "Updated Items:<ul>"
for updateAction in updateMenuXMenuItemActions:
emailBody += getUpdatedMenuXMenuItemLine(
emailerDb, updateAction)
if updateItems == True:
emailBody += "</ul>"
#SUITES
unitActions = getAppliedActionsByType(appliedActions, "setup",
"units")
if len(unitActions) > 0:
sendEmail = True
emailBody += "<h3>UNITS:</h3>"
addUnitActions = getInsertActions(unitActions)
if len(addUnitActions) > 0:
emailBody += "New Units:<ul>"
for addAction in addUnitActions:
levyTempPointerUid = addAction[0][13]
emailBody += getNewUnitLine(emailerDb,
levyTempPointerUid)
emailBody += "</ul>"
editUnitActions = getUpdateActions(unitActions)
if len(editUnitActions) > 0:
emailBody += "Updated Units:<ul>"
for editAction in editUnitActions:
emailBody += getUpdatedUnitLine(emailerDb, editAction)
emailBody += "</ul>"
else:
print "NO UNIT UPDATES WERE MADE"
emailBody += "</html>"
print str(venueUid) + " OUTPUT: " + emailBody
if sendEmail:
emailAddresses = emailerDb.getEmailAddresses(venueUid)
for address in emailAddresses:
address = address[0]
#gmail.sendGmail("*****@*****.**", "fkTUfbmv2YVy", "*****@*****.**", address, 'Integration Complete', emailBody, 'please open this in an HTML compatable email client')
MailGun.sendEmail("*****@*****.**", address,
"Integration Complete", emailBody)
else:
print "Don't send email, nothing to tell the venue about"
else:
print "No actions were applied, Canceling Email"
def __init__(self, levyDB):
self.db = DbConnection().connection
self.levyDB = levyDB
notequery = QtSql.QSqlQuery()
notequery.prepare(QtCore.QString("INSERT INTO note (citekey, note) VALUES (?, ?)"))
notequery.bindValue(0, QtCore.QVariant(currentRef))
notequery.bindValue(1, QtCore.QVariant(note))
notequery.exec_()
if notequery.lastError().type():
print "Error inserting note: [%s] %s" % (citekey, note)
if __name__ == "__main__":
app = QtGui.QApplication(sys.argv)
fileHandle = open("/micehome/jlerch/Documents/MICE/articles/morris-water-maze/lit_notes.txt", 'r')
fileContents = fileHandle.read()
db = DbConnection()
db.guiPrompt()
starts = []
ends = []
refStarter = re.compile(r'^====')
noteStarter = re.compile(r'^\*')
spaceStarter = re.compile(r'^\s')
lines = fileContents.split("\n")
counter = 0
note = None
currentRef = None
notecounter = 0
class Migrate(object): def __init__(self, data, node, fan): self.db = DbConnection() self.data = data self.node = node self.fan = fan def db_create(self): self.db.create_node_table() self.db.create_data_table() self.db.create_fan_table() def db_drop(self): self.db.drop_node_table() self.db.drop_data_table() self.db.drop_fan_table() def db_migrate(self): with open(self.node) as f: reader = csv.DictReader(f, delimiter=',') for row in reader: description = row["Description"] # dummy data nr = NodeRecord(description, description, description, description, 1) self.db.insert_node_record(nr) with open(self.data) as f: reader = csv.DictReader(f, delimiter=',') for row in reader: co2 = row["CO2"] temp = row["Temperature"] hum = row["Humidity"] light = row["Light Value"] node_id = row["Time"] dr = DataRecord(co2, temp, hum, light, node_id) self.db.insert_data_record(dr) with open(self.fan) as f: reader = csv.DictReader(f, delimiter=',') for row in reader: fan_1 = row["fan_1"] fan_2 = row["fan_2"] fan_3 = row["fan_3"] fan_4 = row["fan_4"] fan_5 = row["fan_5"] fn = FanRecord(fan_1, fan_2, fan_3, fan_4, fan_5) self.db.insert_fan_record(fn)
def __init__(self, data, node, fan): self.db = DbConnection() self.data = data self.node = node self.fan = fan
from TrainingEventDB import TrainingEventDB
import HipChat
from config import CheckMateConfig
import redis
checkmateconfig = CheckMateConfig()
host = checkmateconfig.REDIS_ORDERS_HOST
port = checkmateconfig.REDIS_ORDERS_PORT
db = checkmateconfig.REDIS_ORDERS_DB
password = checkmateconfig.REDIS_ORDERS_PASSWORD
redisInstance = redis.Redis(host, port, db, password)
for key in redisInstance.keys('*:orders_by*'):
redisInstance.delete(key)
conn = DbConnection().connection
dbCore = TrainingEventDB(conn)
trainingEvents = dbCore.getTrainingEvents()
HipChat.sendMessage("Moving " + str(len(trainingEvents)) + " training events", "Training", 447878, "purple")
for eventUid, venueUid in trainingEvents:
print "Event UID: " + str(eventUid) + " Venue UID: " + str(venueUid)
#move the event to today
dbCore.moveEventToToday(eventUid)
#clear the orders
dbCore.clearEventOrders(eventUid)
print(f'Received document snapshot (DEVICE): {doc.id}. Pump: {p_state} Lamp: {l_state} Heater: {h_state}')
sensor_data_uploader.set_updated(False)
spi = spidev.SpiDev()
spi.open(0, 0)
spi.max_speed_hz = 1000000
GPIO.setmode(GPIO.BOARD)
GPIO.setwarnings(False)
# LED Azul de Prendido
GPIO.setup(38, GPIO.OUT, initial=1)
min_max_per_plant = json.load(codecs.open('min_max.json', 'r', 'utf-8-sig'))
cred = './credentials.json'
firebase_admin.initialize_app(credentials.Certificate(cred))
conn = DbConnection(db_name='local.db')
fsm = FirestoreManager(cred=cred, col_name='crops')
mnt = FirestoreManager(cred=cred, col_name='monitor')
mnt.retrieve_doc(doc_id='plant')
power_sensor = PowerSupplySensor(pin=36, conn=conn)
image_processor = ImageProcessor(conn=conn)
photo_sensor = PhotoSensor(spi=spi, conn=conn, lamp_pin=22, pin=0)
humidity_sensor = HumiditySensor(spi=spi, conn=conn, pump_pin=18, pin=1)
temperature_sensor = TemperatureSensor(conn=conn, heater_pin=11, pin=22)
sensor_data_uploader = SensorDataUploader(conn=conn, fsm=fsm)
query_watch = None
species = None
while True:
