def __init__(self,
collection,
start_date,
end_date,
chunk_size=10000,
max_items=None):
self.db = settings.get_db()
self.source = self.db[collection]
# total number of batches of data in the db/collection
if not max_items:
self.len = self.source.find({
'Document.g_timestamp': {
'$gte': start_date,
'$lt': end_date
}
}).count()
else:
self.len = max_items
self.slices = int(self.len / chunk_size)
self.data = [] # always keep 2 slices 1 to read from and 1 as a buffer
self.lock = threading.Lock()
self.cond = threading.Condition()
self.available = True
self.start = start_date
self.end = end_date
self.offset = 0
self.chunk_size = chunk_size
self.data = [self.__fetch__() for _ in xrange(2)]
self.order = []
def main():
# Arguments:
form = cgi.FieldStorage()
if form.getvalue('unique_id'):
uid = cgi.escape(form.getvalue('unique_id'))
card_id = cgi.escape(form.getvalue('card_id'))
db = settings.get_db()
# Print basic HTML stuff:
base.begin()
base.header(title='Adding a unique ID...')
base.header_redirect("module.py?db={0}&card_id={1}".format(db, card_id))
base.top(db)
# Add:
try:
con = connect(True, db)
cur = con.cursor()
cur.execute('UPDATE Card SET unique_id="{0}" WHERE card_id={1}'.format(uid, card_id))
con.commit()
con.close()
except Exception as err:
print "<span style='color: red'>ERROR</span><br>"
print err
# print("<h3>Serial number already exists!</h3>")
base.bottom()
def main():
# Arguments:
form = cgi.FieldStorage()
sn = form.getvalue("serial_number")
db = settings.get_db()
if sn:
try:
con = connect(True, db)
cur = con.cursor()
cur.execute("INSERT INTO Card SET sn = '{0}'; ".format(sn))
con.commit()
con.close()
except Exception as ex:
base.begin()
base.header(title='{0}: add module'.format(db))
base.top(db)
print ex
print '<center><h3 style="color:red"><i>Serial number "{0}" already exists! It was not added.</i></h3></center>'.format(sn)
else:
cardid = module.fetch_cardid_from_sn(db,sn)
base.begin()
base.header_redirect("module.py?db={0}&card_id={1}".format(db,cardid))
base.top(db)
else:
base.begin()
base.header_redirect("home.py?db={0}".format(db),1)
base.top(db)
print '<center><h3 style="color:red"><i>Tried to input null serial number. Do not do that.</i></h3></center>'.format(sn)
base.bottom()
def __init__(self,
script: MlScript,
source_collection,
targets_definitions,
timestamp_column=None,
use_feature_generation=False,
index_column=None):
self.script = script
self.prediction_log = []
self.db = settings.get_db()
self.predictions = self.db['predictions']
self.client = script.client
# target_columns, timestamp_column
fbuilder_config = {
'db': self.db,
'collection': {
'key': source_collection
},
'targets': targets_definitions,
'use_feature_generation': use_feature_generation
}
self.features_builder = FeaturesBuilder(fbuilder_config, MongoBufferedReader)
self.features_builder.set_script(script)
self.targets = [x['column'] for x in targets_definitions]
self.timestamp_column = timestamp_column
self.data_consumer = None
self.feature_generation_enabled = True
self.targets_models = dict()
def main():
# Arguments:
form = cgi.FieldStorage()
sn = cgi.escape(form.getvalue('serial_num')) # This is coming from a form.
cardid = cgi.escape(form.getvalue('card_id'))
person_id = base.cleanCGInumber(form.getvalue('person_id'))
db = settings.get_db()
note = cgi.escape(form.getvalue('note')) if form.getvalue('note') else False
# base.begin()
# print person_id
# print note
# base.bottom()
if not person_id:
base.error(db, "You must select a tester to be associated with this note.")
else:
if not note:
base.error(db, "You left the note field empty. You're not allowed to make an empty note.")
else:
# Begin:
base.begin()
base.header_redirect("module.py?db={0}&card_id={1}#notes".format(db, cardid))
base.top(db)
# Add note:
result = add_note(db, sn, note, person_id)
if result:
print "added note:<br>"
print note
# base.header_redirect_module_notes(card_id, serial_num)
base.bottom()
def main():
# Arguments:
form = cgi.FieldStorage()
testtype_name = form.getvalue("testName")
testtype_short = form.getvalue("descShort")
testtype_long = form.getvalue("descLong")
testtype_req = (False, True)[bool(form.getvalue("required"))]
db = settings.get_db()
# Begin:
base.begin()
base.header_redirect("erudition.py?db={0}".format(db),1)
base.top(db)
# Insert:
con = connect(True, db)
cur = con.cursor()
if testtype_name and testtype_long:
cur.execute("INSERT INTO Test_Type (name, required, desc_short, desc_long, relative_order) VALUES ('{0}', {1}, '{2}', '{3}', 1);".format(testtype_name, int(testtype_req), testtype_short, testtype_long))
con.commit()
con.close()
else:
print '<center><h3 style="color:red"><i> ERROR: Null test type or test type description provided. Please input again.</i></h3></center>'
base.bottom()
def run():
db = get_db()
db.session.add(User(name="Flask", email="*****@*****.**"))
db.session.commit()
users = User.query.all()
def main(): # Identify DB from URL: db = settings.get_db() base.begin() # Print the preamble and opening html tag. base.header(title='ePorridge') # Print the header. base.top(db) # Print the top portion of the body (title, buttons). This remains the same for every page. print_home(db) # Print what should appear on the home page. (Defined above.) base.bottom() # Print footer (if applicable) and closing body and html tags.
def main():
# Arguments:
form = cgi.FieldStorage()
cardid = base.cleanCGInumber(form.getvalue('card_id'))
db = settings.get_db()
# Basic:
base.begin()
base.header(title='{0}: module'.format(db))
base.top(db)
# Form:
print_form(db, cardid)
base.bottom()
def execute_mongo_crud(self,
c_name,
requests,
exec_type,
position=None,
step=10000):
"""执行mongo CRUD操作"""
method_map = {
'update': 'bulk_write',
'insert': 'insert_many',
}
for i in range(0, len(requests), step):
self.execute(get_db()[c_name], method_map[exec_type],
requests[i:i + step])
del requests
def main():
db = settings.get_db()
base.begin()
base.header(title='{0}: tests'.format(db))
base.top(db)
print_test_info_table(db)
print '<br><br>'
print_testtype_form(db)
print_card_info_table(db)
print '<br><br>'
print_cardinfotype_form(db)
base.bottom()
def main():
# Arguments:
db = settings.get_db()
# Basic:
base.begin()
base.header(title='{0}: summary'.format(db))
base.top(db)
# Modules table:
print_stat_table(db)
print_tests_table(db)
print_modules_table(db)
# End:
base.bottom()
def gen_features(self, msg):
from db.data_handlers import MongoBufferedReader
params = msg.get('params')
model_id = params.get('model_id')
from ml.feature_extraction import FeaturesBuilder
log.info("Generating features: " + str(params))
# we dont create tasks anymore..
# with db:
# task = FeatureGenerationTasks.create(Status=0, ModelId=model_id)
# task.save()
params['db'] = settings.get_db()
fe = FeaturesBuilder(params, MongoBufferedReader)
callback = partial(Server.on_features_generated,
server=self,
task=task)
self.ppool.apply_async(fe.generate_features, callback=callback)
return {'task_id': task.Id}
def main():
# Arguments
form = cgi.FieldStorage()
cardid = base.cleanCGInumber(form.getvalue('card_id'))
suggested_test = base.cleanCGInumber(form.getvalue('suggested'))
db = settings.get_db()
sn = module.fetch_sn_from_cardid(db, cardid)
# Basic:
base.begin()
base.header(title='{0}: test'.format(db))
base.top(db)
# Test form:
print_test_form(db, cardid, sn, suggested_test)
base.bottom()
def get_tide_times(date, intent):
database = settings.get_db()
db_cursor = database.cursor()
date = date.replace('-', '')
db_cursor.execute('SELECT * FROM tides WHERE the_date={}'.format(date))
tide_times = db_cursor.fetchone()
if 'high' in intent:
tide_times = tide_times[1:3]
which_tide = 'high tides'
if 'low' in intent:
tide_times = tide_times[3:]
which_tide = 'low tides'
print(intent, "tide times -------------- ", tide_times)
return {
'fulfillmentText':
'The {} will be at {} and {}.'.format(which_tide, tide_times[0],
tide_times[1])
}
def main():
# Arguments:
form = cgi.FieldStorage()
sn = form.getvalue("serial_number")
db = settings.get_db()
## KLUDGE for special ease-of-access for "hf_qie" DBs:
if "hf_qie" in db:
if len(sn) == 3:
sn = "3040000000000500" + sn
if sn:
card_id = module.fetch_cardid_from_sn(db, sn)
if card_id:
base.error(db, "Finding the module ...", redirect="module.py?db={0}&card_id={1}".format(db, card_id), redirect_time=0)
else:
base.error(db, "There is no module in this database with a serial number matching \"{0}\".".format(sn))
else:
base.error(db, "What did you expect to happen?")
return True
def main():
# Arguments:
form = cgi.FieldStorage()
test_id = base.cleanCGInumber(form.getvalue('test_id'))
comment = form.getvalue('comment')
db = settings.get_db()
card_id = test.fetch_cardid_from_testid(db, test_id)
testtype_id = test.fetch_testtypeid_from_testid(db, test_id)
# Revoke:
revoke_test(db, test_id, revoke=True, comment=comment)
# Basic:
base.begin()
# base.header(title='{0}: revoke'.format(db))
base.header_redirect("module.py?db={0}&card_id={1}#test-{2}".format(db, card_id, testtype_id))
base.top(db)
base.bottom()
def main():
# Arguments:
form = cgi.FieldStorage()
person_id = base.cleanCGInumber(form.getvalue("person_id"))
test_type = base.cleanCGInumber(form.getvalue("test_type"))
card_id = base.cleanCGInumber(form.getvalue("card_id"))
success = (False, True)[bool(form.getvalue("success"))]
db = settings.get_db()
## Comments:
comments = form.getvalue("comments")
if comments:
comments = cgi.escape(comments)
if person_id != 0:
# Basic:
base.begin()
base.header_redirect("module.py?db={0}&card_id={1}".format(db, card_id))
base.top(db)
# Add the test to the DB:
test_id = add_test_basic(db, person_id, test_type, card_id, success, comments)
## Attachments:
for i in range(1, 4):
afile = form['attach{0}'.format(i)]
if (afile.filename):
adesc= form.getvalue("attachdesc{0}".format(i))
if adesc:
adesc = cgi.escape(adesc)
acomment= form.getvalue("attachcomment{0}".format(i))
if acomment:
acomment = cgi.escape(acomment)
add_test_attachment(db, test_id, afile, adesc, acomment)
base.bottom()
else:
base.error(db, "You must select a tester from the available list.")
else:
base.error(db, "You must input comments before the test can be added.")
def main():
form = cgi.FieldStorage()
infoName = cgi.escape(form.getvalue("characteristic","no name found"))
descShort = cgi.escape(form.getvalue("descShort","no description found"))
descLong = cgi.escape(form.getvalue("descLong","no description found"))
db = settings.get_db()
base.begin()
base.header_redirect('erudition.py?db={0}#cardInfoTable'.format(db),1)
base.top(db)
con = connect(True,db)
cur = con.cursor(buffered=True)
if infoName and descLong:
cur.execute("INSERT INTO Card_Info_Types (Info_Name,Info_Desc_Short,Info_Desc_Long) VALUES ('{0}','{1}','{2}');".format(infoName,descShort,descLong))
con.commit()
con.close()
else:
print '<center><h3 style="color:red"><i> ERROR: Null characteristic or characteristic info. Please input again. </i></h3></center>'
base.bottom()
def main():
db = settings.get_db()
base.begin()
base.header(title='{0}: testers'.format(db))
base.top(db)
print '<div class="row">'
print '<div class="col-md-12">'
print '<table class="table" style="width:650px" align="center">'
print '<tr>'
print '<th > Name </th>'
print '<th > ID </th>'
print '</tr>'
userInfo = fetch_user_info(db)
for user in userInfo:
print '<tr>'
print u'<td align="left">{0}</td>'.format(user['name']).encode("utf-8")
print '<td align="left">',user['id'],'</td>'
print '</tr>'
print '</table>'
print '</div>'
print '</div>'
print '<br><br>'
print '<form action="add_tester.py?db={0}" method="post" enctype="multipart/form-data">'.format(db)
print '\t<div class="col-md-6">'
print '\t\t<b>Add tester</b><br>'
print '\t\ttester name:<br>'
print '\t\t<textarea name="testerName" cols="35" rows="1"></textarea><br>'
print '\t\t<input type="submit" value="Submit">'
print '\t</div>'
print '</form>'
base.bottom()
def main():
# Arguments:
form = cgi.FieldStorage()
tester_name = form.getvalue("testerName")
db = settings.get_db()
# Begin:
base.begin()
base.header_redirect("testers.py?db={0}".format(db),1)
base.top(db)
# Connect to DB:
con = connect(True, db)
cur = con.cursor(buffered=True)
#print "INSERT INTO people (person_name, person_id) VALUES ('{0}',{1})".format(testerName,testerID)
if tester_name:
cur.execute("INSERT INTO People (person_name) VALUES ('{0}')".format(tester_name))
con.commit()
con.close()
else:
print "<center><h3 style='color:red'><i>ERROR: Null tester name submitted. Database not updated.</i></h3></center>"
base.bottom()
def fetch_record(query, collection):
db = settings.get_db()
col = db[collection]
doc = col.find_one(query)
return doc
def __init__(self):
self.db = settings.get_db()
self.encoding_helper = EncodingHelper()
import click
import jieba
import jieba.analyse
from settings import get_db, collections
db = get_db()
def handler_jieba_text_rank():
collection = db['article-juejin']
jieba.analyse.set_stop_words('stopword.txt')
for data in collection.find():
body = data['body']
doc_id = data['document_id']
print(doc_id)
ranks = []
for x, w in jieba.analyse.textrank(body,
topK=30,
withWeight=True,
allowPOS=('ns', 'n')):
ranks.append({x: w})
data = {'$set': {'jieba_text_rank': ranks}}
collection.update_one({'document_id': doc_id}, data, upsert=True)
@click.command()
@click.option('--collection',
'-c',
#!/usr/bin/python
import cgi
import base
import module_functions
from connect import connect
import settings
import os.path
import sys
form = cgi.FieldStorage()
attach_id = base.cleanCGInumber(form.getvalue('attach_id'))
db = settings.get_db()
con = connect(True, db)
cur = con.cursor()
cur.execute("SELECT test_id, attachmime, originalname FROM Attachments WHERE attach_id=%d" % (attach_id));
if not cur.with_rows:
print "Content-type: text/html\n"
base.header("Attachment Request Error")
base.top()
print "<h1>Attachment not available</h1>"
base.bottom()
else:
thevals=cur.fetchall();
#attpath=settings.getAttachmentPathFor(thevals[0][0],attach_id) # Kelvin (16/08/16) - this function is not in settings.py....
attpath=settings.get_attachment_path(db,thevals[0][0],attach_id)
if not os.path.isfile(attpath):
print "Content-type: text/html\n"