def read_from_db(data_file):
'''Reads the data from database and exports to specified JSON file'''
global LIMIT_COUNT
count = 1
json_docs = []
db = MongoDB()
cursor = db.query({})
if cursor:
for doc in cursor:
pprint.pprint(doc)
json_doc = json.dumps(doc, default=json_util.default)
json_docs.append(json_doc)
if LIMIT_COUNT != 0 and count > LIMIT_COUNT:
print "\nLimited IDs Read \n"
break
count += 1
db.close()
print count-1, "Records Found. \n"
json_data_file = [json.loads(j_doc, object_hook=json_util.object_hook) for j_doc in json_docs]
write_path = os.path.join(*[DIR, "data", data_file])
with open(write_path, 'w') as outfile:
json.dump(json_data_file, outfile, default=date_handler, indent=4, sort_keys=True)
def insert_to_db():
'''Inserts scrapped data to database'''
global LIMIT_COUNT
count = 1
idlist = read_json('import.json')
db = MongoDB()
print "Initiate Scraping for ", len(idlist), "records ...\n\n"
for storeid in idlist:
print "Scraping Data: " + str(count) + "\n"
extn = scrap_data(storeid["_id"])
extn['updated'] = datetime.datetime.utcnow()
print extn
print "Inserted", db.insert({"_id":storeid["_id"]},
{"$set":extn, "$setOnInsert": {"created": datetime.datetime.utcnow()}}), "data.\n"
if LIMIT_COUNT != 0 and count > LIMIT_COUNT:
print "Update Limit Has Been Set \n"
break
count += 1
print count-1, "records updated. \n"
db.close()
def __init__(self, update_rank_list):
self.db = MongoDB(update_rank_list=update_rank_list)
self.target_dict = {}
# print('Update once when initialized & take a look at time')
start_time = time.time()
self.whole_data_bundle()
self.period_seconds = int(time.time() - start_time) * 2
def __init__(self, thug_version):
BaseLogging.__init__(self)
self.HPFeeds = HPFeeds()
self.MAEC = MAEC(thug_version)
self.MongoDB = MongoDB()
self.shellcodes = set()
async def on_ready(self):
"""
Implementing discord.Client on_ready() that is called when the bot is ready
We do any additional post-initialization set-up here
"""
print('Logged on as {0}!'.format(self.user))
self.db = MongoDB()
def __init__(self, update_rank_list):
self.db = MongoDB(update_rank_list=update_rank_list)
self.total_rank_list = []
# self.mid_list = []
# self.message_room_persentage_dict = {}
# self.man_chart_dict = {}
# self.man_status_dict = {}
# self.radar_dict = {}
self.huolonglive_tracker = {}
self.NRS = no_ram_saver()
'start socket process.. if this works'
python_ws_client(self.db)
print('Update once when initialized & take a look at time')
def get_posts():
db = MongoDB("facebook")
search_term = request.args.get('q')
data = None
with db.open("posts") as dbpost:
find_ = re.compile(search_term, re.IGNORECASE)
data = list(
dbpost.find({
"name": "tatnh",
"content": {
"$regex": find_
}
}))
if not data:
return "NULL"
list_content = []
for ct in data:
if "content" in ct:
content = ct["content"]
if search_term.lower() in content.lower():
content = content
list_content.append(
{"content": content.encode("utf-8").decode("utf-8")})
return jsonify({"data": list_content})
def __init__(self):
self.command = Command()
print(type(self.command))
self.db = MongoDB()
self.SPEAKER = 'SPEAKER'
self.music_cmd = [
"Playing... - func => playingmusic", "명령 : 1. 그만 | 2. 다음",
"명령을 확인중...", "다시 말해 주세요.", "재생을 정지합니다."
]
self.filename = [
'music/Do It.mp3', 'music/noma - Color.mp3', 'music/Sakura.mp3',
'music/Dawn.mp3', 'music/Tomorrow.mp3'
]
self.music_num = random.randrange(0, 5)
self.initMixer()
self.r = sr.Recognizer()
self.count = 0
self.count = self.music_num
print(self.count)
self.playmusic(self.filename[self.music_num])
#!/bin/python
from DataProcessor import DataProcessor
from Utils import Utils
from Logger import Logger
from MongoDB import MongoDB
Utils.createFolderIfNotExists(DataProcessor.TMP_FOLDER)
LOGGER = Logger(DataProcessor.TMP_FOLDER, verbose=True, name='processor')
Utils(DataProcessor.TMP_FOLDER, LOGGER)
mongo = MongoDB('127.0.0.1', 27017, LOGGER, user='******', password='******')
mongo.startQueue(id=0)
print(mongo.getQueueConsumerId())
processor = DataProcessor(mongo, LOGGER)
processor.filterAndNormalizeFullDataset()
#!/bin/python
from DataProcessor import DataProcessor
from Utils import Utils
from Logger import Logger
from MongoDB import MongoDB
Utils.createFolderIfNotExists(DataProcessor.TMP_FOLDER)
LOGGER=Logger(DataProcessor.TMP_FOLDER,verbose=False,name='processor')
Utils(DataProcessor.TMP_FOLDER,LOGGER)
LOGGER.info('Starting Data Processor...')
if Utils.runningOnDockerContainer(){
mongo_addr='mongo'
}else{
mongo_addr='127.0.0.1'
}
mongo=MongoDB(mongo_addr,27017,LOGGER,user='******',password='******')
processor=DataProcessor(mongo,LOGGER)
mongo.startQueue(id=0)
LOGGER.info('Started Data Processor...OK')
LOGGER.info('Listening on queue as {}'.format(mongo.getQueueConsumerId()))
processor.loopOnQueue()
return tweet
def processQuote(tweet, isRT):
pass
def insertTweet (tweet):
pass
"""
tweet = getTweet('917946195410128897')
print(sys.getdefaultencoding())
print (tweet.count())
for t in tweet:
print (t["text"])
"""
db = MongoDB(mongoDB)
RTcount = 0
quoteCount = 0
tweets = db.find(MongoDB.TWEETS_COLLECTION) #getAllTweet()
rts = db.find(MongoDB.TWEETS_COLLECTION, {"retweeted_status": { "$exists" : True}}) #getRTs()
quotes = getQuotes()
originals = getOriginals()
replys = getReplys()
for t in tweets:
isRT = False
if CONST_RT in t:
isRT = True
RTcount += 1
processRT(t)
def __init__(self,):
self.connection= MongoDB('mongodb://localhost:27017/', "mydatabase")
488476334, 87873398, 2242195, 6984908, 400829593, 11291800, 403813584,
432148649, 38024934, 85081706, 661374, 252080324, 359923328, 44927036,
6059217, 516503980, 62631261, 14357419, 14311544, 399861511, 658937,
431101646, 37956041, 313582767, 2178793, 2984759, 499473607, 38794541,
22523561, 32818399, 11772051, 284186543, 396156032, 32173803, 505544254,
363407083, 259250816, 295366519, 37381305, 79017483, 24748424, 394828898,
401383106, 22911369, 398289768, 381557606, 482535850, 1659020, 82093851,
7821115, 37524392, 226891674, 35050726, 88394741, 101259914, 166868,
8173836, 33686029, 110816348, 298937273, 381731506, 8311502, 4335297,
35192221, 288313, 60529793, 34645723, 57488281, 2536, 35644357, 325492218,
289219253, 99583737, 13843791, 282249327, 345668787, 11254795, 40084787,
8415238, 105298708, 20945902, 2938618, 32818720, 87245460, 35587289,
422645812, 33768156, 287863513, 11614571, 15772897, 384016308, 349875109,
1439169, 536826, 349826940, 35565671, 28284601, 33623729, 5702283, 1089449,
321882059, 488870190, 2984278, 522334, 1430724, 505187254, 107924025,
8177500, 29090935, 33528990, 65937481, 7170201, 1581043, 273675639,
73820643, 416103130, 12385555, 17943863, 15364076, 520845958, 158483664,
13546605, 356414, 7920216, 8302421, 249574102, 349455465, 28945970,
20319918, 11288846, 20180718, 15983970, 5366591, 352076508, 86605561,
434838993, 13204992, 7815, 171763777, 347505620, 9529702, 292017937,
349977, 21910697, 327569968, 393709604, 7400053, 249762351, 35698830,
20378379, 3996044, 91689527, 301128262, 1484324, 9597720, 463677, 71275337,
313716538, 14566476, 24221899, 683642, 492059392, 335999484, 275525520,
282789550, 38765647, 213965443, 283070257, 872047, 51724265
]
db = MongoDB()
db.reset_threshord_of(list100000, 100000)
db.reset_threshord_of(list10000, 10000)
db.reset_threshord_of(list2000, 2000)
db.reset_threshord_of(list5000, 5000)
from flask import *
from flask_cors import *
from MongoDB import MongoDB
# runtime configurations
HOST = 'localhost'
PORT = 5000
m = MongoDB(url="localhost:27017", database="college", doc="student")
app = Flask(__name__)
@app.route('/', methods=['GET'])
def get_students():
print(m.size())
val = jsonify(m.find_all())
return val
@app.route('/<id>', methods=['GET'])
def get_student_by_id(id):
val = m.find_by_id(id)
return val
@app.route('/', methods=['POST'])
def add_student():
student = request.get_json()
m.add(student)
return student
def __init__(self):
# MEMBERS
# self.motor = MotorControl() #servo motor
self.db = MongoDB()
self.lock = threading.Lock()
self.timedCaptureBool = False
self.threadInProgress = threading.Event()
# GUI ELEMENTS
self.root = Tk()
self.root.title('ZRecognition')
self.root.geometry('800x600')
self.root.resizable(False, False)
# FRAME DEFINITIONS
self.imageFrame = Frame(self.root, padx=5, pady=5)
self.inputFrame = Frame(self.root, padx=5, pady=5)
self.logFrame = Frame(self.root, padx=5, pady=5)
# WIDGET DEFINITIONS
self.canvasColor = Canvas(self.imageFrame)
self.canvasDigi = Canvas(self.imageFrame)
self.manualCapture = Button(
self.inputFrame,
text='Manual Capture',
height=1,
command=lambda: threading.Thread(target=lambda: self.ManualCapture(
)).start() if not self.threadInProgress.isSet() else None)
self.timedCapture = Button(
self.inputFrame,
text='Timed Capture',
height=1,
command=lambda: threading.Thread(target=lambda: self.TimedCapture(
)).start() if not self.threadInProgress.isSet() else None)
self.stopCapture = Button(self.inputFrame,
text='Stop Timed Capture',
height=1,
command=lambda: self.StopTimer())
self.timedCaptureTimer = Text(self.inputFrame, width=10, height=2)
self.log = sctx.ScrolledText(self.logFrame, height=10, state=DISABLED)
self.var = IntVar(self.inputFrame)
self.var.set(3) # initial value
self.option = OptionMenu(self.inputFrame, self.var, 3, 4, 5, 6, 7)
# PLACING ALL ELEMENTS INTO FORM
# IMAGE FRAME
self.imageFrame.grid(row=0, column=0, sticky=N + S + E + W)
self.canvasColor.grid(row=0, column=0, sticky=N + S)
self.canvasDigi.grid(row=0, column=1, sticky=N + S)
# INPUT FRAME
self.inputFrame.grid(row=1, column=0, sticky=N + S + E + W)
self.manualCapture.grid(row=0, column=0, sticky=N + S + E + W)
self.timedCapture.grid(row=0, column=1, sticky=N + S + E + W)
self.stopCapture.grid(row=1, column=0, sticky=N + S + E + W)
self.option.grid(row=1, column=1)
# LOG FRAME
self.logFrame.grid(row=2, column=0, sticky=N + S + E + W)
self.log.grid(row=0, column=0, sticky=N + S + E + W)
# GRID WEIGHT FOR EXPANDING
# MAIN FRAME
temp = self.root.grid_size()
for r in range(temp[1]): # ROWS
Grid.rowconfigure(self.root, r, weight=1)
for c in range(temp[0]):
Grid.columnconfigure(self.root, c, weight=1)
# CANVAS FRAME
temp = self.imageFrame.grid_size()
for r in range(temp[1]): # ROWS
Grid.rowconfigure(self.imageFrame, r, weight=1)
for c in range(temp[0]):
Grid.columnconfigure(self.imageFrame, c, weight=1)
# INPUT FRAME
temp = self.inputFrame.grid_size()
for r in range(temp[1]): # ROWS
Grid.rowconfigure(self.inputFrame, r, weight=1)
for c in range(temp[0]):
Grid.columnconfigure(self.inputFrame, c, weight=1)
# LOG FRAME
Grid.rowconfigure(self.logFrame, 0, weight=1)
Grid.columnconfigure(self.logFrame, 0, weight=1)
# Conexion for Mongodb Atlas
from MongoDB import MongoDB
connector = MongoDB(
"mongodb://*****:*****@db-argentum-shard-00-00-8pr3z.gcp.mongodb.net:27017,db-argentum-shard-00-01-8pr3z.gcp.mongodb.net:27017,db-argentum-shard-00-02-8pr3z.gcp.mongodb.net:27017/db-mechanicus?ssl=true&replicaSet=DB-Argentum-shard-0&authSource=admin&retryWrites=true",
'db-mechanicus')
from MongoDB import MongoDB
import re
#dictionary_regex = MongoDB("regex")
dictionary = MongoDB("substring_Length3to8")
def subStringFinder(word):
for i in range(1, len(word)):
for j in range(0, len(word) - i):
wordToAdd = word[j:j + i + 1]
if (len(wordToAdd) > 2 and len(wordToAdd) < 9):
dictionary.add(wordToAdd)
def regexFinder(word):
regexList = []
with open(
"/Users/thomasbekman/Documents/Research/Regex_toFormat.txt") as f:
for line in f:
formattedLine = "^" + line[:-1] + "$" # The [:-1] is to deal with new lin charecters
if (line.find("\n") == -1):
formattedLine = "^" + line + "$"
regexList.append(formattedLine)
for regex in regexList:
# print("Im the Rgex: "+ regex)
if (re.match(regex, str(word)) is not None):
dictionary_regex.add(regex)
break
def __init__(self):
self.connection = MongoDB('mongodb://localhost:27017/', "mydatabase")
self.Logger = FileLogger()
