def __init__(self, config_path, cla_dict):
trulia_conf = TruliaConfLoader.TruliaConfLoader(config_path)
self.load_trulia_params(trulia_conf)
self.db_mgr = DatabaseManager.DatabaseManager(config_path)
kv_store = cla_dict['kv_store']
if kv_store == '':
self.kv_mgr = None
elif kv_store == 'h':
print "loading HBase manager",
import HBaseManager
self.kv_mgr = HBaseManager.HBaseManager()
print "completed"
elif kv_store == 'r':
print "loading Redis manager",
import RedisManager
self.kv_mgr = RedisManager.RedisManager()
print "completed"
fluentd_rx = cla_dict['fluentd_rx']
if fluentd_rx == '':
self.fluentd_enabled = False
print "FluentD not enabled"
elif fluentd_rx == 'f':
print "loading fluentd for local fs"
sender.setup('fs')
self.fluentd_enabled = True
print "FluentD enabled for local filesystem"
elif fluentd_rx == 'h':
print "loading fluentd for hdfs"
sender.setup('hdfs')
self.fluentd_enabled = True
print "FluentD enabled for HDFS"
def run(self):
self.__database_manager = DatabaseManager.DatabaseManager()
self.__scanner_data_provider = ScannerDataProvider.ScannerDataProvider(
)
self.__scanner_data_provider.start()
self.__running_lock.acquire()
self.__is_running = True
self.__running_lock.release()
self.__thread_timer = time.time()
self.__data_timer = time.time()
while True:
if time.time() - self.__thread_timer > 1000.0 / 1000.0:
self.__running_lock.acquire()
condition = self.__is_running
self.__running_lock.release()
if bool(condition) is False:
break
self.__thread_timer = time.time()
if time.time() - self.__data_timer > 100.0 / 1000.0:
dict_scanner_data = self.__scanner_data_provider.get_scanner_data(
)
self.__store_in_db(dict_scanner_data)
self.__store_in_queue(dict_scanner_data)
self.__data_timer = time.time()
# Wait for the threads to stop
self.__scanner_data_provider.stop()
self.__scanner_data_provider.join()
print '[Data Manager] scanner_data_provider stopped '
def __init__(self, cogs, output_path, db_args, **args):
super().__init__(**args)
self.db_manager = DatabaseManager.DatabaseManager(db_args, output_path)
self.connected = False
self.output_path = output_path
for cog in cogs:
self.load_extension(cog)
def __init__(self, config_path):
trulia_conf = TruliaConfLoader.TruliaConfLoader(config_path)
self.load_trulia_params(trulia_conf)
self.db_mgr = DatabaseManager.DatabaseManager(config_path)
self.curr_key_idx = 0
# lock for threads to use to add to val_strs
self.lock = threading.Lock()
self.val_strs = list()
def dbConnect (self,db_name):
db_username = gv.db_username
db_password = gv.db_password
db_host = gv.db_host
#db_name = gv.name_database
db_port = gv.db_port
global databaseObject
databaseObject = DatabaseManager(db_username, db_password,db_host,db_port, db_name)
databaseObject.Connect()
def dbConnect(self):
db_username = '******'
db_password = '******'
db_host = '127.0.0.1'
db_name = gv.databaseName
db_port = '3306'
global databaseObject
databaseObject = DatabaseManager(db_username, db_password, db_host,
db_port, db_name)
databaseObject.Connect()
def dbConnect (self):
db_username = gv.userName
db_password = gv.password
db_host = gv.dbHost
db_name = gv.databaseName
db_port = gv.dbPort
db_connector = gv.dbConnector
global databaseObject
databaseObject = DatabaseManager(db_connector, db_username, db_password,db_host,db_port, db_name)
databaseObject.Connect()
def __init__(self):
self.__active_user_list = []
self.__sys_data = ERPDate.ERPDate(1, 1)
self.__raw_material_order_manager = RawMaterialOrderManager.RawMaterialOrderManager(
)
self.__repository_manager = RepositoryManager.RepositoryManager()
self.__database_manager = DatabaseManager.DatabaseManager()
self.__request_msg = []
self.__fcn_dic = []
self.__init_system()
self.test_main()
def start(self, strat, capital, commission):
self.initDatabase()
self.initPipelineTables()
self.initStatisticsTables()
self.initTradingTable()
self.initBookTables()
entryConditions, exitConditions = stratFuncs
strat = stra.Strategy(stratName, entryConditions, exitConditions)
self.databaseManager = dbm.DatabaseManager(self.dbRef, self.conn,
strat, auth)
self.databaseManager.setTradingParameters(self.exchange, self.ticker)
def CreateDatabaseManager(appID, result_p):
expDBDirPath = os.environ["CommonProgramFiles"] + "\\VOCALOID5\\Explib"
if(expDBDirPath == None):
return None
elif(expDBDirPath == ""):
return None
try:
databaseManager = DatabaseManager.DatabaseManager(DatabaseManagerIF.VDM_createDatabaseManager(appID, expDBDirPath, result_p))
if(result_p.contents.value != VDMError.VDMError.NotAny):
return None
else:
return databaseManager
except:
return None
def __collect_data(self):
"""
get data from MySQLdb server
"""
if bool(self.__db_connected) is False:
self.__database_manager = DatabaseManager.DatabaseManager()
self.__db_connected = True
user_settings = \
self.__database_manager.get_data_from_database('HOME_SCANNER_USER_SETTINGS')
if len(user_settings) > 0:
if len(user_settings[0]) > 7:
self.__temperature_threshold = int(user_settings[0][1])
self.__humidity_threshold = int(user_settings[0][2])
self.__gas_threshold = int(user_settings[0][3])
self.__video_enabled = int(user_settings[0][4])
self.__face_detection_enabled = int(user_settings[0][5])
self.__motion_detection_enabled = int(user_settings[0][6])
self.__human_detection_enabled = int(user_settings[0][7])
def upload_notifications(self, notifications_queue):
"""
Upload the notifications via DB MGR
"""
current_thread = threading.currentThread()
__database_manager = DatabaseManager.DatabaseManager()
__thread_timer = time.time()
while getattr(current_thread, 'is_running', True):
if time.time() - __thread_timer > 200.0 / 1000.0:
__thread_timer = time.time()
try:
notifications = notifications_queue.get(False)
except Queue.Empty:
continue
for elem in notifications:
__database_manager.insert_data_in_database(
elem,
'HOME_SCANNER_NOTIFICATIONS'
)
notifications_queue.task_done()
def __init__(self, dbName):
self.dbManager = DatabaseManager(dbName)
self.initIPTables()
self.deviceDict = dict()
self.loadRulesFromDB()
import random
import time
import threading
from DatabaseManager import *
db = DatabaseManager(None, "db.csv")
dict_lock = threading.Lock()
users = "joe josh john matt mike charles charlotte maggie martha jackie jamie andy alex tom dick harry steven susan nancy".split(
)
passwords = "lorem ipsum dolor sit amet consectetur adipiscing elit sed do eiusmod tempor incididunt ut labore et dolore magna aliqua".split(
)
passkeys = {}
global failures, globcount, END
failures = 0
globcount = 0
END = False
def dlock():
dict_lock.acquire()
def dunlock():
dict_lock.release()
def createsome(i):
table.put(
hbase_base_key + '-' + key, {
'i:city': str(city),
'i:sc': str(state_code),
'i:lat:': str(lat),
'i:lon': str(lon)
})
def get_dom_dict(dom):
pass
sys.path.append('../common/')
import DatabaseManager
dm = DatabaseManager.DatabaseManager('../conf/')
import time
conn = happybase.Connection('ip-172-31-11-76.us-west-1.compute.internal')
conn.delete_table('city_stats_26june14', disable=True)
conn.create_table('city_stats_26june14', {'cf': {}, 'i': {}})
table = conn.table('city_stats_26june14')
#table = conn.table('city_stats')
# TODO cleanup
fluent_path = "../extern/fluent-logger-python"
sys.path.append(fluent_path)
from fluent import sender
from fluent import event
def setupUI(self):
layout = QtGui.QVBoxLayout()
layout.setContentsMargins(20, 20, 20, 10)
layout.addWidget(self.table)
layout.addSpacing(50)
bttnLayout = QtGui.QHBoxLayout()
bttnLayout.addStretch()
bttnLayout.addWidget(self.bttnBack)
layout.addLayout(bttnLayout)
self.setLayout(layout)
def goBack(self):
if self.__parent is not None:
self.__parent.goBack()
if __name__ == "__main__":
DatabaseManager.db = DatabaseManager.DatabaseManager(
host="sql12.freemysqlhosting.net",
username="******",
password="******",
databaseName="sql12269310")
app = QtGui.QApplication(sys.argv)
w = ShowDesigationWidget()
w.show()
sys.exit(app.exec_())
from DatabaseManager import *
from functions import *
from SpecRatioFunctions import *
try:
opts, args = getopt.getopt(sys.argv, "n:", ["name="])
except getopt.GetoptError:
print('main.py -n <pc name>')
sys.exit(2)
# Get the name of the pc/user and add to database
pc_name = args[2]
# Initialize connection with AWS RDS database
db = DatabaseManager()
# Add user to the pc table
pc_id = db.add_pc_to_db(pc_name)
# Get the runtime for each benchmark (benchmarks are sorted by their id in the database)
print("The benchmarks are running, please wait...")
ordered_runtimes = get_ordered_runtimes()
# Get the reference runtime for each benchmark
reference_runtimes = db.get_reference_times()
if not reference_runtimes:
exit(500)
# Add the result for each benchmark in the results table
spec_ratios = []
tweets_data_path = "data/"
tweets_file_name = "twitter_data_sampled"
tweets_file_name_original = tweets_file_name
if database_selection == 1:
tweets_file_name = input("Enter name of database(Default: '%s'):" %
(tweets_file_name))
if tweets_file_name == "":
tweets_file_name = tweets_file_name_original
media_folder = tweets_data_path + tweets_file_name + "/media/"
dc = DataCrawler.DataCrawler()
dm = DatabaseManager.DatabaseManager(tweets_data_path, tweets_file_name,
database_selection)
if database_selection == 2:
media_folder = "PostgreSQL/media/"
table = input(
"(1) url\n(2) media\n(3) icard\n(4) user url\n(5) user images\n\nselect a table to process (default: 1):"
)
if table == None: table = 1
else:
if table == "":
table = 1
table = int(table)
offset = input("Choose a offset (default: 0):")
if offset == None:
def __init__(self, trulia_home_path, kv_store='r'):
self.common_path = trulia_home_path + "/common/"
self.config_path = trulia_home_path + "/conf/"
self.debug = False
sys.path.append(self.common_path)
import RestCallHandler
if kv_store == 'h':
import HBaseManager
self.kv_store_mgr = HBaseManager.HBaseManager()
elif kv_store == 'r':
import RedisManager
self.kv_store_mgr = RedisManager.RedisManager()
# RDBMS Manager
import DatabaseManager
self.database_manager = DatabaseManager.DatabaseManager(
self.config_path)
self.app = Flask(__name__)
@self.app.route('/data/state/volume', methods=['GET'])
def data_state_volume():
# get data state parameters to the right of ?q= as string from flask.request
params = request.args.get('q', None)
return RestCallHandler.handle_data_state_volume_query(
self.kv_store_mgr, self.database_manager, params)
@self.app.route('/data/state/average', methods=['GET'])
def data_state_average():
# get data state parameters to the right of ?q= as string from flask.request
params = request.args.get('q', None)
return RestCallHandler.handle_data_state_average_query(
self.kv_store_mgr, self.database_manager, params)
@self.app.route('/data/county/volume', methods=['GET'])
def data_county_volume():
# get data county parameters to the right of ?q= as string from flask.request
params = request.args.get('q', None)
return RestCallHandler.handle_data_county_volume_query(
self.kv_store_mgr, self.database_manager, params)
@self.app.route('/data/county/average', methods=['GET'])
def data_county_average():
# get data county parameters to the right of ?q= as string from flask.request
params = request.args.get('q', None)
return RestCallHandler.handle_data_county_average_query(
self.kv_store_mgr, self.database_manager, params)
@self.app.route('/data/city/volume', methods=['GET'])
def data_city_volume():
# get data city parameters to the right of ?q= as string from flask.request
params = request.args.get('q', None)
return RestCallHandler.handle_data_city_volume_query(
self.kv_store_mgr, self.database_manager, params)
@self.app.route('/data/city/average', methods=['GET'])
def data_city_average():
# get data city parameters to the right of ?q= as string from flask.request
params = request.args.get('q', None)
return RestCallHandler.handle_data_city_average_query(
self.kv_store_mgr, self.database_manager, params)
@self.app.route('/data/zipcode/volume', methods=['GET'])
def data_zipcode_volume():
# get data zipcode parameters to the right of ?q= as string from flask.request
params = request.args.get('q', None)
return RestCallHandler.handle_data_zipcode_volume_query(
self.kv_store_mgr, self.database_manager, params)
@self.app.route('/data/zipcode/average', methods=['GET'])
def data_zipcode_average():
# get data zipcode parameters to the right of ?q= as string from flask.request
params = request.args.get('q', None)
return RestCallHandler.handle_data_zipcode_average_query(
self.kv_store_mgr, self.database_manager, params)
def transfer_analysed_files(self, analysed_files_queue):
"""
transfer the analysed files to webserver location and
delete the mp4 files
"""
current_thread = threading.currentThread()
self.__database_manager = DatabaseManager.DatabaseManager()
while getattr(current_thread, 'is_running', True):
try:
file_name = analysed_files_queue.get(False)
except Queue.Empty:
time.sleep(1)
continue
# delete the mp4 file
clear_command = []
clear_command.append('rm')
clear_command.append(file_name)
clear_process = subprocess.Popen(clear_command)
while clear_process.poll() is None:
time.sleep(0.1)
# move the avi file to website location
file_name = os.path.splitext(file_name)[0] + '.h264'
converted_file_name = os.path.splitext(file_name)[0] + '.mp4'
convert_command = []
convert_command.append(MP4BOX)
convert_command.append("-add")
convert_command.append(file_name)
convert_command.append(converted_file_name)
convert_process = subprocess.Popen(convert_command)
while convert_process.poll() is None:
time.sleep(0.1)
# delete the h264 file
clear_command = []
clear_command.append('rm')
clear_command.append(file_name)
clear_process = subprocess.Popen(clear_command)
while clear_process.poll() is None:
time.sleep(0.1)
move_command = []
move_command.append('mv')
move_command.append(converted_file_name)
move_command.append('/var/www/html/videos/')
move_process = subprocess.Popen(move_command)
while move_process.poll() is None:
time.sleep(0.1)
timest = time.time()
timestamp = datetime.datetime.fromtimestamp(timest).strftime('%Y-%m-%d %H:%M:%S')
db_file_record = [converted_file_name, timestamp]
self.__database_manager.insert_data_in_database(
db_file_record,
'HOME_SCANNER_VIDEO_FILES'
)
analysed_files_queue.task_done()
#!/usr/bin/env python2
from pprint import pprint
import DatabaseManager
print("~~~ Testing DatabaseManager ~~~")
dm = DatabaseManager.DatabaseManager()
dm.get_data('author_ln')
dm.get_data('reviewer')
dm.get_matches("analysis_sw",1,"SAS")
dm.get_matches("reviewer",0,6)
dm.get_searches("analysis_sw",1,'SAS')
dm.get_searches('article_doi',1,'10.1097')
dm.get_searches('article_title',1,'cancer')
dm.get_metadata()
dm.get_ml_data("analysis_processes_clear")
print("~~~ Testing XMLExtractor ~~~")
import XMLExtractor
xe = XMLExtractor.XMLExtractor()
from bs4 import BeautifulSoup
bs = BeautifulSoup("<ImportantInformation><BestDinosaur>triceratops</BestDinosaur><BestCountry>Ireland</BestCountry></ImportantInformation>",'lxml')
