def __init__(self, config_path, cla_dict): trulia_conf = TruliaConfLoader.TruliaConfLoader(config_path) self.load_trulia_params(trulia_conf) self.db_mgr = DatabaseManager.DatabaseManager(config_path) kv_store = cla_dict['kv_store'] if kv_store == '': self.kv_mgr = None elif kv_store == 'h': print "loading HBase manager", import HBaseManager self.kv_mgr = HBaseManager.HBaseManager() print "completed" elif kv_store == 'r': print "loading Redis manager", import RedisManager self.kv_mgr = RedisManager.RedisManager() print "completed" fluentd_rx = cla_dict['fluentd_rx'] if fluentd_rx == '': self.fluentd_enabled = False print "FluentD not enabled" elif fluentd_rx == 'f': print "loading fluentd for local fs" sender.setup('fs') self.fluentd_enabled = True print "FluentD enabled for local filesystem" elif fluentd_rx == 'h': print "loading fluentd for hdfs" sender.setup('hdfs') self.fluentd_enabled = True print "FluentD enabled for HDFS"
def run(self): self.__database_manager = DatabaseManager.DatabaseManager() self.__scanner_data_provider = ScannerDataProvider.ScannerDataProvider( ) self.__scanner_data_provider.start() self.__running_lock.acquire() self.__is_running = True self.__running_lock.release() self.__thread_timer = time.time() self.__data_timer = time.time() while True: if time.time() - self.__thread_timer > 1000.0 / 1000.0: self.__running_lock.acquire() condition = self.__is_running self.__running_lock.release() if bool(condition) is False: break self.__thread_timer = time.time() if time.time() - self.__data_timer > 100.0 / 1000.0: dict_scanner_data = self.__scanner_data_provider.get_scanner_data( ) self.__store_in_db(dict_scanner_data) self.__store_in_queue(dict_scanner_data) self.__data_timer = time.time() # Wait for the threads to stop self.__scanner_data_provider.stop() self.__scanner_data_provider.join() print '[Data Manager] scanner_data_provider stopped '
def __init__(self, cogs, output_path, db_args, **args): super().__init__(**args) self.db_manager = DatabaseManager.DatabaseManager(db_args, output_path) self.connected = False self.output_path = output_path for cog in cogs: self.load_extension(cog)
def __init__(self, config_path): trulia_conf = TruliaConfLoader.TruliaConfLoader(config_path) self.load_trulia_params(trulia_conf) self.db_mgr = DatabaseManager.DatabaseManager(config_path) self.curr_key_idx = 0 # lock for threads to use to add to val_strs self.lock = threading.Lock() self.val_strs = list()
def dbConnect (self,db_name): db_username = gv.db_username db_password = gv.db_password db_host = gv.db_host #db_name = gv.name_database db_port = gv.db_port global databaseObject databaseObject = DatabaseManager(db_username, db_password,db_host,db_port, db_name) databaseObject.Connect()
def dbConnect(self): db_username = '******' db_password = '******' db_host = '127.0.0.1' db_name = gv.databaseName db_port = '3306' global databaseObject databaseObject = DatabaseManager(db_username, db_password, db_host, db_port, db_name) databaseObject.Connect()
def dbConnect (self): db_username = gv.userName db_password = gv.password db_host = gv.dbHost db_name = gv.databaseName db_port = gv.dbPort db_connector = gv.dbConnector global databaseObject databaseObject = DatabaseManager(db_connector, db_username, db_password,db_host,db_port, db_name) databaseObject.Connect()
def __init__(self): self.__active_user_list = [] self.__sys_data = ERPDate.ERPDate(1, 1) self.__raw_material_order_manager = RawMaterialOrderManager.RawMaterialOrderManager( ) self.__repository_manager = RepositoryManager.RepositoryManager() self.__database_manager = DatabaseManager.DatabaseManager() self.__request_msg = [] self.__fcn_dic = [] self.__init_system() self.test_main()
def start(self, strat, capital, commission): self.initDatabase() self.initPipelineTables() self.initStatisticsTables() self.initTradingTable() self.initBookTables() entryConditions, exitConditions = stratFuncs strat = stra.Strategy(stratName, entryConditions, exitConditions) self.databaseManager = dbm.DatabaseManager(self.dbRef, self.conn, strat, auth) self.databaseManager.setTradingParameters(self.exchange, self.ticker)
def CreateDatabaseManager(appID, result_p): expDBDirPath = os.environ["CommonProgramFiles"] + "\\VOCALOID5\\Explib" if(expDBDirPath == None): return None elif(expDBDirPath == ""): return None try: databaseManager = DatabaseManager.DatabaseManager(DatabaseManagerIF.VDM_createDatabaseManager(appID, expDBDirPath, result_p)) if(result_p.contents.value != VDMError.VDMError.NotAny): return None else: return databaseManager except: return None
def __collect_data(self): """ get data from MySQLdb server """ if bool(self.__db_connected) is False: self.__database_manager = DatabaseManager.DatabaseManager() self.__db_connected = True user_settings = \ self.__database_manager.get_data_from_database('HOME_SCANNER_USER_SETTINGS') if len(user_settings) > 0: if len(user_settings[0]) > 7: self.__temperature_threshold = int(user_settings[0][1]) self.__humidity_threshold = int(user_settings[0][2]) self.__gas_threshold = int(user_settings[0][3]) self.__video_enabled = int(user_settings[0][4]) self.__face_detection_enabled = int(user_settings[0][5]) self.__motion_detection_enabled = int(user_settings[0][6]) self.__human_detection_enabled = int(user_settings[0][7])
def upload_notifications(self, notifications_queue): """ Upload the notifications via DB MGR """ current_thread = threading.currentThread() __database_manager = DatabaseManager.DatabaseManager() __thread_timer = time.time() while getattr(current_thread, 'is_running', True): if time.time() - __thread_timer > 200.0 / 1000.0: __thread_timer = time.time() try: notifications = notifications_queue.get(False) except Queue.Empty: continue for elem in notifications: __database_manager.insert_data_in_database( elem, 'HOME_SCANNER_NOTIFICATIONS' ) notifications_queue.task_done()
def __init__(self, dbName): self.dbManager = DatabaseManager(dbName) self.initIPTables() self.deviceDict = dict() self.loadRulesFromDB()
import random import time import threading from DatabaseManager import * db = DatabaseManager(None, "db.csv") dict_lock = threading.Lock() users = "joe josh john matt mike charles charlotte maggie martha jackie jamie andy alex tom dick harry steven susan nancy".split( ) passwords = "lorem ipsum dolor sit amet consectetur adipiscing elit sed do eiusmod tempor incididunt ut labore et dolore magna aliqua".split( ) passkeys = {} global failures, globcount, END failures = 0 globcount = 0 END = False def dlock(): dict_lock.acquire() def dunlock(): dict_lock.release() def createsome(i):
table.put( hbase_base_key + '-' + key, { 'i:city': str(city), 'i:sc': str(state_code), 'i:lat:': str(lat), 'i:lon': str(lon) }) def get_dom_dict(dom): pass sys.path.append('../common/') import DatabaseManager dm = DatabaseManager.DatabaseManager('../conf/') import time conn = happybase.Connection('ip-172-31-11-76.us-west-1.compute.internal') conn.delete_table('city_stats_26june14', disable=True) conn.create_table('city_stats_26june14', {'cf': {}, 'i': {}}) table = conn.table('city_stats_26june14') #table = conn.table('city_stats') # TODO cleanup fluent_path = "../extern/fluent-logger-python" sys.path.append(fluent_path) from fluent import sender from fluent import event
def setupUI(self): layout = QtGui.QVBoxLayout() layout.setContentsMargins(20, 20, 20, 10) layout.addWidget(self.table) layout.addSpacing(50) bttnLayout = QtGui.QHBoxLayout() bttnLayout.addStretch() bttnLayout.addWidget(self.bttnBack) layout.addLayout(bttnLayout) self.setLayout(layout) def goBack(self): if self.__parent is not None: self.__parent.goBack() if __name__ == "__main__": DatabaseManager.db = DatabaseManager.DatabaseManager( host="sql12.freemysqlhosting.net", username="******", password="******", databaseName="sql12269310") app = QtGui.QApplication(sys.argv) w = ShowDesigationWidget() w.show() sys.exit(app.exec_())
from DatabaseManager import * from functions import * from SpecRatioFunctions import * try: opts, args = getopt.getopt(sys.argv, "n:", ["name="]) except getopt.GetoptError: print('main.py -n <pc name>') sys.exit(2) # Get the name of the pc/user and add to database pc_name = args[2] # Initialize connection with AWS RDS database db = DatabaseManager() # Add user to the pc table pc_id = db.add_pc_to_db(pc_name) # Get the runtime for each benchmark (benchmarks are sorted by their id in the database) print("The benchmarks are running, please wait...") ordered_runtimes = get_ordered_runtimes() # Get the reference runtime for each benchmark reference_runtimes = db.get_reference_times() if not reference_runtimes: exit(500) # Add the result for each benchmark in the results table spec_ratios = []
tweets_data_path = "data/" tweets_file_name = "twitter_data_sampled" tweets_file_name_original = tweets_file_name if database_selection == 1: tweets_file_name = input("Enter name of database(Default: '%s'):" % (tweets_file_name)) if tweets_file_name == "": tweets_file_name = tweets_file_name_original media_folder = tweets_data_path + tweets_file_name + "/media/" dc = DataCrawler.DataCrawler() dm = DatabaseManager.DatabaseManager(tweets_data_path, tweets_file_name, database_selection) if database_selection == 2: media_folder = "PostgreSQL/media/" table = input( "(1) url\n(2) media\n(3) icard\n(4) user url\n(5) user images\n\nselect a table to process (default: 1):" ) if table == None: table = 1 else: if table == "": table = 1 table = int(table) offset = input("Choose a offset (default: 0):") if offset == None:
def __init__(self, trulia_home_path, kv_store='r'): self.common_path = trulia_home_path + "/common/" self.config_path = trulia_home_path + "/conf/" self.debug = False sys.path.append(self.common_path) import RestCallHandler if kv_store == 'h': import HBaseManager self.kv_store_mgr = HBaseManager.HBaseManager() elif kv_store == 'r': import RedisManager self.kv_store_mgr = RedisManager.RedisManager() # RDBMS Manager import DatabaseManager self.database_manager = DatabaseManager.DatabaseManager( self.config_path) self.app = Flask(__name__) @self.app.route('/data/state/volume', methods=['GET']) def data_state_volume(): # get data state parameters to the right of ?q= as string from flask.request params = request.args.get('q', None) return RestCallHandler.handle_data_state_volume_query( self.kv_store_mgr, self.database_manager, params) @self.app.route('/data/state/average', methods=['GET']) def data_state_average(): # get data state parameters to the right of ?q= as string from flask.request params = request.args.get('q', None) return RestCallHandler.handle_data_state_average_query( self.kv_store_mgr, self.database_manager, params) @self.app.route('/data/county/volume', methods=['GET']) def data_county_volume(): # get data county parameters to the right of ?q= as string from flask.request params = request.args.get('q', None) return RestCallHandler.handle_data_county_volume_query( self.kv_store_mgr, self.database_manager, params) @self.app.route('/data/county/average', methods=['GET']) def data_county_average(): # get data county parameters to the right of ?q= as string from flask.request params = request.args.get('q', None) return RestCallHandler.handle_data_county_average_query( self.kv_store_mgr, self.database_manager, params) @self.app.route('/data/city/volume', methods=['GET']) def data_city_volume(): # get data city parameters to the right of ?q= as string from flask.request params = request.args.get('q', None) return RestCallHandler.handle_data_city_volume_query( self.kv_store_mgr, self.database_manager, params) @self.app.route('/data/city/average', methods=['GET']) def data_city_average(): # get data city parameters to the right of ?q= as string from flask.request params = request.args.get('q', None) return RestCallHandler.handle_data_city_average_query( self.kv_store_mgr, self.database_manager, params) @self.app.route('/data/zipcode/volume', methods=['GET']) def data_zipcode_volume(): # get data zipcode parameters to the right of ?q= as string from flask.request params = request.args.get('q', None) return RestCallHandler.handle_data_zipcode_volume_query( self.kv_store_mgr, self.database_manager, params) @self.app.route('/data/zipcode/average', methods=['GET']) def data_zipcode_average(): # get data zipcode parameters to the right of ?q= as string from flask.request params = request.args.get('q', None) return RestCallHandler.handle_data_zipcode_average_query( self.kv_store_mgr, self.database_manager, params)
def transfer_analysed_files(self, analysed_files_queue): """ transfer the analysed files to webserver location and delete the mp4 files """ current_thread = threading.currentThread() self.__database_manager = DatabaseManager.DatabaseManager() while getattr(current_thread, 'is_running', True): try: file_name = analysed_files_queue.get(False) except Queue.Empty: time.sleep(1) continue # delete the mp4 file clear_command = [] clear_command.append('rm') clear_command.append(file_name) clear_process = subprocess.Popen(clear_command) while clear_process.poll() is None: time.sleep(0.1) # move the avi file to website location file_name = os.path.splitext(file_name)[0] + '.h264' converted_file_name = os.path.splitext(file_name)[0] + '.mp4' convert_command = [] convert_command.append(MP4BOX) convert_command.append("-add") convert_command.append(file_name) convert_command.append(converted_file_name) convert_process = subprocess.Popen(convert_command) while convert_process.poll() is None: time.sleep(0.1) # delete the h264 file clear_command = [] clear_command.append('rm') clear_command.append(file_name) clear_process = subprocess.Popen(clear_command) while clear_process.poll() is None: time.sleep(0.1) move_command = [] move_command.append('mv') move_command.append(converted_file_name) move_command.append('/var/www/html/videos/') move_process = subprocess.Popen(move_command) while move_process.poll() is None: time.sleep(0.1) timest = time.time() timestamp = datetime.datetime.fromtimestamp(timest).strftime('%Y-%m-%d %H:%M:%S') db_file_record = [converted_file_name, timestamp] self.__database_manager.insert_data_in_database( db_file_record, 'HOME_SCANNER_VIDEO_FILES' ) analysed_files_queue.task_done()
#!/usr/bin/env python2 from pprint import pprint import DatabaseManager print("~~~ Testing DatabaseManager ~~~") dm = DatabaseManager.DatabaseManager() dm.get_data('author_ln') dm.get_data('reviewer') dm.get_matches("analysis_sw",1,"SAS") dm.get_matches("reviewer",0,6) dm.get_searches("analysis_sw",1,'SAS') dm.get_searches('article_doi',1,'10.1097') dm.get_searches('article_title',1,'cancer') dm.get_metadata() dm.get_ml_data("analysis_processes_clear") print("~~~ Testing XMLExtractor ~~~") import XMLExtractor xe = XMLExtractor.XMLExtractor() from bs4 import BeautifulSoup bs = BeautifulSoup("<ImportantInformation><BestDinosaur>triceratops</BestDinosaur><BestCountry>Ireland</BestCountry></ImportantInformation>",'lxml')