def init(): global DB_DOCKER_PORTS global SERVICE_INSTANCES_LIST try: # SERVICE_INSTANCES_LIST => "MEMORY DB" LOG.info( '[lifecycle.data.app.db] [init] Initializing SERVICE_INSTANCES_LIST ...' ) SERVICE_INSTANCES_LIST = [] # DB_DOCKER_PORTS: PORTS DATABASE for each of the Lifecycles / agents => "PHYSICAL DB" LOG.info( '[lifecycle.data.app.db] [init] Initializing DB_DOCKER_PORTS ...') DB_DOCKER_PORTS = Base(config.dic['LM_WORKING_DIR_VOLUME'] + config.dic['DB_DOCKER_PORTS'] ) #Base(config.dic['DB_DOCKER_PORTS']) # create new base with field names if not DB_DOCKER_PORTS.exists(): DB_DOCKER_PORTS.create('port', 'mapped_to') else: DB_DOCKER_PORTS.open() records = DB_DOCKER_PORTS() except: LOG.exception( '[lifecycle.data.app.db] [init] Exception: Error while initializing db components' )
def __init__(self): db = Base('rjdb.pdl') if db.exists(): db.open() else: db.create(*FIELDS) self.db = db
def init(): global DB_SHARING_MODEL global DB_USER_PROFILE try: # DB_SHARING_MODEL: LOG.info( "[usermgnt.data.standalone.db] [init] Initializing DB_SHARING_MODEL [" + config.dic['UM_WORKING_DIR_VOLUME'] + config.dic['DB_SHARING_MODEL'] + "] ...") DB_SHARING_MODEL = Base(config.dic['UM_WORKING_DIR_VOLUME'] + config.dic['DB_SHARING_MODEL']) if not DB_SHARING_MODEL.exists(): # create new base with field names DB_SHARING_MODEL.create('id', 'user_id', 'device_id', 'max_apps', 'battery_limit') else: DB_SHARING_MODEL.open() # DB_USER_PROFILE: LOG.info( "[usermgnt.data.standalone.db] [init] Initializing DB_USER_PROFILE [" + config.dic['UM_WORKING_DIR_VOLUME'] + config.dic['DB_USER_PROFILE'] + "] ...") DB_USER_PROFILE = Base(config.dic['UM_WORKING_DIR_VOLUME'] + config.dic['DB_USER_PROFILE']) if not DB_USER_PROFILE.exists(): # create new base with field names DB_USER_PROFILE.create('id', 'user_id', 'device_id', 'service_consumer', 'resource_contributor') else: DB_USER_PROFILE.open() except: LOG.exception( '[usermgnt.data.standalone.db] [init] Exception: Error while initializing db components' )
def state_extraction(): db = Base("scnscraper/abap.pydb") if db.exists(): db.open() record = db(type = "Question") print("# discussion scraped: " + str(record.__len__())) print("Answered: " + str(db(resolve = "Answered.").__len__())) print("Answered with solution: "+ str(db(resolve = "solution").__len__())) print("Not Answered: " + str(db(resolve = "Not Answered.").__len__())) print("Assumed Answered: " + str(db(resolve = "Assumed Answered.").__len__()))
def load_db(self, check=True, fix=False, save_to_file=False): self.log('Opening {0} database(s)'.format(len(self.db_files)), logging.INFO) for db_name, db_file in self.db_files.items(): _db = Base(db_file, save_to_file=save_to_file) _db.open() self.log('Database {0} opened, records #: {1}'.format(db_name, len(_db)), logging.DEBUG) self.db.update({db_name: _db}) _db.create_index('uid') _db.create_index('type') if check is True: self.check_db(fix)
def test_open_memory_with_existing_filename(self): self.filter_db = Base(test_db_name, save_to_file=True) self.filter_db.create('unique_id', 'name', "active", mode="override") self.filter_db.commit() db = Base(test_db_name, save_to_file=False) db.open() self.assertEqual(db.fields, ['unique_id', 'name', "active"]) db = Base(test_db_name, save_to_file=False) db.create('unique_id2', 'name2', "active2", mode="override") self.assertEqual(db.fields, ['unique_id2', 'name2', "active2"])
def state_extraction(): db = Base("scnscraper/abap.pydb") if db.exists(): db.open() record = db(type="Question") print("# discussion scraped: " + str(record.__len__())) print("Answered: " + str(db(resolve="Answered.").__len__())) print("Answered with solution: " + str(db(resolve="solution").__len__())) print("Not Answered: " + str(db(resolve="Not Answered.").__len__())) print("Assumed Answered: " + str(db(resolve="Assumed Answered.").__len__()))
def load_db(self, check=True, fix=False, save_to_file=False): self.log('Opening {0} database(s)'.format(len(self.db_files)), logging.INFO) for db_name, db_file in self.db_files.items(): _db = Base(db_file, save_to_file=save_to_file) _db.open() self.log('Database {0} opened, records #: {1}'.format(db_name, len(_db)), logging.DEBUG) self.db.update({db_name: _db}) _db.create_index('uid') _db.create_index('type') self.log("Db {0}: printing simple strawman prediction accuracy for answers with max upvotes as best answer:".format(db_name), logging.INFO) self._strawman(_db) if check is True: self.check_db(fix)
def load_db(self, check=True, fix=False, save_to_file=False): self.log('Opening {0} database(s)'.format(len(self.db_files)), logging.INFO) for db_name, db_file in self.db_files.items(): _db = Base(db_file, save_to_file=save_to_file) _db.open() self.log( 'Database {0} opened, records #: {1}'.format( db_name, len(_db)), logging.DEBUG) self.db.update({db_name: _db}) _db.create_index('uid') _db.create_index('type') if check is True: self.check_db(fix)
class ZipcodeDB(object): PYDBLITE_DB_FILE = './zipcode.db' def __init__(self, pydblite_db_file=None): if pydblite_db_file is None: pydblite_db_file = ZipcodeDB.PYDBLITE_DB_FILE try: self._db = Base(pydblite_db_file) self._db.open() except Exception: self._db = None def query_by_zip_code(self, zip_cd): logger.debug( "**************** entering ZipcodeDB.query_zipcode_db_by_zip_code") record = {} # Empty dict if not found if self._db is not None: records = self._db(zip_cd=zip_cd) if len(records) == 1: record = records[0] return record def get_timezone_for_zip_code(self, zip_code): logger.debug("**************** entering ZipcodeDB.get_timezone") ret_val = 'NoTZ/' + zip_code data = self.query_by_zip_code(zip_code) if data: # timezone is really just an offset timezone = data['data']['timezone'] dst = data['data']['dst'] tz_dic = { '-5+1': 'US/Eastern', '-5+0': 'US/East-Indiana', '-6+1': 'US/Central', '-7+1': 'US/Mountain', '-7+0': 'US/Arizona', '-8+1': 'US/Pacific', '-9+1': 'US/Alaska', '-10+0': 'US/Hawaii', '-10+1': 'US/Aleutian' } key = timezone + '+' + dst if key in tz_dic: ret_val = tz_dic[key] return ret_val
def init(): global DB_LM_SERVICE_INSTANCES try: # DB_LM: LM DATABASE ("PHYSICAL DB") LOG.info('[lifecycle.data.app.lm_db] [init] Initializing DB_LM ...') DB_LM_SERVICE_INSTANCES = Base(config.dic['LM_WORKING_DIR_VOLUME'] + config.dic['DB_STANDALONE_MODE'] + "_service_instances") # create new base with field names if not DB_LM_SERVICE_INSTANCES.exists(): DB_LM_SERVICE_INSTANCES.create('id', 'service_instance') else: DB_LM_SERVICE_INSTANCES.open() except: LOG.exception( '[lifecycle.data.app.lm_db] [init] Exception: Error while initializing db components' )
def load_db(self, check=True, fix=False, save_to_file=False): self.log('Opening {0} database(s)'.format(len(self.db_files)), logging.INFO) for db_name, db_file in self.db_files.items(): _db = Base(db_file, save_to_file=save_to_file) _db.open() self.log( 'Database {0} opened, records #: {1}'.format( db_name, len(_db)), logging.DEBUG) self.db.update({db_name: _db}) _db.create_index('uid') _db.create_index('type') self.log( "Db {0}: printing simple strawman prediction accuracy for answers with max upvotes as best answer:" .format(db_name), logging.INFO) self._strawman(_db) if check is True: self.check_db(fix)
try: if state: state_name = instances[state] else: state_name = None break except KeyError: print("State {} is not available!".format(state)) # calculate end date. end_date = start_date + relativedelta(years=int(num_of_year)) # day number between start and end date. day_num = (end_date - start_date).days # open table date_table = date_table.open() # iterate date and process data. for item in range(day_num): date = (start_date + timedelta(days=item)).date() # check date is exists or not.If exists then continue. if len(date_table(date=date)) > 0: continue sequence = len(date_table) + 1 week_day_num = date.weekday() + 1 day_name = date.strftime("%A") day_short_name = date.strftime("%a") month_day_num = date.day month_name = date.strftime("%B") month_short_name = date.strftime("%b") year_num = date.year month_num = date.month
class DataStoring(): #Inizialize an instantiated object by opening json file and the database def __init__(self): self.out_file = open("scnscraper/abap.json", "a") self.out_file.close() self.db = Base("scnscraper/abap.pydb") if self.db.exists(): self.db.open() else: self.db.create('url', 'uid', 'type', 'author', 'title', 'date_time', 'tags', 'views', 'answers', 'resolve', 'upvotes', 'text') #for each thread scraped, insert it into db def insert_items_into_db(self, threads): for thread in threads: item = SapItem() # New Item instance item = thread try: # Insert into db self.db.insert(url = str(item["url"]), uid = str(item["uid"]), type= str(item["type"] ), author=str(item["author"]), title = str(item["title"]), date_time = str(item["date_time"] ),tags = str(item["tags"] ), views = str(item["views"] ), answers = str(item["answers"] ), resolve = str(item["resolve"] ), upvotes = str(item["upvotes"] ), text = str(item["text"])) except UnicodeEncodeError: print("Unicode Encode Exception!") #save changes on disk self.db.commit() # for each thread scraped, initialize the string to insert into json file def threads_to_str(self, threads): out_string = "[ " if threads.__len__() == 0: return "" for thread in threads: item = SapItem() item = thread try: out_string += "{ url: '" + str(item["url"] ) + "', " + "uid: '" + str(item["uid"] ) + "', "\ "type: '" + str(item["type"] ) + "', "\ "author: '"+ str(item["author"]) + "', " \ "title: '"+ str(item["title"]) + "', "\ "date_time: '"+ str(item["date_time"] ) + "', " \ "tags: '"+ str(item["tags"] ) + "', " \ "views: '"+ str(item["views"] ) + "', "\ "answers: '"+ str(item["answers"] ) + "', " \ "resolve: '"+ str(item["resolve"] ) + "', " \ "upvotes: '"+ str(item["upvotes"] ) + "', "\ "text: '" + str(item["text"]) + "' }\n" except UnicodeEncodeError: print("Unicode Encode Exception!") out_string += " ]\n\n" return out_string #for each thread scraped, insert it into json file def insert_items_into_file(self, threads): try: self.out_file = open("scnscraper/abap.json", "a") # open in append mode #convert into string and insert into file self.out_file.write(self.threads_to_str(threads)) self.out_file.close() except: print('Exception in writing file') self.out_file.close() # read the web page index def read_index_from_file(self): if os.path.exists('scnscraper/index.txt'): with open('scnscraper/index.txt') as f: index = int(f.readline()) f.close() else: f = open('scnscraper/index.txt', 'w') index = 2 f.write(str(index)) f.close() return index # Write the web page index def write_index_into_file(self, i): f = open('scnscraper/index.txt', 'w') f.write(str(i)) f.close() # Convert the content of json file into a new db def from_json_to_db(self): thread = '' db = Base("scnscraper/abap.pydb", save_to_file= True) # create new base with field names db.create('url', 'uid', 'type', 'author', 'title', 'date_time', 'tags', 'views', 'answers', 'resolve', 'upvotes', 'text', mode='override') i=0 with open('scnsraper/threads.json', 'r') as file: for line in file: if(line.endswith(" }\n")): thread += line tokens = re.search(r"url:\s'(.*?)',\suid:\s'(.*?)',\stype:\s'(.*?)',\sauthor:\s'(.*?)',\stitle:\s'(.*?)',\sdate_time:\s'(.*?)',\stags:\s'(.*?)',\sviews:\s'(.*?)',\sanswers:\s'(.*?)',\sresolve:\s'(.*?)',\supvotes:\s'(.*?)', text:\s'((.|\n)*)'\s}", str(thread)) if tokens is not None: db.insert(url = tokens.group(1), uid = tokens.group(2), type= tokens.group(3), author=tokens.group(4), title = tokens.group(5), date_time = tokens.group(6), tags = tokens.group(7), views = tokens.group(8), answers = tokens.group(9), resolve = tokens.group(10), upvotes = tokens.group(11), text = tokens.group(12)) db.commit() print ('\n--------------------------------------------\n') thread = '' if(line.startswith(" ]")): print("new page") thread = '' if(line.endswith('\n') and (not line.startswith(" ]\n\n")) and (not line.endswith(" }\n"))): thread += line def state_extraction(): db = Base("scnscraper/abap.pydb") if db.exists(): db.open() record = db(type = "Question") print("# discussion scraped: " + str(record.__len__())) print("Answered: " + str(db(resolve = "Answered.").__len__())) print("Answered with solution: "+ str(db(resolve = "solution").__len__())) print("Not Answered: " + str(db(resolve = "Not Answered.").__len__())) print("Assumed Answered: " + str(db(resolve = "Assumed Answered.").__len__())) state_extraction = staticmethod(state_extraction)
class DataStoring(): #Inizialize an instantiated object by opening json file and the database def __init__(self): self.out_file = open("scnscraper/abap.json", "a") self.out_file.close() self.db = Base("scnscraper/abap.pydb") if self.db.exists(): self.db.open() else: self.db.create('url', 'uid', 'type', 'author', 'title', 'date_time', 'tags', 'views', 'answers', 'resolve', 'upvotes', 'text') #for each thread scraped, insert it into db def insert_items_into_db(self, threads): for thread in threads: item = SapItem() # New Item instance item = thread try: # Insert into db self.db.insert(url=str(item["url"]), uid=str(item["uid"]), type=str(item["type"]), author=str(item["author"]), title=str(item["title"]), date_time=str(item["date_time"]), tags=str(item["tags"]), views=str(item["views"]), answers=str(item["answers"]), resolve=str(item["resolve"]), upvotes=str(item["upvotes"]), text=str(item["text"])) except UnicodeEncodeError: print("Unicode Encode Exception!") #save changes on disk self.db.commit() # for each thread scraped, initialize the string to insert into json file def threads_to_str(self, threads): out_string = "[ " if threads.__len__() == 0: return "" for thread in threads: item = SapItem() item = thread try: out_string += "{ url: '" + str(item["url"] ) + "', " + "uid: '" + str(item["uid"] ) + "', "\ "type: '" + str(item["type"] ) + "', "\ "author: '"+ str(item["author"]) + "', " \ "title: '"+ str(item["title"]) + "', "\ "date_time: '"+ str(item["date_time"] ) + "', " \ "tags: '"+ str(item["tags"] ) + "', " \ "views: '"+ str(item["views"] ) + "', "\ "answers: '"+ str(item["answers"] ) + "', " \ "resolve: '"+ str(item["resolve"] ) + "', " \ "upvotes: '"+ str(item["upvotes"] ) + "', "\ "text: '" + str(item["text"]) + "' }\n" except UnicodeEncodeError: print("Unicode Encode Exception!") out_string += " ]\n\n" return out_string #for each thread scraped, insert it into json file def insert_items_into_file(self, threads): try: self.out_file = open("scnscraper/abap.json", "a") # open in append mode #convert into string and insert into file self.out_file.write(self.threads_to_str(threads)) self.out_file.close() except: print('Exception in writing file') self.out_file.close() # read the web page index def read_index_from_file(self): if os.path.exists('scnscraper/index.txt'): with open('scnscraper/index.txt') as f: index = int(f.readline()) f.close() else: f = open('scnscraper/index.txt', 'w') index = 2 f.write(str(index)) f.close() return index # Write the web page index def write_index_into_file(self, i): f = open('scnscraper/index.txt', 'w') f.write(str(i)) f.close() # Convert the content of json file into a new db def from_json_to_db(self): thread = '' db = Base("scnscraper/abap.pydb", save_to_file=True) # create new base with field names db.create('url', 'uid', 'type', 'author', 'title', 'date_time', 'tags', 'views', 'answers', 'resolve', 'upvotes', 'text', mode='override') i = 0 with open('scnsraper/threads.json', 'r') as file: for line in file: if (line.endswith(" }\n")): thread += line tokens = re.search( r"url:\s'(.*?)',\suid:\s'(.*?)',\stype:\s'(.*?)',\sauthor:\s'(.*?)',\stitle:\s'(.*?)',\sdate_time:\s'(.*?)',\stags:\s'(.*?)',\sviews:\s'(.*?)',\sanswers:\s'(.*?)',\sresolve:\s'(.*?)',\supvotes:\s'(.*?)', text:\s'((.|\n)*)'\s}", str(thread)) if tokens is not None: db.insert(url=tokens.group(1), uid=tokens.group(2), type=tokens.group(3), author=tokens.group(4), title=tokens.group(5), date_time=tokens.group(6), tags=tokens.group(7), views=tokens.group(8), answers=tokens.group(9), resolve=tokens.group(10), upvotes=tokens.group(11), text=tokens.group(12)) db.commit() print('\n--------------------------------------------\n') thread = '' if (line.startswith(" ]")): print("new page") thread = '' if (line.endswith('\n') and (not line.startswith(" ]\n\n")) and (not line.endswith(" }\n"))): thread += line def state_extraction(): db = Base("scnscraper/abap.pydb") if db.exists(): db.open() record = db(type="Question") print("# discussion scraped: " + str(record.__len__())) print("Answered: " + str(db(resolve="Answered.").__len__())) print("Answered with solution: " + str(db(resolve="solution").__len__())) print("Not Answered: " + str(db(resolve="Not Answered.").__len__())) print("Assumed Answered: " + str(db(resolve="Assumed Answered.").__len__())) state_extraction = staticmethod(state_extraction)
from readinstrument import MyTrade import persist_last_value api_key = os.getenv("API_KEY") token = os.getenv("PUB_TOKEN") user = os.getenv("USER_ID") # Initialise. # kws = WebSocket("your_api_key", "your_public_token", "logged_in_user_id") # kws = WebSocket(api_key, token, user) kws = WebSocket("lzxojcmp16le5ep8", "9dd813876abe0c06dfca066221f5d1da", "DD1846") # Initialize DB. db = Base(db_name, sqlite_compat=True) if db.exists(): db.open() else: db.create('time', 'instrument_token', 'last_price', 'mode', 'tradeable') # Save Initial Time now = datetime.datetime.now() tmp_filename = PREVIOUS_TIME+str(int(time.time())) persist_last_value.save_object(tmp_filename, now) factor = 1 minutes = 5 trade = MyTrade(fac=factor, c_min=minutes)