def load_db(self, check=True, fix=False, save_to_file=False): self.log('Opening {0} database(s)'.format(len(self.db_files)), logging.INFO) for db_name, db_file in self.db_files.items(): _db = Base(db_file, save_to_file=save_to_file) _db.open() self.log('Database {0} opened, records #: {1}'.format(db_name, len(_db)), logging.DEBUG) self.db.update({db_name: _db}) _db.create_index('uid') _db.create_index('type') if check is True: self.check_db(fix)
def load_db(self, check=True, fix=False, save_to_file=False): self.log('Opening {0} database(s)'.format(len(self.db_files)), logging.INFO) for db_name, db_file in self.db_files.items(): _db = Base(db_file, save_to_file=save_to_file) _db.open() self.log('Database {0} opened, records #: {1}'.format(db_name, len(_db)), logging.DEBUG) self.db.update({db_name: _db}) _db.create_index('uid') _db.create_index('type') self.log("Db {0}: printing simple strawman prediction accuracy for answers with max upvotes as best answer:".format(db_name), logging.INFO) self._strawman(_db) if check is True: self.check_db(fix)
def pydblite(): from pydblite.pydblite import Base db = Base('dummy', save_to_file=False) # create new base with field names db.create('name', 'age', 'size') # insert new record db.insert(name='homer', age=23, size=1.84) # records are dictionaries with a unique integer key __id__ # simple selection by field value records = db(name="homer") # complex selection by list comprehension res = [r for r in db if 30 > r['age'] >= 18 and r['size'] < 2] print("res:", res) # delete a record or a list of records r = records[0] db.delete(r) list_of_records = [] r = db.insert(name='homer', age=23, size=1.84) list_of_records.append(db[r]) r = db.insert(name='marge', age=36, size=1.94) list_of_records.append(db[r]) # or generator expression for r in (r for r in db if r['name'] in ('homer', 'marge')): # print "record:", r pass db.delete(list_of_records) rec_id = db.insert(name='Bart', age=15, size=1.34) record = db[rec_id] # the record such that record['__id__'] == rec_id # delete a record by its id del db[rec_id] # create an index on a field db.create_index('age') # update rec_id = db.insert(name='Lisa', age=13, size=1.24) # direct access by id record = db[rec_id] db.update(record, age=24) # add and drop fields db.add_field('new_field', default=0) db.drop_field('name') # save changes on disk db.commit()
def pydblite(): from pydblite.pydblite import Base db = Base('dummy', save_to_file=False) # create new base with field names db.create('name', 'age', 'size') # insert new record db.insert(name='homer', age=23, size=1.84) # records are dictionaries with a unique integer key __id__ # simple selection by field value records = db(name="homer") # complex selection by list comprehension res = [r for r in db if 30 > r['age'] >= 18 and r['size'] < 2] print "res:", res # delete a record or a list of records r = records[0] db.delete(r) list_of_records = [] r = db.insert(name='homer', age=23, size=1.84) list_of_records.append(db[r]) r = db.insert(name='marge', age=36, size=1.94) list_of_records.append(db[r]) # or generator expression for r in (r for r in db if r['name'] in ('homer', 'marge')): # print "record:", r pass db.delete(list_of_records) rec_id = db.insert(name='Bart', age=15, size=1.34) record = db[rec_id] # the record such that record['__id__'] == rec_id # delete a record by its id del db[rec_id] # create an index on a field db.create_index('age') # update rec_id = db.insert(name='Lisa', age=13, size=1.24) # direct access by id record = db[rec_id] db.update(record, age=24) # add and drop fields db.add_field('new_field', default=0) db.drop_field('name') # save changes on disk db.commit()
def load_db(self, check=True, fix=False, save_to_file=False): self.log('Opening {0} database(s)'.format(len(self.db_files)), logging.INFO) for db_name, db_file in self.db_files.items(): _db = Base(db_file, save_to_file=save_to_file) _db.open() self.log( 'Database {0} opened, records #: {1}'.format( db_name, len(_db)), logging.DEBUG) self.db.update({db_name: _db}) _db.create_index('uid') _db.create_index('type') if check is True: self.check_db(fix)
def load_db(self, check=True, fix=False, save_to_file=False): self.log('Opening {0} database(s)'.format(len(self.db_files)), logging.INFO) for db_name, db_file in self.db_files.items(): _db = Base(db_file, save_to_file=save_to_file) _db.open() self.log( 'Database {0} opened, records #: {1}'.format( db_name, len(_db)), logging.DEBUG) self.db.update({db_name: _db}) _db.create_index('uid') _db.create_index('type') self.log( "Db {0}: printing simple strawman prediction accuracy for answers with max upvotes as best answer:" .format(db_name), logging.INFO) self._strawman(_db) if check is True: self.check_db(fix)
class MemoryDict(object): def __init__(self, name="", key="", value=['col']): self.db = Base(name, save_to_file=False) self.db.create(key, *value) self.db.create_index(key) self.key = key self.value = value def give_me_elem(self, key): return eval('self.db._' + self.key + '[key]') def is_in(self, key): return len(self.give_me_elem(key)) > 0 def insert(self, key="", value=[""]): record = self.give_me_elem(key) if len(record) > 0: for i in range(len(self.value)): record[0][self.value[i]] = value[i] else: self.db.insert(key, *value) def pop(self, key): record = self.give_me_elem(key) if len(record) > 0: rec_id = record[0]['__id__'] del self.db[rec_id] def iteritems(self): return list(self.db) def len(self): return len(self.db) def print_all(self): for r in self.db: print(r)
class PathSegmentDB(object): """Simple database for paths using PyDBLite""" def __init__(self, segment_ttl=None, max_res_no=None, labels=None): # pragma: no cover """ :param int segment_ttl: The TTL for each record in the database (in s) or None to just use the segment's expiration time. :param int max_res_no: Number of results returned for a query. :param dict labels: Labels added to the exported metrics. The following labels are supported: - server_id: A unique identifier of the server that is exporting - isd_as: The ISD_AS of where the server is running - type: A generic label for the type of the revocations. """ self._db = None self._lock = threading.Lock() self._segment_ttl = segment_ttl self._max_res_no = max_res_no self._labels = labels if self._labels: self._init_metrics() self._setup_db() def _init_metrics(self): # pragma: no cover SEGS_TOTAL.labels(**self._labels).set(0) SEGS_BYTES.labels(**self._labels).set(0) SEGS_ADDED.labels(**self._labels).inc(0) SEGS_REMOVED.labels(**self._labels).inc(0) def _setup_db(self): # pragma: no cover with self._lock: self._db = Base("", save_to_file=False) self._db.create('record', 'id', 'first_isd', 'first_as', 'last_isd', 'last_as', 'sibra', mode='override') self._db.create_index('id') self._db.create_index('last_isd') self._db.create_index('last_as') def __getitem__(self, seg_id): # pragma: no cover """Return a path object by segment id.""" with self._lock: recs = self._db(id=seg_id) if recs: return recs[0]['record'].pcb return None def __contains__(self, seg_id): # pragma: no cover with self._lock: recs = self._db(id=seg_id) return len(recs) > 0 def flush(self): # pragma: no cover """Removes all records from the database.""" if self._labels: SEGS_REMOVED.labels(**self._labels).inc(len(self)) SEGS_TOTAL.labels(**self._labels).set(0) SEGS_BYTES.labels(**self._labels).set(0) self._setup_db() def update(self, pcb, reverse=False): """ Insert path into database. Return the result of the operation. """ first_ia = pcb.first_ia() last_ia = pcb.last_ia() if reverse: first_ia, last_ia = last_ia, first_ia if self._segment_ttl: now = int(SCIONTime.get_time()) record = PathSegmentDBRecord(pcb, now + self._segment_ttl) else: record = PathSegmentDBRecord(pcb) with self._lock: recs = self._db(id=record.id, sibra=pcb.is_sibra()) assert len(recs) <= 1, "PathDB contains > 1 path with the same ID" if not recs: self._db.insert(record, record.id, first_ia[0], first_ia[1], last_ia[0], last_ia[1], pcb.is_sibra()) logging.debug("Added segment from %s to %s: %s", first_ia, last_ia, pcb.short_desc()) if self._labels: SEGS_ADDED.labels(**self._labels).inc() SEGS_TOTAL.labels(**self._labels).inc() SEGS_BYTES.labels(**self._labels).inc(len(pcb)) return DBResult.ENTRY_ADDED cur_rec = recs[0]['record'] if pcb.get_expiration_time() < cur_rec.pcb.get_expiration_time(): return DBResult.NONE old_pcb = cur_rec.pcb cur_rec.pcb = pcb if self._segment_ttl: cur_rec.exp_time = now + self._segment_ttl else: cur_rec.exp_time = pcb.get_expiration_time() if self._labels: SEGS_ADDED.labels(**self._labels).inc() SEGS_BYTES.labels(**self._labels).inc(len(pcb) - len(old_pcb)) return DBResult.ENTRY_UPDATED def delete(self, segment_id): """Deletes a path segment with a given ID.""" with self._lock: recs = self._db(id=segment_id) if not recs: return DBResult.NONE self._db.delete(recs) assert len(recs) == 1 if self._labels: SEGS_REMOVED.labels(**self._labels).inc() SEGS_TOTAL.labels(**self._labels).dec() SEGS_BYTES.labels(**self._labels).dec( len(recs[0]['record'].pcb)) return DBResult.ENTRY_DELETED def delete_all(self, segment_ids): """ Deletes paths with the given IDs and returns the number of deletions. :param list segment_ids: The segment IDs to remove. :returns: The number of deletions. :rtype: int """ deletions = 0 for seg_id in segment_ids: if self.delete(seg_id) == DBResult.ENTRY_DELETED: deletions += 1 return deletions def __call__(self, *args, full=False, **kwargs): """ Selection by field values. Returns a sorted (path fidelity) list of paths according to the criterias specified. :param bool full: Return list of results not bounded by self._max_res_no. """ kwargs = self._parse_call_kwargs(kwargs) with self._lock: recs = self._db(*args, **kwargs) valid_recs = self._exp_call_records(recs) return self._sort_call_pcbs(full, valid_recs) def _parse_call_kwargs(self, kwargs): # pragma: no cover first_ia = kwargs.pop("first_ia", None) if first_ia: kwargs["first_isd"] = first_ia[0] kwargs["first_as"] = first_ia[1] last_ia = kwargs.pop("last_ia", None) if last_ia: kwargs["last_isd"] = last_ia[0] kwargs["last_as"] = last_ia[1] if "sibra" not in kwargs: kwargs["sibra"] = False return kwargs def _exp_call_records(self, recs): """Remove expired segments from the db.""" now = int(SCIONTime.get_time()) ret = [] expired = [] for r in recs: if r['record'].exp_time < now: expired.append(r) logging.debug("Path-Segment expired: %s", r['record'].pcb.short_desc()) continue ret.append(r) if expired: self._db.delete(expired) return ret def _sort_call_pcbs(self, full, valid_recs): # pragma: no cover seg_recs = sorted([r['record'] for r in valid_recs], key=lambda x: x.fidelity) if self._max_res_no and not full: seg_recs = seg_recs[:self._max_res_no] return [r.pcb for r in seg_recs] def __len__(self): # pragma: no cover with self._lock: return len(self._db)
'day_short_name', 'month_week_num', 'month_week_begin_date', 'month_week_end_date', 'quarter_week_num', 'quarter_week_begin_date', 'quarter_week_end_date', 'year_week_num', 'year_week_begin_date', 'year_week_end_date', 'month_day_num', 'month_num', 'month_name', 'month_short_name', 'month_begin_date', 'month_end_date', 'quarter_day_num', 'quarter_num', 'quarter_name', 'quarter_begin_date', 'quarter_end_date', 'year_day_num', 'year_num', 'year_begin_date', 'year_end_date', 'dd_mon_yyyy', 'dd_month_yyyy', 'mon_dd_yyyy', 'month_dd_yyyy', 'dd_mm_yyyy', 'mm_dd_yyyy', 'mm_dd_yy', 'dd_mm_yy', 'm_d_yy', 'd_m_yy', 'weekday_flag', 'week_first_day_flag', 'week_last_day_flag', 'month_first_day_flag', 'month_last_day_flag', 'quarter_first_day_flag', 'quarter_last_day_flag', 'year_first_day_flag', 'year_last_day_flag', 'leap_year_flag', 'is_holiday', 'holiday_name', 'nth_weekday') # create index date_table.create_index('date', 'julian_date_num', 'sequence', 'nth_weekday') # validate date input. while True: # input start date string start_date_string = input("Enter start date with format mm/dd/yyyy: ") try: valid_date = time.strptime(start_date_string, '%m/%d/%Y') # convert date string to date object. start_date = dateparser.parse(start_date_string) break except ValueError: print('Invalid date!.Please enter valid date') continue
def create_db(): db = Base('dummy', save_to_file=False) db.create('sid', 'channel', 'pid') db.create_index('sid') db.create_index('channel') return db
'dd_mm_yy', 'm_d_yy', 'd_m_yy', 'weekday_flag', 'week_first_day_flag', 'week_last_day_flag', 'month_first_day_flag', 'month_last_day_flag', 'quarter_first_day_flag', 'quarter_last_day_flag', 'year_first_day_flag', 'year_last_day_flag', 'leap_year_flag' ) # create index date_table.create_index('date','julian_date_num','sequence') # validate date input. while True: # imput start date string start_date_string = input("Enter start date with format mm/dd/yyyy: ") try: valid_date = time.strptime(start_date_string, '%m/%d/%Y') # convert date string to date object. start_date = dateparser.parse(start_date_string) break except ValueError: print('Invalid date!.Please enter valid date') continue