class TinydbStorage(interface.StorageInterface):
def __init__(self, db_path):
super(TinydbStorage, self).__init__()
self.db = TinyDB(db_path)
def create_table(self, table_name):
self._assert_no_table(table_name)
table = self.db.table(table_name)
table.insert({'table_exists': True})
def get_tables(self):
return self.db.tables()
def get(self, table_name, key):
self._assert_table(table_name)
table = self.db.table(table_name)
element = Query()
result = table.search(element.key == key)
if len(result) == 0:
raise interface.DataException("Key {} not found in table".format(key))
return result[0]['val']
def set(self, table_name, key, val):
self._assert_table(table_name)
table = self.db.table(table_name)
element = Query()
table.remove(element.key == key)
table.insert({'key': key, 'val': val})
def append(self, table_name, list_key, val):
self._assert_table(table_name)
table = self.db.table(table_name)
l = self.get(table_name, list_key)
#TODO: check if l is a list maybe
if val in l:
raise interface.DataException("Value: {0} already exists in list: {1}".format(val, list_key))
l.append(val)
element = Query()
table.update({'val': l}, element.key == list_key)
def remove(self, table_name, list_key, val):
#TODO: do
raise NotImplementedError("Storage module must implement this")
def _assert_no_table(self, table_name):
table = self.db.table(table_name)
if len(table) > 0:
raise interface.DataException("Table already exists: {}".format(table_name))
def _assert_table(self, table_name):
table = self.db.table(table_name)
if len(table) == 0:
raise interface.DataException("Table does not exist: {}".format(table_name))
def testBasicOperation():
def addone(x):
x['int'] += 1
default_db = TinyDB('./db_location/default.json')
real_table = default_db.table('real')
print(f"{'*'*20}打开了数据库{default_db.name}{'*'*20}")
for i in range(7):
default_db.insert({"int": i + 1, "char": chr(65 + i)})
print("对每一个元素进行打印操作：")
print("default_db 中每一个int字段加1")
default_db.update(addone)
print("对每一个元素进行打印操作：")
print("default_db中的所有表段为：", default_db.tables())
print("default_db中所有的数据为：", default_db.all())
default_db.truncate()
print(f"{'*'*20}清除了所有表{'*'*20}")
print("db中所有的表段为：", default_db.tables())
print("default_db中所有的数据为：", default_db.all())
print(f"{'*'*20}关闭了表{default_db.name}{'*'*20}")
default_db.close()
def api_list(campus):
"""
`/list` with [GET] handles a single request to list department or
course keys from the database
It takes an optional query parameter `dept` which is first checked
for existence and then returns the dept keys.
However, if `course` is also selected, it will return only the data
of that course within the department.
:param campus: (str) The campus to retrieve data from
:return: 200 - Found entry and returned keys successfully
to the user.
:return: 404 - Could not find entry
:return:
"""
if campus not in CAMPUS_LIST:
return 'Error! Could not find campus in database', 404
db = TinyDB(join(DB_ROOT, f'{CAMPUS_LIST[campus]}_database.json'))
raw = request.args
qp = {k: v.upper() for k, v in raw.items()}
if 'dept' not in qp:
return jsonify(', '.join(db.tables())), 200
qp_dept = qp['dept']
if qp_dept in db.tables():
table = db.table(f'{qp_dept}')
keys = set().union(*(d.keys() for d in table.all()))
return jsonify(', '.join(keys)), 200
return 'Error! Could not list', 404
def test_non_default_table():
db = TinyDB(storage=MemoryStorage)
assert [TinyDB.DEFAULT_TABLE] == list(db.tables())
db = TinyDB(storage=MemoryStorage, default_table='non-default')
assert set(['non-default']) == db.tables()
db.purge_tables()
TinyDB.DEFAULT_TABLE = 'non-default'
db = TinyDB(storage=MemoryStorage)
assert set(['non-default']) == db.tables()
def test_non_default_table():
db = TinyDB(storage=MemoryStorage)
assert [TinyDB.DEFAULT_TABLE] == list(db.tables())
db = TinyDB(storage=MemoryStorage, default_table='non-default')
assert set(['non-default']) == db.tables()
db.purge_tables()
TinyDB.DEFAULT_TABLE = 'non-default'
db = TinyDB(storage=MemoryStorage)
assert set(['non-default']) == db.tables()
def get_one(db: TinyDB, data: dict, filters: dict):
"""
This is a helper used by the `/get` route to extract course data.
It works for both [GET] and [POST] and fetches data from the database
:param db: (TinyDB) Database to retrieve data from
:param data: (dict) The query param or the POST body dict
:param filters: (dict) A optional dictionary of filters to be
passed to filter_courses()
:return: course: (dict) A singular course listing from the database
(if it passes filters)
"""
data_dept = data['dept']
if data_dept in db.tables():
table = db.table(f'{data_dept}')
entries = table.all()
if 'course' not in data:
return entries
data_course = data['course']
try:
course = next((e[f'{data_course}'] for e in entries
if f'{data_course}' in e))
if filters:
filter_courses(filters, course)
except StopIteration:
return dict()
return course if course else dict()
def calculate():
db = TinyDB('data/db_holding.json')
for cube_symbol in db.tables():
HoldingTable = db.table(cube_symbol)
for row in HoldingTable:
print(row)
def get_users(cls):
# get all users
db = TinyDB(config.user_file)
user_list = db.tables()
data = map(unicode.lower, user_list)
data.remove('_default')
return data
class DatabaseHelper(metaclass=Singleton):
def __init__(self):
self._tinydb = TinyDB("UserData.json")
#creates new table if it doesn't exist yet, or returns
#existing or cached table
def createTable(self, table_name):
return self._tinydb.table(table_name)
#remove table from database
def removeTable(self, name: str):
self._tinydb.purge_table(name)
#returns list of all table names in database
def getTables(self):
return self._tinydb.tables()
#insert new row values into a table, returns id
def insert(self, table_name: str, row: dict):
return self.createTable(table_name).insert(row)
#get a row of values given table name, and field-value search
def get(self, table_name: str, field: str, value):
return self.createTable(table_name).get(where(field) == value)
#update a field value based on table name
def update(self, table_name: str, searchField: str, searchValue,
updateField: str, updateValue):
table = self.createTable(table_name)
table.update({updateField: updateValue},
where(searchField) == searchValue)
def test_purge_table():
db = TinyDB(storage=MemoryStorage)
assert [TinyDB.DEFAULT_TABLE] == list(db.tables())
db.purge_table(TinyDB.DEFAULT_TABLE)
assert [] == list(db.tables())
table_name = 'some-other-table'
db = TinyDB(storage=MemoryStorage)
db.table(table_name)
assert set([TinyDB.DEFAULT_TABLE, table_name]) == db.tables()
db.purge_table(table_name)
assert set([TinyDB.DEFAULT_TABLE]) == db.tables()
db.purge_table('non-existent-table-name')
assert set([TinyDB.DEFAULT_TABLE]) == db.tables()
def test_purge_table():
db = TinyDB(storage=MemoryStorage)
assert [TinyDB.DEFAULT_TABLE] == list(db.tables())
db.purge_table(TinyDB.DEFAULT_TABLE)
assert [] == list(db.tables())
table_name = 'some-other-table'
db = TinyDB(storage=MemoryStorage)
db.table(table_name)
assert set([TinyDB.DEFAULT_TABLE, table_name]) == db.tables()
db.purge_table(table_name)
assert set([TinyDB.DEFAULT_TABLE]) == db.tables()
db.purge_table('non-existent-table-name')
assert set([TinyDB.DEFAULT_TABLE]) == db.tables()
def main():
if not exists(DB_DIR):
makedirs(DB_DIR, exist_ok=True)
if not exists(join(DB_DIR, 'html')):
makedirs(join(DB_DIR, 'html'), exist_ok=True)
codes = generate_term_codes()
print_c(f'Loaded {color(Fore.CYAN, len(codes))} term codes\n')
prefix = PREFIXES[0] if not DEBUG else PREFIXES[1]
if DEBUG:
codes = codes[:5]
print_c(f'Scraping session cookie…\r')
cookies = scrape_cookies()
print_c(f"Scraped session cookie {color(Fore.YELLOW, cookies['CPSESSID'])}\n{'-'*79}\n")
temp_path = join(DB_DIR, 'temp.json')
try:
for term in codes:
print_c(f" [{term}] [{color(Fore.YELLOW, 'MINING…')}] Scraping…\r")
temp = TinyDB(temp_path)
dept_data = mine_dept_data(term, write=False)
print_c(f" [{term}] [{color(Fore.YELLOW, 'MINING…')}] " +
f"Parsing {len(dept_data)} departments…\r")
failed = False
for idx, variant in enumerate(ADVANCED_FORM_DATA):
content = mine_table_data(term, variant, dept_data, cookies, write=False)
if advanced_parse(content, db=temp, term=term):
break
elif idx == len(ADVANCED_FORM_DATA) - 1:
failed = True
if rename(temp_path, join(DB_DIR, f'{prefix}_{term}_database.json')):
remove(temp_path)
db = TinyDB(join(DB_DIR, f'{prefix}_{term}_database.json'))
num_courses = sum([len(db.table(t).all()) for t in db.tables()])
if failed:
print_c(f" [{term}] [{color(Fore.RED, 'ERROR!!')}] Payload failed…\n")
else:
print_c(f" [{term}] [{color(Fore.GREEN, 'SUCCESS')}] Mined {num_courses} courses\n")
except (KeyboardInterrupt, TimeoutException) as e:
print_c(f"{color(Fore.GREEN, e)}\n")
kill_driver()
remove(temp_path)
finally:
kill_driver()
def restore(self):
database = self.application.db
bak = TinyDB(st.bak)
for x in database.tables():
if self.application.collection(x) == True:
database.purge_table(x)
if x in bak.tables():
table = database.table(x)
table.insert_multiple(bak.table(x).all())
def test_create_db(self):
database_path = "./test_data/test_db.json"
create_db_and_main_tables(database_path)
db = TinyDB(database_path)
tables = sorted(list(db.tables()))
self.assertEqual(tables[0], "Operation")
self.assertEqual(tables[1], "User")
def main():
if not exists(DB_ROOT):
makedirs(DB_ROOT, exist_ok=True)
content = urlopen(f'file://{abspath(SCHEDULE)}') if exists(SCHEDULE) else mine(write=True)
db = TinyDB(join(DB_ROOT, 'database.json'))
parse(content, db=db)
print(db.tables())
class Database(object):
def __init__(self, path):
self._path = path
self._db = TinyDB(storage=MemoryStorage)
self._json = open_database(path)
self._endpoints = self._generate_tables()
def _generate_tables(self):
"""Generate tables from database
Returns:
list: table names
"""
resources = [name for name in self._json]
for name in resources:
table = tinydb_make_table(self._db, name)
tinydb_populate_table(table, self._json[name])
return resources
@property
def json(self):
"""Return Dict Repr for the database
Returns:
dict: database as dict
"""
return self._json
@property
def endpoints(self):
"""Return a list of the table's name
Returns:
list: table name
"""
return self._endpoints
def resources(self):
"""Return a list of the table's name
Returns:
list: table name
"""
return [table for table in self._db.tables() if table != "_default"]
def resource(self, name):
"""Return the table resource
Returns:
Table: Table resource
"""
return self._db.table(name)
def test_purge_db(self):
database_path = "./test_data/test_db.json"
create_db_and_main_tables(database_path)
purge_db(database_path)
db = TinyDB(database_path)
tables = sorted(list(db.tables()))
print(tables)
self.assertEqual(tables[0], "_default")
self.assertEqual(len(tables), 1)
def main():
if not exists(DB_ROOT):
makedirs(DB_ROOT, exist_ok=True)
for term in TERM_CODES.values():
temp_path = join(DB_ROOT, 'temp.json')
temp = TinyDB(temp_path)
content = mine(term)
parse(content, db=temp)
rename(temp_path, join(DB_ROOT, f'{term}_database.json')) and remove(temp_path)
db = TinyDB(join(DB_ROOT, f'{term}_database.json'))
print(term, db.tables())
def update_tinydb(self, tinyrundb_json, run_json, rerunWhat):
db = TinyDB(tinyrundb_json, sort_keys=True, indent=4, separators=(',', ': '))
db.drop_table('_default')
query = Query()
runcases = json.loads(open(run_json).read())
for case in runcases:
if case['feature'] in db.tables():
feature_table = db.table(case['feature'])
if (rerunWhat is not None) and (len(feature_table.search(query.status == rerunWhat)) > 0):
feature_table.update({'status': 'rerun'})
else:
feature_table = db.table(case['feature'])
feature_table.insert(case)
db.close()
def _convert_to_sqlitedb(self):
old_db = TinyDB(DATABASE_SETTINGS['database'])
self.mem_db = []
total = 0
for table in old_db.tables():
for item in old_db.table(table).all():
total += 1
print "MIGRATING DATABASE"
print "--OLD DATABASE: " + str(total)
i = 0
for table in old_db.tables():
for item in old_db.table(table).all():
if len(item['accn']) < 15:
if int(item['time']) > self.purge_date:
self.mem_db.append(item)
print " Converted: " + str(i) + " of " + str(total)
i += 1
self.dict_db[str(item['time'])] = item
else:
print "REMOVING OLD THING"
print "DATABASE MIGRATION COMPLETE"
print "COMMITING CHANGES"
self.get_last_filed()
class TinyDBDriver:
def __init__(self):
self.db = TinyDB('covid.json')
self.table_1 = "testing_stats_1"
self.table_2 = "testing_stats_2"
self.primary_table = None
self.secondary_table = None
def get_db(self):
return self.db
def determine_table_names(self):
tables = self.db.tables()
print(">>>>>>>>>>>")
print(tables)
print(">>>>>>>>>>>")
response = {}
if self.table_2 in tables:
self.primary_table = self.table_2
self.secondary_table = self.table_1
else:
self.primary_table = self.table_1
self.secondary_table = self.table_2
def get_primary_table(self):
return self.primary_table
def write_to_tinydb(self, payloads):
self.determine_table_names()
print("Before writing records :: Primary table = " +
self.primary_table + ", Secondary table = " +
self.secondary_table)
print("Creating new table => " + self.secondary_table)
table = self.db.table(self.secondary_table)
table.insert_multiple(payloads)
print("Inserted " + str(len(payloads)) + " records into " +
self.secondary_table)
self.db.drop_table(self.primary_table)
print("Dropped older table =>" + self.primary_table)
temp_table = self.secondary_table
self.secondary_table = self.primary_table
self.primary_table = temp_table
print("After writing records :: Primary table = " +
self.primary_table + ", Secondary table = " +
self.secondary_table)
return {"table": self.primary_table, "count": len(table.all())}
class Application(tornado.web.Application):
def __init__(self):
self.db = TinyDB(st.json)
handlers = [(r'/', NaviHandler), (r'/login', LoginHandler),
(r'/logout', LogoutHandler), (r'/title', TitleHandler),
(r'/headline/api', HeadlineApi),
(r'/read/api/([a-zA-Z0-9_]+)/([0-9]+)', ArticleApi),
(r'/write/api/([a-zA-Z0-9_]+)', ArticleApi),
(r'/help', HelpHandler), (r'/master/*', MasterHandler),
(r'/alert', AlertHandler),
(r'/([a-zA-Z0-9_]+)', IndexHandler),
(r'/([a-zA-Z0-9_]+)/([0-9]+)/*', IndexHandler),
(r'/([a-zA-Z0-9_]+)/admin/([0-9]+)/*', AdminHandler),
(r'/([a-zA-Z0-9_]+)/admin/([a-z]+)/*', AdminConfHandler),
(r'/([a-zA-Z0-9_]+)/userdel', UserHandler),
(r'/([a-zA-Z0-9_]+)/search', SearchHandler),
(r'/([a-zA-Z0-9_]+)/regist', RegistHandler)]
settings = {
'template_path': os.path.join(os.path.dirname(__file__),
'templates'),
'static_path': os.path.join(os.path.dirname(__file__), 'static'),
'ui_modules': {
'Footer': FooterModule
},
'cookie_secret': 'bZJc2sWbQLKo6GkHn/VB9oXwQt8SOROkRvJ5/xJ89Eo=',
'xsrf_cookies': True,
'debug': True,
'login_url': '/login'
}
tornado.web.Application.__init__(self, handlers, **settings)
def gpos(self, dbname, page):
params = self.db.get(where('kinds') == 'conf')
pos = int(page)
if pos <= 0:
pos = 0
elif (pos - 1) * params['count'] >= len(self.db.table(dbname)):
pos = 0
return pos
def collection(self, name):
if name in self.db.tables():
return True
else:
return False
def main():
if not exists(DB_DIR):
makedirs(DB_DIR, exist_ok=True)
for term in CURRENT_TERM_CODES.values():
temp = TinyDB(storage=MemoryStorage)
content = mine(term)
parse(content, db=temp)
db = TinyDB(join(DB_DIR, f'{term}_database.json'))
db.storage.write(temp.storage.read())
depts = ', '.join(db.tables())
log_info(f'Scraped term {term}', pad=False, details={
'depts': depts,
})
db.close()
def generate():
'''
Generate a test database from the fetched HTML
'''
if not exists(DB_DIR):
makedirs(DB_DIR, exist_ok=True)
with open(f'{join(DB_DIR, SCHEDULE)}', "r") as file:
temp_path = join(DB_DIR, 'temp.json')
temp = TinyDB(temp_path)
content = file.read()
parse(content, db=temp)
if rename(temp_path, TEST_DB):
remove(temp_path)
db = TinyDB(TEST_DB)
print('Departments:', ', '.join(list(db.tables())))
class TinyMongoDatabase(object):
"""Representation of a Pymongo database"""
def __init__(self, database, foldername, storage):
"""Initialize a TinyDB file named as the db name in the given folder
"""
self._foldername = foldername
self.tinydb = TinyDB(os.path.join(foldername, database + u".json"),
storage=storage)
def __getattr__(self, name):
"""Gets a new or existing collection"""
return TinyMongoCollection(name, self)
def __getitem__(self, name):
"""Gets a new or existing collection"""
return TinyMongoCollection(name, self)
def collection_names(self):
"""Get a list of all the collection names in this database"""
return list(self.tinydb.tables())
def api_list_url(campus):
'''
`/urls` with [GET] returns a tree of all departments, their courses, and the courses' endpoints to hit.
:param campus: (str) The campus to retrieve data from
:return: 200 - Should always return
'''
if campus not in CAMPUS_LIST:
return 'Error! Could not find campus in database', 404
db = TinyDB(join(DB_ROOT, f'{CAMPUS_LIST[campus]}_database.json'))
data = defaultdict(list)
for dept in db.tables():
table = db.table(dept)
keys = set().union(*(d.keys() for d in table.all()))
data[f'{dept}'].append({k: generate_url(dept, k) for k in keys})
return jsonify(data), 200
class TinyMongoDatabase(object):
"""Representation of a Pymongo database"""
def __init__(self, database, foldername, storage):
"""Initialize a TinyDB file named as the db name in the given folder
"""
self._foldername = foldername
self.tinydb = TinyDB(
os.path.join(foldername, database + u".json"),
storage=storage
)
def __getattr__(self, name):
"""Gets a new or existing collection"""
return TinyMongoCollection(name, self)
def __getitem__(self, name):
"""Gets a new or existing collection"""
return TinyMongoCollection(name, self)
def collection_names(self):
"""Get a list of all the collection names in this database"""
return list(self.tinydb.tables())
def open_subject():
print("=" * 60)
print("hello user.")
if not os.path.exists('./data/'):
os.makedirs('./data/')
db = TinyDB("./data/database.json", indent=4)
available_tables = db.tables()
i = 0
subject_order = {}
print("choose one of your subjects:")
for subject in available_tables:
if subject != "_default" and subject != "":
print("(" + str(i) + ") " + subject)
subject_order[subject] = i
i += 1
subject_number = input(" => ")
for sub in subject_order:
if subject_order[sub] == int(subject_number):
subject = sub
table = db.table(subject.lower()).all()
db.close()
return subject
def generate_reports(self, dbfile):
'''
'''
# get run duration
t1 = self.runtime_stamp
t2 = self.end_time
stime = datetime(
int(t1[:4]), int(t1[4:6]), int(t1[6:8]), int(t1[9:11]),
int(t1[11:13]), int(t1[13:15]))
etime = datetime(
int(t2[:4]), int(t2[4:6]), int(t2[6:8]), int(t2[9:11]),
int(t2[11:13]), int(t2[13:15]))
run_duration = str(etime - stime)
print('Run Duration: {}'.format(run_duration))
# generate cucumber report json file
db = TinyDB(dbfile, sort_keys=True, indent=4, separators=(',', ': '))
db.drop_table('_default')
query = Query()
cucumber_report_json = []
for table in db.tables():
group = db.table(table)
reportList = group.search(query.status != 'crashed')
feature_report = None
for case in reportList:
if os.path.exists(case['result_json']): # process new result
element = json.loads(open(case['result_json']).read())[0]
os.rename(case['result_json'], case['result_json'] + '.processed')
elif os.path.exists(case['result_json'] + '.processed'): # process existing result
element = json.loads(open(case['result_json'] + '.processed').read())[0]
else:
group.update({'status': 'crashed'}, doc_ids=[case.doc_id])
if not feature_report:
feature_report = element
else:
feature_report['elements'].append(element['elements'][0])
def test_drop_table():
db = TinyDB(storage=MemoryStorage)
default_table_name = db.table(db.default_table_name).name
assert [] == list(db.tables())
db.insert({'a': 1})
assert [default_table_name] == list(db.tables())
db.drop_table(default_table_name)
assert [] == list(db.tables())
table_name = 'some-other-table'
db = TinyDB(storage=MemoryStorage)
db.table(table_name).insert({'a': 1})
assert {table_name} == db.tables()
db.drop_table(table_name)
assert set() == db.tables()
assert table_name not in db._tables
db.drop_table('non-existent-table-name')
assert set() == db.tables()
class CanStorage:
__data_base = TinyDB
__current_sequence_table = TinyDB.table
__current_sequence = None
__max_sequence = None
__ready_to_store = False
def __init__(self, a_file_path):
"""
Opens (or creates) a data base file that that the instance of a
CanStorage interacts with.
:param a_file_path:
Path and file name. Note: path _has_ to exist, if not the program will
exit non-gracefully.
:return:
N/A
"""
self.__data_base = TinyDB(a_file_path)
# Check if we have a current sequence stored in the filemajigger
sequence_table = self.__data_base.table('sequence_counter')
sequence_check = sequence_table.search(where('sequence'))
# If a previous sequence exist we increment the max by one
if sequence_check:
self.__max_sequence = max(sequence_check)['sequence']
# If this is the first entry set current sequence to 0
else:
self.__max_sequence = 0
def print_debug_info(self):
"""
Provides debug information about contents of data base.
:return:
N/A
"""
print self.__data_base.all()
print self.__data_base.tables()
def __init_storage(self):
"""
Initialises a new storage table. Increments the sequence counter, stores
it for future use and creates a new named table for the new sequence of
data to be stored.
:return:
N/A
"""
self.__current_sequence = self.__max_sequence + 1
# Store the current sequence to db for next time the file is opened
sequence_table = self.__data_base.table('sequence_counter')
sequence_table.insert({'sequence': self.__current_sequence})
# Create new table entry for this sequence
sequence_name = 'sequence' + str(self.__current_sequence)
self.__current_sequence_table = self.__data_base.table(sequence_name)
self.__ready_to_store = True
def store(self, a_dict_or_list_entry):
"""
Stores a data entry in the currently opened data base table. If the
storage is not initialised it will call the initialising function to
create a new table for the current sequence of data to be stored.
:param a_dict_or_list_entry:
Either a list containing several dictionary entries or a single
dictionary entry containing a 'data_id' filed.
:return:
N/A
"""
if not self.__ready_to_store:
self.__init_storage()
# Check if we're storing a list or a dictionary
if type(a_dict_or_list_entry) == list:
# Cycle through all dictionaries stored in list
for list_entry in a_dict_or_list_entry:
# Get and remove the key from the dict
data_key = list_entry['data_id']
list_entry.pop('data_id', 0)
# Store the passed dictionary with its key being the data_id
# field
self.__current_sequence_table.insert({data_key: list_entry})
elif type(a_dict_or_list_entry) == dict:
# Get and remove the key from the dict
data_key = a_dict_or_list_entry['data_id']
a_dict_or_list_entry.pop('data_id', 0)
# Store the passed dictionary with its key being the data_id field
self.__current_sequence_table.insert({data_key:
a_dict_or_list_entry})
else:
exit('CanParser.store() expects list or dict entries!')
def load(self, a_sequence_number, a_key):
"""
Provides access to the data stored for the specified sequence number and
the specified key ('data_id').
:param a_sequence_number:
The sequence number of interest.
:param a_key:
A 'data_id' key containing the data we are interested in retrieving.
:return:
data_list_for_key containing a list of dictionary objects.
Will return an empty list of the sequence number is invalid.
"""
data_list_for_key = list()
if a_sequence_number <= self.__max_sequence:
sequence_name = 'sequence' + str(a_sequence_number)
selected_table = self.__data_base.table(sequence_name)
data_list_for_key = selected_table.search(where(a_key))
return data_list_for_key
def get_max_sequence(self):
"""
Give a user the number of data sequences stored in the data base.
:return:
Number of sequences currently stored.
"""
return self.__max_sequence
def get_data_types(self, a_sequence_number):
"""
Returns all the data types that are stored in a given data sequence
entry.
:param a_sequence_number:
The data sequence the user is interested in retrieving a list of
different data entries for.
:return:
key_list containing the unique 'data_id's available in the specified
sequence number.
Will return an empty list of the sequence number is invalid.
"""
key_list = list()
# Only return for valid sequence numbers!
if a_sequence_number <= self.__max_sequence:
sequence_name = 'sequence' + str(a_sequence_number)
selected_table = self.__data_base.table(sequence_name)
all_items = selected_table.all()
for item in all_items:
if item.keys()[0] not in key_list:
key_list.append(item.keys()[0])
return key_list
class DatabaseAccess(object):
'''
Raw database access. This module might be unnecessary, it's just a simple
wrapper around TinyDB. It doesn't isolate the Query abstractions of TinyDB
so we don't get any portability or independence from TinyDB. But it is a
place to change storage, middleware, etc.
'''
def __init__(self, filename):
'''
Initialize a database with a file.
:param filename: database filename.
'''
self.filename = str(filename)
self.db = TinyDB(self.filename, storage=CachingMiddleware(JSONStorage))
def close(self):
'''Close the database file.'''
self.db.close()
def table(self, tableName):
return self.db.table(tableName)
# def insert(self, *args, **kwargs):
# '''Insert an object into the database.'''
# return self.db.insert(*args, **kwargs)
# def get(self, *args, **kwargs):
# '''Get an object from the database.'''
# return self.db.Get(*args, **kwargs)
# def contains(self, *args, **kwargs):
# '''Test an object from the database.'''
# return self.db.Get(*args, **kwargs)
# def search(self, *args, **kwargs):
# '''Query for objects.'''
# return self.db.search(*args, **kwargs)
# def remove(self, *args, **kwargs):
# '''Remove objects by query.'''
# return self.db.remove(*args, **kwargs)
def setSingleton(self, kind, model):
id = None
self.db.table(kind).purge()
id = self.db.table(kind).insert(model)
return id
def getSingleton(self, kind):
objs = self.db.table(kind).all()
if len(objs) == 0:
return None
return objs[0]
def getArgs(self):
return self.getSingleton('fashion.prime.args')
def isVerbose(self):
args = self.getArgs()
if args is None:
return False
return args["verbose"]
def isDebug(self):
args = self.getArgs()
if args is None:
return False
return args["debug"]
def kinds(self):
k = self.db.tables()
k.remove("_default")
return k
class dbs:
db = None
filename = None
db_name = 'data'
table = '_default'
new_data = False
def __init__(self, filename=None):
if filename:
self.filename = filename
def open(self, filename=''):
if filename:
self.filename = filename
try:
self.db = TinyDB(self.filename)
self.new_data = True
except:
self.db = None
self.filename = ''
def status(self):
if self.filename:
txt = '\nThe JSON data file is %s.\n\n' % self.filename
if self.db is not None:
if len(self.db.tables()) > 0:
txt += 'There are %s tables in this data file as follows:\n' % len(
self.db.tables())
total_records = 0
for table_name in self.db.tables():
total_records += len(self.db.table(table_name))
txt += " A table named '%s' with %s records.\n" % (
table_name, len(self.db.table(table_name)))
txt += ' In total there are %s records in the data file.\n' % total_records
txt += "The current table is '%s'.\n" % self.table
else:
txt += 'There are no tables in this data file.\n'
else:
txt += 'The data file is empty or has not been initialized.\n'
else:
txt = '\nA data file has not been opened.\n'
return (txt)
def fields(self, tablename=''):
fieldnames = []
for row in self.db.table(tablename if tablename else self.table):
for fieldname in row.keys():
if not fieldname in fieldnames:
fieldnames.append(fieldname)
return (fieldnames)
def save(self, data_record):
try:
self.db.table(self.table).insert(data_record)
self.new_data = True
return (True)
except:
return (False)
#def __len__(self):
# return(len(self.db) if self.db else 0)
def names(self, fieldname):
return ([row[fieldname] for row in self.db.table(self.table)])
def replace(self, data_record):
pass
def duplicate(self, data_record):
pass
def delete(self, doc_id):
self.db.table(self.table).remove(doc_ids=[doc_id])
self.new_data = True
def delete_all(self):
self.db.table(self.table).purge()
self.new_data = True
def get_unitid(self, name):
unit, idno = name.split('-')
p = self.db.table(self.table).search((where('unit') == unit.strip())
& (where('id') == idno.strip()))
if p:
return (p)
else:
return (None)
def last_record(self):
try:
last = self.db.table(self.table).all()[-1]
except:
last = None
return (last)
class GamesRepository:
"""A TinyDB JSON database"""
DB = 'games'
NEW = 'new'
OLD = 'old'
DIFF = 'diff'
def __init__(self):
self.db_path = None
self._setup()
self.db = TinyDB(self.db_path)
self._create_tables()
def get_new_games(self) -> List[dict]:
return self._get_games(self.NEW)
def get_old_games(self) -> List[dict]:
return self._get_games(self.OLD)
def get_diff(self) -> List[dict]:
return self._get_games(self.DIFF)
def replace_new_games(self, games: List[dict]):
self._replace_games(self.NEW, games)
def replace_old_games(self, games: List[dict]):
self._replace_games(self.OLD, games)
def replace_diff(self, games: List[dict]):
self._replace_games(self.DIFF, games)
def find_by_platform(self, table_name: str, name: str) -> List[dict]:
table = self._get_table(table_name)
return table.search(where('platform') == name)
def find_by_service(self, table_name: str, name: str) -> List[dict]:
table = self._get_table(table_name)
return table.search(where('service') == name)
def _get_games(self, table_name: str) -> List[dict]:
table = self._get_table(table_name)
return [game for game in table.all()]
def _replace_games(self, table_name: str, games: List[dict]):
table = self._get_table(table_name)
self._delete_table(table_name)
self._create(table_name)
for game in games:
table.insert(game)
def _setup(self):
db_dir = get_games_path()
self.db_path = os.path.join(db_dir, "{}.json".format(self.DB))
self._mkdir(db_dir)
def _create_tables(self):
self._create(self.OLD)
self._create(self.NEW)
self._create(self.DIFF)
def _delete_table(self, table_name: str):
"""Purges table in DB"""
if table_name in self.db.tables():
self.db.table(table_name).truncate()
def _get_table(self, table_name: str) -> TinyDB.table:
if table_name not in self.db.tables():
logger.error("No table name: {}".format(table_name))
raise TableException("No table name: {}".format(table_name))
return self.db.table(table_name)
def _create(self, table_name: str) -> int:
"""Creates a table in DB"""
if table_name not in self.db.tables():
return self._touch_table(self.db.table(table_name))
@staticmethod
def _touch_table(table) -> int:
"""Inserts a dummy data in the table to make it readable"""
return table.remove(doc_ids=[table.insert({})])
@staticmethod
def _mkdir(dir_path: str):
"""Creates a directory if one does not exist"""
try:
os.mkdir(dir_path)
except FileExistsError:
pass
class TinyRunDB(BaseDB):
def __init__(self, conn_str):
self.name = 'TinyRunDB'
self.conn_str = conn_str
self.default_table = 'linchpin'
def _opendb(self):
self.middleware = CachingMiddleware(JSONStorage)
self.middleware.WRITE_CACHE_SIZE = 500
self.db = TinyDB(self.conn_str, storage=self.middleware,
default_table=self.default_table)
def __str__(self):
if self.conn_str:
return "{0} at {2}".format(self.name, self.conn_str)
return "{0} at {1}".format(self.name, 'None')
@property
def schema(self):
return self._schema
@schema.setter
def schema(self, schema):
self._schema = dict()
self._schema.update(schema)
@usedb
def init_table(self, table):
t = self.db.table(name=table)
return t.insert(self.schema)
@usedb
def update_record(self, table, run_id, key, value):
t = self.db.table(name=table)
return t.update(add(key, value), eids=[run_id])
@usedb
def get_record(self, table, action='up', run_id=None):
t = self.db.table(name=table)
if not run_id:
run_id = len(t.all())
if not run_id:
return (None, 0)
for rid in range(int(run_id), 0, -1):
record = t.get(eid=int(rid))
if record['action'] == action:
return (record, int(rid))
return (None, 0)
@usedb
def get_records(self, table, count=10):
records = {}
if table in self.db.tables():
t = self.db.table(name=table)
start = len(t)
end = start - count
for i in xrange(start, end, -1):
records[i] = t.get(doc_id=i)
return records
@usedb
def get_tables(self):
tables = self.db.tables()
tables.remove(self.default_table)
return tables
def remove_record(self, table, key, value):
pass
def search(self, table, key=None):
t = self.db.table(name=table)
if key:
return t.search(key)
return t.all()
def query(self, table, query):
pass
def purge(self, table=None):
if table:
return self.db.purge_table(table)
return self.db.purge_tables()
def _closedb(self):
self.db.close()
class HashDatabase:
BLANK_NTLMHASH = "31d6cfe0d16ae931b73c59d7e0c089c0"
def __init__(self,
db_name,
domain,
raise_if_table_doesnt_exist=True,
only_enabled=False,
only_users=False):
self.db = None
self.table = None
self.only_enabled = (Query().enabled.exists() if only_enabled else
Query().ntlmhash.exists()) & (
Query().enabled == True if only_enabled else
Query().ntlmhash.exists())
self.only_users = (Query().username.exists() if only_users else Query(
).ntlmhash.exists()) & (Query().username.test(lambda v: not v.endswith(
"$")) if only_users else Query().ntlmhash.exists())
serialization = SerializationMiddleware()
serialization.register_serializer(DateTimeSerializer(), "datetime")
self.db = TinyDB(db_name, storage=CachingMiddleware(serialization))
tables = list(self.db.tables())
if raise_if_table_doesnt_exist and domain not in tables:
raise DomainDoesntExist(
"Hashes for domain '{}' do not exist in database.".format(
domain), tables)
self.table = self.db.table(domain)
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
self.db.close()
@property
def counts(self):
total = self.table.count(self.only_enabled & self.only_users)
local_users = self.table.count((~Query().historic.exists()) & (Query(
).username.test(lambda v: "\\" not in v and not v.endswith("$")))
& self.only_users)
domain_users = self.table.count((~Query().historic.exists()) & (
Query().username.test(lambda v: "\\" in v and not v.endswith("$")))
& self.only_users)
computers = self.table.count(
Query().username.test(lambda v: v.endswith("$")))
return total, local_users, domain_users, computers
@property
def user_counts(self):
enabled_users = self.table.search((Query().enabled == True) & (
Query().username.test(lambda v: not v.endswith("$"))))
disabled_users = self.table.search((Query().enabled == False) & (
Query().username.test(lambda v: not v.endswith("$"))))
return len(enabled_users), len(disabled_users)
@property
def password_stats(self):
cracked = self.table.count((Query().password.exists())
& (Query().password != "") & self.only_users
& self.only_enabled)
blank = self.table.count(
Query().ntlmhash == HashDatabase.BLANK_NTLMHASH)
historic = self.table.count((Query().historic.exists())
& self.only_enabled & self.only_users)
return cracked, blank, historic
@property
def all_passwords(self):
results = self.table.search((Query().password.exists())
& (Query().password != "")
& self.only_users & self.only_enabled)
return [(result["password"],
zxcvbn.password_strength(result["password"])["score"])
for result in results]
@property
def password_composition_stats(self):
alphanum = string.ascii_letters + string.digits
only_alpha = self.table.count(Query().password.test(
lambda p: p != "" and all(c in alphanum for c in p)))
with_special = self.table.count(Query().password.test(
lambda p: p != "" and any(c not in alphanum for c in p)))
only_digits = self.table.count(Query().password.test(
lambda p: p != "" and all(c in string.digits for c in p)))
return only_alpha, with_special, only_digits
def get_historic_passwords(self, limit=10):
results = sorted(self.table.search((Query().password.exists())
& (Query().password != "")
& (Query().historic.exists())
& (Query().username.exists())
& self.only_enabled),
key=lambda r: r["username"])
passwords = ((user, len(list(count)))
for user, count in itertools.groupby(
results, lambda r: r["username"]))
return sorted(list((user, self.__get_passwords_for_user(user))
for user, count in passwords),
key=lambda (user, passwords): len(passwords),
reverse=True)[:limit]
def get_passwords(self, sortby, reverse=True, limit=10):
results = sorted(
self.table.search((Query().password.exists()) & self.only_users
& self.only_enabled),
key=lambda r: r["password"])
passwords = ((password, len(list(count)))
for password, count in itertools.groupby(
results, lambda r: r["password"]))
return sorted(list(
(password, count, zxcvbn.password_strength(password)["score"],
self.__get_users_with_password(password))
for password, count in passwords),
key=sortby,
reverse=reverse)[:limit]
def get_passwords_where(self, where):
return self.table.search((Query().password.exists())
& (Query().password.test(where))
& self.only_users & self.only_enabled)
def update_hash_password(self, hash, password):
self.table.update(
{
"ntlmhash": hash,
"password": password,
"updated": datetime.now()
},
Query().ntlmhash == hash)
def insert(self, record):
record["created"] = datetime.now()
self.table.insert(record)
def __get_users_with_password(self, password):
users = self.table.search((Query().password.exists())
& (Query().username.exists())
& (Query().password == password)
& self.only_users & self.only_enabled)
shuffle(users)
return users
def __get_passwords_for_user(self, user):
passwords = sorted(self.table.search((Query().password.exists())
& (Query().password != "")
& (Query().username.exists())
& (Query().username == user)
& self.only_enabled),
key=lambda r: r["password"])
grouped_passwords = ((password, users)
for password, users in itertools.groupby(
passwords, lambda r: r["password"]))
return list(grouped_passwords)
class DatabaseAccess(object):
'''
Raw database access. This module might be unnecessary, it's just a simple
wrapper around TinyDB. It doesn't isolate the Query abstractions of TinyDB
so we don't get any portability or independence from TinyDB. But it is a
place to change storage, middleware, etc.
'''
def __init__(self, filename):
'''
Initialize a database with a file.
:param filename: database filename.
'''
self.filename = str(filename)
self.db = TinyDB(self.filename, storage=CachingMiddleware(JSONStorage))
def close(self):
'''Close the database file.'''
self.db.close()
def table(self, tableName):
return self.db.table(tableName)
# def insert(self, *args, **kwargs):
# '''Insert an object into the database.'''
# return self.db.insert(*args, **kwargs)
# def get(self, *args, **kwargs):
# '''Get an object from the database.'''
# return self.db.Get(*args, **kwargs)
# def contains(self, *args, **kwargs):
# '''Test an object from the database.'''
# return self.db.Get(*args, **kwargs)
# def search(self, *args, **kwargs):
# '''Query for objects.'''
# return self.db.search(*args, **kwargs)
# def remove(self, *args, **kwargs):
# '''Remove objects by query.'''
# return self.db.remove(*args, **kwargs)
def setSingleton(self, kind, model):
id = None
self.db.table(kind).purge()
id = self.db.table(kind).insert(model)
return id
def getSingleton(self, kind):
objs = self.db.table(kind).all()
if len(objs) == 0:
return None
return objs[0]
def getArgs(self):
return self.getSingleton('fashion.prime.args')
def isVerbose(self):
args = self.getArgs()
if args is None:
return False
return args["verbose"]
def isDebug(self):
args = self.getArgs()
if args is None:
return False
return args["debug"]
def kinds(self):
k = self.db.tables()
k.remove("_default")
return k
class TinyRunDB(BaseDB):
def __init__(self, conn_str):
self.name = 'TinyRunDB'
self.conn_str = conn_str
self.default_table = 'linchpin'
def _opendb(self):
self.middleware = CachingMiddleware(JSONStorage)
self.middleware.WRITE_CACHE_SIZE = 500
self.db = TinyDB(self.conn_str, storage=self.middleware,
default_table=self.default_table)
def __str__(self):
if self.conn_str:
return "{0} at {2}".format(self.name, self.conn_str)
return "{0} at {1}".format(self.name, 'None')
@property
def schema(self):
return self._schema
@schema.setter
def schema(self, schema):
self._schema = dict()
self._schema.update(schema)
@usedb
def init_table(self, table):
t = self.db.table(name=table)
return t.insert(self.schema)
@usedb
def update_record(self, table, run_id, key, value):
t = self.db.table(name=table)
tx_rec = t.get(eid=run_id).get("outputs", [])
if len(tx_rec) > 0 and isinstance(value, list):
# fetch the resources dict, index
# by filtering them from outputs list
res_list = [(idx, x) for idx, x in enumerate(tx_rec)
if "resources" in x]
if len(res_list) != 0:
res_idx = res_list[0][0]
resources = res_list[0][1]
if "resources" in value[0]:
de = defaultdict(list, resources["resources"])
for i, j in value[0]["resources"].items():
de[i].extend(j)
de = {"resources": de}
tx_rec[res_idx] = de
return t.update(tinySet(key, [de]), eids=[run_id])
return t.update(add(key, value), eids=[run_id])
@usedb
def get_tx_record(self, tx_id):
t = self.db.table(name='linchpin')
return t.get(eid=tx_id)
@usedb
def get_tx_records(self, tx_ids):
txs = {}
t = self.db.table(name='linchpin')
for tx_id in tx_ids:
txs[tx_id] = t.get(eid=tx_id)
return txs
@usedb
def get_record(self, table, action='up', run_id=None):
t = self.db.table(name=table)
if not run_id:
run_id = len(t.all())
if not run_id:
return (None, 0)
for rid in range(int(run_id), 0, -1):
record = t.get(eid=int(rid))
if record and record['action'] == action:
return (record, int(rid))
else:
record = t.get(eid=int(run_id))
if record:
return(record, int(run_id))
return (None, 0)
@usedb
def get_records(self, table, count=10):
records = {}
if table in self.db.tables():
t = self.db.table(name=table)
if len(t.all()):
start = len(t)
if count == 'all':
end = 0
else:
end = start - count
for i in xrange(start, end, -1):
records[i] = t.get(doc_id=i)
return records
@usedb
def get_tables(self):
tables = self.db.tables()
tables.remove(self.default_table)
return tables
def remove_record(self, table, key, value):
pass
def search(self, table, key=None):
t = self.db.table(name=table)
if key:
return t.search(key)
return t.all()
def query(self, table, query):
pass
def purge(self, table=None):
if table:
return self.db.purge_table(table)
return self.db.purge_tables()
def _closedb(self):
self.db.close()
from flask import Flask
from tinydb import TinyDB
from storage import YAMLStorage
app = Flask(__name__)
db = TinyDB('data.yaml', storage=YAMLStorage)
print(db.tables())
