def __init__(self, name="", key="", value=['col']):
self.db = Base(name, save_to_file=False)
self.db.create(key, *value)
self.db.create_index(key)
self.key = key
self.value = value
def __init__(self, pydblite_db_file=None):
if pydblite_db_file is None:
pydblite_db_file = ZipcodeDB.PYDBLITE_DB_FILE
try:
self._db = Base(pydblite_db_file)
self._db.open()
except Exception:
self._db = None
def _setup_db(self): # pragma: no cover
with self._lock:
self._db = Base("", save_to_file=False)
self._db.create('record', 'id', 'first_isd', 'first_as', 'last_isd',
'last_as', 'sibra', mode='override')
self._db.create_index('id')
self._db.create_index('last_isd')
self._db.create_index('last_as')
def state_extraction():
db = Base("scnscraper/abap.pydb")
if db.exists():
db.open()
record = db(type = "Question")
print("# discussion scraped: " + str(record.__len__()))
print("Answered: " + str(db(resolve = "Answered.").__len__()))
print("Answered with solution: "+ str(db(resolve = "solution").__len__()))
print("Not Answered: " + str(db(resolve = "Not Answered.").__len__()))
print("Assumed Answered: " + str(db(resolve = "Assumed Answered.").__len__()))
def __init__(self):
self.out_file = open("scnscraper/abap.json", "a")
self.out_file.close()
self.db = Base("scnscraper/abap.pydb")
if self.db.exists():
self.db.open()
else:
self.db.create('url', 'uid', 'type', 'author', 'title',
'date_time', 'tags', 'views', 'answers', 'resolve',
'upvotes', 'text')
def test_open_memory_with_existing_filename(self):
self.filter_db = Base(test_db_name, save_to_file=True)
self.filter_db.create('unique_id', 'name', "active", mode="override")
self.filter_db.commit()
db = Base(test_db_name, save_to_file=False)
db.open()
self.assertEqual(db.fields, ['unique_id', 'name', "active"])
db = Base(test_db_name, save_to_file=False)
db.create('unique_id2', 'name2', "active2", mode="override")
self.assertEqual(db.fields, ['unique_id2', 'name2', "active2"])
def create():
"""
Creates a pydblite database in memory.
:return: Base class
"""
from pydblite.pydblite import Base
db = Base("temp-db", save_to_file=False)
db.create("hash", "path")
return db
def test_sqlite_compat_insert_list(self):
self.filter_db = Base(test_db_name, save_to_file=False, sqlite_compat=True)
self.filter_db.create('unique_id', 'name', "active", mode="override")
status = [(8, "testname", 0)]
# Insert 1 entries
rec = self.filter_db.insert(status)
self.assertEqual(rec, None)
self.assertEqual(len(self.filter_db), 1)
self.assertEqual(self.filter_db[0]["unique_id"], 8)
self.assertEqual(self.filter_db[0]["name"], "testname")
self.assertEqual(self.filter_db[0]["active"], 0)
def state_extraction():
db = Base("scnscraper/abap.pydb")
if db.exists():
db.open()
record = db(type="Question")
print("# discussion scraped: " + str(record.__len__()))
print("Answered: " + str(db(resolve="Answered.").__len__()))
print("Answered with solution: " +
str(db(resolve="solution").__len__()))
print("Not Answered: " +
str(db(resolve="Not Answered.").__len__()))
print("Assumed Answered: " +
str(db(resolve="Assumed Answered.").__len__()))
def inventoryshape(classdir):
filecount = len(glob(classdir + "/*.jpg"))
print('determining optimal image resolution...')
db = Base('shape', save_to_file=False)
db.create('filename', 'height', 'width', 'count')
index = 0
for imagePath in glob(classdir + "/*.jpg"):
index = index + 1
try:
img = cv2.imread(imagePath)
filename = os.path.basename(imagePath)
shape = img.shape
h = shape[0]
w = shape[1]
pre = db(height=h, width=w)
# see if there is already an image of this shape in the DB...
if pre:
# ...if so - update the count
rec_id = pre[0]['__id__']
counter = int(pre[0]['count'])
counter = counter + 1
record = db[rec_id]
db.update(record, count=counter)
else:
# ...if not - insert the new shape
db.insert(filename=filename, height=h, width=w, count=1)
sys.stdout.write("reading shape for image #{} of {} \r".format(
index, filecount))
sys.stdout.flush()
time.sleep(0.1)
except Exception, e:
print('error processing image {}: {}'.format(imagePath, e))
continue
class InMemoryDBLite(InMemoryDB):
"""Class that implements all steps from Dextra's programming challange.
Uses pydblite in-memory engine.
"""
def __init__(self, name: str):
logger.debug('Initializing DB.')
self.connected = False
self.name = name
self.db = Base(name, save_to_file=False)
def connect(self):
logger.debug(f'Connecting to [{self.name}].')
# When using pydblite in-memory engine, is unnecessary
# connect to a db, so we just set the flag to true
self.connected = True
def disconnect(self):
logger.debug(f'Disconnecting from [{self.name}].')
if not self.connected:
raise Exception('Not connected to db.')
else:
# When using pydblite in-memory engine, is unnecessary
# disconnect from a db, so we just set the flag to false
self.connected = False
def create_schema(self, *args):
logger.debug(f'Crating schema into [{self.name}].')
if not self.connected:
raise Exception('Not connected to db.')
else:
r = self.db.create(*args, mode='override')
self.db.commit()
return r
def insert(self, item: dict):
logger.debug(f'Inserting item into [{self.name}].')
if not self.connected:
raise Exception('Not connected to db.')
else:
r = self.db.insert(**item)
self.db.commit()
return r
def insert_multiple(self, items: list):
logger.debug(f'Inserting multiple items into [{self.name}].')
if not self.connected:
raise Exception('Not connected to db.')
else:
for item in items:
r = self.db.insert(**item)
self.db.commit()
return r
def setupdb(ndbl='ndb.pd1', hdbl='hdb.pd1'):
global ndb
global hdb
ndb = Base(ndbl)
hdb = Base(hdbl)
ndb.create('NID', 'Range', mode="open")
hdb.create('UID', 'HID', 'NID', 'ip', 'bridge')
class ZipcodeDB(object):
PYDBLITE_DB_FILE = './zipcode.db'
def __init__(self, pydblite_db_file=None):
if pydblite_db_file is None:
pydblite_db_file = ZipcodeDB.PYDBLITE_DB_FILE
try:
self._db = Base(pydblite_db_file)
self._db.open()
except Exception:
self._db = None
def query_by_zip_code(self, zip_cd):
logger.debug(
"**************** entering ZipcodeDB.query_zipcode_db_by_zip_code")
record = {} # Empty dict if not found
if self._db is not None:
records = self._db(zip_cd=zip_cd)
if len(records) == 1:
record = records[0]
return record
def get_timezone_for_zip_code(self, zip_code):
logger.debug("**************** entering ZipcodeDB.get_timezone")
ret_val = 'NoTZ/' + zip_code
data = self.query_by_zip_code(zip_code)
if data:
# timezone is really just an offset
timezone = data['data']['timezone']
dst = data['data']['dst']
tz_dic = {
'-5+1': 'US/Eastern',
'-5+0': 'US/East-Indiana',
'-6+1': 'US/Central',
'-7+1': 'US/Mountain',
'-7+0': 'US/Arizona',
'-8+1': 'US/Pacific',
'-9+1': 'US/Alaska',
'-10+0': 'US/Hawaii',
'-10+1': 'US/Aleutian'
}
key = timezone + '+' + dst
if key in tz_dic:
ret_val = tz_dic[key]
return ret_val
def from_json_to_db(self):
thread = ''
db = Base("scnscraper/abap.pydb", save_to_file= True)
# create new base with field names
db.create('url', 'uid', 'type', 'author',
'title', 'date_time', 'tags', 'views',
'answers', 'resolve', 'upvotes', 'text', mode='override')
i=0
with open('scnsraper/threads.json', 'r') as file:
for line in file:
if(line.endswith(" }\n")):
thread += line
tokens = re.search(r"url:\s'(.*?)',\suid:\s'(.*?)',\stype:\s'(.*?)',\sauthor:\s'(.*?)',\stitle:\s'(.*?)',\sdate_time:\s'(.*?)',\stags:\s'(.*?)',\sviews:\s'(.*?)',\sanswers:\s'(.*?)',\sresolve:\s'(.*?)',\supvotes:\s'(.*?)', text:\s'((.|\n)*)'\s}", str(thread))
if tokens is not None:
db.insert(url = tokens.group(1), uid = tokens.group(2), type= tokens.group(3),
author=tokens.group(4), title = tokens.group(5), date_time = tokens.group(6),
tags = tokens.group(7), views = tokens.group(8), answers = tokens.group(9),
resolve = tokens.group(10), upvotes = tokens.group(11), text = tokens.group(12))
db.commit()
print ('\n--------------------------------------------\n')
thread = ''
if(line.startswith(" ]")):
print("new page")
thread = ''
if(line.endswith('\n') and (not line.startswith(" ]\n\n")) and (not line.endswith(" }\n"))):
thread += line
def init():
global DB_DOCKER_PORTS
global SERVICE_INSTANCES_LIST
try:
# SERVICE_INSTANCES_LIST => "MEMORY DB"
LOG.info(
'[lifecycle.data.app.db] [init] Initializing SERVICE_INSTANCES_LIST ...'
)
SERVICE_INSTANCES_LIST = []
# DB_DOCKER_PORTS: PORTS DATABASE for each of the Lifecycles / agents => "PHYSICAL DB"
LOG.info(
'[lifecycle.data.app.db] [init] Initializing DB_DOCKER_PORTS ...')
DB_DOCKER_PORTS = Base(config.dic['LM_WORKING_DIR_VOLUME'] +
config.dic['DB_DOCKER_PORTS']
) #Base(config.dic['DB_DOCKER_PORTS'])
# create new base with field names
if not DB_DOCKER_PORTS.exists():
DB_DOCKER_PORTS.create('port', 'mapped_to')
else:
DB_DOCKER_PORTS.open()
records = DB_DOCKER_PORTS()
except:
LOG.exception(
'[lifecycle.data.app.db] [init] Exception: Error while initializing db components'
)
def __init__(self):
db = Base('rjdb.pdl')
if db.exists():
db.open()
else:
db.create(*FIELDS)
self.db = db
def __init__(self):
self.out_file = open("scnscraper/abap.json", "a")
self.out_file.close()
self.db = Base("scnscraper/abap.pydb")
if self.db.exists():
self.db.open()
else:
self.db.create('url', 'uid', 'type', 'author', 'title', 'date_time', 'tags',
'views', 'answers', 'resolve', 'upvotes', 'text')
def test_open_file_with_existing_dir(self):
os.mkdir(test_db_name)
db = Base(test_db_name, save_to_file=True)
# A dir with that name exists
self.assertRaises(IOError,
db.create,
'unique_id',
'name',
"active",
mode="open")
def load_db(self, check=True, fix=False, save_to_file=False):
self.log('Opening {0} database(s)'.format(len(self.db_files)),
logging.INFO)
for db_name, db_file in self.db_files.items():
_db = Base(db_file, save_to_file=save_to_file)
_db.open()
self.log(
'Database {0} opened, records #: {1}'.format(
db_name, len(_db)), logging.DEBUG)
self.db.update({db_name: _db})
_db.create_index('uid')
_db.create_index('type')
if check is True:
self.check_db(fix)
class MemoryQueue(object):
def __init__(self, name="", col_names=['col']):
self.db = Base(name, save_to_file=False)
self.db.create(*col_names)
self.st = 0
self.en = 0
def pop(self):
if (self.st == self.en):
print('Queue Empty')
ret = self.db[self.st]
del self.db[self.st]
self.st += 1
return ret
def top(self):
if (self.st == self.en):
print('Queue Empty')
ret = self.db[self.st]
return ret
def push(self, arg=['']):
self.db.insert(*arg)
self.en += 1
def print_queue(self):
for r in self.db:
print(r)
def is_empty(self):
return self.st == self.en
def detectduplicates(classdir):
# Create an in-memory database
db = Base('fingerprinter', save_to_file=False)
db.create('filename', 'hash')
filecount = len(glob(classdir + "/*.jpg"))
duplicatecount = 0
print("creating image fingerprints for de-duplication ...")
index = 0
for imagePath in glob(classdir + "/*.jpg"):
index = index + 1
try:
if os.path.exists(imagePath):
image = Image.open(imagePath)
h = str(imagehash.dhash(image))
filename = os.path.basename(imagePath)
sys.stdout.write(
"fingerprint created for image # {} of {} \r".format(
index, filecount))
sys.stdout.flush()
time.sleep(0.1)
pre = db(hash=h)
if pre:
# This image is a duplicate - delete it
duplicatecount = duplicatecount + 1
os.remove(classdir + "/" + filename)
else:
db.insert(filename=filename, hash=h)
except Exception, e:
print('Error in detectduplicates() function: {}'.format(e))
continue
def __init__(self, segment_ttl=None, max_res_no=None): # pragma: no cover
"""
:param int segment_ttl:
The TTL for each record in the database (in s) or None to just use
the segment's expiration time.
:param int max_res_no: Number of results returned for a query.
"""
self._db = Base("", save_to_file=False)
self._db.create('record',
'id',
'first_isd',
'first_as',
'last_isd',
'last_as',
'sibra',
mode='override')
self._db.create_index('id')
self._db.create_index('last_isd')
self._db.create_index('last_as')
self._lock = threading.Lock()
self._segment_ttl = segment_ttl
self._max_res_no = max_res_no
def load_db(self, check=True, fix=False, save_to_file=False):
self.log('Opening {0} database(s)'.format(len(self.db_files)), logging.INFO)
for db_name, db_file in self.db_files.items():
_db = Base(db_file, save_to_file=save_to_file)
_db.open()
self.log('Database {0} opened, records #: {1}'.format(db_name, len(_db)), logging.DEBUG)
self.db.update({db_name: _db})
_db.create_index('uid')
_db.create_index('type')
if check is True:
self.check_db(fix)
def setupdb(ndbl = 'ndb.pd1', hdbl = 'hdb.pd1'):
global ndb
global hdb
ndb = Base(ndbl)
hdb = Base(hdbl)
ndb.create('NID', 'Range', mode="open")
hdb.create('UID', 'HID', 'NID', 'ip', 'bridge')
def from_json_to_db(self):
thread = ''
db = Base("scnscraper/abap.pydb", save_to_file=True)
# create new base with field names
db.create('url',
'uid',
'type',
'author',
'title',
'date_time',
'tags',
'views',
'answers',
'resolve',
'upvotes',
'text',
mode='override')
i = 0
with open('scnsraper/threads.json', 'r') as file:
for line in file:
if (line.endswith(" }\n")):
thread += line
tokens = re.search(
r"url:\s'(.*?)',\suid:\s'(.*?)',\stype:\s'(.*?)',\sauthor:\s'(.*?)',\stitle:\s'(.*?)',\sdate_time:\s'(.*?)',\stags:\s'(.*?)',\sviews:\s'(.*?)',\sanswers:\s'(.*?)',\sresolve:\s'(.*?)',\supvotes:\s'(.*?)', text:\s'((.|\n)*)'\s}",
str(thread))
if tokens is not None:
db.insert(url=tokens.group(1),
uid=tokens.group(2),
type=tokens.group(3),
author=tokens.group(4),
title=tokens.group(5),
date_time=tokens.group(6),
tags=tokens.group(7),
views=tokens.group(8),
answers=tokens.group(9),
resolve=tokens.group(10),
upvotes=tokens.group(11),
text=tokens.group(12))
db.commit()
print('\n--------------------------------------------\n')
thread = ''
if (line.startswith(" ]")):
print("new page")
thread = ''
if (line.endswith('\n') and (not line.startswith(" ]\n\n"))
and (not line.endswith(" }\n"))):
thread += line
def load_db(self, check=True, fix=False, save_to_file=False):
self.log('Opening {0} database(s)'.format(len(self.db_files)), logging.INFO)
for db_name, db_file in self.db_files.items():
_db = Base(db_file, save_to_file=save_to_file)
_db.open()
self.log('Database {0} opened, records #: {1}'.format(db_name, len(_db)), logging.DEBUG)
self.db.update({db_name: _db})
_db.create_index('uid')
_db.create_index('type')
self.log("Db {0}: printing simple strawman prediction accuracy for answers with max upvotes as best answer:".format(db_name), logging.INFO)
self._strawman(_db)
if check is True:
self.check_db(fix)
def load_db(self, check=True, fix=False, save_to_file=False):
self.log('Opening {0} database(s)'.format(len(self.db_files)),
logging.INFO)
for db_name, db_file in self.db_files.items():
_db = Base(db_file, save_to_file=save_to_file)
_db.open()
self.log(
'Database {0} opened, records #: {1}'.format(
db_name, len(_db)), logging.DEBUG)
self.db.update({db_name: _db})
_db.create_index('uid')
_db.create_index('type')
self.log(
"Db {0}: printing simple strawman prediction accuracy for answers with max upvotes as best answer:"
.format(db_name), logging.INFO)
self._strawman(_db)
if check is True:
self.check_db(fix)
def init():
global DB_LM_SERVICE_INSTANCES
try:
# DB_LM: LM DATABASE ("PHYSICAL DB")
LOG.info('[lifecycle.data.app.lm_db] [init] Initializing DB_LM ...')
DB_LM_SERVICE_INSTANCES = Base(config.dic['LM_WORKING_DIR_VOLUME'] +
config.dic['DB_STANDALONE_MODE'] +
"_service_instances")
# create new base with field names
if not DB_LM_SERVICE_INSTANCES.exists():
DB_LM_SERVICE_INSTANCES.create('id', 'service_instance')
else:
DB_LM_SERVICE_INSTANCES.open()
except:
LOG.exception(
'[lifecycle.data.app.lm_db] [init] Exception: Error while initializing db components'
)
def test_sqlite_compat(self):
db = Base(test_db_name, save_to_file=False, sqlite_compat=True)
db.create('unique_id', 'name', "active", mode="open")
self.reset_status_values_for_filter()
# Insert 7 entries
res = db.insert(self.status)
self.assertEqual(res, None)
self.assertEqual(len(db), 7)
status = [(8, "testname", 0)]
res = db.insert(status)
self.assertEqual(res, None)
self.assertEqual(len(db), 8)
class MemoryDict(object):
def __init__(self, name="", key="", value=['col']):
self.db = Base(name, save_to_file=False)
self.db.create(key, *value)
self.db.create_index(key)
self.key = key
self.value = value
def give_me_elem(self, key):
return eval('self.db._' + self.key + '[key]')
def is_in(self, key):
return len(self.give_me_elem(key)) > 0
def insert(self, key="", value=[""]):
record = self.give_me_elem(key)
if len(record) > 0:
for i in range(len(self.value)):
record[0][self.value[i]] = value[i]
else:
self.db.insert(key, *value)
def pop(self, key):
record = self.give_me_elem(key)
if len(record) > 0:
rec_id = record[0]['__id__']
del self.db[rec_id]
def iteritems(self):
return list(self.db)
def len(self):
return len(self.db)
def print_all(self):
for r in self.db:
print(r)
class DataStoring():
#Inizialize an instantiated object by opening json file and the database
def __init__(self):
self.out_file = open("scnscraper/abap.json", "a")
self.out_file.close()
self.db = Base("scnscraper/abap.pydb")
if self.db.exists():
self.db.open()
else:
self.db.create('url', 'uid', 'type', 'author', 'title', 'date_time', 'tags',
'views', 'answers', 'resolve', 'upvotes', 'text')
#for each thread scraped, insert it into db
def insert_items_into_db(self, threads):
for thread in threads:
item = SapItem() # New Item instance
item = thread
try:
# Insert into db
self.db.insert(url = str(item["url"]), uid = str(item["uid"]), type= str(item["type"] ),
author=str(item["author"]), title = str(item["title"]),
date_time = str(item["date_time"] ),tags = str(item["tags"] ),
views = str(item["views"] ), answers = str(item["answers"] ),
resolve = str(item["resolve"] ), upvotes = str(item["upvotes"] ),
text = str(item["text"]))
except UnicodeEncodeError:
print("Unicode Encode Exception!")
#save changes on disk
self.db.commit()
# for each thread scraped, initialize the string to insert into json file
def threads_to_str(self, threads):
out_string = "[ "
if threads.__len__() == 0:
return ""
for thread in threads:
item = SapItem()
item = thread
try:
out_string += "{ url: '" + str(item["url"] ) + "', " + "uid: '" + str(item["uid"] ) + "', "\
"type: '" + str(item["type"] ) + "', "\
"author: '"+ str(item["author"]) + "', " \
"title: '"+ str(item["title"]) + "', "\
"date_time: '"+ str(item["date_time"] ) + "', " \
"tags: '"+ str(item["tags"] ) + "', " \
"views: '"+ str(item["views"] ) + "', "\
"answers: '"+ str(item["answers"] ) + "', " \
"resolve: '"+ str(item["resolve"] ) + "', " \
"upvotes: '"+ str(item["upvotes"] ) + "', "\
"text: '" + str(item["text"]) + "' }\n"
except UnicodeEncodeError:
print("Unicode Encode Exception!")
out_string += " ]\n\n"
return out_string
#for each thread scraped, insert it into json file
def insert_items_into_file(self, threads):
try:
self.out_file = open("scnscraper/abap.json", "a") # open in append mode
#convert into string and insert into file
self.out_file.write(self.threads_to_str(threads))
self.out_file.close()
except:
print('Exception in writing file')
self.out_file.close()
# read the web page index
def read_index_from_file(self):
if os.path.exists('scnscraper/index.txt'):
with open('scnscraper/index.txt') as f:
index = int(f.readline())
f.close()
else:
f = open('scnscraper/index.txt', 'w')
index = 2
f.write(str(index))
f.close()
return index
# Write the web page index
def write_index_into_file(self, i):
f = open('scnscraper/index.txt', 'w')
f.write(str(i))
f.close()
# Convert the content of json file into a new db
def from_json_to_db(self):
thread = ''
db = Base("scnscraper/abap.pydb", save_to_file= True)
# create new base with field names
db.create('url', 'uid', 'type', 'author',
'title', 'date_time', 'tags', 'views',
'answers', 'resolve', 'upvotes', 'text', mode='override')
i=0
with open('scnsraper/threads.json', 'r') as file:
for line in file:
if(line.endswith(" }\n")):
thread += line
tokens = re.search(r"url:\s'(.*?)',\suid:\s'(.*?)',\stype:\s'(.*?)',\sauthor:\s'(.*?)',\stitle:\s'(.*?)',\sdate_time:\s'(.*?)',\stags:\s'(.*?)',\sviews:\s'(.*?)',\sanswers:\s'(.*?)',\sresolve:\s'(.*?)',\supvotes:\s'(.*?)', text:\s'((.|\n)*)'\s}", str(thread))
if tokens is not None:
db.insert(url = tokens.group(1), uid = tokens.group(2), type= tokens.group(3),
author=tokens.group(4), title = tokens.group(5), date_time = tokens.group(6),
tags = tokens.group(7), views = tokens.group(8), answers = tokens.group(9),
resolve = tokens.group(10), upvotes = tokens.group(11), text = tokens.group(12))
db.commit()
print ('\n--------------------------------------------\n')
thread = ''
if(line.startswith(" ]")):
print("new page")
thread = ''
if(line.endswith('\n') and (not line.startswith(" ]\n\n")) and (not line.endswith(" }\n"))):
thread += line
def state_extraction():
db = Base("scnscraper/abap.pydb")
if db.exists():
db.open()
record = db(type = "Question")
print("# discussion scraped: " + str(record.__len__()))
print("Answered: " + str(db(resolve = "Answered.").__len__()))
print("Answered with solution: "+ str(db(resolve = "solution").__len__()))
print("Not Answered: " + str(db(resolve = "Not Answered.").__len__()))
print("Assumed Answered: " + str(db(resolve = "Assumed Answered.").__len__()))
state_extraction = staticmethod(state_extraction)
def test_open_memory(self):
db = Base(":memory:")
self.assertFalse(db.save_to_file)
def test_open_existing(self):
db = Base(test_db_name, save_to_file=True)
db.create('unique_id', 'name', "active", mode="open")
db.insert("123", "N", True)
db.commit()
# Just verify that it works to open an existing db.
# The column names are ignored, therefore they should
# equal the old column names
db = Base(test_db_name, save_to_file=True)
db.create('unique_id2', 'name2', "active2", mode="open")
rec = db.insert("123", "N", True)
db.commit()
self.assertEqual(db.fields, ['unique_id', 'name', "active"])
# mode="override" will overwrite existing db
db = Base(test_db_name, save_to_file=True)
db.create('unique_id', 'name', "active", mode="override")
db.commit()
self.assertEqual(len(self.filter_db), 0)
# Equals passing mode=None
self.assertRaises(IOError, db.create, 'unique_id', 'name', "active")
self.assertRaises(ValueError, db.create, 'unique_id', 'name', "active", mode="invalidmode")
def test_open(self):
db = Base('dummy', save_to_file=False)
db.create('name', 'age', 'size')
db.insert(name='homer', age=23, size=1.84)
def setUp(self): # NOQA
self.first_record_id = 0
filter_db = Base(test_db_name, save_to_file=False)
filter_db.create('unique_id', 'name', "active", mode="override")
self.filter_db = filter_db
class PyDbLiteTestCase(Generic, unittest.TestCase):
def setUp(self): # NOQA
self.first_record_id = 0
filter_db = Base(test_db_name, save_to_file=False)
filter_db.create('unique_id', 'name', "active", mode="override")
self.filter_db = filter_db
def tearDown(self): # NOQA
if os.path.isfile(test_db_name):
os.remove(test_db_name)
elif os.path.isdir(test_db_name):
os.rmdir(test_db_name)
def setup_db_for_filter(self):
self.reset_status_values_for_filter()
for d in self.status:
res = self.filter_db.insert(**d)
self.assertEqual(res, 6)
def test_open(self):
db = Base('dummy', save_to_file=False)
db.create('name', 'age', 'size')
db.insert(name='homer', age=23, size=1.84)
def test_open_file_with_existing_dir(self):
os.mkdir(test_db_name)
db = Base(test_db_name, save_to_file=True)
# A dir with that name exists
self.assertRaises(IOError, db.create, 'unique_id', 'name', "active", mode="open")
def test_open_existing(self):
db = Base(test_db_name, save_to_file=True)
db.create('unique_id', 'name', "active", mode="open")
db.insert("123", "N", True)
db.commit()
# Just verify that it works to open an existing db.
# The column names are ignored, therefore they should
# equal the old column names
db = Base(test_db_name, save_to_file=True)
db.create('unique_id2', 'name2', "active2", mode="open")
rec = db.insert("123", "N", True)
db.commit()
self.assertEqual(db.fields, ['unique_id', 'name', "active"])
# mode="override" will overwrite existing db
db = Base(test_db_name, save_to_file=True)
db.create('unique_id', 'name', "active", mode="override")
db.commit()
self.assertEqual(len(self.filter_db), 0)
# Equals passing mode=None
self.assertRaises(IOError, db.create, 'unique_id', 'name', "active")
self.assertRaises(ValueError, db.create, 'unique_id', 'name', "active", mode="invalidmode")
def test_open_memory(self):
db = Base(":memory:")
self.assertFalse(db.save_to_file)
def test_open_memory_with_existing_filename(self):
self.filter_db = Base(test_db_name, save_to_file=True)
self.filter_db.create('unique_id', 'name', "active", mode="override")
self.filter_db.commit()
db = Base(test_db_name, save_to_file=False)
db.open()
self.assertEqual(db.fields, ['unique_id', 'name', "active"])
db = Base(test_db_name, save_to_file=False)
db.create('unique_id2', 'name2', "active2", mode="override")
self.assertEqual(db.fields, ['unique_id2', 'name2', "active2"])
def test_insert_list(self):
status = (8, "testname", 0)
# Insert 7 entries
rec = self.filter_db.insert(status)
self.assertEqual(rec, 0)
self.assertEqual(self.filter_db[rec]["unique_id"], status)
def test_sqlite_compat_insert_list(self):
self.filter_db = Base(test_db_name, save_to_file=False, sqlite_compat=True)
self.filter_db.create('unique_id', 'name', "active", mode="override")
status = [(8, "testname", 0)]
# Insert 1 entries
rec = self.filter_db.insert(status)
self.assertEqual(rec, None)
self.assertEqual(len(self.filter_db), 1)
self.assertEqual(self.filter_db[0]["unique_id"], 8)
self.assertEqual(self.filter_db[0]["name"], "testname")
self.assertEqual(self.filter_db[0]["active"], 0)
def test_sqlite_compat(self):
db = Base(test_db_name, save_to_file=False, sqlite_compat=True)
db.create('unique_id', 'name', "active", mode="open")
self.reset_status_values_for_filter()
# Insert 7 entries
res = db.insert(self.status)
self.assertEqual(res, None)
self.assertEqual(len(db), 7)
status = [(8, "testname", 0)]
res = db.insert(status)
self.assertEqual(res, None)
self.assertEqual(len(db), 8)
import cProfile
from pydblite.pydblite import Base
from workalendar.core import Calendar, MON, TUE, WED, THU, FRI, SAT, SUN
from ..retrieve_data import (retrieve_nth_weekday_loop, retrieve_nth_weekday_map
retrieve_nth_weekday_list_comp, retrieve_nth_weekday_all_fields)
date_table = Base('temporal_data.pdl')
date_table = date_table.create(mode="open")
# nth_weekday = input("Give the year, month, weekday, nth number(2010, 2, 2, 3): ")
nth_weekday = (2010, 2, 2, 3)
cProfile.run('retrieve_nth_weekday_map(date_table, nth_weekday)')
cProfile.run('retrieve_nth_weekday_loop(date_table, nth_weekday)')
cProfile.run('retrieve_nth_weekday_all_fields(date_table, nth_weekday)')
cProfile.run('retrieve_nth_weekday_list_comp(date_table, nth_weekday)')
cProfile.run('Calendar.get_nth_weekday_in_month(2010, 2, FRI, 3)')
def pydblite():
from pydblite.pydblite import Base
db = Base('dummy', save_to_file=False)
# create new base with field names
db.create('name', 'age', 'size')
# insert new record
db.insert(name='homer', age=23, size=1.84)
# records are dictionaries with a unique integer key __id__
# simple selection by field value
records = db(name="homer")
# complex selection by list comprehension
res = [r for r in db if 30 > r['age'] >= 18 and r['size'] < 2]
print "res:", res
# delete a record or a list of records
r = records[0]
db.delete(r)
list_of_records = []
r = db.insert(name='homer', age=23, size=1.84)
list_of_records.append(db[r])
r = db.insert(name='marge', age=36, size=1.94)
list_of_records.append(db[r])
# or generator expression
for r in (r for r in db if r['name'] in ('homer', 'marge')):
# print "record:", r
pass
db.delete(list_of_records)
rec_id = db.insert(name='Bart', age=15, size=1.34)
record = db[rec_id] # the record such that record['__id__'] == rec_id
# delete a record by its id
del db[rec_id]
# create an index on a field
db.create_index('age')
# update
rec_id = db.insert(name='Lisa', age=13, size=1.24)
# direct access by id
record = db[rec_id]
db.update(record, age=24)
# add and drop fields
db.add_field('new_field', default=0)
db.drop_field('name')
# save changes on disk
db.commit()
