def test_get_and_set():
""" Tests the get() and set() methods of PupDB. """
database = PupDB(TEST_DB_PATH)
key, val = 'test_key', list(range(100))
database.set(key, val)
assert database.get(key) == val
def __init__(self, data_range, database=None):
""" Instance Initialization """
super(PupDBWriterThread, self).__init__()
self.data_range = data_range
if database is None:
self.database = PupDB(TEST_DB_PATH)
else:
self.database = database
def write_to_db(data_range):
""" Method performs write to the database."""
database = PupDB(TEST_DB_PATH)
for i in data_range:
database.set(i, i)
logging.info('Process %s wrote range(%s, %s) to DB successfully.',
os.getpid(), data_range.start, data_range.stop)
def test_dumps():
""" Tests the dumps() method of PupDB. """
database = PupDB(TEST_DB_PATH)
range_list = list(range(10))
for i in range_list:
database.set(i, i)
db_dict = {str(i): i for i in range_list}
assert database.dumps() == json.dumps(db_dict, sort_keys=True)
def test_keys():
""" Tests the keys() method of PupDB. """
database = PupDB(TEST_DB_PATH)
range_list = [str(i) for i in range(10)]
for i in range_list:
database.set(i, i)
db_keys_list = list(database.keys())
assert sorted(db_keys_list) == range_list
def test_length():
"""
Tests whether len() on the PupDB instance returns it's proper length.
"""
database = PupDB(TEST_DB_PATH)
num_items = 100
range_list = list(range(num_items))
for i in range_list:
database.set(i, i)
assert len(database) == 100
def test_values():
""" Tests the values() method of PupDB. """
database = PupDB(TEST_DB_PATH)
range_list = list(range(10))
for i in range_list:
database.set(i, i)
db_values_list = list(database.values())
assert sorted(db_values_list) == range_list
def test_items():
""" Tests the items() method of PupDB. """
database = PupDB(TEST_DB_PATH)
range_list = list(range(10))
for i in range_list:
database.set(i, i)
items_list = [(str(i), i) for i in range_list]
db_items_list = list(database.items())
assert sorted(db_items_list) == items_list
def test_get_database():
"""
Tests whether _get_database() returns the dict for the DB.
"""
database = PupDB(TEST_DB_PATH)
num_items = 100
range_list = list(range(num_items))
for i in range_list:
database.set(i, i)
# pylint: disable=protected-access
db_dict = database._get_database()
assert json.dumps({str(i): i for i in range_list}, sort_keys=True) == \
json.dumps(db_dict, sort_keys=True)
def test_mt_get_and_set_mi():
"""
Tests the get() and set() methods of PupDB,
in a multi-threaded scenario, where each thread has it's own
instance (object) of PupDB.
"""
data_ranges = [
range(1, 50),
range(50, 100),
range(100, 150),
range(150, 201)
]
writers = []
# Write from multiple threads.
for data_range in data_ranges:
writer = PupDBWriterThread(data_range)
writers.append(writer)
writer.start()
for writer in writers:
writer.join()
# Verify if all keys have been written properly.
database = PupDB(TEST_DB_PATH)
assert len(database) == 200
def init_module():
""" Initializes the Flask App. """
app = Flask(__name__)
app.response_class = CustomResponse
database = PupDB(os.environ.get('PUPDB_FILE_PATH') or 'pupdb.json')
return app, database
def test_flush_database():
"""
Tests whether _flush_database() writes to DB.
"""
database = PupDB(TEST_DB_PATH)
num_items = 100
range_list = list(range(num_items))
data_dict = {str(i): i for i in range_list}
# pylint: disable=protected-access
database._flush_database(data_dict)
db_dict = database._get_database()
assert json.dumps(data_dict, sort_keys=True) == \
json.dumps(db_dict, sort_keys=True)
def __init__(self,
name=None,
fn=None,
use_secret=CRYPT_USE_SECRET,
path_secret=PATH_CRYPT_SECRET,
encrypt_values=False,
encryptor_func=None,
decryptor_func=None):
# defaults
if not name and fn: name = os.path.basename(fn).replace('.', '_')
self.name, self.fn = name, fn
# use secret? for salting
if use_secret and path_secret:
if not os.path.exists(path_secret):
self.secret = get_random_binary_id()
from comrad.backend.keymaker import make_key_discreet
self.log('shhh! creating secret:',
make_key_discreet(self.secret))
with open(path_secret, 'wb') as of:
of.write(self.secret)
else:
with open(path_secret, 'rb') as f:
self.secret = f.read()
else:
self.secret = b''
self.encrypt_values = encrypt_values
if self.secret and encrypt_values and (not encryptor_func
or not decryptor_func):
from comrad.backend.keymaker import ComradSymmetricKeyWithPassphrase
self.key = ComradSymmetricKeyWithPassphrase(passphrase=self.secret)
encryptor_func = self.key.encrypt
decryptor_func = self.key.decrypt
self.encryptor_func = encryptor_func
self.decryptor_func = decryptor_func
# self.store = FilesystemStore(self.fn)
# self.store = RedisStore(redis.StrictRedis())
# self.db = Vedis(self.fn)
# self.db = WalrusLite(self.fn)
# import hirlite
# self.db = hirlite.Rlite(path=self.fn)
from pupdb.core import PupDB
self.db = PupDB(self.fn)
def test_remove():
""" Tests the remove() method of PupDB. """
database = PupDB(TEST_DB_PATH)
for i in range(10):
database.set(i, i)
# Removing key 0 from the database.
database.remove(0)
for i in range(10):
if i == 0:
assert database.get(i) is None
else:
assert database.get(i) is not None
# Check the behavaiour for a non-existent key.
with pytest.raises(KeyError):
database.remove(1000)
class PupDBWriterThread(Thread):
"""
This class represents an instance of a thread that writes
to the database.
"""
def __init__(self, data_range, database=None):
""" Instance Initialization """
super(PupDBWriterThread, self).__init__()
self.data_range = data_range
if database is None:
self.database = PupDB(TEST_DB_PATH)
else:
self.database = database
def run(self):
""" Method performs write to the database."""
for i in self.data_range:
self.database.set(i, i)
def test_mp_get_and_set():
"""
Tests the get() and set() methods of PupDB,
in a multi-processing scenario.
"""
data_ranges = [(0, 100), (100, 200), (200, 300), (300, 400)]
for data_range in data_ranges:
proc = subprocess.Popen(
'python writer_process.py {} {}'.format(*data_range), shell=True)
proc.wait()
database = PupDB(TEST_DB_PATH)
assert len(database) == 400
def test_mt_get_and_set_si():
"""
Tests the get() and set() methods of PupDB,
in a multi-threaded scenario, where all threads share a
single instance (object) of PupDB.
PupDB currently does not support multiple threads using the same
PupDB instance, so the database file will get corrupted due to
race condition.
If you want to use PupDB with multiple threads, maintain separate
PupDB instance for each thread.
"""
v_major, v_minor, _, _, _ = sys.version_info
if v_major < 3 or (v_major == 3 and v_minor <= 4):
error_cls = ValueError
else:
error_cls = json.decoder.JSONDecodeError
with pytest.raises(error_cls):
data_ranges = [
range(1, 50),
range(50, 100),
range(100, 150),
range(150, 201)
]
writers = []
database = PupDB(TEST_DB_PATH)
# Write from multiple threads.
for data_range in data_ranges:
writer = PupDBWriterThread(data_range, database)
writers.append(writer)
writer.start()
for writer in writers:
writer.join()
# Verify if all keys have been written properly.
assert len(database) == 200
def Main(credentials):
"""Main function declaration.
requires:
credentials: tuple (Twitter credentials)
"""
defineLoggers()
storage_db = PupDB(
PupDB_FILENAME) # activate PupDB file for persistent storage
last_tweet = storage_db.get(PupDB_MRTkey)
last_level = storage_db.get(PupDB_MRLkey)
if last_tweet is None: # Pre-load empty database
last_tweet = str(LOCAL_NOW_STRING())
last_level = MINIMUM_CONCERN_LEVEL
storage_db.set(PupDB_MRTkey, last_tweet)
storage_db.set(PupDB_MRLkey, last_level)
forecast_level = last_level
storage_db.set(PupDB_MRFkey, forecast_level)
# initialization complete. Begin main loop.
# check river database and update if needed
# tweet status if needed and update tweet database
# update any local display devices
return False # should never end
def main():
parser = ArgumentParser()
parser.add_argument('--reportdir',
default='reports',
metavar='D',
help='directory containing report files')
parser.add_argument(
'--xactfile',
default=None,
metavar='F',
help='csv file containing financial transaction report')
parser.add_argument('--xerofile',
default=None,
metavar='F',
help='output csv file for xero pre-coded transactions')
parser.add_argument('--pupdbfile',
default=None,
metavar='F',
help='json file for pupdb key-value store')
parser.add_argument('--dryrun',
'-n',
action='store_true',
help='dont make any actual changes - just run through')
parser.add_argument('--verbose',
'-v',
action='store_true',
help='print verbose messages')
args = parser.parse_args()
reportdir = args.reportdir
if not os.path.isdir(reportdir):
reportdir = os.path.join(xerodir, args.reportdir)
if not os.path.isdir(reportdir):
raise RuntimeError('cannot locate reports directory!')
if args.verbose:
print('[reports found in directory {} (realpath={})]'.format(
reportdir, os.path.realpath(reportdir)),
file=sys.stderr)
xactfile = args.xactfile
if xactfile is None:
xactfile, _ = latest_report(
'transactions', reportdir, r'^transactions_(\d{8})\.csv$',
lambda m: datetime.strptime(m.group(1), '%Y%m%d'), args.verbose)
if xactfile is None:
raise RuntimeError('cannot locate transaction file!')
if args.verbose:
print('[transaction report file = {} (realpath={})]'.format(
xactfile, os.path.realpath(xactfile)),
file=sys.stderr)
xerofile = args.xerofile
if xerofile is None:
xerofile = os.path.join(
xerodir,
datetime.now().strftime('xero-upload-%Y%m%d.csv'))
if args.verbose:
print('[Xero output csv file = {} (realpath={})]'.format(
xerofile, os.path.realpath(xerofile)),
file=sys.stderr)
pupdbfile = args.pupdbfile
if pupdbfile is None:
pupdbfile = os.path.join(xerodir, 'uploaded.json')
if args.verbose:
print('[pupdb json file = {} (realpath={})]'.format(
pupdbfile, os.path.realpath(pupdbfile)),
file=sys.stderr)
config = load_config(prefix=xerodir)
db = PupDB(pupdbfile)
fieldnames = [
'Date',
'Amount',
'Payee',
'Description',
'Reference',
'Cheque Number',
'Account code',
'Tax Rate (Display Name)',
'Tracking1',
'Tracking2',
'Transaction Type',
'Analysis code',
]
voucher_desc = 'Get Active Kids Voucher Program'
with open(xactfile, 'r', newline='') as infile:
reader = DictReader(infile)
output_records = {}
already_uploaded = {}
order_numbers = []
order_item_ids = []
total_netamount = Decimal('0.00')
total_phqfee = Decimal('0.00')
total_subtotal = Decimal('0.00')
total_pending = Decimal('0.00')
total_gvapplied = Decimal('0.00')
for inrec in reader:
org = inrec['Organisation']
role = inrec['Role']
org_to = inrec['Organisation Registering To']
pstatus = inrec['Payout Status']
netamount = Decimal(inrec['Net Amount'][1:])
if (org != 'Shooters Basketball Club' or role != 'Player'
or org_to != 'Shooters Basketball Club'
or pstatus != 'DISBURSED'):
if args.verbose:
print('skip (bad rec): org={}, role={}, org_to={}, '
'pstatus={}'.format(org, role, org_to, pstatus),
file=sys.stderr)
if pstatus == 'DISBURSEMENT_PENDING':
total_pending += netamount
continue
rtype = inrec['Type of Registration']
rname = inrec['Registration']
ptype = inrec['Product Type']
for rdesc in config['types']:
if (rdesc['rtype'] == rtype and rdesc['rname'] == rname
and rdesc['ptype'] == ptype):
break
else:
raise RuntimeError(
'type not found: rtype={}, rname={}, ptype={}'.format(
rtype, rname, ptype))
dfmt = '%d/%m/%Y'
sname = inrec['Season Name']
xdate = to_date(inrec['Date'], dfmt)
name = inrec['Name']
onum = inrec['Order Number']
oid = inrec['Order Item ID']
oprice = Decimal(inrec['Order Item Price'][1:])
gvname = inrec['Government Voucher Name']
if gvname != '':
if gvname != 'Get Active Kids':
raise RuntimeError('bad gov voucher: {}'.format(gvname))
sgva = inrec['Government Voucher Amount']
gvamount = Decimal(sgva[1:])
if gvamount != Decimal('200.00'):
raise RuntimeError(
'GAK voucher not $200: {:.2f}'.format(gvamount))
sgvaa = inrec['Government Voucher Amount Applied']
gvapplied = Decimal(sgvaa[1:])
else:
gvamount = Decimal('0.00')
gvapplied = Decimal('0.00')
product = inrec['Product Name']
quantity = int(inrec['Quantity'])
subtotal = Decimal(inrec['Subtotal'][1:])
phqfee = Decimal(inrec['PlayHQ Fee'][1:])
pdate = to_date(inrec['Payout Date'], '%Y-%m-%d')
pid = inrec['Payout ID']
if rdesc['rid'] == 'clinic':
sku = inrec['Merchandise SKU']
if sku not in rdesc['skus']:
raise RuntimeError('sku not found: {} not in {}'.format(
sku, rdesc['skus']))
# Term N, YYYY
m = re.match(r'^Term ([1-4]), (\d{4})$', sname)
if m is None:
raise RuntimeError(
'clinic record has bad season name ({})'.format(sname))
tracking1 = rdesc['tracking1'].format(*m.groups())
tracking2 = rdesc['tracking2']
elif rdesc['rid'] == 'registration':
feename = inrec['Fee Name']
for fdesc in rdesc['fees']:
if fdesc['name'] == feename:
famount = Decimal(fdesc['amount'])
break
else:
raise RuntimeError(
'fee not found: rtype={}, rname={}, ptype={}'.format(
rtype, rname, ptype))
if quantity != 1:
raise RuntimeError('registration with quantity != 1!')
if famount != oprice:
raise RuntimeError(
'fee amount mismatch ({:.2f}!={:.2f})'.format(
famount, oprice))
# (Winter|Summer) YYYY
m = re.match(r'^(Winter|Summer) (\d{4})$', sname)
if m is None:
raise RuntimeError(
'rego record has bad season name ({})'.format(sname))
wors, year = m.groups()
if wors == 'Summer':
year = '{}/{:02d}'.format(year, int(year) - 2000 + 1)
tracking1 = rdesc['tracking1'].format(year, wors)
tracking2 = rdesc['tracking2']
else:
raise RuntimeError('bad rego id {}!'.format(rdesc['rid']))
if (oprice - gvapplied) * quantity != subtotal:
raise RuntimeError(
'oprice({:.2f})*quantity({})!=subtotal({:.2f})'.format(
oprice, quantity, subtotal))
if subtotal != netamount + phqfee:
raise RuntimeError(
'subtotal({:.2f})!=netamount({:.2f})+phqfee({:.2f})'.
format(
subtotal,
netamount,
phqfee,
))
if onum in order_numbers:
raise RuntimeError('duplicate order number {}!'.format(onum))
order_numbers.append(onum)
if oid in order_item_ids:
raise RuntimeError('duplicate order item id {}!'.format(oid))
order_item_ids.append(oid)
if db.get(pid) is not None:
is_already_uploaded = True
if pid not in already_uploaded:
already_uploaded[pid] = []
if args.verbose:
print('already uploaded: name={}, pdate={}, pid={}'.format(
name, pdate, pid),
file=sys.stderr)
else:
is_already_uploaded = False
if pid not in output_records:
output_records[pid] = []
total_netamount += netamount
total_phqfee += phqfee
total_subtotal += subtotal
total_gvapplied += gvapplied
desc = '{} - ${:.2f} x {:d}'.format(product, oprice, quantity)
orec = {
'Date': xdate.strftime(dfmt),
'Amount': '{:.2f}'.format(subtotal),
'Payee': name,
'Description': '{}: subtotal'.format(desc),
'Reference': '{} on {}'.format(pid, pdate.strftime(dfmt)),
'Cheque Number': onum,
'Account code': config['sales_account'],
'Tax Rate (Display Name)': config['taxrate'],
'Tracking1': tracking1,
'Tracking2': tracking2,
'Transaction Type': 'credit',
'Analysis code': oid,
}
if is_already_uploaded:
already_uploaded[pid].append(orec)
else:
output_records[pid].append(orec)
if phqfee != Decimal('0.00'):
orec = {
'Date': xdate.strftime(dfmt),
'Amount': '-{:.2f}'.format(phqfee),
'Payee': name,
'Description': '{}: playhq fees'.format(desc),
'Reference': '{} on {}'.format(pid, pdate.strftime(dfmt)),
'Cheque Number': onum,
'Account code': config['fees_account'],
'Tax Rate (Display Name)': config['taxrate'],
'Tracking1': tracking1,
'Tracking2': tracking2,
'Transaction Type': 'debit',
'Analysis code': oid,
}
if is_already_uploaded:
already_uploaded[pid].append(orec)
else:
output_records[pid].append(orec)
if gvapplied != Decimal('0.00'):
orec = {
'Date': xdate.strftime(dfmt),
'Amount': '{:.2f}'.format(gvapplied),
'Payee': voucher_desc,
'Description': '{}: get active for {}'.format(desc, name),
'Reference': '{} on {}'.format(pid, pdate.strftime(dfmt)),
'Cheque Number': onum,
'Account code': config['other_revenue_account'],
'Tax Rate (Display Name)': config['taxrate'],
'Tracking1': tracking1,
'Tracking2': tracking2,
'Transaction Type': 'credit',
'Analysis code': oid,
}
if is_already_uploaded:
already_uploaded[pid].append(orec)
else:
output_records[pid].append(orec)
for pid, oreclist1 in already_uploaded.items():
total_amount1 = sum(
Decimal(orec['Amount']) for orec in oreclist1
if orec['Payee'] != voucher_desc)
dbval = db.get(pid)
if dbval is None:
raise RuntimeError('db get of pid {} failed!'.format(pid))
oreclist2 = loads(dbval)
if not isinstance(oreclist2, list):
if args.verbose:
print('cannot check old record: pid={}, amount=${:.2f}'.format(
pid, total_amount1),
file=sys.stderr)
continue
total_amount2 = sum(
Decimal(orec['Amount']) for orec in oreclist2
if orec['Payee'] != voucher_desc)
if total_amount1 != total_amount2:
raise RuntimeError(
'pid {} total amount mismatch (${:.2f} != ${:.2f})!'.format(
pid, total_amount1, total_amount2))
if args.verbose:
print('checked already uploaded: pid={}, amount=${:.2f}'.format(
pid, total_amount1),
file=sys.stderr)
if args.verbose and total_pending > 0:
print('total pending = ${:.2f}'.format(total_pending), file=sys.stderr)
if len(output_records) == 0:
print('No records were collected.', file=sys.stderr)
return 0
if total_subtotal - total_phqfee != total_netamount:
raise RuntimeError('total({:.2f})-fees({:.2f})!=net({:.2f})'.format(
total_subtotal, total_phqfee, total_netamount))
if args.verbose:
print('{} payment ids were collected.'.format(len(output_records)),
file=sys.stderr)
print('subtotal = ${:.2f}'.format(total_subtotal), file=sys.stderr)
print('phqfee = ${:.2f}'.format(total_phqfee), file=sys.stderr)
print('netamount = ${:.2f}'.format(total_netamount), file=sys.stderr)
print('gov vouchers = ${:.2f}'.format(total_gvapplied),
file=sys.stderr)
for pid, oreclist in output_records.items():
amount = Decimal(0.0)
for outrec in oreclist:
if outrec['Payee'] != voucher_desc:
amount += Decimal(outrec['Amount'])
print(' Payment Id {} = ${:.2f}'.format(pid, amount),
file=sys.stderr)
if args.dryrun:
return 0
if os.path.exists(xerofile):
raise RuntimeError('will not overwrite file {}'.format(xerofile))
with open(xerofile, 'w', newline='') as outfile:
writer = DictWriter(outfile, fieldnames=fieldnames)
writer.writeheader()
for pid, oreclist in output_records.items():
for outrec in oreclist:
writer.writerow(outrec)
for pid, oreclist in output_records.items():
db.set(pid, dumps(oreclist))
return 0
class Crypt(Logger):
def __init__(self,
name=None,
fn=None,
use_secret=CRYPT_USE_SECRET,
path_secret=PATH_CRYPT_SECRET,
encrypt_values=False,
encryptor_func=None,
decryptor_func=None):
# defaults
if not name and fn: name = os.path.basename(fn).replace('.', '_')
self.name, self.fn = name, fn
# use secret? for salting
if use_secret and path_secret:
if not os.path.exists(path_secret):
self.secret = get_random_binary_id()
from comrad.backend.keymaker import make_key_discreet
self.log('shhh! creating secret:',
make_key_discreet(self.secret))
with open(path_secret, 'wb') as of:
of.write(self.secret)
else:
with open(path_secret, 'rb') as f:
self.secret = f.read()
else:
self.secret = b''
self.encrypt_values = encrypt_values
if self.secret and encrypt_values and (not encryptor_func
or not decryptor_func):
from comrad.backend.keymaker import ComradSymmetricKeyWithPassphrase
self.key = ComradSymmetricKeyWithPassphrase(passphrase=self.secret)
encryptor_func = self.key.encrypt
decryptor_func = self.key.decrypt
self.encryptor_func = encryptor_func
self.decryptor_func = decryptor_func
# self.store = FilesystemStore(self.fn)
# self.store = RedisStore(redis.StrictRedis())
# self.db = Vedis(self.fn)
# self.db = WalrusLite(self.fn)
# import hirlite
# self.db = hirlite.Rlite(path=self.fn)
from pupdb.core import PupDB
self.db = PupDB(self.fn)
def log(self, *x, **y):
if LOG_GET_SET:
super().log(*x)
def hash(self, binary_data):
# return binary_data
return b64enc_s(hasher(b64dec(binary_data), self.secret))
def force_binary(self, k_b):
if k_b is None: return None
if type(k_b) == str: k_b = k_b.encode()
if type(k_b) != bytes: k_b = k_b.decode()
return k_b
def package_key(self, k, prefix=''):
return b64enc_s(prefix) + b64enc_s(k)
def package_val(self, v, encrypt=None):
if encrypt is None: encrypt = self.encrypt_values
v_b = self.force_binary(v)
if encrypt:
try:
v_b = self.encryptor_func(v_b)
except ThemisError as e:
self.log('!! ENCRYPTION ERROR:', e)
v = b64enc_s(v)
return v
def unpackage_val(self, v, encrypt=None):
v_b = b64dec(v)
if encrypt is None: encrypt = self.encrypt_values
if encrypt:
try:
v_b = self.decryptor_func(v_b)
except ThemisError as e:
self.log('!! DECRYPTION ERROR:', e)
return v_b
def has(self, k, prefix=''):
got = self.get(k, prefix=prefix)
# self.log('has got',got)
return bool(got)
def set(self, k, v, prefix='', override=False, encrypt=True):
if self.has(k, prefix=prefix) and not override:
self.log(
f"I'm afraid I can't let you do that, overwrite someone's data!\n\nat {prefix}{k} = {v}"
)
return False #(False,None,None)
k_b = self.package_key(k, prefix=prefix)
k_b_hash = self.hash(k_b)
v_b = self.package_val(v, encrypt=(self.encrypt_values and encrypt))
if not override:
self.log(f'''Crypt.set(\n\t{k_b}\n\n\t{k_b_hash}\n\n\t{v_b}\n)''')
#self.store.put(k_b_hash,v_b)
#with self.db.transaction():
# self.db[k_b_hash]=v_b
# return self.db.command('set',k_b_hash,v_b)
return self.db.set(k_b_hash, v_b)
# return True
def exists(self, k, prefix=''):
return self.has(k, prefix=prefix)
def key2hash(self, k, prefix=''):
return self.hash(self.package_key(prefix + k))
def delete(self, k, prefix=''):
k_b = self.package_key(k, prefix=prefix)
k_b_hash = self.hash(k_b)
# v = self.db.command('del',k_b_hash)
v = self.db.remove(k_b_hash)
self.log('<--', v)
return v
def get(self, k, prefix=''):
k_b = self.package_key(k, prefix=prefix)
k_b_hash = self.hash(k_b)
# v=self.db.get(k_b_hash)
self.log('getting k', k, 'with prefix', prefix)
self.log('getting k_b', k_b)
self.log('getting k_b_hash', k_b_hash)
# v = self.db.command('get',k_b_hash)
v = self.db.get(k_b_hash)
self.log('<--', v)
v_b = self.unpackage_val(v)
return v_b