def test_get_and_set():
""" Tests the get() and set() methods of PupDB. """
database = PupDB(TEST_DB_PATH)
key, val = 'test_key', list(range(100))
database.set(key, val)
assert database.get(key) == val
def write_to_db(data_range):
""" Method performs write to the database."""
database = PupDB(TEST_DB_PATH)
for i in data_range:
database.set(i, i)
logging.info('Process %s wrote range(%s, %s) to DB successfully.',
os.getpid(), data_range.start, data_range.stop)
def test_values():
""" Tests the values() method of PupDB. """
database = PupDB(TEST_DB_PATH)
range_list = list(range(10))
for i in range_list:
database.set(i, i)
db_values_list = list(database.values())
assert sorted(db_values_list) == range_list
def test_keys():
""" Tests the keys() method of PupDB. """
database = PupDB(TEST_DB_PATH)
range_list = [str(i) for i in range(10)]
for i in range_list:
database.set(i, i)
db_keys_list = list(database.keys())
assert sorted(db_keys_list) == range_list
def test_length():
"""
Tests whether len() on the PupDB instance returns it's proper length.
"""
database = PupDB(TEST_DB_PATH)
num_items = 100
range_list = list(range(num_items))
for i in range_list:
database.set(i, i)
assert len(database) == 100
def test_dumps():
""" Tests the dumps() method of PupDB. """
database = PupDB(TEST_DB_PATH)
range_list = list(range(10))
for i in range_list:
database.set(i, i)
db_dict = {str(i): i for i in range_list}
assert database.dumps() == json.dumps(db_dict, sort_keys=True)
def test_items():
""" Tests the items() method of PupDB. """
database = PupDB(TEST_DB_PATH)
range_list = list(range(10))
for i in range_list:
database.set(i, i)
items_list = [(str(i), i) for i in range_list]
db_items_list = list(database.items())
assert sorted(db_items_list) == items_list
def test_get_database():
"""
Tests whether _get_database() returns the dict for the DB.
"""
database = PupDB(TEST_DB_PATH)
num_items = 100
range_list = list(range(num_items))
for i in range_list:
database.set(i, i)
# pylint: disable=protected-access
db_dict = database._get_database()
assert json.dumps({str(i): i for i in range_list}, sort_keys=True) == \
json.dumps(db_dict, sort_keys=True)
def test_remove():
""" Tests the remove() method of PupDB. """
database = PupDB(TEST_DB_PATH)
for i in range(10):
database.set(i, i)
# Removing key 0 from the database.
database.remove(0)
for i in range(10):
if i == 0:
assert database.get(i) is None
else:
assert database.get(i) is not None
# Check the behavaiour for a non-existent key.
with pytest.raises(KeyError):
database.remove(1000)
class PupDBWriterThread(Thread):
"""
This class represents an instance of a thread that writes
to the database.
"""
def __init__(self, data_range, database=None):
""" Instance Initialization """
super(PupDBWriterThread, self).__init__()
self.data_range = data_range
if database is None:
self.database = PupDB(TEST_DB_PATH)
else:
self.database = database
def run(self):
""" Method performs write to the database."""
for i in self.data_range:
self.database.set(i, i)
def Main(credentials):
"""Main function declaration.
requires:
credentials: tuple (Twitter credentials)
"""
defineLoggers()
storage_db = PupDB(
PupDB_FILENAME) # activate PupDB file for persistent storage
last_tweet = storage_db.get(PupDB_MRTkey)
last_level = storage_db.get(PupDB_MRLkey)
if last_tweet is None: # Pre-load empty database
last_tweet = str(LOCAL_NOW_STRING())
last_level = MINIMUM_CONCERN_LEVEL
storage_db.set(PupDB_MRTkey, last_tweet)
storage_db.set(PupDB_MRLkey, last_level)
forecast_level = last_level
storage_db.set(PupDB_MRFkey, forecast_level)
# initialization complete. Begin main loop.
# check river database and update if needed
# tweet status if needed and update tweet database
# update any local display devices
return False # should never end
class Crypt(Logger):
def __init__(self,
name=None,
fn=None,
use_secret=CRYPT_USE_SECRET,
path_secret=PATH_CRYPT_SECRET,
encrypt_values=False,
encryptor_func=None,
decryptor_func=None):
# defaults
if not name and fn: name = os.path.basename(fn).replace('.', '_')
self.name, self.fn = name, fn
# use secret? for salting
if use_secret and path_secret:
if not os.path.exists(path_secret):
self.secret = get_random_binary_id()
from comrad.backend.keymaker import make_key_discreet
self.log('shhh! creating secret:',
make_key_discreet(self.secret))
with open(path_secret, 'wb') as of:
of.write(self.secret)
else:
with open(path_secret, 'rb') as f:
self.secret = f.read()
else:
self.secret = b''
self.encrypt_values = encrypt_values
if self.secret and encrypt_values and (not encryptor_func
or not decryptor_func):
from comrad.backend.keymaker import ComradSymmetricKeyWithPassphrase
self.key = ComradSymmetricKeyWithPassphrase(passphrase=self.secret)
encryptor_func = self.key.encrypt
decryptor_func = self.key.decrypt
self.encryptor_func = encryptor_func
self.decryptor_func = decryptor_func
# self.store = FilesystemStore(self.fn)
# self.store = RedisStore(redis.StrictRedis())
# self.db = Vedis(self.fn)
# self.db = WalrusLite(self.fn)
# import hirlite
# self.db = hirlite.Rlite(path=self.fn)
from pupdb.core import PupDB
self.db = PupDB(self.fn)
def log(self, *x, **y):
if LOG_GET_SET:
super().log(*x)
def hash(self, binary_data):
# return binary_data
return b64enc_s(hasher(b64dec(binary_data), self.secret))
def force_binary(self, k_b):
if k_b is None: return None
if type(k_b) == str: k_b = k_b.encode()
if type(k_b) != bytes: k_b = k_b.decode()
return k_b
def package_key(self, k, prefix=''):
return b64enc_s(prefix) + b64enc_s(k)
def package_val(self, v, encrypt=None):
if encrypt is None: encrypt = self.encrypt_values
v_b = self.force_binary(v)
if encrypt:
try:
v_b = self.encryptor_func(v_b)
except ThemisError as e:
self.log('!! ENCRYPTION ERROR:', e)
v = b64enc_s(v)
return v
def unpackage_val(self, v, encrypt=None):
v_b = b64dec(v)
if encrypt is None: encrypt = self.encrypt_values
if encrypt:
try:
v_b = self.decryptor_func(v_b)
except ThemisError as e:
self.log('!! DECRYPTION ERROR:', e)
return v_b
def has(self, k, prefix=''):
got = self.get(k, prefix=prefix)
# self.log('has got',got)
return bool(got)
def set(self, k, v, prefix='', override=False, encrypt=True):
if self.has(k, prefix=prefix) and not override:
self.log(
f"I'm afraid I can't let you do that, overwrite someone's data!\n\nat {prefix}{k} = {v}"
)
return False #(False,None,None)
k_b = self.package_key(k, prefix=prefix)
k_b_hash = self.hash(k_b)
v_b = self.package_val(v, encrypt=(self.encrypt_values and encrypt))
if not override:
self.log(f'''Crypt.set(\n\t{k_b}\n\n\t{k_b_hash}\n\n\t{v_b}\n)''')
#self.store.put(k_b_hash,v_b)
#with self.db.transaction():
# self.db[k_b_hash]=v_b
# return self.db.command('set',k_b_hash,v_b)
return self.db.set(k_b_hash, v_b)
# return True
def exists(self, k, prefix=''):
return self.has(k, prefix=prefix)
def key2hash(self, k, prefix=''):
return self.hash(self.package_key(prefix + k))
def delete(self, k, prefix=''):
k_b = self.package_key(k, prefix=prefix)
k_b_hash = self.hash(k_b)
# v = self.db.command('del',k_b_hash)
v = self.db.remove(k_b_hash)
self.log('<--', v)
return v
def get(self, k, prefix=''):
k_b = self.package_key(k, prefix=prefix)
k_b_hash = self.hash(k_b)
# v=self.db.get(k_b_hash)
self.log('getting k', k, 'with prefix', prefix)
self.log('getting k_b', k_b)
self.log('getting k_b_hash', k_b_hash)
# v = self.db.command('get',k_b_hash)
v = self.db.get(k_b_hash)
self.log('<--', v)
v_b = self.unpackage_val(v)
return v_b
def main():
parser = ArgumentParser()
parser.add_argument('--reportdir',
default='reports',
metavar='D',
help='directory containing report files')
parser.add_argument(
'--xactfile',
default=None,
metavar='F',
help='csv file containing financial transaction report')
parser.add_argument('--xerofile',
default=None,
metavar='F',
help='output csv file for xero pre-coded transactions')
parser.add_argument('--pupdbfile',
default=None,
metavar='F',
help='json file for pupdb key-value store')
parser.add_argument('--dryrun',
'-n',
action='store_true',
help='dont make any actual changes - just run through')
parser.add_argument('--verbose',
'-v',
action='store_true',
help='print verbose messages')
args = parser.parse_args()
reportdir = args.reportdir
if not os.path.isdir(reportdir):
reportdir = os.path.join(xerodir, args.reportdir)
if not os.path.isdir(reportdir):
raise RuntimeError('cannot locate reports directory!')
if args.verbose:
print('[reports found in directory {} (realpath={})]'.format(
reportdir, os.path.realpath(reportdir)),
file=sys.stderr)
xactfile = args.xactfile
if xactfile is None:
xactfile, _ = latest_report(
'transactions', reportdir, r'^transactions_(\d{8})\.csv$',
lambda m: datetime.strptime(m.group(1), '%Y%m%d'), args.verbose)
if xactfile is None:
raise RuntimeError('cannot locate transaction file!')
if args.verbose:
print('[transaction report file = {} (realpath={})]'.format(
xactfile, os.path.realpath(xactfile)),
file=sys.stderr)
xerofile = args.xerofile
if xerofile is None:
xerofile = os.path.join(
xerodir,
datetime.now().strftime('xero-upload-%Y%m%d.csv'))
if args.verbose:
print('[Xero output csv file = {} (realpath={})]'.format(
xerofile, os.path.realpath(xerofile)),
file=sys.stderr)
pupdbfile = args.pupdbfile
if pupdbfile is None:
pupdbfile = os.path.join(xerodir, 'uploaded.json')
if args.verbose:
print('[pupdb json file = {} (realpath={})]'.format(
pupdbfile, os.path.realpath(pupdbfile)),
file=sys.stderr)
config = load_config(prefix=xerodir)
db = PupDB(pupdbfile)
fieldnames = [
'Date',
'Amount',
'Payee',
'Description',
'Reference',
'Cheque Number',
'Account code',
'Tax Rate (Display Name)',
'Tracking1',
'Tracking2',
'Transaction Type',
'Analysis code',
]
voucher_desc = 'Get Active Kids Voucher Program'
with open(xactfile, 'r', newline='') as infile:
reader = DictReader(infile)
output_records = {}
already_uploaded = {}
order_numbers = []
order_item_ids = []
total_netamount = Decimal('0.00')
total_phqfee = Decimal('0.00')
total_subtotal = Decimal('0.00')
total_pending = Decimal('0.00')
total_gvapplied = Decimal('0.00')
for inrec in reader:
org = inrec['Organisation']
role = inrec['Role']
org_to = inrec['Organisation Registering To']
pstatus = inrec['Payout Status']
netamount = Decimal(inrec['Net Amount'][1:])
if (org != 'Shooters Basketball Club' or role != 'Player'
or org_to != 'Shooters Basketball Club'
or pstatus != 'DISBURSED'):
if args.verbose:
print('skip (bad rec): org={}, role={}, org_to={}, '
'pstatus={}'.format(org, role, org_to, pstatus),
file=sys.stderr)
if pstatus == 'DISBURSEMENT_PENDING':
total_pending += netamount
continue
rtype = inrec['Type of Registration']
rname = inrec['Registration']
ptype = inrec['Product Type']
for rdesc in config['types']:
if (rdesc['rtype'] == rtype and rdesc['rname'] == rname
and rdesc['ptype'] == ptype):
break
else:
raise RuntimeError(
'type not found: rtype={}, rname={}, ptype={}'.format(
rtype, rname, ptype))
dfmt = '%d/%m/%Y'
sname = inrec['Season Name']
xdate = to_date(inrec['Date'], dfmt)
name = inrec['Name']
onum = inrec['Order Number']
oid = inrec['Order Item ID']
oprice = Decimal(inrec['Order Item Price'][1:])
gvname = inrec['Government Voucher Name']
if gvname != '':
if gvname != 'Get Active Kids':
raise RuntimeError('bad gov voucher: {}'.format(gvname))
sgva = inrec['Government Voucher Amount']
gvamount = Decimal(sgva[1:])
if gvamount != Decimal('200.00'):
raise RuntimeError(
'GAK voucher not $200: {:.2f}'.format(gvamount))
sgvaa = inrec['Government Voucher Amount Applied']
gvapplied = Decimal(sgvaa[1:])
else:
gvamount = Decimal('0.00')
gvapplied = Decimal('0.00')
product = inrec['Product Name']
quantity = int(inrec['Quantity'])
subtotal = Decimal(inrec['Subtotal'][1:])
phqfee = Decimal(inrec['PlayHQ Fee'][1:])
pdate = to_date(inrec['Payout Date'], '%Y-%m-%d')
pid = inrec['Payout ID']
if rdesc['rid'] == 'clinic':
sku = inrec['Merchandise SKU']
if sku not in rdesc['skus']:
raise RuntimeError('sku not found: {} not in {}'.format(
sku, rdesc['skus']))
# Term N, YYYY
m = re.match(r'^Term ([1-4]), (\d{4})$', sname)
if m is None:
raise RuntimeError(
'clinic record has bad season name ({})'.format(sname))
tracking1 = rdesc['tracking1'].format(*m.groups())
tracking2 = rdesc['tracking2']
elif rdesc['rid'] == 'registration':
feename = inrec['Fee Name']
for fdesc in rdesc['fees']:
if fdesc['name'] == feename:
famount = Decimal(fdesc['amount'])
break
else:
raise RuntimeError(
'fee not found: rtype={}, rname={}, ptype={}'.format(
rtype, rname, ptype))
if quantity != 1:
raise RuntimeError('registration with quantity != 1!')
if famount != oprice:
raise RuntimeError(
'fee amount mismatch ({:.2f}!={:.2f})'.format(
famount, oprice))
# (Winter|Summer) YYYY
m = re.match(r'^(Winter|Summer) (\d{4})$', sname)
if m is None:
raise RuntimeError(
'rego record has bad season name ({})'.format(sname))
wors, year = m.groups()
if wors == 'Summer':
year = '{}/{:02d}'.format(year, int(year) - 2000 + 1)
tracking1 = rdesc['tracking1'].format(year, wors)
tracking2 = rdesc['tracking2']
else:
raise RuntimeError('bad rego id {}!'.format(rdesc['rid']))
if (oprice - gvapplied) * quantity != subtotal:
raise RuntimeError(
'oprice({:.2f})*quantity({})!=subtotal({:.2f})'.format(
oprice, quantity, subtotal))
if subtotal != netamount + phqfee:
raise RuntimeError(
'subtotal({:.2f})!=netamount({:.2f})+phqfee({:.2f})'.
format(
subtotal,
netamount,
phqfee,
))
if onum in order_numbers:
raise RuntimeError('duplicate order number {}!'.format(onum))
order_numbers.append(onum)
if oid in order_item_ids:
raise RuntimeError('duplicate order item id {}!'.format(oid))
order_item_ids.append(oid)
if db.get(pid) is not None:
is_already_uploaded = True
if pid not in already_uploaded:
already_uploaded[pid] = []
if args.verbose:
print('already uploaded: name={}, pdate={}, pid={}'.format(
name, pdate, pid),
file=sys.stderr)
else:
is_already_uploaded = False
if pid not in output_records:
output_records[pid] = []
total_netamount += netamount
total_phqfee += phqfee
total_subtotal += subtotal
total_gvapplied += gvapplied
desc = '{} - ${:.2f} x {:d}'.format(product, oprice, quantity)
orec = {
'Date': xdate.strftime(dfmt),
'Amount': '{:.2f}'.format(subtotal),
'Payee': name,
'Description': '{}: subtotal'.format(desc),
'Reference': '{} on {}'.format(pid, pdate.strftime(dfmt)),
'Cheque Number': onum,
'Account code': config['sales_account'],
'Tax Rate (Display Name)': config['taxrate'],
'Tracking1': tracking1,
'Tracking2': tracking2,
'Transaction Type': 'credit',
'Analysis code': oid,
}
if is_already_uploaded:
already_uploaded[pid].append(orec)
else:
output_records[pid].append(orec)
if phqfee != Decimal('0.00'):
orec = {
'Date': xdate.strftime(dfmt),
'Amount': '-{:.2f}'.format(phqfee),
'Payee': name,
'Description': '{}: playhq fees'.format(desc),
'Reference': '{} on {}'.format(pid, pdate.strftime(dfmt)),
'Cheque Number': onum,
'Account code': config['fees_account'],
'Tax Rate (Display Name)': config['taxrate'],
'Tracking1': tracking1,
'Tracking2': tracking2,
'Transaction Type': 'debit',
'Analysis code': oid,
}
if is_already_uploaded:
already_uploaded[pid].append(orec)
else:
output_records[pid].append(orec)
if gvapplied != Decimal('0.00'):
orec = {
'Date': xdate.strftime(dfmt),
'Amount': '{:.2f}'.format(gvapplied),
'Payee': voucher_desc,
'Description': '{}: get active for {}'.format(desc, name),
'Reference': '{} on {}'.format(pid, pdate.strftime(dfmt)),
'Cheque Number': onum,
'Account code': config['other_revenue_account'],
'Tax Rate (Display Name)': config['taxrate'],
'Tracking1': tracking1,
'Tracking2': tracking2,
'Transaction Type': 'credit',
'Analysis code': oid,
}
if is_already_uploaded:
already_uploaded[pid].append(orec)
else:
output_records[pid].append(orec)
for pid, oreclist1 in already_uploaded.items():
total_amount1 = sum(
Decimal(orec['Amount']) for orec in oreclist1
if orec['Payee'] != voucher_desc)
dbval = db.get(pid)
if dbval is None:
raise RuntimeError('db get of pid {} failed!'.format(pid))
oreclist2 = loads(dbval)
if not isinstance(oreclist2, list):
if args.verbose:
print('cannot check old record: pid={}, amount=${:.2f}'.format(
pid, total_amount1),
file=sys.stderr)
continue
total_amount2 = sum(
Decimal(orec['Amount']) for orec in oreclist2
if orec['Payee'] != voucher_desc)
if total_amount1 != total_amount2:
raise RuntimeError(
'pid {} total amount mismatch (${:.2f} != ${:.2f})!'.format(
pid, total_amount1, total_amount2))
if args.verbose:
print('checked already uploaded: pid={}, amount=${:.2f}'.format(
pid, total_amount1),
file=sys.stderr)
if args.verbose and total_pending > 0:
print('total pending = ${:.2f}'.format(total_pending), file=sys.stderr)
if len(output_records) == 0:
print('No records were collected.', file=sys.stderr)
return 0
if total_subtotal - total_phqfee != total_netamount:
raise RuntimeError('total({:.2f})-fees({:.2f})!=net({:.2f})'.format(
total_subtotal, total_phqfee, total_netamount))
if args.verbose:
print('{} payment ids were collected.'.format(len(output_records)),
file=sys.stderr)
print('subtotal = ${:.2f}'.format(total_subtotal), file=sys.stderr)
print('phqfee = ${:.2f}'.format(total_phqfee), file=sys.stderr)
print('netamount = ${:.2f}'.format(total_netamount), file=sys.stderr)
print('gov vouchers = ${:.2f}'.format(total_gvapplied),
file=sys.stderr)
for pid, oreclist in output_records.items():
amount = Decimal(0.0)
for outrec in oreclist:
if outrec['Payee'] != voucher_desc:
amount += Decimal(outrec['Amount'])
print(' Payment Id {} = ${:.2f}'.format(pid, amount),
file=sys.stderr)
if args.dryrun:
return 0
if os.path.exists(xerofile):
raise RuntimeError('will not overwrite file {}'.format(xerofile))
with open(xerofile, 'w', newline='') as outfile:
writer = DictWriter(outfile, fieldnames=fieldnames)
writer.writeheader()
for pid, oreclist in output_records.items():
for outrec in oreclist:
writer.writerow(outrec)
for pid, oreclist in output_records.items():
db.set(pid, dumps(oreclist))
return 0