def move_file_and_test(self, src_f, targ_f):
"""Helper function that performs a dupe file move and tests it."""
# Initial state
self.cr_load("dupes-initial")
sm = SettingsManager(self.loc_in_cr('kpawebgen.json'))
db_path = os.path.join(sm.abspath('kpa_dir'), shadow)
index_p = self.loc_in_cr("KPA/index.xml")
kpa_p = self.loc_in_cr("KPA")
self.cr_nexist(db_path)
# Create shadow DB
u = Updater(sm.abspath('kpa_dir'), shadow)
did = u.update()
self.assertTrue(did)
conn = sqlite3.connect(db_path)
def getall(q, args=()):
return conn.execute(q, args).fetchall()
def count_images():
return getall("""SELECT COUNT(*) FROM image_id""")[0][0]
(srcID, srcMD5) = getall(
"""SELECT imageID, md5 FROM image_id WHERE path = ?""",
(src_f,))[0]
oldCount = count_images()
# Move a file (in filesystem and XML)
with open(index_p, 'r') as f:
xml = f.read()
xml = xml.replace(src_f, targ_f)
with open(index_p, 'w') as f:
f.write(xml)
os.rename(os.path.join(kpa_p, src_f),
os.path.join(kpa_p, targ_f))
# Update shadow DB
u = Updater(sm.abspath('kpa_dir'), shadow)
did = u.update()
self.assertTrue(did)
# Verify move in shadow DB
self.assertEqual(getall("""
SELECT action, oldPath, newPath
FROM changelog
WHERE changelog.logID = (SELECT MAX(logID) FROM changelog);"""),
[('move', src_f, targ_f)])
(targID, targMD5) = getall(
"""SELECT imageID, md5 FROM image_id WHERE path = ?""",
(targ_f,))[0]
self.assertEqual(targID, srcID)
self.assertEqual(targMD5, srcMD5)
self.assertEqual(count_images(), oldCount)
conn.close()
class TestSettingsManager(TestBase):
#== Util ==#
s = None
"""SettingsManager for main testing config file."""
def setUp(self):
self.s = SettingsManager('unit/settings/various.json', self.get_dir())
#== Tests ==#
def test_missing(self):
"""Missing config files must throw an error."""
self.expect_error(lambda:SettingsManager('does_not_exist.json'), 'Invalid path to settings file')
def test_override(self):
"""User properties must override defaults."""
self.assertEqual(self.s.require('gal_db_path'), 'foo.bar')
def test_require(self):
"""Retrieval of required non-empty properties."""
self.expect_error(lambda: self.s.require('12345'), 'missing key',
'Require throws error for missing key')
self.assertEqual(self.s.require('data_dir'), 'data')
def test_catlist(self):
"""Retrieval of lists."""
self.assertEqual(self.s.require('cats_unconditional', list), ['Foo','*Bar '])
def test_default_string(self):
"""String is expected by default."""
self.expect_error(lambda:self.s.require('cats_unconditional'),
"Config file needed <type")
def test_abspath(self):
"""Computation of absolute path from absolute and relative paths."""
self.assertEqual(self.s.abspath('test/abs'), '/tmp')
actual = self.s.abspath('test/rel')
expected = os.path.join(self.get_dir(), 'unit/settings/beyond')
self.assertEqual(os.path.realpath(actual), os.path.realpath(expected))
def test_read_standard(self):
"""Read a KPA database and check that the result makes sense."""
self.cr_load("standard")
sm = SettingsManager(self.loc_in_cr('kpawebgen.json'))
r = Reader(sm.abspath('kpa_dir'))
m = r.read_latest_data()
self.assertEqual(7, len(m.images))
beetle = (i for i in m.images if i['path'] == 'IMG_5179.JPG').next()
beetle_tags = beetle['tags']
del beetle['tags']
beetle_timestamp = datetime.datetime(2009, 6, 25, 7, 45, 50)
expected_beetle = {'imageID':None,
'md5':'34f7adfce54bd34bf5f0266dd2c7b800',
'path':'IMG_5179.JPG',
'width':800,
'height':600,
'angle':0,
'label':'IMG_5179',
'description':'Regular public.',
'stackOrder':None,
'stackID':None,
'startDate':beetle_timestamp,
'endDate':beetle_timestamp}
self.assertEqual(beetle, expected_beetle)
# next() throws if no match
locat = (i for i in m.cats if i['catName'] == 'Location').next()
inn_tag = (i for i in m.tags if i['tagName'] == 'Orange Hill Beach Inn' and
i['catID'] == locat['catID']).next()
island_tag = (i for i in m.tags
if i['catID'] == locat['catID'] and
i['tagName'] == 'San Salvador Island').next()
self.assertEqual(11, len(beetle_tags)) # no inherit
beetle_tags.index({'tagID':inn_tag['tagID'],
'catID':locat['catID']})
self.assertEqual(3, len(m.tag_tags))
m.tag_tags.index({'catID':locat['catID'],
'setID':island_tag['tagID'],
'tagID':inn_tag['tagID']})
class KPAWebGen(object):
"""Core code for gallery generator."""
#== Setup ==#
# A sequence is an array of 1 or more actions.
# Actions can be specified by names.
def __init__(self):
# source of lookup table
actions = [('read', self.act_read),
('localweb', self.act_localweb),
('publish', self.act_publish)]
# save off the ordered names and the lookup table
self.action_names = zip(*actions)[0]
self.action_lookup = dict(actions)
# TODO: Python subproject shouldn't know about global directory structure
root = '../../..'
self.clj_path = ResourceManager.locate(root + '/clj')
action_names = None
"""Names of program actions, in order."""
action_lookup = None
"""Dict of action names to functions."""
sequence_all = 'all'
"""Specifies that all actions should be taken (in order.)"""
clj_path = None
"""Location of Clojure jar."""
settings = None
"""SettingsManager object."""
# String -> List<String>
def get_sequence(self, seq_name):
"""Expand requested sequence into action names. If invalid sequence name,
throw error with message about allowed names."""
if seq_name == self.sequence_all:
return self.action_names
if seq_name not in self.action_names:
ls = ", ".join(list(self.action_names)+[self.sequence_all])
raise Exception("Sequence must be one of: %s" % (ls,))
return [seq_name]
# String List<String> ->
def dispatch(self, config_loc, sequence):
"""Dispatch on action names."""
self.settings = SettingsManager(config_loc)
for name in sequence:
print("Running action: %s" % name, file=sys.stdout)
self.action_lookup[name]()
def invoke_clojure_action(self, task):
# We have to use lein instead of launching an uberjar since some of
# the integration tests are in Clojure, and therefore need to call
# the .py script...
# Once everything is in Clojure, this won't be an issue.
subprocess.check_call(['lein', 'trampoline', 'run',
'--task', task,
'--config', self.settings.config['_config_path'],
'--debug', str(Global.debug).lower()],
shell=False, cwd=self.clj_path)
#== Actions ==#
def act_read(self):
"""Update our DB with the latest KPA data."""
did = Updater(self.settings.abspath('kpa_dir'),
self.settings.require('shadow_path')).update(self.force_read)
if not did:
print("Skipping read since DB has not changed.", file=sys.stdout)
def act_localweb(self):
"""Generate local website."""
self.invoke_clojure_action('localweb')
def act_publish(self):
"""Publish to remote website."""
self.invoke_clojure_action('publish-s3')
#== Interfaces ==#
force_read = False
def main(self, args):
"""Validate arguments (minus script name) and call core code."""
logging.debug('Main args: %s' % args)
if len(args) < 2:
if len(args) != 1 or args[0] != '--help':
print("Wrong number of arguments.", file=sys.stderr)
self.die_usage()
config_loc = args[0]
sequence_name = args[1]
if len(args) == 3 and args[2] == '--force-read':
self.force_read = True
try:
sequence = self.get_sequence(sequence_name)
except Exception as e:
KPAWebGen.die_top("Could not determine requested action: " + str(e),
Constants.ERR_ACTION_INVALID)
try:
self.dispatch(config_loc, sequence)
except Exception as e:
msg = "Error(%s): %s" % (e.__class__.__name__, e)
KPAWebGen.die_top(msg, Constants.ERR_UNKNOWN_REASON)
@staticmethod
def die_usage():
usage = """Usage:
kpawebgen --help
kpawebgen config-file sequence [options]
Options:
--force-read
Update the shadow DB even if it dopesn't appear to need it."""
KPAWebGen.die_top(usage, Constants.ERR_BAD_ARGS, isExc=False)
@staticmethod
def die_top(msgs=None, code=Constants.ERR_NORMAL_EXIT, isExc=True):
"""Die at the top level. Do not call from core code."""
if Global.debug and isExc:
traceback.print_exc(file=sys.stdout)
if isinstance(msgs, basestring):
msgs = [msgs]
if msgs:
for m in msgs:
print(m, file=sys.stderr)
sys.exit(code)
def test_firstrun(self):
"""Must create a DB3 if none exists."""
self.cr_load("firstrun")
sm = SettingsManager(self.loc_in_cr('kpawebgen.json'))
db_path = os.path.join(sm.abspath('kpa_dir'), shadow)
u = Updater(sm.abspath('kpa_dir'), shadow)
self.cr_nexist(db_path)
did = u.update()
self.assertTrue(did)
self.cr_exist(db_path)
conn = sqlite3.connect(db_path)
def getall(q):
return conn.execute(q).fetchall()
self.assertNotEqual(getall('SELECT xmlChecksum FROM metadata')[0][0], None)
self.assertEqual(getall('SELECT nextImageID FROM metadata'), [(7,)])
self.assertEqual(getall('SELECT count(*) FROM image_id'), [(7,)])
self.assertEqual(getall('SELECT min(imageID) FROM image_id'), [(0,)])
self.assertEqual(getall('SELECT max(imageID) FROM image_id'), [(6,)])
self.assertEqual(getall('SELECT count(*) FROM image_int'), [(7,)])
self.assertEqual(getall('SELECT count(*) FROM image_ext'), [(7,)])
self.assertEqual(getall('SELECT count(*) FROM changelog'), [(7,)])
self.assertEqual(getall('SELECT DISTINCT oldPath, oldMD5 FROM changelog'),
[(None,None)])
hornetID = getall("""SELECT imageID FROM image_id
WHERE path='IMG_9610.JPG'""")[0][0]
hornetInt = conn.execute("SELECT * FROM image_int WHERE imageID=?",
(hornetID,)).fetchall()[0]
conn.close()
# Create the baseline for several changes
self.dump_db(db_path, 'baseline.dump')
db_backup = self.loc_in_cr('shadow.db3.firstrun')
subprocess.call(['cp', db_path, db_backup])
# No changes yet
u = Updater(sm.abspath('kpa_dir'), shadow)
did = u.update()
self.assertFalse(did) # Don't update if nothing has changed
# Alter index: Delete and move
self.patch("KPA/index.xml", "patches/delete+move.index.xml.patch")
u = Updater(sm.abspath('kpa_dir'), shadow)
did = u.update()
self.assertTrue(did)
self.dump_db(db_path, 'delete+move.dump')
self.validate_diff("baseline.dump", "patches/delete+move.dump.patch",
"delete+move.dump")
self.patch("KPA/index.xml", "patches/delete+move.index.xml.patch",
undo=True)
subprocess.call(['rm', db_path])
subprocess.call(['cp', db_backup, db_path])
# Alter index: Create and edit
self.patch("KPA/index.xml", "patches/create+edit.index.xml.patch")
u = Updater(sm.abspath('kpa_dir'), shadow)
did = u.update()
self.assertTrue(did)
self.dump_db(db_path, 'create+edit.dump')
self.validate_diff("baseline.dump", "patches/create+edit.dump.patch",
"create+edit.dump")
self.patch("KPA/index.xml", "patches/create+edit.index.xml.patch",
undo=True)
subprocess.call(['rm', db_path])
subprocess.call(['cp', db_backup, db_path])
