def branch_hier_enforce_branch(self, branch_tup, branch_update_enabled):
'''
Checks the user's access to a branch at the requested revision, or
revisions, in the case of a Diff.
We always check the user's access at the latest revision, to understand
what their access to the current branch really is.
'''
log.verbose1('self.req.revision.rev: %s' % (self.req.revision.rev, ))
if isinstance(self.req.revision.rev, revision.Current):
if branch_update_enabled:
min_access = Access_Level.editor
else:
min_access = Access_Level.viewer
self.access_level_id = self.branch_hier_enforce_revision(
branch_tup, self.req.revision.rev, min_access)
elif isinstance(self.req.revision.rev, revision.Historic):
g.assurt(not branch_update_enabled)
self.branch_hier_enforce_revision(branch_tup,
self.req.revision.rev,
Access_Level.viewer)
# In historic mode, access is always viewer
self.access_level_id = Access_Level.viewer
elif isinstance(self.req.revision.rev, revision.Diff):
# The user needs view access or better to both revisions.
acl_viewer = Access_Level.viewer
g.assurt(not branch_update_enabled)
rev = revision.Historic(self.req.revision.rev.rid_old)
self.branch_hier_enforce_revision(branch_tup, rev, acl_viewer)
rev = revision.Historic(self.req.revision.rev.rid_new)
self.branch_hier_enforce_revision(branch_tup, rev, acl_viewer)
# In historic mode, access is always viewer
self.access_level_id = Access_Level.viewer
elif isinstance(self.req.revision.rev, revision.Updated):
log.error(
'Updated revision requested. This is only secure for scripts.')
raise GWIS_Error('GWIS does not support revision.Updated.')
elif isinstance(self.req.revision.rev, revision.Comprehensive):
self.access_level_id = self.branch_hier_enforce_revision(
branch_tup, revision.Current(), Access_Level.viewer)
else:
g.assurt(False)
def setup_qbs(self):
username = self.wtem.created_by
db = db_glue.new()
if self.wtem.revision_id is None:
rev = revision.Current()
else:
rev = revision.Historic(self.wtem.revision_id, allow_deleted=False)
(branch_id, branch_hier) = branch.Many.branch_id_resolve(db,
self.wtem.branch_id, branch_hier_rev=rev)
if not branch_hier:
raise Exception(
'Branch with stack_id %d not found in database at %s.'
% (self.wtem.branch_id, rev.short_name(),))
self.qb = Item_Query_Builder(db, username, branch_hier, rev)
self.qb.item_mgr = self.qb.item_mgr
Query_Overlord.finalize_query(self.qb)
def setup_qb_src(self, all_errs):
qb_src = None
username = self.mjob.wtem.created_by
# The source qb is just for reading...
db = db_glue.new()
# ... but we'll be making temporary tables of stack IDs, so start a
# transaction.
db.transaction_begin_rw()
# The byways in the conflated were not marked deleted when they were
# exported for conflation, so we don't need to look for deleted.
# NOTE: The original MetC import script used based rev off
# self.target_branch.last_merge_rid rather than what's in the
# config file.
g.assurt(self.spf_conf.revision_id)
revision_id = self.spf_conf.revision_id
rev = revision.Historic(revision_id, allow_deleted=False)
# Make the branch_hier.
(branch_id,
branch_hier) = branch.Many.branch_id_resolve(db,
self.mjob.wtem.branch_id,
branch_hier_rev=rev)
# Put it all together.
if branch_id is None:
all_errs.append('setup_qb_src: not a branch: %s at %s' % (
self.mjob.wtem.branch_id,
str(rev),
))
# Don't forget to close. Not too big a deal, but oddly, if we don't,
# the next attempt by this thread to get the db will result in the
# same DB() object being created and the same self.conn returned,
# and then db_glue complains that it's got that self and conn in
# conn_lookup.
db.close()
db = None
else:
g.assurt(branch_hier)
qb_src = Item_Query_Builder(db, username, branch_hier, rev)
# It's nice to have both the raw, opaque hexadecimal geometry as well
# as the WKT geometry, since not all APIs are that flexible, and also
# because it's easier to work with WKT in Python and OSGeo (and also
# because [lb] hasn't seen an OGR fcn. to convert raw PostGIS geom,
# but he's probably not looking hard enough).
qb_src.filters.skip_geometry_raw = False
qb_src.filters.skip_geometry_svg = True
qb_src.filters.skip_geometry_wkt = False
qb_src.item_mgr = Item_Manager()
# FIXME: Is this right? What about tgraph?
qb_src.item_mgr.load_cache_attachments(qb_src)
Query_Overlord.finalize_query(qb_src)
# Check that user has viewer access on the source branch.
source_branch = self.verify_branch_access(qb_src,
Access_Level.viewer,
all_errs)
# NOTE: The job itself is already access-controlled, so generally the
# user has arbiter access to the branch at the Current revision.
self.qb_src = qb_src
def make_branch(self):
# FIXME: If user specifies --branch, we should make the new branch
# descend from the specified branch.
import_callback, export_callback = self.get_branch_callbacks()
tile_skins = self.get_tile_skin_names()
# FIXME: Ensure that name is unique! I.e., check the new branch name and
# check that the required group names are available.
if self.cli_opts.is_basemap:
parent_id = None
else:
parent_id = self.qb.branch_hier[0][0]
last_merge_rid = self.cli_opts.last_merge_rid or self.qb.item_mgr.rid_new
log.info('Making new branch: "%s" / parent_id: %s / last_merge_rid: %s'
% (self.cli_opts.new_branch_name, parent_id, last_merge_rid,))
new_branch = branch.One(
qb=self.qb,
row={
# item_versioned
'system_id' : None, # assigned later
'branch_id' : None, # assigned later
'stack_id' : self.qb.item_mgr.get_next_client_id(),
'version' : 0,
'deleted' : False,
'reverted' : False,
'name' : self.cli_opts.new_branch_name,
'valid_start_rid' : None,
'valid_until_rid' : None,
# branch
'parent_id' : parent_id,
'last_merge_rid' : last_merge_rid,
'conflicts_resolved' : True,
'import_callback' : import_callback,
'export_callback' : export_callback,
'tile_skins' : tile_skins,
# Skipping: coverage_area. See gen_tilecache_cfg.py.
}
)
# Make the user who's running the script the branch owner.
# And give the new branch groups access, too.
g.assurt(self.qb.user_group_id)
# Also make an entry for the Public, so it's easy to set this.
# MAYBE: Maybe these are the groups and there are no other choices?
# That would really simply the client and other operations involving
# editing GIA records...
pub_grp_id = group.Many.public_group_id(self.qb.db)
#
target_groups = {
self.qb.user_group_id : Access_Level.owner,
self.sid_editors : Access_Level.editor,
self.sid_arbiters : Access_Level.arbiter,
self.sid_owners : Access_Level.owner,
pub_grp_id : Access_Level.denied,
}
# All branches are access-style 'permissive'. Cyclopath might some day
# support, e.g., 'usr_choice', on sub-branches, i.e., so an agency can
# let their users create their own branches. But right now, since
# branches are pretty specially wired into the system and require special
# scripts to setup and maintain, we'll just stick with 'permissive'.
new_branch.access_style_id = Access_Style.permissive
# access_infer_id is set from item_stack.save_core.get_access_infer().
# NOTE: We set the valid_start_rid to 1 so that the branch is viewable
# at historic revisions, i.e., that the user sees what the parent
# branch looked like back then. If we didn't do this, it makes some
# operations fail, i.e., valid_start_rid cannot be greater than
# last_merge_rid, so importing fails if the user cannot see the
# branch at the last_merge_rid.
# Skipping: self.qb.item_mgr.rid_new.
first_rid = 1
new_branch.prepare_and_save_item(self.qb,
target_groups=target_groups,
rid_new=first_rid,
ref_item=None)
log.info('Created branch: %s (%d)'
% (new_branch.name, new_branch.stack_id,))
# Make the branch_hier.
revision_id = self.qb.item_mgr.rid_new
rev = revision.Historic(revision_id, allow_deleted=False)
(branch_id, branch_hier) = branch.Many.branch_id_resolve(self.qb.db,
new_branch.stack_id, branch_hier_rev=rev)
self.qb.branch_hier_set(branch_hier)
# not needed: self.qb.revision = branch_hier[0][1]
# MEH: Set self.cli_args.branch_hier? No one should be using branch_hier
# from cli_args, so, why bother... let the assurts fly instead.
# Whatever: self.cli_args.branch_hier = branch_hier
self.the_branch = new_branch
def load_make_qb_new(self, rid_latest):
g.assurt(rid_latest > 0)
rid_min = self.rid_max
self.rid_max = rid_latest
if isinstance(self.revision, revision.Current):
if rid_min > 0:
update_only = True
else:
g.assurt(isinstance(self.revision, revision.Historic))
self.rid_max = self.revision.rid
rev = None
branch_hier = None
if rid_min == self.rid_max:
# The caller should already have checked that we have work to do.
log.error('load_make_qb_new: rid_min == self.rid_max')
rev_hist = None
else:
# We always need a historic revision, since we always update the attr
# and tag cache.
rev_hist = revision.Historic(self.rid_max)
# If rid_min is already set, do an Update.
if rid_min > 0:
log.debug('load_make_qb_new: fr. %d to %d' % (
rid_min,
self.rid_max,
))
g.assurt(isinstance(self.revision, revision.Current))
# If we've already loaded byways, we're updating the map,
# and we want to fetch changed byways, including deleted or
# restricted-access byways, so we can remove those edges from the
# transportation graph.
rev_fetch = revision.Updated(rid_min, self.rid_max)
else:
# We're loading the map for the first time.
rev_fetch = rev_hist
qb_fetch = None
if rev_hist is not None:
branch_hier = branch.Many.branch_hier_build(
self.update_db, self.branch_hier[0][0], rev_hist)
qb_fetch = Item_Query_Builder(self.update_db, self.username,
branch_hier, rev_fetch)
# The Item_Manager class will make a table of all changed items by
# stack_id, and it'll join that against a normal Historic query, so
# we need to keep the username for the Historic query.
# NO:
# if isinstance(rev_fetch, revision.Updated):
# qb_fetch.username = None
# qb_fetch.filters.gia_userless = True
# Because we're using revision.Updated, we need to tell search_get_sql
# not to worry.
qb_fetch.request_is_local = True
qb_fetch.request_is_script = False # True if user running it.
# This populates the user gids and sets up geometry queries. Neither
# or which should be necessary.
Query_Overlord.finalize_query(qb_fetch)
if rev_fetch != rev_hist:
qb_hist = Item_Query_Builder(self.update_db, self.username,
branch_hier, rev_hist)
Query_Overlord.finalize_query(qb_hist)
else:
qb_hist = qb_fetch
# Load the link_value caches for the byways, since we need tags and
# attributes for the cost function.
qb_fetch.item_mgr = Item_Manager()
# NOTE: Whether rev is Historic or Updated, we'll load attrs and tags
# for a specific revision ID. For Historic, we'll load them for the
# historic rev ID, and for Updated, we'll load 'em for rid_max.
# BUG nnnn: With too many tags... we'll want to have a service
# running to handle web requests (so they can always be resident)?
# bahh...
qb_fetch.item_mgr.load_cache_attachments(qb_hist)
return qb_fetch
def branch_hier_build(db,
branch_id,
mainline_rev,
diff_group=None,
latter_group=None):
# Build the branch hierarchy, which is a list of tuples. The first
# element is the branch ID and the second is the last_merge revision ID.
# The first branch ID is the leaf branch, then its parent, etc., and the
# baseline is the last item in the list, i.e., branch_hier[0] is the
# leaf branch ID and branch_hier[-1] is the baseline branch ID.
g.assurt(branch_id)
g.assurt(mainline_rev is not None)
# The tuples indicate at which revision to fetch items from a branch.
# So the last_merge_rid applies not to the self-same branch but to
# its ascendant. We start with the leaf branch, which is descendantless.
cur_branch_id = branch_id
prev_last_merge_rid = None
# Walk the hierarchy from leaf to root.
branch_hier = []
not_at_baseline = True
latter_group_i = 0
while not_at_baseline:
# We set a limit on the length of branch hierarchies in the CONFIG.
if len(branch_hier) > conf.max_parent_chain_len:
log.warning('Leaf branch %d has very long parent chain' %
(branch_id))
raise GWIS_Error('Branch hierarchy has too many generations.')
# Each branch must be checked out at the relevant revision.
# EXPLAIN: Link to the Wiki docs (and make the Wiki docs first).
# Explaining how to make the branch hier could use some
# graphics....
if prev_last_merge_rid is None:
if (isinstance(mainline_rev, revision.Current)
or isinstance(mainline_rev, revision.Historic)
or isinstance(mainline_rev, revision.Comprehensive)):
rev = mainline_rev
else:
if isinstance(mainline_rev, revision.Diff):
if (diff_group == 'latter') or (diff_group is None):
rev_id = mainline_rev.rid_new
elif diff_group == 'former':
rev_id = mainline_rev.rid_old
else:
g.assurt(False)
elif isinstance(mainline_rev, revision.Updated):
rev_id = mainline_rev.rid_max
else:
g.assurt(False)
rev = revision.Historic(rev_id)
else:
rev = revision.Historic(prev_last_merge_rid)
# Grab the parent branch id
# DOCS: All calls to psycopg2's sql() must use %s and not %d.
# FIXME: Rename parent_id 2 parent_stack_id, 2 conform to 'the others'
# FIXME: I was doing db.sql("blah %s", rev.as_sql_where()) but I get
# unicode, and wrapping with str() didn't help:
# AND E'((iv.valid_until_rid = 2000000000)
# AND (NOT iv.deleted))'
basic_sql = ("""
SELECT
iv.stack_id
, iv.name
, br.parent_id
, br.last_merge_rid
FROM
branch AS br
JOIN
item_versioned as iv
USING (system_id)
WHERE
br.stack_id = %d
AND %%s
""" % (cur_branch_id, ))
br_sql = basic_sql % (rev.as_sql_where('iv'), )
# 2014.08.19: Weird exception happening... but so far
# just on test server?
# dump.20140813-12:25:48.456171_e3aa5_EXCEPT
# dump.20140816-18:44:21.885138_e3aa5_EXCEPT
# 1 Traceback (most recent call last):
# 2 File "/ccp/dev/cycloplan_test/pyserver/gwis/request.py",
# line 454, in process_req
# 3 self.command_process_req()
# 4 File "/ccp/dev/cycloplan_test/pyserver/gwis/request.py",
# line 513, in command_process_req
# 5 self.decode_gwis()
# 6 File "/ccp/dev/cycloplan_test/pyserver/gwis/request.py",
# line 799, in decode_gwis
# 7 self.parts.decode_gwis()
# 8 File "/ccp/dev/cycloplan_test/pyserver/gwis/request.py",
# line 62, in decode_gwis
# 9 self.branch.decode_gwis()
# 10 File "/ccp/dev/cycloplan_test/pyserver/gwis/query_branch.py",
# line 52, in decode_gwis
# 11 self.req.revision.rev)
# 12 File "/ccp/dev/cycloplan_test/pyserver/item/feat/branch.py",
# line 513, in branch_hier_build
# 13 rows = db.sql(br_sql)
# 14 File "/ccp/dev/cycloplan_test/pyserver/util_/db_glue.py",
# line 1321, in sql
# 15 self.curs.execute(sql, parms)
# 16 ProgrammingError: relation "branch" does not exist
# 17 LINE 8: branch AS br
try:
rows = db.sql(br_sql)
except Exception, e:
log.error('Unexcepted exception: %s' % (str(e), ))
log.error('br_sql: %s' % (br_sql, ))
raise
if not rows:
# This means the branch didn't exist at that revision, so just
# run up the parent until we find a branch that does, and then
# start the hierarchy from there.
# EXPLAIN: When/Why does this happen?
log.debug('Branch ID %d does not exist at %s' % (
cur_branch_id,
mainline_rev.short_name(),
))
# We shouldn't have started to build the hier at this point.
if branch_hier:
log.error(
'branch_hier_build: parent br. %d missing at %s' % (
cur_branch_id,
mainline_rev.short_name(),
))
g.assurt(False)
# Note that if the user wants Current, the branch ID really d.n.e.
if not isinstance(rev, revision.Current):
g.assurt(isinstance(rev, revision.Historic))
# Get the parent's branch ID using the first vers of the branch.
par_sql = basic_sql % ('iv.version = 1', )
rows = db.sql(par_sql)
# Back up scope so we also raise on revision.Current.
if not rows:
raise GWIS_Error('Branch ID %d does not exist.' %
(cur_branch_id, ))
# Now we've got one row and rev is Historic.
g.assurt(len(rows) == 1)
# Setup the next iteration.
#
# [lb] isn't quite sure of the appropriate behavior here, but it
# shouldn't really matter: if the requested revision is before the
# first version's last_merge_rid, do we use last_merge_rid, or the
# requsted rid (and show an earlier version of the parent)? Since
# the branch did not exist at the requested revision, we can just
# assume, if the branch existed, it was updated at every revision,
# so just serve the parent branch at the requested revision.
last_merge_rid = rows[0]['last_merge_rid']
if last_merge_rid <= rev.rid:
prev_last_merge_rid = last_merge_rid
else:
prev_last_merge_rid = rev.rid
# NOTE: The convention is that a child branch's stack ID is always
# greater than its parent. This simplifies some of the SQL
# used with stacked branching.
parent_id = rows[0]['parent_id']
g.assurt(cur_branch_id > parent_id)
cur_branch_id = parent_id
else:
g.assurt(len(rows) == 1)
# Store the branch_hier tuple.
brh_tup = (
cur_branch_id,
rev,
rows[0]['name'],
)
log.verbose(' >> br. hier tup: (%d, %s, %s)' % (
brh_tup[0],
brh_tup[1].short_name(),
brh_tup[2],
))
branch_hier.append(brh_tup)
# Setup the next iteration.
prev_last_merge_rid = rows[0]['last_merge_rid']
# NOTE: The convention is that a child branch's stack ID is always
# greater than its parent. This simplifies some of the SQL
# used with stacked branching.
if rows[0]['parent_id']:
g.assurt(cur_branch_id > rows[0]['parent_id'])
cur_branch_id = rows[0]['parent_id']
# If the cur_branch_id is now not something, it means we're done.
if not cur_branch_id:
not_at_baseline = False # Now we're not not at the baseline.