def attribute_load(self, qb, attr_name):
# DEPRECATED: See qb.item_mgr.cache_attrs/cache_attrnames.
# 2012.08.14: This is still used by link_post... probably okay if it just
# happens once per checkout...
# MAYBE: Can/Should we preload attributes for all pyserver instances?
# Would only make sense if it runs before URL requests are received,
# i.e., if Apache loads pyserver and then just forks it for each request.
attr = None
if (qb.item_mgr is not None) and (qb.item_mgr.loaded_cache):
try:
# Note that we're caching to the qb object so that this fcn. is
# thread-, er, apache-thread-safe.
attr = qb.item_mgr.cache_attrnames[attr_name]
except KeyError:
pass
if attr is None:
# CAVEAT: Callers should call qb.item_mgr.load_cache_attachments(qb)
# if they intend to load most of the attributes. If the caller
# is just intending to load a few attributes, this fcn. is
# fine, but if you want all the attributes, load the cache.
qb_cur = qb.clone()
qb_cur.revision = revision.Current()
qb_cur.branch_hier[0] = (
qb_cur.branch_hier[0][0],
qb_cur.revision,
qb_cur.branch_hier[0][2],
)
# Can call qb_cur.revision.setup_gids(qb_cur.db, qb_cur.username) or
qb_cur.branch_hier_set(qb_cur.branch_hier)
Query_Overlord.finalize_query(qb_cur)
attrs = attribute.Many()
attrs.search_by_internal_name(attr_name, qb_cur)
log.verbose('attribute_load: %s (%d)' % (
attr_name,
len(attrs),
))
if len(attrs):
g.assurt(len(attrs) == 1)
attr = attrs[0]
if qb.item_mgr.loaded_cache:
qb.item_mgr.cache_attrnames[attr_name] = attr
if attr is not None:
self.attr_stack_id = attr.stack_id
self.the_attr = attr
else:
# This happens on import... but import not longer uses this fcn.
# This fcn. is just called for checking out a thread and posts,
# e.g., to get the stack ID of '/post/revision', so this is probably a
# problem.
log.warning('attribute_load: Attribute not found: %s' %
(attr_name, ))
self.attr_stack_id = 0
self.the_attr = None
def reset_current_qb(self):
self.current_qb = self.qb.clone(db_clone=False)
branch_hier = copy.copy(self.current_qb.branch_hier)
branch_hier[0] = (
branch_hier[0][0], revision.Current(), branch_hier[0][2],)
self.current_qb.branch_hier_set(branch_hier)
def add_members(self):
log.info('Adding branch user group memberships.')
# MAYBE: Use this script to update the public basemap after the
# v1-v2 upgrade? i.e., don't create group memberships in the
# upgrade scripts!
# Group Memberships are saved with the basemap branch ID. We don't need
# to clone the db, and if we did, we'd want to relock the 'revision'
# table (i.e., grac_mgr expects: db.locked_tables == ['revision',]).
basemap_qb = self.qb.clone(db_clone=False)
# Use the basemap branch. At the Current revision.
parentest = basemap_qb.branch_hier[-1]
branch_hier = [(parentest[0], revision.Current(), parentest[2],),]
basemap_qb.branch_hier_set(branch_hier)
common_row = {
# From item_versioned
'system_id' : None, # assigned later
'branch_id' : None, # assigned later
'version' : 0,
'deleted' : False,
'reverted' : False,
'name' : '',
'valid_start_rid' : 1,
'valid_until_rid' : None,
# From groupy_base
#'group_name' : None,#self.cli_opts.new_branch_name,
# From group_membership
'opt_out' : False,
#'group_desc' : '',
#'group_scope' : Access_Scope.shared,
'access_level_id' : Access_Level.editor,
}
usernames = list(set(self.cli_opts.editors
+ self.cli_opts.arbiters
+ self.cli_opts.owners))
self.add_members_to_group(basemap_qb, common_row,
self.sid_editors, usernames)
usernames = list(set(self.cli_opts.arbiters
+ self.cli_opts.owners))
self.add_members_to_group(basemap_qb, common_row,
self.sid_arbiters, usernames)
usernames = list(set(self.cli_opts.owners))
self.add_members_to_group(basemap_qb, common_row,
self.sid_owners, usernames)
def get_qb_for_branch(self, branch_id):
try:
branch_qb, the_branch = self.branch_qbs[branch_id]
log.debug('get_qb_for_branch: existing qb: user: %s / branch_id: %s'
% (self.username, branch_id,))
except KeyError:
rev = revision.Current()
branch_hier = branch.Many.branch_hier_build(self.db, branch_id, rev)
branch_qb = Item_Query_Builder(self.db, self.username, branch_hier,
rev)
g.assurt(branch_id == branch_hier[0][0])
access_ok = True
the_branch = None
branches = branch.Many()
branches.search_by_stack_id(branch_id, branch_qb)
if branches:
g.assurt(len(branches) == 1)
the_branch = branches[0]
if not the_branch.can_view():
log.warning(
'send_email_: user cannot access leafy branch: %s / %s'
% (self.username, branch_id,))
access_ok = False
parent_branches = branch_hier[1:]
for parent in parent_branches:
branches = branch.Many()
branches.search_by_stack_id(parent[0], branch_qb)
if branches:
g.assurt(len(branches) == 1)
if not branches[0].can_view():
log.warning(
'send_email_: user cannot access parent branch: %s / %s'
% (self.username, branches[0].stack_id,))
access_ok = False
if (the_branch is None) or (not access_ok):
branch_qb = None
self.branch_qbs[branch_id] = (branch_qb, the_branch,)
log.debug('get_qb_for_branch: new qb: user: %s / branch_id: %s / %s'
% (self.username, branch_id, branch_qb,))
return (branch_qb, the_branch,)
def branch_hier_enforce_branch(self, branch_tup, branch_update_enabled):
'''
Checks the user's access to a branch at the requested revision, or
revisions, in the case of a Diff.
We always check the user's access at the latest revision, to understand
what their access to the current branch really is.
'''
log.verbose1('self.req.revision.rev: %s' % (self.req.revision.rev, ))
if isinstance(self.req.revision.rev, revision.Current):
if branch_update_enabled:
min_access = Access_Level.editor
else:
min_access = Access_Level.viewer
self.access_level_id = self.branch_hier_enforce_revision(
branch_tup, self.req.revision.rev, min_access)
elif isinstance(self.req.revision.rev, revision.Historic):
g.assurt(not branch_update_enabled)
self.branch_hier_enforce_revision(branch_tup,
self.req.revision.rev,
Access_Level.viewer)
# In historic mode, access is always viewer
self.access_level_id = Access_Level.viewer
elif isinstance(self.req.revision.rev, revision.Diff):
# The user needs view access or better to both revisions.
acl_viewer = Access_Level.viewer
g.assurt(not branch_update_enabled)
rev = revision.Historic(self.req.revision.rev.rid_old)
self.branch_hier_enforce_revision(branch_tup, rev, acl_viewer)
rev = revision.Historic(self.req.revision.rev.rid_new)
self.branch_hier_enforce_revision(branch_tup, rev, acl_viewer)
# In historic mode, access is always viewer
self.access_level_id = Access_Level.viewer
elif isinstance(self.req.revision.rev, revision.Updated):
log.error(
'Updated revision requested. This is only secure for scripts.')
raise GWIS_Error('GWIS does not support revision.Updated.')
elif isinstance(self.req.revision.rev, revision.Comprehensive):
self.access_level_id = self.branch_hier_enforce_revision(
branch_tup, revision.Current(), Access_Level.viewer)
else:
g.assurt(False)
def output_cfg_for_branch(self, branch_):
username = conf.anonymous_username
rev = revision.Current()
branch_hier = branch.Many.branch_hier_build(self.qb.db,
branch_.stack_id, rev)
qb = Item_Query_Builder(self.qb.db, username, branch_hier, rev)
# See byway.Many.branch_coverage_area_update and item_mgr.revision_save:
# Computing the rubber-band polygon -- the so-called coverage area --
# takes a number of seconds to compute. So best not to block a user's
# GWIS call but to update the coverate area separately.
# MAYBE: Move this to Mr. Do!
# FIXME: revision.Revision.geosummary_update should move here, too?
# Or maybe to new Mr. Do! job?
# See if we should bother doing this.
needs_update = self.check_branch_last_rid_changed(qb, branch_)
if self.cli_opts.force:
needs_update = True
if needs_update:
self.branch_coverage_area_update(qb, branch_)
# Now that we've got the branch bbox, generate the layers for each
# branch-skin combo.
skin_names = branch_.get_skin_names()
for skin_name in skin_names:
log.debug('Processing branch named "%s" (%d) / skin "%s"' % (
branch_.name,
branch_.stack_id,
skin_name,
))
self.out_cfg_for_brskin(qb, branch_, skin_name)
if needs_update:
self.update_branch_last_rid(qb, branch_)
def setup_qbs(self):
username = self.wtem.created_by
db = db_glue.new()
if self.wtem.revision_id is None:
rev = revision.Current()
else:
rev = revision.Historic(self.wtem.revision_id, allow_deleted=False)
(branch_id, branch_hier) = branch.Many.branch_id_resolve(db,
self.wtem.branch_id, branch_hier_rev=rev)
if not branch_hier:
raise Exception(
'Branch with stack_id %d not found in database at %s.'
% (self.wtem.branch_id, rev.short_name(),))
self.qb = Item_Query_Builder(db, username, branch_hier, rev)
self.qb.item_mgr = self.qb.item_mgr
Query_Overlord.finalize_query(self.qb)
def __init__(self, req):
Query_Base.__init__(self, req)
self.rev = revision.Current()
def setup_qb_cur(self, all_errs, min_acl=Access_Level.viewer):
# For both import and export, qb_src is used to retrieve items from the
# database, and qb_cur is used to check the user's group accesses and
# maybe to search for regions if a restrictive bbox is being imposed.
# But qb_cur is also used during import to save changes to the database;
# qb_cur is not used during export to save anything to the database.
#
# NOTE: On import, we row-lock on the grac tables, group_membership
# and new_item_policy. We also row-lock the destination branch.
# So other operations might block while this code runs.
# CONFIRM: We don't lock anything on export, right?
qb_cur = None
username = self.mjob.wtem.created_by
db = db_glue.new()
rev = revision.Current(allow_deleted=False)
(branch_id,
branch_hier) = branch.Many.branch_id_resolve(db,
self.mjob.wtem.branch_id,
branch_hier_rev=rev)
if branch_id is None:
# EXPLAIN: How come we don't raise here, like we do in the else?
# Or, why doesn't the else block use all_errs?
# See: raise_error_on_branch.
# And if you look at export_cyclop.substage_initialize,
# you'll see that it assurts not all_errs, so I guess
# it expects us to raise.
all_errs.append('setup_qb_cur: not a branch: %s at %s' % (
self.mjob.wtem.branch_id,
str(rev),
))
else:
g.assurt(branch_hier)
g.assurt(branch_id == branch_hier[0][0])
raise_error_on_branch = False
if not self.spf_conf.branch_name:
# This happens on export, since export_cyclop.substage_initialize
# only sets branch_id when setting up the qbs. This is because it
# uses the merge_job's branch_id, and since merge_job is just an
# item_versioned item, all it has is its branch_id, as items do
# not also store the branch name.
self.spf_conf.branch_name = branch_hier[0][2]
elif self.spf_conf.branch_name != branch_hier[0][2]:
# The branch name in the shapefile should match.
log.error('setup_qb_cur: branch_name mismatch: %s / %s' % (
self.spf_conf.branch_name,
branch_hier[0][2],
))
raise_error_on_branch = True
# else, the branch_name in the conf matches the one we loaded by ID.
#
if self.spf_conf.branch_id != branch_id:
# But the branch ID we can tolerate being wrong.
log.warning('setup_qb_cur: unexpected spf_conf.branch_id: %s' %
(self.spf_conf.branch_id, ))
# For the Metc Bikeways shapefile, this just means [lb] hasn't
# update the branch ID attribute in the shapefile...
g.assurt(self.spf_conf.branch_name)
(try_branch_id,
try_branch_hier) = branch.Many.branch_id_resolve(
db, self.spf_conf.branch_name, branch_hier_rev=rev)
if try_branch_id == branch_id:
log.warning('setup_qb_cur: ok: overriding branch_id: %s' %
(branch_id, ))
self.spf_conf.branch_id = branch_id
else:
log.error(
'setup_qb_cur: try_branch_id != branch_id: %s != %s' %
(
try_branch_id,
branch_id,
))
raise_error_on_branch = True
if raise_error_on_branch:
if conf.break_on_assurt:
import pdb
pdb.set_trace()
raise GWIS_Error(
'Shapefile branch ID and name do not match job details: '
'work_item: %s/%s | shapefile: %s/%s' % (
branch_hier[0][2],
branch_hier[0][0],
self.spf_conf.branch_name,
self.spf_conf.branch_id,
))
qb_cur = Item_Query_Builder(db, username, branch_hier, rev)
# Load both the raw geometry and the WKT geometry; we need to be
# flexible.
qb_cur.filters.skip_geometry_raw = False
qb_cur.filters.skip_geometry_svg = True
qb_cur.filters.skip_geometry_wkt = False
# To save things, we need to set the group ID explicitly.
self.user_group_id = User.private_group_id(qb_cur.db, username)
qb_cur.user_group_id = self.user_group_id
qb_cur.item_mgr = Item_Manager()
# Load the attachment cache now. On import, if we create new
# attributes (see metc_bikeways_defs.py), we'll keep it updated.
qb_cur.item_mgr.load_cache_attachments(qb_cur)
Query_Overlord.finalize_query(qb_cur)
# FIXME: This comment. I like it. But it's not true... yet.
# Getting row lock in branches_prepare. So don't table lock.
#
# Start the transaction, since the grac_mgr does some row locking.
# We'll keep the rows locked until we've verified permissions.
# FIXME: Verify you rollback and start a new 'revision' lock...
# or maybe just start a new 'revision' lock? or can you
# write to a Shapfile first and zip through the Shapefile
# to save quickly and not hold the lock so long?
# BUG nnnn: Investigate using a row-level branch lock; for now,
# just lock rev.
qb_cur.db.transaction_begin_rw()
qb_cur.grac_mgr = Grac_Manager()
load_grp_mmbrshps = True
qb_cur.grac_mgr.prepare_mgr('user', qb_cur, load_grp_mmbrshps)
# FIXME: Does qb_src need grac_mgr?
#self.qb_src.grac_mgr = qb_cur.grac_mgr
# Check user's minimum access level.
target_branch = self.verify_branch_access(qb_cur, min_acl,
all_errs)
g.assurt(target_branch.stack_id == self.spf_conf.branch_id)
if (self.spf_conf.branch_name and
(self.spf_conf.branch_name != qb_cur.branch_hier[0][2])):
log.warning('Unexpected spf_conf.branch_name: %s' %
(self.spf_conf.branch_name, ))
self.spf_conf.branch_name = qb_cur.branch_hier[0][2]
self.qb_cur = qb_cur
log.debug('setup_qb_cur: spf_conf: %s' % (str(self.spf_conf), ))
def setup_links(self):
# First count some table rows and double-check the upgrade so far. We
# want to be confident we're getting all the CcpV1 records and making
# appropriate CcpV2 records.
try:
self.setup_links_sanity_check()
except:
log.warning('setup_links: old CcpV1 already dropped; moving on...')
# Now get the unique set of usernames. We're going to create items owned
# by certain users, and we'll need to setup resources for each user, like
# the query_builder and the grac_mgr.
usernames_sql = ("""
SELECT DISTINCT (username)
FROM item_watcher_bug_nnnn
ORDER BY username
""")
# NOTE: We're not bothering with dont_fetchall.
# There are only a few hundred rows...
rows = self.qb.db.sql(usernames_sql)
log.debug('setup_links: found %d unique users with watchers' %
(len(rows), ))
if not rows:
log.error('setup_links: nothing found')
g.assurt(false)
for row in rows:
username = row['username']
# Hmm. There's no user.One() class to load a user. It's all custom.
user_rows = self.qb.db.sql(
"SELECT login_permitted FROM user_ WHERE username = %s" %
(self.qb.db.quoted(username), ))
g.assurt(len(user_rows) == 1)
if not user_rows[0]['login_permitted']:
log.debug('setup_links: skipping: !user_.login_permitted: %s' %
(username, ))
continue
log.verbose2('setup_links: processing username: %s' % (username, ))
g.assurt(isinstance(self.qb.revision, revision.Current))
rev_cur = revision.Current()
user_qb = Item_Query_Builder(self.qb.db, username,
self.qb.branch_hier, rev_cur)
user_qb.grac_mgr = Grac_Manager()
user_qb.grac_mgr.prepare_mgr('user', user_qb)
#
g.assurt(user_qb.username
and (user_qb.username != conf.anonymous_username))
user_qb.user_group_id = User.private_group_id(
user_qb.db, user_qb.username)
#
# Use the same item_mgr so we pull client stack IDs from the same
# pool.
user_qb.item_mgr = self.qb.item_mgr
# Finalize the query. This sets revision.gids so it'll include the
# user's private group (and the anonymous and All Users groups).
Query_Overlord.finalize_query(user_qb)
# We can still get deleted regions and add links for them.
user_qb.revision.allow_deleted = True
# Finally, update the database. Oi, there's a lot of setup!
self.setup_links_for_user(user_qb)
# The way Item_Query_Builder works, it usually wires the branch_hier
# revision to the revision revision.
g.assurt(self.qb.branch_hier[0][1] == rev_cur)
# We'll reuse the branch_hier so clear this user's gids.
self.qb.branch_hier[0][1].gids = None
revision=None,
viewport=None,
quieter=False,
)
(clopts, args) = op.parse_args()
items_fetched = item_factory.get_item_module(clopts.item_type).Many()
if (not clopts.quieter):
print('...checking revision')
rev = None
if (clopts.revision):
rev = revision.Revision.revision_object_get(rev)
else:
rev = revision.Current()
if (not clopts.quieter):
print('...checking viewport')
# NOTE Ignoring bbox_exclude
vp = None
areq = None
req = gwis.request.Request(areq)
if (clopts.viewport):
bbox = clopts.viewport
vp = query_viewport.Query_Viewport(req)
vp.parse_strs(bbox, None)
elif (not clopts.viewport_none):
bbox = '483690,4978196,487454,4981236'
vp = query_viewport.Query_Viewport(req)
def update_branch(self):
#branch_id = branch.Many.public_branch_id(self.qb)
#revision_id = self.qb.item_mgr.rid_new
#rev = revision.Historic(revision_id, allow_deleted=False)
rev = revision.Current()
(branch_id, branch_hier) = branch.Many.branch_id_resolve(self.qb.db,
self.cli_opts.update_branch, branch_hier_rev=rev)
# Be sure to get the item_stack table or, when we save, access_style_id
# won't be set and our not null constraint will complain.
self.qb.filters.include_item_stack = True
branches = branch.Many()
branches.search_by_stack_id(branch_id, self.qb)
if len(branches) != 1:
raise Exception('Branch named "%s" not found.'
% (self.cli_opts.update_branch,))
g.assurt(len(branches) == 1)
the_branch = branches[0]
# Currently, all branches are 'permissive'.
g.assurt(the_branch.access_style_id == Access_Style.permissive)
#
import_callback, export_callback = self.get_branch_callbacks()
if ((import_callback or export_callback)
or (self.cli_opts.tile_skins is not None)):
if import_callback or export_callback:
log.debug('Overwriting import_callback: was: "%s" / now: "%s"'
% (the_branch.import_callback, import_callback,))
the_branch.import_callback = import_callback
log.debug('Overwriting export_callback: was: "%s" / now: "%s"'
% (the_branch.export_callback, export_callback,))
the_branch.export_callback = export_callback
if self.cli_opts.tile_skins is not None:
tile_skins = self.get_tile_skin_names()
log.debug('Overwriting tile_skins: was: "%s" / now: "%s"'
% (the_branch.tile_skins, tile_skins,))
the_branch.tile_skins = tile_skins
# MAYBE: Call prepare_and_save_item? Or just do it ourselves?
# NOTE: grac_mgr.prepare_existing_from_stack_id calls
# validize, which calls groups_access_load_from_db...
# which we just call ourselves. I think this works.
is_new_item = False
the_branch.validize(self.qb, is_new_item,
item_base.One.dirty_reason_item_user,
ref_item=None)
rid_new = self.qb.item_mgr.rid_new
the_branch.version_finalize_and_increment(self.qb, rid_new)
the_branch.save(self.qb, rid_new)
self.the_branch = the_branch
def regions_coalesce_geometry(req=None, from_qb=None):
# Calculate the region-filter geometry. Do it now so we can cache the
# value and use an explicit value in the SQL where clause. (This is
# bug nnnn, don't use SQL fcns. in WHERE clauses).
#
# Note that the region-filter is either the geometry of one or more
# regions the client specifically indicate, or the geometry is the
# geometry of regions that the user is watching.
#
# Note also that we use the Current revision of the regions and ignore
# whatever revision the client might really be requesting. This is by
# design, so users can make new regions and use them on historical
# queries. It also makes working with the autocomplete edit box in
# the client easier, since it only has to get the list of current
# regions.
g.assurt((req is not None) ^ (from_qb is not None))
if from_qb is None:
g.assurt(False) # Deprecated.
db = req.db
username = req.client.username
branch_hier = req.branch.branch_hier
the_rev = req.revision.rev
filters = req.filters
# NOTE: Cannot call as_iqb(addons=False), because that fcn. calls
# finalize_query(). So we make our own qb, below.
else:
db = from_qb.db
username = from_qb.username
branch_hier = from_qb.branch_hier
the_rev = from_qb.revision
filters = from_qb.filters
if not isinstance(the_rev, revision.Current):
the_rev = revision.Current()
branch_hier = branch.Many.branch_hier_build(
db, branch_hier[0][0], the_rev)
g.assurt(id(the_rev) == id(branch_hier[0][1]))
else:
# [lb] is not sure this is always true. Just curious...
g.assurt(id(the_rev) == id(branch_hier[0][1]))
g.assurt(not isinstance(the_rev, revision.Comprehensive))
qb = Item_Query_Builder(db, username, branch_hier, the_rev)
g.assurt(qb.revision == the_rev)
# Use from_qb.item_mgr?
# Cannot import Item_Manager:
# Nope: qb.item_mgr = Item_Manager()
g.assurt(from_qb is not None)
g.assurt(from_qb.item_mgr is not None)
qb.item_mgr = from_qb.item_mgr
# Create a Many() object to perform the search
# 2013.03.29: MAYBE: We're really fetching regions...
region_geoms = region.Many()
# Tell the sql builder not to double-check the multi geom
region_geoms.search_for_geom(qb, filters.filter_by_regions,
filters.filter_by_watch_geom)
g.assurt(len(region_geoms) == 1) # Because of the aggregrate, ST_Union.
# Normally geometry_svg gets set but only for an outer select.
# Our select had nothing nested, so geometry is raw only.
geom = region_geoms[0].geometry
if geom is None:
# If there's no geometry, we won't find anything, so short-circuit.
# MAYBE: Alert user that region name was not found, or no watched
# regions were found? Except we're called from a filter...
# so the lack of results should be clue enough.
log.debug('fetch_n_save: GWIS_Nothing_Found: nothing found')
raise GWIS_Nothing_Found()
log.debug('fetch_n_save: len(geom): %s' % (len(geom),))
# Set the geometry we calculated in the source qb.
filters.only_in_multi_geometry = geom
