def attribute_load(self, qb, attr_name): # DEPRECATED: See qb.item_mgr.cache_attrs/cache_attrnames. # 2012.08.14: This is still used by link_post... probably okay if it just # happens once per checkout... # MAYBE: Can/Should we preload attributes for all pyserver instances? # Would only make sense if it runs before URL requests are received, # i.e., if Apache loads pyserver and then just forks it for each request. attr = None if (qb.item_mgr is not None) and (qb.item_mgr.loaded_cache): try: # Note that we're caching to the qb object so that this fcn. is # thread-, er, apache-thread-safe. attr = qb.item_mgr.cache_attrnames[attr_name] except KeyError: pass if attr is None: # CAVEAT: Callers should call qb.item_mgr.load_cache_attachments(qb) # if they intend to load most of the attributes. If the caller # is just intending to load a few attributes, this fcn. is # fine, but if you want all the attributes, load the cache. qb_cur = qb.clone() qb_cur.revision = revision.Current() qb_cur.branch_hier[0] = ( qb_cur.branch_hier[0][0], qb_cur.revision, qb_cur.branch_hier[0][2], ) # Can call qb_cur.revision.setup_gids(qb_cur.db, qb_cur.username) or qb_cur.branch_hier_set(qb_cur.branch_hier) Query_Overlord.finalize_query(qb_cur) attrs = attribute.Many() attrs.search_by_internal_name(attr_name, qb_cur) log.verbose('attribute_load: %s (%d)' % ( attr_name, len(attrs), )) if len(attrs): g.assurt(len(attrs) == 1) attr = attrs[0] if qb.item_mgr.loaded_cache: qb.item_mgr.cache_attrnames[attr_name] = attr if attr is not None: self.attr_stack_id = attr.stack_id self.the_attr = attr else: # This happens on import... but import not longer uses this fcn. # This fcn. is just called for checking out a thread and posts, # e.g., to get the stack ID of '/post/revision', so this is probably a # problem. log.warning('attribute_load: Attribute not found: %s' % (attr_name, )) self.attr_stack_id = 0 self.the_attr = None
def reset_current_qb(self): self.current_qb = self.qb.clone(db_clone=False) branch_hier = copy.copy(self.current_qb.branch_hier) branch_hier[0] = ( branch_hier[0][0], revision.Current(), branch_hier[0][2],) self.current_qb.branch_hier_set(branch_hier)
def add_members(self): log.info('Adding branch user group memberships.') # MAYBE: Use this script to update the public basemap after the # v1-v2 upgrade? i.e., don't create group memberships in the # upgrade scripts! # Group Memberships are saved with the basemap branch ID. We don't need # to clone the db, and if we did, we'd want to relock the 'revision' # table (i.e., grac_mgr expects: db.locked_tables == ['revision',]). basemap_qb = self.qb.clone(db_clone=False) # Use the basemap branch. At the Current revision. parentest = basemap_qb.branch_hier[-1] branch_hier = [(parentest[0], revision.Current(), parentest[2],),] basemap_qb.branch_hier_set(branch_hier) common_row = { # From item_versioned 'system_id' : None, # assigned later 'branch_id' : None, # assigned later 'version' : 0, 'deleted' : False, 'reverted' : False, 'name' : '', 'valid_start_rid' : 1, 'valid_until_rid' : None, # From groupy_base #'group_name' : None,#self.cli_opts.new_branch_name, # From group_membership 'opt_out' : False, #'group_desc' : '', #'group_scope' : Access_Scope.shared, 'access_level_id' : Access_Level.editor, } usernames = list(set(self.cli_opts.editors + self.cli_opts.arbiters + self.cli_opts.owners)) self.add_members_to_group(basemap_qb, common_row, self.sid_editors, usernames) usernames = list(set(self.cli_opts.arbiters + self.cli_opts.owners)) self.add_members_to_group(basemap_qb, common_row, self.sid_arbiters, usernames) usernames = list(set(self.cli_opts.owners)) self.add_members_to_group(basemap_qb, common_row, self.sid_owners, usernames)
def get_qb_for_branch(self, branch_id): try: branch_qb, the_branch = self.branch_qbs[branch_id] log.debug('get_qb_for_branch: existing qb: user: %s / branch_id: %s' % (self.username, branch_id,)) except KeyError: rev = revision.Current() branch_hier = branch.Many.branch_hier_build(self.db, branch_id, rev) branch_qb = Item_Query_Builder(self.db, self.username, branch_hier, rev) g.assurt(branch_id == branch_hier[0][0]) access_ok = True the_branch = None branches = branch.Many() branches.search_by_stack_id(branch_id, branch_qb) if branches: g.assurt(len(branches) == 1) the_branch = branches[0] if not the_branch.can_view(): log.warning( 'send_email_: user cannot access leafy branch: %s / %s' % (self.username, branch_id,)) access_ok = False parent_branches = branch_hier[1:] for parent in parent_branches: branches = branch.Many() branches.search_by_stack_id(parent[0], branch_qb) if branches: g.assurt(len(branches) == 1) if not branches[0].can_view(): log.warning( 'send_email_: user cannot access parent branch: %s / %s' % (self.username, branches[0].stack_id,)) access_ok = False if (the_branch is None) or (not access_ok): branch_qb = None self.branch_qbs[branch_id] = (branch_qb, the_branch,) log.debug('get_qb_for_branch: new qb: user: %s / branch_id: %s / %s' % (self.username, branch_id, branch_qb,)) return (branch_qb, the_branch,)
def branch_hier_enforce_branch(self, branch_tup, branch_update_enabled): ''' Checks the user's access to a branch at the requested revision, or revisions, in the case of a Diff. We always check the user's access at the latest revision, to understand what their access to the current branch really is. ''' log.verbose1('self.req.revision.rev: %s' % (self.req.revision.rev, )) if isinstance(self.req.revision.rev, revision.Current): if branch_update_enabled: min_access = Access_Level.editor else: min_access = Access_Level.viewer self.access_level_id = self.branch_hier_enforce_revision( branch_tup, self.req.revision.rev, min_access) elif isinstance(self.req.revision.rev, revision.Historic): g.assurt(not branch_update_enabled) self.branch_hier_enforce_revision(branch_tup, self.req.revision.rev, Access_Level.viewer) # In historic mode, access is always viewer self.access_level_id = Access_Level.viewer elif isinstance(self.req.revision.rev, revision.Diff): # The user needs view access or better to both revisions. acl_viewer = Access_Level.viewer g.assurt(not branch_update_enabled) rev = revision.Historic(self.req.revision.rev.rid_old) self.branch_hier_enforce_revision(branch_tup, rev, acl_viewer) rev = revision.Historic(self.req.revision.rev.rid_new) self.branch_hier_enforce_revision(branch_tup, rev, acl_viewer) # In historic mode, access is always viewer self.access_level_id = Access_Level.viewer elif isinstance(self.req.revision.rev, revision.Updated): log.error( 'Updated revision requested. This is only secure for scripts.') raise GWIS_Error('GWIS does not support revision.Updated.') elif isinstance(self.req.revision.rev, revision.Comprehensive): self.access_level_id = self.branch_hier_enforce_revision( branch_tup, revision.Current(), Access_Level.viewer) else: g.assurt(False)
def output_cfg_for_branch(self, branch_): username = conf.anonymous_username rev = revision.Current() branch_hier = branch.Many.branch_hier_build(self.qb.db, branch_.stack_id, rev) qb = Item_Query_Builder(self.qb.db, username, branch_hier, rev) # See byway.Many.branch_coverage_area_update and item_mgr.revision_save: # Computing the rubber-band polygon -- the so-called coverage area -- # takes a number of seconds to compute. So best not to block a user's # GWIS call but to update the coverate area separately. # MAYBE: Move this to Mr. Do! # FIXME: revision.Revision.geosummary_update should move here, too? # Or maybe to new Mr. Do! job? # See if we should bother doing this. needs_update = self.check_branch_last_rid_changed(qb, branch_) if self.cli_opts.force: needs_update = True if needs_update: self.branch_coverage_area_update(qb, branch_) # Now that we've got the branch bbox, generate the layers for each # branch-skin combo. skin_names = branch_.get_skin_names() for skin_name in skin_names: log.debug('Processing branch named "%s" (%d) / skin "%s"' % ( branch_.name, branch_.stack_id, skin_name, )) self.out_cfg_for_brskin(qb, branch_, skin_name) if needs_update: self.update_branch_last_rid(qb, branch_)
def setup_qbs(self): username = self.wtem.created_by db = db_glue.new() if self.wtem.revision_id is None: rev = revision.Current() else: rev = revision.Historic(self.wtem.revision_id, allow_deleted=False) (branch_id, branch_hier) = branch.Many.branch_id_resolve(db, self.wtem.branch_id, branch_hier_rev=rev) if not branch_hier: raise Exception( 'Branch with stack_id %d not found in database at %s.' % (self.wtem.branch_id, rev.short_name(),)) self.qb = Item_Query_Builder(db, username, branch_hier, rev) self.qb.item_mgr = self.qb.item_mgr Query_Overlord.finalize_query(self.qb)
def __init__(self, req): Query_Base.__init__(self, req) self.rev = revision.Current()
def setup_qb_cur(self, all_errs, min_acl=Access_Level.viewer): # For both import and export, qb_src is used to retrieve items from the # database, and qb_cur is used to check the user's group accesses and # maybe to search for regions if a restrictive bbox is being imposed. # But qb_cur is also used during import to save changes to the database; # qb_cur is not used during export to save anything to the database. # # NOTE: On import, we row-lock on the grac tables, group_membership # and new_item_policy. We also row-lock the destination branch. # So other operations might block while this code runs. # CONFIRM: We don't lock anything on export, right? qb_cur = None username = self.mjob.wtem.created_by db = db_glue.new() rev = revision.Current(allow_deleted=False) (branch_id, branch_hier) = branch.Many.branch_id_resolve(db, self.mjob.wtem.branch_id, branch_hier_rev=rev) if branch_id is None: # EXPLAIN: How come we don't raise here, like we do in the else? # Or, why doesn't the else block use all_errs? # See: raise_error_on_branch. # And if you look at export_cyclop.substage_initialize, # you'll see that it assurts not all_errs, so I guess # it expects us to raise. all_errs.append('setup_qb_cur: not a branch: %s at %s' % ( self.mjob.wtem.branch_id, str(rev), )) else: g.assurt(branch_hier) g.assurt(branch_id == branch_hier[0][0]) raise_error_on_branch = False if not self.spf_conf.branch_name: # This happens on export, since export_cyclop.substage_initialize # only sets branch_id when setting up the qbs. This is because it # uses the merge_job's branch_id, and since merge_job is just an # item_versioned item, all it has is its branch_id, as items do # not also store the branch name. self.spf_conf.branch_name = branch_hier[0][2] elif self.spf_conf.branch_name != branch_hier[0][2]: # The branch name in the shapefile should match. log.error('setup_qb_cur: branch_name mismatch: %s / %s' % ( self.spf_conf.branch_name, branch_hier[0][2], )) raise_error_on_branch = True # else, the branch_name in the conf matches the one we loaded by ID. # if self.spf_conf.branch_id != branch_id: # But the branch ID we can tolerate being wrong. log.warning('setup_qb_cur: unexpected spf_conf.branch_id: %s' % (self.spf_conf.branch_id, )) # For the Metc Bikeways shapefile, this just means [lb] hasn't # update the branch ID attribute in the shapefile... g.assurt(self.spf_conf.branch_name) (try_branch_id, try_branch_hier) = branch.Many.branch_id_resolve( db, self.spf_conf.branch_name, branch_hier_rev=rev) if try_branch_id == branch_id: log.warning('setup_qb_cur: ok: overriding branch_id: %s' % (branch_id, )) self.spf_conf.branch_id = branch_id else: log.error( 'setup_qb_cur: try_branch_id != branch_id: %s != %s' % ( try_branch_id, branch_id, )) raise_error_on_branch = True if raise_error_on_branch: if conf.break_on_assurt: import pdb pdb.set_trace() raise GWIS_Error( 'Shapefile branch ID and name do not match job details: ' 'work_item: %s/%s | shapefile: %s/%s' % ( branch_hier[0][2], branch_hier[0][0], self.spf_conf.branch_name, self.spf_conf.branch_id, )) qb_cur = Item_Query_Builder(db, username, branch_hier, rev) # Load both the raw geometry and the WKT geometry; we need to be # flexible. qb_cur.filters.skip_geometry_raw = False qb_cur.filters.skip_geometry_svg = True qb_cur.filters.skip_geometry_wkt = False # To save things, we need to set the group ID explicitly. self.user_group_id = User.private_group_id(qb_cur.db, username) qb_cur.user_group_id = self.user_group_id qb_cur.item_mgr = Item_Manager() # Load the attachment cache now. On import, if we create new # attributes (see metc_bikeways_defs.py), we'll keep it updated. qb_cur.item_mgr.load_cache_attachments(qb_cur) Query_Overlord.finalize_query(qb_cur) # FIXME: This comment. I like it. But it's not true... yet. # Getting row lock in branches_prepare. So don't table lock. # # Start the transaction, since the grac_mgr does some row locking. # We'll keep the rows locked until we've verified permissions. # FIXME: Verify you rollback and start a new 'revision' lock... # or maybe just start a new 'revision' lock? or can you # write to a Shapfile first and zip through the Shapefile # to save quickly and not hold the lock so long? # BUG nnnn: Investigate using a row-level branch lock; for now, # just lock rev. qb_cur.db.transaction_begin_rw() qb_cur.grac_mgr = Grac_Manager() load_grp_mmbrshps = True qb_cur.grac_mgr.prepare_mgr('user', qb_cur, load_grp_mmbrshps) # FIXME: Does qb_src need grac_mgr? #self.qb_src.grac_mgr = qb_cur.grac_mgr # Check user's minimum access level. target_branch = self.verify_branch_access(qb_cur, min_acl, all_errs) g.assurt(target_branch.stack_id == self.spf_conf.branch_id) if (self.spf_conf.branch_name and (self.spf_conf.branch_name != qb_cur.branch_hier[0][2])): log.warning('Unexpected spf_conf.branch_name: %s' % (self.spf_conf.branch_name, )) self.spf_conf.branch_name = qb_cur.branch_hier[0][2] self.qb_cur = qb_cur log.debug('setup_qb_cur: spf_conf: %s' % (str(self.spf_conf), ))
def setup_links(self): # First count some table rows and double-check the upgrade so far. We # want to be confident we're getting all the CcpV1 records and making # appropriate CcpV2 records. try: self.setup_links_sanity_check() except: log.warning('setup_links: old CcpV1 already dropped; moving on...') # Now get the unique set of usernames. We're going to create items owned # by certain users, and we'll need to setup resources for each user, like # the query_builder and the grac_mgr. usernames_sql = (""" SELECT DISTINCT (username) FROM item_watcher_bug_nnnn ORDER BY username """) # NOTE: We're not bothering with dont_fetchall. # There are only a few hundred rows... rows = self.qb.db.sql(usernames_sql) log.debug('setup_links: found %d unique users with watchers' % (len(rows), )) if not rows: log.error('setup_links: nothing found') g.assurt(false) for row in rows: username = row['username'] # Hmm. There's no user.One() class to load a user. It's all custom. user_rows = self.qb.db.sql( "SELECT login_permitted FROM user_ WHERE username = %s" % (self.qb.db.quoted(username), )) g.assurt(len(user_rows) == 1) if not user_rows[0]['login_permitted']: log.debug('setup_links: skipping: !user_.login_permitted: %s' % (username, )) continue log.verbose2('setup_links: processing username: %s' % (username, )) g.assurt(isinstance(self.qb.revision, revision.Current)) rev_cur = revision.Current() user_qb = Item_Query_Builder(self.qb.db, username, self.qb.branch_hier, rev_cur) user_qb.grac_mgr = Grac_Manager() user_qb.grac_mgr.prepare_mgr('user', user_qb) # g.assurt(user_qb.username and (user_qb.username != conf.anonymous_username)) user_qb.user_group_id = User.private_group_id( user_qb.db, user_qb.username) # # Use the same item_mgr so we pull client stack IDs from the same # pool. user_qb.item_mgr = self.qb.item_mgr # Finalize the query. This sets revision.gids so it'll include the # user's private group (and the anonymous and All Users groups). Query_Overlord.finalize_query(user_qb) # We can still get deleted regions and add links for them. user_qb.revision.allow_deleted = True # Finally, update the database. Oi, there's a lot of setup! self.setup_links_for_user(user_qb) # The way Item_Query_Builder works, it usually wires the branch_hier # revision to the revision revision. g.assurt(self.qb.branch_hier[0][1] == rev_cur) # We'll reuse the branch_hier so clear this user's gids. self.qb.branch_hier[0][1].gids = None
revision=None, viewport=None, quieter=False, ) (clopts, args) = op.parse_args() items_fetched = item_factory.get_item_module(clopts.item_type).Many() if (not clopts.quieter): print('...checking revision') rev = None if (clopts.revision): rev = revision.Revision.revision_object_get(rev) else: rev = revision.Current() if (not clopts.quieter): print('...checking viewport') # NOTE Ignoring bbox_exclude vp = None areq = None req = gwis.request.Request(areq) if (clopts.viewport): bbox = clopts.viewport vp = query_viewport.Query_Viewport(req) vp.parse_strs(bbox, None) elif (not clopts.viewport_none): bbox = '483690,4978196,487454,4981236' vp = query_viewport.Query_Viewport(req)
def update_branch(self): #branch_id = branch.Many.public_branch_id(self.qb) #revision_id = self.qb.item_mgr.rid_new #rev = revision.Historic(revision_id, allow_deleted=False) rev = revision.Current() (branch_id, branch_hier) = branch.Many.branch_id_resolve(self.qb.db, self.cli_opts.update_branch, branch_hier_rev=rev) # Be sure to get the item_stack table or, when we save, access_style_id # won't be set and our not null constraint will complain. self.qb.filters.include_item_stack = True branches = branch.Many() branches.search_by_stack_id(branch_id, self.qb) if len(branches) != 1: raise Exception('Branch named "%s" not found.' % (self.cli_opts.update_branch,)) g.assurt(len(branches) == 1) the_branch = branches[0] # Currently, all branches are 'permissive'. g.assurt(the_branch.access_style_id == Access_Style.permissive) # import_callback, export_callback = self.get_branch_callbacks() if ((import_callback or export_callback) or (self.cli_opts.tile_skins is not None)): if import_callback or export_callback: log.debug('Overwriting import_callback: was: "%s" / now: "%s"' % (the_branch.import_callback, import_callback,)) the_branch.import_callback = import_callback log.debug('Overwriting export_callback: was: "%s" / now: "%s"' % (the_branch.export_callback, export_callback,)) the_branch.export_callback = export_callback if self.cli_opts.tile_skins is not None: tile_skins = self.get_tile_skin_names() log.debug('Overwriting tile_skins: was: "%s" / now: "%s"' % (the_branch.tile_skins, tile_skins,)) the_branch.tile_skins = tile_skins # MAYBE: Call prepare_and_save_item? Or just do it ourselves? # NOTE: grac_mgr.prepare_existing_from_stack_id calls # validize, which calls groups_access_load_from_db... # which we just call ourselves. I think this works. is_new_item = False the_branch.validize(self.qb, is_new_item, item_base.One.dirty_reason_item_user, ref_item=None) rid_new = self.qb.item_mgr.rid_new the_branch.version_finalize_and_increment(self.qb, rid_new) the_branch.save(self.qb, rid_new) self.the_branch = the_branch
def regions_coalesce_geometry(req=None, from_qb=None): # Calculate the region-filter geometry. Do it now so we can cache the # value and use an explicit value in the SQL where clause. (This is # bug nnnn, don't use SQL fcns. in WHERE clauses). # # Note that the region-filter is either the geometry of one or more # regions the client specifically indicate, or the geometry is the # geometry of regions that the user is watching. # # Note also that we use the Current revision of the regions and ignore # whatever revision the client might really be requesting. This is by # design, so users can make new regions and use them on historical # queries. It also makes working with the autocomplete edit box in # the client easier, since it only has to get the list of current # regions. g.assurt((req is not None) ^ (from_qb is not None)) if from_qb is None: g.assurt(False) # Deprecated. db = req.db username = req.client.username branch_hier = req.branch.branch_hier the_rev = req.revision.rev filters = req.filters # NOTE: Cannot call as_iqb(addons=False), because that fcn. calls # finalize_query(). So we make our own qb, below. else: db = from_qb.db username = from_qb.username branch_hier = from_qb.branch_hier the_rev = from_qb.revision filters = from_qb.filters if not isinstance(the_rev, revision.Current): the_rev = revision.Current() branch_hier = branch.Many.branch_hier_build( db, branch_hier[0][0], the_rev) g.assurt(id(the_rev) == id(branch_hier[0][1])) else: # [lb] is not sure this is always true. Just curious... g.assurt(id(the_rev) == id(branch_hier[0][1])) g.assurt(not isinstance(the_rev, revision.Comprehensive)) qb = Item_Query_Builder(db, username, branch_hier, the_rev) g.assurt(qb.revision == the_rev) # Use from_qb.item_mgr? # Cannot import Item_Manager: # Nope: qb.item_mgr = Item_Manager() g.assurt(from_qb is not None) g.assurt(from_qb.item_mgr is not None) qb.item_mgr = from_qb.item_mgr # Create a Many() object to perform the search # 2013.03.29: MAYBE: We're really fetching regions... region_geoms = region.Many() # Tell the sql builder not to double-check the multi geom region_geoms.search_for_geom(qb, filters.filter_by_regions, filters.filter_by_watch_geom) g.assurt(len(region_geoms) == 1) # Because of the aggregrate, ST_Union. # Normally geometry_svg gets set but only for an outer select. # Our select had nothing nested, so geometry is raw only. geom = region_geoms[0].geometry if geom is None: # If there's no geometry, we won't find anything, so short-circuit. # MAYBE: Alert user that region name was not found, or no watched # regions were found? Except we're called from a filter... # so the lack of results should be clue enough. log.debug('fetch_n_save: GWIS_Nothing_Found: nothing found') raise GWIS_Nothing_Found() log.debug('fetch_n_save: len(geom): %s' % (len(geom),)) # Set the geometry we calculated in the source qb. filters.only_in_multi_geometry = geom