def oauth(request): # retrieve destination parameters from Globus redirect # example URL with added parameters from Globus redirect: # /globus/oauth/?label=&verify_checksum=on&submitForm=&folder[0]=tmp&endpoint=cinquiniluca#mymac&path=/~/&ep=GC&lock=ep&method=get&folderlimit=1&action=http://localhost:8000/globus/oauth/ request.session[TARGET_ENDPOINT] = getQueryDict(request).get('endpoint','#') request.session[TARGET_FOLDER] = getQueryDict(request).get('path','/~/') + getQueryDict(request).get('folder[0]', '') # default value: user home directory print 'User selected destionation endpoint:%s, path:%s, folder:%s' % (request.session[TARGET_ENDPOINT], getQueryDict(request).get('path','/~/'), getQueryDict(request).get('folder[0]', '')) # Redirect the user to Globus OAuth server to get an authorization code if the user approves the access request. globus_authorize_url = establishFlow(request).step1_get_authorize_url() print "Redirecting to: %s" % globus_authorize_url return HttpResponseRedirect(globus_authorize_url)
def oauth(request): # retrieve destination parameters from Globus redirect # example URL with added parameters from Globus redirect: # /globus/oauth/?label=&verify_checksum=on&submitForm=&folder[0]=tmp&endpoint=cinquiniluca#mymac&path=/~/&ep=GC&lock=ep&method=get&folderlimit=1&action=http://localhost:8000/globus/oauth/ print request.GET request.session[TARGET_ENDPOINT] = getQueryDict(request).get('endpoint','#') request.session[TARGET_FOLDER] = getQueryDict(request).get('path','/~/') + getQueryDict(request).get('folder[0]', '') # default value: user home directory print 'User selected destionation endpoint:%s, path:%s, folder:%s' % (request.session[TARGET_ENDPOINT], getQueryDict(request).get('path','/~/'), getQueryDict(request).get('folder[0]', '')) # Redirect the user to Globus OAuth server to get an authorization code if the user approves the access request. globus_authorize_url = establishFlow(request).step1_get_authorize_url() print "Redirecting to: %s" % globus_authorize_url return HttpResponseRedirect(globus_authorize_url)
def search_config(request, searchConfig, extra={}, fromRedirectFlag=False): """ Project-specific search view that processes all GET/POST requests. Parses GET/POST requests parameters and combines them with the project fixed constraints. Delegates to 'search_get' and 'search_post'. Pre-seeded search URLs are automatically processed (i.e. GET requests with additional HTTP parameters, but NOT after a POST redirect). """ # print extra arguments #for key, value in extra.items(): # print 'extra key=%s value=%s' % (key, value) # create search input object from request parameters ONLY # NOTE: HTTP parameters MUST be part of project sarch configuration searchInput = _buildSearchInputFromHttpRequest(request, searchConfig) # GET/POST switch queryDict = getQueryDict(request) print "Search() view: HTTP Request method=%s fromRedirectFlag flag=%s HTTP parameters=%s" % (request.method, fromRedirectFlag, queryDict) if request.method == 'GET': # GET pre-seeded search URL -> invoke POST immediately if len(queryDict.keys()) > 0 and not fromRedirectFlag: return search_post(request, searchInput, searchConfig, extra) else: return search_get(request, searchInput, searchConfig, extra, fromRedirectFlag) else: return search_post(request, searchInput, searchConfig, extra)
def search_config(request, searchConfig, extra={}, fromRedirectFlag=False): """ Project-specific search view that processes all GET/POST requests. Parses GET/POST requests parameters and combines them with the project fixed constraints. Delegates to 'search_get' and 'search_post'. Pre-seeded search URLs are automatically processed (i.e. GET requests with additional HTTP parameters, but NOT after a POST redirect). """ # print extra arguments #for key, value in extra.items(): # print 'extra key=%s value=%s' % (key, value) # create search input object from request parameters ONLY # NOTE: HTTP parameters MUST be part of project sarch configuration searchInput = _buildSearchInputFromHttpRequest(request, searchConfig) # GET/POST switch queryDict = getQueryDict(request) print "Search() view: HTTP Request method=%s fromRedirectFlag flag=%s HTTP parameters=%s" % ( request.method, fromRedirectFlag, queryDict) if request.method == 'GET': # GET pre-seeded search URL -> invoke POST immediately if len(queryDict.keys()) > 0 and not fromRedirectFlag: return search_post(request, searchInput, searchConfig, extra) else: return search_get(request, searchInput, searchConfig, extra, fromRedirectFlag) else: return search_post(request, searchInput, searchConfig, extra)
def membership_list_requested(request, project_short_name): # load project project = get_object_or_404(Project, short_name__iexact=project_short_name) # check permission if not userHasAdminPermission(request.user, project): return HttpResponseForbidden(PERMISSION_DENIED_MESSAGE) # load user group group = project.getUserGroup() # optional 'match' argument match = getQueryDict(request).get('match', None) # works for GET or POST if match: # lookup specific user _users = [mr.user for mr in MembershipRequest.objects.filter(group=group).order_by('user__last_name')] users = [user for user in _users if (match in user.first_name.lower() or match in user.last_name.lower() or match in user.username.lower() or match in user.email.lower())] else: # load all users that have requested membership users = [mr.user for mr in MembershipRequest.objects.filter(group=group).order_by('user__last_name')] title = '%s Pending Users' % project.short_name view_name = 'membership_list_requested' return render_membership_page(request, project, users, title, view_name)
def datacart_pid(request, site_id, user_id): # check User object user = get_object_or_404(User, pk=user_id) # security check if not request.user.id != user_id: raise Exception("User not authorized to modify datacart") pid_messaging_service_credentials = [] priority = 1 for cred in settings.PID_CREDENTIALS: parts = cred.split('|') if len(parts) == 6: ssl_enabled = False if parts[5].strip().upper() == 'TRUE': ssl_enabled = True pid_messaging_service_credentials.append({'url': parts[0].strip(), 'port': parts[1].strip(), 'vhost': parts[2].strip(), 'user': parts[3].strip(), 'password': parts[4].strip(), 'ssl_enabled': ssl_enabled, 'priority': priority}) priority += 1 # get list of dataset_pids and dataset_ids try: datacart = DataCart.objects.get(user=user) except DataCart.DoesNotExist: datacart = None queryDict = getQueryDict(request) ids = queryDict.getlist('id') dataset_ids = {} if datacart: for item in datacart.items.all(): # filter selected datasets only if item.identifier in ids: dataset_ids[item.identifier.split('|')[0]] = item.getValue('pid') # call PID library to generate collection PID connector = esgfpid.Connector(handle_prefix=settings.PID_PREFIX, messaging_service_exchange_name=settings.PID_MESSAGING_SERVICE_EXCHANGE, messaging_service_credentials=pid_messaging_service_credentials, ) connector.start_messaging_thread() pid = connector.create_data_cart_pid(dataset_ids) connector.finish_messaging_thread() print 'Generated data cart PID for %d datasets: %s' % (len(ids), pid) return HttpResponse(json.dumps(pid), content_type="application/json")
def search(request, project_short_name): """ Entry point for all search requests (GET/POST). Loads project-specific configuration. """ # store this URL at session scope so other pages can reload the last search request.session[LAST_SEARCH_URL] = request.get_full_path( ) # relative search page URL + optional query string fromRedirectFlag = False if request.session.get(SEARCH_REDIRECT, None): fromRedirectFlag = True # remove POST redirect flag from session del request.session[SEARCH_REDIRECT] # retrieve project from database project = get_object_or_404(Project, short_name__iexact=project_short_name) # check permission if project.private: if request.user.is_anonymous(): return HttpResponseRedirect( reverse('login') + "?next=%s" % request.path) else: if not userHasUserPermission(request.user, project): return HttpResponseForbidden(PERMISSION_DENIED_MESSAGE) config = _getSearchConfig(request, project) if config: # config.printme() # pass on project as extra argument to search # also include possible custom template return search_config(request, config, extra={ 'project': project, 'title2': '%s Data Search' % project.short_name, 'template': getQueryDict(request).get(TEMPLATE, None) }, fromRedirectFlag=fromRedirectFlag) # search is not configured for this project else: messages = [ 'Searching is not enabled for this project.', 'Please contact the project administrators for further assistance.' ] return render(request, 'cog/common/message.html', { 'project': project, 'messages': messages, 'title2': 'Data Search' })
def datacart_wget(request, site_id, user_id): # load User object user = get_object_or_404(User, pk=user_id) # security check if not request.user.id != user_id: raise Exception("User not authorized to use datacart") # retrieve list of selected dataset ids queryDict = getQueryDict(request) ids = queryDict.getlist('id') # map of dataset ids grouped by index node response_data = {} # loop over datacart items try: datacart = DataCart.objects.get(user=user) except DataCart.DoesNotExist: datacart = None if datacart: for item in datacart.items.all(): # filter selected datasets only if item.identifier in ids: # group selected dataset by index_node index_node = item.getValue('index_node') wget_key = index_node shard = item.getValue('shard') if shard is not None and len(shard.strip()) > 0: wget_key += "|" + shard if wget_key not in response_data: response_data[wget_key] = [] response_data[wget_key].append(item.identifier) ''' Example response_data: { u'esgf-node.jpl.nasa.gov|localhost:8982': [u'CMAC.NASA-GSFC.AIRS.mon.v1|esg-datanode.jpl.nasa.gov'], u'esgf-node.jpl.nasa.gov': [u'obs4MIPs.NASA-JPL.QuikSCAT.mon.v1|esgf-node.jpl.nasa.gov'], u'pcmdi9.llnl.gov': [ u'cmip5.output1.INM.inmcm4.1pctCO2.day.atmos.day.r1i1p1.v20110323|pcmdi9.llnl.gov', u'cmip5.output1.INM.inmcm4.esmHistorical.fx.atmos.fx.r0i0p0.v20110927|pcmdi9.llnl.gov', u'cmip5.output1.INM.inmcm4.1pctCO2.day.ocean.day.r1i1p1.v20110323|pcmdi9.llnl.gov' ] } ''' return HttpResponse(json.dumps(response_data), content_type='application/json')
def datacart_wget(request, site_id, user_id): # load User object user = get_object_or_404(User, pk=user_id) # security check if not request.user.id != user_id: raise Exception("User not authorized to use datacart") # retrieve list of selected dataset ids queryDict = getQueryDict(request) ids = queryDict.getlist('id') # map of dataset ids grouped by index node response_data = {} # loop over datacart items try: datacart = DataCart.objects.get(user=user) except DataCart.DoesNotExist: datacart = None if datacart: for item in datacart.items.all(): # filter selected datasets only if item.identifier in ids: # group selected dataset by index_node index_node = item.getValue('index_node') wget_key = index_node shard = item.getValue('shard') if shard is not None and len(shard.strip())>0: wget_key += "|" + shard if wget_key not in response_data: response_data[wget_key] = [] response_data[wget_key].append(item.identifier) ''' Example response_data: { u'esgf-node.jpl.nasa.gov|localhost:8982': [u'CMAC.NASA-GSFC.AIRS.mon.v1|esg-datanode.jpl.nasa.gov'], u'esgf-node.jpl.nasa.gov': [u'obs4MIPs.NASA-JPL.QuikSCAT.mon.v1|esgf-node.jpl.nasa.gov'], u'pcmdi9.llnl.gov': [ u'cmip5.output1.INM.inmcm4.1pctCO2.day.atmos.day.r1i1p1.v20110323|pcmdi9.llnl.gov', u'cmip5.output1.INM.inmcm4.esmHistorical.fx.atmos.fx.r0i0p0.v20110927|pcmdi9.llnl.gov', u'cmip5.output1.INM.inmcm4.1pctCO2.day.ocean.day.r1i1p1.v20110323|pcmdi9.llnl.gov' ] } ''' return HttpResponse(json.dumps(response_data), content_type='application/json')
def get_openid(request): """ Retrieves the user openid from either the request query string, or the request cookies. """ queryDict = getQueryDict(request) if queryDict.get('openid', None): return queryDict['openid'] elif request.COOKIES.get('openid', None): return request.COOKIES['openid'] else: return ''
def _getUnsavedProjectSubFolders(project, request): """ Function to create the project top-level sub-folders, in the appropriate state, WITHOUT PERSISTING THEM TO THE DATABASE. """ folders = [] for key, value in TOP_SUB_FOLDERS.items(): folder = Folder(name=value, project=project, active=False) if request is not None and ("folder_%s" % value) in getQueryDict(request).keys(): folder.active = True folders.append(folder) return folders
def membership_process(request, project_short_name): # load project project = get_object_or_404(Project, short_name__iexact=project_short_name) # check permission if not userHasAdminPermission(request.user, project): return HttpResponseForbidden(PERMISSION_DENIED_MESSAGE) queryDict = getQueryDict(request) for (name, value) in queryDict.items(): if name.startswith(NEW_MEMBERSHIP) or name.startswith(OLD_MEMBERSHIP) or name.startswith(NO_MEMBERSHIP): (prefix, group_name, user_id) = name.split(":") group = get_object_or_404(Group, name=group_name) user = get_object_or_404(User, pk=user_id) # HTTP POST parameter from form check-box, all checks are treated as new # process checkbox as a new user if name.startswith(NEW_MEMBERSHIP): status = addMembership(user, group, admin=request.user) #only email if user not already a member if status == RESULT_SUCCESS: notifyUserOfMembershipGranted(project, group, user, request) # process hidden input field that indicates current membership # HTTP POST parameter from form hidden field # if user has a role, then {{isEnrolled}} turns on the hidden field with value = "on" elif name.startswith(OLD_MEMBERSHIP): try: # don't delete from group if checkbox is still checked (e.g. new membership) new_membership = queryDict[encodeMembershipPar(NEW_MEMBERSHIP, group.name, user.id)] except KeyError: # checkbox is empty, so remove from group status = cancelMembership(user, group, admin=request.user) if status == RESULT_SUCCESS: notifyUserOfGroupRemoval(project, group, user) # HTTP GET parameter (when delete link clicked) elif name.startswith(NO_MEMBERSHIP): # TODO check group here, should remove from all groups status = cancelMembership(user, group) if status == RESULT_SUCCESS: notifyUserOfGroupRemoval(project, group, user) # redirect to the original listing that submitted the processing view_name = queryDict['view_name'] return HttpResponseRedirect(reverse(view_name, kwargs={'project_short_name': project_short_name})+"?status=success")
def pagination_url(request, page_number): '''Constructs the previous/next URL for a paginated page including all GET request parameters.''' # copy all current GET/POST request parameters params = getQueryDict(request).copy() # replace/add 'page' parameter params['page'] = page_number # remove 'csrfmiddlewaretoken' from GET URL if params.get('csrfmiddlewaretoken', None): del params['csrfmiddlewaretoken'] # build full URL return '%s?%s' % (request.path, params.urlencode())
def _buildSearchInputFromHttpRequest(request, searchConfig): """Assembles the search input from the HTTP request only (NO project fixed constraints yet).""" queryDict = getQueryDict(request) # populate input with search constraints from HTTP request # only use facet keys from project configuration searchInput = SearchInput() for facetGroup in searchConfig.facetProfile.facetGroups: for key in facetGroup.getKeys(): if queryDict.get(key, None): for value in queryDict.getlist(key): if value: searchInput.addConstraint(key, value) # text if queryDict.get('query', None): searchInput.query = queryDict['query'] # type if queryDict.get('type', None): searchInput.type = queryDict['type'] # replica=True/False if queryDict.get('replica', None) == 'on': searchInput.replica = True # latest=True/False if queryDict.get('latest', None) == 'on': searchInput.latest = False # local=True/False + replica=True if queryDict.get('local', None) == 'on': searchInput.local = True searchInput.replica = True # NEW: local search ALWAYS implies ALL replicas # offset, limit if queryDict.get('offset', 0): searchInput.offset = int(queryDict['offset']) if queryDict.get('limit', 0): searchInput.limit = int(queryDict['limit']) # max_version, min_version # NOTE: implies search on All Versions ( searchInput.latest = False ) if queryDict.get('max_version', ''): searchInput.max_version = queryDict['max_version'].strip() searchInput.latest = False if queryDict.get('min_version', ''): searchInput.min_version = queryDict['min_version'].strip() searchInput.latest = False return searchInput
def admin_users(request): # optional parameters (via GET or POST) queryDict = getQueryDict(request) sortby = queryDict.get('sortby', 'username') # default to sort by 'username' match = queryDict.get('match', None) if match: users = getAdminUsersThatMatch(match, sortby=sortby) else: users = User.objects.all().order_by(sortby) title = 'List System/Node Users' return render(request, 'cog/admin/admin_users.html', {'users': paginate(users, request, max_counts_per_page=50), 'title': title})
def admin_users(request): # optional parameters (via GET or POST) queryDict = getQueryDict(request) sortby = queryDict.get('sortby', 'username') # default to sort by 'username' match = queryDict.get('match', None) if match: users = getAdminUsersThatMatch(match, sortby=sortby) else: users = User.objects.all().order_by(sortby) title = 'List System/Node Users' return render(request, 'cog/admin/admin_users.html', { 'users': paginate(users, request, max_counts_per_page=50), 'title': title })
def membership_list_all(request, project_short_name): # load project project = get_object_or_404(Project, short_name__iexact=project_short_name) # check permission if not userHasAdminPermission(request.user, project): return HttpResponseForbidden(PERMISSION_DENIED_MESSAGE) # load all users - that match... match = getQueryDict(request).get('match', None) # works for GET or POST if match: users = getUsersThatMatch(match) else: users = User.objects.all().order_by('last_name') title = 'Assign Project Members From "List Of All Node Users"' view_name = 'membership_list_all' return render_membership_page(request, project, users, title, view_name)
def search(request, project_short_name): """ Entry point for all search requests (GET/POST). Loads project-specific configuration. """ # store this URL at session scope so other pages can reload the last search request.session[LAST_SEARCH_URL] = request.get_full_path() # relative search page URL + optional query string fromRedirectFlag = False if request.session.get(SEARCH_REDIRECT, None): fromRedirectFlag = True # remove POST redirect flag from session del request.session[SEARCH_REDIRECT] # retrieve project from database project = get_object_or_404(Project, short_name__iexact=project_short_name) # check permission if project.private: if request.user.is_anonymous(): return HttpResponseRedirect(reverse('login')+"?next=%s" % request.path) else: if not userHasUserPermission(request.user, project): return HttpResponseForbidden(PERMISSION_DENIED_MESSAGE) config = _getSearchConfig(request, project) if config: # config.printme() # pass on project as extra argument to search # also include possible custom template return search_config(request, config, extra={'project': project, 'title2': '%s Data Search'% project.short_name, 'template':getQueryDict(request).get(TEMPLATE,None) }, fromRedirectFlag=fromRedirectFlag) # search is not configured for this project else: messages = ['Searching is not enabled for this project.', 'Please contact the project administrators for further assistance.'] return render(request, 'cog/common/message.html', {'project': project, 'messages': messages, 'title2': 'Data Search'})
def save_user_tag(request): # POST: when local user submits form, GET: when remote user is redirected to this node if request.method == 'POST' or request.method == 'GET': # retrieve tag queryDict = getQueryDict(request) tagName = queryDict['tag'] redirect = queryDict['redirect'] print 'Saving user tag: %s' % tagName print 'Eventually redirecting to: %s' % redirect if isUserLocal(request.user): try: tag = ProjectTag.objects.get(name__iexact=tagName) except ObjectDoesNotExist: tag = ProjectTag.objects.create(name=tagName) print 'Created new tag: %s' % tag # add this tag to the user preferences utags = request.user.profile.tags if tag not in utags.all(): utags.add(tag) request.user.profile.save() print 'Tag: %s added to user: %s' % (tagName, request.user) # set session flag to preselect a tab request.session['PROJECT_BROWSER_TAB'] = 3 return HttpResponseRedirect(redirect) # redirect request to user home node else: url = "http://%s%s?tag=%s&redirect=%s" % ( request.user.profile.site.domain, reverse('save_user_tag'), tagName, redirect) print 'Redirecting save request to URL=%s' % url # set session flag to eventually force reloading of user tags request.session['LAST_ACCESSED'] = 0 # also set session flag to preselect a tab request.session['PROJECT_BROWSER_TAB'] = 3 request.session.save() return HttpResponseRedirect(url)
def save_user_tag(request): # POST: when local user submits form, GET: when remote user is redirected to this node if request.method == 'POST' or request.method == 'GET': # retrieve tag queryDict = getQueryDict(request) tagName = queryDict['tag'] redirect = queryDict['redirect'] print 'Saving user tag: %s' % tagName print 'Eventually redirecting to: %s' % redirect if isUserLocal(request.user): try: tag = ProjectTag.objects.get(name__iexact=tagName) except ObjectDoesNotExist: tag = ProjectTag.objects.create(name=tagName) print 'Created new tag: %s' % tag # add this tag to the user preferences utags = request.user.profile.tags if tag not in utags.all(): utags.add(tag) request.user.profile.save() print 'Tag: %s added to user: %s' % (tagName, request.user) # set session flag to preselect a tab request.session['PROJECT_BROWSER_TAB'] = 3 return HttpResponseRedirect(redirect) # redirect request to user home node else: url = "http://%s%s?tag=%s&redirect=%s" % (request.user.profile.site.domain, reverse('save_user_tag'), tagName, redirect) print 'Redirecting save request to URL=%s' % url # set session flag to eventually force reloading of user tags request.session['LAST_ACCESSED'] = 0 # also set session flag to preselect a tab request.session['PROJECT_BROWSER_TAB'] = 3 request.session.save() return HttpResponseRedirect(url)
def doc_remove(request, doc_id): # retrieve document from database doc = get_object_or_404(Doc, pk=doc_id) project = doc.project # check permission if not userHasContributorPermission(request.user, project): return HttpResponseForbidden(PERMISSION_DENIED_MESSAGE) # delete doc altogether delete_doc(doc) # redirect to original page, or to project home if not found redirect = getQueryDict(request).get('redirect', None) if redirect is None: redirect = reverse('project_home', kwargs={'project_short_name': project.short_name.lower()}) # redirect to project home page #return HttpResponseRedirect( reverse('doc_list', kwargs={'project_short_name': project.short_name.lower() } ) ) return HttpResponseRedirect(redirect)
def doc_remove(request, doc_id): # retrieve document from database doc = get_object_or_404(Doc, pk=doc_id) project = doc.project # check permission if not userHasContributorPermission(request.user, project): return HttpResponseForbidden(PERMISSION_DENIED_MESSAGE) # delete doc altogether delete_doc(doc) # redirect to original page, or to project home if not found redirect = getQueryDict(request).get('redirect', None) if redirect is None: redirect = reverse( 'project_home', kwargs={'project_short_name': project.short_name.lower()}) # redirect to project home page #return HttpResponseRedirect( reverse('doc_list', kwargs={'project_short_name': project.short_name.lower() } ) ) return HttpResponseRedirect(redirect)
def membership_list_enrolled(request, project_short_name): # load project project = get_object_or_404(Project, short_name__iexact=project_short_name) # check permission if not userHasAdminPermission(request.user, project): return HttpResponseForbidden(PERMISSION_DENIED_MESSAGE) # optional 'match' argument match = getQueryDict(request).get('match', None) # works for GET or POST if match: # filter all users by 'match' _users = getUsersThatMatch(match) # filter all users by project users = [user for user in _users if (user in project.getUserGroup().user_set.all() or user in project.getAdminGroup().user_set.all())] else: users = list(project.getUsers()) title = '%s Current Users' % project.short_name view_name = 'membership_list_enrolled' return render_membership_page(request, project, users, title, view_name)
def datacart_pid(request, site_id, user_id): # check User object user = get_object_or_404(User, pk=user_id) # security check if not request.user.id != user_id: raise Exception("User not authorized to modify datacart") pid_messaging_service_credentials = [] priority = 1 for cred in settings.PID_CREDENTIALS: parts = cred.split('|') if len(parts) == 6: ssl_enabled = False if parts[5].strip().upper() == 'TRUE': ssl_enabled = True pid_messaging_service_credentials.append({ 'url': parts[0].strip(), 'port': parts[1].strip(), 'vhost': parts[2].strip(), 'user': parts[3].strip(), 'password': parts[4].strip(), 'ssl_enabled': ssl_enabled, 'priority': priority }) priority += 1 # get list of dataset_pids and dataset_ids try: datacart = DataCart.objects.get(user=user) except DataCart.DoesNotExist: datacart = None queryDict = getQueryDict(request) ids = queryDict.getlist('id') dataset_ids = {} if datacart: for item in datacart.items.all(): # filter selected datasets only if item.identifier in ids: dataset_ids[item.identifier.split('|')[0]] = item.getValue( 'pid') # call PID library to generate collection PID connector = esgfpid.Connector( handle_prefix=settings.PID_PREFIX, messaging_service_exchange_name=settings. PID_MESSAGING_SERVICE_EXCHANGE, messaging_service_credentials=pid_messaging_service_credentials, ) connector.start_messaging_thread() pid = connector.create_data_cart_pid(dataset_ids) connector.finish_messaging_thread() print 'Generated data cart PID for %d datasets: %s' % (len(ids), pid) return HttpResponse(json.dumps(pid), content_type="application/json")
def download(request): ''' View that initiates the Globus download workflow by collecting and optionally sub-selecting the GridFTP URLs to be downloaded. This view can be invoked via GET (link from search page, one dataset only) or POST (link from data cart page, multiple datasets at once). Example URL: http://localhost:8000/globus/download/ ?dataset=obs4MIPs.NASA-JPL.AIRS.mon.v1%[email protected],obs4MIPs.NASA-JPL.MLS.mon.v1%[email protected] &method=web ''' # retrieve request parameters datasets = getQueryDict(request).get('dataset','').split(",") # optional query filter query = getQueryDict(request).get('query',None) # maximum number of files to query for, if specified limit = request.GET.get('limit', DOWNLOAD_LIMIT) # map of (data_node, list of GridFTP URLs to download) download_map = {} # loop over requested datasets for dataset in datasets: # query each index_node for all files belonging to that dataset (dataset_id, index_node) = str(dataset).split('@') params = [ ('type',"File"), ('dataset_id',dataset_id), ('offset','0'), ('limit',limit), ('fields','url'), ("format", "application/solr+json") ] if query is not None and len(query.strip())>0: params.append( ('query', query) ) # optional shard shard = request.GET.get('shard', '') if shard is not None and len(shard.strip()) > 0: params.append(('shards', shard+"/solr")) # '&shards=localhost:8982/solr' else: params.append(("distrib", "false")) url = "http://"+index_node+"/esg-search/search?"+urllib.urlencode(params) print 'Searching for files at URL: %s' % url jobj = getJson(url) # parse response for GridFTP URls if jobj is not None: for doc in jobj['response']['docs']: access = {} for url in doc['url']: # example URLs: # 'http://esg-datanode.jpl.nasa.gov/thredds/fileServer/esg_dataroot/obs4MIPs/observations/atmos/husNobs/mon/grid/NASA-JPL/AIRS/v20110608/husNobs_AIRS_L3_RetStd-v5_200209-201105.nc|application/netcdf|HTTPServer' # 'http://esg-datanode.jpl.nasa.gov/thredds/dodsC/esg_dataroot/obs4MIPs/observations/atmos/husNobs/mon/grid/NASA-JPL/AIRS/v20110608/husNobs_AIRS_L3_RetStd-v5_200209-201105.nc.html|application/opendap-html|OPENDAP' # 'globus:8a3f3166-e9dc-11e5-97d6-22000b9da45e//esg_dataroot/obs4MIPs/observations/atmos/husNobs/mon/grid/NASA-JPL/AIRS/v20110608/husNobs_AIRS_L3_RetStd-v5_200209-201105.nc|Globus|Globus' # 'gsiftp://esg-datanode.jpl.nasa.gov:2811//esg_dataroot/obs4MIPs/observations/atmos/husNobs/mon/grid/NASA-JPL/AIRS/v20110608/husNobs_AIRS_L3_RetStd-v5_200209-201105.nc|application/gridftp|GridFTP' parts = url.split('|') access[parts[2].lower()] = parts[0] if 'globus' in access: m = re.match('globus:([^/]*)(.*)', access['globus']) if m: gendpoint_name = m.group(1) path = m.group(2) if not gendpoint_name in download_map: download_map[gendpoint_name] = [] # insert empty list of paths download_map[gendpoint_name].append(path) else: print 'The file is not accessible through Globus' else: return HttpResponseServerError("Error querying for files URL") # store map in session request.session[GLOBUS_DOWNLOAD_MAP] = download_map print 'Stored Globus Download Map=%s at session scope' % download_map # redirect after post (to display page) return HttpResponseRedirect( reverse('globus_transfer') )
def getHttpParamValue(request, name): '''Retrieves an HTTP parameter value from either the GET or POST request dictionary.''' return getQueryDict(request).get(name, '')
def post_update(request, post_id): # retrieve post object from database post = get_object_or_404(Post, pk=post_id) # check permission if not userCanPost(request.user, post): return HttpResponseForbidden(PERMISSION_DENIED_MESSAGE) # check lock lock = getLock(post) if isLockedOut(request.user, lock): return getPostIsLockedRedirect(request, post.project, post, lock) if request.method == 'GET': # create/renew lock lock = createLock(post, request.user) # extract page partial URL if post.type == Post.TYPE_PAGE: post.url = get_project_page_sub_url(post.project, post.url) # create form object from model form = PostForm(post.type, post.project, instance=post) return render_post_form(request, form, post.project, post.type, lock=lock) else: # update existing database model with form data form = PostForm(post.type, post.project, request.POST, instance=post) # check versions queryDict = getQueryDict(request) if post.version != int(queryDict.get('version', -1)): print 'database version=%s form version=%s' % (post.version, queryDict.get('version', -1)) return getLostLockRedirect(request, post.project, post, lock) #print "1 PAGE URL=%s" % form.data['url'] if form.is_valid(): # TODO: if hit cancel, form does not render, what is missing from save? # build instance from form data post = form.save(commit=False) form_data = form.clean() if form_data.get("save_only"): # save instance post.save() return render_post_form(request, form, post.project, post.type, lock=lock) else: # increment version post.version += 1 # rebuild full page URL if post.type == Post.TYPE_PAGE: post.url = get_project_page_full_url(post.project, post.url) # change the author to the last editor post.author = request.user # update date post.update_date = now() # save instance post.save() # create project-topic relation if not existing already if post.topic is not None: createProjectTopicIfNotExisting(post.project, post.topic) # release lock deleteLock(post) # send post update signal post.send_signal(SIGNAL_OBJECT_UPDATED) # redirect to post (GET-POST-REDIRECT) if post.type != Post.TYPE_HYPERLINK: return redirect_to_post(request, post) # or to project home page else: return HttpResponseRedirect(reverse('project_home', args=[post.project.short_name.lower()])) else: print form.errors return render_post_form(request, form, post.project, post.type, lock=lock)
def search_post(request, searchInput, searchConfig, extra={}): """ View that processes a search POST request. Stores results in session, together with special SEARCH_REDIRECT session flag. Then redirects to the search GET URL. """ facetProfile = searchConfig.facetProfile searchService = searchConfig.searchService queryDict = getQueryDict(request) # validate user input (valid, error_message) = searchInput.isValid() if valid: # add project fixed constraints print 'Search POST: adding fixed project constraints' _searchInput = _addConfigConstraints(searchInput, searchConfig) _searchInput.printme() # set retrieval of all facets in profile _searchInput.facets = facetProfile.getAllKeys() # execute query for results, facets try: (url, xml) = searchService.search(_searchInput) searchOutput = deserialize(xml, facetProfile) # searchOutput.printme() # initialize new session data from extra argument dictionary data = extra data[SEARCH_INPUT] = searchInput # IMPORTANT: store in session the ORIGINAL search input WITHOUT project constraints data[SEARCH_OUTPUT] = searchOutput data[SEARCH_URL] = url data[FACET_PROFILE] = facetProfile # data[FACET_PROFILE] = sorted( facetProfile.getKeys() ) # sort facets by key except HTTPError: print "HTTP Request Error" data = request.session[SEARCH_DATA] data[SEARCH_INPUT] = searchInput data[ERROR_MESSAGE] = "Error: search may not be properly configured. Contact the Project " \ "Administrator." # invalid user input else: print "Invalid Search Input" # re-use previous data (output, profile and any extra argument) from session data = request.session[SEARCH_DATA] # override search input from request data[SEARCH_INPUT] = searchInput # add error data[ERROR_MESSAGE] = error_message # store data in session request.session[SEARCH_DATA] = data # update search path sp = request.session.get(SEARCH_PATH, []) # for key, values in searchInput.constraints.items(): # note: request parameters do NOT include the project fixed constraints req_constraints = [] # latest constraints from request for key, value in queryDict.items(): if not key in SEARCH_PATH_EXCLUDE and value != 'on': # value from 'checkbox_...' if value is not None and len(value) > 0: # disregard empty facet print 'key=%s value=%s' % (key, value) constraint = (key, value) req_constraints.append(constraint) if not constraint in sp: sp.append(constraint) # remove obsolete constraints sp = [item for item in sp if item in req_constraints] request.session[SEARCH_PATH] = sp # use POST-REDIRECT-GET pattern # flag the redirect in session request.session[SEARCH_REDIRECT] = True #return HttpResponseRedirect(request.get_full_path()) # relative search page URL + optional query string # GET redirect if extra.get(TEMPLATE, None): return HttpResponseRedirect(request.get_full_path()) # keep the query parameters else: return HttpResponseRedirect(request.path) # remove query parameters
def post_add(request, project_short_name, owner=None): """ View to create a Post object within a context project. Optionally, an owner object can be specified, which is assigned a reference to the newly created Post through its method .setPost(Post) """ # load project project = get_object_or_404(Project, short_name__iexact=project_short_name) # check permission if not userHasContributorPermission(request.user, project): return HttpResponseForbidden(PERMISSION_DENIED_MESSAGE) # retrieve type postType = getQueryDict(request).get('type') if request.method == 'GET': # create empty Post object, pre-populate project and type post = Post() post.project = project post.type = postType # optionally assign parent Post parent_id = request.GET.get('parent_id', None) if parent_id: ppost = get_object_or_404(Post, pk=parent_id) post.parent = ppost post.topic = ppost.topic # set fixed fields for hyperlinks #if postType == Post.TYPE_HYPERLINK: # post.template = None # post.is_private = False # post.is_restricted = False # create form from instance # note extra argument project to customize the queryset! form = PostForm(postType, project, instance=post) return render_post_form(request, form, project, postType) else: # create form object from form data form = PostForm(postType, project, request.POST) if form.is_valid(): # create a new post object but don't save it to the database yet post = form.save(commit=False) # modify the post object post.author = request.user # update date post.update_date = now() # page: build full page URL if post.type == Post.TYPE_PAGE: post.url = get_project_page_full_url(project, post.url) elif post.type != Post.TYPE_HYPERLINK: # assign temporary value before object id is assigned post.url = datetime.now() # assign post order, if top-level # note that the post.topic may be None if post.parent is None: pages = Post.objects.filter(project=project).filter(topic=post.topic).filter(parent=None).\ filter(Q(type=Post.TYPE_PAGE) | Q(type=Post.TYPE_HYPERLINK)).order_by('order') post.order = len(pages) + 1 else: post.order = 0 # save post object to the database (GET-POST-REDIRECT) post.save() # assign post URL and save again if post.type == Post.TYPE_BLOG or post.type == Post.TYPE_NOTES: post.url = reverse('post_detail', args=[post.id]) post.save() # create project-topic relation if not existing already if post.topic is not None: createProjectTopicIfNotExisting(project, post.topic) # assign this reference to owner if owner is not None: owner.setPost(post) owner.save() # send post update signal post.send_signal(SIGNAL_OBJECT_CREATED) # redirect to post (GET-POST-REDIRECT) if post.type != Post.TYPE_HYPERLINK: return redirect_to_post(request, post) # or to project home page else: return HttpResponseRedirect( reverse('project_home', args=[project_short_name.lower()])) # invalid data else: print form.errors return render_post_form(request, form, project, postType)
def post_update(request, post_id): # retrieve post object from database post = get_object_or_404(Post, pk=post_id) # check permission if not userCanPost(request.user, post): return HttpResponseForbidden(PERMISSION_DENIED_MESSAGE) # check lock lock = getLock(post) if isLockedOut(request.user, lock): return getPostIsLockedRedirect(request, post.project, post, lock) if request.method == 'GET': # create/renew lock lock = createLock(post, request.user) # extract page partial URL if post.type == Post.TYPE_PAGE: post.url = get_project_page_sub_url(post.project, post.url) # create form object from model form = PostForm(post.type, post.project, instance=post) return render_post_form(request, form, post.project, post.type, lock=lock) else: # update existing database model with form data form = PostForm(post.type, post.project, request.POST, instance=post) # check versions queryDict = getQueryDict(request) if post.version != int(queryDict.get('version', -1)): print 'database version=%s form version=%s' % ( post.version, queryDict.get('version', -1)) return getLostLockRedirect(request, post.project, post, lock) #print "1 PAGE URL=%s" % form.data['url'] if form.is_valid(): # TODO: if hit cancel, form does not render, what is missing from save? # build instance from form data post = form.save(commit=False) form_data = form.clean() if form_data.get("save_only"): # save instance post.save() return render_post_form(request, form, post.project, post.type, lock=lock) else: # increment version post.version += 1 # rebuild full page URL if post.type == Post.TYPE_PAGE: post.url = get_project_page_full_url( post.project, post.url) # change the author to the last editor post.author = request.user # update date post.update_date = now() # save instance post.save() # create project-topic relation if not existing already if post.topic is not None: createProjectTopicIfNotExisting(post.project, post.topic) # release lock deleteLock(post) # send post update signal post.send_signal(SIGNAL_OBJECT_UPDATED) # redirect to post (GET-POST-REDIRECT) if post.type != Post.TYPE_HYPERLINK: return redirect_to_post(request, post) # or to project home page else: return HttpResponseRedirect( reverse('project_home', args=[post.project.short_name.lower()])) else: print form.errors return render_post_form(request, form, post.project, post.type, lock=lock)
def download(request): ''' View that initiates the Globus download workflow by collecting and optionally sub-selecting the GridFTP URLs to be downloaded. This view can be invoked via GET (link from search page, one dataset only) or POST (link from data cart page, multiple datasets at once). Example URL: http://localhost:8000/globus/download/ ?dataset=obs4MIPs.NASA-JPL.AIRS.mon.v1%[email protected],obs4MIPs.NASA-JPL.MLS.mon.v1%[email protected] &method=web ''' # retrieve request parameters datasets = getQueryDict(request).get('dataset', '').split(",") # optional query filter query = getQueryDict(request).get('query', None) # maximum number of files to query for, if specified limit = request.GET.get('limit', DOWNLOAD_LIMIT) # map of (data_node, list of GridFTP URLs to download) download_map = {} # loop over requested datasets for dataset in datasets: # query each index_node for all files belonging to that dataset (dataset_id, index_node) = str(dataset).split('@') params = [('type', "File"), ('dataset_id', dataset_id), ('offset', '0'), ('limit', limit), ('fields', 'url'), ("format", "application/solr+json")] if query is not None and len(query.strip()) > 0: params.append(('query', query)) # optional shard shard = request.GET.get('shard', '') if shard is not None and len(shard.strip()) > 0: params.append( ('shards', shard + "/solr")) # '&shards=localhost:8982/solr' else: params.append(("distrib", "false")) url = "http://" + index_node + "/esg-search/search?" + urllib.urlencode( params) print 'Searching for files at URL: %s' % url jobj = getJson(url) # parse response for GridFTP URls if jobj is not None: for doc in jobj['response']['docs']: access = {} for url in doc['url']: # example URLs: # 'http://esg-datanode.jpl.nasa.gov/thredds/fileServer/esg_dataroot/obs4MIPs/observations/atmos/husNobs/mon/grid/NASA-JPL/AIRS/v20110608/husNobs_AIRS_L3_RetStd-v5_200209-201105.nc|application/netcdf|HTTPServer' # 'http://esg-datanode.jpl.nasa.gov/thredds/dodsC/esg_dataroot/obs4MIPs/observations/atmos/husNobs/mon/grid/NASA-JPL/AIRS/v20110608/husNobs_AIRS_L3_RetStd-v5_200209-201105.nc.html|application/opendap-html|OPENDAP' # 'globus:8a3f3166-e9dc-11e5-97d6-22000b9da45e//esg_dataroot/obs4MIPs/observations/atmos/husNobs/mon/grid/NASA-JPL/AIRS/v20110608/husNobs_AIRS_L3_RetStd-v5_200209-201105.nc|Globus|Globus' # 'gsiftp://esg-datanode.jpl.nasa.gov:2811//esg_dataroot/obs4MIPs/observations/atmos/husNobs/mon/grid/NASA-JPL/AIRS/v20110608/husNobs_AIRS_L3_RetStd-v5_200209-201105.nc|application/gridftp|GridFTP' parts = url.split('|') access[parts[2].lower()] = parts[0] if 'globus' in access: m = re.match('globus:([^/]*)(.*)', access['globus']) if m: gendpoint_name = m.group(1) path = m.group(2) if not gendpoint_name in download_map: download_map[gendpoint_name] = [ ] # insert empty list of paths download_map[gendpoint_name].append(path) else: print 'The file is not accessible through Globus' else: return HttpResponseServerError("Error querying for files URL") # store map in session request.session[GLOBUS_DOWNLOAD_MAP] = download_map print 'Stored Globus Download Map=%s at session scope' % download_map # redirect after post (to display page) return HttpResponseRedirect(reverse('globus_transfer'))
def search_post(request, searchInput, searchConfig, extra={}): """ View that processes a search POST request. Stores results in session, together with special SEARCH_REDIRECT session flag. Then redirects to the search GET URL. """ facetProfile = searchConfig.facetProfile searchService = searchConfig.searchService queryDict = getQueryDict(request) # validate user input (valid, error_message) = searchInput.isValid() if valid: # add project fixed constraints print 'Search POST: adding fixed project constraints' _searchInput = _addConfigConstraints(searchInput, searchConfig) _searchInput.printme() # set retrieval of all facets in profile _searchInput.facets = facetProfile.getAllKeys() # execute query for results, facets try: (url, xml) = searchService.search(_searchInput) searchOutput = deserialize(xml, facetProfile) # searchOutput.printme() # initialize new session data from extra argument dictionary data = extra data[ SEARCH_INPUT] = searchInput # IMPORTANT: store in session the ORIGINAL search input WITHOUT project constraints data[SEARCH_OUTPUT] = searchOutput data[SEARCH_URL] = url data[FACET_PROFILE] = facetProfile # data[FACET_PROFILE] = sorted( facetProfile.getKeys() ) # sort facets by key except HTTPError: print "HTTP Request Error" data = request.session[SEARCH_DATA] data[SEARCH_INPUT] = searchInput data[ERROR_MESSAGE] = "Error: search may not be properly configured. Contact the Project " \ "Administrator." # invalid user input else: print "Invalid Search Input" # re-use previous data (output, profile and any extra argument) from session data = request.session[SEARCH_DATA] # override search input from request data[SEARCH_INPUT] = searchInput # add error data[ERROR_MESSAGE] = error_message # store data in session request.session[SEARCH_DATA] = data # update search path sp = request.session.get(SEARCH_PATH, []) # for key, values in searchInput.constraints.items(): # note: request parameters do NOT include the project fixed constraints req_constraints = [] # latest constraints from request for key, value in queryDict.items(): if not key in SEARCH_PATH_EXCLUDE and value != 'on': # value from 'checkbox_...' if value is not None and len(value) > 0: # disregard empty facet print 'key=%s value=%s' % (key, value) constraint = (key, value) req_constraints.append(constraint) if not constraint in sp: sp.append(constraint) # remove obsolete constraints sp = [item for item in sp if item in req_constraints] request.session[SEARCH_PATH] = sp # use POST-REDIRECT-GET pattern # flag the redirect in session request.session[SEARCH_REDIRECT] = True #return HttpResponseRedirect(request.get_full_path()) # relative search page URL + optional query string # GET redirect if extra.get(TEMPLATE, None): return HttpResponseRedirect( request.get_full_path()) # keep the query parameters else: return HttpResponseRedirect(request.path) # remove query parameters
def post_add(request, project_short_name, owner=None): """ View to create a Post object within a context project. Optionally, an owner object can be specified, which is assigned a reference to the newly created Post through its method .setPost(Post) """ # load project project = get_object_or_404(Project, short_name__iexact=project_short_name) # check permission if not userHasContributorPermission(request.user, project): return HttpResponseForbidden(PERMISSION_DENIED_MESSAGE) # retrieve type postType = getQueryDict(request).get('type') if request.method == 'GET': # create empty Post object, pre-populate project and type post = Post() post.project = project post.type = postType # optionally assign parent Post parent_id = request.GET.get('parent_id', None) if parent_id: ppost = get_object_or_404(Post, pk=parent_id) post.parent = ppost post.topic = ppost.topic # set fixed fields for hyperlinks #if postType == Post.TYPE_HYPERLINK: # post.template = None # post.is_private = False # post.is_restricted = False # create form from instance # note extra argument project to customize the queryset! form = PostForm(postType, project, instance=post) return render_post_form(request, form, project, postType) else: # create form object from form data form = PostForm(postType, project, request.POST) if form.is_valid(): # create a new post object but don't save it to the database yet post = form.save(commit=False) # modify the post object post.author = request.user # update date post.update_date = now() # page: build full page URL if post.type == Post.TYPE_PAGE: post.url = get_project_page_full_url(project, post.url) elif post.type != Post.TYPE_HYPERLINK: # assign temporary value before object id is assigned post.url = datetime.now() # assign post order, if top-level # note that the post.topic may be None if post.parent is None: pages = Post.objects.filter(project=project).filter(topic=post.topic).filter(parent=None).\ filter(Q(type=Post.TYPE_PAGE) | Q(type=Post.TYPE_HYPERLINK)).order_by('order') post.order = len(pages)+1 else: post.order = 0 # save post object to the database (GET-POST-REDIRECT) post.save() # assign post URL and save again if post.type == Post.TYPE_BLOG or post.type == Post.TYPE_NOTES: post.url = reverse('post_detail', args=[post.id]) post.save() # create project-topic relation if not existing already if post.topic is not None: createProjectTopicIfNotExisting(project, post.topic) # assign this reference to owner if owner is not None: owner.setPost(post) owner.save() # send post update signal post.send_signal(SIGNAL_OBJECT_CREATED) # redirect to post (GET-POST-REDIRECT) if post.type != Post.TYPE_HYPERLINK: return redirect_to_post(request, post) # or to project home page else: return HttpResponseRedirect(reverse('project_home', args=[project_short_name.lower()])) # invalid data else: print form.errors return render_post_form(request, form, project, postType)