def video(self): data = None if self.video_source and self.video_id: # Check for cached data data = self._video_cache if not data: data = { 'source': self.video_source, 'id': self.video_id, 'url': self.video_url, 'embeddable_url': self.embeddable_video_url, 'duration': 0, 'uploaded_at': '', 'thumbnail': '', } if self.video_source == 'youtube': youtube_video = requests.get( f'https://www.googleapis.com/youtube/v3/videos?part=snippet,contentDetails&id={self.video_id}&key={app.config["YOUTUBE_API_KEY"]}' ).json() if not youtube_video or 'items' not in youtube_video: raise VideoException( "Unable to fetch data, please check the youtube url or API key" ) elif not youtube_video['items']: # Response has zero item for our given video ID. # This will happen if the video has been removed from YouTube. self._source_video_exists = False else: youtube_video = youtube_video['items'][0] data['duration'] = parse_duration( youtube_video['contentDetails'] ['duration']).total_seconds() data['uploaded_at'] = parse_isoformat( youtube_video['snippet']['publishedAt'], naive=False) data['thumbnail'] = youtube_video['snippet'][ 'thumbnails']['medium']['url'] elif self.video_source == 'vimeo': vimeo_video = requests.get( f'https://vimeo.com/api/v2/video/{self.video_id}.json') if vimeo_video.status_code == 200: vimeo_video = vimeo_video.json()[0] data['duration'] = vimeo_video['duration'] # Vimeo returns naive datetime, we will add utc timezone to it data['uploaded_at'] = parse_isoformat( vimeo_video['upload_date'], delimiter=' ').replace(tzinfo=utc) data['thumbnail'] = vimeo_video['thumbnail_medium'] else: # Video doesn't exist on Vimeo anymore self._source_video_exists = False self._video_cache = data # using _video_cache setter to set cache return data
def import_from(self, data): super(ContentMixin, self).import_from(data) self.start_datetime = parse_isoformat(data['start_datetime']) self.end_datetime = parse_isoformat(data['end_datetime']) self.timezone = data['timezone'] self.location_name = data['location_name'] self.location_address = data['location_address'] self.mapmarker = data['mapmarker'] self.capacity = data['capacity'] self.allow_waitlisting = data['allow_waitlisting'] self.allow_maybe = data['allow_maybe']
def test_parse_isoformat(self): assert parse_isoformat( '1882-12-11T00:00:00.1234Z') == datetime.datetime( 1882, 12, 11, 0, 0, 0, 123400) assert parse_isoformat('1882-12-11T00:00:00Z'), datetime.datetime( 1882, 12, 11, 0, 0) assert parse_isoformat('1882-12-11T00:00:00.1234Z', naive=False) == datetime.datetime(1882, 12, 11, 0, 0, 0, 123400, tzinfo=UTC) assert parse_isoformat('1882-12-11T00:00:00Z', naive=False) == datetime.datetime(1882, 12, 11, 0, 0, tzinfo=UTC) with self.assertRaises(ParseError): parse_isoformat('2019-05-03T05:02:26.340937Z\'') with self.assertRaises(ParseError): parse_isoformat('2019-05-03T05:02:26.340937Z\'', naive=False)
def import_from(self, data): self.uuid = data['uuid'] self.name = data['name'] self.title = data['title'] self.author = data.get('author') self.published_at = parse_isoformat(data['published_at']) self.properties = data['properties']
def _video_cache(self): data = redis_store.hgetall(self.video_cache_key) if data: if 'uploaded_at' in data and data['uploaded_at']: data['uploaded_at'] = parse_isoformat(data['uploaded_at'], naive=False) if 'duration' in data and data['duration']: data['duration'] = int(data['duration']) return data
def import_from(self, data): super(ContentMixin, self).import_from(data) last = self.last_revision() if last and last.updated_at >= parse_isoformat(data['revision_updated_at']): # Don't import if data is older than or the same as the last revision return revision = self.revise() revision.title = data['title'] revision.description = data['description'] revision.content = data['content'] revision.template = data['template']
def _data(self): if not hasattr(self, '_data_cached'): # Get JSON and cache locally try: r = requests.get('http://funnel.hasgeek.com/%s/json' % self.funnel_name) data = r.json() if callable(r.json) else r.json sectionmap = dict([(s['title'], s['name']) for s in data['sections']]) for proposal in data['proposals']: proposal['submitted'] = parse_isoformat(proposal['submitted']) proposal['section_name'] = sectionmap.get(proposal['section']) v = proposal['votes'] proposal['votes'] = '+%d' % v if v > 0 else '%d' % v self._data_cached = data except ConnectionError: self._data_cached = { 'proposals': [], 'sections': [], 'space': {}, } return self._data_cached
def index(basequery=None, md5sum=None, tag=None, domain=None, location=None, title=None, showall=True, statuses=None, batched=True, ageless=False, template_vars={}): is_siteadmin = lastuser.has_permission('siteadmin') if basequery is None: is_index = True else: is_index = False now = datetime.utcnow() if basequery is None and not (g.user or g.kiosk or (g.board and not g.board.require_login)): showall = False batched = False if basequery is None: basequery = JobPost.query # Apply request.args filters data_filters = {} f_types = request.args.getlist('t') while '' in f_types: f_types.remove('') if f_types: data_filters['types'] = f_types basequery = basequery.join(JobType).filter(JobType.name.in_(f_types)) f_categories = request.args.getlist('c') while '' in f_categories: f_categories.remove('') if f_categories: data_filters['categories'] = f_categories basequery = basequery.join(JobCategory).filter( JobCategory.name.in_(f_categories)) r_locations = request.args.getlist('l') if location: r_locations.append(location['geonameid']) f_locations = [] remote_location = getbool(request.args.get('anywhere')) or False for rl in r_locations: if isinstance(rl, int) and rl > 0: f_locations.append(rl) elif rl == 'anywhere': remote_location = True elif rl.isdigit(): f_locations.append(int(rl)) elif rl: ld = location_geodata(rl) if ld: f_locations.append(ld['geonameid']) remote_location_query = basequery.filter( JobPost.remote_location == True) # NOQA if f_locations: locations_query = basequery.join(JobLocation).filter( JobLocation.geonameid.in_(f_locations)) else: locations_query = basequery.join(JobLocation) if f_locations and remote_location: data_filters['locations'] = f_locations data_filters['anywhere'] = True recency = JobPost.datetime > datetime.utcnow() - agelimit basequery = locations_query.filter(recency).union( remote_location_query.filter(recency)) elif f_locations: data_filters['locations'] = f_locations basequery = locations_query elif remote_location: data_filters['anywhere'] = True # Only works as a positive filter: you can't search for jobs that are NOT anywhere basequery = remote_location_query if 'currency' in request.args and request.args[ 'currency'] in CURRENCY.keys(): currency = request.args['currency'] data_filters['currency'] = currency basequery = basequery.filter(JobPost.pay_currency == currency) pay_graph = currency else: pay_graph = False if getbool(request.args.get('equity')): # Only works as a positive filter: you can't search for jobs that DON'T pay in equity data_filters['equity'] = True basequery = basequery.filter(JobPost.pay_equity_min != None) # NOQA if 'pay' in request.args or ('pmin' in request.args and 'pmax' in request.args): if 'pay' in request.args: f_pay = string_to_number(request.args['pay']) f_min = int(f_pay * 0.90) f_max = int(f_pay * 1.30) else: # Legacy URL with min/max values f_min = string_to_number(request.args['pmin']) f_max = string_to_number(request.args['pmax']) f_pay = f_min # Use min for pay now if f_pay is not None and f_min is not None and f_max is not None: data_filters['pay'] = f_pay basequery = basequery.filter(JobPost.pay_cash_min < f_max, JobPost.pay_cash_max >= f_min) else: f_pay = None f_min = None f_max = None if getbool(request.args.get('archive')): ageless = True data_filters['archive'] = True statuses = POSTSTATUS.ARCHIVED search_domains = None if request.args.get('q'): q = for_tsquery(request.args['q']) try: # TODO: Can we do syntax validation without a database roundtrip? db.session.query(db.func.to_tsquery(q)).all() except ProgrammingError: db.session.rollback() g.event_data['search_syntax_error'] = (request.args['q'], q) if not request.is_xhr: flash( _(u"Search terms ignored because this didn’t parse: {query}" ).format(query=q), 'danger') else: # Query's good? Use it. data_filters['query'] = q search_domains = Domain.query.filter( Domain.search_vector.match(q, postgresql_regconfig='english'), Domain.is_banned == False).options( db.load_only('name', 'title', 'logo_url')).all() # NOQA basequery = basequery.filter( JobPost.search_vector.match(q, postgresql_regconfig='english')) if data_filters: g.event_data['filters'] = data_filters showall = True batched = True # getposts sets g.board_jobs, used below posts = getposts(basequery, pinned=True, showall=showall, statuses=statuses, ageless=ageless).all() if is_siteadmin or (g.user and g.user.flags.get('is_employer_month')): cache_viewcounts(posts) if posts: employer_name = posts[0].company_name else: employer_name = u'a single employer' if g.user: g.starred_ids = set( g.user.starred_job_ids(agelimit if not ageless else None)) else: g.starred_ids = set() jobpost_ab = session_jobpost_ab() # Make lookup slightly faster in the loop below since 'g' is a proxy board = g.board if board: board_jobs = g.board_jobs else: board_jobs = {} if is_index and posts and not g.kiosk: # Group posts by email_domain on index page only, when not in kiosk mode grouped = OrderedDict() for post in posts: pinned = post.pinned if board is not None: blink = board_jobs.get( post.id ) # board_jobs only contains the last 30 days, no archive if blink is not None: pinned = blink.pinned if pinned: # Make pinned posts appear in a group of one grouped.setdefault(('s', post.hashid), []).append( (pinned, post, bgroup(jobpost_ab, post))) elif post.status == POSTSTATUS.ANNOUNCEMENT: # Make announcements also appear in a group of one grouped.setdefault(('a', post.hashid), []).append( (pinned, post, bgroup(jobpost_ab, post))) elif post.domain.is_webmail: grouped.setdefault(('ne', post.md5sum), []).append( (pinned, post, bgroup(jobpost_ab, post))) else: grouped.setdefault(('nd', post.email_domain), []).append( (pinned, post, bgroup(jobpost_ab, post))) pinsandposts = None else: grouped = None if g.board: pinsandposts = [] for post in posts: pinned = post.pinned if board is not None: blink = board_jobs.get( post.id ) # board_jobs only contains the last 30 days, no archive if blink is not None: pinned = blink.pinned pinsandposts.append((pinned, post, bgroup(jobpost_ab, post))) else: pinsandposts = [(post.pinned, post, bgroup(jobpost_ab, post)) for post in posts] # Pick a header campaign (only if not kiosk or an XHR reload) pay_graph_data = None if not g.kiosk: if g.preview_campaign: header_campaign = g.preview_campaign else: geonameids = g.user_geonameids + f_locations header_campaign = Campaign.for_context(CAMPAIGN_POSITION.HEADER, board=g.board, user=g.user, anon_user=g.anon_user, geonameids=geonameids) if pay_graph: pay_graph_data = make_pay_graph(pay_graph, posts, rmin=f_min, rmax=f_max) else: header_campaign = None loadmore = False if batched: # Figure out where the batch should start from startdate = None if 'startdate' in request.values: try: startdate = parse_isoformat(request.values['startdate']) except ValueError: pass batchsize = 32 # list of posts that were pinned at the time of first load pinned_hashids = request.args.getlist('ph') # Depending on the display mechanism (grouped or ungrouped), extract the batch if grouped: if not startdate: startindex = 0 for row in grouped.values(): # break when a non-pinned post is encountered if (not row[0][0]): break else: pinned_hashids.append(row[0][1].hashid) else: # Loop through group looking for start of next batch. See below to understand the # nesting structure of 'grouped' for startindex, row in enumerate(grouped.values()): # Skip pinned posts when looking for starting index if (row[0][1].hashid not in pinned_hashids and row[0][1].datetime < startdate): break batch = grouped.items()[startindex:startindex + batchsize] if startindex + batchsize < len(grouped): # Get the datetime of the last group's first item # batch = [((type, domain), [(pinned, post, bgroup), ...])] # batch[-1] = ((type, domain), [(pinned, post, bgroup), ...]) # batch[-1][1] = [(pinned, post, bgroup), ...] # batch[-1][1][0] = (pinned, post, bgroup) # batch[-1][1][0][1] = post loadmore = batch[-1][1][0][1].datetime grouped = OrderedDict(batch) elif pinsandposts: if not startdate: startindex = 0 for row in pinsandposts: # break when a non-pinned post is encountered if not row[0]: break else: pinned_hashids.append(row[1].hashid) else: for startindex, row in enumerate(pinsandposts): # Skip pinned posts when looking for starting index if (row[1].hashid not in pinned_hashids and row[1].datetime < startdate): break batch = pinsandposts[startindex:startindex + batchsize] if startindex + batchsize < len(pinsandposts): # batch = [(pinned, post), ...] loadmore = batch[-1][1].datetime pinsandposts = batch if grouped: g.impressions = { post.id: (pinflag, post.id, is_bgroup) for group in grouped.itervalues() for pinflag, post, is_bgroup in group } elif pinsandposts: g.impressions = { post.id: (pinflag, post.id, is_bgroup) for pinflag, post, is_bgroup in pinsandposts } # Test values for development: # if not g.user_geonameids: # g.user_geonameids = [1277333, 1277331, 1269750] if not location and 'l' not in request.args and g.user_geonameids and ( g.user or g.anon_user) and ( (not g.board.auto_locations) if g.board else True): # No location filters? Prompt the user ldata = location_geodata(g.user_geonameids) location_prompts = [ ldata[geonameid] for geonameid in g.user_geonameids if geonameid in ldata ] else: location_prompts = [] query_params = request.args.to_dict(flat=False) if loadmore: query_params.update({ 'startdate': loadmore.isoformat() + 'Z', 'ph': pinned_hashids }) return dict(pinsandposts=pinsandposts, grouped=grouped, now=now, newlimit=newlimit, title=title, md5sum=md5sum, domain=domain, location=location, employer_name=employer_name, showall=showall, is_index=is_index, header_campaign=header_campaign, loadmore=loadmore, location_prompts=location_prompts, search_domains=search_domains, query_params=query_params, is_siteadmin=is_siteadmin, pay_graph_data=pay_graph_data, paginated=index_is_paginated(), template_vars=template_vars)
def fetch_jobposts(request_args, request_values, filters, is_index, board, board_jobs, gkiosk, basequery, md5sum, domain, location, title, showall, statusfilter, batched, ageless, template_vars, search_query=None, query_string=None): if basequery is None: basequery = JobPost.query # Apply request.args filters data_filters = {} f_types = filters.get('t') or request_args.getlist('t') while '' in f_types: f_types.remove('') if f_types: data_filters['types'] = f_types basequery = basequery.join(JobType).filter(JobType.name.in_(f_types)) f_categories = filters.get('c') or request_args.getlist('c') while '' in f_categories: f_categories.remove('') if f_categories: data_filters['categories'] = f_categories basequery = basequery.join(JobCategory).filter(JobCategory.name.in_(f_categories)) f_domains = filters.get('d') or request_args.getlist('d') while '' in f_domains: f_domains.remove('') if f_domains: basequery = basequery.join(Domain).filter(Domain.name.in_(f_domains)) f_tags = filters.get('k') or request_args.getlist('k') while '' in f_tags: f_tags.remove('') if f_tags: basequery = basequery.join(JobPostTag).join(Tag).filter(Tag.name.in_(f_tags)) data_filters['location_names'] = r_locations = filters.get('l') or request_args.getlist('l') if location: r_locations.append(location['geonameid']) f_locations = [] remote_location = getbool(filters.get('anywhere') or request_args.get('anywhere')) or False if remote_location: data_filters['location_names'].append('anywhere') for rl in r_locations: if isinstance(rl, int) and rl > 0: f_locations.append(rl) elif rl == 'anywhere': remote_location = True elif rl.isdigit(): f_locations.append(int(rl)) elif rl: ld = location_geodata(rl) if ld: f_locations.append(ld['geonameid']) remote_location_query = basequery.filter(JobPost.remote_location == True) # NOQA if f_locations: locations_query = basequery.join(JobLocation).filter(JobLocation.geonameid.in_(f_locations)) else: locations_query = basequery.join(JobLocation) if f_locations and remote_location: data_filters['locations'] = f_locations data_filters['anywhere'] = True recency = JobPost.state.LISTED basequery = locations_query.filter(recency).union(remote_location_query.filter(recency)) elif f_locations: data_filters['locations'] = f_locations basequery = locations_query elif remote_location: data_filters['anywhere'] = True # Only works as a positive filter: you can't search for jobs that are NOT anywhere basequery = remote_location_query currency = filters.get('currency') or request_args.get('currency') if currency in CURRENCY.keys(): data_filters['currency'] = currency basequery = basequery.filter(JobPost.pay_currency == currency) pay_graph = currency else: pay_graph = False if getbool(filters.get('equity') or request_args.get('equity')): # Only works as a positive filter: you can't search for jobs that DON'T pay in equity data_filters['equity'] = True basequery = basequery.filter(JobPost.pay_equity_min != None) # NOQA if filters.get('pay') or 'pay' in request_args or ('pmin' in request_args and 'pmax' in request_args): if 'pay' in request_args or filters.get('pay'): f_pay = filters['pay'] if filters.get('pay') else string_to_number(request_args['pay']) if f_pay is not None: f_min = int(f_pay * 0.90) f_max = int(f_pay * 1.30) else: f_min = None f_max = None else: # Legacy URL with min/max values f_min = string_to_number(request_args['pmin']) f_max = string_to_number(request_args['pmax']) f_pay = f_min # Use min for pay now if f_pay is not None and f_min is not None and f_max is not None: data_filters['pay'] = f_pay basequery = basequery.filter(JobPost.pay_cash_min < f_max, JobPost.pay_cash_max >= f_min) else: f_pay = None f_min = None f_max = None if getbool(request_args.get('archive')): ageless = True data_filters['archive'] = True statusfilter = JobPost.state.ARCHIVED if query_string: data_filters['query'] = search_query data_filters['query_string'] = query_string basequery = basequery.filter(JobPost.search_vector.match(search_query, postgresql_regconfig='english')) if data_filters: showall = True batched = True posts = getposts(basequery, pinned=True, showall=showall, statusfilter=statusfilter, ageless=ageless).all() if getbool(request_args.get('embed')): embed = True if posts: limit = string_to_number(request_args.get('limit')) if limit is not None: posts = posts[:limit] else: posts = posts[:8] else: embed = False if posts: employer_name = posts[0].company_name else: employer_name = u'a single employer' jobpost_ab = session_jobpost_ab() if is_index and posts and not gkiosk and not embed: # Group posts by email_domain on index page only, when not in kiosk mode grouped = OrderedDict() for post in posts: pinned = post.pinned if board is not None: blink = board_jobs.get(post.id) # board_jobs only contains the last 30 days, no archive if blink is not None: pinned = blink.pinned if pinned: # Make pinned posts appear in a group of one grouped.setdefault(('s', post.hashid), []).append( (pinned, post, bgroup(jobpost_ab, post))) elif post.state.ANNOUNCEMENT: # Make announcements also appear in a group of one grouped.setdefault(('a', post.hashid), []).append( (pinned, post, bgroup(jobpost_ab, post))) elif post.domain.is_webmail: grouped.setdefault(('ne', post.md5sum), []).append( (pinned, post, bgroup(jobpost_ab, post))) else: grouped.setdefault(('nd', post.email_domain), []).append( (pinned, post, bgroup(jobpost_ab, post))) pinsandposts = None else: grouped = None if board: pinsandposts = [] for post in posts: pinned = post.pinned if board is not None: blink = board_jobs.get(post.id) # board_jobs only contains the last 30 days, no archive if blink is not None: pinned = blink.pinned pinsandposts.append((pinned, post, bgroup(jobpost_ab, post))) else: pinsandposts = [(post.pinned, post, bgroup(jobpost_ab, post)) for post in posts] # Pick a header campaign (only if not kiosk or an XHR reload) pay_graph_data = None loadmore = False if batched: # Figure out where the batch should start from startdate = None if 'startdate' in request_values: try: startdate = parse_isoformat(request_values['startdate']) except TypeError: abort(400) except ValueError: pass batchsize = 32 # list of posts that were pinned at the time of first load pinned_hashids = request_args.getlist('ph') # Depending on the display mechanism (grouped or ungrouped), extract the batch if grouped: if not startdate: startindex = 0 for row in grouped.values(): # break when a non-pinned post is encountered if (not row[0][0]): break else: pinned_hashids.append(row[0][1].hashid) else: # Loop through group looking for start of next batch. See below to understand the # nesting structure of 'grouped' for startindex, row in enumerate(grouped.values()): # Skip pinned posts when looking for starting index if (row[0][1].hashid not in pinned_hashids and row[0][1].datetime < startdate): break batch = grouped.items()[startindex:startindex + batchsize] if startindex + batchsize < len(grouped): # Get the datetime of the last group's first item # batch = [((type, domain), [(pinned, post, bgroup), ...])] # batch[-1] = ((type, domain), [(pinned, post, bgroup), ...]) # batch[-1][1] = [(pinned, post, bgroup), ...] # batch[-1][1][0] = (pinned, post, bgroup) # batch[-1][1][0][1] = post loadmore = batch[-1][1][0][1].datetime grouped = OrderedDict(batch) elif pinsandposts: if not startdate: startindex = 0 for row in pinsandposts: # break when a non-pinned post is encountered if not row[0]: break else: pinned_hashids.append(row[1].hashid) else: for startindex, row in enumerate(pinsandposts): # Skip pinned posts when looking for starting index if (row[1].hashid not in pinned_hashids and row[1].datetime < startdate): break batch = pinsandposts[startindex:startindex + batchsize] if startindex + batchsize < len(pinsandposts): # batch = [(pinned, post), ...] loadmore = batch[-1][1].datetime pinsandposts = batch query_params = request_args.to_dict(flat=False) if loadmore: query_params.update({'startdate': loadmore.isoformat() + 'Z', 'ph': pinned_hashids}) if location: data_filters['location_names'].append(location['name']) query_params.update({'l': location['name']}) if pay_graph: pay_graph_data = make_pay_graph(pay_graph, posts, rmin=f_min, rmax=f_max) return dict(posts=posts, pinsandposts=pinsandposts, grouped=grouped, newlimit=newlimit, title=title, md5sum=md5sum, domain=domain, location=location, employer_name=employer_name, showall=showall, f_locations=f_locations, loadmore=loadmore, query_params=query_params, data_filters=data_filters, pay_graph_data=pay_graph_data, paginated=index_is_paginated(), template_vars=template_vars, embed=embed)
def fetch_jobposts(request_args, request_values, filters, is_index, board, board_jobs, gkiosk, basequery, md5sum, domain, location, title, showall, statusfilter, batched, ageless, template_vars, search_query=None, query_string=None): if basequery is None: basequery = JobPost.query # Apply request.args filters data_filters = {} f_types = filters.get('t') or request_args.getlist('t') while '' in f_types: f_types.remove('') if f_types: data_filters['types'] = f_types basequery = basequery.join(JobType).filter(JobType.name.in_(f_types)) f_categories = filters.get('c') or request_args.getlist('c') while '' in f_categories: f_categories.remove('') if f_categories: data_filters['categories'] = f_categories basequery = basequery.join(JobCategory).filter(JobCategory.name.in_(f_categories)) f_domains = filters.get('d') or request_args.getlist('d') while '' in f_domains: f_domains.remove('') if f_domains: basequery = basequery.join(Domain).filter(Domain.name.in_(f_domains)) f_tags = filters.get('k') or request_args.getlist('k') while '' in f_tags: f_tags.remove('') if f_tags: basequery = basequery.join(JobPostTag).join(Tag).filter(Tag.name.in_(f_tags)) data_filters['location_names'] = r_locations = filters.get('l') or request_args.getlist('l') if location: r_locations.append(location['geonameid']) f_locations = [] remote_location = getbool(filters.get('anywhere') or request_args.get('anywhere')) or False if remote_location: data_filters['location_names'].append('anywhere') for rl in r_locations: if isinstance(rl, int) and rl > 0: f_locations.append(rl) elif rl == 'anywhere': remote_location = True elif rl.isdigit(): f_locations.append(int(rl)) elif rl: ld = location_geodata(rl) if ld: f_locations.append(ld['geonameid']) remote_location_query = basequery.filter(JobPost.remote_location == True) # NOQA if f_locations: locations_query = basequery.join(JobLocation).filter(JobLocation.geonameid.in_(f_locations)) else: locations_query = basequery.join(JobLocation) if f_locations and remote_location: data_filters['locations'] = f_locations data_filters['anywhere'] = True recency = JobPost.state.LISTED basequery = locations_query.filter(recency).union(remote_location_query.filter(recency)) elif f_locations: data_filters['locations'] = f_locations basequery = locations_query elif remote_location: data_filters['anywhere'] = True # Only works as a positive filter: you can't search for jobs that are NOT anywhere basequery = remote_location_query currency = filters.get('currency') or request_args.get('currency') if currency in CURRENCY.keys(): data_filters['currency'] = currency basequery = basequery.filter(JobPost.pay_currency == currency) pay_graph = currency else: pay_graph = False if getbool(filters.get('equity') or request_args.get('equity')): # Only works as a positive filter: you can't search for jobs that DON'T pay in equity data_filters['equity'] = True basequery = basequery.filter(JobPost.pay_equity_min != None) # NOQA if filters.get('pay') or 'pay' in request_args or ('pmin' in request_args and 'pmax' in request_args): if 'pay' in request_args or filters.get('pay'): f_pay = filters['pay'] if filters.get('pay') else string_to_number(request_args['pay']) if f_pay is not None: f_min = int(f_pay * 0.90) f_max = int(f_pay * 1.30) else: f_min = None f_max = None else: # Legacy URL with min/max values f_min = string_to_number(request_args['pmin']) f_max = string_to_number(request_args['pmax']) f_pay = f_min # Use min for pay now if f_pay is not None and f_min is not None and f_max is not None: data_filters['pay'] = f_pay basequery = basequery.filter(JobPost.pay_cash_min < f_max, JobPost.pay_cash_max >= f_min) else: f_pay = None f_min = None f_max = None if getbool(request_args.get('archive')): ageless = True data_filters['archive'] = True statusfilter = JobPost.state.ARCHIVED if query_string: data_filters['query'] = search_query data_filters['query_string'] = query_string basequery = basequery.filter(JobPost.search_vector.match(search_query, postgresql_regconfig='english')) if data_filters: showall = True batched = True posts = getposts(basequery, pinned=True, showall=showall, statusfilter=statusfilter, ageless=ageless).all() if posts: employer_name = posts[0].company_name else: employer_name = u'a single employer' jobpost_ab = session_jobpost_ab() if is_index and posts and not gkiosk: # Group posts by email_domain on index page only, when not in kiosk mode grouped = OrderedDict() for post in posts: pinned = post.pinned if board is not None: blink = board_jobs.get(post.id) # board_jobs only contains the last 30 days, no archive if blink is not None: pinned = blink.pinned if pinned: # Make pinned posts appear in a group of one grouped.setdefault(('s', post.hashid), []).append( (pinned, post, bgroup(jobpost_ab, post))) elif post.state.ANNOUNCEMENT: # Make announcements also appear in a group of one grouped.setdefault(('a', post.hashid), []).append( (pinned, post, bgroup(jobpost_ab, post))) elif post.domain.is_webmail: grouped.setdefault(('ne', post.md5sum), []).append( (pinned, post, bgroup(jobpost_ab, post))) else: grouped.setdefault(('nd', post.email_domain), []).append( (pinned, post, bgroup(jobpost_ab, post))) pinsandposts = None else: grouped = None if board: pinsandposts = [] for post in posts: pinned = post.pinned if board is not None: blink = board_jobs.get(post.id) # board_jobs only contains the last 30 days, no archive if blink is not None: pinned = blink.pinned pinsandposts.append((pinned, post, bgroup(jobpost_ab, post))) else: pinsandposts = [(post.pinned, post, bgroup(jobpost_ab, post)) for post in posts] # Pick a header campaign (only if not kiosk or an XHR reload) pay_graph_data = None loadmore = False if batched: # Figure out where the batch should start from startdate = None if 'startdate' in request_values: try: startdate = parse_isoformat(request_values['startdate']) except ValueError: pass batchsize = 32 # list of posts that were pinned at the time of first load pinned_hashids = request_args.getlist('ph') # Depending on the display mechanism (grouped or ungrouped), extract the batch if grouped: if not startdate: startindex = 0 for row in grouped.values(): # break when a non-pinned post is encountered if (not row[0][0]): break else: pinned_hashids.append(row[0][1].hashid) else: # Loop through group looking for start of next batch. See below to understand the # nesting structure of 'grouped' for startindex, row in enumerate(grouped.values()): # Skip pinned posts when looking for starting index if (row[0][1].hashid not in pinned_hashids and row[0][1].datetime < startdate): break batch = grouped.items()[startindex:startindex + batchsize] if startindex + batchsize < len(grouped): # Get the datetime of the last group's first item # batch = [((type, domain), [(pinned, post, bgroup), ...])] # batch[-1] = ((type, domain), [(pinned, post, bgroup), ...]) # batch[-1][1] = [(pinned, post, bgroup), ...] # batch[-1][1][0] = (pinned, post, bgroup) # batch[-1][1][0][1] = post loadmore = batch[-1][1][0][1].datetime grouped = OrderedDict(batch) elif pinsandposts: if not startdate: startindex = 0 for row in pinsandposts: # break when a non-pinned post is encountered if not row[0]: break else: pinned_hashids.append(row[1].hashid) else: for startindex, row in enumerate(pinsandposts): # Skip pinned posts when looking for starting index if (row[1].hashid not in pinned_hashids and row[1].datetime < startdate): break batch = pinsandposts[startindex:startindex + batchsize] if startindex + batchsize < len(pinsandposts): # batch = [(pinned, post), ...] loadmore = batch[-1][1].datetime pinsandposts = batch query_params = request_args.to_dict(flat=False) if loadmore: query_params.update({'startdate': loadmore.isoformat() + 'Z', 'ph': pinned_hashids}) if location: data_filters['location_names'].append(location['name']) query_params.update({'l': location['name']}) if pay_graph: pay_graph_data = make_pay_graph(pay_graph, posts, rmin=f_min, rmax=f_max) return dict(posts=posts, pinsandposts=pinsandposts, grouped=grouped, newlimit=newlimit, title=title, md5sum=md5sum, domain=domain, location=location, employer_name=employer_name, showall=showall, f_locations=f_locations, loadmore=loadmore, query_params=query_params, data_filters=data_filters, pay_graph_data=pay_graph_data, paginated=index_is_paginated(), template_vars=template_vars)
def index(basequery=None, type=None, category=None, md5sum=None, domain=None, location=None, title=None, showall=True, statuses=None, tag=None, batched=True, ageless=False): if basequery is None: is_index = True else: is_index = False now = datetime.utcnow() if basequery is None and not (g.user or g.kiosk or (g.board and not g.board.require_login)): showall = False batched = False if basequery is None: basequery = JobPost.query # Apply request.args filters data_filters = {} f_types = request.args.getlist('t') while '' in f_types: f_types.remove('') if f_types: data_filters['types'] = f_types basequery = basequery.join(JobType).filter(JobType.name.in_(f_types)) f_categories = request.args.getlist('c') while '' in f_categories: f_categories.remove('') if f_categories: data_filters['categories'] = f_categories basequery = basequery.join(JobCategory).filter(JobCategory.name.in_(f_categories)) r_locations = request.args.getlist('l') f_locations = [] remote_location = getbool(request.args.get('anywhere')) or False for rl in r_locations: if rl == 'anywhere': remote_location = True elif rl.isdigit(): f_locations.append(int(rl)) elif rl: ld = location_geodata(rl) if ld: f_locations.append(ld['geonameid']) remote_location_query = basequery.filter(JobPost.remote_location == True) # NOQA locations_query = basequery.join(JobLocation).filter(JobLocation.geonameid.in_(f_locations)) if f_locations and remote_location: data_filters['locations'] = f_locations data_filters['anywhere'] = True recency = JobPost.datetime > datetime.utcnow() - agelimit basequery = locations_query.filter(recency).union(remote_location_query.filter(recency)) elif f_locations: data_filters['locations'] = f_locations basequery = locations_query elif remote_location: data_filters['anywhere'] = True # Only works as a positive filter: you can't search for jobs that are NOT anywhere basequery = remote_location_query if 'currency' in request.args and request.args['currency'] in CURRENCY.keys(): currency = request.args['currency'] data_filters['currency'] = currency basequery = basequery.filter(JobPost.pay_currency == currency) pay_graph = currency else: pay_graph = False if getbool(request.args.get('equity')): # Only works as a positive filter: you can't search for jobs that DON'T pay in equity data_filters['equity'] = True basequery = basequery.filter(JobPost.pay_equity_min != None) # NOQA if 'pmin' in request.args and 'pmax' in request.args: f_min = string_to_number(request.args['pmin']) f_max = string_to_number(request.args['pmax']) if f_min is not None and f_max is not None: data_filters['pay_min'] = f_min data_filters['pay_max'] = f_max basequery = basequery.filter(JobPost.pay_cash_min < f_max, JobPost.pay_cash_max >= f_min) else: f_min = None f_max = None if getbool(request.args.get('archive')): ageless = True data_filters['archive'] = True statuses = POSTSTATUS.ARCHIVED search_domains = None if request.args.get('q'): q = for_tsquery(request.args['q']) try: # TODO: Can we do syntax validation without a database roundtrip? db.session.query(db.func.to_tsquery(q)).all() except ProgrammingError: db.session.rollback() g.event_data['search_syntax_error'] = (request.args['q'], q) if not request.is_xhr: flash(_(u"Search terms ignored because this didn’t parse: {query}").format(query=q), 'danger') else: # Query's good? Use it. data_filters['query'] = q search_domains = Domain.query.filter( Domain.search_vector.match(q, postgresql_regconfig='english'), Domain.is_banned == False).options( db.load_only('name', 'title', 'logo_url')).all() # NOQA basequery = basequery.filter(JobPost.search_vector.match(q, postgresql_regconfig='english')) if data_filters: g.event_data['filters'] = data_filters showall = True batched = True # getposts sets g.board_jobs, used below posts = getposts(basequery, pinned=True, showall=showall, statuses=statuses, ageless=ageless).all() # Cache viewcounts (admin view or not) cache_viewcounts(posts) if posts: employer_name = posts[0].company_name else: employer_name = u'a single employer' if g.user: g.starred_ids = set(g.user.starred_job_ids(agelimit)) else: g.starred_ids = set() jobpost_ab = session_jobpost_ab() # Make lookup slightly faster in the loop below since 'g' is a proxy board = g.board if board: board_jobs = g.board_jobs else: board_jobs = {} if is_index and posts and not g.kiosk: # Group posts by email_domain on index page only, when not in kiosk mode grouped = OrderedDict() for post in posts: pinned = post.pinned if board is not None: blink = board_jobs.get(post.id) # board_jobs only contains the last 30 days, no archive if blink is not None: pinned = blink.pinned if pinned: # Make pinned posts appear in a group of one grouped.setdefault(('s', post.hashid), []).append( (pinned, post, bgroup(jobpost_ab, post))) elif post.status == POSTSTATUS.ANNOUNCEMENT: # Make announcements also appear in a group of one grouped.setdefault(('a', post.hashid), []).append( (pinned, post, bgroup(jobpost_ab, post))) elif post.domain.is_webmail: grouped.setdefault(('ne', post.md5sum), []).append( (pinned, post, bgroup(jobpost_ab, post))) else: grouped.setdefault(('nd', post.email_domain), []).append( (pinned, post, bgroup(jobpost_ab, post))) pinsandposts = None else: grouped = None if g.board: pinsandposts = [] for post in posts: pinned = post.pinned if board is not None: blink = board_jobs.get(post.id) # board_jobs only contains the last 30 days, no archive if blink is not None: pinned = blink.pinned pinsandposts.append((pinned, post, bgroup(jobpost_ab, post))) else: pinsandposts = [(post.pinned, post, bgroup(jobpost_ab, post)) for post in posts] # Pick a header campaign (only if not kiosk or an XHR reload) pay_graph_data = None if not g.kiosk and not request.is_xhr: if g.preview_campaign: header_campaign = g.preview_campaign else: if location: geonameids = g.user_geonameids + [location['geonameid']] else: geonameids = g.user_geonameids header_campaign = Campaign.for_context(CAMPAIGN_POSITION.HEADER, board=g.board, user=g.user, anon_user=g.anon_user, geonameids=geonameids) if pay_graph: pay_graph_data = make_pay_graph(pay_graph, posts, rmin=f_min, rmax=f_max) else: header_campaign = None loadmore = False if batched: # Figure out where the batch should start from startdate = None if 'startdate' in request.values: try: startdate = parse_isoformat(request.values['startdate']) except ValueError: pass if request.method == 'GET': batchsize = 31 # Skipping one for the special stickie that's on all pages else: batchsize = 32 # Depending on the display mechanism (grouped or ungrouped), extract the batch if grouped: if not startdate: startindex = 0 else: # Loop through group looking for start of next batch. See below to understand the # nesting structure of 'grouped' for startindex, row in enumerate(grouped.values()): # Skip examination of pinned listings (having row[0][0] = True) if (not row[0][0]) and row[0][1].datetime < startdate: break batch = grouped.items()[startindex:startindex + batchsize] if startindex + batchsize < len(grouped): # Get the datetime of the last group's first item # batch = [((type, domain), [(pinned, post, bgroup), ...])] # batch[-1] = ((type, domain), [(pinned, post, bgroup), ...]) # batch[-1][1] = [(pinned, post, bgroup), ...] # batch[-1][1][0] = (pinned, post, bgroup) # batch[-1][1][0][1] = post loadmore = batch[-1][1][0][1].datetime grouped = OrderedDict(batch) elif pinsandposts: if not startdate: startindex = 0 else: for startindex, row in enumerate(pinsandposts): # Skip pinned posts when looking for starting index if (not row[0]) and row[1].datetime < startdate: break batch = pinsandposts[startindex:startindex + batchsize] if startindex + batchsize < len(pinsandposts): # batch = [(pinned, post), ...] loadmore = batch[-1][1].datetime pinsandposts = batch if grouped: g.impressions = {post.id: (pinflag, post.id, is_bgroup) for group in grouped.itervalues() for pinflag, post, is_bgroup in group} elif pinsandposts: g.impressions = {post.id: (pinflag, post.id, is_bgroup) for pinflag, post, is_bgroup in pinsandposts} return render_template('index.html', pinsandposts=pinsandposts, grouped=grouped, now=now, newlimit=newlimit, jobtype=type, jobcategory=category, title=title, md5sum=md5sum, domain=domain, employer_name=employer_name, location=location, showall=showall, tag=tag, is_index=is_index, header_campaign=header_campaign, loadmore=loadmore, search_domains=search_domains, is_siteadmin=lastuser.has_permission('siteadmin'), pay_graph_data=pay_graph_data)
def test_parse_isoformat(self): assert parse_isoformat("1882-12-11T00:00:00.1234Z") == datetime.datetime(1882, 12, 11, 0, 0, 0, 123400) assert parse_isoformat("1882-12-11T00:00:00Z"), datetime.datetime(1882, 12, 11, 0, 0) assert parse_isoformat("1882-12-11T00:00:00.1234Z", naive=False) == datetime.datetime(1882, 12, 11, 0, 0, 0, 123400, tzinfo=UTC) assert parse_isoformat("1882-12-11T00:00:00Z", naive=False), datetime.datetime(1882, 12, 11, 0, 0, tzinfo=UTC)
def test_parse_isoformat(self): self.assertEqual(parse_isoformat("1882-12-11T00:00:00.1234Z"), datetime.datetime(1882, 12, 11, 0, 0, 0, 123400)) self.assertEqual(parse_isoformat("1882-12-11T00:00:00Z"), datetime.datetime(1882, 12, 11, 0, 0))
def test_parse_isoformat(self): self.assertEqual(parse_isoformat("1882-12-11T00:00:00.1234Z"), datetime.datetime(1882, 12, 11, 0, 0, 0, 123400)) self.assertEqual(parse_isoformat("1882-12-11T00:00:00Z"), datetime.datetime(1882, 12, 11, 0, 0))
def _sync(self): if self.node.source != 'doattend': yield "Unsupported data source, aborting.\n" return if not self.node.sourceid or not self.node.api_key: yield "Source event id and API key are required.\n" return # All good, start pulling data... data_url = 'http://doattend.com/api/events/%s/participants_list.json?api_key=%s' % ( self.node.sourceid, self.node.api_key) yield "Receiving data from DoAttend..." r = requests.get(data_url) data = r.json() if callable(r.json) else r.json yield " OK\n" yield "Participant count: %d\n" % len(data['participants']) yield "Previously synced count: %d\n\n" % len(self.node.participants) by_ticket = {} local_tickets = set() upstream_tickets = set() unindexed = [] for participant in self.node.participants: if participant.ticket is not None: by_ticket[participant.ticket] = participant local_tickets.add(participant.ticket) else: unindexed.append(participant) plist = data['participants'] plist.reverse() # DoAttend list is sorted by most-recent first for p in plist: upstream_tickets.add(p['Ticket_Number']) participant = by_ticket.get(p['Ticket_Number']) if participant is None: participant = Participant(participant_list=self.node) db.session.add(participant) participant.ticket = p['Ticket_Number'].strip() by_ticket[participant.ticket] = participant local_tickets.add(participant.ticket) syncinfo = { 'datetime': parse_isoformat(p['Date']), 'fullname': p['Name'].strip() if isinstance(p['Name'], basestring) else p['Name'], 'email': p['Email'].strip() if isinstance(p['Email'], basestring) else p['Email'], 'ticket_type': p['Ticket_Name'].strip() if isinstance(p['Ticket_Name'], basestring) else p['Ticket_Name'], } pinfo = p.get('participant_information', []) if isinstance(pinfo, dict): pinfo = [pinfo] for keyval in pinfo: key = keyval['desc'] value = keyval.get('info') if key == 'Job Title': syncinfo['jobtitle'] = value.strip() if isinstance(value, basestring) else value elif key == 'Company': syncinfo['company'] = value.strip() if isinstance(value, basestring) else value elif key == 'Twitter Handle': syncinfo['twitter'] = value.strip() if isinstance(value, basestring) else value elif key == 'City': syncinfo['city'] = value.strip() if isinstance(value, basestring) else value elif key == 'T-shirt size': syncinfo['tshirt_size'] = value.split('-', 1)[0].strip() if isinstance(value, basestring) else value edited = False for key, value in syncinfo.items(): if getattr(participant, key) != value: setattr(participant, key, value) if 'key' == 'email': participant.user = None edited = True if edited: if participant.id is None: yield "New participant (#%s): %s\n" % (participant.ticket, participant.fullname) else: yield "Edited participant (#%s): %s\n" % (participant.ticket, participant.fullname) # Check for deleted participants removed_tickets = local_tickets - upstream_tickets for ticket in removed_tickets: participant = by_ticket.get(ticket) if participant: yield "Removed participant (#%s): %s\n" % (ticket, participant.fullname) db.session.delete(participant) db.session.commit() yield '\nAll done.'
def test_parse_isoformat(self): assert parse_isoformat( '1882-12-11T00:00:00.1234Z') == datetime.datetime( 1882, 12, 11, 0, 0, 0, 123400) assert parse_isoformat('1882-12-11T00:00:00Z'), datetime.datetime( 1882, 12, 11, 0, 0) assert parse_isoformat('1882-12-11T00:00:00.1234Z', naive=False) == datetime.datetime(1882, 12, 11, 0, 0, 0, 123400, tzinfo=UTC) assert parse_isoformat('1882-12-11T00:00:00Z', naive=False) == datetime.datetime(1882, 12, 11, 0, 0, tzinfo=UTC) assert parse_isoformat('1882-12-11T00:00:00-06:39', naive=False) == datetime.datetime( 1882, 12, 11, 0, 0, 0, tzinfo=TestTZ()) assert parse_isoformat('1882-12-11T00:00:00-06:39', naive=True) == datetime.datetime( 1882, 12, 11, 6, 39, 0) assert parse_isoformat('1882-12-11T00:00:00', naive=True) == datetime.datetime( 1882, 12, 11, 0, 0, 0) with pytest.raises(ValueError): # lacking the T delimiter assert parse_isoformat( '1882-12-11 00:00:00.1234Z') == datetime.datetime( 1882, 12, 11, 0, 0, 0, 123400) # will pass with delimiter assert parse_isoformat('1882-12-11 00:00:00.1234Z', delimiter=' ') == datetime.datetime( 1882, 12, 11, 0, 0, 0, 123400) assert parse_isoformat("2012-05-21 23:06:08", naive=False, delimiter=' ') == datetime.datetime( 2012, 5, 21, 23, 6, 8) with pytest.raises(ParseError): parse_isoformat('2019-05-03T05:02:26.340937Z\'') with pytest.raises(ParseError): parse_isoformat('2019-05-03T05:02:26.340937Z\'', naive=False)