def test_perform_indexing_api_request_failure_index(initialized_db, set_secscan_config): secscan = V4SecurityScanner(app, instance_keys, storage) secscan._secscan_api = mock.Mock() secscan._secscan_api.state.return_value = {"state": "abc"} secscan._secscan_api.index.side_effect = APIRequestFailure() next_token = secscan.perform_indexing() assert next_token is None assert ManifestSecurityStatus.select().count() == 0 # Set security scanner to return good results and attempt indexing again secscan._secscan_api.index.side_effect = None secscan._secscan_api.index.return_value = ( { "err": None, "state": IndexReportState.Index_Finished }, "abc", ) next_token = secscan.perform_indexing() assert next_token.min_id == Manifest.select(fn.Max( Manifest.id)).scalar() + 1 assert ManifestSecurityStatus.select().count() == Manifest.select( fn.Max(Manifest.id)).count()
def get_transaction_list_details(transaction_list): """Return complete data set on a specified transaction.""" query = (Files().select( Files.transaction.alias('upload_id'), fn.Max(Transactions.updated).alias('upload_date'), fn.Min(Files.mtime).alias('file_date_start'), fn.Max(Files.mtime).alias('file_date_end'), fn.Min(Transactions.submitter).alias('uploaded_by_id'), fn.Sum(Files.size).alias('bundle_size'), fn.Count(Files.id).alias('file_count'), fn.Min(Transactions.updated).alias('upload_datetime'), fn.Min(Transactions.proposal).alias('proposal_id'), fn.Min(Transactions.instrument).alias('instrument_id')).join( Transactions).where( Files.transaction << transaction_list).group_by( Files.transaction)) return { str(r['upload_id']): { 'upload_id': str(r['upload_id']), 'upload_date': r['upload_date'].date().strftime('%Y-%m-%d'), 'file_date_start': r['file_date_start'].date().strftime('%Y-%m-%d'), 'file_date_end': r['file_date_end'].date().strftime('%Y-%m-%d'), 'uploaded_by_id': int(r['uploaded_by_id']), 'bundle_size': int(r['bundle_size']), 'file_count': int(r['file_count']), 'upload_datetime': r['upload_date'].strftime('%Y-%m-%d %H:%M:%S'), 'proposal_id': r['proposal_id'], 'instrument_id': r['instrument_id'] } for r in query.dicts() }
def _get_earliest_latest(item_type, item_list, time_basis): accepted_item_types = list( set( list(QueryBase.object_type_mappings.keys()) + list(QueryBase.object_type_mappings.values()))) accepted_time_basis_types = [ 'submitted', 'modified', 'created', 'submit', 'modified', 'create', 'submit_time', 'modified_time', 'create_time', 'submitted_date', 'modified_date', 'created_date', ] item_type = QueryBase.object_type_mappings.get(item_type) time_basis = time_basis.lower() if item_type not in accepted_item_types or time_basis not in accepted_time_basis_types: raise HTTPError('400 Invalid Query') short_time_basis = time_basis[:5] time_basis = { 'submi': lambda x: 'submitted', 'modif': lambda x: 'modified', 'creat': lambda x: 'created' }[short_time_basis](short_time_basis) search_field = getattr(TransSIP, '{0}'.format(item_type)) if time_basis == 'submitted': query = TransSIP().select( fn.Min(TransSIP.updated).alias('earliest'), fn.Max(TransSIP.updated).alias('latest'), ) if time_basis in ['modified', 'created']: time_basis_field = getattr(Files, '{0}time'.format(time_basis[:1])) query = Files().select( fn.Min(time_basis_field).alias('earliest'), fn.Max(time_basis_field).alias('latest'), ).join(TransSIP, on=(TransSIP.id == Files.transaction)) query = query.where(search_field << item_list) row = query.get() if row.earliest is None or row.latest is None: message = '' raise HTTPError('404 Not Found', message) return { 'earliest': row.earliest.strftime('%Y-%m-%d %H:%M:%S'), 'latest': row.latest.strftime('%Y-%m-%d %H:%M:%S') }
def test_perform_indexing_api_request_index_error_response(initialized_db, set_secscan_config): secscan = V4SecurityScanner(app, instance_keys, storage) secscan._secscan_api = mock.Mock() secscan._secscan_api.state.return_value = {"state": "xyz"} secscan._secscan_api.index.return_value = ( {"err": "something", "state": IndexReportState.Index_Error}, "xyz", ) next_token = secscan.perform_indexing() assert next_token.min_id == Manifest.select(fn.Max(Manifest.id)).scalar() + 1 assert ManifestSecurityStatus.select().count() == Manifest.select(fn.Max(Manifest.id)).count() for mss in ManifestSecurityStatus.select(): assert mss.index_status == IndexStatus.FAILED
def get_transaction_list_details(transaction_list): """Return complete data set on a specified transaction.""" # pylint: disable=no-member query = (Files().select( Files.transaction.alias('upload_id'), fn.Max(TransSIP.updated).alias('upload_date'), fn.Min(Files.mtime).alias('file_date_start'), fn.Max(Files.mtime).alias('file_date_end'), fn.Min(TransSIP.submitter).alias('uploaded_by_id'), fn.Sum(Files.size).alias('bundle_size'), fn.Count(Files.id).alias('file_count'), fn.Min(TransSIP.updated).alias('upload_datetime'), fn.Min(TransSIP.project).alias('project_id'), fn.Min(TransSIP.instrument).alias('instrument_id')).join( TransSIP, on=(TransSIP.id == Files.transaction )).where(Files.transaction << transaction_list).group_by( Files.transaction)) # pylint: enable=no-member return { str(r['upload_id']): { 'upload_id': str(r['upload_id']), 'upload_date': r['upload_date'].date().strftime('%Y-%m-%d'), 'file_date_start': SummarizeByDate.utc_to_local( r['file_date_start']).date().strftime('%Y-%m-%d'), 'file_date_end': SummarizeByDate.utc_to_local( r['file_date_end']).date().strftime('%Y-%m-%d'), 'uploaded_by_id': int(r['uploaded_by_id']), 'bundle_size': int(r['bundle_size']), 'file_count': int(r['file_count']), 'upload_datetime': SummarizeByDate.utc_to_local( r['upload_date']).strftime('%Y-%m-%d %H:%M:%S'), 'project_id': r['project_id'], 'instrument_id': r['instrument_id'] } for r in query.dicts() }
def get_last1h_devices(self): if scanner.last_scan is None: self.chat.reply("⚠️ Scanner is not started yet") now = datetime.now() results = ScanResult\ .select()\ .where(ScanResult.time > now - timedelta(hours=1))\ .join(Device, JOIN.LEFT_OUTER)\ .join(Person, JOIN.LEFT_OUTER)\ .group_by(ScanResult.mac_addr)\ .having(fn.Max(ScanResult.time) == ScanResult.time)\ .order_by(-ScanResult.time, NodeList((Person.name, SQL('IS NULL'))), Person.name) msg_text = "Active in the last hour devices list\nAs of %s\n" % now.strftime( "%Y.%m.%d %X") for k, g in groupby(results, lambda x: x.time): age = int((now - k).seconds / 60) msg_text += "\n<b>%s min ago</b>\n" % str( age) if age > 0 else "\n<b>Now</b>\n" for r in g: if r.device: d = r.device msg_text += "• %s (%s) \n" % (d.owner.name if d.owner else "N/A", d.name or "N/A") else: msg_text += "• <code>%s</code>\n" % r.mac_addr self.chat.reply( msg_text, parse_mode='HTML', )
def get_max_trend_date_by_category(category: list = None): result = PTrends.select(fn.Max(PTrends.valid_date)).join(PTorrent) if category: if not isinstance(category, list): category = [category] result = result.where((PTorrent.category.in_(category))) return result.scalar()
def _get_last_known_transaction(): txn_id = (TransSIP .select(fn.Max(TransSIP.id).alias('id')) .where(TransSIP.deleted >> None) .dicts() .get()) return {'latest_transaction_id': txn_id['id']}
def get_last_scheduled_datetime(cls) -> datetime: last_scheduled_post = Story.select(fn.Max(Story.scheduled_datetime)).scalar() if last_scheduled_post: return last_scheduled_post else: return datetime.now()
def test_perform_indexing_api_request_failure_index(initialized_db): app.config["SECURITY_SCANNER_V4_NAMESPACE_WHITELIST"] = ["devtable"] expected_manifests = ( Manifest.select(fn.Max(Manifest.id)) .join(Repository) .join(User) .where(User.username == "devtable") ) secscan = V4SecurityScanner(app, instance_keys, storage) secscan._secscan_api = mock.Mock() secscan._secscan_api.state.return_value = "abc" secscan._secscan_api.index.side_effect = APIRequestFailure() next_token = secscan.perform_indexing() assert next_token is None assert ManifestSecurityStatus.select().count() == 0 # Set security scanner to return good results and attempt indexing again secscan._secscan_api.index.side_effect = None secscan._secscan_api.index.return_value = ( {"err": None, "state": IndexReportState.Index_Finished}, "abc", ) next_token = secscan.perform_indexing() assert next_token.min_id == expected_manifests.scalar() + 1 assert ManifestSecurityStatus.select().count() == expected_manifests.count()
def perform_indexing_recent_manifests(self, batch_size=None): try: indexer_state = self._secscan_api.state() except APIRequestFailure: return None if not batch_size: batch_size = self.app.config.get("SECURITY_SCANNER_V4_BATCH_SIZE", 0) reindex_threshold = datetime.utcnow() - timedelta( seconds=self.app.config.get( "SECURITY_SCANNER_V4_REINDEX_THRESHOLD", 86400)) end_index = Manifest.select(fn.Max(Manifest.id)).scalar() start_index = max(end_index - batch_size, 1) iterator = self._get_manifest_iterator( indexer_state, start_index, end_index, batch_size=max(batch_size // 20, 1), reindex_threshold=reindex_threshold, ) self._index(iterator, reindex_threshold)
def test_manifest_iterator(initialized_db, set_secscan_config, index_status, indexer_state, seconds, expect_zero): secscan = V4SecurityScanner(app, instance_keys, storage) for manifest in Manifest.select(): with db_transaction(): ManifestSecurityStatus.delete().where( ManifestSecurityStatus.manifest == manifest, ManifestSecurityStatus.repository == manifest.repository, ).execute() ManifestSecurityStatus.create( manifest=manifest, repository=manifest.repository, error_json={}, index_status=index_status, indexer_hash="old hash", indexer_version=IndexerVersion.V4, last_indexed=datetime.utcnow() - timedelta(seconds=seconds), metadata_json={}, ) iterator = secscan._get_manifest_iterator( indexer_state, Manifest.select(fn.Min(Manifest.id)).scalar(), Manifest.select(fn.Max(Manifest.id)).scalar(), ) count = 0 for candidate, abt, num_remaining in iterator: count = count + 1 if expect_zero: assert count == 0 else: assert count != 0
def perform_indexing(self, start_token=None, batch_size=None): try: indexer_state = self._secscan_api.state() except APIRequestFailure: return None if not batch_size: batch_size = self.app.config.get("SECURITY_SCANNER_V4_BATCH_SIZE", 0) reindex_threshold = datetime.utcnow() - timedelta( seconds=self.app.config.get( "SECURITY_SCANNER_V4_REINDEX_THRESHOLD", 86400)) max_id = Manifest.select(fn.Max(Manifest.id)).scalar() start_index = (start_token.min_id if start_token is not None else Manifest.select(fn.Min(Manifest.id)).scalar()) if max_id is None or start_index is None or start_index > max_id: return None iterator = self._get_manifest_iterator( indexer_state, start_index, max_id, batch_size=batch_size, reindex_threshold=reindex_threshold, ) self._index(iterator, reindex_threshold) return ScanToken(max_id + 1)
def runner(): last_repo = PuppetFiles.select(fn.Max(PuppetFiles.repositoryName)).scalar() repositories = os.listdir(Constants.PUPPET_REPOSITORIES_PATH) for repository in repositories: if last_repo is not None and repository <= last_repo: print("Ignoring " + repository) else: print("Adding " + repository) entries = [] for path in glob.glob(Constants.PUPPET_REPOSITORIES_PATH + "/" + repository + "/**/*.pp", recursive=True): entries.append(( path, repository, )) print(path, repository) with db.atomic(): PuppetFiles.insert_many( entries, fields=[PuppetFiles.path, PuppetFiles.repositoryName]).execute() files = (PuppetFiles.select().where((PuppetFiles.isAnalyzed == False))) return files
def updateVideo(self): # fill in general part of Video Info self.task_name = self.tasks[self.video.task_id] self.video_name = self.video.file_name self.video_length = (model.database.Kinematic .select() .where(model.database.Kinematic.video_id == self.video.id) .select(fn.Max(model.database.Kinematic.frame)) .scalar() ) # physical location of the video file file_name = os.path.join( self.dir_config, 'tasks', self.task_name, 'video', self.video_name + '_capture2.avi') self.load(file_name) # update the Gestures tab self.gesture_store.clear() self.gesture_spans = list() for x in model.database.Transcript.select().where(model.database.Transcript.video_id == self.video.id).order_by(model.database.Transcript.start) : gesture_store_item = [x.gesture_id, self.gestures[x.gesture_id], x.start, x.end] self.gesture_store.append(gesture_store_item) self.gesture_spans.append(gesture_store_item) # update subject info self.label_subject.set_markup('\n'.join([ '<b>Subject Code</b>: ' + self.video.file_name.rpartition('_')[2][0], '<b>Trial</b>: ' + self.video.file_name[-1], '<b>Skill Level</b>: ' + model.database.SKILL_LEVELS_DICT[self.video.skill_level].replace('<', '<').replace('>', '>'), '<b>Global Rating Score</b>: {}/30 ({:.2f}%)'.format(self.video.grs_total, int(self.video.grs_total) / 0.30 ), ' <b>Respect for tissue</b>: ' + self.getGrs(self.video.grs_tissue, model.database.GRS_TISSUE), ' <b>Suture/needle handling</b>: ' + self.getGrs(self.video.grs_suture, model.database.GRS_SUTURE), ' <b>Time and motion</b>: ' + self.getGrs(self.video.grs_time, model.database.GRS_TIME), ' <b>Flow of operation</b>: ' + self.getGrs(self.video.grs_flow, model.database.GRS_FLOW), ' <b>Overall performance</b>: ' + self.getGrs(self.video.grs_performance, None), ' <b>Quality of final product</b>: ' + self.getGrs(self.video.grs_quality, None), ])) # download kinematics self.kinematics = dict() self.kinematics_range = dict() for x in model.database.Kinematic.select().where(model.database.Kinematic.video_id == self.video.id).order_by(model.database.Kinematic.frame) : self.kinematics[x.frame] = x colname_i = 0 for colname in model.kinematics.columns : self.kinematics_range[colname] = max( -min(getattr(self.kinematics[x],colname) for x in self.kinematics), max (getattr(self.kinematics[x],colname) for x in self.kinematics)) #colname_i += 1 #print(colname_i) self.updateGesturePlot(self.arm_type) self.onKspToggled()
def get_true_goals(): """ 获取每个小组净胜球最大的队伍 """ teams = Points.select(Points.group, Points.team, fn.Max(Points.true_goal)).where( Points.team == Points.team).group_by(Points.group) data = [g.get_dict() for g in teams] return json_data(data)
def get_high_and_average_scores(): high_score, average_score = (ScoreEntry .select(fn.Max(ScoreEntry.final_score), fn.Avg(ScoreEntry.final_score)) .scalar(as_tuple=True)) if (high_score, average_score) == (None, None): high_score, average_score = 0, 0 return high_score, average_score
def get_max_batch(self, source): max_id = webapp_job_batch.select( fn.Max(webapp_job_batch.id).alias('max')).where( webapp_job_batch.crawler_source == source) if (max_id[0].max is None): return 1 else: return max_id[0].max + 1
def max_id(cls): """ Get the max id on the table. Returns: int """ return cls.select(fn.Max(cls.id)).scalar()
def compute_location_ratings(labels=HAND_LABELED_EVENTS, task_compute_index=None): # Create a new index for this computation last_compute_index = LocationRating.select( fn.Max(LocationRating.compute_index)).scalar() or 0 compute_index = last_compute_index + 1 # Determine what will be the compute index of the task periods that ratings are matched to. # This will become the latest compute index if it hasn't been specified. if task_compute_index is None: task_compute_index = TaskPeriod.select(fn.Max( TaskPeriod.compute_index)).scalar() # Create a list to hold all ratings that couldn't be matched to a task period. # At the end, we want to return these, in case it's important for the caller to know # which events we couldn't create rating records for. unmatched_ratings = [] for event in LocationEvent.select(): # Check to see whether this is a rating event rating_match = re.match("^Rating: (\d)+$", event.event_type) if rating_match: # If this is a rating event, extract the rating rating = int(rating_match.group(1)) rating_created = create_location_rating( compute_index=compute_index, task_compute_index=task_compute_index, event=event, rating=rating, labels=labels, ) # If a rating wasn't created, this probably couldn't be matched to a task. # Save a record of which event failed to be matched to a task and which user # this event happened for. if not rating_created: unmatched_ratings.append({ 'user_id': event.user_id, 'event_id': event.id, }) return unmatched_ratings
def fetch_boxes(arxiv_id: ArxivId, schema: str, version: Optional[int], types: List[str]) -> Optional[RegionsByPageAndType]: # Discover the most recent version of data in the database for the paper. setup_database_connections(schema) if version is None: version_number = (Version.select(fn.Max( Version.index)).join(Paper).where( Paper.arxiv_id == arxiv_id).scalar()) if version_number is None: logging.warning( # pylint: disable=logging-not-lazy "There are no entities for paper %s in database schema %s", arxiv_id, schema, ) return None version = int(version_number) # Load bounding boxes from rows in the tables. rows = (EntityModel.select( EntityModel.id, EntityModel.type, BoundingBoxModel.left, BoundingBoxModel.top, BoundingBoxModel.width, BoundingBoxModel.height, BoundingBoxModel.page, ).join(Paper).switch(EntityModel).join(BoundingBoxModel).where( EntityModel.version == version, Paper.arxiv_id == arxiv_id, EntityModel.type << types, ).dicts()) boxes_by_entity_db_id: Dict[str, List[BoundingBox]] = defaultdict(list) types_by_entity_db_id: Dict[str, str] = {} for row in rows: boxes_by_entity_db_id[row["id"]].append( BoundingBox( row["left"], row["top"], row["width"], row["height"], row["page"], )) types_by_entity_db_id[row["id"]] = row["type"] regions: RegionsByPageAndType = defaultdict(list) for db_id, bounding_boxes in boxes_by_entity_db_id.items(): by_page = group_by_page(bounding_boxes) for page, page_boxes in by_page.items(): key = (page, types_by_entity_db_id[db_id]) rectangles = frozenset([ FloatRectangle(b.left, b.top, b.width, b.height) for b in page_boxes ]) regions[key].append(rectangles) return regions
def getNextOrderID(): id = Order.select(fn.Max(Order.id)).scalar() if id is None: id = 1 else: id += 1 return id
def _backfill_manifests(self): try: Manifest.select().where( Manifest.layers_compressed_size >> None).get() except Manifest.DoesNotExist: logger.debug("Manifest backfill worker has completed; skipping") return False iterator = yield_random_entries( lambda: Manifest.select().where(Manifest.layers_compressed_size >> None), Manifest.id, 250, Manifest.select(fn.Max(Manifest.id)).scalar(), 1, ) for manifest_row, abt, _ in iterator: if manifest_row.layers_compressed_size is not None: logger.debug("Another worker preempted this worker") abt.set() continue logger.debug("Setting layers compressed size for manifest %s", manifest_row.id) layers_compressed_size = -1 config_media_type = None manifest_bytes = Bytes.for_string_or_unicode( manifest_row.manifest_bytes) try: parsed = parse_manifest_from_bytes( manifest_bytes, manifest_row.media_type.name, validate=False) layers_compressed_size = parsed.layers_compressed_size if layers_compressed_size is None: layers_compressed_size = 0 config_media_type = parsed.config_media_type or None except ManifestException as me: logger.warning( "Got exception when trying to parse manifest %s: %s", manifest_row.id, me) assert layers_compressed_size is not None updated = (Manifest.update( layers_compressed_size=layers_compressed_size, config_media_type=config_media_type, ).where(Manifest.id == manifest_row.id, Manifest.layers_compressed_size >> None).execute()) if updated != 1: logger.debug("Another worker preempted this worker") abt.set() continue return True
async def ws_index(request, websocket): subscribe(websocket, "jobs") # avoid fetch "log" field from the db to reduce memory usage selected_fields = ( Job.id, Job.name, Job.url_or_path, Job.state, Job.created_time, Job.started_time, Job.end_time, ) JobAlias = Job.alias() subquery = JobAlias.select(*selected_fields)\ .where(JobAlias.state << ("done", "failure", "canceled", "error"))\ .group_by(JobAlias.url_or_path)\ .select(fn.Max(JobAlias.id).alias("max_id")) latest_done_jobs = Job.select(*selected_fields)\ .join(subquery, on=(Job.id == subquery.c.max_id))\ .order_by(-Job.id) subquery = JobAlias.select(*selected_fields)\ .where(JobAlias.state == "scheduled")\ .group_by(JobAlias.url_or_path)\ .select(fn.Min(JobAlias.id).alias("min_id")) next_scheduled_jobs = Job.select(*selected_fields)\ .join(subquery, on=(Job.id == subquery.c.min_id))\ .order_by(-Job.id) # chunks initial data by batch of 30 to avoid killing firefox data = chunks( itertools.chain( map(model_to_dict, next_scheduled_jobs.iterator()), map(model_to_dict, Job.select().where(Job.state == "running").iterator()), map(model_to_dict, latest_done_jobs.iterator())), 30) first_chunck = next(data) await websocket.send( ujson.dumps({ "action": "init_jobs", "data": first_chunck, # send first chunk })) for chunk in data: await websocket.send( ujson.dumps({ "action": "init_jobs_stream", "data": chunk, })) await websocket.wait_closed()
def last_change_date(cls): """Find the last changed date for the object.""" # pylint: disable=no-value-for-parameter last_change_date = cls.select(fn.Max(cls.updated)).scalar() # pylint: enable=no-value-for-parameter last_change_string = last_change_date \ if last_change_date is not None else '1970-01-01 00:00:00' last_change_string = last_change_date.isoformat(' ') \ if isinstance(last_change_date, datetime.datetime) else parser.parse(last_change_string).isoformat(' ') return text_type(last_change_string)
def get_last_refreshed_on_time(): """ Time of the most recently *refreshed* feed """ last_checked_on = Feed.select().aggregate(fn.Max(Feed.last_checked_on)) if last_checked_on: return datetime_as_epoch(last_checked_on) # Return a fallback value return datetime_as_epoch(datetime.utcnow())
def index(): max_timestamp = ImageInfo.select(fn.Max(ImageInfo.timestamp)).scalar() # truncate the datetime obj start_time = max_timestamp.replace(minute=0, hour=0, second=0, microsecond=0) end_time = start_time + timedelta(days=1) result_tuples = generate_bar_chart_tuples(start_time, end_time) return render_template('bar_chart.html', result_tuples=result_tuples)
def get_citation_count_for_queries(queries, api_key): # Create a new fetch index. last_fetch_index = Publication.select( fn.Max(Publication.fetch_index)).scalar() or 0 fetch_index = last_fetch_index + 1 for query in queries: # Fetch the citation count! get_citation_count(query, fetch_index, api_key)
def get_timepaths_for_dataset(dataset, limit=10): '''Identify all time paths in a dataset. Returns all of them if there are 10 or less. Returns min/max/count if more than 10.''' all_paths = DataSet.select().where(DataSet.name==dataset.name, DataSet.project==dataset.project, DataSet.metaarg_guid==dataset.metaarg_guid) total_records = all_paths.count() if total_records < limit: timepaths = list(all_paths.order_by(DataSet.timepath).select(DataSet.timepath).tuples()) timepaths = [t[0] for t in timepaths] return {'allpaths': timepaths} else: min_value, max_value = timepaths.select(fn.Min(DataSet.timepath), fn.Max(DataSet.timepath)).scalar(as_tuple=True) return {'cnt': total_records, 'min_value': min_value, 'max_value': max_value}
def main(tags, *args, **kwargs): # Create a new fetch index. last_fetch_index = QuestionSnapshot.select( fn.Max(QuestionSnapshot.fetch_index)).scalar() or 0 fetch_index = last_fetch_index + 1 with open(tags) as tag_file: tag_list = [t.strip() for t in tag_file.readlines()] for tag in tag_list: fetch_questions_for_tag(tag, fetch_index)