def runTestsAndPrintReport(trainingCategories, testingCategories): allPositiveTrainingReviews = list(); allNegativeTrainingReviews = list(); allPositiveTestingReviews = list(); allNegativeTestingReviews = list(); for category in trainingCategories: (allPositiveTrainingReviewsTmp, allNegativeTrainingReviewsTmp) = getAllClassifiedReviews(AmazonReviewCorpusReader(), category) allPositiveTrainingReviews += allPositiveTrainingReviewsTmp; allNegativeTrainingReviews += allNegativeTrainingReviewsTmp; for category in testingCategories: (allPositiveTestingReviewsTmp, allNegativeTestingReviewsTmp) = getAllClassifiedReviews(AmazonReviewCorpusReader(), category) allPositiveTestingReviews += allPositiveTestingReviewsTmp; allNegativeTestingReviews += allNegativeTestingReviewsTmp; datasets = [ClassifiedDataset(0.1, 'POS', allPositiveTrainingReviews, allPositiveTestingReviews), ClassifiedDataset(0.1, 'NEG', allNegativeTrainingReviews, allNegativeTestingReviews) ] print 'training: %s vs testing: %s' % (trainingCategories, testingCategories); report = Report(datasets, factories) reportStats = report.getReport(); Report.printReport(reportStats);
def update_ui(): if items[current].get('audit_radcat'): radcat_entry.set("RADCAT{}".format(items[current]['audit_radcat'])) fu_entry.set(items[current]['audit_radcat3'] == "Yes") unscored_btn.state(['!disabled']) else: radcat_entry.set('') fu_entry.set(False) item = items[current] r = Report(text=item['report_text']) # logging.debug(r.text) # extractions = r.extractions() # item['radcat'] = int(extractions.get('radcat')) # item['radcat3'] = "Yes" if extractions.get('radcat3') else "No" complete = [k for k in items if k.get('audit_radcat')] task_label_str.set("Report {} of {} ({} complete)\nStatus: {}".format( current + 1, len(items), len(complete), item['status'])) report_text['state'] = 'normal' report_text.delete('1.0', 'end') report_text.insert('1.0', r.anonymized()) report_text['state'] = 'disabled'
def run(self): # Set input files InputUtils.set_processor(self) InputUtils.set_memory(self) InputUtils.set_tasks(self) self.setup_tasks() # Run simulator... while self.time < self.end_sim_time: if self.verbose == System.V_DETAIL: print(f'\ntime = {self.time}') self.print_queue() # time 부터 (time+1)동안 실행될 task 코어의 개수만큼 고르기. exec_task_list = [] if len(self.queue) < self.CPU.n_core: # 큐에 있는 것 모두 실행가능(코어의 개수보다 적으므로) for tup in self.queue: exec_task_list.append(tup[1]) self.queue = [] # self.CPU.n_core - len(self.queue)개의 코어는 idle로 실행 for i in range(self.CPU.n_core - len(self.queue)): self.CPU.exec_idle(time=1) else: for i in range(self.CPU.n_core): exec_task_list.append(self.pop_queue()) # for active tasks (1 unit 실행) for exec_task in exec_task_list: exec_task.exec_active(system=self, time=1) # for other idle tasks (전력 소모 계산 및 1초 흐르기) for i in range(len(self.queue)): task = self.queue[i][1] task.exec_idle(time=1, update_deadline=True) self.queue[i] = (task.calc_priority(), task) heapq.heapify(self.queue) # 재정렬 필요 for tup in self.wait_period_queue: tup[1].exec_idle(time=1, update_deadline=False) self.add_utilization() # 실행된 task의 주기 끝났는지 확인해서 끝났으면 초기화 시키고 wait으로 for exec_task in exec_task_list: if exec_task.det_remain == 0: exec_task.period_start += exec_task.period exec_task.det_remain = exec_task.det exec_task.deadline = exec_task.period self.push_wait_period_queue(exec_task) else: self.push_queue(exec_task) self.check_queued_tasks() self.time += 1 self.check_wait_period_queue() report = Report(self) report.print_console() return report
def make_report(n_best, n_worst): report = Report() make_header(report) details = make_details(data, control.test_months, n_best, n_worst) for line in details.iterlines(): report.append(line) return report
def reportBasicInfo(self, printOnScreen=True): featuresType = [] featuresMin = [] featuresMax = [] featuresMean = [] for feature in self.features: featuresType.append(type(self.dataFrame[feature].iloc[0]).__name__) featuresMin.append(self.dataFrame[feature].min()) featuresMax.append(self.dataFrame[feature].max()) if type(self.dataFrame[feature].iloc[0]).__name__ != 'str': featuresMean.append(self.dataFrame[feature].mean()) else: featuresMean.append('none') name = 'BasicInfo' + str(self.numberOfReports('Basic') + 1) report = Report( name, cols=zip(self.features, featuresType, featuresMin, featuresMax, featuresMean, self.n_nulls), headers=["Feature", "Type", "Min", "Max", "Mean", "Num Nulls"], typeReport='Basic') if printOnScreen: report.showReport() self.reports.append(report)
def PrintReport(self): data = list(self.worldData.plants) data.extend(list(self.worldData.grazers)) data.extend(list(self.worldData.predators)) data.extend(list(self.worldData.obstacles)) report = Report() report.create(data, self.simTime)
def main(): # Delete portfolio.db if it exists os.remove("portfolio.db") print("portfolio.db removed successfully") # Get data paths stock_filename = "data_stocks.csv" bond_filename = "data_bonds.csv" # Initialize dataReader dataReader = DataReader(stock_filename, bond_filename) # Get stock and bond data stockData = dataReader.getStockData() bondData = dataReader.getBondData() # Initialize an investor investor = Investor("Bob", "Smith", "123 Fake St, Denver, CO 80221", "303.777.1234") # Initialize a portfolio portfolio = Portfolio(investor) # Add the stocks and bonds to the portfolio portfolio.addStocks(stockData) portfolio.addBonds(bondData) # Initialize a report report = Report(portfolio) # Print the report report.print()
def make_report(n_best, n_worst): report = Report() make_header(report) details, extra_info = make_details(data, control.test_months, n_best, n_worst) for line in details.iterlines(): report.append(line) make_plt(data, extra_info, n_best, n_worst) return report
def reportPPTX(cf,servers,switches,zfs): print("* Generate Final Report ...") date = datetime.now().strftime("%d%m%Y-%H%M%S") name = cf.variables['pptx_report'] + '/' + 'Exalogic_Report_'+ date + '.pptx' r = Report() r.loadreport(date,name,"Informe de desempeño de nodos Exalogic", servers, switches, zfs, cf) r.generatepptxreport() print("* ENJOY :) * ")
def __init__(self, api_aid=None, api_sec=None, api_org=None, api_url=None, api_token=None, api_token_expires=None): """ """ # logger self.log_level = { 'debug': logging.DEBUG, 'info': logging.INFO, 'warning': logging.WARNING, 'error': logging.ERROR, 'critical': logging.CRITICAL } self.formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s (%(funcName)s:%(lineno)d)' ) self.tcl = tc_logger() # debugging self._memory_monitor = True # credentials self._api_aid = api_aid self._api_sec = api_sec self._api_token = api_token self._api_token_expires = api_token_expires # user defined values self._api_org = api_org self._api_url = api_url self._api_result_limit = 200 # default values self._activity_log = False self._api_request_timeout = 30 self._api_retries = 5 # maximum of 5 minute window self._api_sleep = 59 # seconds self._bulk_on_demand = False self._enable_report = False self._indicators_regex = indicators_regex self._proxies = {'https': None} self._retype = type(re.compile('')) # config items self._report = [] self._verify_ssl = False # initialize request session handle self._session = Session() # instantiate report object self.report = Report() # save custom types for later self._indicator_parser = IndicatorObjectParser(self)
def reportTest(fileName): with open(fileName, "r") as f: s = f.read() chronicles = Chronicles() chronicles.fromJSON(s) report = Report(chronicles) report.image_file_types = [".png"] report.htmlReport("../test/report.html") webbrowser.open("../test/report.html")
def _make_report(self, counters): r = Report() r.append('Records retained while reducing input file') for path, counter in counters.iteritems(): r.append(' ') r.append('path %s' % path) for tag, value in counter.iteritems(): r.append('%30s: %d' % (tag, value)) return r
def __init__(self, header_lines, column_defs, print_as_spaces, verbose=True): self._report = Report() self._header(header_lines) self._ct = ColumnsTable(column_defs, verbose) self._print_as_spaces = print_as_spaces
def CreateFeeReport(self, validfromdate, receiver, calcfees, income, notes): ch = Report() ch.createFeeReport( validfromdate=validfromdate, receiver=receiver, fees=calcfees, income=income, notes=notes )
def __init__(self): self.__report = Report() self.__csvFile = None self.__arrayOfDates = self.__report.getDates() self.__avgTemperature = None self.__avgHumidity = None self.__status = None self.__message = None
def reportInfoRelevancies(self, printOnScreen=True): name = 'RelevanciesInfo' + str(self.numberOfReports('Relevancies') + 1) report = Report(name, cols=zip(self.features, self.regressor.feature_importances_), headers=["Feature", "Relevancy"], typeReport='Relevancies') if printOnScreen: report.showReport() self.reports.append(report)
def main(): # Get data path filename = "allStocks.json" # Initialize dataReader and get data dataReader = DataReader(filename) data = dataReader.getData() # Initialize a report and call print report = Report(data) report.print()
def main(): analyze = Analyze() data = analyze.market.update_chart_data(start=DAY, period=HOUR / 12) report = Report() header = [ 'date', 'volume', 'open', 'high', 'low', 'close', 'quoteVolume', 'weightedAverage' ] report.write_csv(data=data, header=header, file_name='somefile') print("Done!")
def test_report_structure(self): lista = ["pippo", 5, "PAOLO"] thisdict = {"brand": "Ford", "model": "Mustang", "year": 1964} thisdict2 = {"brand": "Ferrari", "model": "408", "year": 1997} rp = ReportPagine(lista, thisdict, thisdict2) thisdict3 = {"brand": "Armani", "model": "jeans", "year": 2000} thisdict4 = { "brand": "The north face", "model": "Jacket", "year": 2010 } rf = ReportFoto(thisdict3, thisdict4) report = Report("sito", rp, rf) print(report.toJSON())
def __init__(self, k, validation_month, ensemble_weighting, column_definitions, test): self._column_definitions = column_definitions self._report = Report() self._test = test self._header(k, validation_month, ensemble_weighting) cd = self._column_definitions.defs_for_columns( 'description', 'mae_validation', 'mae_query', 'mare_validation', 'mare_query', ) self._ct = ColumnsTable(columns=cd, verbose=True)
def make_report(title, ordered_cities): def make_detail_line(city): return { 'city': city, 'median_price': median_prices[city], 'median_price_index': median_prices_indices[city], 'n_trades': n_trades[city], 'n_trades_index': n_trades_indices[city], } c = ColumnsTable(( ('city', 30, '%30s', ('', '', '', '', '', 'City'), 'city name'), ('median_price', 7, '%7.0f', ('', '', '', '', 'median', 'price'), 'median price in city'), ('median_price_index', 7, '%7.2f', ('median', 'price', '/', 'overall', 'median', 'price'), 'median price as fraction of overall median price'), ('n_trades', 7, '%7.0f', ('', '', '', '', 'number', 'trades'), 'number of trades across all months'), ('n_trades_index', 7, '%7.2f', ('number', 'trades', '/ ', 'overall', 'median', 'trades'), 'median number trades as fraction of overall median number of trades' ), )) for city in ordered_cities: c.append_detail(**make_detail_line(city)) c.append_legend(40) r = Report() r.append(title) r.append(' ') for line in c.iterlines(): r.append(line) return r
def __init__(self, validation_month, k, column_definitions, test): self._report = Report() self._header(validation_month, k) self._column_definitions = column_definitions self._test = test cd = self._column_definitions.defs_for_columns( 'median_absolute_error', 'model', 'n_months_back', 'max_depth', 'n_estimators', 'max_features', 'learning_rate', ) self._ct = ColumnsTable(columns=cd, verbose=True)
def doPoll(self): oneDay = 60*60*24 startTime = (time.time() - 7*oneDay) * 1000 reports = self.api.getReports(startTime).json() if (len(reports) == 0): return for report in reports: id = report['id'] fights = self.api.getFights(id) if not report['id'] in self.data.reports.keys(): self.data.reports[id] = Report( id, report['title'], report['owner'], int(report['start'])) print("New log!") if len(self.data.reports[id].fights) < len(fights): print("New fight!") self.data.reports[id].dirty = True for fight in fights: fightId = fight['id'] if not fightId in self.data.reports[id].fights.keys(): newFight = Fight(fightId, fight['name'],id) self.data.reports[id].addFight(newFight) for id, report in self.data.reports.items(): if report.isDirty(): self.data.reports[id].message = report.getFormattedChatMessage() if report.startTime < (time.time() - 14*oneDay) * 1000: self.data.reports.pop(id, None)
def generateUniqueKeys(params, data): report_name = Report.getReportName(params) structure_keymap = fn.getNestedElement(params, 'structure_keymap') unique_keys = [] for idx in range(0, len(data)): row = data[idx] key = {} key['facility_type'] = fn.getNestedElement(row, 'facility_type') key['facility_code'] = fn.getNestedElement(row, 'facility_code') key['budget_type_code'] = fn.getNestedElement(row, 'budget_type_code') key['object_code'] = fn.getNestedElement(row, 'object_code') key['drug_code'] = fn.getNestedElement(row, 'drug_code') key['item_group_code'] = fn.getNestedElement(row, 'item_group_code') key['state_code'] = fn.getNestedElement(row, 'state_code') push_data = [] for sm in global_group_order[report_name]: mapped_key = global_group_order_kepmap[sm] push_data.append(key[mapped_key]) unique_keys.append('_'.join(push_data)) # Logger.v('unique_keys', len(unique_keys)); unique_keys = sorted(list(set(unique_keys))) # Logger.v('unique_keys', len(unique_keys)); return unique_keys
def __init__(self): wx.Frame.__init__(self, None, -1, "PyLoad", size=(800, 600)) self.InitProject() self.project = Project() self.reportPath = 'reports/last-report.db' self.report = Report(self.reportPath) # set self.report to None if you don't want to generate report #self.report = None self.path = None self.nb = NoteBook(self, -1, self.project, self.report) self.InitIcons() self.UseMenuBar() self.UseToolBar() self.Bind(wx.EVT_CLOSE, self.OnClose) self.Bind(EVT_PLAY_STOPPED, self.OnPlayStopped) register_changed_callback(self.SetChanged) #TODO: put in tabs' constructors self.nb.recordTab.tree.project = self.project self.nb.editTab.specialsPanel.project = self.project self.proxy = None
def generateItemDetail(params, data): result = {} report_name = Report.getReportName(params) split_uk = fn.getNestedElement(params, 'split_uk') for sm in global_group_order[report_name][:len(split_uk)]: if sm == 'facility_type': smk = 'facility' else: smk = sm if sm == 'facility_type': id_ = '{0}_type'.format(smk) name_ = '{0}_type'.format(smk) name_2 = '{0}_type'.format(smk) code_ = '{0}_type'.format(smk) else: id_ = '{0}_seq_no'.format(smk) name_ = '{0}_name'.format(smk) name_2 = '{0}_desc'.format(smk) code_ = '{0}_code'.format(smk) try: result['name'] = data[0][name_] except Exception as e: result['name'] = fn.getNestedElement( data[0], name_2, fn.getNestedElement(data[0], code_)) result['id'] = data[0][id_] result['code'] = data[0][code_] return result
def generateChildren(params, data): report_name = Report.getReportName(params) min_purchase_amount = fn.getNestedElement(params, 'filter.min_purchase_amount', 0) if type(min_purchase_amount) == str and min_purchase_amount: min_purchase_amount = int(min_purchase_amount) children = [] for d in data: # Logger.v('d', d); obj_ = {} # Logger.v('global_children_key',global_children_key) # Logger.v('report_name',report_name) for gck in global_children_key[report_name]: # Logger.v('gck', gck) value = fn.getNestedElement(d, gck) if value or value == 0: obj_[gck] = value else: if report_name == 'budget': # Logger.v('Report.generateChildren: {0} not found, sum up after data cleaning process.'.format(gck)); if gck == 'total_allocation': obj_[gck] = d['first_allocation'] + d[ 'additional_allocation'] elif gck == 'balance_amount': obj_[gck] = d['first_allocation'] + d[ 'additional_allocation'] - d['pending_amount'] - d[ 'liablity_amount'] - d['utilized_amount'] # Logger.v('obj_', obj_); if report_name == 'procurement': if obj_['purchase_amount'] > min_purchase_amount: children.append(obj_) else: children.append(obj_) return children
def preprocessData(params, data): report_name = Report.getReportName(params) key_to_join = fn.getNestedElement(global_key_to_join, report_name) df = pd.DataFrame(data, dtype=str) # Logger.v('df', df); joined_key = [] joined_ = [] joined_columns_list = [key_to_join[0]] df['first_allocation'] = pd.to_numeric(df['first_allocation']) df['additional_allocation'] = pd.to_numeric(df['additional_allocation']) df['pending_amount'] = pd.to_numeric(df['pending_amount']) df['utilized_amount'] = pd.to_numeric(df['utilized_amount']) df['liablity_amount'] = pd.to_numeric(df['liablity_amount']) df['trans_in_amount'] = pd.to_numeric(df['trans_in_amount']) df['trans_out_amount'] = pd.to_numeric(df['trans_out_amount']) df['deduction_amount'] = pd.to_numeric(df['deduction_amount']) df['current_actual_amount'] = pd.to_numeric(df['current_actual_amount']) df['total_allocation'] = pd.to_numeric(df['total_allocation']) df['balance_amount'] = pd.to_numeric(df['balance_amount']) # df.info(); for idx in range(0, len(key_to_join)): ktj = key_to_join[idx] joined_key.append(ktj) if idx > 0: joined_.append(['_'.join(joined_key[:-1]), ktj]) columns = joined_[idx - 1] joined_columns = '_'.join(columns) joined_columns_list.append(joined_columns) df[joined_columns] = df[columns[0]].str.cat(df[columns[1]], sep="|") return df
def __init__(self, k, ensemble_weighting, column_definitions, test): self._column_definitions = column_definitions self._test = test self._report = Report() self._header(k, ensemble_weighting) cd = self._column_definitions.defs_for_columns( 'validation_month', 'mae_index0', 'mae_ensemble', 'mae_best_next_month', 'median_price', 'fraction_median_price_next_month_index0', 'fraction_median_price_next_month_ensemble', 'fraction_median_price_next_month_best', ) self._ct = ColumnsTable(columns=cd, verbose=True)
def report(self): r = Report() for bus in self.bus_list: field_ops = bus.get_field_ops() for op in field_ops: op.report(r) print r
def __init__(self): self.config = ConfigManager() self.db = DBManager() self.report = Report(self.config, self.db) if self.config.hostsToPing[0] is '' and len( self.config.hostsToPing) is 1: print("No hosts to ping") sys.exit() # Perform pings for host in self.config.hostsToPing: self.ping(host) # self.db.PrintResultsTable() # Used for testing self.report.SendReport() # Send report if needed
def __init__(self, pr: int, branch: str, date: str, e_date: float, feature_variables: int, meta_variables: int, passed: bool, pr_id: int, num_samples: int, sha: str, time_elapsed: str, user: str, email: str, status: str, report: str = None): self.pr = pr self.branch = branch self.date = date self.e_date = e_date self.feature_variables = feature_variables self.meta_variables = meta_variables self.passed = passed self.pr_id = pr_id self.num_samples = num_samples self.sha = sha self.time_elapsed = time_elapsed self.user = user self.email = email self.status = status self.report = Report(report)
def make_report(summary): r = Report() format_header = '%40s %8s %8s %8s %8s %8s %8s %8s' format_detail = '%40s %8.0f %8.0f %8.0f %8.0f %8d %8d %8.0f' r.append(format_header % ('numeric feature', 'min', 'median', 'mean', 'max', 'distinct', 'NaN', 'std')) for row_name, row_value in summary.iterrows(): r.append(format_detail % ( row_name, row_value['min'], row_value['50%'], row_value['mean'], row_value['max'], row_value['number_distinct'], row_value['number_nan'], row_value['std'])) return r
def run(self): global nodes logging.info("Starting FjoSpidie 2.0") starttime = datetime.now() ids_engine = None tempdir = tempfile.mkdtemp(dir="/mnt/fjospidie") if self.config.suricata: from engine.SuricataEngine import SuricataEngine else: from engine.SnortEngine import SnortEngine report = Report(starttime, self.config) proxy_port = random.randint(20000, 65534) start_url = urlparse(self.config.url) nodes.append(Node(start_url.hostname)) nodes[0].set_status(200) if not self.config.nopcap: pcap_engine = PcapEngine(self.config, report, tempdir) pcap_engine.start() webrunner = WebRunner(report) har = webrunner.run_webdriver(start_url, proxy_port, self.config, tempdir) if not self.config.nopcap: pcap_engine.stop() pcap_path = pcap_engine.pcap_path connections = webrunner.find_external_connections(har) entries = har.entries report.insert_entries(entries) if self.config.parsers: for parser in self.config.parsers, : package = "fjospidie.engine.parser.{}".format(parser) try: imported = importlib.import_module(package) parser_class = getattr(imported, parser) parser_engine = parser_class(self.config, report, entries) parsers.append(parser_engine) parser_engine.start() except Exception, e: logging.error("Error starting parser {}: {}".format( parser, e)) next
def testNIC(nic, dict, descr): defaults = { 'bsizes': [64, 128, 256, 512, 1024, 2048, 4096, 8192, 16 * 1024] } print(descr) r = Report(("cpu read", "tot read", "cpu write", "tot write")) r.printHeading() bsizes = dict['bsizes'] if 'bsizes' in dict else defaults['bsizes'] for bs in bsizes: tr = nic.read_time(bs) cr = nic.read_cpu(bs) tw = nic.write_time(bs) cw = nic.write_cpu(bs) r.printLatency(bs, (cr, tr, cw, tw)) print("")
def testHBA(hba, dict, descr): defaults = { 'bsizes': [512, 4096, 128 * 1024, 4096 * 1024] } print(descr) r = Report(("cpu read", "tot read", "cpu write", "tot write")) r.printHeading() bsizes = dict['bsizes'] if 'bsizes' in dict else defaults['bsizes'] for bs in bsizes: tr = hba.read_time(bs) cr = hba.read_cpu(bs) tw = hba.write_time(bs) cw = hba.write_cpu(bs) r.printLatency(bs, (cr, tr, cw, tw)) print("")
def make_chart_a(control, data): 'return a Report' def make_header(report): report.append('Median Absolute Errors for Most Accurate Models') report.append('By Month') report.append('By Feature Group') report.append(' ') def make_details(data, control): 'return a ColumnsTable' def append_feature_group_description(ct): ct.append_line(' ') ct.append_line('Features groups;') ct.append_line('s : only size features') ct.append_line('sw : only size and wealth features') ct.append_line('swp : only size, wealth, and property features') ct.append_line('swpn : all features: size, wealth, property, and neighborhood') ct = ColumnsTable(( ('month', 6, '%6s', ('', 'month'), 'training month'), ('features', 8, '%8s', ('features', 'group'), 'group of features'), ('model', 5, '%5s', ('best', 'model'), 'family of best model'), ('mae', 6, '%6.0f', ('', 'mae'), 'mae of best model in month using features'), ), verbose=True, ) for month in control.test_months: for features in control.feature_groups: mae_model = data[month][features] ct.append_detail( month=month, features=features, model=mae_model.model, mae=mae_model.mae, ) ct.append_detail() # blank line separates each month ct.append_legend() append_feature_group_description(ct) return ct report = Report() make_header(report) for line in make_details(data, control).iterlines(): report.append(line) return report
def tptest(disk, dict, descr="Estimated Throughput"): """ run a standard set of throughputs against a specified device disk -- device to be tested dict -- FioRsize ... size of test file FioRdepths ... list of request depths FioRbs ... list of block sizes filesize -- size of the file used for the test depth -- number of queued parallel operations """ dflt = { # default throughput test parameters 'FioRsize': 16 * GIG, 'FioRdepth': [1, 32], 'FioRbs': [4096, 128 * 1024, 4096 * 1024], } sz = dict['FioRsize'] if 'FioRsize' in dict else dflt['FioRsize'] depths = dict['FioRdepth'] if 'FioRdepth' in dict else dflt['FioRdepth'] bsizes = dict['FioRbs'] if 'FioRbs' in dict else dflt['FioRbs'] r = Report(("seq read", "seq write", "rnd read", "rnd write")) for depth in depths: print("%s (%s), depth=%d" % (descr, disk.desc, depth)) r.printHeading() for bs in bsizes: # run the simulations tsr = disk.avgTime(bs, sz, read=True, seq=True, depth=depth) tsw = disk.avgTime(bs, sz, read=False, seq=True, depth=depth) trr = disk.avgTime(bs, sz, read=True, seq=False, depth=depth) trw = disk.avgTime(bs, sz, read=False, seq=False, depth=depth) # compute the corresponding bandwidths bsr = bs * SECOND / tsr bsw = bs * SECOND / tsw brr = bs * SECOND / trr brw = bs * SECOND / trw r.printBW(bs, (bsr, bsw, brr, brw)) # compute the corresponding IOPS isr = SECOND / tsr isw = SECOND / tsw irr = SECOND / trr irw = SECOND / trw r.printIOPS(0, (isr, isw, irr, irw)) # print out the latencies r.printLatency(0, (tsr, tsw, trr, trw)) print("")
def testGetReport(self): rep = Report() self.assertTrue(rep.saveReport("Escola Santa Clara","Juan","professor",0,1,0,1,0,2,1,0,0,1)) form = {} form['qualification'] = 'professor' form['teacherName'] = 'Juan' form['question0'] = 0 form['question1'] = 1 form['question2'] = 0 form['question3'] = 1 form['question4'] = 0 form['question5'] = 2 form['question6'] = 1 form['question7'] = 0 form['question8'] = 0 form['question9'] = 1 self.assertEqual(rep.getReport('Escola Santa Clara'),form)
def CreateQuarterAnnouncement(self, date): setattr(self.db, 'querydate', date) isodate = '{:%Y-%m-%d}'.format(date) children = self.db.getchildren_for_quarterannouncement(isodate) ch = Report() ch.createQAReport(children, isodate)
def __init__(self): self.seek_list = None self.comment_file = None self.index_file = None self.symbol_to_encoding_dict = None self.cids = None self.comment_offsets_cid = None self.comment_offsets = None self.comment_term_counts = None self.comment_csv_reader = None self.authors_list = None self.articles_list = None self.reply_to_index = None self.collection_term_count = 0 self.stemmer = Stemmer.Stemmer('english') self.tokenizer = nltk.tokenize.ToktokTokenizer() self.report = Report()
def lint(self): wire_to_field_op = {} # maps wires to the field_op that output it specified_outputs = {} # maps wires to buses field_ops = [] # check that each wire is specified no more than once for bus in self.bus_list: for field_op in bus.get_field_ops(): field_ops.append(field_op) output_wires = field_op.output_wires() # print "field_op %s" % field_op # print " %s" % output_wires for output in output_wires: if (output in specified_outputs): print "LINT: bus %s re-specifies wire %s, already set by bus %s" % ( bus, output, specified_outputs[output]) else: specified_outputs[output] = bus wire_to_field_op[output] = field_op # check that each wire is specified at least once for idx in range(self.total_wire_count): wire = Wire(idx) if (wire not in specified_outputs): print "LINT: no bus specifies output %s" % wire # check that each field_op computes a value that eventually reaches # an output. useful_field_ops = set( filter(lambda f: isinstance(f, FieldOutput), field_ops)) done_field_ops = set() while (len(useful_field_ops) > 0): #print "Considering %s field_ops" % len(useful_field_ops) required_wires = set() for field_op in useful_field_ops: #print "inputs of %s are %s" % (field_op, field_op.input_wires()) required_wires.update(field_op.input_wires()) done_field_ops.update(useful_field_ops) #print "Considering %s required_wires" % len(required_wires) useful_field_ops = set() for wire in required_wires: print wire field_op = wire_to_field_op[wire] if (field_op not in done_field_ops): useful_field_ops.add(field_op) all_field_ops = set(field_ops) unused_field_ops = all_field_ops.difference(done_field_ops) r = Report() for field_op in unused_field_ops: field_op.report(r) if (len(r) > 0): print "LINT: %d unused field ops; cost:\n%s" % ( len(unused_field_ops), r) # print "LINT: %s" % unused_field_ops print "(info) Linted %s field ops from %s buses" % (len(field_ops), len(self.bus_list))
def make_chart_b(control, data): "return a Report" def make_header(report): report.append("Mean Probability of a Feature Being Included in a Decision Tree") report.append("Across the Entire Ensemble of Decisions Trees") report.append("For Most Accurate Model in Each Training Month") report.append(" ") def make_mean_importance_by_feature(test_months): "return dict[feature_name] = float, the mean importance of the feature" feature_names = Features().ege_names(control.arg.features) mean_importance = {} # key = feature_name for feature_index, feature_name in enumerate(feature_names): # build vector of feature_importances for feature_name feature_importances = np.zeros(len(test_months)) # for feature_name for month_index, test_month in enumerate(test_months): month_importances = data[ReductionKey(test_month)] # for each feature all_feature_importances = month_importances.importances["feature_importances"] feature_importances[month_index] = all_feature_importances[feature_index] mean_importance[feature_name] = np.mean(feature_importances) return mean_importance def make_details(data, test_months): "return a ColumnTable" columns_table = ColumnsTable( ( ("mean_prob", 5, "%5.2f", ("mean", "prob"), "mean probability feature appears in a decision tree"), ("feature_name", 40, "%40s", (" ", "feature name"), "name of feature"), ), verbose=True, ) mean_importance = make_mean_importance_by_feature(test_months) for feature_name in sorted(mean_importance, key=mean_importance.get, reverse=True): columns_table.append_detail(mean_prob=mean_importance[feature_name] * 100.0, feature_name=feature_name) columns_table.append_legend() return columns_table report = Report() make_header(report) details = make_details(data, control.test_months) for line in details.iterlines(): report.append(line) return report
def report_for_file(self, file_name): print 'report for: ' + file_name windows = self._windows methods = self._methods report = Report(file_name, windows, methods) raw_txt = Reader.readFromFile(file_name) #print raw_txt words = Reader.extractWords(raw_txt, "russian") keywords = Reader.meter(words) self._keywords = keywords self._terms = words #инициализация отчета термами с tf for term in self._terms: report.add_term_tf(term, keywords[term]) for window in windows: for method in methods: print method, window (array, graph) = self.get_rw_for(method, window) report._graph = graph #todo graph как св-во, пересмотреть логику for v in array: term = v.term_value report.add_term_rw_stats(term, method, window, v.term_weight_rw) self._reports[file_name] = report
def report(self): """ Returns a detailed report on the packing of this structure. """ report = Report() report.add_html_body(self.html_report()) if self.options["reportmode"] == "html": return [report.get_html(), ""] elif self.options["reportmode"] == "text": return [report.get_text(), ""] elif self.options["reportmode"] == "both": return [report.get_text(), report.get_html()] else: raise "unknown report mode option '%s'" % options["reportmode"]
def report(self,options): """ Returns a detailed report on the packing of this structure. """ report = Report() report.add_html_body(self.html_report(options)) if options['reportmode'] == 'html': return [report.get_html(),None] elif options['reportmode'] == 'text': return [report.get_text(),None] elif options['reportmode'] == 'both': return [report.get_text(),report.get_html()] else: raise "unknown report mode option '%s'"%options['reportmode']
def __init__(self, api_aid, api_sec, api_org, api_url): """ """ # logger self.log_level = { 'debug': logging.DEBUG, 'info': logging.INFO, 'warning': logging.WARNING, 'error': logging.ERROR, 'critical': logging.CRITICAL} self.formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s (%(funcName)s:%(lineno)d)') self.tcl = tc_logger() # debugging self._memory_monitor = True # credentials self._api_aid = api_aid self._api_sec = api_sec # user defined values self._api_org = api_org self._api_url = api_url self._api_result_limit = 200 # default values self._activity_log = False self._api_request_timeout = 30 self._api_retries = 5 # maximum of 5 minute window self._api_sleep = 59 # seconds self._bulk_on_demand = False self._enable_report = False self._indicators_regex = indicators_regex self._proxies = {'https': None} self._retype = type(re.compile('')) # config items self._report = [] self._verify_ssl = False # initialize request session handle self._session = Session() # instantiate report object self.report = Report()
def tptest(disk, filesize, depth=1, bsizes=(4096, 128 * 1024, 4096 * 1024)): """ run a standard set of throughputs against a specified device disk -- device to be tested filesize -- size of the file used for the test depth -- number of queued parallel operations """ r = Report(("seq read", "seq write", "rnd read", "rnd write")) r.printHeading() for bs in bsizes: tsr = disk.avgTime(bs, filesize, read=True, seq=True, depth=depth) tsw = disk.avgTime(bs, filesize, read=False, seq=True, depth=depth) trr = disk.avgTime(bs, filesize, read=True, seq=False, depth=depth) trw = disk.avgTime(bs, filesize, read=False, seq=False, depth=depth) # compute the corresponding bandwidths bsr = bs * SECOND / tsr bsw = bs * SECOND / tsw brr = bs * SECOND / trr brw = bs * SECOND / trw r.printBW(bs, (bsr, bsw, brr, brw)) r.printIOPS(bs, (bsr, bsw, brr, brw)) r.printLatency(bs, (tsr, tsw, trr, trw))
def get_surfdist_report(self,surfdist,options): bins = options['surfdist_bins'] maxdepth = options['surfdist_maxdepth'] step = maxdepth/bins count = [0] * bins bigger = 0 total_count = 0 for s in surfdist: if s == -1 or s == 9999: continue for i in range(bins): if s<=(i+1)*step: total_count += 1 count[i] += 1 break if s>maxdepth: bigger += 1 # write table report = """<h2>Distances to next surface atom</h2> <table> <tr><th>bin</th><th>dist</th><th>#atoms</th></tr> """ for i in range(bins): report += "<tr><td>%i</td><td>%5.2f</td><td>%i</td></tr>\n"%(i+1,step*(i+1),count[i]) report += "</table><p>total atoms counted: %i<br>"%total_count report += "\natoms out of range: %i</p>\n"%bigger hreport = Report() hreport.add_html_body(report) if options['reportmode'] == 'html': return [hreport.get_html(),None] elif options['reportmode'] == 'text': return [hreport.get_text(),None] elif options['reportmode'] == 'both': return [hreport.get_text(),hreport.get_html()] else: raise "unknown report mode option '%s'"%options['reportmode'] return hreport
def make_chart_stats(data, control, filter_f): 'return Report with statistics for years and months that obey the filter' r = Report() r.append('Prices by Month') r.append('') ct = ColumnsTable(( ('year', 4, '%4d', (' ', ' ', 'year'), 'year of transaction'), ('month', 5, '%5d', (' ', ' ', 'month'), 'month of transaction'), ('mean_price', 6, '%6.0f', (' ', ' mean', 'price'), 'mean price in dollars'), ('median_price', 6, '%6.0f', (' ', 'median', 'price'), 'median price in dollars'), ('mean_price_ratio', 6, '%6.3f', (' mean', ' price', ' ratio'), 'ratio of price in current month to prior month'), ('median_price_ratio', 6, '%6.3f', ('median', ' price', ' ratio'), 'ratio of price in current month to prior month'), ('number_trades', 6, '%6d', ('number', 'of', 'trades'), 'number of trades in the month'), )) prior_mean_price = None prior_median_price = None for year in xrange(2003, 2010): for month in xrange(1, 13): if filter_f(year, month): value = data[make_reduction_key(year, month)] mean_price = value['mean'] median_price = value['median'] number_trades = value['count'] ct.append_detail( year=year, month=month, mean_price=mean_price, median_price=median_price, mean_price_ratio=None if prior_mean_price is None else mean_price / prior_mean_price, median_price_ratio=None if prior_median_price is None else median_price / prior_median_price, number_trades=number_trades, ) prior_mean_price = mean_price prior_median_price = median_price ct.append_legend() for line in ct.iterlines(): r.append(line) return r
def loop(): global CURRENT_DAY global CURRENT_MON global CURRENT_HOUR global CROP global SYSTEM_CONTROLLER global SOIL_HUMIDITY global CURRENT_TEMPERATURE global IS_RAINING global CURRENT_ILLUMINATION global READINGS_DELAY global DEBUG global DEBUG_DAY global DEBUG_MONTH global DEBUG_HOUR global SYSTEM_OVERRIDE global CURRENT_CROP_REPORT global CURRENT_SYSTEM_REPORT global UPDATE_SYSTEM while True: if read_rain(): UPDATE_SYSTEM = True if read_temperature(): UPDATE_SYSTEM = True if read_soil_humiture(): UPDATE_SYSTEM = True if read_illumination(): UPDATE_SYSTEM = True if UPDATE_SYSTEM: print_line('UPDATING SYSTEM') CURRENT_DAY = datetime.datetime.today().day CURRENT_MON = datetime.datetime.today().month CURRENT_HOUR = datetime.datetime.today().hour # Overriding date/time if debugging if DEBUG: override_debug_values() if DEBUG_DAY is not False: CURRENT_DAY = DEBUG_DAY print_line('DEBUGGING WITH DAY AS: ' + str(CURRENT_DAY)) if DEBUG_MONTH is not False: CURRENT_MON = DEBUG_MONTH print_line('DEBUGGING WITH MONTH AS: ' + str(CURRENT_MON)) if DEBUG_HOUR is not False: CURRENT_HOUR = DEBUG_HOUR print_line('DEBUGGING WITH HOUR AS: ' + str(CURRENT_HOUR)) report = Report(SOIL_HUMIDITY, CURRENT_TEMPERATURE, IS_RAINING, CURRENT_ILLUMINATION) report.build_crop_status(CROP, CURRENT_MON, CURRENT_DAY, CURRENT_HOUR) crop_report = report.get() update_status = SYSTEM_CONTROLLER.update_status(crop_report, SYSTEM_OVERRIDE) CURRENT_CROP_REPORT = build_crop_report(crop_report) CURRENT_SYSTEM_REPORT = '' for status in update_status: print_line(' .' + status) CURRENT_SYSTEM_REPORT += ' .' + status + '\n' UPDATE_SYSTEM = False time.sleep(READINGS_DELAY)
def compare(self): for dirname, dirnames, filenames in walk(self.__report_folder_path): report_under_test = Report(dirname) pdf_results = TemplateVariables() ppt_results = TemplateVariables() xls_results = TemplateVariables() doc_results = TemplateVariables() for filename in filenames: convert_to_images(dirname, filename) if '\\2010' in dirname: for reportdir, reportdirs, reportfiles in walk(dirname): for report in reportfiles: report_path = join(reportdir, report) etalon_report = report_path.replace('\\2010\\', '\\2005\\') if report.endswith(FileFormat.PDF + FileFormat.POWERPOINT + FileFormat.WORD): self.__run_bcomp__(report_path, etalon_report, self.__text_compare_settings) bat_name = create_batch(report_path, etalon_report) if Report.get_report_name(reportdir) == report_under_test.report_title and \ report.endswith(FileFormat.PDF): pdf_results.old_report = etalon_report pdf_results.new_report = report_path pdf_results.bat_list.append(bat_name) pdf_results.html_list.append(self.compare_report) elif report.endswith(".ppt"): ppt_results.old_report = etalon_report ppt_results.new_report = report_path ppt_results.bat_list.append(bat_name) ppt_results.html_list.append(self.compare_report) elif report.endswith(".doc"): doc_results.old_report = etalon_report doc_results.new_report = report_path doc_results.bat_list.append(bat_name) doc_results.html_list.append(self.compare_report) elif report.endswith(FileFormat.EXCEL): xls_results.old_report = etalon_report xls_results.new_report = report_path elif report.endswith(FileFormat.PNG): self.__run_bcomp__(report_path, etalon_report, self.__picture_compare_settings) bcomp_clean(reportdir, self.compare_report) bat_name = create_batch(report_path, etalon_report) if reportdir.endswith("-xls"): xls_results.bat_list.append(bat_name) xls_results.html_list.append(self.compare_report) if reportdir.endswith("-pdf"): pdf_results.bat_list.append(bat_name) pdf_results.html_list.append(self.compare_report) elif reportdir.endswith("-ppt"): ppt_results.bat_list.append(bat_name) ppt_results.html_list.append(self.compare_report) elif reportdir.endswith("-doc"): doc_results.bat_list.append(bat_name) doc_results.html_list.append(self.compare_report) template_vars = { "report_title": report_under_test.report_title, "pdf_old": pdf_results.old_report, "pdf_new": pdf_results.new_report, "pdf_bats": pdf_results.bat_list, "pdf_htmls": pdf_results.html_list, "excel_old": xls_results.old_report, "excel_new": xls_results.new_report, "xls_bats": xls_results.bat_list, "xls_htmls": xls_results.html_list, "ppt_old": ppt_results.old_report, "ppt_new": ppt_results.new_report, "ppt_bats": ppt_results.bat_list, "ppt_htmls": ppt_results.html_list, "doc_old": doc_results.old_report, "doc_new": doc_results.new_report, "doc_bats": doc_results.bat_list, "doc_htmls": doc_results.html_list, } Template(template_vars, dirname, report_under_test.report_title).create_template()
def translate(self, message): hexMessageIN = [] isRequest = [] # turn into bitstrings for i in message: value = BitArray(hex=i[0].replace(" ", "")) hexMessageIN.append(value) isRequest.append(i[1]) # detect multiple messages hexMessages = [] temp = [] for i in hexMessageIN: print(i[0:16].hex) if i[0:16].hex == BitArray("0x0564").hex: if len(temp) > 0: hexMessages.append(list(temp)) temp = [] temp.append(i) if len(temp) > 0: hexMessages.append(list(temp)) allMessages = Report("All Messages", "The entirity of the message", "", "") messageCount = -1 for hexMessage in hexMessages: messageCount += 1 hexString = "" for msg in hexMessage: hexString += msg.hex + ", " thisMessage = Report("Message {}".format(messageCount), "", "", hexString) # verify correctness try: if not DataLinkTranslator.DataLayerCorrect(hexMessage[0]): thisMessage.AddNext( Report("ERROR", "This message is not verifyably DNP3, or may be malformed", message[0]), "" ) return Report("Invalid Message", "", "", "") except: return Report("Invalid Input", "", "", "") # remove CRC bits for everything (Andy took 'em out) # for i in hexMessage: # i = DataLinkTranslator.StripCRCBits(i) # Get message length thisMessage.length = DataLinkTranslator.DataLayerLength(hexMessage[0][:]) thisMessage.MessageLength = thisMessage.length.uint thisMessage.AddNext( Report( "Message length", "Number of octets this message contains that are not CRC related", str(thisMessage.length.uint), "", ) ) # Get Control Field control = DataLinkTranslator.DataLayerControl(hexMessage[0][:]) thisMessage.AddNext( Report("Message Control Data", "Function operations and qualifiers", str(control.hex), "") ) thisMessage.Next[-1].AddNext(DataLinkTranslator.DataLayerControlReport(control)) # message sender thisMessage.sender = DataLinkTranslator.DataLayerSource(hexMessage[0][:]) thisMessage.AddNext(Report("Message Sender", "ID for sender", str(thisMessage.sender.hex), "")) # message reciever thisMessage.reciever = DataLinkTranslator.DataLayerDestination(hexMessage[0][:]) thisMessage.AddNext(Report("Message Reciever", "ID for Reciever", str(thisMessage.reciever.hex), "")) # message transport layer thisMessage.transport = "" if hexMessage[1][0]: thisMessage.transport += " FINAL " if hexMessage[1][1]: thisMessage.transport += " FIRST " thisMessage.AddNext( Report( "Transport Function", "Links together large messages in sequence", (thisMessage.transport + "Seq {}").format(hexMessage[1][2:8].uint), "", ) ) hexMessage[1] = hexMessage[1][8:] # technically a block, so sue me fragment = 1 baseLayer = thisMessage while fragment < len(hexMessage): bucket = [] # requests contain no actual data # just outlines for what is expected in responses if isRequest[index]: bucket.append(Report("Object Header", "Prefix information on Application layer", "", "")) flags = getAppRequestHeader(hexMessage[fragment]) for i in flags: bucket[-1].Next.append(i) bucket.append(translateFuncCode(getFuncCode(hexMessage[fragment]))) # skip internal indications, does not exist in requests # responses...well... else: bucket.append(Report("Object Header", "Prefix information on Application layer", "", "")) flags = getAppRequestHeader(hexMessage[fragment]) for i in flags: bucket[-1].Next.append(i) bucket.append(translateFuncCode(getFuncCode(hexMessage[fragment]))) for i in bucket: temp = i if not isinstance(i, Report): temp = Report(i, "", "", "") baseLayer.Next[-1].AddNext(temp) baseLayer.AddNext(Report("Application Fragment {}".format(fragment - 1), "", "", "")) fragment += 1 allMessages.AddNext(thisMessage) print(allMessages) return allMessages
class ThreatConnect: """ """ def __init__(self, api_aid, api_sec, api_org, api_url): """ """ # logger self.log_level = { 'debug': logging.DEBUG, 'info': logging.INFO, 'warning': logging.WARNING, 'error': logging.ERROR, 'critical': logging.CRITICAL} self.formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s (%(funcName)s:%(lineno)d)') self.tcl = tc_logger() # debugging self._memory_monitor = True # credentials self._api_aid = api_aid self._api_sec = api_sec # user defined values self._api_org = api_org self._api_url = api_url self._api_result_limit = 200 # default values self._activity_log = False self._api_request_timeout = 30 self._api_retries = 5 # maximum of 5 minute window self._api_sleep = 59 # seconds self._bulk_on_demand = False self._enable_report = False self._indicators_regex = indicators_regex self._proxies = {'https': None} self._retype = type(re.compile('')) # config items self._report = [] self._verify_ssl = False # initialize request session handle self._session = Session() # instantiate report object self.report = Report() # # Memory Testing # # self._p = psutil.Process(os.getpid()) # self._memory = self._p.memory_info().rss def _api_request_headers(self, ro): """ """ timestamp = int(time.time()) signature = "{0}:{1}:{2}".format(ro.path_url, ro.http_method, timestamp) # python 2.7, does not work on 3.x and not tested on 2.6 # hmac_signature = hmac.new(self._api_sec, signature, digestmod=hashlib.sha256).digest() # authorization = 'TC {0}:{1}'.format(self._api_aid, base64.b64encode(hmac_signature)) # python 3.x hmac_signature = hmac.new(self._api_sec.encode(), signature.encode(), digestmod=hashlib.sha256).digest() authorization = 'TC {0}:{1}'.format(self._api_aid, base64.b64encode(hmac_signature).decode()) ro.add_header('Timestamp', timestamp) ro.add_header('Authorization', authorization) def api_filter_handler(self, resource_obj, filter_objs): """ """ data_set = None if not filter_objs: # build api call (no filters) default_request_object = resource_obj.default_request_object data_set = self.api_response_handler(resource_obj, default_request_object) else: # # process each filter added to the resource object for retrieve # first_run = True # # each resource object can have x filter objects with an operator to join or intersect results # for filter_obj in filter_objs: obj_list = [] # temp storage for results on individual filter objects owners = filter_obj.owners if len(owners) == 0: # handle filters with no owners owners = [self._api_org] # use default org # iterate through all owners for o in owners: self.tcl.debug('owner: {0!s}'.format(o)) if len(filter_obj) > 0: # request object are for api filters for ro in filter_obj: if ro.owner_allowed: ro.set_owner(o) results = self.api_response_handler(resource_obj, ro) if ro.resource_type not in [ResourceType.OWNERS, ResourceType.VICTIMS, ResourceType.BATCH_JOBS]: # TODO: should this be done? # post filter owners for obj in results: if obj.owner_name.upper() != o.upper(): results.remove(obj) obj_list.extend(results) else: ro = filter_obj.default_request_object if ro.owner_allowed: ro.set_owner(o) results = self.api_response_handler(resource_obj, ro) if ro.resource_type not in [ResourceType.OWNERS, ResourceType.VICTIMS]: # TODO: should this be done? # post filter owners for obj in results: if obj.owner_name.upper() != o.upper(): results.remove(obj) obj_list.extend(results) # # post filters # pf_obj_set = set(obj_list) self.tcl.debug('count before post filter: {0:d}'.format(len(obj_list))) for pfo in filter_obj.post_filters: self.tcl.debug('pfo: {0!s}'.format(pfo)) # # Report Entry # report_entry = ReportEntry() report_entry.add_post_filter_object(pfo) # current post filter method filter_method = getattr(resource_obj, pfo.method) # current post filter results post_filter_results = set(filter_method(pfo.filter, pfo.operator, pfo.description)) pf_obj_set = pf_obj_set.intersection(post_filter_results) self.report.add(report_entry) # set obj_list to post_filter results if filter_obj.post_filters_len > 0: obj_list = list(pf_obj_set) self.tcl.debug('count after post filter: {0:d}'.format(len(obj_list))) # no need to join or intersect on first run if first_run: data_set = set(obj_list) first_run = False continue # # depending on the filter type the result will be intersected or joined # if filter_obj.operator is FilterSetOperator.AND: data_set = data_set.intersection(obj_list) elif filter_obj.operator is FilterSetOperator.OR: data_set.update(set(obj_list)) # # only add to report if these results should be tracked (exclude attribute, tags, etc) # self.report.add_filtered_results(len(data_set)) # # after intersection or join add the objects to the resource object # for obj in data_set: resource_obj.add_obj(obj) def api_request(self, ro): """ """ api_response = None fail_msg = None h_content_length = None h_content_type = None start = datetime.now() # # enable activity log # if self._activity_log: ro.enable_activity_log() # # prepare request # url = '{0!s}{1!s}'.format(self._api_url, ro.request_uri) api_request = Request(ro.http_method, url, data=ro.body, params=ro.payload) request_prepped = api_request.prepare() # # generate headers # ro.set_path_url(request_prepped.path_url) self._api_request_headers(ro) request_prepped.prepare_headers(ro.headers) # # Debug # self.tcl.debug('request_object: {0!s}'.format(ro)) self.tcl.debug('url: {0!s}'.format(url)) self.tcl.debug('path url: {0!s}'.format(request_prepped.path_url)) # # api request (gracefully handle temporary communications issues with the API) # for i in range(1, self._api_retries + 1, 1): try: api_response = self._session.send( request_prepped, verify=self._verify_ssl, timeout=self._api_request_timeout, proxies=self._proxies, stream=False) break except exceptions.ReadTimeout as e: self.tcl.error('Error: {0!s}'.format(e)) self.tcl.error('The server may be experiencing delays at the moment.') self.tcl.info('Pausing for {0!s} seconds to give server time to catch up.'.format(self._api_sleep)) time.sleep(self._api_sleep) self.tcl.info('Retry {0!s} ....'.format(i)) if i == self._api_retries: self.tcl.critical('Exiting: {0!s}'.format(e)) raise RuntimeError(e) except exceptions.ConnectionError as e: self.tcl.error('Error: {0!s}'.format(e)) self.tcl.error('Connection Error. The server may be down.') self.tcl.info('Pausing for {0!s} seconds to give server time to catch up.'.format(self._api_sleep)) time.sleep(self._api_sleep) self.tcl.info('Retry {0!s} ....'.format(i)) if i == self._api_retries: self.tcl.critical('Exiting: {0!s}'.format(e)) raise RuntimeError(e) except socket.error as e: self.tcl.critical('Exiting: {0!s}'.format(e)) raise RuntimeError(e) # # header values # if 'content-length' in api_response.headers: h_content_length = api_response.headers['content-length'] if 'content-type' in api_response.headers: h_content_type = api_response.headers['content-type'] # # raise exception on *critical* errors # non_critical_errors = [ b'The MD5 for this File is invalid, a File with this MD5 already exists', # 400 (application/json) b'The SHA-1 for this File is invalid, a File with this SHA-1 already exists', # 400 (application/json) b'The SHA-256 for this File is invalid, a File with this SHA-256 already exists', # 400 (application/json) b'The requested resource was not found', # 404 (application/json) b'Could not find resource for relative', # 500 (text/plain) b'The requested Security Label was not removed - access was denied', # 401 (application/json) ] # # TODO: work out some logic to improve the API error handling, possible area where API could improve # # valid status codes 200, 201, 202 # if api_response.status_code in [400, 401, 403, 500, 503]: if api_response.status_code not in [200, 201, 202]: # check for non critical errors that have bad status codes nce_found = False fail_msg = api_response.content for nce in non_critical_errors: # api_response_dict['message'] not in non_critical_errors: if re.findall(nce, api_response.content): nce_found = True break if ro.failure_callback is not None: ro.failure_callback(api_response.status_code) # raise error on bad status codes that are not defined as nce if not nce_found: self.tcl.critical('Status Code: {0:d}'.format(api_response.status_code)) self.tcl.critical('Failed API Response: {0!s}'.format(api_response.content)) if ro.failure_callback is not None: ro.failure_callback(api_response.status_code) raise RuntimeError(api_response.content) # # set response encoding (best guess) # if api_response.encoding is None: api_response.encoding = api_response.apparent_encoding # # Debug # self.tcl.debug('url: %s', api_response.url) self.tcl.debug('status_code: %s', api_response.status_code) self.tcl.debug('content-length: %s', h_content_length) self.tcl.debug('content-type: %s', h_content_type) # # Report # self.report.add_api_call() # count api calls self.report.add_request_time(datetime.now() - start) self.tcl.debug('Request Time: {0!s}'.format(datetime.now() - start)) if self._enable_report: report_entry = ReportEntry() report_entry.add_request_object(ro) report_entry.set_request_url(api_response.url) report_entry.set_status_code(api_response.status_code) report_entry.set_failure_msg(fail_msg) self.report.add(report_entry) # # return response # return api_response def api_response_handler(self, resource_obj, ro): """ """ # # initialize vars # api_response_dict = {} obj_list = [] # only track filter counts on request from this method ro.enable_track() # # debug # self.tcl.debug('Results Limit: {0!s}'.format(self._api_result_limit)) # only resource supports pagination if ro.resource_pagination: ro.set_result_limit(self._api_result_limit) ro.set_result_start(0) else: ro.set_remaining_results(1) while ro.remaining_results > 0: # # api request # api_response = self.api_request(ro) # self.tcl.debug('Results Content: {0!s}'.format(api_response.content)) self.tcl.debug('Status Code: {0!s}'.format(api_response.status_code)) self.tcl.debug('Content Type: {0!s}'.format(api_response.headers['content-type'])) # # Process API response # if api_response.headers['content-type'] == 'application/json': api_response_dict = api_response.json() # try and free memory for next api request api_response.close() del api_response # doesn't appear to clear memory # # BULK INDICATOR (does not have status) # if 'indicator' in api_response_dict: if ro.resource_type == ResourceType.INDICATORS: data = api_response_dict['indicator'] for item in data: obj_list.append(parse_indicator( item, resource_obj, ro.description, ro.request_uri, self._indicators_regex)) if len(obj_list) % 500 == 0: self.tcl.debug('obj_list len: {0!s}'.format(len(obj_list))) elif api_response_dict['status'] == 'Failure': # handle failed request (404 Resource not Found) if 'message' in api_response_dict: self.tcl.error('{0!s} "{1!s}"'.format(api_response_dict['message'], ro.description)) ro.set_remaining_results(0) continue # # ADVERSARIES # elif ro.resource_type == ResourceType.ADVERSARIES: data = api_response_dict['data']['adversary'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: obj_list.append( parse_group(item, ResourceType.ADVERSARIES, resource_obj, ro.description, ro.request_uri)) # # INDICATORS # elif ro.resource_type == ResourceType.INDICATORS: data = api_response_dict['data']['indicator'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: obj_list.append(parse_indicator( item, resource_obj, ro.description, ro.request_uri, self._indicators_regex)) # # ADDRESSES # elif ro.resource_type == ResourceType.ADDRESSES: data = api_response_dict['data']['address'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: obj_list.append(parse_indicator( item, resource_obj, ro.description, ro.request_uri, self._indicators_regex)) # # DOCUMENTS # elif ro.resource_type == ResourceType.DOCUMENTS: data = api_response_dict['data']['document'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: obj_list.append( parse_group( item, ResourceType.DOCUMENTS, resource_obj, ro.description, ro.request_uri)) # # EMAILS # elif ro.resource_type == ResourceType.EMAILS: data = api_response_dict['data']['email'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: obj_list.append( parse_group( item, ResourceType.EMAILS, resource_obj, ro.description, ro.request_uri)) # # EMAIL ADDRESSES # elif ro.resource_type == ResourceType.EMAIL_ADDRESSES: data = api_response_dict['data']['emailAddress'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: obj_list.append(parse_indicator( item, resource_obj, ro.description, ro.request_uri, self._indicators_regex)) # # GROUPS # elif ro.resource_type == ResourceType.GROUPS: data = api_response_dict['data']['group'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: obj_list.append( parse_group(item, ResourceType.GROUPS, resource_obj, ro.description, ro.request_uri)) # # FILES # elif ro.resource_type == ResourceType.FILES: data = api_response_dict['data']['file'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: obj_list.append(parse_indicator( item, resource_obj, ro.description, ro.request_uri, self._indicators_regex)) # # HOSTS # elif ro.resource_type == ResourceType.HOSTS: data = api_response_dict['data']['host'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: obj_list.append(parse_indicator( item, resource_obj, ro.description, ro.request_uri, self._indicators_regex)) # # DNSResolutions # elif ro.resource_type == ResourceType.DNS_RESOLUTIONS: data = api_response_dict['data']['dnsResolution'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: if 'addresses' in item: # don't process dns resolutions that have no addresses obj_list.append(parse_dns_resolution(item)) # # INCIDENTS # elif ro.resource_type == ResourceType.INCIDENTS: data = api_response_dict['data']['incident'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: obj_list.append( parse_group(item, ResourceType.INCIDENTS, resource_obj, ro.description, ro.request_uri)) # # METRICS # # elif ro.resource_type == ResourceType.OWNER_METRICS: # data = api_response_dict['data']['ownerMetric'] # if not isinstance(data, list): # data = [data] # for single results to be a list # for item in data: # obj_list.append( # parse_metrics(item, resource_obj, ro.description, ro.request_uri)) # # MINE # # elif ro.resource_type == ResourceType.OWNER_MINE: # data = api_response_dict['data']['owner'] # if not isinstance(data, list): # data = [data] # for single results to be a list # for item in data: # obj_list.append( # parse_metrics(item, resource_obj, ro.description, ro.request_uri)) # # MEMBERS # # elif ro.resource_type == ResourceType.OWNER_MEMBERS: # data = api_response_dict['data']['user'] # if not isinstance(data, list): # data = [data] # for single results to be a list # for item in data: # obj_list.append( # parse_metrics(item, resource_obj, ro.description, ro.request_uri)) # # OWNERS # elif ro.resource_type == ResourceType.OWNERS: data = api_response_dict['data']['owner'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: obj_list.append( parse_owner(item, resource_obj, ro.description, ro.request_uri)) # # SIGNATURES # elif ro.resource_type == ResourceType.SIGNATURES: data = api_response_dict['data']['signature'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: obj_list.append( parse_group(item, ResourceType.SIGNATURES, resource_obj, ro.description, ro.request_uri)) # # TASKS # elif ro.resource_type == ResourceType.TASKS: data = api_response_dict['data']['task'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: obj_list.append( parse_task(item, ResourceType.TASKS, resource_obj, ro.description, ro.request_uri)) # # THREATS # elif ro.resource_type == ResourceType.THREATS: data = api_response_dict['data']['threat'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: obj_list.append( parse_group(item, ResourceType.THREATS, resource_obj, ro.description, ro.request_uri)) # # URLS # elif ro.resource_type == ResourceType.URLS: data = api_response_dict['data']['url'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: obj_list.append(parse_indicator( item, resource_obj, ro.description, ro.request_uri, self._indicators_regex)) # # VICTIMS # elif ro.resource_type == ResourceType.VICTIMS: data = api_response_dict['data']['victim'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: # victims data comes back with no owner, manually add owner here item['owner'] = ro.owner obj_list.append(parse_victim(item, resource_obj, ro.description, ro.request_uri)) # # BatchJobs # elif ro.resource_type == ResourceType.BATCH_JOBS: data = api_response_dict['data']['batchStatus'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: # victims data comes back with no owner, manually add owner here item['owner'] = ro.owner obj_list.append(parse_batch_job(item, resource_obj, ro.description, ro.request_uri)) elif api_response.headers['content-type'] == 'text/plain': self.tcl.error('{0!s} "{1!s}"'.format(api_response.content, ro.description)) ro.set_remaining_results(0) continue # add_obj resource_pagination if required if ro.resource_pagination: # get the number of results returned by the api if ro.result_start == 0: ro.set_remaining_results(api_response_dict['data']['resultCount'] - ro.result_limit) else: ro.set_remaining_results(ro.remaining_results - ro.result_limit) # increment the start position ro.set_result_start(ro.result_start + ro.result_limit) else: ro.set_remaining_results(0) self.tcl.debug('Result Count: {0!s}'.format(len(obj_list))) self.report.add_unfiltered_results(len(obj_list)) return obj_list # # api / sdk settings # def result_pagination(self, ro, identifier): data = [] ro.set_result_limit(self._api_result_limit) ro.set_result_start(0) while ro.remaining_results > 0: api_response = self.api_request(ro) if api_response.headers['content-type'] == 'application/json': api_response_dict = api_response.json() if api_response_dict['status'] == 'Success': data.extend(api_response_dict['data'][identifier]) # get the number of results returned by the api if ro.result_start == 0: ro.set_remaining_results(api_response_dict['data']['resultCount'] - ro.result_limit) else: ro.set_remaining_results(ro.remaining_results - ro.result_limit) # increment the start position ro.set_result_start(ro.result_start + ro.result_limit) return data def report_enable(self): """ """ self._enable_report = True def report_disable(self): """ """ self._enable_report = False def set_activity_log(self, data_bool): """ enable or disable api activity log """ if isinstance(data_bool, bool): self._activity_log = data_bool def set_api_request_timeout(self, data_int): """ set timeout value for the requests module """ if isinstance(data_int, int): self._api_request_timeout = data_int else: raise AttributeError(ErrorCodes.e0110.value.format(data_int)) def set_api_retries(self, data): """ set the number of api retries before exception is raised """ if isinstance(data, int): self._api_retries = data else: raise AttributeError(ErrorCodes.e0120.value.format(data)) def set_api_sleep(self, data): """ set the amount of time between retries """ if isinstance(data, int): self._api_sleep = data else: raise AttributeError(ErrorCodes.e0130.value.format(data)) def set_api_result_limit(self, data_int): """ set the number of result to return per api request (500 max) """ if isinstance(data_int, int): self._api_result_limit = data_int else: raise AttributeError(ErrorCodes.e0140.value.format(data_int)) def set_proxies(self, proxy_address, proxy_port, proxy_user=None, proxy_pass=None): """ define proxy server to use with the requests module """ # "http": "http://*****:*****@10.10.1.10:3128/", # accept host with http(s) or without proxy_method = 'http://' if re.match('^http', proxy_address): proxy_method, proxy_host = proxy_address.split('//') proxy_method += '//' proxy_address = proxy_host # TODO: add validation if proxy_user is not None and proxy_pass is not None: self._proxies['https'] = '{0!s}{1!s}:{2!s}@{3!s}:{4!s}'.format( proxy_method, proxy_user, proxy_pass, proxy_address, proxy_port) else: self._proxies['https'] = '{0!s}{1!s}:{2!s}'.format( proxy_method, proxy_address, proxy_port) def get_proxies(self): """ get proxy settings """ return self._proxies def set_tcl_file(self, fqpn, level='info'): """ set the log file destination and log level """ file_path = os.path.dirname(fqpn) if os.access(file_path, os.W_OK): if self.tcl.level > self.log_level[level]: self.tcl.setLevel(self.log_level[level]) fh = logging.FileHandler(fqpn) # fh.set_name('tc_log_file') # not supported in python 2.6 if level in self.log_level.keys(): fh.setLevel(self.log_level[level]) else: fh.setLevel(self.log_level['info']) fh.setFormatter(self.formatter) self.tcl.addHandler(fh) # def set_tcl_level(self, level): # """ """ # if level in self.log_level.keys(): # if self.tcl.level > self.log_level[level]: # self.tcl.setLevel(self.log_level[level]) # self.tcl.handlers[0].setLevel(self.log_level[level]) def set_tcl_console_level(self, level): """ set the console log level """ if level in self.log_level.keys(): if self.tcl.level > self.log_level[level]: self.tcl.setLevel(self.log_level[level]) ch = logging.StreamHandler() # ch.set_name('console') # not supported in python 2.6 ch.setLevel(self.log_level[level]) ch.setFormatter(self.formatter) self.tcl.addHandler(ch) def set_indicator_regex(self, type_enum, compiled_regex): """ overwrite default SDK regex """ self.tcl.debug('overwrite regex for {0!s}'.format(type_enum.name)) if not isinstance(type_enum, IndicatorType): raise AttributeError(ErrorCodes.e0150.value.format(type_enum)) if not isinstance(compiled_regex, list): compiled_regex = [compiled_regex] cr_list = [] for cr in compiled_regex: if isinstance(cr, self._retype): cr_list.append(cr) else: raise AttributeError(ErrorCodes.e0160.value.format(cr)) self._indicators_regex[type_enum.name] = cr_list # # Resources # def adversaries(self): """ return an adversary container object """ return Adversaries(self) def bulk(self): """ return a bulk container object """ return Bulk(self) def bulk_indicators(self, on_demand=False): """ return a bulk indicator container object """ return BulkIndicators(self, on_demand) def documents(self): """ return a document container object """ return Documents(self) def emails(self): """ return an email container object """ return Emails(self) def groups(self): """ return an group container object """ return Groups(self) def incidents(self): """ return an incident container object """ return Incidents(self) def indicators(self): """ return an indicator container object """ return Indicators(self) def owners(self): """ return an owner container object """ return Owners(self) def signatures(self): """ return a signature container object """ return Signatures(self) def tasks(self): """ return a task container object """ return Tasks(self) def threats(self): """ return a threat container object """ return Threats(self) def victims(self): """ return a victim container object """ return Victims(self) def batch_jobs(self): return BatchJobs(self)
def play_project(project, reporter = None, project_path = 'unkown', variables = None): assert isinstance(project, Project) if project.current_special == None: raise RuntimeError('No special to start!') user_count = project.user_count iteration_count = project.iteration_count special = project.current_special user_factory = project.user_factory iteration_factory = project.iteration_factory global_factory = project.global_factory if reporter != None: from Report import Report if type(reporter) in (str, unicode): reporter = Report(reporter) assert isinstance(reporter, Report) import datetime start_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') info = { 'Start Time': start_time, 'User Count': user_count, 'Iteration Count': iteration_count, 'Special': special.label, 'Project Path': project_path, } import ReportManager ReportManager.start_report(reporter = reporter, project = project, info = info) import PlayPolicy policy = PlayPolicy.IterationBasedPlayPolicy( player = special, user_count = user_count, iteration_count = iteration_count, user_factory = user_factory, iteration_factory = iteration_factory, global_factory = global_factory, reporter = reporter ) import sys sys.path.append(os.path.join(sys.path[0], 'runtime')) sys.path.append(os.path.join(sys.path[0], 'plugin')) if variables: from Scope import Scope scope = Scope() scope.variables = variables policy.play(scope) else: policy.play() if len(sys.path) >= 2 and sys.path[-2:] == ['runtime', 'plugin']: sys.path.pop() sys.path.pop() if reporter: reporter.finish()
def restore(self, diff_snapshot): report = Report() for report_class in self.list_subclass: report.union(report_class.restore(diff_snapshot)) return report
def fstest(fs, filesize=16 * MEG, depth=1, direct=False, sync=False, crtdlt=False, bsizes=(4096, 128 * 1024, 4096 * 1024)): """ compute & display standard fio to filesystem on a disk fs -- file system to be tested filesize -- size of file in which I/O is being done depth -- number of concurrent requests direct -- I/O is direct (not buffered) sync -- updates are immediately flushed """ if crtdlt: (tc, bwc, loadc) = fs.create(sync=sync) (td, bwd, loadd) = fs.delete(sync=sync) r = Report(("create", "delete")) r.printHeading() r.printIOPS(1, (bwc, bwd)) r.printLatency(1, (tc, td)) r = Report(("seq read", "seq write", "rnd read", "rnd write")) r.printHeading() for bs in bsizes: (tsr, bsr, lsr) = fs.read(bs, filesize, seq=True, depth=depth, direct=direct) (tsw, bsw, lsw) = fs.write(bs, filesize, seq=True, depth=depth, direct=direct, sync=sync) (trr, brr, lrr) = fs.read(bs, filesize, seq=False, depth=depth, direct=direct) (trw, brw, lrw) = fs.write(bs, filesize, seq=False, depth=depth, direct=direct, sync=sync) r.printBW(bs, (bsr, bsw, brr, brw)) r.printIOPS(bs, (bsr, bsw, brr, brw)) r.printLatency(bs, (tsr, tsw, trr, trw))
def servertest(fs, depth=1, crtdlt=False, bsizes=(4096, 128 * 1024, 4096 * 1024)): """ compute & display standard test results """ if crtdlt: tc = fs.create() td = fs.delete() r = Report(("create", "delete")) r.printHeading() r.printIOPS(1, (SECOND / tc, SECOND / td)) r.printLatency(1, (tc, td)) r = Report(("seq read", "seq write", "rnd read", "rnd write")) r.printHeading() for bs in bsizes: (tsr, bsr, rload) = fs.read(bs, depth=depth, seq=True) (tsw, bsw, wload) = fs.write(bs, depth=depth, seq=True) (trr, brr, rload) = fs.read(bs, depth=depth, seq=False) (trw, brw, wload) = fs.write(bs, depth=depth, seq=False) r.printBW(bs, (bsr, bsw, brr, brw)) r.printIOPS(bs, (bsr, bsw, brr, brw)) r.printLatency(bs, (tsr, tsw, trr, trw))
def testSaveReport(self): rep = Report() self.assertTrue(rep.saveReport("Escola Santa Clara","Juan","professor",0,1,0,1,0,2,1,0,0,1))
# # basic unit test exerciser # if __name__ == '__main__': cpu = makeCPU([]) print("%s w/%dGB of DDR3-%d RAM" % (cpu.desc, cpu.mem_size / GIG, cpu.mem_speed)) print print(" thread switch %dus" % (cpu.thread_us())) print(" process switch %dus" % (cpu.proc_us())) print(" DMA start/intr %dus" % (cpu.dma_us())) from Report import Report r = Report(("mem-rd", "mem-wrt", "process", "instrs")) print r.printHeading() sizes = [1024, 4096, 128*1024, 1024*1024] for bs in sizes: mem_r = cpu.mem_read(bs) mem_w = cpu.mem_write(bs) mem_p = cpu.process(bs) mem_x = cpu.execute(bs) r.printLatency(bs, (mem_r, mem_w, mem_p, mem_x)) r = Report(("sha-1", "comp", "decomp", "RAID-6")) print r.printHeading() sizes = [1024, 4096, 128*1024, 1024*1024] for bs in sizes: