def test_run_field_rules(): config = { "rules": { "field_level_rules": [ { "name": "DescriptionRule", "run": True }, { "name": "YesNoNameRule", "run": True }, ] }, } linter = LookMlLinter(config) lookml = LookML("test/minimal_multiline.view.lkml") v = lookml.views()[0] print("v", v) out = linter.run_field_rules(v, "dimension", "dimensions", "xxx", []) assert out[0] == { "file": "xxx", "rule": "DescriptionRule", "passed": 1, "type": "dimension", "fieldname": "city_code", }
def test_run_field_rules(): config = { "rules": { "field_level_rules": [ { "name": "DescriptionRule", "run": True }, { "name": "YesNoNameRule", "run": True }, ] }, } linter = LookMlLinter(config) lookml = LookML("test/minimal_multiline.view.lkml") v = lookml.views()[0] print("v", v) out = linter.run_field_rules(v, 'dimension', 'dimensions', "xxx", []) assert out[0] == { 'file': 'xxx', 'rule': 'DescriptionRule', 'passed': 1, 'type': 'dimension', 'fieldname': 'city_code' }
def run(self): ''' run the set of file and field-level rules against all files in the file glob Returns: nothing. Saves two CSV files, specified in the config ''' file_out = [] field_out = [] timestr = datetime.datetime.now().isoformat() no_orphans_rule = None if "NoOrphansRule" in self.other_rules_to_run(): no_orphans_rule = NoOrphansRule(self.config) globstrings = self.config['infile_globs'] for globstring in globstrings: filepaths = glob.glob(globstring) for filepath in filepaths: simple_filepath = os.path.basename(filepath) logging.info("Processing %s", filepath) lookml = LookML(filepath) file_out = self.run_file_rules(lookml, simple_filepath, file_out) if lookml.has_views(): v = lookml.views()[0] field_out = self.run_field_rules(v, 'dimension', 'dimensions', simple_filepath, field_out) field_out = self.run_field_rules(v, 'dimension_group', 'dimension_groups', simple_filepath, field_out) field_out = self.run_field_rules(v, 'measure', 'measures', simple_filepath, field_out) if no_orphans_rule: no_orphans_rule.process_lookml(lookml) #add some metadata for each of the records we created above [ f.update({'glob': globstring}) for f in field_out + file_out if not 'glob' in f ] # for this rule, we can only assess who failed after all files are processed if no_orphans_rule: file_out = no_orphans_rule.finish_up(file_out) if 'simple_biquery' in self.config['output']: simple_bq_writer = SimpleBqWriter() if 'bigquery' in self.config['output']: bq_writer = BqWriter() if len(file_out) > 0: df = pd.DataFrame(file_out) df['time'] = timestr df['repo'] = self.config['git']['url'] if 'csv' in self.config['output']: self.write_file_csv(df) if 'simple_biquery' in self.config['output']: simple_bq_writer.upload(df, self.config, 'file_destination_table') if 'bigquery' in self.config['output']: bq_writer.upload(df, self.config, 'file_destination_table') if len(field_out) > 0: df = pd.DataFrame(field_out) df['time'] = timestr df['repo'] = self.config['git']['url'] if 'csv' in self.config['output']: self.write_field_csv(df) if 'simple_biquery' in self.config['output']: simple_bq_writer.upload(df, self.config, 'field_destination_table') if 'bigquery' in self.config['output']: bq_writer.upload(df, self.config, 'field_destination_table') return file_out, field_out