def merge(input, output): filepath_to_package_name = lambda filepath: filepath[filepath.rindex( os.sep) + 1:filepath.rindex('.json')] # List rules input_rules = list_filter( is_app_rules, list_map(lambda item: input + os.sep + item, os.listdir(input))) existing_packs = list_map( filepath_to_package_name, list_filter( is_app_rules, list_map(lambda item: output + os.sep + 'apps' + os.sep + item, os.listdir(output + os.sep + 'apps')))) skipped_file = 0 finished_file = 0 for input_rule in input_rules: package_name = filepath_to_package_name(input_rule) # If existing this rules in output path, skip it. if (package_name in existing_packs): skipped_file += 1 else: shutil.copy( input_rule, output + os.sep + 'apps' + os.sep + package_name + '.json') finished_file += 1 print('Finished merging. Skipped %d files.' \ ' Copied %d files.' % (skipped_file, finished_file))
def close_existing_rules_issues(github, rules_path): repo = github.get_repo(ISSUE_REPO) # Get issues only created by auto wizard issues = list_filter( lambda issue: issue.title.startswith( \ '[New rules request][AUTO]'), repo.get_issues(state='open').get_page(0) ) # Get existing rules package names package_name = list_map( lambda item: item[item.rindex(os.sep) + 1:item.rindex('.json')], list_filter( is_app_rules, list_map(lambda item: rules_path + os.sep + 'apps' + os.sep + item, os.listdir(rules_path + os.sep + 'apps')))) print('Start closing issues...') count = 0 for issue in issues: if issue.title[issue.title.rindex(' ') + 1:] in package_name: issue.edit(state="closed") count += 1 if count == 0: print('No issues to close.') else: print('Closed %d issues.' % count)
def add_ids_for_observers(path): rules = list_filter( is_app_rules, list_map(lambda item: path + os.sep + 'apps' + os.sep + item, os.listdir(path + os.sep + 'apps'))) # Update rules for rule in rules: model = {} try: with codecs.open(rule, mode='r', encoding='utf-8') as f: model = json.loads(f.read()) f.close() changed = False if 'observers' in model.keys(): for observer in model['observers']: if not 'id' in observer.keys(): temp_id = observer['description'] count = 0 while contains_id_in_observer(model['observers'], \ temp_id + '_' + str(count)): count += 1 observer['id'] = temp_id + '_' + str(count) changed = True if changed: with codecs.open(rule, mode='w', encoding='utf-8') as f: f.write(json.dumps(model, indent=2, ensure_ascii=False)) f.close() except Exception as e: print('Failed to update ' + rule)
def convert(input): rules_path = input rules = list_filter( is_app_rules, list_map(lambda item: rules_path + os.sep + item, os.listdir(rules_path))) print('Found rules count: %d' % (len(rules))) # Make output path output_path = input + os.sep + 'output' if os.path.isfile(output_path): os.remove(output_path) if not os.path.exists(output_path): os.mkdir(output_path) print('Output to ' + output_path) # Convert and write out results for rule in rules: with codecs.open(rule, mode='r', encoding='utf-8') as f: model = json.loads(f.read(), object_pairs_hook=OrderedDict) data_converter.convert_old_data(model) with codecs.open(output_path + os.sep + model['package'] + '.json', mode='w', encoding='utf-8') as out: out.write(json.dumps(model, indent=2, ensure_ascii=False)) out.close() f.close() print('Finished converting.')
def make_verfied_list(path): rules = list_filter( is_app_rules, list_map( lambda item: path + os.sep + 'apps' + os.sep + item, os.listdir(path + os.sep + 'apps') ) ) # Get all verified apps package name verified_apps = [] for rule in rules: with codecs.open(rule, mode='r', encoding='utf-8') as f: print(rule) model = json.loads(f.read()) if 'verified' in model.keys() and model['verified']: verified_apps.append({ 'package_name': model['package'] }) f.close() print('Found verified apps count: %d' % (len(verified_apps))) # Write to output.json with codecs.open( path + os.sep + 'verified_apps.output.json', mode='w', encoding='utf-8') as out: print('Output to ' + path + os.sep + 'verified_apps.output.json') out.write(json.dumps(verified_apps, indent=2, ensure_ascii=False)) out.close() print('Finished making list.')
def download_issues(github): repo = github.get_repo(ISSUE_REPO) # Get issues only created by auto wizard issues = list_filter( lambda issue: issue.title.startswith( \ '[New rules request][AUTO]'), repo.get_issues(state='open').get_page(0) ) issues = list_filter_not(is_issue_need_discussion, issues) # Make output path output_path = os.getcwd() + os.sep + 'output' if os.path.isfile(output_path): os.remove(output_path) if not os.path.exists(output_path): os.mkdir(output_path) print('Output to ' + output_path) total = len(issues) current = 0 print_progress = lambda a, b: \ print('Start downloading rules... %d/%d' % (a, b), end='') CODE_BLOCK = '```' print_progress(current, total) # Download json output from issues for issue in issues: current += 1 # Get information from issue package_name = issue.title[issue.title.rindex(' ') + 1:] print(' Package: ' + package_name, end='\r') if (not os.path.isfile(output_path + os.sep + package_name + '.json')): body = repo.get_issue(issue.number).body content = body[body.index(CODE_BLOCK) + len(CODE_BLOCK):body.rindex(CODE_BLOCK)] # Try to convert old data try: content = json.dumps(data_converter.convert_old_data( json.loads(content, object_pairs_hook=OrderedDict)), indent=2, ensure_ascii=False) except: pass # Add to cache with codecs.open(output_path + os.sep + package_name + '.json', mode='w', encoding='utf-8') as f: f.write(content) f.close() print_progress(current, total) # Done downloading print('\nDownloaded %d rules' % (current)) print('\nFinished downloading issues. Remember to check if rules are ' \ 'vaild.')