def annotate_scmsources(): url_regex = re.compile('^(.*)$') line_regex_strict = re.compile( r"^let\s+scmnr\.(\d+)\s*=\s*\{'type':\s*'(\w+)',\s*'url':\s*'([^']+)'\s*\}\s*" ) line_snr_regex = re.compile(r'(\d+)') line_url_regex = re.compile(r"'url': '([^']+)'") line_scm_regex = re.compile(r"'type': '([^']+)'") prnt( '┌ X if line contains information besides repository URL and SCM used') prnt('│┌ ? or ! if line failes to be parsed by regular expressions') prnt('││┌ A if scm type is unknown') prnt('│││┌ F if match failes') prnt( '││││┌ O if match deduced from the description is different, M if there are no' ) prnt( '│││││┌ E if exception occurred, C if printing candidate on this line') prnt( '┴┴┴┴┴┴┬─────────────────────────────────────────────────────────────────────────' ) with lshg.MercurialRemoteParser() as remote_parser: set_remote_parser(remote_parser) for line in process_scmsources(False): try: numcolumns = 5 best_candidate = None line = line.decode('utf-8') logger.info('Checking line ' + line) match = line_regex_strict.match(line) if match: write(' ') numcolumns -= 2 snr, scm, url = match.groups() else: write('X') numcolumns -= 1 if args['--no-extra-check']: raise NotLoggedError snr = line_snr_regex.search(line) scm = line_scm_regex.search(line) url = line_url_regex.search(line) if snr: snr = snr.group(1) if scm: scm = scm.group(1) if url: url = url.group(1) if not snr: write('!') numcolumns -= 1 raise ValueError('snr key not found') if not scm: write('?A') numcolumns -= 2 url = None elif not url: write('?-') numcolumns -= 2 scm = None else: write(' ') numcolumns -= 1 if scm not in scm_matches: write('A') numcolumns -= 1 url = None scm = None else: write(' ') numcolumns -= 1 voinfo = db[snr] vofiles = set(get_file_list(voinfo)) if url: match = url_regex.match(url) candidate = scm_matches[scm](match, voinfo) files = get_scm_file_list(candidate) prefix, key2 = compare_file_lists(candidate, vofiles, files) write(' ' if key2 else 'F') numcolumns -= 1 else: write('-') numcolumns -= 1 best_candidate = find_repo_candidate(voinfo, vofiles) if not best_candidate: write('M') numcolumns -= 1 elif not (url and best_candidate.scm_url == url and best_candidate.scm == scm): write('O') numcolumns -= 1 else: write(' ') numcolumns -= 1 best_candidate = None write(' ') numcolumns -= 1 except Exception as e: if not isinstance(e, NotLoggedError): logger.exception(e) write(('-' * numcolumns) + 'E') finally: # XXX line ends with \n, thus not writing `+ '\n'` here. write('│' + line) if best_candidate and args['--print-other-candidate']: write( "-----C│let scmnr.%-4u = {'type': '%s', 'url': '%s'}\n" % (int(snr), best_candidate.scm, best_candidate.scm_url))
def annotate_scmsources(): url_regex = re.compile('^(.*)$') line_regex_strict = re.compile(r"^let\s+scmnr\.(\d+)\s*=\s*\{'type':\s*'(\w+)',\s*'url':\s*'([^']+)'\s*\}\s*") line_snr_regex = re.compile(r'(\d+)') line_url_regex = re.compile(r"'url': '([^']+)'") line_scm_regex = re.compile(r"'type': '([^']+)'") prnt ('┌ X if line contains information besides repository URL and SCM used') prnt ('│┌ ? or ! if line failes to be parsed by regular expressions') prnt ('││┌ A if scm type is unknown') prnt ('│││┌ F if match failes') prnt ('││││┌ O if match deduced from the description is different, M if there are no') prnt ('│││││┌ E if exception occurred, C if printing candidate on this line') prnt ('┴┴┴┴┴┴┬─────────────────────────────────────────────────────────────────────────') with lshg.MercurialRemoteParser() as remote_parser: set_remote_parser(remote_parser) for line in process_scmsources(False): try: numcolumns = 5 best_candidate = None line = line.decode('utf-8') logger.info('Checking line ' + line) match = line_regex_strict.match(line) if match: write(' ') numcolumns -= 2 snr, scm, url = match.groups() else: write('X') numcolumns -= 1 if args['--no-extra-check']: raise NotLoggedError snr = line_snr_regex.search(line) scm = line_scm_regex.search(line) url = line_url_regex.search(line) if snr: snr = snr.group(1) if scm: scm = scm.group(1) if url: url = url.group(1) if not snr: write('!') numcolumns -= 1 raise ValueError('snr key not found') if not scm: write('?A') numcolumns -= 2 url = None elif not url: write('?-') numcolumns -= 2 scm = None else: write(' ') numcolumns -= 1 if scm not in scm_matches: write('A') numcolumns -= 1 url = None scm = None else: write(' ') numcolumns -= 1 voinfo = db[snr] vofiles = set(get_file_list(voinfo)) if url: match = url_regex.match(url) candidate = scm_matches[scm](match, voinfo) files = get_scm_file_list(candidate) prefix, key2 = compare_file_lists(candidate, vofiles, files) write(' ' if key2 else 'F') numcolumns -= 1 else: write('-') numcolumns -= 1 best_candidate = find_repo_candidate(voinfo, vofiles) if not best_candidate: write('M') numcolumns -= 1 elif not (url and best_candidate.scm_url == url and best_candidate.scm == scm): write('O') numcolumns -= 1 else: write(' ') numcolumns -= 1 best_candidate = None write(' ') numcolumns -= 1 except Exception as e: if not isinstance(e, NotLoggedError): logger.exception(e) write(('-' * numcolumns) + 'E') finally: # XXX line ends with \n, thus not writing `+ '\n'` here. write('│' + line) if best_candidate and args['--print-other-candidate']: write("-----C│let scmnr.%-4u = {'type': '%s', 'url': '%s'}\n" % (int(snr), best_candidate.scm, best_candidate.scm_url))
def main(): scmnrs = process_scmsources() omitted = load_scmnrs_json(scmnrs, omitted_name) found = scmnrs.copy() scm_generated = load_scmnrs_json(scmnrs, scm_generated_name) not_found = load_scmnrs_json(scmnrs, not_found_name, set) if not args['--no-descriptions']: try: with open(description_hashes_name) as DHF: description_hashes = json.load(DHF) except IOError: description_hashes = {} else: description_hashes = {} if args['scripts']: keys = args['<SID>'] not_found -= set(keys) elif args['last']: i = args['last'] _keys = reversed(sorted(db, key=int)) keys = [] while i: keys.append(next(_keys)) i -= 1 else: keys = reversed(sorted(db, key=int)) _process_voinfo = partial(process_voinfo, scm_generated, found, not_found, description_hashes) with lshg.MercurialRemoteParser() as remote_parser: set_remote_parser(remote_parser) if args['recheck']: for key in scm_generated: _process_voinfo(key, db[key], recheck=True) else: for key in keys: if not args['--force'] and key in scmnrs: if args['new']: break else: continue logger.info('Considering key {0}'.format(key)) _process_voinfo(key, db[key]) if not args['--no-descriptions']: logger.info('Starting descriptions check') for key in keys: voinfo = db[key] changed = False if key not in description_hashes: h = get_voinfo_hash(voinfo) description_hashes[key] = h changed = True if key in found: continue if not changed: h = get_voinfo_hash(voinfo) changed = (h != description_hashes.get(key)) if changed: logger.info( 'Hash for key {0} changed, checking it'.format( key)) description_hashes[key] = h else: logger.info('New hash for key {0}'.format(key)) if changed: _process_voinfo(key, voinfo) return scm_generated, not_found, description_hashes
def main(): scmnrs = process_scmsources() omitted = load_scmnrs_json(scmnrs, omitted_name) found = scmnrs.copy() scm_generated = load_scmnrs_json(scmnrs, scm_generated_name) not_found = load_scmnrs_json(scmnrs, not_found_name, set) if not args['--no-descriptions']: try: with open(description_hashes_name) as DHF: description_hashes = json.load(DHF) except IOError: description_hashes = {} else: description_hashes = {} if args['scripts']: keys = args['<SID>'] not_found -= set(keys) elif args['last']: i = args['last'] _keys = reversed(sorted(db, key=int)) keys = [] while i: keys.append(next(_keys)) i -= 1 else: keys = reversed(sorted(db, key=int)) _process_voinfo = partial(process_voinfo, scm_generated,found,not_found,description_hashes) with lshg.MercurialRemoteParser() as remote_parser: set_remote_parser(remote_parser) if args['recheck']: for key in scm_generated: _process_voinfo(key, db[key], recheck=True) else: for key in keys: if not args['--force'] and key in scmnrs: if args['new']: break else: continue logger.info('Considering key {0}'.format(key)) _process_voinfo(key, db[key]) if not args['--no-descriptions']: logger.info('Starting descriptions check') for key in keys: voinfo = db[key] changed = False if key not in description_hashes: h = get_voinfo_hash(voinfo) description_hashes[key] = h changed = True if key in found: continue if not changed: h = get_voinfo_hash(voinfo) changed = (h != description_hashes.get(key)) if changed: logger.info('Hash for key {0} changed, checking it'.format(key)) description_hashes[key] = h else: logger.info('New hash for key {0}'.format(key)) if changed: _process_voinfo(key, voinfo) return scm_generated, not_found, description_hashes