def __init__(self, args):
super().__init__()
self.updated_since = args.updated_since
self.dump_dir = DUMP_DIR if os.path.isdir(DUMP_DIR) else None
self.loop_delay = int(args.loop_delay)
self.db = args.db
if args.command == 'github_crawler':
if args.repository:
self.repository_el_re = "%s/%s" % (
args.org.lstrip('^'),
args.repository.lstrip('^'),
)
else:
self.repository_el_re = args.org.lstrip('^') + '/.*'
self.prf = pullrequest.PRsFetcher(
GithubGraphQLQuery(token_getter=args.token_getter),
args.base_url,
args.org,
args.repository,
)
elif args.command == 'gerrit_crawler':
self.repository_el_re = args.repository.lstrip('^')
self.prf = review.ReviewesFetcher(
args.base_url,
args.repository,
args.insecure,
login=args.login,
password=args.password,
)
def __init__(self, args) -> None:
super().__init__()
self.updated_since = args.updated_since
self.dump_dir = DUMP_DIR if os.path.isdir(DUMP_DIR) else None
self.loop_delay = int(args.loop_delay)
self.db: db.ELmonocleDB = args.db
self.prf: Union[pullrequest.PRsFetcher, review.ReviewesFetcher]
if args.command == "github_crawler":
if args.repository:
self.repository_el_re = "%s/%s" % (
args.org.lstrip("^"),
args.repository.lstrip("^"),
)
else:
self.repository_el_re = args.org.lstrip("^") + "/.*"
self.prf = pullrequest.PRsFetcher(
GithubGraphQLQuery(token_getter=args.token_getter),
args.base_url,
args.org,
args.repository,
)
elif args.command == "gerrit_crawler":
self.repository_el_re = args.repository.lstrip("^")
self.prf = review.ReviewesFetcher(
args.base_url,
args.repository,
args.insecure,
login=args.login,
password=args.password,
)
def __init__(self,
args,
elastic_conn='localhost:9200',
elastic_timeout=10):
super().__init__()
self.updated_since = args.updated_since
self.dump_dir = DUMP_DIR if os.path.isdir(DUMP_DIR) else None
self.loop_delay = int(args.loop_delay)
self.db = ELmonocleDB(elastic_conn=elastic_conn,
index=args.index,
timeout=elastic_timeout)
if args.command == 'github_crawler':
if args.repository:
self.repository_el_re = "%s/%s" % (
args.org.lstrip('^'),
args.repository.lstrip('^'),
)
else:
self.repository_el_re = args.org.lstrip('^') + '/.*'
self.prf = pullrequest.PRsFetcher(GithubGraphQLQuery(args.token),
args.base_url, args.org,
args.repository)
elif args.command == 'gerrit_crawler':
self.repository_el_re = args.repository.lstrip('^')
self.prf = review.ReviewesFetcher(args.base_url, args.repository)
def extract_and_compare(self, name):
input_pr, xtrd_ref = load_change(name)
pr_fetcher = pullrequest.PRsFetcher(None, 'https://github.com', None, None)
xtrd = pr_fetcher.extract_objects([input_pr], None)
ddiff = DeepDiff(xtrd_ref, xtrd, ignore_order=True)
if ddiff:
raise DiffException(ddiff)
def test_load_buggy(self):
"""
Github crawler extracts buggy prs
"""
pr_fetcher = pullrequest.PRsFetcher(None, 'https://github.com', None, None)
datasets_dir = Path(DATASETS)
for fn in datasets_dir.glob('github_*.json'):
dataset = load_dataset(fn)
xtrd = pr_fetcher.extract_objects([dataset], None)
self.assertNotEqual(xtrd, [])
def test_load_buggy(self):
"""
Github crawler extracts buggy prs
"""
pr_fetcher = pullrequest.PRsFetcher(None, "https://github.com", None,
None)
datasets_dir = Path(DATASETS)
for fn in datasets_dir.glob("github_*.json"):
self.log.info("Loading buggy PR from %s " % fn)
dataset = load_dataset(fn)
xtrd = pr_fetcher.extract_objects([dataset], lambda x, y: None)
self.assertNotEqual(xtrd, [])
def extract_and_compare(self, name: str) -> None:
input_pr, xtrd_ref = load_change(name)
pr_fetcher = pullrequest.PRsFetcher(None, "https://github.com", None,
None)
xtrd: List[Union[Change,
Event]] = pr_fetcher.extract_objects([input_pr], None)
ddiff = DeepDiff(
xtrd_ref,
[change_or_event_to_dict(x) for x in xtrd],
ignore_order=True,
)
if ddiff:
raise DiffException(ddiff)
def __init__(self, args):
self.updated_since = args.updated_since
self.loop_delay = int(args.loop_delay)
self.get_one = getattr(args, 'id', None)
self.db = ELmonocleDB()
if args.command == 'github_crawler':
self.get_one_rep = getattr(args, 'repository', None)
self.org = args.org
self.repository_el_re = args.org.lstrip('^') + '.*'
self.prf = pullrequest.PRsFetcher(
GithubGraphQLQuery(args.token),
args.host, args.org)
elif args.command == 'gerrit_crawler':
self.repository_el_re = args.repository.lstrip('^')
self.prf = review.ReviewesFetcher(
args.host, args.repository)