def main(argv): params = argv if len(params) == 1: params = IrishNameParser().names else: params = argv[1:] counter = 0 profile_downloader = PublicProfileDownloader() for param in params: profile_downloader.download(param) counter += NUM if counter > 500: time.sleep(120) counter -= 500 DBHelper.commitAndClose()
def run(num, file_name): rg = RG(file_name) profile_paths = getPublicProfiles(limit=num) try: for path in profile_paths: parser = ProfileParser(path) profile = parser.parseHtml() # print profile.extra_profile_list # Utils.putExtraProfilesIntoDB(profile.extra_profile_list) rg.add(profile) DBHelper.dataSetRDF(profile.file_name, rdf=1) except Exception: traceback.print_exc() rg.save(format='xml', file_name=file_name) rg.close() DBHelper.commitAndClose() else: rg.save(format='xml', file_name=file_name) rg.close() DBHelper.commitAndClose()
