def main(): google_service_account_path = Path(__file__).parent / 'google_service_account.json' google_service_account_json = os.getenv('GOOGLE_SERVICE_ACCOUNT') or google_service_account_path.read_text() google_service_account = json.loads(google_service_account_json) google_scope = ['https://spreadsheets.google.com/feeds', 'https://www.googleapis.com/auth/drive'] credentials = ServiceAccountCredentials.from_json_keyfile_dict(google_service_account, google_scope) doc_key = '1TO5Yzk0-4V_RzRK5Jr9I_pF5knZsEZrNn2HKTXrHgls' doc = gspread.authorize(credentials).open_by_key(doc_key) records = doc.worksheet('jobs').get_all_records(default_blank=None) with db: for model in [Job, JobError, JobDropped]: model.drop_table() model.create_table() for record in records: Job.create(**coerce_record(record)) Pool().map(run_spider, [ 'linkedin', 'stackoverflow', 'startupjobs', ])
def main(): doc_key = '1TO5Yzk0-4V_RzRK5Jr9I_pF5knZsEZrNn2HKTXrHgls' records = download_sheet(doc_key, 'jobs') with db: for model in [Job, JobError, JobDropped]: model.drop_table() model.create_table() for record in records: Job.create(**coerce_record(record)) Pool().map(run_spider, [ 'linkedin', 'stackoverflow', 'startupjobs', ])
def create_job(id, **kwargs): return Job.create( id=str(id), timestamp=kwargs.get('timestamp', datetime(2019, 7, 6, 20, 24, 3)), company_name=kwargs.get('company_name', 'Honza Ltd.'), job_type=kwargs.get('job_type', 'internship'), title=kwargs.get('title', 'Junior Software Engineer'), company_link=kwargs.get('company_link', 'https://example.com'), email=kwargs.get('email', '*****@*****.**'), location=kwargs.get('location', 'Brno, Czech Republic'), description=kwargs.get('description', '**Really long** description.'), is_approved=kwargs.get('is_approved', True), is_sent=kwargs.get('is_sent', False), )
def create_job(id, **kwargs): return Job.create( id=str(id), posted_at=kwargs.get('posted_at', datetime(2019, 7, 6, 20, 24, 3)), company_name=kwargs.get('company_name', 'Honza Ltd.'), employment_types=kwargs.get('employment_types', frozenset(['internship'])), title=kwargs.get('title', 'Junior Software Engineer'), company_link=kwargs.get('company_link', 'https://example.com'), email=kwargs.get('email', '*****@*****.**'), location=kwargs.get('location', 'Brno, Czech Republic'), description=kwargs.get('description', '**Really long** description.'), source=kwargs.get('source', 'juniorguru'), is_approved=kwargs.get('is_approved', True), is_sent=kwargs.get('is_sent', False), is_expired=kwargs.get('is_expired', False), )
def create_job(id, **kwargs): return Job.create(**prepare_job_data(id, **kwargs))