def main():
google_service_account_path = Path(__file__).parent / 'google_service_account.json'
google_service_account_json = os.getenv('GOOGLE_SERVICE_ACCOUNT') or google_service_account_path.read_text()
google_service_account = json.loads(google_service_account_json)
google_scope = ['https://spreadsheets.google.com/feeds',
'https://www.googleapis.com/auth/drive']
credentials = ServiceAccountCredentials.from_json_keyfile_dict(google_service_account, google_scope)
doc_key = '1TO5Yzk0-4V_RzRK5Jr9I_pF5knZsEZrNn2HKTXrHgls'
doc = gspread.authorize(credentials).open_by_key(doc_key)
records = doc.worksheet('jobs').get_all_records(default_blank=None)
with db:
for model in [Job, JobError, JobDropped]:
model.drop_table()
model.create_table()
for record in records:
Job.create(**coerce_record(record))
Pool().map(run_spider, [
'linkedin',
'stackoverflow',
'startupjobs',
])
def main():
doc_key = '1TO5Yzk0-4V_RzRK5Jr9I_pF5knZsEZrNn2HKTXrHgls'
records = download_sheet(doc_key, 'jobs')
with db:
for model in [Job, JobError, JobDropped]:
model.drop_table()
model.create_table()
for record in records:
Job.create(**coerce_record(record))
Pool().map(run_spider, [
'linkedin',
'stackoverflow',
'startupjobs',
])
def create_job(id, **kwargs):
return Job.create(
id=str(id),
timestamp=kwargs.get('timestamp', datetime(2019, 7, 6, 20, 24, 3)),
company_name=kwargs.get('company_name', 'Honza Ltd.'),
job_type=kwargs.get('job_type', 'internship'),
title=kwargs.get('title', 'Junior Software Engineer'),
company_link=kwargs.get('company_link', 'https://example.com'),
email=kwargs.get('email', '*****@*****.**'),
location=kwargs.get('location', 'Brno, Czech Republic'),
description=kwargs.get('description', '**Really long** description.'),
is_approved=kwargs.get('is_approved', True),
is_sent=kwargs.get('is_sent', False),
)
def create_job(id, **kwargs):
return Job.create(
id=str(id),
posted_at=kwargs.get('posted_at', datetime(2019, 7, 6, 20, 24, 3)),
company_name=kwargs.get('company_name', 'Honza Ltd.'),
employment_types=kwargs.get('employment_types',
frozenset(['internship'])),
title=kwargs.get('title', 'Junior Software Engineer'),
company_link=kwargs.get('company_link', 'https://example.com'),
email=kwargs.get('email', '*****@*****.**'),
location=kwargs.get('location', 'Brno, Czech Republic'),
description=kwargs.get('description', '**Really long** description.'),
source=kwargs.get('source', 'juniorguru'),
is_approved=kwargs.get('is_approved', True),
is_sent=kwargs.get('is_sent', False),
is_expired=kwargs.get('is_expired', False),
)
def create_job(id, **kwargs):
return Job.create(**prepare_job_data(id, **kwargs))