def test_adie_events2dicts(self) -> None:
"""Basic usage."""
collection = adie_events.adie_events2dicts(
path.join(path.dirname(__file__), 'testdata/adie-events.json'))
event_protos = dict(
mongo.collection_to_proto_mapping(collection, event_pb2.Event))
self.assertLess({'a0w1W00000Mdkz2QAB', 'a0w1W00000MdkaRQAR'},
set(event_protos))
self.assertEqual(13, len(event_protos), msg=event_protos)
event = event_protos['a0w1W00000Mdkz2QAB']
self.assertEqual("Atelier de la création d'entreprise", event.title)
self.assertEqual('Anglet', event.city_name)
self.assertEqual(
'***Ça parle de quoi ?***\n'
'\n'
'ENA accueil\n'
'\n'
'***Ça se passe où ?***\n'
'\n'
"Agence Adie d'Anglet\n"
"Résidence de l'Alliance - Centre Jorlis, 3, rue du Pont de l'Aveugle, 64600 Anglet\n"
'\n'
'***Quand ?***\n'
'\n'
'le lundi 23 septembre 2019 de 14h00 à 16h00\n', event.description)
self.assertEqual('le 23 septembre', event.timing_text)
self.assertEqual('2019-09-23', event.start_date)
self.assertAlmostEqual(43.499046, event.latitude, places=5)
event_special_date = event_protos['a0w1W00000MdkaRQAR']
self.assertEqual('le 06 août', event_special_date.timing_text)
self.assertEqual('2019-08-06', event_special_date.start_date)
def test_events2dicts(self) -> None:
"""Basic usage."""
json.dump([
{
'title': 'Event A',
'organiser': 'Pascal',
'latitude': 45.75,
'longitude': 4.85,
'price': 0,
'id': 'event-a',
'slug': 'slug-event-a',
'date': '2017-08-19',
'category': ['Trouver un job'],
},
], self.events_file)
self.events_file.close()
collection = workup_events.events2dicts(
self.events_json_path,
io.StringIO('''departement_id,max_latitude,max_longitude,min_latitude,min_longitude
38,45.8667,6.18333,44.75,4.76667
69,46.2833,5.1116,45.45,4.3
75,48.86,2.34445,48.86,2.34445
'''))
event_protos = dict(mongo.collection_to_proto_mapping(
collection, event_pb2.Event))
self.assertEqual({'event-a'}, set(event_protos))
event = event_protos['event-a']
self.assertEqual('Event A', event.title)
self.assertEqual('https://www.workuper.com/events/slug-event-a', event.link)
self.assertEqual(['for-departement(38,69)'], event.filters)
def test_csv2dicts(self):
"""Test basic usage of the csv2dicts function."""
offers = reorient_jobbing.csv2dicts(self.market_score, self.offers_csv,
self.rome_job_groups,
self.item_arborescence,
self.referentiel_apellation)
offers_proto = dict(
mongo.collection_to_proto_mapping(
offers, reorient_jobbing_pb2.LocalJobbingStats))
self.assertEqual(['45'], list(offers_proto.keys()))
loiret_offers = offers_proto['45']
# The maximum number of jobs required is 5.
self.assertEqual(5, len(loiret_offers.departement_job_stats.jobs))
self.assertEqual(
['A1401', 'N1101', 'N4104', 'N4102', 'A1413'],
[job.rome_id for job in loiret_offers.departement_job_stats.jobs])
self.assertEqual(
[69, 61, 60, 56, 55],
[job.offers for job in loiret_offers.departement_job_stats.jobs])
self.assertEqual(
'Aide arboricole',
loiret_offers.departement_job_stats.jobs[1].feminine_name)
self.assertEqual('Maître / Maîtresse de chai',
loiret_offers.departement_job_stats.jobs[3].name)
self.assertEqual(
'Maître de chai',
loiret_offers.departement_job_stats.jobs[3].masculine_name)
def test_basic_usage(self) -> None:
"""Basic usage."""
requirements = job_offers_requirements.csv2dicts(
path.join(self.testdata_folder, 'job_offers.csv'),
path.join(self.testdata_folder, 'column_names.txt'))
requirements_proto = dict(
mongo.collection_to_proto_mapping(requirements,
job_pb2.JobRequirements))
f1106 = requirements_proto['F1106']
self.assertEqual(2, len(f1106.diplomas), f1106.diplomas)
self.assertEqual('Bac+2', f1106.diplomas[1].name)
self.assertEqual(job_pb2.BTS_DUT_DEUG, f1106.diplomas[1].diploma.level)
self.assertEqual(25, f1106.diplomas[1].percent_suggested)
self.assertEqual(0, f1106.diplomas[1].percent_required)
self.assertEqual([2],
[e.office_skills_level for e in f1106.office_skills])
self.assertEqual(11, f1106.office_skills[0].percent_suggested)
self.assertEqual(
[job_pb2.CAR],
[license.driving_license for license in f1106.driving_licenses])
self.assertEqual(11, f1106.driving_licenses[0].percent_suggested)
self.assertEqual(100, f1106.driving_licenses[0].percent_required)
self.assertEqual([
job_pb2.CDD_OVER_3_MONTHS, job_pb2.CDI,
job_pb2.CDD_LESS_EQUAL_3_MONTHS
], [e.contract_type for e in f1106.contract_types])
self.assertEqual(['10686', '11753', '10688', '16733', '19658'],
[j.code_ogr for j in f1106.specific_jobs])
self.assertEqual(44, f1106.specific_jobs[0].percent_suggested)
def test_get_cities_dicts(self, mock_get):
"""Basic usage."""
mock_get().json.return_value = json.loads('''[
{
"idinsee": "01001",
"nom": "Abergement-Clémenciat",
"url": "\\/territoires\\/auvergne-rhone-alpes\\/ain\\/abergement\\/1001\\/1"
},
{
"idinsee":"32013",
"nom":"Auch",
"url":"\\/territoires\\/occitanie\\/gers\\/auch\\/32013\\/12474"
},
{
"idinsee":"2A004",
"nom":"Ajaccio",
"url":"\\/territoires\\/corse\\/corse-du-sud\\/ajaccio\\/992004\\/37369"
}
]''')
cities = dict(mongo.collection_to_proto_mapping(
eterritoire.get_cities_dicts(),
association_pb2.SimpleLink))
self.assertEqual({'01001', '32013', '2A004'}, cities.keys())
self.assertEqual(
'/territoires/corse/corse-du-sud/ajaccio/992004/37369',
cities['2A004'].path)
def test_too_many_skills(self) -> None:
"""Make sure we do not get too many skills per rome prefix."""
base = airtablemock.Airtable('the-base', 'apikey42')
for i in range(25):
base.create(
'the-table', {
'name': f'Skill {i}',
'description':
'Chercher activement des idées pour aider les gens',
'TIME_TO_MARKET': 55,
'BREADTH_OF_JOBS': 82,
'JOB_SATISFACTION': 58,
'BETTER_INCOME': 33,
'NO_AUTOMATISATION': 36,
'value_score': i + 0.93,
'rome_prefixes': 'J12',
})
skills_per_rome_pefix = dict(
mongo.collection_to_proto_mapping(
skills_for_future.get_skills_per_rome_prefix(
'the-base', 'the-table'), skill_pb2.JobSkills))
self.assertEqual(
['Skill 24', 'Skill 23', 'Skill 22', 'Skill 21', 'Skill 20'],
[skill.name for skill in skills_per_rome_pefix['J12'].skills])
def test_adie_events2dicts(self):
"""Basic usage."""
collection = adie_events.adie_events2dicts(
path.join(path.dirname(__file__), 'testdata/adie-evenements.html'))
event_protos = dict(
mongo.collection_to_proto_mapping(collection, event_pb2.Event))
self.assertEqual({'2018-02_0', '2018-02_1'}, set(event_protos))
event = event_protos['2018-02_0']
self.assertEqual(
"Zoom sur le microcrédit et l'accompagnement de l'Adie.",
event.title)
self.assertEqual('Annecy', event.city_name)
self.assertEqual(
'***Ça parle de quoi ?***\n'
'\n'
"Posez vos questions, l'Adie vous répond ! En partenariat avec CMA. "
"Pour s'inscrire, appelez le 04 50 23 92 22.\n"
'\n'
'***Ça se passe où ?***\n'
'\n'
"Chambre des métiers et de l'artisanat \n"
'28, avenue de France, 74000 Annecy\n'
'\n'
'***Quand ?***\n'
'\n'
'le lundi 5 février \n'
'de 14h à 17h', event.description)
self.assertEqual('le 5 février', event.timing_text)
self.assertAlmostEqual(45.9100539, event.latitude, places=5)
def test_basic_usage(self, mock_emploi_store):
"""Basic usage."""
mock_resource = mock_emploi_store.Client().get_package().get_resource()
mock_resource.records.return_value = [
{
'DEPARTEMENT_CODE': '69',
'ROME_PROFESSION_CARD_CODE': 'A1234'
},
{
'DEPARTEMENT_CODE': '69',
'ROME_PROFESSION_CARD_CODE': 'A1234'
},
{
'DEPARTEMENT_CODE': '69',
'ROME_PROFESSION_CARD_CODE': 'A1234'
},
{
'DEPARTEMENT_CODE': '01',
'ROME_PROFESSION_CARD_CODE': 'A1234'
},
]
counts = recent_job_offers_count.download_and_count()
counts_proto = dict(
mongo.collection_to_proto_mapping(counts, job_pb2.LocalJobStats))
self.assertEqual(set(['69:A1234', '01:A1234']), set(counts_proto))
self.assertEqual(3, counts_proto['69:A1234'].num_available_job_offers)
self.assertEqual(1, counts_proto['01:A1234'].num_available_job_offers)
def test_csv2dicts(self):
"""Test basic usage of the csv2dicts function."""
offers = seasonal_jobbing.csv2dicts(self.offers_csv)
offers_proto = dict(
mongo.collection_to_proto_mapping(
offers, seasonal_jobbing_pb2.MonthlySeasonalJobbingStats))
self.assertEqual([3], list(offers_proto.keys()))
march_proto = offers_proto[3]
self.assertEqual(
['06'], [d.departement_id for d in march_proto.departement_stats])
first_departement_proto = march_proto.departement_stats[0]
self.assertEqual('06', first_departement_proto.departement_id)
self.assertEqual(793,
first_departement_proto.departement_seasonal_offers)
self.assertEqual(11, len(first_departement_proto.job_groups))
self.assertEqual('Personnel de cuisine',
first_departement_proto.job_groups[0].name.strip())
self.assertEqual(166, first_departement_proto.job_groups[0].offers)
self.assertEqual('G1602',
first_departement_proto.job_groups[0].rome_id)
def test_extract_offers_per_cities(self) -> None:
"""Basic usage of extract_offers_per_cities."""
cities = offers_per_city.extract_offers_per_cities(
self.offers_csv,
self.colnames_csv,
'2015-01-01',
data_folder=path.join(path.dirname(__file__), 'testdata'))
city_protos = dict(
mongo.collection_to_proto_mapping(cities,
commute_pb2.HiringCities))
self.assertEqual(['F1106'], list(city_protos.keys()))
f_cities = city_protos['F1106'].hiring_cities
self.assertEqual(['Parthenay', 'PARTHENAY-mod', 'Lyon'],
[h.city.name for h in f_cities])
offer_rates = [h.offers_per_inhabitant for h in f_cities]
self.assertEqual(sorted(offer_rates, reverse=True), offer_rates)
best_city = f_cities[0]
self.assertEqual('Parthenay', best_city.city.name)
self.assertEqual('79202', best_city.city.city_id)
self.assertEqual('79', best_city.city.departement_id)
self.assertEqual(10478, best_city.city.population)
self.assertEqual(2, best_city.offers)
self.assertAlmostEqual(2 / 10478,
best_city.offers_per_inhabitant,
places=5)
self.assertAlmostEqual(46.65, best_city.city.latitude, places=5)
self.assertAlmostEqual(-0.25, best_city.city.longitude, places=5)
# Test that the arrondissement gets removed for LYON 06
self.assertEqual('Lyon', f_cities[2].city.name)
def test_csv2dicts(self):
"""Test basic usage of the csv2dicts function."""
missions = civic_service.csv2dicts(self.civic_service_missions_csv,
self.today)
missions_proto = dict(
mongo.collection_to_proto_mapping(
missions, association_pb2.VolunteeringMissions))
self.assertEqual(['45', '59', '63'],
sorted(list(missions_proto.keys())))
loiret_missions = missions_proto['45']
# The maximum number of missions required is 5.
self.assertEqual(5, len(loiret_missions.missions))
self.assertEqual([
'Solidarité', 'Éducation pour tous', 'Solidarité', 'Solidarité',
'Environnement'
], [mission.domain for mission in loiret_missions.missions])
self.assertEqual([
'Direction départementale des finances publiques de la charente',
'Association brin de ficelle', 'Ufcv auvergne/rhône-alpes',
'Pole emploi midi pyrenees', 'Mairie de paris'
], [mission.association_name for mission in loiret_missions.missions])
self.assertEqual('1ᵉʳ mars 2018',
loiret_missions.missions[1].start_date)
self.assertEqual(
'Contribuer à la qualité des relations des services ' +
'des finances publiques avec leurs usagers',
loiret_missions.missions[0].title)
def test_get_cities_dicts(self, mock_requests: 'requests_mock._RequestObjectProxy') -> None:
"""Basic usage."""
mock_requests.get(
'http://www.eterritoire.fr/webservice/listeCommunes.php',
json=json.loads('''[
{
"idinsee": "01001",
"nom": "Abergement-Clémenciat",
"url": "\\/territoires\\/auvergne-rhone-alpes\\/ain\\/abergement\\/1001\\/1"
},
{
"idinsee":"32013",
"nom":"Auch",
"url":"\\/territoires\\/occitanie\\/gers\\/auch\\/32013\\/12474"
},
{
"idinsee":"2A004",
"nom":"Ajaccio",
"url":"\\/territoires\\/corse\\/corse-du-sud\\/ajaccio\\/992004\\/37369"
}
]'''))
cities = dict(mongo.collection_to_proto_mapping(
eterritoire.get_cities_dicts(),
association_pb2.SimpleLink))
self.assertEqual({'01001', '32013', '2A004'}, cities.keys())
self.assertEqual(
'/territoires/corse/corse-du-sud/ajaccio/992004/37369',
cities['2A004'].path)
def test_csv2dicts(self) -> None:
"""Test basic usage of the csv2dicts function."""
collection = rome_mobility.csv2dicts(self.rome_csv_pattern)
protos = dict(
mongo.collection_to_proto_mapping(collection,
discovery_pb2.JobsExploration))
# Point checks.
self.assertEqual([
'10976', '10979', '10992', '10994', '11005', '11010', '11036',
'11037', '11044', '11046', '11047', '11052', '11059', '15441',
'F1402', 'G1102', 'G1201', 'G1202'
], sorted(protos))
g1202_proto = protos['G1202']
self.assertEqual(
['G1201', 'G1203', 'G1403', 'K1206', 'K1601', 'K2105', 'K2111'],
sorted(g.job_group.rome_id for g in g1202_proto.job_groups))
g1403 = [
g for g in g1202_proto.job_groups if g.job_group.rome_id == 'G1403'
].pop()
self.assertEqual(
"Gestion de structure de loisirs ou d'hébergement touristique",
g1403.job_group.name)
self.assertEqual(discovery_pb2.EVOLUTION, g1403.mobility_type)
samples = [j.code_ogr for j in g1403.job_group.samples]
self.assertEqual(3, len(samples))
self.assertEqual(3, len(set(samples)))
self.assertLess(
set(samples),
set([
'14216', '14217', '14234', '14237', '14301', '14322', '14326',
'14356', '14418', '14969', '14970', '14357'
]))
self.assertEqual(3, len(set(j.name for j in g1403.job_group.samples)))
# Check exploration suggestion for a job group.
k1601 = [
g.job_group for g in g1202_proto.job_groups
if g.job_group.rome_id == 'K1601'
].pop()
self.assertEqual(['16228', '38584'],
sorted(j.code_ogr for j in k1601.samples))
self.assertEqual([u'Ludothécaire', u'Responsable de ludothèque'],
sorted(j.name for j in k1601.samples))
# Check exploration suggestions for a single job.
_10979_proto = protos['10979']
self.assertEqual(['E1101'],
sorted(g.job_group.rome_id
for g in _10979_proto.job_groups))
self.assertEqual('Animation de site multimédia',
_10979_proto.job_groups[0].job_group.name)
def test_collection_to_proto_mapping_wrong_field(self):
"""Use of unknown proto field in a dict."""
iterator = mongo.collection_to_proto_mapping([
{
'_id': '75056',
'unkownField': 'Pascal'
},
], user_pb2.User)
self.assertRaises(mongo.json_format.ParseError, next, iterator)
def test_collection_to_proto_mapping_hidden_field(self) -> None:
"""Use of hidden unknown proto field in a dict."""
iterator = mongo.collection_to_proto_mapping([
{
'_id': '75056',
'_hiddenField': 'Pascal'
},
], test_pb2.User)
user_id, unused_user = next(iterator)
self.assertEqual('75056', user_id)
def test_make_dicts(self):
"""Basic usage."""
dicts = regions.make_dicts(
path.join(_TESTDATA_FOLDER, 'geo/insee_france_regions.tsv'),
path.join(_TESTDATA_FOLDER, 'geo/region_prefix.tsv'))
protos = dict(mongo.collection_to_proto_mapping(dicts, geo_pb2.Region))
self.assertEqual({'84'}, protos.keys())
self.assertEqual('en ', protos['84'].prefix)
self.assertEqual('Auvergne-Rhône-Alpes', protos['84'].name)
def test_basic(self) -> None:
"""Basic usage."""
found_las = dict(
mongo.collection_to_proto_mapping(
local_authorities.csv2dicts(self.testdata_wards),
geo_pb2.Departement))
self.assertEqual(6, len(found_las), msg=found_las.keys())
# Point check.
barking = found_las['E09000002']
self.assertEqual('Barking and Dagenham', barking.name, msg=barking)
self.assertEqual('in ', barking.prefix, msg=barking)
def test_csv2dicts(self):
"""Test basic usage of the csv2dicts function."""
collection = city_locations.csv2dicts(self.stats_filename)
protos = dict(
mongo.collection_to_proto_mapping(collection, geo_pb2.FrenchCity))
self.assertEqual(11, len(protos))
# Point check.
city = protos['39001']
self.assertAlmostEqual(47.0667, city.latitude, places=5)
self.assertAlmostEqual(5.38333, city.longitude, places=5)
def test_csv2dicts(self):
"""Test basic usage of the csv2dicts function."""
collection = job_offers.csv2dicts(self.job_offers_csv, self.colnames_csv)
self.assertEqual(4, len(collection))
collection_proto = dict(mongo.collection_to_proto_mapping(
collection, job_offer_counts_pb2.JobOfferCounts))
# Point check, csv designed to match these numbers.
city_id = '79202'
city = collection_proto['F1106:c' + city_id]
self.assertEqual(city_id, city.city.city_id)
self.assertEqual(2, city.city_count)
self.assertEqual(6, city.departement_count)
self.assertEqual(7, city.region_count)
def test_csv2dicts(self):
"""Test basic usage of the csv2dicts function."""
collection = unverified_data_zones.csv2dicts(self.testdata_folder)
protos = dict(mongo.collection_to_proto_mapping(
collection, user_pb2.UnverifiedDataZone))
self.assertEqual(533, len(protos))
# Point check.
key = hashlib.md5('03120:A1301'.encode('utf-8')).hexdigest()
unverified_zone_entry = protos[key]
self.assertEqual('03120', unverified_zone_entry.postcodes)
self.assertEqual('A1301', unverified_zone_entry.rome_id)
def test_make_dicts(self):
"""Basic usage."""
dicts = departements.make_dicts(
path.join(_TESTDATA_FOLDER, 'geo/insee_france_departements.tsv'),
path.join(_TESTDATA_FOLDER,
'geo/insee_france_oversee_collectivities.tsv'),
path.join(_TESTDATA_FOLDER, 'geo/departement_prefix.tsv'))
protos = dict(
mongo.collection_to_proto_mapping(dicts, geo_pb2.Departement))
self.assertEqual({'01', '69', '74'}, protos.keys())
self.assertEqual('en ', protos['74'].prefix)
self.assertEqual('Rhône', protos['69'].name)
def test_collection_to_proto_mapping_dupes(self):
"""Use of duplicates in the collection."""
iterator = mongo.collection_to_proto_mapping([
{
'_id': '75056',
'userId': 'Pascal'
},
{
'_id': '75056',
'userId': 'Pascal'
},
], user_pb2.User)
next(iterator)
self.assertRaises(KeyError, next, iterator)
def test_get_skills_per_rome_prefix(self) -> None:
"""Test the importer for skill for future."""
base = airtablemock.Airtable('the-base', 'apikey42')
base.create(
'the-table', {
'name': 'Orientation client',
'description':
'Chercher activement des idées pour aider les gens',
'TIME_TO_MARKET': 55,
'BREADTH_OF_JOBS': 82,
'JOB_SATISFACTION': 58,
'BETTER_INCOME': 33,
'NO_AUTOMATISATION': 36,
'value_score': 3.93,
'rome_prefixes': 'J12, B11, B15',
})
base.create(
'the-table', {
'name': 'Enseignement',
'description':
"Partager ses compétence et les enseigner à d'autres",
'TIME_TO_MARKET': 70,
'BREADTH_OF_JOBS': 67,
'JOB_SATISFACTION': 79,
'BETTER_INCOME': 54,
'NO_AUTOMATISATION': 21,
'value_score': 5.49,
'rome_prefixes': 'B11, B15, K21',
})
skills_per_rome_pefix = dict(
mongo.collection_to_proto_mapping(
skills_for_future.get_skills_per_rome_prefix(
'the-base', 'the-table'), skill_pb2.JobSkills))
self.assertEqual({'B11', 'B15', 'K21', 'J12'},
skills_per_rome_pefix.keys())
# Point checks
self.assertEqual(
['Enseignement', 'Orientation client'],
[skill.name for skill in skills_per_rome_pefix['B11'].skills])
self.assertEqual([
skill_pb2.JOB_SATISFACTION,
skill_pb2.TIME_TO_MARKET,
], skills_per_rome_pefix['B11'].skills[0].assets)
def test_get_missions_dict(
self, mock_check: mock.MagicMock,
mock_requests: 'requests_mock._RequestObjectProxy') -> None:
"""Basic usage."""
mock_requests.get(
'https://www.tousbenevoles.org/linkedin_webservice/xml/linkedin.xml',
headers={'user-agent': 'Bayes Impact collaboration'},
text=
('''
<jobs>
<job>
<JobTitle>Bénévolat : Cool Mission</JobTitle>
<JobId>12345</JobId>
<JobDescription>Mission proposée par Bayes Impact<br />'''
'''<b>Informations complémentaires</b>Nothing</JobDescription>
<applyURL>https://www.example.com/yes?param=true</applyURL>
<PostalCode>69006</PostalCode>
</job>
<job>
<JobTitle>Bénévolat : Cool Mission #2</JobTitle>
<JobId>4200</JobId>
<JobDescription>Mission proposée par Bayes Impact<br />'''
'''<b>Informations complémentaires</b>Nothing</JobDescription>
<applyURL>https://www.example.com/no</applyURL>
<PostalCode>75002</PostalCode>
</job>
</jobs>
'''))
# Coverage check is skipped for this test.
mock_check.return_value = True
missions = dict(
mongo.collection_to_proto_mapping(
volunteering_missions.get_missions_dicts(),
association_pb2.VolunteeringMissions))
self.assertTrue(mock_requests.called)
self.assertEqual({'69', '75'}, missions.keys())
self.assertEqual(['Cool Mission #2'],
[m.title for m in missions['75'].missions])
self.assertEqual('https://www.example.com/no?utm_source=bob-emploi',
missions['75'].missions[0].link)
self.assertEqual(
'https://www.example.com/yes?param=true&utm_source=bob-emploi',
missions['69'].missions[0].link)
def test_csv2dicts(self) -> None:
"""Test basic usage of the csv2dicts function."""
airtable = airtablemock.Airtable('app-mybase')
airtable.create('tbl-my-job-group-info', {
'soc_prefix': '543',
'covidRisk': 'COVID_RISKY',
})
collection = local_diagnosis.csv2dicts(
postings_csv=os.path.join(TESTDATA_FOLDER, 'emsi_postings_counts_2019_area4-occ4.csv'),
occupations_csv=os.path.join(
TESTDATA_FOLDER, 'emsi_occupation_counts_2019_area4-occ4.csv'),
jobs_xls=os.path.join(TESTDATA_FOLDER, 'soc/soc2010.xls'),
career_jumps_csv=os.path.join(TESTDATA_FOLDER, 'soc/career_changers_matrix.csv'),
info_by_prefix_airtable='app-mybase:tbl-my-job-group-info',
salary_filename=os.path.join(
TESTDATA_FOLDER, 'salaries_by_region_2020.xlsx'),
wards_ons_csv=os.path.join(TESTDATA_FOLDER, 'wards.csv'),
geonames_txt=os.path.join(TESTDATA_FOLDER, 'geonames.txt'),
geonames_admin_txt=os.path.join(TESTDATA_FOLDER, 'geonames_admin.txt')
)
protos = dict(mongo.collection_to_proto_mapping(
collection, job_pb2.LocalJobStats))
self.assertEqual(7, len(protos), msg=protos.keys())
# Point checks.
wolverhampton111 = protos['E08000031:1115']
self.assertEqual(2, wolverhampton111.imt.yearly_avg_offers_per_10_candidates)
self.assertEqual('Around £25,424', wolverhampton111.imt.median_salary.short_text)
self.assertEqual(25424, wolverhampton111.imt.median_salary.median_salary)
self.assertEqual(1, wolverhampton111.num_less_stressful_departements)
self.assertEqual(
['2111'], [jg.job_group.rome_id for jg in wolverhampton111.less_stressful_job_groups])
self.assertEqual(
['Chemical scientists'],
[jg.job_group.name for jg in wolverhampton111.less_stressful_job_groups])
# Check empty valued.
self.assertEqual(0, protos['E07000121:1115'].imt.yearly_avg_offers_per_10_candidates)
# Check zero-valued.
self.assertEqual(-1, protos['E07000120:1116'].imt.yearly_avg_offers_per_10_candidates)
# Check saved by the posting durations.
self.assertEqual(4, protos['E07000119:1115'].imt.yearly_avg_offers_per_10_candidates)
def test_collection_to_proto_mapping(self):
"""Basic usage of collection_to_proto_mapping function."""
protos = dict(
mongo.collection_to_proto_mapping([
{
'_id': '75056',
'userId': 'Pascal'
},
{
'_id': '69123',
'userId': 'Stephan'
},
], user_pb2.User))
self.assertEqual(['69123', '75056'], sorted(protos.keys()))
self.assertEqual('Pascal', protos['75056'].user_id)
self.assertEqual('Stephan', protos['69123'].user_id)
def json2dicts(events_file_name: str, french_regions_tsv: str,
prefix_tsv: str) -> list[dict[str, Any]]:
"""Convert salons from pole-emploi API to json compatible with
online_salon_pb2.OnlineSalon proto.
"""
# Rules are defined here: https://airtable.com/tbl6eAgUh8JGoiYnp/viwO0TJnWjTPexmsS
rules = [
_OnlineSalonRule(rule, french_regions_tsv, prefix_tsv)
for _, rule in mongo.collection_to_proto_mapping(
airtable_to_protos.airtable2dicts(collection_name='localisation',
base_id='appXmyc7yYj0pOcae',
table='tbl6eAgUh8JGoiYnp',
proto='SalonFilterRule',
view='Ready to Import'),
online_salon_pb2.SalonFilterRule)
]
with open(events_file_name, encoding='utf-8') as json_data:
salons = typing.cast(list[dict[str, Any]], json.load(json_data))
for salon in salons:
salon['start_date'] = _isodate_from_string(salon['dateDebut'])
salon['application_start_date'] = _isodate_from_string(
salon['dateDebutCandidature'])
salon['application_end_date'] = _isodate_from_string(
salon['dateFinCandidature'], is_end_of_day=True)
salon['locations'] = _get_city(
french_regions_tsv, prefix_tsv,
typing.cast(str, salon.get('localisation', '')))
salon = _aggregate_rule_results(salon, rules)
if not salon['locations']:
logging.warning('Missing locations on salon\n%s', salon)
# TODO(cyrille): Add test for not missing case.
if not salon.get('jobGroupIds'):
logging.warning('Missing job groups on salon\n%s', salon)
for old, new in _FIELD_RENAMER.items():
try:
salon[new] = salon.pop(old)
except KeyError:
continue
for field in _FIELDS_TO_DROP:
salon.pop(field, None)
return salons
def test_csv2dicts(self) -> None:
"""Test basic usage of the csv2dicts function."""
jobs = reorient_jobbing.csv2dicts(
self.job_zones_tsv, self.occupation_names_txt,
self.market_scores_csv)
jobs_proto = dict(mongo.collection_to_proto_mapping(
jobs, reorient_jobbing_pb2.LocalJobbingStats))
self.assertIn(36061, list(jobs_proto.keys()))
manhattan_jobs = jobs_proto[36061]
self.assertEqual(1, len(manhattan_jobs.departement_job_stats.jobs))
self.assertEqual(['35-2011'], [
job.rome_id for job in manhattan_jobs.departement_job_stats.jobs])
self.assertEqual(
'Cooks, Fast Food', manhattan_jobs.departement_job_stats.jobs[0].name)
def test_basic_usage(self):
"""Basic usage."""
changes = job_offers_changes.csv2dicts(
path.join(self.testdata_folder, 'job_offers.csv'),
path.join(self.testdata_folder, 'column_names.txt'))
changes_proto = dict(mongo.collection_to_proto_mapping(
changes, job_pb2.LocalJobStats))
self.assertEqual({'78:F1106', '79:F1106', '888:F1106'}, set(changes_proto))
proto = changes_proto['79:F1106']
self.assertEqual(500, proto.job_offers_change)
self.assertEqual(6, proto.num_job_offers_last_year)
self.assertEqual({2014: 1, 2015: 6}, proto.num_job_offers_per_year)
proto = changes_proto['78:F1106']
self.assertEqual({2015: 1}, proto.num_job_offers_per_year)
def test_bad_format_file(self) -> None:
"""Test with an input file wih a bad format."""
collection = local_diagnosis.csv2dicts(
hires_csv=path.join(_TESTDATA_FOLDER, 'emsi_hires_wrong.csv'),
job_seekers_csv=path.join(_TESTDATA_FOLDER,
'emsi_job_seekers_counts_dec_2019.csv'),
carreer_changers_tsv=path.join(
_TESTDATA_FOLDER,
f'onet_{local_diagnosis.ONET_VERSION}/Career_Changers_Matrix.txt'
),
soc_definition_xls=path.join(_TESTDATA_FOLDER,
'soc/soc_2010_definitions.xls'))
with self.assertRaises(ValueError) as error:
dict(
mongo.collection_to_proto_mapping(collection,
job_pb2.LocalJobStats))
self.assertIn('Incorrect dataset format:', str(error.exception))
