def test_duplicates(self): """Avoid creating duplicate rows for the same translation.""" airtable.Airtable('appkEc8N0Bw4Uok43', '').create('translations', { 'string': 'Already translated', }) bob_advice_base = airtable.Airtable('appXmyc7yYj0pOcae', '') bob_advice_base.create( 'advice_modules', { 'advice_id': 'first-advice', 'title': 'Already translated', 'explanations': 'New translation', }) bob_advice_base.create('advice_modules', { 'advice_id': 'second-advice', 'title': 'New translation', }) collect_strings.main('apikey') translations = \ list(airtable.Airtable('appkEc8N0Bw4Uok43', '').iterate('translations')) self.assertEqual( ['Already translated', 'New translation'], sorted(t.get('fields', {}).get('string') for t in translations))
def test_main(self): """Basic usage.""" airtable.Airtable('appkEc8N0Bw4Uok43', '').create('translations', { 'string': 'Already translated', }) upload_pot.main( path.join(path.dirname(__file__), 'testdata/frontend-server-strings.pot'), 'api-key') translations = \ list(airtable.Airtable('appkEc8N0Bw4Uok43', '').iterate('translations')) self.assertEqual( ['Already translated', 'New message to translate'], sorted(t.get('fields', {}).get('string') for t in translations)) new_message = next( t for t in translations if t['fields']['string'] == 'New message to translate') self.assertEqual('frontend-server-strings.pot', new_message['fields']['origin']) self.assertEqual('bob_emploi/frontend/server/scoring.py#70', new_message['fields']['origin_id'])
def test_namespace_prefix(self) -> None: """Add namespace prefix before uploading.""" airtable.Airtable('appkEc8N0Bw4Uok43', '').create( 'translations', { 'string': 'Already translated', 'en': 'Already translated in English', }) airtable.Airtable('appkEc8N0Bw4Uok43', '').create( 'translations', { 'string': 'frontend-client-strings:Already translated', 'en': 'Already translated in English', }) upload_strings_to_translate.main( (path.join( path.dirname(__file__), 'testdata/strings/frontend-client-strings.json'), '--api-key', 'api-key', '--lang', 'fr', '--prefix-with-namespace'), ) translations = \ list(airtable.Airtable('appkEc8N0Bw4Uok43', '').iterate('translations')) self.assertEqual([ 'Already translated', 'frontend-client-strings:Already translated', 'frontend-client-strings:New message to translate' ], sorted(t.get('fields', {}).get('string') for t in translations)) new_message = next(t for t in translations if t['fields']['string'] == 'frontend-client-strings:New message to translate') self.assertEqual('New message to translate in French', new_message['fields'].get('fr'))
def test_main(self): """Test collection of strings in various tables and fields.""" bob_advice_base = airtable.Airtable('appXmyc7yYj0pOcae', '') bob_advice_base.create( 'advice_modules', { 'advice_id': 'first-advice', 'explanations (for client)': 'my explanation', 'title': 'First Advice', }) bob_advice_base.create('advice_modules', { 'advice_id': 'second-advice', 'title': 'Second Advice', }) bob_advice_base.create('diagnostic_sentences', { 'sentence_template': 'A sentence template', }) collect_strings.main('apikey') translations = \ list(airtable.Airtable('appkEc8N0Bw4Uok43', '').iterate('translations')) self.assertEqual([ 'A sentence template', 'First Advice', 'Second Advice', 'my explanation' ], sorted(t.get('fields', {}).get('string') for t in translations))
def test_strings_from_json(self) -> None: """Upload translatable strings from JSON file.""" airtable.Airtable('appkEc8N0Bw4Uok43', '').create('translations', { 'string': 'Already translated', }) upload_strings_to_translate.main( (path.join(path.dirname(__file__), 'testdata/strings/frontend-client-strings.json'), '--api-key', 'api-key'), ) translations = \ list(airtable.Airtable('appkEc8N0Bw4Uok43', '').iterate('translations')) self.assertEqual( ['Already translated', 'New message to translate'], sorted(t.get('fields', {}).get('string') for t in translations)) new_message = next( t for t in translations if t['fields']['string'] == 'New message to translate') self.assertEqual('frontend-client-strings.json', new_message['fields']['origin']) self.assertEqual('1', new_message['fields']['origin_id'])
def test_duplicates(self) -> None: """Avoid creating duplicate rows for the same translation.""" airtable.Airtable('appkEc8N0Bw4Uok43', '').create('translations', { 'string': 'Already translated', }) bob_advice_base = airtable.Airtable('appXmyc7yYj0pOcae', '') bob_advice_base.create( 'advice_modules', { 'advice_id': 'first-advice', 'title': 'Already translated', 'explanations': 'New translation', 'trigger_scoring_model': 'constant(2)', }) bob_advice_base.create( 'advice_modules', { 'advice_id': 'second-advice', 'title': 'New translation', 'trigger_scoring_model': 'constant(2)', }) collect_strings.main(['apikey']) self._assert_all_translations([ 'Already translated', 'New translation', 'adviceModules:first-advice:title', 'adviceModules:second-advice:title', ])
def test_multiple_bases(self) -> None: """Test using the same table name in 2 different bases.""" base1 = airtable.Airtable('first-base', '') base1.create('table', {'number': 6}) base2 = airtable.Airtable('second-base', '') base2.create('table', {'number': 7}) records = base1.iterate('table') self.assertEqual([6], [record['fields']['number'] for record in records])
def test_multiple_clients(self) -> None: """Test accessing a table from different clients.""" base1 = airtable.Airtable('first-base', '') base1.create('table', {'number': 8}) other_client_base1 = airtable.Airtable('first-base', '') other_client_base1.create('table', {'number': 9}) records = base1.iterate('table') self.assertEqual([8, 9], [record['fields']['number'] for record in records])
def _load_domains_from_airtable(base_id, table, view=None): """Load domain data from AirTable. Args: base_id: the ID of your AirTable app. table: the name of the table to import. Returns: A map from sector names to domain names. """ if not AIRTABLE_API_KEY: raise ValueError( 'No API key found. Create an airtable API key at ' 'https://airtable.com/account and set it in the AIRTABLE_API_KEY ' 'env var.') client = airtable.Airtable(base_id, AIRTABLE_API_KEY) domains = {} errors = [] for record in client.iterate(table, view=view): fields = record['fields'] sector = fields.get('name') if not sector: continue domain = fields.get('domain_name') if not domain: errors.append(ValueError( 'Sector "{}" on record "{}" has no domain_name set.' .format(sector, record['id']))) continue domains[sector] = domain if errors: raise ValueError('{:d} errors while importing from Airtable:\n{}'.format( len(errors), '\n'.join(str(error) for error in errors))) return domains
def _load_crosswalk_airtable(base_id: str, table: str, view: Optional[str] = None) -> pd.DataFrame: """Load FAP -> SOC crosswalk from AirTable. Args: base_id: the ID of your AirTable app. table: the name of the table to import. """ api_key = os.getenv('AIRTABLE_API_KEY') if not api_key: raise ValueError( 'No API key found. Create an airtable API key at ' 'https://airtable.com/account and set it in the AIRTABLE_API_KEY ' 'env var.') client = airtable.Airtable(base_id, api_key) mappings: list[Tuple[str, str]] = [] errors: list[KeyError] = [] for record in client.iterate(table, view=view): try: for fap_prefix in record['fields']['FAP prefixes']: mappings.append( (record['fields']['O*NET-SOC Code'], fap_prefix)) except KeyError as error: errors.append(error) if errors: raise KeyError( f'{len(errors):d} errors while importing from Airtable:\n' + '\n'.join(str(error) for error in errors)) return pd.DataFrame(mappings, columns=('soc_2018', 'fap_prefix'))
def airtable2dicts(base_id, table, proto, view=None): """Import the suggestions in MongoDB. Args: base_id: the ID of your AirTable app. table: the name of the table to import. proto: the name of the proto type. view: optional - the name of the view to import. Returns: an iterable of dict with the JSON values of the proto. """ converter = PROTO_CLASSES[proto] api_key = os.getenv('AIRTABLE_API_KEY') if not api_key: raise ValueError( 'No API key found. Create an airtable API key at ' 'https://airtable.com/account and set it in the AIRTABLE_API_KEY ' 'env var.') client = airtable.Airtable(base_id, api_key) records = list(client.iterate(table, view=view)) previous_key = None for record in records: sort_key = converter.sort_key(record) if previous_key is not None and sort_key < previous_key: raise ValueError( 'Records are not sorted properly: go to Airtable and apply the sorting for the ' 'view.') previous_key = sort_key proto_records = [converter.convert_record(r) for r in records] return validate(proto_records, converter.proto_type)
def main( api_key, base_id='appMRMtWV61Kibt37', table='Job Groups', field='Networking as first application mode', data_folder='data'): """Update an AirTable field based on IMT data. Args: api_key: the API key to access AirTable (see https://airtable.com/account). base_id: the ID of the AirTable base to update (see https://airtable.com/api). table: the name of the AirTable table to update. field: the name of the AirTable field to update. data_folder: the folder containing the scraped IMT data. """ imt = cleaned_data.scraped_imt(data_folder) # Create the set of ROME ID of job groups that use network as their first # application mode. job_groups_use_network = set() for unused_index, row in imt.reset_index().drop_duplicates('rome_id').iterrows(): if not row.applicationModes: continue if any(m['first'] != 'PERSONAL_OR_PROFESSIONAL_CONTACTS' for m in row.applicationModes.values()): continue job_groups_use_network.add(row.rome_id) # Upload them to AirTable. client = airtable.Airtable(base_id, api_key) records = client.iterate(table) for record in records: if record.get('fields').get('code_rome') not in job_groups_use_network: continue client.update(table, record.get('id'), {field: True})
def test_space_checks( self, unused_mock_requests: 'requests_mock._RequestObjectProxy') -> None: """Check that translations are checked for whitespaces before being used.""" client = airtable.Airtable('appkEc8N0Bw4Uok43', '') record_id = next(record['id'] for record in client.iterate('tblQL7A5EgRJWhQFo') if record['fields']['string'].startswith( 'Ce message vous a été envoyé')) client.delete('tblQL7A5EgRJWhQFo', record_id) client.create( 'tblQL7A5EgRJWhQFo', { 'string': 'Ce message vous a été envoyé à la suite du recueil de vos données sur BOB ' 'EMPLOI. La fréquence des envois est limitée, mais peut être plus importante en ' 'fonction de l’action de l’association. Vous avez la possibilité d’exercer vos droits ' 'd’accès, de rectification et de suppression en écrivant à l’adresse suivante : ' '[email protected].', 'en': 'This message contains a space before an exclamation mark !', 'fr@tu': "Ce message t'a été envoyé", }) with self.assertRaises(ValueError) as error: translate_mailjet.main(('--lang', 'en', 'reset-password')) self.assertIn('a space before an exclamation mark** **!', str(error.exception) + str(error.exception.__cause__))
def test_get_max_records(self) -> None: """Test the max records feature of the get method.""" base = airtable.Airtable('base', '') base.create('table', {'number': 1}) base.create('table', {'number': 2}) base.create('table', {'number': 3}) base.create('table', {'number': 4}) base.create('table', {'number': 5}) response = base.get('table', limit=2, max_records=3) self.assertEqual( [1, 2], [record['fields']['number'] for record in response['records']]) self.assertEqual(2, response.get('offset')) response = base.get('table', limit=2, offset=typing.cast(int, response.get('offset')), max_records=3) self.assertEqual( [3], [record['fields']['number'] for record in response['records']]) self.assertNotIn('offset', response)
def test_client_collection(self) -> None: """Test collecting strings for client-side translation from a specific table.""" bob_advice_base = airtable.Airtable('appXmyc7yYj0pOcae', '') bob_advice_base.create( 'contact_lead', { 'card_content': 'Translation needed', 'email_template': "This is a templated email that doesn't need translation", 'name': 'I need to be translated', }) bob_advice_base.create('email_templates', { 'reason': "That's why", 'title': 'Please, translate me', }) bob_advice_base.create( 'advice_modules', { 'advice_id': 'first-advice', 'title': 'Title to translate', 'static_explanations': 'New translation', 'explanations': 'No translation needed', }) collect_strings.main( ['apiKey', '--collection', 'client-advice_modules']) self._assert_all_translations([ 'adviceModules:first-advice:static_explanations', 'adviceModules:first-advice:title' ])
def test_strings_from_test_files(self) -> None: """Do not upload strings from test files.""" airtable.Airtable('appkEc8N0Bw4Uok43', '').create('translations', { 'string': 'Already translated', }) upload_pot.main( path.join(path.dirname(__file__), 'testdata/frontend-server-strings-test.pot'), 'api-key') translations = \ list(airtable.Airtable('appkEc8N0Bw4Uok43', '').iterate('translations')) self.assertEqual( ['Already translated', 'New message to translate in test and in prod'], sorted(t.get('fields', {}).get('string') for t in translations))
def index(): # get the airtable-database at = airtable.Airtable('appG0JiXjmaxZgcQ7', 'key1GfTb9F6DsLhys') # for all categories add one thingy to the cat list allItems = at.get('Items').get('records') cat = [] for category in at.get('Category').get('records'): catItems = [] # add the items which are in this category to the item-list for i in allItems: if i['fields'].get('Category')[0] == category['id']: # if there is a blog for this item, then add it b = i['fields'].get('BlogText') if b is None: b='' else: b=b[0] catItems.append({'id': i['fields'].get('ID'), 'name': i['fields'].get('Name'), 'weight': i['fields'].get('Weight'), 'time': i['fields'].get('TimeID')[0], 'blog': b}) # add id, name and all items of this category to the cat-list cat.append( {'id': category['fields'].get('ID'), 'name': category['fields'].get('Name'), 'catItems': catItems}) return render_template('calculator.html', categories=cat)
def _load_items_from_airtable( proto_name: str, job_groups: Iterable[str], airtable_connection: Optional[str], rome_prefix_field: str) \ -> Optional[Mapping[str, List[Dict[str, Any]]]]: if not airtable_connection: return None parts = airtable_connection.split(':') if len(parts) <= 2: base_id, table = parts view = None else: base_id, table, view = parts items: Dict[str, List[Dict[str, Any]]] = {job_group: [] for job_group in job_groups} converter = airtable_to_protos.PROTO_CLASSES[proto_name] client = airtable.Airtable(base_id, AIRTABLE_API_KEY) for record in client.iterate(table, view=view): item = converter.convert_record(record) del item['_id'] job_group_prefixes = record['fields'].get(rome_prefix_field, '') for job_group_prefix in job_group_prefixes.split(','): job_group_prefix = job_group_prefix.strip() if not job_group_prefix: continue for job_group, items_for_group in items.items(): if job_group.startswith(job_group_prefix): items_for_group.append(item) return items
def _load_strict_diplomas_from_airtable(base_id: str, table: str, view: Optional[str] = None ) -> pandas.Series: """Load strict requirement for diplomas data from AirTable. Args: base_id: the ID of your AirTable app. table: the name of the table to import. Returns: A series indexed on ROME code with boolean values. """ if not AIRTABLE_API_KEY: raise ValueError( 'No API key found. Create an airtable API key at ' 'https://airtable.com/account and set it in the AIRTABLE_API_KEY ' 'env var.') client = airtable.Airtable(base_id, AIRTABLE_API_KEY) diploma_required = { record['fields'].get('code_rome'): record['fields'].get('is_diploma_strictly_required', False) for record in client.iterate(table, view=view) } return pandas.Series(diploma_required)
def main(api_key: str, base_id: str = 'appMRMtWV61Kibt37', table: str = 'VAE_Stats', data_folder: str = 'data', filename: str = '', sheetname: str = 'Figure 5 web') -> None: """Update an AirTable field based on data from XLS file.""" if not filename: filename = path.join(data_folder, 'vae-2018.xls') file = pd.ExcelFile(path.join(data_folder, filename)) vae_stats = file.parse(sheetname, header=1).dropna() vae_stats.set_index(vae_stats.columns[0], inplace=True) diplomas = { diploma.Index: diploma[-1] for diploma in vae_stats.itertuples() } # Upload them to AirTable. client = airtable.Airtable(base_id, api_key) records = client.iterate(table) for record in records: try: value = diplomas.pop(record.get('fields').get('name')) except KeyError: continue client.update(table, record.get('id'), {_FIELDNAME: value}) for name, value in diplomas.items(): client.create(table, {'name': name, _FIELDNAME: value})
def _load_assets_from_airtable(base_id, table, view=None): """Load assets data from AirTable. Args: base_id: the ID of your AirTable app. table: the name of the table to import. """ if not AIRTABLE_API_KEY: raise ValueError( 'No API key found. Create an airtable API key at ' 'https://airtable.com/account and set it in the AIRTABLE_API_KEY ' 'env var.') client = airtable.Airtable(base_id, AIRTABLE_API_KEY) assets = [] errors = [] for record in client.iterate(table, view=view): try: assets.append(_load_asset_from_airtable(record['fields'])) except ValueError as error: errors.append(error) if errors: raise ValueError('{:d} errors while importing from Airtable:\n{}'.format( len(errors), '\n'.join(str(error) for error in errors))) return dict(assets)
def test_invalid_record( self, mock_logging: mock.MagicMock, mock_requests: 'requests_mock._RequestObjectProxy') -> None: """Test collecting strings with a record not passing checks.""" mock_requests.post('https://slack.example.com/webhook') bob_advice_base = airtable.Airtable('appXmyc7yYj0pOcae', '') bob_advice_base.create( 'contact_lead', { 'card_content': ' Translation needed ', 'email_template': "This is a templated email that doesn't need translation", 'name': 'I need to be translated', }) collect_strings.main(['apiKey', '--collection', 'contact_lead']) self._assert_all_translations([]) self.assertTrue(mock_logging.call_count, msg=mock_logging.call_args_list) self.assertEqual(1, mock_requests.call_count) self.assertEqual( { 'attachments': [{ 'mrkdwn_in': ['text'], 'title': 'Automatic String Collect', 'text': 'Here is the report:\nAll the collections have been collected.' + '\nErrors in collection:\ncontact_lead: 1\n' }], }, mock_requests.request_history[0].json(), )
def test_delete_unused(self) -> None: """Delete unused translations.""" bob_advice_base = airtable.Airtable('appXmyc7yYj0pOcae', '') bob_advice_base.create('advice_modules', { 'title': 'Other module', 'advice_id': 'other' }) module = bob_advice_base.create( 'advice_modules', { 'advice_id': 'first-advice', 'static_explanations': 'Original text', 'title': 'Title to translate', }) collect_strings.main(['apiKey', '--collection', 'client']) bob_advice_base.delete('advice_modules', module['id']) bob_advice_base.create('advice_modules', { 'advice_id': 'second-advice', 'static_explanations': 'New text', }) collect_strings.main( ['apiKey', '--collection', 'client', '--unused', 'delete']) self._assert_all_translations([ 'adviceModules:other:title', 'adviceModules:second-advice:static_explanations', ])
def _load_prefix_info_from_airtable(job_groups, base_id, table, view=None): """Load info by prefix from AirTable. Args: job_groups: an iterable of job groups. base_id: the ID of your AirTable app. table: the name of the table to import. Returns: A pandas DataFrame keyed by job group with the fields. """ if not AIRTABLE_API_KEY: raise ValueError( 'No API key found. Create an airtable API key at ' 'https://airtable.com/account and set it in the AIRTABLE_API_KEY ' 'env var.') columns = ['inDomain'] info = pandas.DataFrame(index=job_groups, columns=columns) client = airtable.Airtable(base_id, AIRTABLE_API_KEY) sorted_records = sorted( client.iterate(table, view=view), key=lambda record: str(record['fields'].get('rome_prefix'))) for record in sorted_records: fields = record['fields'] rome_prefix = fields.get('rome_prefix') if not rome_prefix: continue for column in columns: if column not in fields: continue info.loc[info.index.str.startswith(rome_prefix), column] = fields[column] return info.fillna('')
def test_get_limit(self) -> None: """Test the limit feature of the get method.""" base = airtable.Airtable('base', '') base.create('table', {'number': 1}) base.create('table', {'number': 2}) base.create('table', {'number': 3}) base.create('table', {'number': 4}) base.create('table', {'number': 5}) response = base.get('table', limit=2) self.assertEqual( [1, 2], [record['fields']['number'] for record in response['records']]) self.assertEqual(2, response.get('offset')) response = base.get('table', limit=2, offset=2) self.assertEqual( [3, 4], [record['fields']['number'] for record in response['records']]) self.assertEqual(4, response.get('offset')) response = base.get('table', limit=2, offset=4) self.assertEqual( [5], [record['fields']['number'] for record in response['records']]) self.assertNotIn('offset', response)
def index(): if request.method == 'POST': noBottles = 18 * int(request.form['noBottles']) else: noBottles = 0 app.logger.warning('A warning occurred (%d apples)', 42) # formvar=request.args.get('formfield', '') at = airtable.Airtable('appG0JiXjmaxZgcQ7', 'key1GfTb9F6DsLhys') allItems = at.get('Items') cat = [] for category in at.get('Category').get('records'): catItems = [] # add the items which are in this category to the item-list for i in allItems.get('records'): if i['fields'].get('Category')[0] == category['id']: time = i['fields'].get('TimeID')[0] catItems.append({'id': i['fields'].get('ID'), 'name': i['fields'].get('Name'), 'weight': i['fields'].get('Weight'), 'time': i['fields'].get('TimeID')[0]}) # add id, name and all items of this category to the cat-list cat.append({'id': category['fields'].get('ID'), 'name': category['fields'].get('Name'), 'catItems': catItems}) return render_template('calculator_test.html', categories=cat)
def test_invalid_converter( self, mock_logging: mock.MagicMock, mock_requests: 'requests_mock._RequestObjectProxy') -> None: """Test collecting strings with a broken converter.""" mock_requests.post('https://slack.example.com/webhook') bob_advice_base = airtable.Airtable('appXmyc7yYj0pOcae', '') bob_advice_base.create('contact_lead', { 'title': 'Title to translate', }) collect_strings.main(['apiKey', '--collection', 'contact_lead']) self._assert_all_translations([]) mock_logging.assert_called_once() self.assertEqual(1, mock_requests.call_count) self.assertEqual( { 'attachments': [{ 'mrkdwn_in': ['text'], 'title': 'Automatic String Collect', 'text': 'Here is the report:\nAll the collections have been collected.' + '\nErrors in collection:\ncontact_lead: 1\n' }], }, mock_requests.request_history[0].json(), )
def extract_airtable_shows(): ''' Load shows from airtable into a Postgres table Airtable shows are identified by their IMDB ID. This task stores the IDs and ratings only. ''' hook = PostgresHook(postgres_conn_id='postgres_movies') base = airtable.Airtable(base_id, api_key) ratings = [] for record in base.iterate(shows_table): fields = record['fields'] rating = fields.get('Mat Rating') imdb_url = fields.get('IMDB') if rating is not None and validate_imdb_url(imdb_url): ratings.append((imdb_url, rating)) logger.info('Rated %d movies', len(ratings)) hook.insert_rows('my_ratings', rows=ratings, target_fields=('imdb_url', 'stars'))
def put_data(data): """ Puts the specified data in to the Airtable. Expects data to be in JSON """ at_write = airtable.Airtable(app.config['AIRTABLE_BASE'], app.config['AIRTABLE_WRITE_KEY']) return at_write.create(app.config['AIRTABLE_TABLE'] , data)
def test_collect_all(self, mock_post: mock.MagicMock) -> None: """Test collection of strings in various tables and fields.""" bob_advice_base = airtable.Airtable('appXmyc7yYj0pOcae', '') bob_advice_base.create( 'advice_modules', { 'advice_id': 'first-advice', 'explanations (for client)': 'my explanation', 'title': 'First Advice', }) bob_advice_base.create('advice_modules', { 'advice_id': 'second-advice', 'title': 'Second Advice', }) bob_advice_base.create('diagnostic_sentences', { 'sentence_template': 'A sentence template', 'order': 1, }) collect_strings.main(['apikey']) self._assert_all_translations([ 'A sentence template', 'First Advice', 'Second Advice', 'my explanation' ]) mock_post.assert_not_called()