def _update_employees(employee_dicts): """Given a JSON string in the format "[{employee info 1}, {employee info 2}, ...]", create new employee records and update existing records as necessary. Then determine whether any employees have been terminated since the last update, and mark these employees as such. """ logging.info('Updating employees...') employees = [] for d in employee_dicts: existing_employee = Employee.query(Employee.username == d['username']).get() if existing_employee is None: new_employee = Employee.create_from_dict(d, persist=False) employees.append(new_employee) else: existing_employee.update_from_dict(d) # If the user is in the S3 dump, then the user is no longer # terminated. existing_employee.terminated = False employees.append(existing_employee) # write to db every 500 employees so we save memory if len(employees) == 500: ndb.put_multi(employees) del employees employees = [] if employees: ndb.put_multi(employees) logging.info('Done.')
def _update_employees(employee_dicts): """Given a JSON string in the format "[{employee info 1}, {employee info 2}, ...]", create new employee records and update existing records as necessary. Then determine whether any employees have been terminated since the last update, and mark these employees as such. """ logging.info('Updating employees... {}MB'.format(memory_usage().current())) db_employee_dict = { employee.username: employee for employee in Employee.query() } all_employees, new_employees = [], [] current_usernames = set() for d in employee_dicts: existing_employee = db_employee_dict.get(d['username']) if existing_employee is None: new_employee = Employee.create_from_dict(d, persist=False) all_employees.append(new_employee) new_employees.append(new_employee) else: existing_employee.update_from_dict(d) # If the user is in the S3 dump, then the user is no longer # terminated. existing_employee.terminated = False all_employees.append(existing_employee) current_usernames.add(d['username']) if len(all_employees) % 200 == 0: logging.info('Processed {} employees, {}MB'.format( len(all_employees), memory_usage().current())) ndb.put_multi(all_employees) # Figure out if there are any employees in the DB that aren't in the S3 # dump. These are terminated employees, and we need to mark them as such. db_usernames = set(db_employee_dict.keys()) terminated_usernames = db_usernames - current_usernames terminated_employees = [] for username in terminated_usernames: employee = db_employee_dict[username] employee.terminated = True terminated_employees.append(employee) ndb.put_multi(terminated_employees) logging.info('Done updating employees. {}MB'.format( memory_usage().current()))
def _update_employees(employee_dicts): """Given a JSON string in the format "[{employee info 1}, {employee info 2}, ...]", create new employee records and update existing records as necessary. Then determine whether any employees have been terminated since the last update, and mark these employees as such. """ logging.info('Updating employees...') db_employee_dict = { employee.username: employee for employee in Employee.query() } all_employees, new_employees = [], [] current_usernames = set() for d in employee_dicts: existing_employee = db_employee_dict.get(d['username']) if existing_employee is None: new_employee = Employee.create_from_dict(d, persist=False) all_employees.append(new_employee) new_employees.append(new_employee) elif existing_employee.is_employee_data_changed(d, is_terminated=False): existing_employee.update_from_dict(d) all_employees.append(existing_employee) current_usernames.add(d['username']) logging.info('Processed {} employees'.format(len(all_employees))) # Only updates the changed and new employees data ndb.put_multi(all_employees) # Figure out if there are any employees in the DB that aren't in the S3 # dump. These are terminated employees, and we need to mark them as such. db_usernames = set(db_employee_dict.keys()) terminated_usernames = db_usernames - current_usernames terminated_employees = [] for username in terminated_usernames: employee = db_employee_dict[username] # Only updating non-terminated employees if not employee.terminated: employee.terminated = True terminated_employees.append(employee) ndb.put_multi(terminated_employees) logging.info('Done updating employees.')
def _update_employees(employee_dicts): """Given a JSON string in the format "[{employee info 1}, {employee info 2}, ...]", create new employee records and update existing records as necessary. Then determine whether any employees have been terminated since the last update, and mark these employees as such. """ logging.info('Updating employees...') all_employees, new_employees = [], [] current_usernames = set() for d in employee_dicts: existing_employee = Employee.query(Employee.username == d['username']).get() if existing_employee is None: new_employee = Employee.create_from_dict(d, persist=False) all_employees.append(new_employee) new_employees.append(new_employee) else: existing_employee.update_from_dict(d) # If the user is in the S3 dump, then the user is no longer # terminated. existing_employee.terminated = False all_employees.append(existing_employee) current_usernames.add(d['username']) ndb.put_multi(all_employees) # Figure out if there are any employees in the DB that aren't in the S3 # dump. These are terminated employees, and we need to mark them as such. usernames_to_employees = dict( (employee.username, employee) for employee in Employee.query() ) db_usernames = set(usernames_to_employees.keys()) terminated_usernames = db_usernames - current_usernames terminated_employees = [] for u in terminated_usernames: employee = usernames_to_employees[u] employee.terminated = True terminated_employees.append(employee) ndb.put_multi(terminated_employees) logging.info('Done.')
def create_employee( username='******', department='Engineering', first_name='John', last_name='Doe', photo_url=None, ): if photo_url is None: photo_url = 'http://example.com/photos/{0}.jpg'.format(username) return Employee.create_from_dict({ 'username': username, 'department': department, 'first_name': first_name, 'last_name': last_name, 'photo_url': photo_url, })