def _update_employees(employee_dicts):
"""Given a JSON string in the format "[{employee info 1}, {employee info 2}, ...]",
create new employee records and update existing records as necessary.
Then determine whether any employees have been terminated since the last update,
and mark these employees as such.
"""
logging.info('Updating employees...')
employees = []
for d in employee_dicts:
existing_employee = Employee.query(Employee.username == d['username']).get()
if existing_employee is None:
new_employee = Employee.create_from_dict(d, persist=False)
employees.append(new_employee)
else:
existing_employee.update_from_dict(d)
# If the user is in the S3 dump, then the user is no longer
# terminated.
existing_employee.terminated = False
employees.append(existing_employee)
# write to db every 500 employees so we save memory
if len(employees) == 500:
ndb.put_multi(employees)
del employees
employees = []
if employees:
ndb.put_multi(employees)
logging.info('Done.')
def _update_employees(employee_dicts):
"""Given a JSON string in the format "[{employee info 1}, {employee info 2}, ...]",
create new employee records and update existing records as necessary.
Then determine whether any employees have been terminated since the last update,
and mark these employees as such.
"""
logging.info('Updating employees... {}MB'.format(memory_usage().current()))
db_employee_dict = {
employee.username: employee
for employee in Employee.query()
}
all_employees, new_employees = [], []
current_usernames = set()
for d in employee_dicts:
existing_employee = db_employee_dict.get(d['username'])
if existing_employee is None:
new_employee = Employee.create_from_dict(d, persist=False)
all_employees.append(new_employee)
new_employees.append(new_employee)
else:
existing_employee.update_from_dict(d)
# If the user is in the S3 dump, then the user is no longer
# terminated.
existing_employee.terminated = False
all_employees.append(existing_employee)
current_usernames.add(d['username'])
if len(all_employees) % 200 == 0:
logging.info('Processed {} employees, {}MB'.format(
len(all_employees),
memory_usage().current()))
ndb.put_multi(all_employees)
# Figure out if there are any employees in the DB that aren't in the S3
# dump. These are terminated employees, and we need to mark them as such.
db_usernames = set(db_employee_dict.keys())
terminated_usernames = db_usernames - current_usernames
terminated_employees = []
for username in terminated_usernames:
employee = db_employee_dict[username]
employee.terminated = True
terminated_employees.append(employee)
ndb.put_multi(terminated_employees)
logging.info('Done updating employees. {}MB'.format(
memory_usage().current()))
def _update_employees(employee_dicts):
"""Given a JSON string in the format "[{employee info 1}, {employee info 2}, ...]",
create new employee records and update existing records as necessary.
Then determine whether any employees have been terminated since the last update,
and mark these employees as such.
"""
logging.info('Updating employees...')
db_employee_dict = {
employee.username: employee
for employee in Employee.query()
}
all_employees, new_employees = [], []
current_usernames = set()
for d in employee_dicts:
existing_employee = db_employee_dict.get(d['username'])
if existing_employee is None:
new_employee = Employee.create_from_dict(d, persist=False)
all_employees.append(new_employee)
new_employees.append(new_employee)
elif existing_employee.is_employee_data_changed(d, is_terminated=False):
existing_employee.update_from_dict(d)
all_employees.append(existing_employee)
current_usernames.add(d['username'])
logging.info('Processed {} employees'.format(len(all_employees)))
# Only updates the changed and new employees data
ndb.put_multi(all_employees)
# Figure out if there are any employees in the DB that aren't in the S3
# dump. These are terminated employees, and we need to mark them as such.
db_usernames = set(db_employee_dict.keys())
terminated_usernames = db_usernames - current_usernames
terminated_employees = []
for username in terminated_usernames:
employee = db_employee_dict[username]
# Only updating non-terminated employees
if not employee.terminated:
employee.terminated = True
terminated_employees.append(employee)
ndb.put_multi(terminated_employees)
logging.info('Done updating employees.')
def _update_employees(employee_dicts):
"""Given a JSON string in the format "[{employee info 1}, {employee info 2}, ...]",
create new employee records and update existing records as necessary.
Then determine whether any employees have been terminated since the last update,
and mark these employees as such.
"""
logging.info('Updating employees...')
all_employees, new_employees = [], []
current_usernames = set()
for d in employee_dicts:
existing_employee = Employee.query(Employee.username == d['username']).get()
if existing_employee is None:
new_employee = Employee.create_from_dict(d, persist=False)
all_employees.append(new_employee)
new_employees.append(new_employee)
else:
existing_employee.update_from_dict(d)
# If the user is in the S3 dump, then the user is no longer
# terminated.
existing_employee.terminated = False
all_employees.append(existing_employee)
current_usernames.add(d['username'])
ndb.put_multi(all_employees)
# Figure out if there are any employees in the DB that aren't in the S3
# dump. These are terminated employees, and we need to mark them as such.
usernames_to_employees = dict(
(employee.username, employee)
for employee
in Employee.query()
)
db_usernames = set(usernames_to_employees.keys())
terminated_usernames = db_usernames - current_usernames
terminated_employees = []
for u in terminated_usernames:
employee = usernames_to_employees[u]
employee.terminated = True
terminated_employees.append(employee)
ndb.put_multi(terminated_employees)
logging.info('Done.')
def create_employee(
username='******',
department='Engineering',
first_name='John',
last_name='Doe',
photo_url=None,
):
if photo_url is None:
photo_url = 'http://example.com/photos/{0}.jpg'.format(username)
return Employee.create_from_dict({
'username': username,
'department': department,
'first_name': first_name,
'last_name': last_name,
'photo_url': photo_url,
})
