def get_or_create(self, email, **kwargs):
""" Get or create User from json
:param email: email
:param kwargs: params for extra field
:return: User object
"""
def get_email():
# if email doesn't exist, generate it
return email or '{0}@beneple.com'.format(kwargs['username'])
try:
obj = User.objects.get(email=get_email())
except User.DoesNotExist:
obj = User(email=get_email(),
password=kwargs['password'],
clearance=self.convert_group_permission(
kwargs['groups']),
extra={
'legacy': {
'date_joined': kwargs['date_joined'],
'first_name': kwargs['first_name'],
'last_name': kwargs['last_name'],
'username': kwargs['username'],
}
})
obj.save(metadata=Metadata.empty())
return obj
def complement_form_dependent(self, employee_obj, **kwargs):
obj = employee_obj
obj.extra.setdefault('dependants', [])
dependant_names = [d['name'] for d in obj.extra['dependants']]
if kwargs['name'] not in dependant_names:
relation = kwargs['relation']
relation = relation if relation and relation != 'None' else 'Other'
obj.extra['dependants'].append({
'birth_date': kwargs['birth_date'],
'name': kwargs['name'],
'nationality': kwargs['nationality'],
'relation': relation,
'emirates_id': {
'number': kwargs['emirates_id'],
'expiry_date': kwargs['emirates_id_expiry_date'],
},
'passport': {
'number': kwargs['passport_number'],
'expiry_date': kwargs['passport_expiry_date'],
},
'visa': {
'number': kwargs['visa_number'],
'expiry_date': kwargs['visa_expiry_date'],
'image_url': kwargs['visa_document']
},
})
obj.extra['legacy']['dependants'] = self.get_extra_legacy(
kwargs, 'dependants')
obj.extra = self.clean_and_validate_extra(obj.extra)
obj.save(metadata=Metadata.empty())
return obj
def get_or_create(self, title, url, **kwargs):
""" Get or create Org from json.
:param title: name filed
:param url: domain field
:param kwargs: other params
:return: Org object
"""
try:
obj = Org.objects.get(name=title, domain=url)
except Org.DoesNotExist:
extra_legacy = self.get_extra_legacy(kwargs)
obj = Org(name=title, domain=url, extra={'legacy': extra_legacy})
working_start_hour = kwargs.get('working_start_hour')
if working_start_hour:
obj.extra["working_hours_start"] = parser.parse(
working_start_hour).time().isoformat()
working_hours_end = kwargs.get('working_hours_end')
if working_hours_end:
obj.extra["working_hours_end"] = parser.parse(
working_hours_end).time().isoformat()
validate_json(obj.extra, self.schema)
obj.save(metadata=Metadata.empty())
return obj
def complement_form_dependent(self, employee_obj, **kwargs):
obj = employee_obj
obj.extra.setdefault('dependants', [])
dependant_names = [d['name'] for d in obj.extra['dependants']]
if kwargs['name'] not in dependant_names:
relation = kwargs['relation']
relation = relation if relation and relation != 'None' else 'Other'
obj.extra['dependants'].append({
'birth_date': kwargs['birth_date'],
'name': kwargs['name'],
'nationality': kwargs['nationality'],
'relation': relation,
'emirates_id': {
'number': kwargs['emirates_id'],
'expiry_date': kwargs['emirates_id_expiry_date'],
},
'passport': {
'number': kwargs['passport_number'],
'expiry_date': kwargs['passport_expiry_date'],
},
'visa': {
'number': kwargs['visa_number'],
'expiry_date': kwargs['visa_expiry_date'],
'image_url': kwargs['visa_document']
},
})
obj.extra['legacy']['dependants'] = self.get_extra_legacy(
kwargs, 'dependants'
)
obj.extra = self.clean_and_validate_extra(obj.extra)
obj.save(metadata=Metadata.empty())
return obj
def get_or_create(self, title, url, **kwargs):
""" Get or create Org from json.
:param title: name filed
:param url: domain field
:param kwargs: other params
:return: Org object
"""
try:
obj = Org.objects.get(name=title, domain=url)
except Org.DoesNotExist:
extra_legacy = self.get_extra_legacy(kwargs)
obj = Org(name=title, domain=url, extra={'legacy': extra_legacy})
working_start_hour = kwargs.get('working_start_hour')
if working_start_hour:
obj.extra["working_hours_start"] = parser.parse(
working_start_hour).time().isoformat()
working_hours_end = kwargs.get('working_hours_end')
if working_hours_end:
obj.extra["working_hours_end"] = parser.parse(
working_hours_end).time().isoformat()
validate_json(obj.extra, self.schema)
obj.save(metadata=Metadata.empty())
return obj
def get_or_create(self, email, **kwargs):
""" Get or create User from json
:param email: email
:param kwargs: params for extra field
:return: User object
"""
def get_email():
# if email doesn't exist, generate it
return email or '{0}@beneple.com'.format(kwargs['username'])
try:
obj = User.objects.get(email=get_email())
except User.DoesNotExist:
obj = User(
email=get_email(), password=kwargs['password'],
clearance=self.convert_group_permission(kwargs['groups']),
extra={'legacy': {
'date_joined': kwargs['date_joined'],
'first_name': kwargs['first_name'],
'last_name': kwargs['last_name'],
'username': kwargs['username'],
}}
)
obj.save(metadata=Metadata.empty())
return obj
def complement_form_bankinfo(self, employee_obj, **kwargs):
obj = employee_obj
bank_schema = self.schema['properties']['bank_account']['properties']
obj.extra['bank_account'] = self.get_extra_by_schema(
kwargs, bank_schema)
obj.extra['legacy']['bank_account'] = self.get_extra_legacy(
kwargs, 'bank_account')
obj.extra = self.clean_and_validate_extra(obj.extra)
obj.save(metadata=Metadata.empty())
return obj
def complement_form_bankinfo(self, employee_obj, **kwargs):
obj = employee_obj
bank_schema = self.schema['properties']['bank_account']['properties']
obj.extra['bank_account'] = self.get_extra_by_schema(kwargs,
bank_schema)
obj.extra['legacy']['bank_account'] = self.get_extra_legacy(
kwargs, 'bank_account'
)
obj.extra = self.clean_and_validate_extra(obj.extra)
obj.save(metadata=Metadata.empty())
return obj
def complement_from_employment(self, employee_obj, **kwargs):
""" Update Employee object, get data from employee.employment
:param employee_obj: Employee model instance
:param kwargs: data params
:return:
"""
obj = employee_obj
try:
salary = Decimal(kwargs['salary'].replace(',', '')) \
if kwargs['salary'] else Decimal(0)
if len(salary.as_tuple().digits) > 10:
# if count of digits more 10 - don't save in db
salary = Decimal(0)
obj.monthlysalary = salary
except InvalidOperation:
pass
obj.extra['title'] = kwargs['position'] or '---'
if 'documents' not in obj.extra:
obj.extra['documents'] = {}
obj.extra['documents'].update({
'visa': {
'number': kwargs['visa_number'],
'expiry_date': kwargs['visa_expiry_date'],
'image_url': kwargs['visa_document'],
},
'work_contract': {
'start_date': kwargs['contract_start_date'],
'end_date': kwargs['contract_end_date'],
},
})
extra_legacy = self.get_extra_legacy(kwargs, 'employment')
obj.extra['legacy'].update({'employment': extra_legacy})
obj.extra = self.clean_and_validate_extra(obj.extra)
obj.save(metadata=Metadata.empty())
return obj
def complement_from_employment(self, employee_obj, **kwargs):
""" Update Employee object, get data from employee.employment
:param employee_obj: Employee model instance
:param kwargs: data params
:return:
"""
obj = employee_obj
try:
salary = Decimal(kwargs['salary'].replace(',', '')) \
if kwargs['salary'] else Decimal(0)
if len(salary.as_tuple().digits) > 10:
# if count of digits more 10 - don't save in db
salary = Decimal(0)
obj.monthlysalary = salary
except InvalidOperation:
pass
obj.extra['title'] = kwargs['position'] or '---'
if 'documents' not in obj.extra:
obj.extra['documents'] = {}
obj.extra['documents'].update({
'visa': {
'number': kwargs['visa_number'],
'expiry_date': kwargs['visa_expiry_date'],
'image_url': kwargs['visa_document'],
},
'work_contract': {
'start_date': kwargs['contract_start_date'],
'end_date': kwargs['contract_end_date'],
},
})
extra_legacy = self.get_extra_legacy(kwargs, 'employment')
obj.extra['legacy'].update({'employment': extra_legacy})
obj.extra = self.clean_and_validate_extra(obj.extra)
obj.save(metadata=Metadata.empty())
return obj
def load_old_data():
with open(os.path.join(settings.BASE_DIR, 'old_db_data.json')) as f:
json_data = json.load(f)
# loading old data
data = {}
for i in json_data:
data.setdefault(i['model'], {})
data[i['model']].update({i['pk']: i['fields']})
print 'data for import loaded'
# load json schema
org_schema = load_schema(__file__, 'org/org.schema.json')
employee_schema = load_schema(__file__, 'org/employee.schema.json')
del (employee_schema['required'])
# init converters
user_manager = UserManager(data['auth.user'])
org_manager = OrgManager(data['aythan.organization'], org_schema)
employee_manager = EmployeeManager(data['employee.employee'],
employee_schema)
# import auth.user
for user_id, user_data in data['auth.user'].iteritems():
user_manager.get_or_create(**user_data)
if DEBUG:
break
print 'import auth.user is complete'
# import aythan.organization
for org_id, org_data in data['aythan.organization'].iteritems():
user_id = org_data.get('user')
if user_id:
# link to org user equivalent of admin rights in new systems
user = user_manager.get_by_old_user_id(user_id)
user.clearance = 'admin'
user.save(metadata=Metadata.empty())
org_manager.get_or_create(**org_data)
if DEBUG:
break
print 'import aythan.organization is complete'
# load employee.employee
for employee_id, employee_data in data['employee.employee'].iteritems():
user = user_manager.get_by_old_user_id(employee_data['user'])
org = None
org_id = employee_data.get('organization')
if org_id:
org = org_manager.get_or_create(
**data['aythan.organization'][org_id])
employee_manager.get_or_create(user_obj=user,
org_obj=org,
**employee_data)
if DEBUG:
break
print 'import employee.employee is complete'
# load employee.employment
for e_id, employment_data in data['employee.employment'].iteritems():
employee = employee_manager.get_by_old_employee_id(
employment_data['employee'])
if employee_data.get('line_manager'):
# create Superiority link for User
line_manager_user = employee_manager.get_user_by_old_employee_id(
employee_data['line_manager'])
line_manager = employee_manager.get_or_create(
user_obj=line_manager_user)
Superiority(subordinate=employee,
superior=line_manager).save(metadata=Metadata.empty())
employee_manager.complement_from_employment(employee,
**employment_data)
if DEBUG:
break
print 'import employee.employment is complete'
# load employee.bankinfo
for eb_id, eb_data in data['employee.bankinfo'].iteritems():
employee = employee_manager.get_by_old_employee_id(eb_data['employee'])
employee_manager.complement_form_bankinfo(employee, **eb_data)
if DEBUG:
break
print 'import employee.bankinfo is complete'
# load employee.dependent
for ed_id, ed_data in data['employee.dependent'].iteritems():
employee = employee_manager.get_by_old_employee_id(ed_data['employee'])
employee_manager.complement_form_dependent(employee, **ed_data)
if DEBUG:
break
print 'import employee.dependent is complete'
def get_or_create(self, user_obj, org_obj=None, **kwargs):
""" Get or create Employee object.
:param user_obj: User model instance
:param org_obj: Org model instance
:param kwargs: other params
:return: Employee object
"""
try:
obj = Employee.objects.get(employeeuser__user=user_obj)
except Employee.DoesNotExist:
m = Metadata.empty()
pers_email = kwargs['personal_email']
gender = kwargs['gender']
obj = Employee(
displayname=u'{0} {1}'.format(
user_obj.extra['legacy']['first_name'],
user_obj.extra['legacy']['last_name']).strip(),
active=kwargs['status'],
extra=delete_empty_dict_values({
'birth_date':
kwargs['birth_date'],
'emergency_contact': {
'name': kwargs['ec_name'],
'phone_number': kwargs['ec_phone'],
},
'gender':
gender.lower() if gender else None,
'contact': {
'home': {
'address': kwargs['home_address'],
'email':
pers_email if is_valid_email(pers_email) else '',
'phone_number': kwargs['phone'],
}
},
'documents': {
'labor_card': {
'number': kwargs['labour_card'],
'expiry_date': kwargs['labour_expiry_date'],
},
'passport': {
'number': kwargs['passport_number'],
'expiry_date': kwargs['passport_expiry_date'],
},
'emirates_id': {
'number': kwargs['emirates_id'],
'expiry_date': kwargs['emirates_id_expiry_date'],
},
},
'languages':
[lang.strip() for lang in kwargs['languages'].split(',')]
if kwargs['languages'] else [],
'nationality':
kwargs['nationality'],
'legacy': {
'employee': self.get_extra_legacy(kwargs, 'employee')
}
}),
)
obj.extra['marital_status'] = {
'Married': 'married',
'Single': 'unmarried',
}.get(kwargs['marital_status']) or "other"
obj.extra['religion'] = {
'Christianity': 'Christian',
'Islam': 'Muslim',
}.get(kwargs['religion']) or 'Other'
validate_json(obj.extra, self.schema)
obj.save(metadata=m)
EmployeeUser(employee=obj, user=user_obj).save(metadata=m)
if org_obj:
OrgEmployee(employee=obj, org=org_obj).save(metadata=m)
return obj
def load_old_data():
with open(os.path.join(settings.BASE_DIR, 'old_db_data.json')) as f:
json_data = json.load(f)
# loading old data
data = {}
for i in json_data:
data.setdefault(i['model'], {})
data[i['model']].update({i['pk']: i['fields']})
print 'data for import loaded'
# load json schema
org_schema = load_schema(__file__, 'org/org.schema.json')
employee_schema = load_schema(__file__, 'org/employee.schema.json')
del(employee_schema['required'])
# init converters
user_manager = UserManager(data['auth.user'])
org_manager = OrgManager(data['aythan.organization'], org_schema)
employee_manager = EmployeeManager(data['employee.employee'],
employee_schema)
# import auth.user
for user_id, user_data in data['auth.user'].iteritems():
user_manager.get_or_create(**user_data)
if DEBUG:
break
print 'import auth.user is complete'
# import aythan.organization
for org_id, org_data in data['aythan.organization'].iteritems():
user_id = org_data.get('user')
if user_id:
# link to org user equivalent of admin rights in new systems
user = user_manager.get_by_old_user_id(user_id)
user.clearance = 'admin'
user.save(metadata=Metadata.empty())
org_manager.get_or_create(**org_data)
if DEBUG:
break
print 'import aythan.organization is complete'
# load employee.employee
for employee_id, employee_data in data['employee.employee'].iteritems():
user = user_manager.get_by_old_user_id(employee_data['user'])
org = None
org_id = employee_data.get('organization')
if org_id:
org = org_manager.get_or_create(
**data['aythan.organization'][org_id]
)
employee_manager.get_or_create(user_obj=user, org_obj=org,
**employee_data)
if DEBUG:
break
print 'import employee.employee is complete'
# load employee.employment
for e_id, employment_data in data['employee.employment'].iteritems():
employee = employee_manager.get_by_old_employee_id(
employment_data['employee']
)
if employee_data.get('line_manager'):
# create Superiority link for User
line_manager_user = employee_manager.get_user_by_old_employee_id(
employee_data['line_manager'])
line_manager = employee_manager.get_or_create(
user_obj=line_manager_user)
Superiority(
subordinate=employee, superior=line_manager
).save(metadata=Metadata.empty())
employee_manager.complement_from_employment(
employee, **employment_data
)
if DEBUG:
break
print 'import employee.employment is complete'
# load employee.bankinfo
for eb_id, eb_data in data['employee.bankinfo'].iteritems():
employee = employee_manager.get_by_old_employee_id(eb_data['employee'])
employee_manager.complement_form_bankinfo(employee, **eb_data)
if DEBUG:
break
print 'import employee.bankinfo is complete'
# load employee.dependent
for ed_id, ed_data in data['employee.dependent'].iteritems():
employee = employee_manager.get_by_old_employee_id(ed_data['employee'])
employee_manager.complement_form_dependent(employee, **ed_data)
if DEBUG:
break
print 'import employee.dependent is complete'
def get_or_create(self, user_obj, org_obj=None, **kwargs):
""" Get or create Employee object.
:param user_obj: User model instance
:param org_obj: Org model instance
:param kwargs: other params
:return: Employee object
"""
try:
obj = Employee.objects.get(employeeuser__user=user_obj)
except Employee.DoesNotExist:
m = Metadata.empty()
pers_email = kwargs['personal_email']
gender = kwargs['gender']
obj = Employee(
displayname=u'{0} {1}'.format(
user_obj.extra['legacy']['first_name'],
user_obj.extra['legacy']['last_name']
).strip(),
active=kwargs['status'],
extra=delete_empty_dict_values({
'birth_date': kwargs['birth_date'],
'emergency_contact': {
'name': kwargs['ec_name'],
'phone_number': kwargs['ec_phone'],
},
'gender': gender.lower() if gender else None,
'contact': {'home': {
'address': kwargs['home_address'],
'email': pers_email if is_valid_email(pers_email) else '',
'phone_number': kwargs['phone'],
}},
'documents': {
'labor_card': {
'number': kwargs['labour_card'],
'expiry_date': kwargs['labour_expiry_date'],
},
'passport': {
'number': kwargs['passport_number'],
'expiry_date': kwargs['passport_expiry_date'],
},
'emirates_id': {
'number': kwargs['emirates_id'],
'expiry_date': kwargs['emirates_id_expiry_date'],
},
},
'languages': [
lang.strip() for lang in kwargs['languages'].split(',')
] if kwargs['languages'] else [],
'nationality': kwargs['nationality'],
'legacy': {
'employee': self.get_extra_legacy(kwargs, 'employee')
}
}),
)
obj.extra['marital_status'] = {
'Married': 'married',
'Single': 'unmarried',
}.get(kwargs['marital_status']) or "other"
obj.extra['religion'] = {
'Christianity': 'Christian',
'Islam': 'Muslim',
}.get(kwargs['religion']) or 'Other'
validate_json(obj.extra, self.schema)
obj.save(metadata=m)
EmployeeUser(employee=obj, user=user_obj).save(metadata=m)
if org_obj:
OrgEmployee(employee=obj, org=org_obj).save(metadata=m)
return obj