class HatchbuckParser:
"""
An object that does all the parsing for/with Hatchbuck.
"""
def __init__(self, args):
self.args = args
self.stats = {}
self.hatchbuck = None
def main(self):
"""Parsing gets kicked off here"""
logging.debug("starting with arguments: %s", self.args)
self.init_hatchbuck()
self.parse_files()
def show_summary(self):
"""Show some statistics"""
logging.info(self.stats)
def init_hatchbuck(self):
"""Initialize hatchbuck API incl. authentication"""
if not self.args.hatchbuck:
logging.error("No hatchbuck_key found.")
sys.exit(1)
self.hatchbuck = Hatchbuck(self.args.hatchbuck, noop=self.args.noop)
def parse_files(self):
"""Start parsing files"""
if self.args.file:
for file in self.args.file:
logging.debug("parsing file %s", file)
self.parse_file(file)
elif self.args.dir:
for direc in self.args.dir:
logging.debug("using directory %s", direc)
for file in os.listdir(direc):
if file.endswith(".vcf"):
file_path = os.path.join(direc, file)
logging.info("parsing file %s", file_path)
try:
self.parse_file(file_path)
except binascii.Error as error:
logging.error("error parsing: %s", error)
else:
logging.info("Nothing to do.")
# pylint: disable=too-many-branches
# pylint: disable=too-many-locals
# pylint: disable=too-many-statements
def parse_file(self, file):
"""
Parse a single address book file
"""
prin = pprint.PrettyPrinter()
self.stats = {}
for vob in vobject.readComponents(open(file)):
content = vob.contents
if self.args.verbose:
logging.debug("parsing %s:", file)
prin.pprint(content)
if "n" not in content:
self.stats["noname"] = self.stats.get("noname", 0) + 1
return
if "email" not in content or not re.match(
r"^[^@]+@[^@]+\.[^@]+$", content["email"][0].value):
self.stats["noemail"] = self.stats.get("noemail", 0) + 1
return
self.stats["valid"] = self.stats.get("valid", 0) + 1
# aggregate stats what kind of fields we have available
for i in content:
# if i in c:
self.stats[i] = self.stats.get(i, 0) + 1
emails = []
for email in content.get("email", []):
if re.match(r"^[^@äöü]+@[^@]+\.[^@]+$", email.value):
emails.append(email.value)
profile_list = []
for email in emails:
profile = self.hatchbuck.search_email(email)
if profile:
profile_list.append(profile)
else:
continue
# No contacts found
if not profile_list:
# create new contact
profile = dict()
profile["firstName"] = content["n"][0].value.given
profile["lastName"] = content["n"][0].value.family
if "title" in content:
profile["title"] = content["title"][0].value
if "org" in content:
profile["company"] = content["org"][0].value
profile["subscribed"] = True
profile["status"] = {"name": "Lead"}
if self.args.source:
profile["source"] = {"id": self.args.source}
# override hatchbuck sales rep username if set
# (default: api key owner)
if self.args.user:
profile["salesRep"] = {"username": self.args.user}
profile["emails"] = []
for email in content.get("email", []):
if not re.match(r"^[^@äöü]+@[^@]+\.[^@]+$", email.value):
continue
if "WORK" in email.type_paramlist:
kind = "Work"
elif "HOME" in email.type_paramlist:
kind = "Home"
else:
kind = "Other"
profile["emails"].append({
"address": email.value,
"type": kind
})
profile = self.hatchbuck.create(profile)
logging.info("added contact: %s", profile)
for profile in profile_list:
if profile["firstName"] == "" or "@" in profile["firstName"]:
profile = self.hatchbuck.profile_add(
profile, "firstName", None,
content["n"][0].value.given)
if profile["lastName"] == "" or "@" in profile["lastName"]:
profile = self.hatchbuck.profile_add(
profile, "lastName", None,
content["n"][0].value.family)
if "title" in content and profile.get("title", "") == "":
profile = self.hatchbuck.profile_add(
profile, "title", None, content["title"][0].value)
if "company" in profile:
if "org" in content and profile.get("company", "") == "":
profile = self.hatchbuck.profile_add(
profile, "company", None, content["org"][0].value)
if profile["company"] == "":
# empty company name ->
# maybe we can guess the company name from the email
# address?
# logging.warning("empty company with emails: %s",
# profile['emails'])
pass
# clean up company name
if re.match(r";$", profile["company"]):
logging.warning("found unclean company name: %s",
profile["company"])
if re.match(r"\|", profile["company"]):
logging.warning("found unclean company name: %s",
profile["company"])
for addr in content.get("adr", []):
address = {
"street": addr.value.street,
"zip_code": addr.value.code,
"city": addr.value.city,
"country": addr.value.country,
}
try:
if "WORK" in addr.type_paramlist:
kind = "Work"
elif "HOME" in addr.type_paramlist:
kind = "Home"
else:
kind = "Other"
except AttributeError:
# if there is no type at all
kind = "Other"
logging.debug("adding address %s %s", address, profile)
profile = self.hatchbuck.profile_add_address(
profile, address, kind)
for telefon in content.get("tel", []):
# number cleanup
number = telefon.value
for rep in "()-\xa0":
# clean up number
number = number.replace(rep, "")
number = number.replace("+00", "+").replace("+0", "+")
try:
if "WORK" in telefon.type_paramlist:
kind = "Work"
elif "HOME" in telefon.type_paramlist:
kind = "Home"
else:
kind = "Other"
except AttributeError:
# if there is no type at all
kind = "Other"
redundant = False
try:
phonenumber = phonenumbers.parse(number, None)
pformatted = phonenumbers.format_number(
phonenumber,
phonenumbers.PhoneNumberFormat.INTERNATIONAL)
except phonenumbers.phonenumberutil.NumberParseException:
# number could not be parsed, e.g. because it is a
# local number without country code
logging.warning(
"could not parse number %s as %s in %s, "
"trying to guess country from address",
telefon.value,
number,
self.hatchbuck.short_contact(profile),
)
pformatted = number
# try to guess the country from the addresses
countries_found = []
for addr in profile.get("addresses", []):
if (addr.get("country", False) and addr["country"]
not in countries_found):
countries_found.append(addr["country"])
logging.debug("countries found %s", countries_found)
if len(countries_found) == 1:
# lets try to parse the number with the country
countrycode = countries.lookup(
countries_found[0]).alpha_2
logging.debug("countrycode %s", countrycode)
try:
phonenumber = phonenumbers.parse(
number, countrycode)
pformatted = phonenumbers.format_number(
phonenumber,
phonenumbers.PhoneNumberFormat.
INTERNATIONAL,
)
logging.debug("guess %s", pformatted)
profile = self.hatchbuck.profile_add(
profile,
"phones",
"number",
pformatted,
{"type": kind},
)
# if we got here we now have a full number
continue
except phonenumbers.phonenumberutil.NumberParseException:
logging.warning(
"could not parse number %s as %s using country %s in %s",
telefon.value,
number,
countrycode,
self.hatchbuck.short_contact(profile),
)
pformatted = number
# check that there is not an international/longer
# number there already
# e.g. +41 76 4000 464 compared to 0764000464
# skip the 0 in front
num = number.replace(" ", "")[1:]
for tel2 in profile["phones"]:
# check for suffix match
if tel2["number"].replace(" ", "").endswith(num):
logging.warning(
"not adding number %s from %s because it "
"is a suffix of existing %s",
num,
self.hatchbuck.short_contact(profile),
tel2["number"],
)
redundant = True
break
if not redundant:
profile = self.hatchbuck.profile_add(
profile, "phones", "number", pformatted,
{"type": kind})
# clean & deduplicate all phone numbers
profile = self.hatchbuck.clean_all_phone_numbers(profile)
for skype in content.get("x-skype", []):
profile = self.hatchbuck.profile_add(
profile,
"instantMessaging",
"address",
skype.value,
{"type": "Skype"},
)
for msn in content.get("x-msn", []):
profile = self.hatchbuck.profile_add(
profile,
"instantMessaging",
"address",
msn.value,
{"type": "Messenger"},
)
for msn in content.get("x-msnim", []):
profile = self.hatchbuck.profile_add(
profile,
"instantMessaging",
"address",
msn.value,
{"type": "Messenger"},
)
for twitter in content.get("x-twitter", []):
if "twitter.com" in twitter.value:
value = twitter.value
else:
value = "http://twitter.com/" + twitter.value.replace(
"@", "")
profile = self.hatchbuck.profile_add(
profile, "socialNetworks", "address", value,
{"type": "Twitter"})
for url in content.get("url", []) + content.get(
"x-socialprofile", []):
value = url.value
if not value.startswith("http"):
value = "http://" + value
if "facebook.com" in value:
profile = self.hatchbuck.profile_add(
profile,
"socialNetworks",
"address",
value,
{"type": "Facebook"},
)
elif "twitter.com" in value:
profile = self.hatchbuck.profile_add(
profile,
"socialNetworks",
"address",
value,
{"type": "Twitter"},
)
else:
profile = self.hatchbuck.profile_add(
profile, "website", "websiteUrl", value)
for bday in content.get("bday", []):
date = {
"year": bday.value[0:4],
"month": bday.value[5:7],
"day": bday.value[8:10],
}
profile = self.hatchbuck.profile_add_birthday(
profile, date)
if self.args.tag:
if not self.hatchbuck.profile_contains(
profile, "tags", "name", self.args.tag):
self.hatchbuck.add_tag(profile["contactId"],
self.args.tag)
# get the list of unique contacts IDs to detect if there are
# multiple contacts in hatchbuck for this one contact in CardDAV
profile_contactids = []
message = ""
for profile in profile_list:
if profile["contactId"] not in profile_contactids:
profile_contactids.append(profile["contactId"])
email_profile = " "
for email_add in profile.get("emails", []):
email_profile = email_add["address"] + " "
number_profile = " "
for phone_number in profile.get("phones", []):
number_profile = phone_number["number"] + " "
message += ("{0} {1} ({2}, {3}, {4})".format(
profile["firstName"],
profile["lastName"],
email_profile,
number_profile,
profile["contactUrl"],
) + ", ")
if len(profile_contactids) > 1:
# there are duplicates
NotificationService().send_message(
"Duplicates: %s from file: %s" % (message[:-2], file))
def test_clean_all_phone_numbers(self):
hatchbuck = Hatchbuck("")
hatchbuck.noop = True
profile1 = {
"phones": [
{
"id": "1",
"number": "+(414) 454-5 53 00",
"type": "Work",
"typeId": "asdf",
},
{
"id": "2",
"number": "+(414) 454-5 53 00",
"type": "Home",
"typeId": "qwer",
},
{
"id": "3",
"number": "(044) 545-53-00",
"type": "Work",
"typeId": "asdf",
},
]
}
self.assertEqual(
hatchbuck.clean_all_phone_numbers(profile1),
{
"phones": [
{
"id": "1",
"number": "+41 44 545 53 00",
"type": "Work",
"typeId": "asdf",
}
]
},
)
profile2 = {
"phones": [
{
"id": "1",
"number": "(044) 545-53-00",
"type": "Work",
"typeId": "asdf",
}
],
"addresses": [{"country": "Switzerland"}],
}
self.assertEqual(
hatchbuck.clean_all_phone_numbers(profile2),
{
"phones": [
{
"id": "1",
"number": "+41 44 545 53 00",
"type": "Work",
"typeId": "asdf",
}
],
"addresses": [{"country": "Switzerland"}],
},
)
profile3 = {
"phones": [
{
"id": "1",
"number": "(044) 545-53-00",
"type": "Work",
"typeId": "asdf",
}
],
"addresses": [{"country": "Switzerland "}], # note the space
}
self.assertEqual(
hatchbuck.clean_all_phone_numbers(profile3),
{
"phones": [
{
"id": "1",
"number": "+41 44 545 53 00",
"type": "Work",
"typeId": "asdf",
}
],
"addresses": [{"country": "Switzerland "}],
},
)
from hatchbuck import Hatchbuck import pprint import sys import logging LOGFORMAT = "%(asctime)s - %(name)s - %(levelname)s - %(message)s" logging.basicConfig(level=logging.DEBUG, format=LOGFORMAT) pp = pprint.PrettyPrinter() hatchbuck = Hatchbuck(sys.argv[1], noop=False) profile = hatchbuck.search_email(sys.argv[2]) pp.pprint(profile["phones"]) profile = hatchbuck.clean_all_phone_numbers(profile) pp.pprint(profile["phones"])
def main(noop=False, partnerfilter=False, verbose=False):
"""
Fetch Odoo ERP customer info
"""
if verbose:
logging.basicConfig(level=logging.DEBUG, format=LOGFORMAT)
else:
logging.basicConfig(level=logging.INFO, format=LOGFORMAT)
hatchbuck = Hatchbuck(os.environ.get("HATCHBUCK_APIKEY"), noop=noop)
odoo = odoorpc.ODOO(os.environ.get("ODOO_HOST"),
protocol="jsonrpc+ssl",
port=443)
odoodb = os.environ.get("ODOO_DB", False)
if not odoodb:
# if the DB is not configured pick the first in the list
# this takes another round-trip to the API
odoodbs = odoo.db.list()
odoodb = odoodbs[0]
odoo.login(odoodb, os.environ.get("ODOO_USERNAME"),
os.environ.get("ODOO_PASSWORD"))
# logging.debug(odoo.env)
partnerenv = odoo.env["res.partner"]
# search for all partner contacts that are people, not companies
odoo_filter = [("is_company", "=", False)]
# optionally filter by name, mostly used for debugging
if partnerfilter:
odoo_filter.append(("name", "ilike", partnerfilter))
partner_ids = partnerenv.search(odoo_filter)
# logging.debug(partner_ids)
for pid in partner_ids:
for child in partnerenv.browse(pid):
# browse() is similar to a database result set/pointer
# it should be able to take a list of IDs
# but it timeouted for my largish list of IDs$
# so we're browsing one ID at a time...
# child = person/contact in a company
# Available fields:
# __last_update
# active
# bank_ids
# birthdate
# calendar_last_notif_ack
# category_id
# child_ids
# city
# color
# comment
# commercial_partner_id
# company_id
# contact_address
# contract_ids
# contracts_count
# country_id
# create_date
# create_uid
# credit
# credit_limit
# customer
# date
# debit
# debit_limit
# display_name
# ean13
# email
# employee
# fax
# function
# has_image
# image
# image_medium
# image_small
# invoice_ids
# is_company
# journal_item_count
# lang
# last_reconciliation_date
# meeting_count
# meeting_ids
# message_follower_ids
# message_ids
# message_is_follower
# message_last_post
# message_summary
# message_unread
# mobile
# name
# notify_email
# opportunity_count
# opportunity_ids
# opt_out
# parent_id
# parent_name
# phone
# phonecall_count
# phonecall_ids
# property_account_payable
# property_account_position
# property_account_receivable
# property_delivery_carrier
# property_payment_term
# property_product_pricelist
# property_product_pricelist_purchase
# property_stock_customer
# property_stock_supplier
# property_supplier_payment_term
# purchase_order_count
# ref
# ref_companies
# sale_order_count
# sale_order_ids
# section_id
# signup_expiration
# signup_token
# signup_type
# signup_url
# signup_valid
# state_id
# street
# street2
# supplier
# supplier_invoice_count
# task_count
# task_ids
# title
# total_invoiced
# type
# tz
# tz_offset
# use_parent_address
# user_id
# user_ids
# vat
# vat_subjected
# website
# write_date
# write_uid
# zip
categories = [cat.name for cat in child.category_id]
logging.info((
child.name, # vorname nachname
# child.title.name, # titel ("Dr.")
# child.function, # "DevOps Engineer"
# child.parent_id.invoice_ids.amount_total,
child.parent_name, # Firmenname
# child.parent_id,
# child.parent_id.name,
# child.email,
# child.mobile,
# child.phone,
categories,
# child.opt_out,
# child.lang,
# child.street,
# child.street2,
# child.zip,
# child.city,
# child.country_id.name,
# child.total_invoiced,
# child.invoice_ids.amount_total,
# partner.total_invoiced,
# partner_turnover,
# child.website,
# child.comment,
))
if not child.email or child.email == "false":
logging.info("no email")
continue
if "@" not in child.email or "." not in child.email:
logging.error("email does not look like email: %s",
child.email)
continue
emails = child.email.replace("mailto:", "")
emails = [x.strip() for x in emails.split(",")]
profile = hatchbuck.search_email_multi(emails)
if profile is None:
logging.debug("user not found in CRM")
if not any([x in STATUSMAP for x in categories]):
# only add contacts that have one of STATUSMAP status
# -> don't add administrative contacts to CRM
logging.info(
"not adding contact because no relevant category")
continue
# create profile
profile = dict()
firstname, lastname = split_name(child.name)
profile["firstName"] = firstname
profile["lastName"] = lastname
profile["subscribed"] = True
# dummy category, will be properly set below
profile["status"] = {"name": "Customer"}
profile["emails"] = []
for addr in emails:
profile["emails"].append({"address": addr, "type": "Work"})
profile = hatchbuck.create(profile)
logging.info("added profile: %s", profile)
if profile is None:
logging.error("adding contact failed: %s", profile)
continue
else:
logging.info("contact found: %s", profile)
for cat in STATUSMAP:
if cat in categories:
# apply the first matching category from STATUSMAP
if noop:
profile["status"] = STATUSMAP[cat]
else:
logging.info("ERP category: %s => CRM status: %s", cat,
STATUSMAP[cat])
profile = hatchbuck.update(profile["contactId"],
{"status": STATUSMAP[cat]})
break
# logging.debug(
# "contact has category vip %s and tag %s",
# "VIP" in categories,
# hatchbuck.profile_contains(profile, "tags", "name", "VIP"),
# )
if "VIP" in categories and not hatchbuck.profile_contains(
profile, "tags", "name", "VIP"):
logging.info("adding VIP tag")
hatchbuck.add_tag(profile["contactId"], "VIP")
if "VIP" not in categories and hatchbuck.profile_contains(
profile, "tags", "name", "VIP"):
logging.info("removing VIP tag")
hatchbuck.remove_tag(profile["contactId"], "VIP")
# update profile with information from odoo
if (profile.get("firstName", "") == ""
or profile.get("firstName", "false") == "false"):
firstname, _ = split_name(child.name)
logging.info("Updating first name: %s", firstname)
profile = hatchbuck.profile_add(profile, "firstName", None,
firstname)
if (profile.get("lastName", "") == ""
or profile.get("lastName", "false") == "false"):
_, lastname = split_name(child.name)
logging.info("Updating last name: %s", lastname)
profile = hatchbuck.profile_add(profile, "lastName", None,
lastname)
for addr in emails:
profile = hatchbuck.profile_add(profile, "emails", "address",
addr, {"type": "Work"})
if profile.get("title", "") == "" and child.function:
logging.info("updating title: %s", child.function)
profile = hatchbuck.profile_add(profile, "title", None,
child.function)
if profile["status"] == "Employee" and os.environ.get(
"EMPLOYEE_COMPANYNAME", False):
logging.info("updating company: %s", child.parent_name)
profile = hatchbuck.profile_add(
profile, "company", None,
os.environ.get("EMPLOYEE_COMPANYNAME"))
elif profile.get("company", "") == "" and child.parent_name:
logging.info("updating company: %s", child.parent_name)
profile = hatchbuck.profile_add(profile, "company", None,
child.parent_name)
if profile.get("company", "") == "":
# empty company name ->
# maybe we can guess the company name
# from the email address?
# logging.warning("empty company with emails: {0}".
# format(profile['emails']))
pass
# clean up company name
if re.match(r";$", profile.get("company", "")):
logging.warning("found unclean company name: %s",
format(profile["company"]))
if re.match(r"\|", profile.get("company", "")):
logging.warning("found unclean company name: %s",
format(profile["company"]))
# Add address
address = {
"street": child.street,
"zip_code": child.zip,
"city": child.city,
"country": child.country_id.name,
}
if profile["status"] == "Employee":
kind = "Home"
else:
kind = "Work"
logging.debug("adding address %s %s", address, profile)
profile = hatchbuck.profile_add_address(profile, address, kind)
# Add website field to Hatchbuck Contact
if child.website:
profile = hatchbuck.profile_add(profile, "website",
"websiteUrl", child.website)
# Add phones and mobile fields to Hatchbuck Contact
if child.phone:
profile = hatchbuck.profile_add(profile, "phones", "number",
child.phone, {"type": kind})
if child.mobile:
profile = hatchbuck.profile_add(profile, "phones", "number",
child.mobile,
{"type": "Mobile"})
# format & deduplicate all phone numbers
profile = hatchbuck.clean_all_phone_numbers(profile)
# Add customFields(comment, amount_total, lang) to
# Hatchbuck Contact
if child.comment:
profile = hatchbuck.profile_add(
profile,
"customFields",
"value",
child.comment,
{"name": "Comments"},
)
if child.lang:
profile = hatchbuck.profile_add(profile, "customFields",
"value", child.lang,
{"name": "Language"})
partner_turnover = "0"
if child.parent_id:
# if the contact belongs to a company include total turnover of the company
partner_turnover = str(round(child.parent_id.total_invoiced))
else:
partner_turnover = str(round(child.total_invoiced))
# looking at
# https://github.com
# /odoo/odoo/blob/master/addons/account/models/partner.py#L260
# this uses account.invoice.report in the background and returns 0
# if the user does not have access to it
# permission: "Accounting & Finance": "Invoicing & Payments"
profile = hatchbuck.profile_add(profile, "customFields", "value",
partner_turnover,
{"name": "Invoiced"})
# Add ERP tag to Hatchbuck Contact
if not hatchbuck.profile_contains(profile, "tags", "name", "ERP"):
hatchbuck.add_tag(profile["contactId"], "ERP")