def registerObject(self, objectData):
if not self.validateFilters(objectData):
self.registerFiltered(objectData)
return
email = objectData.email
if email and SanitationUtils.stringIsEmail(email):
self.registerEmail(objectData, email)
else:
if self.DEBUG_FLAT:
self.registerWarning("invalid email address: %s" % email)
self.registerNoEmail(objectData)
role = objectData.role
if role:
self.registerRole(objectData, role)
else:
# self.registerWarning("invalid role: %s"%role)
self.registerNoRole(objectData)
card = objectData.MYOBID
if card and SanitationUtils.stringIsMYOBID(card):
self.registerCard(objectData, card)
else:
self.registerNoCard(objectData)
username = objectData.username
if username:
self.registerUsername(objectData, username)
else:
if self.DEBUG_FLAT:
self.registerWarning("invalid username: %s" % username)
self.registerNoUsername(objectData)
# company = objectData['Company']
# # if self.DEBUG_FLAT: SanitationUtils.safePrint(repr(objectData), company)
# if company:
# self.registerCompany(objectData, company)
addresses = [objectData.billing_address, objectData.shipping_address]
for address in filter(None, addresses):
if not address.valid:
reason = address.reason
assert reason, "there must be a reason that this address is invalid: " + address
self.registerBadAddress(objectData, address)
name = objectData.name
# print "NAME OF %s IS %s" % (repr(objectData), name.__str__(out_schema="flat"))
if not name.valid:
reason = name.reason
assert reason, "there must be a reason that this name is invalid: " + name
# print "registering bad name: ", SanitationUtils.coerceBytes(name)
self.registerBadName(objectData, name)
# emails = objectData.emails
# if not emails.valid:
# reason = emails.reason
# self.registerBadEmail(objectData, emails)
super(CSVParse_User, self).registerObject(objectData)
def append(self, objectData):
assert issubclass(objectData.__class__, ImportShopMixin), \
"object must be subclass of ImportShopMixin not %s : %s" % (
SanitationUtils.coerceUnicode(objectData.__class__),
SanitationUtils.coerceUnicode(objectData)
)
return super(ShopProdList, self).append(objectData)
def tabulate(self, cols=None, tablefmt=None):
objs = self.objects
sanitizer = self.getSanitizer(tablefmt)
# sanitizer = (lambda x: str(x)) if tablefmt == 'html' else SanitationUtils.makeSafeOutput
if objs:
if not cols: cols = self.reportCols
header = [self.objList_type]
for col in cols.keys():
header += [col]
table = []
for obj in objs:
row = [obj.identifier]
for col in cols.keys():
# if col == 'Address':
# print repr(str(obj.get(col))), repr(sanitizer(obj.get(col)))
row += [ sanitizer(obj.get(col) )or ""]
try:
SanitationUtils.coerceUnicode(row[-1])
except:
Registrar.registerWarning("can't turn row into unicode: %s" % SanitationUtils.coerceUnicode(row))
table += [row]
# table += [[obj.index] + [ sanitizer(obj.get(col) )or "" for col in cols.keys()]]
# print "table", table
# SanitationUtils.safePrint(table)
# return SanitationUtils.coerceUnicode(tabulate(table, headers=header, tablefmt=tablefmt))
return (tabulate(table, headers=header, tablefmt=tablefmt))
# print repr(table)
# print repr(table.encode('utf8'))
# return table.encode('utf8')
else:
Registrar.registerWarning("cannot tabulate Objlist: there are no objects")
return ""
def valuesSimilar(self, col, mValue, sValue):
response = super(SyncUpdate_Cat_Woo, self).valuesSimilar(col, mValue, sValue)
if not response:
if col is 'descsum':
mDesc = SanitationUtils.similarMarkupComparison(mValue)
sDesc = SanitationUtils.similarMarkupComparison(sValue)
if mDesc == sDesc:
response = True
if self.DEBUG_UPDATE: self.registerMessage(self.testToStr(col, mValue.__repr__(), sValue.__repr__(), response))
return response
def printBasicColumns(cls, objects):
obj_list = cls.objectContainer.container()
for _object in objects:
obj_list.append(_object)
cols = cls.objectContainer.container.getBasicCols()
SanitationUtils.safePrint( obj_list.tabulate(
cols,
tablefmt = 'simple'
))
def valuesSimilar(self, col, mValue, sValue):
response = False
if not (mValue or sValue):
response = True
elif not (mValue and sValue):
response = False
#check if they are similar
if SanitationUtils.similarComparison(mValue) == SanitationUtils.similarComparison(sValue):
response = True
if self.DEBUG_UPDATE: self.registerMessage(self.testToStr(col, mValue, sValue, response))
return response
def execSilentCommandAssert(self, command):
self.assertConnect()
stdin, stdout, stderr = self.service.exec_command(command)
if stdin: pass #gets rid of annoying warnings
possible_errors = stdout.readlines() + stderr.readlines()
for error in possible_errors:
if re.match("^Countries.*", error):
print error
continue
assert not error, "command <%s> returned errors: %s" % (
SanitationUtils.coerceUnicode(command),
SanitationUtils.coerceUnicode(error)
)
def printBasicColumns(users):
# print len(users)
usrList = UsrObjList()
for user in users:
usrList.append(user)
# SanitationUtils.safePrint( "BILLING ADDRESS:", repr(user), user['First Name'], user.get('First Name'), user.name.__unicode__(out_schema="flat"))
cols = colData.getBasicCols()
SanitationUtils.safePrint( usrList.tabulate(
cols,
tablefmt = 'simple'
))
def __init__(self, cols, defaults, schema, **kwargs):
if self.DEBUG_MRO:
self.registerMessage('CSVParse_Gen_Tree')
taxoSubs = kwargs.pop('taxoSubs', {})
itemSubs = kwargs.pop('itemSubs', {})
extra_defaults = OrderedDict([
('CVC', '0'),
('code', ''),
('name', ''),
('fullname', ''),
('description', ''),
('HTML Description', ''),
('imglist', [])
])
extra_taxoSubs = OrderedDict([
('', ''),
])
extra_itemSubs = OrderedDict([
('Hot Pink', 'Pink'),
('Hot Lips (Red)', 'Red'),
('Hot Lips', 'Red'),
('Silken Chocolate (Bronze)', 'Bronze'),
('Silken Chocolate', 'Bronze'),
('Moon Marvel (Silver)', 'Silver'),
('Dusty Gold', 'Gold'),
('Screen Printed', ''),
('Embroidered', ''),
])
extra_cols = [schema]
cols = listUtils.combineLists( cols, extra_cols )
defaults = listUtils.combineOrderedDicts( defaults, extra_defaults )
super(CSVParse_Gen_Tree, self).__init__( cols, defaults, **kwargs)
# CSVParse_Gen_Mixin.__init__(self, schema)
metaWidth = kwargs.get('metaWidth', 2)
assert metaWidth >= 2, "metaWidth must be greater than 2 for a GEN subclass"
self.schema = schema
self.taxoSubs = listUtils.combineOrderedDicts( taxoSubs, extra_taxoSubs )
self.itemSubs = listUtils.combineOrderedDicts( itemSubs, extra_itemSubs )
self.taxoRegex = SanitationUtils.compileRegex(self.taxoSubs)
self.itemRegex = SanitationUtils.compileRegex(self.itemSubs)
if self.DEBUG_GEN:
self.registerMessage("taxoDepth: {}".format(self.taxoDepth), 'CSVParse_Gen_Tree.__init__')
self.registerMessage("itemDepth: {}".format(self.itemDepth), 'CSVParse_Gen_Tree.__init__')
self.registerMessage("maxDepth: {}".format(self.maxDepth), 'CSVParse_Gen_Tree.__init__')
self.registerMessage("metaWidth: {}".format(self.metaWidth), 'CSVParse_Gen_Tree.__init__')
self.registerMessage("schema: {}".format(self.schema), 'CSVParse_Gen_Tree.__init__')
def update_meta(self, newmeta):
oldmeta = self.read_meta()
newmeta = dict([(key, SanitationUtils.coerceAscii(value)) for key, value in newmeta.items()])
changed = []
for key in ['title', 'description']:
if SanitationUtils.similarComparison(oldmeta[key]) != SanitationUtils.similarComparison(newmeta[key]):
changed += [key]
if self.DEBUG_IMG:
self.registerMessage(
u"changing imgmeta[%s] from %s to %s" % (key, repr(oldmeta[key]), repr(newmeta[key])),
self.fname
)
if changed:
self.write_meta(newmeta['title'], newmeta['description'])
def testToStr(self, col, val1, val2, res):
return u"testing col %s: %s | %s -> %s" % (
unicode(col),
repr(val1),
repr(val2),
SanitationUtils.boolToTruishString(res)
)
def getSlaveUpdatesNativeRecursive(self, col, updates=None):
if updates == None: updates = OrderedDict()
SanitationUtils.safePrint("getting updates for col %s, updates: %s" % (col, str(updates)))
if col in self.colData.data.keys():
data = self.colData.data[col]
if data.get(self.s_meta_target):
data_s = data.get(self.s_meta_target,{})
if not data_s.get('final') and data_s.get('key'):
updates[data_s.get('key')] = self.newSObject.get(col)
if data.get('aliases'):
data_aliases = data.get('aliases')
for alias in data_aliases:
if self.sColSemiStatic(alias):
continue
updates = self.getSlaveUpdatesNativeRecursive(alias, updates)
return updates
def toHTML(self):
colNames = self.getColNames()
html = u'<table class="shop_table lasercommerce pricing_table table table-striped">'
html += '<thead><tr>'
for col, name in colNames.items():
colClass = SanitationUtils.sanitizeClass(col)
html += '<th class=%s>' % colClass
html += bleach.clean(name)
html += '</th>'
html += '</tr></thead>'
ruleLines = self.getRuleLines()
self.registerMessage("ruleLines: %s" % (ruleLines))
for ruleLineData in ruleLines:
lineType = ruleLineData.get('Discount Type','')
html += '<tr>'
for col in colNames.keys():
value = ruleLineData.get(col, '')
if col == 'Discount' and lineType in ['PDSC']:
value += '%'
html += '<td>'
html += value #bleach.clean(value)
html += '</td>'
html += '</tr>'
html += "</table>"
return html.encode('UTF-8')
def __init__(self, data=None):
if data is None:
data = {}
assert hasattr(data, 'items'), "Container initialized with non-dict"
super(Item, self).__init__([
(key, SanitationUtils.sanitize_container_value(val)) for key, val \
in data.items()
])
def sanitizeValue(self, col, value):
# print "sanitizing", col, repr(value)
if 'phone' in col.lower():
if 'preferred' in col.lower():
if value and len(SanitationUtils.stripNonNumbers(value)) > 1:
# print "value nullified", value
return ""
return value
def __init__(self, data=None):
if data is None:
data = {}
assert hasattr(data, 'items'), "Container initialized with non-dict"
super(Item, self).__init__([
(key, SanitationUtils.sanitize_container_value(val)) for key, val \
in data.items()
])
def tabulate(self, tablefmt=None):
if(self):
prefix, suffix = "", ""
delimeter = "\n"
if tablefmt == 'html':
delimeter = ''
prefix = '<div class="matchList">'
suffix = '</div>'
return prefix + delimeter.join(
[SanitationUtils.coerceBytes(match.tabulate(tablefmt=tablefmt)) for match in self if match]
) + suffix
else:
return ""
def writeSection(title, description, data, length = 0, html_class="results_section"):
sectionID = SanitationUtils.makeSafeClass(title)
description = "%s %s" % (str(length) if length else "No", description)
outFile.write('<div class="%s">'% html_class )
outFile.write('<a data-toggle="collapse" href="#%s" aria-expanded="true" data-target="#%s" aria-controls="%s">' % (sectionID, sectionID, sectionID))
outFile.write('<h2>%s (%d)</h2>' % (title, length))
outFile.write('</a>')
outFile.write('<div class="collapse" id="%s">' % sectionID)
outFile.write('<p class="description">%s</p>' % description)
outFile.write('<p class="data">' )
outFile.write( re.sub("<table>","<table class=\"table table-striped\">",data) )
outFile.write('</p>')
outFile.write('</div>')
outFile.write('</div>')
def displayUpdateList(self, updateList, tablefmt=None):
if updateList:
delimeter = "<br/>" if tablefmt=="html" else "\n"
subject_fmt = "<h4>%s</h4>" if tablefmt=="html" else "%s"
# header = ["Column", "Reason", "Old", "New"]
header = OrderedDict([
('col', 'Column'),
('reason', 'Reason'),
('oldLoserValue', 'Old'),
('oldWinnerValue', 'New'),
('mColTime', 'M TIME'),
('sColTime', 'S TIME'),
])
subjects = {}
for warnings in updateList.values():
for warning in warnings:
subject = warning['subject']
if subject not in subjects.keys():
subjects[subject] = []
warning_fmtd = dict([\
(key, SanitationUtils.sanitizeForTable(val))\
for key, val in warning.items()\
if key in header
])
for key in ['mColTime', 'sColTime']:
try:
rawTime = int(warning[key])
if rawTime:
warning_fmtd[key] = TimeUtils.wpTimeToString(rawTime)
except Exception, e:
pass
subjects[subject].append(warning_fmtd)
tables = []
for subject, warnings in subjects.items():
antiSubject_fmtd = subject_fmt % self.opposite_src(subject)
table = [header]+warnings
table_fmtd = tabulate(table, headers='firstrow', \
tablefmt=tablefmt )
tables.append(delimeter.join([antiSubject_fmtd, table_fmtd]))
return delimeter.join(tables)
def valuesSimilar(self, col, mValue, sValue):
response = super(SyncUpdate_Usr, self).valuesSimilar(col, mValue, sValue)
if not response:
if "phone" in col.lower():
if "preferred" in col.lower():
mPreferred = SanitationUtils.similarTruStrComparison(mValue)
sPreferred = SanitationUtils.similarTruStrComparison(sValue)
# print repr(mValue), " -> ", mPreferred
# print repr(sValue), " -> ", sPreferred
if mPreferred == sPreferred:
response = True
else:
mPhone = SanitationUtils.similarPhoneComparison(mValue)
sPhone = SanitationUtils.similarPhoneComparison(sValue)
plen = min(len(mPhone), len(sPhone))
if plen > 7 and mPhone[-plen] == sPhone[-plen]:
response = True
elif "role" in col.lower():
mRole = SanitationUtils.similarComparison(mValue)
sRole = SanitationUtils.similarComparison(sValue)
if mRole == 'rn':
mRole = ''
if sRole == 'rn':
sRole = ''
if mRole == sRole:
response = True
elif "address" in col.lower() and isinstance(mValue, ContactAddress):
if( mValue != sValue ):
pass
# print "M: ", mValue.__str__(out_schema="flat"), "S: ", sValue.__str__(out_schema="flat")
response = mValue.similar(sValue)
elif "web site" in col.lower():
if SanitationUtils.similarURLComparison(mValue) == SanitationUtils.similarURLComparison(sValue):
response = True
if self.DEBUG_UPDATE: self.registerMessage(self.testToStr(col, mValue.__str__(), sValue.__str__(), response))
return response
def valuesSimilar(self, col, mValue, sValue):
response = super(SyncUpdate_Prod, self).valuesSimilar(col, mValue, sValue)
if col in self.colData.data:
colData = self.colData.data[col]
if colData.get('type'):
if colData.get('type') == 'currency':
mPrice = SanitationUtils.similarCurrencyComparison(mValue)
sPrice = SanitationUtils.similarCurrencyComparison(sValue)
if mPrice == sPrice:
response = True
elif not response:
if col is 'descsum':
mDesc = SanitationUtils.similarMarkupComparison(mValue)
sDesc = SanitationUtils.similarMarkupComparison(sValue)
if mDesc == sDesc:
response = True
elif col is 'CVC':
mCom = SanitationUtils.similarComparison(mValue) or '0'
sCom = SanitationUtils.similarComparison(sValue) or '0'
if mCom == sCom:
response = True
if self.DEBUG_UPDATE: self.registerMessage(self.testToStr(col, mValue.__repr__(), sValue.__repr__(), response))
return response
def sanitizeCell(self, cell):
return SanitationUtils.sanitizeCell(cell)
def __str__(self):
return SanitationUtils.coerce_bytestr(self.__unicode__())
def tabulate(self, cols=None, tablefmt=None):
objlist = self.getObjList()
return SanitationUtils.coerceBytes(objlist.tabulate(cols, tablefmt))
sshClient = paramiko.SSHClient()
sshClient.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
sshClient.connect(**paramikoSSHParams)
stdin, stdout, stderr = sshClient.exec_command(command)
possible_errors = stdout.readlines()
assert not possible_errors, "command returned errors: " + possible_errors
try:
sftpClient = sshClient.open_sftp()
sftpClient.chdir(remoteExportFolder)
fstat = sftpClient.stat(exportFilename)
if fstat:
sftpClient.get(exportFilename, maPath)
except Exception, e:
SanitationUtils.safePrint("ERROR IN SFTP: " + str(e))
finally:
sftpClient.close()
except Exception, e:
SanitationUtils.safePrint("ERROR IN SSH: " + str(e))
finally:
sshClient.close()
maParser = CSVParse_User(
cols = colData.getImportCols(),
defaults = colData.getDefaults(),
contact_schema = 'act',
)
maParser.analyseFile(maPath, maEncoding)
# -*- coding: utf-8 -*-
from utils import SanitationUtils
from kitchen.text import converters
def printandrep(name, thing, back = ""):
fmt_str = "%32s |"
print fmt_str % name, fmt_str % converters.to_bytes(thing), fmt_str % repr(thing), fmt_str % converters.to_bytes(back), fmt_str % converters.to_bytes(unicode(back))
u_str = u"<\U0001F44C'&>"
utf8_str = SanitationUtils.unicodeToUTF8(u_str)
utf8_back = SanitationUtils.utf8ToUnicode(utf8_str)
xml_str = SanitationUtils.unicodeToXml(u_str)
xml_back = SanitationUtils.xmlToUnicode(xml_str)
ascii_str = SanitationUtils.unicodeToAscii(u_str)
ascii_back = SanitationUtils.asciiToUnicode(u_str)
printandrep( "u_str", u_str )
printandrep( "utf8_str", utf8_str, utf8_back )
printandrep( "xml_str", xml_str, xml_back )
printandrep( "ascii_str", ascii_str, ascii_back )
print SanitationUtils.unicodeToUTF8(None)
print SanitationUtils.utf8ToUnicode(None)
print SanitationUtils.unicodeToXml(None)
print SanitationUtils.xmlToUnicode(None)
print SanitationUtils.unicodeToAscii(None)
print SanitationUtils.asciiToUnicode(None)
print SanitationUtils.coerceUnicode(None)
SanitationUtils.safePrint(None)
print converters.to_bytes(SanitationUtils.coerceUnicode("\xf0\x9f\x91\x8c"))
def initContactObjects(self, data):
# emails_kwargs = OrderedDict(filter(None, [\
# ((key, data.get(value)) if data.get(value) else None) for key, value in\
# {
# 'email' : 'E-mail',
# 'personal_email': 'Personal E-mail',
# }.items()
# ]))
#
# self['E-mails'] = EmailFields(
# **emails_kwargs
# )
name_kwargs = OrderedDict(
filter(
None,
[
((key, data.get(value)) if data.get(value) else None)
for key, value in {
"first_name": "First Name",
"middle_name": "Middle Name",
"family_name": "Surname",
"name_prefix": "Name Prefix",
"name_suffix": "Name Suffix",
"contact": "Contact",
"company": "Company",
"city": "City",
"country": "Country",
"state": "State",
}.items()
],
)
)
self["Name"] = ContactName(**name_kwargs)
assert self["Name"] is not None, "contact is missing mandatory fields: something went wrong"
address_kwargs = OrderedDict(
filter(
None,
[
((key, data.get(value)) if data.get(value) else None)
for key, value in {
"line1": "Address 1",
"line2": "Address 2",
"city": "City",
"postcode": "Postcode",
"state": "State",
"country": "Country",
"company": "Company",
}.items()
],
)
)
self["Address"] = ContactAddress(**address_kwargs)
# print "ADDRESS: ", self['Address']
alt_address_kwargs = OrderedDict(
filter(
None,
[
((key, data.get(value)) if data.get(value) else None)
for key, value in {
"line1": "Home Address 1",
"line2": "Home Address 2",
"city": "Home City",
"postcode": "Home Postcode",
"state": "Home State",
"country": "Home Country",
"company": "Company",
}.items()
],
)
)
self["Home Address"] = ContactAddress(**alt_address_kwargs)
# print "HOME ADDRESS: ", self['Home Address']
phone_kwargs = OrderedDict(
filter(
None,
[
((key, data.get(value)) if data.get(value) else None)
for key, value in {
"mob_number": "Mobile Phone",
"tel_number": "Phone",
"fax_number": "Fax",
"mob_pref": "Mobile Phone Preferred",
"tel_pref": "Phone Preferred",
}.items()
],
)
)
self["Phone Numbers"] = ContactPhones(**phone_kwargs)
social_media_kwargs = OrderedDict(
filter(
None,
[
((key, data.get(value)) if data.get(value) else None)
for key, value in {
"facebook": "Facebook Username",
"twitter": "Twitter Username",
"gplus": "GooglePlus Username",
"instagram": "Instagram Username",
"website": "Web Site",
}.items()
],
)
)
self["Social Media"] = SocialMediaFields(**social_media_kwargs)
emails = []
if data.get("E-mail"):
emails = listUtils.combineLists(emails, SanitationUtils.findallEmails(data["E-mail"]))
if data.get("Personal E-mail"):
emails = listUtils.combineLists(emails, SanitationUtils.findallEmails(data.get("Personal E-mail")))
self["E-mail"] = emails.pop(0) if emails else None
self["Personal E-mail"] = ", ".join(emails)
urls = []
if data.get("Web Site"):
urls = listUtils.combineLists(urls, SanitationUtils.findallURLs(data["Web Site"]))
self["Web Site"] = urls.pop(0) if urls else None
# if not self['Emails'].valid:
# self['emails_reason'] = '\n'.join(filter(None, [
# self['Emails'].reason,
# ]))
if not self["Address"].valid or not self["Home Address"].valid:
self["address_reason"] = "\n".join(
filter(
None,
[
"ADDRESS: " + self["Address"].reason if not self["Address"].valid else None,
"HOME ADDRESS: " + self["Home Address"].reason if not self["Home Address"].valid else None,
],
)
)
if not self["Name"].valid:
self["name_reason"] = "\n".join(filter(None, [self["Name"].reason]))
if not self["Phone Numbers"].valid:
self["phone_reason"] = "\n".join(filter(None, [self["Phone Numbers"].reason]))
if not self["Social Media"].valid:
self["social_reason"] = "\n".join(filter(None, [self["Social Media"].reason]))
def main():
global testMode, inFolder, outFolder, logFolder, srcFolder, pklFolder, \
yamlPath, repPath, mFailPath, sFailPath, logPath, zipPath
userFile = cardFile = emailFile = sinceM = sinceS = False
### OVERRIDE CONFIG WITH YAML FILE ###
with open(yamlPath) as stream:
config = yaml.load(stream)
if 'inFolder' in config.keys():
inFolder = config['inFolder']
if 'outFolder' in config.keys():
outFolder = config['outFolder']
if 'logFolder' in config.keys():
logFolder = config['logFolder']
#mandatory
merge_mode = config.get('merge_mode', 'sync')
MASTER_NAME = config.get('master_name', 'MASTER')
SLAVE_NAME = config.get('slave_name', 'SLAVE')
DEFAULT_LAST_SYNC = config.get('default_last_sync')
master_file = config.get('master_file', '')
slave_file = config.get('slave_file', '')
userFile = config.get('userFile')
cardFile = config.get('cardFile')
emailFile = config.get('emailFile')
sinceM = config.get('sinceM')
sinceS = config.get('sinceS')
download_slave = config.get('download_slave')
download_master = config.get('download_master')
update_slave = config.get('update_slave')
update_master = config.get('update_master')
do_filter = config.get('do_filter')
do_problematic = config.get('do_problematic')
do_post = config.get('do_post')
do_sync = config.get('do_sync')
### OVERRIDE CONFIG WITH ARGPARSE ###
parser = argparse.ArgumentParser(description = 'Merge contact records between two databases')
group = parser.add_mutually_exclusive_group()
group.add_argument("-v", "--verbosity", action="count",
help="increase output verbosity")
group.add_argument("-q", "--quiet", action="store_true")
group = parser.add_mutually_exclusive_group()
group.add_argument('--testmode', help='Run in test mode with test databases',
action='store_true', default=None)
group.add_argument('--livemode', help='Run the script on the live databases',
action='store_false', dest='testmode')
group = parser.add_mutually_exclusive_group()
group.add_argument('--download-master', help='download the master data',
action="store_true", default=None)
group.add_argument('--skip-download-master', help='use the local master file instead\
of downloading the master data', action="store_false", dest='download_master')
group = parser.add_mutually_exclusive_group()
group.add_argument('--download-slave', help='download the slave data',
action="store_true", default=None)
group.add_argument('--skip-download-slave', help='use the local slave file instead\
of downloading the slave data', action="store_false", dest='download_slave')
group = parser.add_mutually_exclusive_group()
group.add_argument('--update-master', help='update the master database',
action="store_true", default=None)
group.add_argument('--skip-update-master', help='don\'t update the master database',
action="store_false", dest='update_master')
group = parser.add_mutually_exclusive_group()
group.add_argument('--update-slave', help='update the slave database',
action="store_true", default=None)
group.add_argument('--skip-update-slave', help='don\'t update the slave database',
action="store_false", dest='update_slave')
group = parser.add_mutually_exclusive_group()
group.add_argument('--do-filter', help='filter the databases',
action="store_true", default=None)
group.add_argument('--skip-filter', help='don\'t filter the databases',
action="store_false", dest='do_filter')
group = parser.add_mutually_exclusive_group()
group.add_argument('--do-sync', help='sync the databases',
action="store_true", default=None)
group.add_argument('--skip-sync', help='don\'t sync the databases',
action="store_false", dest='do_sync')
group = parser.add_mutually_exclusive_group()
group.add_argument('--do-problematic', help='make problematic updates to the databases',
action="store_true", default=None)
group.add_argument('--skip-problematic', help='don\'t make problematic updates to the databases',
action="store_false", dest='do_problematic')
group = parser.add_mutually_exclusive_group()
group.add_argument('--do-post', help='post process the contacts',
action="store_true", default=None)
group.add_argument('--skip-post', help='don\'t post process the contacts',
action="store_false", dest='do_post')
parser.add_argument('--m-ssh-host', help='location of master ssh server')
parser.add_argument('--m-ssh-port', type=int, help='location of master ssh port')
parser.add_argument('--limit', type=int, help='global limit of objects to process')
parser.add_argument('--master-file', help='location of master file')
parser.add_argument('--slave-file', help='location of slave file')
parser.add_argument('--card-file')
group = parser.add_argument_group()
group.add_argument('--debug-abstract', action='store_true', dest='debug_abstract')
group.add_argument('--debug-parser', action='store_true', dest='debug_parser')
group.add_argument('--debug-update', action='store_true', dest='debug_update')
group.add_argument('--debug-flat', action='store_true', dest='debug_flat')
group.add_argument('--debug-name', action='store_true', dest='debug_name')
group.add_argument('--debug-address', action='store_true', dest='debug_address')
group.add_argument('--debug-client', action='store_true', dest='debug_client')
group.add_argument('--debug-utils', action='store_true', dest='debug_utils')
group.add_argument('--debug-contact', action='store_true', dest='debug_contact')
args = parser.parse_args()
if args:
print args
if args.verbosity > 0:
Registrar.DEBUG_PROGRESS = True
Registrar.DEBUG_ERROR = True
if args.verbosity > 1:
Registrar.DEBUG_MESSAGE = True
if args.quiet:
Registrar.DEBUG_PROGRESS = False
Registrar.DEBUG_ERROR = False
Registrar.DEBUG_MESSAGE = False
if args.testmode is not None:
testMode = args.testmode
if args.download_slave is not None:
download_slave = args.download_slave
if args.download_master is not None:
download_master = args.download_master
if args.update_slave is not None:
update_slave = args.update_slave
if args.update_master is not None:
update_master = args.update_master
if args.do_filter is not None:
do_filter = args.do_filter
if args.do_sync is not None:
do_sync = args.do_sync
if args.do_problematic is not None:
do_problematic = args.do_problematic
if args.do_post is not None:
do_post = args.do_post
if args.m_ssh_port:
m_ssh_port = args.m_ssh_port
if args.m_ssh_host:
m_ssh_host = args.m_ssh_host
if args.master_file is not None:
download_master = False
master_file = args.master_file
if args.slave_file is not None:
download_slave = False
slave_file = args.slave_file
if args.card_file is not None:
cardFile = args.card_file
do_filter = True
if args.debug_abstract is not None:
Registrar.DEBUG_ABSTRACT = args.debug_abstract
if args.debug_parser is not None:
Registrar.DEBUG_PARSER = args.debug_parser
if args.debug_update is not None:
Registrar.DEBUG_UPDATE = args.debug_update
if args.debug_flat is not None:
Registrar.DEBUG_FLAT = args.debug_flat
if args.debug_name is not None:
Registrar.DEBUG_NAME = args.debug_name
if args.debug_address is not None:
Registrar.DEBUG_ADDRESS = args.debug_address
if args.debug_client is not None:
Registrar.DEBUG_CLIENT = args.debug_client
if args.debug_utils is not None:
Registrar.DEBUG_UTILS = args.debug_utils
if args.debug_contact is not None:
Registrar.DEBUG_CONTACT = args.dest='debug_contact'
global_limit = args.limit
# api config
with open(yamlPath) as stream:
optionNamePrefix = 'test_' if testMode else ''
config = yaml.load(stream)
ssh_user = config.get(optionNamePrefix+'ssh_user')
ssh_pass = config.get(optionNamePrefix+'ssh_pass')
ssh_host = config.get(optionNamePrefix+'ssh_host')
ssh_port = config.get(optionNamePrefix+'ssh_port', 22)
m_ssh_user = config.get(optionNamePrefix+'m_ssh_user')
m_ssh_pass = config.get(optionNamePrefix+'m_ssh_pass')
m_ssh_host = config.get(optionNamePrefix+'m_ssh_host')
m_ssh_port = config.get(optionNamePrefix+'m_ssh_port', 22)
remote_bind_host = config.get(optionNamePrefix+'remote_bind_host', '127.0.0.1')
remote_bind_port = config.get(optionNamePrefix+'remote_bind_port', 3306)
db_user = config.get(optionNamePrefix+'db_user')
db_pass = config.get(optionNamePrefix+'db_pass')
db_name = config.get(optionNamePrefix+'db_name')
db_charset = config.get(optionNamePrefix+'db_charset', 'utf8mb4')
wp_srv_offset = config.get(optionNamePrefix+'wp_srv_offset', 0)
m_db_user = config.get(optionNamePrefix+'m_db_user')
m_db_pass = config.get(optionNamePrefix+'m_db_pass')
m_db_name = config.get(optionNamePrefix+'m_db_name')
m_db_host = config.get(optionNamePrefix+'m_db_host')
m_x_cmd = config.get(optionNamePrefix+'m_x_cmd')
m_i_cmd = config.get(optionNamePrefix+'m_i_cmd')
tbl_prefix = config.get(optionNamePrefix+'tbl_prefix', '')
wp_user = config.get(optionNamePrefix+'wp_user', '')
wp_pass = config.get(optionNamePrefix+'wp_pass', '')
store_url = config.get(optionNamePrefix+'store_url', '')
wc_api_key = config.get(optionNamePrefix+'wc_api_key')
wc_api_secret = config.get(optionNamePrefix+'wc_api_secret')
remote_export_folder = config.get(optionNamePrefix+'remote_export_folder', '')
### DISPLAY CONFIG ###
if Registrar.DEBUG_MESSAGE:
if testMode:
print "testMode enabled"
else:
print "testMode disabled"
if not download_slave:
print "no download_slave"
if not download_master:
print "no download_master"
if not update_master:
print "not updating maseter"
if not update_slave:
print "not updating slave"
if not do_filter:
print "not doing filter"
if not do_sync:
print "not doing sync"
if not do_post:
print "not doing post"
### PROCESS CLASS PARAMS ###
FieldGroup.do_post = do_post
SyncUpdate.setGlobals( MASTER_NAME, SLAVE_NAME, merge_mode, DEFAULT_LAST_SYNC)
TimeUtils.setWpSrvOffset(wp_srv_offset)
### SET UP DIRECTORIES ###
for path in (inFolder, outFolder, logFolder, srcFolder, pklFolder):
if not os.path.exists(path):
os.mkdir(path)
fileSuffix = "_test" if testMode else ""
fileSuffix += "_filter" if do_filter else ""
m_x_filename = "act_x"+fileSuffix+"_"+importName+".csv"
m_i_filename = "act_i"+fileSuffix+"_"+importName+".csv"
s_x_filename = "wp_x"+fileSuffix+"_"+importName+".csv"
remoteExportPath = os.path.join(remote_export_folder, m_x_filename)
if download_master:
maPath = os.path.join(inFolder, m_x_filename)
maEncoding = "utf-8"
else:
# maPath = os.path.join(inFolder, "act_x_test_2016-05-03_23-01-48.csv")
maPath = os.path.join(inFolder, master_file)
# maPath = os.path.join(inFolder, "500-act-records-edited.csv")
# maPath = os.path.join(inFolder, "500-act-records.csv")
maEncoding = "utf8"
if download_slave:
saPath = os.path.join(inFolder, s_x_filename)
saEncoding = "utf8"
else:
saPath = os.path.join(inFolder, slave_file)
# saPath = os.path.join(inFolder, "500-wp-records-edited.csv")
saEncoding = "utf8"
moPath = os.path.join(outFolder, m_i_filename)
repPath = os.path.join(outFolder, "usr_sync_report%s.html" % fileSuffix)
WPresCsvPath = os.path.join(outFolder, "sync_report_wp%s.csv" % fileSuffix)
masterResCsvPath = os.path.join(outFolder, "sync_report_act%s.csv" % fileSuffix)
masterDeltaCsvPath = os.path.join(outFolder, "delta_report_act%s.csv" % fileSuffix)
slaveDeltaCsvPath = os.path.join(outFolder, "delta_report_wp%s.csv" % fileSuffix)
mFailPath = os.path.join(outFolder, "act_fails%s.csv" % fileSuffix)
sFailPath = os.path.join(outFolder, "wp_fails%s.csv" % fileSuffix)
sqlPath = os.path.join(srcFolder, "select_userdata_modtime.sql")
# pklPath = os.path.join(pklFolder, "parser_pickle.pkl" )
pklPath = os.path.join(pklFolder, "parser_pickle%s.pkl" % fileSuffix )
logPath = os.path.join(logFolder, "log_%s.txt" % importName)
zipPath = os.path.join(logFolder, "zip_%s.zip" % importName)
### PROCESS OTHER CONFIG ###
assert store_url, "store url must not be blank"
xmlrpc_uri = store_url + 'xmlrpc.php'
json_uri = store_url + 'wp-json/wp/v2'
actFields = ";".join(ColData_User.getACTImportCols())
jsonConnectParams = {
'json_uri': json_uri,
'wp_user': wp_user,
'wp_pass': wp_pass
}
wcApiParams = {
'api_key':wc_api_key,
'api_secret':wc_api_secret,
'url':store_url
}
sqlConnectParams = {
}
actConnectParams = {
'hostname': m_ssh_host,
'port': m_ssh_port,
'username': m_ssh_user,
'password': m_ssh_pass,
}
actDbParams = {
'db_x_exe':m_x_cmd,
'db_i_exe':m_i_cmd,
'db_name': m_db_name,
'db_host': m_db_host,
'db_user': m_db_user,
'db_pass': m_db_pass,
'fields' : actFields,
}
if sinceM: actDbParams['since'] = sinceM
fsParams = {
'importName': importName,
'remote_export_folder': remote_export_folder,
'inFolder': inFolder,
'outFolder': outFolder
}
#########################################
# Prepare Filter Data
#########################################
print debugUtils.hashify("PREPARE FILTER DATA"), timediff()
if do_filter:
filterFiles = {
'users': userFile,
'emails': emailFile,
'cards': cardFile,
}
filterItems = {}
for key, filterFile in filterFiles.items():
if filterFile:
try:
with open(os.path.join(inFolder,filterFile) ) as filterFileObj:
filterItems[key] = [\
re.sub(r'\s*([^\s].*[^\s])\s*(?:\n)', r'\1', line)\
for line in filterFileObj\
]
except IOError, e:
SanitationUtils.safePrint("could not open %s file [%s] from %s" % (
key,
filterFile,
unicode(os.getcwd())
))
raise e
if sinceM:
filterItems['sinceM'] = TimeUtils.wpStrptime(sinceM)
if sinceS:
filterItems['sinceS'] = TimeUtils.wpStrptime(sinceS)
def __init__(self, cols={}, defaults={}, schema='MY', importName="", \
taxoSubs={}, itemSubs={}, taxoDepth=3, itemDepth=2, metaWidth=2):
if self.DEBUG_MRO:
self.registerMessage(' ')
extra_cols = [ 'WNRC', 'RNRC', 'HTML Description' ]
extra_defaults = OrderedDict([
('Sell', 'S'),
('Tax Code When Sold', 'GST'),
('Sell Price Inclusive', 'X'),
('Income Acct', '41000'),
('Use Desc. On Sale', ''),
('Inactive Item', 'N'),
])
extra_taxoSubs = OrderedDict([
('', ''),
])
extra_itemSubs = OrderedDict([
('Body Butter with Shimmer', 'Body Butter w/ Shimmer'),
('Tan Saver Body Wash', 'Body Wash'),
('Full Body Moisturizer', 'Moisturizer'),
('Moisturizing Body Milk', 'Body Milk'),
('Hair Brush', 'Brush'),
('Training Session', 'Session'),
('Skin Trial Pack', "Pack"),
('Trial Pack', "Pack"),
('Starter Package', 'Pack'),
('Sample Pack', "Pack"),
('Evaluation Package', 'Pack'),
('Spare Pot & Hose', 'Pot, Hose'),
('Spare Pots & Hose', 'Pots, Hose'),
('spare pot + Hose', 'pot, hose'),
('spare pots + Hose', 'pots, hose'),
('extraction fans', 'xfan'),
('Low Voltage', 'LV'),
('Double Sided', '2Sided'),
('TechnoTan', 'TT'),
('VuTan', 'VT'),
('EzeBreathe', 'EZB'),
('Sticky Soul', 'SS'),
('My Tan', 'MT'),
('TanSense', 'TS'),
('Tanning Advantage', 'TA'),
('Tanbience', 'TB'),
('Mosaic Minerals', 'MM'),
('Removal', 'Rem.'),
('Remover', 'Rem.'),
('Application', 'App.'),
('Peach & Vanilla', 'P&V'),
('Tamarillo & Papaya', 'T&P'),
('Tamarillo', 'TAM'),
('Lavander & Rosmary', 'L&R'),
('Lavender & Rosemary', 'L&R'),
('Coconut & Lime', 'C&L'),
('Melon & Cucumber', 'M&C'),
('Coconut Cream', 'CC'),
('Black & Silver', 'B&S'),
('Black & Gold', 'B&G'),
('Hot Pink', 'PNK'),
('Hot Lips (Red)', 'RED'),
('Hot Lips Red', 'RED'),
('Hot Lips', 'RED'),
('Silken Chocolate (Bronze)', 'BRZ'),
('Silken Chocolate', 'BRZ'),
('Moon Marvel (Silver)', 'SLV'),
('Dusty Gold', 'GLD'),
('Black', 'BLK'),
('Light Blue', 'LBLU'),
('Dark Blue', 'DBLU'),
('Blue', 'BLU'),
('Green', 'GRN'),
('Pink', 'PNK'),
('White', 'WHI'),
('Grey', 'GRY'),
('Peach', 'PEA'),
('Bronze', 'BRZ'),
('Silver', 'SLV'),
('Gold', 'GLD'),
('Red', 'RED'),
('Cyclone', 'CYC'),
('Classic', 'CLA'),
('Premier', 'PRE'),
('Deluxe', 'DEL'),
('ProMist Cube', 'CUBE'),
('ProMist', 'PRO'),
('Mini Mist', 'MIN'),
('Choc Fudge', 'CFdg.'),
('Choc Mousse', 'Cmou'),
('Ebony', 'Ebny.'),
('Creme Caramel', 'CCarm.'),
('Caramel', 'Carm.'),
('Cappuccino', 'Capp.'),
('Evaluation', 'Eval.'),
('Package', 'Pack'),
('Sample', 'Samp.'),
('sample', 'Samp.'),
('Tan Care', 'TCare'),
('After Care', "ACare"),
('A-Frame', 'AFrm'),
('X-Frame', 'XFrm'),
('Tear Drop Banner', 'TDBnr'),
('Roll Up Banner', 'RUBnr'),
('Hose Fitting', 'Fit.'),
('Magnetic', 'Mag.'),
('Option ', 'Opt.'),
('Style ', 'Sty.'),
('Insert and Frame', 'ins. frm.'),
('Insert Only', 'ins.'),
('Insert', 'ins.'),
('insert', 'ins.'),
('Frame', 'frm.'),
('Foundation', 'Found.'),
('Economy', 'Econ.'),
('Medium-Dark', 'MDark'),
('Medium Dark', 'MDark'),
('Medium', 'Med.'),
('medium', 'med.'),
('Extra Dark', 'XDark'),
('Extra-Dark', 'XDark'),
('Dark', 'Dark'),
('Tanning', 'Tan.'),
('Extra Small', 'XSml.'),
('Small', 'Sml.'),
('Extra Large', 'XLge.'),
('Large', 'Lge.'),
('Ladies', 'Ld.'),
('Mens', 'Mn.'),
('Non Personalized', 'Std.'),
('Personalized', 'Per.'),
('personalized', 'per.'),
('Personalised', 'Per.'),
('personalised', 'per.'),
('Custom Designed', 'Cust.'),
('Refurbished', 'Refurb.'),
('Compressor', 'Cmpr.'),
('Spray Gun', 'Gun'),
('Permanent', 'Perm.'),
('Shimmering', 'Shim.'),
('Screen Printed', 'SP'),
('Embroidered', 'Embr.'),
('Athletic', 'Athl.'),
('Singlet', 'Sing.'),
('Solution', 'Soln.'),
('Flash Tan', 'FTan'),
('Original', 'Orig.'),
('Exfoliating', 'Exfo.'),
('Disposable', 'Disp.'),
('Retractable', 'Ret.'),
('Synthetic', 'SYN'),
('Natural', 'NAT'),
('Bayonet', 'BAY'),
('Hexagonal', 'Hex.'),
('Product Pack', 'PrPck'),
('Complete', 'Compl.'),
('Fanatic', 'Fan.'),
('one', '1'),
('One', '1'),
('two', '2'),
('Two', '2'),
('three', '3'),
('Three', '3'),
('four', '4'),
('Four', '4'),
# ('for', '4'),
('five', '5'),
('Five', '5'),
('six', '6'),
('Six', '6'),
('seven', '7'),
('seven', '7'),
('eight', '8'),
('Eight', '8'),
('nine', '9'),
('Nine', '9'),
(' Plus', '+'),
(' - ', ' '),
(' Pack / ', ' x '),
('with', 'w/'),
('With', 'w/'),
('Box of', 'Box/'),
(' Fitting for ', ' Fit '),
(' Fits ', ' Fit '),
(' and ', ' & '),
(' And ', ' & '),
# (' (2hr)', ''),
(' (sachet)', ''),
(' (pump bottle)', ''),
(' Bottle with Flip Cap', ''),
(' (jar)', ''),
(' (tube)', ''),
(' (spray)', ''),
(' \xe2\x80\x94 ', ' '),
])
if not importName: importName = time.strftime("%Y-%m-%d %H:%M:%S")
cols = listUtils.combineLists( cols, extra_cols )
defaults = listUtils.combineOrderedDicts( defaults, extra_defaults )
taxoSubs = listUtils.combineOrderedDicts( taxoSubs, extra_taxoSubs )
itemSubs = listUtils.combineOrderedDicts( itemSubs, extra_itemSubs )
if not schema: schema = "MY"
super(CSVParse_MYO, self).__init__( cols, defaults, schema, \
taxoSubs, itemSubs, taxoDepth, itemDepth, metaWidth)
if self.DEBUG_MYO:
self.registerMessage( "csvparse initialized with cols: %s" %
SanitationUtils.coerceUnicode(extra_cols))
from utils import SanitationUtils
import json
from wordpress_json import WordpressJsonWrapper
wp = WordpressJsonWrapper('http://www.technotea.com.au/wp-json/wp/v2', 'neil', 'Stretch@6164')
# wp = WordpressJsonWrapper('http://minimac.ddns.me:11182/wp-json/wp/v2', 'neil', 'Stretch6164')
headers = {
"User-Agent": "curl",
"Content-Type": "text/json",
# ...
}
# posts = wp.get_posts(post_id=16886, headers=headers)
# print json.dumps(posts)
# posts = wp.get_user(user_id=9)
# print json.dumps(posts)
fields_json = '{"client_grade": "Silver", "myob_customer_card_id": "", "direct_brand": "Pending", "myob_card_id": "C000009", "act_role": "RN", "billing_company": null, "first_name":"asdasdadadaadggg" }'
fields_json_base64 = SanitationUtils.encodeBase64(fields_json)
# fields_json_base64 = "eyJ1c2VyX2xvZ2luIjogImFkbWluIiwgImZpcnN0X25hbWUiOiAibm/wn5GMb2Twn5GMbGUiLCAidXNlcl91cmwiOiAiaHR0cDovL3d3dy5sYXNlcnBoaWxlLmNvbS9hc2QifQ=="
posts = wp.update_user(user_id=9, data={'tansync_updated_fields': fields_json_base64})
print json.dumps(posts)
posts = wp.get_user(user_id=9)
print json.dumps(posts)
insort(mDeltaUpdates, syncUpdate)
if syncUpdate.sUpdated and syncUpdate.sDeltas:
insort(sDeltaUpdates, syncUpdate)
if not syncUpdate:
continue
if syncUpdate.sUpdated:
syncSlaveUpdates = syncUpdate.getSlaveUpdates()
if 'E-mail' in syncSlaveUpdates:
newEmail = syncSlaveUpdates['E-mail']
if newEmail in saParser.emails:
mObjects = [mObject]
sObjects = [sObject] + saParser.emails[newEmail]
SanitationUtils.safePrint("duplicate emails", mObjects, sObjects)
emailConflictMatches.addMatch(Match(mObjects, sObjects))
continue
if(not syncUpdate.importantStatic):
if(syncUpdate.mUpdated and syncUpdate.sUpdated):
if(syncUpdate.sMod):
insort(problematicUpdates, syncUpdate)
continue
elif(syncUpdate.mUpdated and not syncUpdate.sUpdated):
insort(nonstaticMUpdates, syncUpdate)
if(syncUpdate.sMod):
insort(problematicUpdates, syncUpdate)
continue
elif(syncUpdate.sUpdated and not syncUpdate.mUpdated):
insort(nonstaticSUpdates, syncUpdate)
def __str__(self):
return SanitationUtils.coerce_bytestr(self.__unicode__())
