def registerObject(self, objectData): if not self.validateFilters(objectData): self.registerFiltered(objectData) return email = objectData.email if email and SanitationUtils.stringIsEmail(email): self.registerEmail(objectData, email) else: if self.DEBUG_FLAT: self.registerWarning("invalid email address: %s" % email) self.registerNoEmail(objectData) role = objectData.role if role: self.registerRole(objectData, role) else: # self.registerWarning("invalid role: %s"%role) self.registerNoRole(objectData) card = objectData.MYOBID if card and SanitationUtils.stringIsMYOBID(card): self.registerCard(objectData, card) else: self.registerNoCard(objectData) username = objectData.username if username: self.registerUsername(objectData, username) else: if self.DEBUG_FLAT: self.registerWarning("invalid username: %s" % username) self.registerNoUsername(objectData) # company = objectData['Company'] # # if self.DEBUG_FLAT: SanitationUtils.safePrint(repr(objectData), company) # if company: # self.registerCompany(objectData, company) addresses = [objectData.billing_address, objectData.shipping_address] for address in filter(None, addresses): if not address.valid: reason = address.reason assert reason, "there must be a reason that this address is invalid: " + address self.registerBadAddress(objectData, address) name = objectData.name # print "NAME OF %s IS %s" % (repr(objectData), name.__str__(out_schema="flat")) if not name.valid: reason = name.reason assert reason, "there must be a reason that this name is invalid: " + name # print "registering bad name: ", SanitationUtils.coerceBytes(name) self.registerBadName(objectData, name) # emails = objectData.emails # if not emails.valid: # reason = emails.reason # self.registerBadEmail(objectData, emails) super(CSVParse_User, self).registerObject(objectData)
def append(self, objectData): assert issubclass(objectData.__class__, ImportShopMixin), \ "object must be subclass of ImportShopMixin not %s : %s" % ( SanitationUtils.coerceUnicode(objectData.__class__), SanitationUtils.coerceUnicode(objectData) ) return super(ShopProdList, self).append(objectData)
def tabulate(self, cols=None, tablefmt=None): objs = self.objects sanitizer = self.getSanitizer(tablefmt) # sanitizer = (lambda x: str(x)) if tablefmt == 'html' else SanitationUtils.makeSafeOutput if objs: if not cols: cols = self.reportCols header = [self.objList_type] for col in cols.keys(): header += [col] table = [] for obj in objs: row = [obj.identifier] for col in cols.keys(): # if col == 'Address': # print repr(str(obj.get(col))), repr(sanitizer(obj.get(col))) row += [ sanitizer(obj.get(col) )or ""] try: SanitationUtils.coerceUnicode(row[-1]) except: Registrar.registerWarning("can't turn row into unicode: %s" % SanitationUtils.coerceUnicode(row)) table += [row] # table += [[obj.index] + [ sanitizer(obj.get(col) )or "" for col in cols.keys()]] # print "table", table # SanitationUtils.safePrint(table) # return SanitationUtils.coerceUnicode(tabulate(table, headers=header, tablefmt=tablefmt)) return (tabulate(table, headers=header, tablefmt=tablefmt)) # print repr(table) # print repr(table.encode('utf8')) # return table.encode('utf8') else: Registrar.registerWarning("cannot tabulate Objlist: there are no objects") return ""
def valuesSimilar(self, col, mValue, sValue): response = super(SyncUpdate_Cat_Woo, self).valuesSimilar(col, mValue, sValue) if not response: if col is 'descsum': mDesc = SanitationUtils.similarMarkupComparison(mValue) sDesc = SanitationUtils.similarMarkupComparison(sValue) if mDesc == sDesc: response = True if self.DEBUG_UPDATE: self.registerMessage(self.testToStr(col, mValue.__repr__(), sValue.__repr__(), response)) return response
def printBasicColumns(cls, objects): obj_list = cls.objectContainer.container() for _object in objects: obj_list.append(_object) cols = cls.objectContainer.container.getBasicCols() SanitationUtils.safePrint( obj_list.tabulate( cols, tablefmt = 'simple' ))
def valuesSimilar(self, col, mValue, sValue): response = False if not (mValue or sValue): response = True elif not (mValue and sValue): response = False #check if they are similar if SanitationUtils.similarComparison(mValue) == SanitationUtils.similarComparison(sValue): response = True if self.DEBUG_UPDATE: self.registerMessage(self.testToStr(col, mValue, sValue, response)) return response
def execSilentCommandAssert(self, command): self.assertConnect() stdin, stdout, stderr = self.service.exec_command(command) if stdin: pass #gets rid of annoying warnings possible_errors = stdout.readlines() + stderr.readlines() for error in possible_errors: if re.match("^Countries.*", error): print error continue assert not error, "command <%s> returned errors: %s" % ( SanitationUtils.coerceUnicode(command), SanitationUtils.coerceUnicode(error) )
def printBasicColumns(users): # print len(users) usrList = UsrObjList() for user in users: usrList.append(user) # SanitationUtils.safePrint( "BILLING ADDRESS:", repr(user), user['First Name'], user.get('First Name'), user.name.__unicode__(out_schema="flat")) cols = colData.getBasicCols() SanitationUtils.safePrint( usrList.tabulate( cols, tablefmt = 'simple' ))
def __init__(self, cols, defaults, schema, **kwargs): if self.DEBUG_MRO: self.registerMessage('CSVParse_Gen_Tree') taxoSubs = kwargs.pop('taxoSubs', {}) itemSubs = kwargs.pop('itemSubs', {}) extra_defaults = OrderedDict([ ('CVC', '0'), ('code', ''), ('name', ''), ('fullname', ''), ('description', ''), ('HTML Description', ''), ('imglist', []) ]) extra_taxoSubs = OrderedDict([ ('', ''), ]) extra_itemSubs = OrderedDict([ ('Hot Pink', 'Pink'), ('Hot Lips (Red)', 'Red'), ('Hot Lips', 'Red'), ('Silken Chocolate (Bronze)', 'Bronze'), ('Silken Chocolate', 'Bronze'), ('Moon Marvel (Silver)', 'Silver'), ('Dusty Gold', 'Gold'), ('Screen Printed', ''), ('Embroidered', ''), ]) extra_cols = [schema] cols = listUtils.combineLists( cols, extra_cols ) defaults = listUtils.combineOrderedDicts( defaults, extra_defaults ) super(CSVParse_Gen_Tree, self).__init__( cols, defaults, **kwargs) # CSVParse_Gen_Mixin.__init__(self, schema) metaWidth = kwargs.get('metaWidth', 2) assert metaWidth >= 2, "metaWidth must be greater than 2 for a GEN subclass" self.schema = schema self.taxoSubs = listUtils.combineOrderedDicts( taxoSubs, extra_taxoSubs ) self.itemSubs = listUtils.combineOrderedDicts( itemSubs, extra_itemSubs ) self.taxoRegex = SanitationUtils.compileRegex(self.taxoSubs) self.itemRegex = SanitationUtils.compileRegex(self.itemSubs) if self.DEBUG_GEN: self.registerMessage("taxoDepth: {}".format(self.taxoDepth), 'CSVParse_Gen_Tree.__init__') self.registerMessage("itemDepth: {}".format(self.itemDepth), 'CSVParse_Gen_Tree.__init__') self.registerMessage("maxDepth: {}".format(self.maxDepth), 'CSVParse_Gen_Tree.__init__') self.registerMessage("metaWidth: {}".format(self.metaWidth), 'CSVParse_Gen_Tree.__init__') self.registerMessage("schema: {}".format(self.schema), 'CSVParse_Gen_Tree.__init__')
def update_meta(self, newmeta): oldmeta = self.read_meta() newmeta = dict([(key, SanitationUtils.coerceAscii(value)) for key, value in newmeta.items()]) changed = [] for key in ['title', 'description']: if SanitationUtils.similarComparison(oldmeta[key]) != SanitationUtils.similarComparison(newmeta[key]): changed += [key] if self.DEBUG_IMG: self.registerMessage( u"changing imgmeta[%s] from %s to %s" % (key, repr(oldmeta[key]), repr(newmeta[key])), self.fname ) if changed: self.write_meta(newmeta['title'], newmeta['description'])
def testToStr(self, col, val1, val2, res): return u"testing col %s: %s | %s -> %s" % ( unicode(col), repr(val1), repr(val2), SanitationUtils.boolToTruishString(res) )
def getSlaveUpdatesNativeRecursive(self, col, updates=None): if updates == None: updates = OrderedDict() SanitationUtils.safePrint("getting updates for col %s, updates: %s" % (col, str(updates))) if col in self.colData.data.keys(): data = self.colData.data[col] if data.get(self.s_meta_target): data_s = data.get(self.s_meta_target,{}) if not data_s.get('final') and data_s.get('key'): updates[data_s.get('key')] = self.newSObject.get(col) if data.get('aliases'): data_aliases = data.get('aliases') for alias in data_aliases: if self.sColSemiStatic(alias): continue updates = self.getSlaveUpdatesNativeRecursive(alias, updates) return updates
def toHTML(self): colNames = self.getColNames() html = u'<table class="shop_table lasercommerce pricing_table table table-striped">' html += '<thead><tr>' for col, name in colNames.items(): colClass = SanitationUtils.sanitizeClass(col) html += '<th class=%s>' % colClass html += bleach.clean(name) html += '</th>' html += '</tr></thead>' ruleLines = self.getRuleLines() self.registerMessage("ruleLines: %s" % (ruleLines)) for ruleLineData in ruleLines: lineType = ruleLineData.get('Discount Type','') html += '<tr>' for col in colNames.keys(): value = ruleLineData.get(col, '') if col == 'Discount' and lineType in ['PDSC']: value += '%' html += '<td>' html += value #bleach.clean(value) html += '</td>' html += '</tr>' html += "</table>" return html.encode('UTF-8')
def __init__(self, data=None): if data is None: data = {} assert hasattr(data, 'items'), "Container initialized with non-dict" super(Item, self).__init__([ (key, SanitationUtils.sanitize_container_value(val)) for key, val \ in data.items() ])
def sanitizeValue(self, col, value): # print "sanitizing", col, repr(value) if 'phone' in col.lower(): if 'preferred' in col.lower(): if value and len(SanitationUtils.stripNonNumbers(value)) > 1: # print "value nullified", value return "" return value
def tabulate(self, tablefmt=None): if(self): prefix, suffix = "", "" delimeter = "\n" if tablefmt == 'html': delimeter = '' prefix = '<div class="matchList">' suffix = '</div>' return prefix + delimeter.join( [SanitationUtils.coerceBytes(match.tabulate(tablefmt=tablefmt)) for match in self if match] ) + suffix else: return ""
def writeSection(title, description, data, length = 0, html_class="results_section"): sectionID = SanitationUtils.makeSafeClass(title) description = "%s %s" % (str(length) if length else "No", description) outFile.write('<div class="%s">'% html_class ) outFile.write('<a data-toggle="collapse" href="#%s" aria-expanded="true" data-target="#%s" aria-controls="%s">' % (sectionID, sectionID, sectionID)) outFile.write('<h2>%s (%d)</h2>' % (title, length)) outFile.write('</a>') outFile.write('<div class="collapse" id="%s">' % sectionID) outFile.write('<p class="description">%s</p>' % description) outFile.write('<p class="data">' ) outFile.write( re.sub("<table>","<table class=\"table table-striped\">",data) ) outFile.write('</p>') outFile.write('</div>') outFile.write('</div>')
def displayUpdateList(self, updateList, tablefmt=None): if updateList: delimeter = "<br/>" if tablefmt=="html" else "\n" subject_fmt = "<h4>%s</h4>" if tablefmt=="html" else "%s" # header = ["Column", "Reason", "Old", "New"] header = OrderedDict([ ('col', 'Column'), ('reason', 'Reason'), ('oldLoserValue', 'Old'), ('oldWinnerValue', 'New'), ('mColTime', 'M TIME'), ('sColTime', 'S TIME'), ]) subjects = {} for warnings in updateList.values(): for warning in warnings: subject = warning['subject'] if subject not in subjects.keys(): subjects[subject] = [] warning_fmtd = dict([\ (key, SanitationUtils.sanitizeForTable(val))\ for key, val in warning.items()\ if key in header ]) for key in ['mColTime', 'sColTime']: try: rawTime = int(warning[key]) if rawTime: warning_fmtd[key] = TimeUtils.wpTimeToString(rawTime) except Exception, e: pass subjects[subject].append(warning_fmtd) tables = [] for subject, warnings in subjects.items(): antiSubject_fmtd = subject_fmt % self.opposite_src(subject) table = [header]+warnings table_fmtd = tabulate(table, headers='firstrow', \ tablefmt=tablefmt ) tables.append(delimeter.join([antiSubject_fmtd, table_fmtd])) return delimeter.join(tables)
def valuesSimilar(self, col, mValue, sValue): response = super(SyncUpdate_Usr, self).valuesSimilar(col, mValue, sValue) if not response: if "phone" in col.lower(): if "preferred" in col.lower(): mPreferred = SanitationUtils.similarTruStrComparison(mValue) sPreferred = SanitationUtils.similarTruStrComparison(sValue) # print repr(mValue), " -> ", mPreferred # print repr(sValue), " -> ", sPreferred if mPreferred == sPreferred: response = True else: mPhone = SanitationUtils.similarPhoneComparison(mValue) sPhone = SanitationUtils.similarPhoneComparison(sValue) plen = min(len(mPhone), len(sPhone)) if plen > 7 and mPhone[-plen] == sPhone[-plen]: response = True elif "role" in col.lower(): mRole = SanitationUtils.similarComparison(mValue) sRole = SanitationUtils.similarComparison(sValue) if mRole == 'rn': mRole = '' if sRole == 'rn': sRole = '' if mRole == sRole: response = True elif "address" in col.lower() and isinstance(mValue, ContactAddress): if( mValue != sValue ): pass # print "M: ", mValue.__str__(out_schema="flat"), "S: ", sValue.__str__(out_schema="flat") response = mValue.similar(sValue) elif "web site" in col.lower(): if SanitationUtils.similarURLComparison(mValue) == SanitationUtils.similarURLComparison(sValue): response = True if self.DEBUG_UPDATE: self.registerMessage(self.testToStr(col, mValue.__str__(), sValue.__str__(), response)) return response
def valuesSimilar(self, col, mValue, sValue): response = super(SyncUpdate_Prod, self).valuesSimilar(col, mValue, sValue) if col in self.colData.data: colData = self.colData.data[col] if colData.get('type'): if colData.get('type') == 'currency': mPrice = SanitationUtils.similarCurrencyComparison(mValue) sPrice = SanitationUtils.similarCurrencyComparison(sValue) if mPrice == sPrice: response = True elif not response: if col is 'descsum': mDesc = SanitationUtils.similarMarkupComparison(mValue) sDesc = SanitationUtils.similarMarkupComparison(sValue) if mDesc == sDesc: response = True elif col is 'CVC': mCom = SanitationUtils.similarComparison(mValue) or '0' sCom = SanitationUtils.similarComparison(sValue) or '0' if mCom == sCom: response = True if self.DEBUG_UPDATE: self.registerMessage(self.testToStr(col, mValue.__repr__(), sValue.__repr__(), response)) return response
def sanitizeCell(self, cell): return SanitationUtils.sanitizeCell(cell)
def __str__(self): return SanitationUtils.coerce_bytestr(self.__unicode__())
def tabulate(self, cols=None, tablefmt=None): objlist = self.getObjList() return SanitationUtils.coerceBytes(objlist.tabulate(cols, tablefmt))
sshClient = paramiko.SSHClient() sshClient.set_missing_host_key_policy(paramiko.AutoAddPolicy()) try: sshClient.connect(**paramikoSSHParams) stdin, stdout, stderr = sshClient.exec_command(command) possible_errors = stdout.readlines() assert not possible_errors, "command returned errors: " + possible_errors try: sftpClient = sshClient.open_sftp() sftpClient.chdir(remoteExportFolder) fstat = sftpClient.stat(exportFilename) if fstat: sftpClient.get(exportFilename, maPath) except Exception, e: SanitationUtils.safePrint("ERROR IN SFTP: " + str(e)) finally: sftpClient.close() except Exception, e: SanitationUtils.safePrint("ERROR IN SSH: " + str(e)) finally: sshClient.close() maParser = CSVParse_User( cols = colData.getImportCols(), defaults = colData.getDefaults(), contact_schema = 'act', ) maParser.analyseFile(maPath, maEncoding)
# -*- coding: utf-8 -*- from utils import SanitationUtils from kitchen.text import converters def printandrep(name, thing, back = ""): fmt_str = "%32s |" print fmt_str % name, fmt_str % converters.to_bytes(thing), fmt_str % repr(thing), fmt_str % converters.to_bytes(back), fmt_str % converters.to_bytes(unicode(back)) u_str = u"<\U0001F44C'&>" utf8_str = SanitationUtils.unicodeToUTF8(u_str) utf8_back = SanitationUtils.utf8ToUnicode(utf8_str) xml_str = SanitationUtils.unicodeToXml(u_str) xml_back = SanitationUtils.xmlToUnicode(xml_str) ascii_str = SanitationUtils.unicodeToAscii(u_str) ascii_back = SanitationUtils.asciiToUnicode(u_str) printandrep( "u_str", u_str ) printandrep( "utf8_str", utf8_str, utf8_back ) printandrep( "xml_str", xml_str, xml_back ) printandrep( "ascii_str", ascii_str, ascii_back ) print SanitationUtils.unicodeToUTF8(None) print SanitationUtils.utf8ToUnicode(None) print SanitationUtils.unicodeToXml(None) print SanitationUtils.xmlToUnicode(None) print SanitationUtils.unicodeToAscii(None) print SanitationUtils.asciiToUnicode(None) print SanitationUtils.coerceUnicode(None) SanitationUtils.safePrint(None) print converters.to_bytes(SanitationUtils.coerceUnicode("\xf0\x9f\x91\x8c"))
def initContactObjects(self, data): # emails_kwargs = OrderedDict(filter(None, [\ # ((key, data.get(value)) if data.get(value) else None) for key, value in\ # { # 'email' : 'E-mail', # 'personal_email': 'Personal E-mail', # }.items() # ])) # # self['E-mails'] = EmailFields( # **emails_kwargs # ) name_kwargs = OrderedDict( filter( None, [ ((key, data.get(value)) if data.get(value) else None) for key, value in { "first_name": "First Name", "middle_name": "Middle Name", "family_name": "Surname", "name_prefix": "Name Prefix", "name_suffix": "Name Suffix", "contact": "Contact", "company": "Company", "city": "City", "country": "Country", "state": "State", }.items() ], ) ) self["Name"] = ContactName(**name_kwargs) assert self["Name"] is not None, "contact is missing mandatory fields: something went wrong" address_kwargs = OrderedDict( filter( None, [ ((key, data.get(value)) if data.get(value) else None) for key, value in { "line1": "Address 1", "line2": "Address 2", "city": "City", "postcode": "Postcode", "state": "State", "country": "Country", "company": "Company", }.items() ], ) ) self["Address"] = ContactAddress(**address_kwargs) # print "ADDRESS: ", self['Address'] alt_address_kwargs = OrderedDict( filter( None, [ ((key, data.get(value)) if data.get(value) else None) for key, value in { "line1": "Home Address 1", "line2": "Home Address 2", "city": "Home City", "postcode": "Home Postcode", "state": "Home State", "country": "Home Country", "company": "Company", }.items() ], ) ) self["Home Address"] = ContactAddress(**alt_address_kwargs) # print "HOME ADDRESS: ", self['Home Address'] phone_kwargs = OrderedDict( filter( None, [ ((key, data.get(value)) if data.get(value) else None) for key, value in { "mob_number": "Mobile Phone", "tel_number": "Phone", "fax_number": "Fax", "mob_pref": "Mobile Phone Preferred", "tel_pref": "Phone Preferred", }.items() ], ) ) self["Phone Numbers"] = ContactPhones(**phone_kwargs) social_media_kwargs = OrderedDict( filter( None, [ ((key, data.get(value)) if data.get(value) else None) for key, value in { "facebook": "Facebook Username", "twitter": "Twitter Username", "gplus": "GooglePlus Username", "instagram": "Instagram Username", "website": "Web Site", }.items() ], ) ) self["Social Media"] = SocialMediaFields(**social_media_kwargs) emails = [] if data.get("E-mail"): emails = listUtils.combineLists(emails, SanitationUtils.findallEmails(data["E-mail"])) if data.get("Personal E-mail"): emails = listUtils.combineLists(emails, SanitationUtils.findallEmails(data.get("Personal E-mail"))) self["E-mail"] = emails.pop(0) if emails else None self["Personal E-mail"] = ", ".join(emails) urls = [] if data.get("Web Site"): urls = listUtils.combineLists(urls, SanitationUtils.findallURLs(data["Web Site"])) self["Web Site"] = urls.pop(0) if urls else None # if not self['Emails'].valid: # self['emails_reason'] = '\n'.join(filter(None, [ # self['Emails'].reason, # ])) if not self["Address"].valid or not self["Home Address"].valid: self["address_reason"] = "\n".join( filter( None, [ "ADDRESS: " + self["Address"].reason if not self["Address"].valid else None, "HOME ADDRESS: " + self["Home Address"].reason if not self["Home Address"].valid else None, ], ) ) if not self["Name"].valid: self["name_reason"] = "\n".join(filter(None, [self["Name"].reason])) if not self["Phone Numbers"].valid: self["phone_reason"] = "\n".join(filter(None, [self["Phone Numbers"].reason])) if not self["Social Media"].valid: self["social_reason"] = "\n".join(filter(None, [self["Social Media"].reason]))
def main(): global testMode, inFolder, outFolder, logFolder, srcFolder, pklFolder, \ yamlPath, repPath, mFailPath, sFailPath, logPath, zipPath userFile = cardFile = emailFile = sinceM = sinceS = False ### OVERRIDE CONFIG WITH YAML FILE ### with open(yamlPath) as stream: config = yaml.load(stream) if 'inFolder' in config.keys(): inFolder = config['inFolder'] if 'outFolder' in config.keys(): outFolder = config['outFolder'] if 'logFolder' in config.keys(): logFolder = config['logFolder'] #mandatory merge_mode = config.get('merge_mode', 'sync') MASTER_NAME = config.get('master_name', 'MASTER') SLAVE_NAME = config.get('slave_name', 'SLAVE') DEFAULT_LAST_SYNC = config.get('default_last_sync') master_file = config.get('master_file', '') slave_file = config.get('slave_file', '') userFile = config.get('userFile') cardFile = config.get('cardFile') emailFile = config.get('emailFile') sinceM = config.get('sinceM') sinceS = config.get('sinceS') download_slave = config.get('download_slave') download_master = config.get('download_master') update_slave = config.get('update_slave') update_master = config.get('update_master') do_filter = config.get('do_filter') do_problematic = config.get('do_problematic') do_post = config.get('do_post') do_sync = config.get('do_sync') ### OVERRIDE CONFIG WITH ARGPARSE ### parser = argparse.ArgumentParser(description = 'Merge contact records between two databases') group = parser.add_mutually_exclusive_group() group.add_argument("-v", "--verbosity", action="count", help="increase output verbosity") group.add_argument("-q", "--quiet", action="store_true") group = parser.add_mutually_exclusive_group() group.add_argument('--testmode', help='Run in test mode with test databases', action='store_true', default=None) group.add_argument('--livemode', help='Run the script on the live databases', action='store_false', dest='testmode') group = parser.add_mutually_exclusive_group() group.add_argument('--download-master', help='download the master data', action="store_true", default=None) group.add_argument('--skip-download-master', help='use the local master file instead\ of downloading the master data', action="store_false", dest='download_master') group = parser.add_mutually_exclusive_group() group.add_argument('--download-slave', help='download the slave data', action="store_true", default=None) group.add_argument('--skip-download-slave', help='use the local slave file instead\ of downloading the slave data', action="store_false", dest='download_slave') group = parser.add_mutually_exclusive_group() group.add_argument('--update-master', help='update the master database', action="store_true", default=None) group.add_argument('--skip-update-master', help='don\'t update the master database', action="store_false", dest='update_master') group = parser.add_mutually_exclusive_group() group.add_argument('--update-slave', help='update the slave database', action="store_true", default=None) group.add_argument('--skip-update-slave', help='don\'t update the slave database', action="store_false", dest='update_slave') group = parser.add_mutually_exclusive_group() group.add_argument('--do-filter', help='filter the databases', action="store_true", default=None) group.add_argument('--skip-filter', help='don\'t filter the databases', action="store_false", dest='do_filter') group = parser.add_mutually_exclusive_group() group.add_argument('--do-sync', help='sync the databases', action="store_true", default=None) group.add_argument('--skip-sync', help='don\'t sync the databases', action="store_false", dest='do_sync') group = parser.add_mutually_exclusive_group() group.add_argument('--do-problematic', help='make problematic updates to the databases', action="store_true", default=None) group.add_argument('--skip-problematic', help='don\'t make problematic updates to the databases', action="store_false", dest='do_problematic') group = parser.add_mutually_exclusive_group() group.add_argument('--do-post', help='post process the contacts', action="store_true", default=None) group.add_argument('--skip-post', help='don\'t post process the contacts', action="store_false", dest='do_post') parser.add_argument('--m-ssh-host', help='location of master ssh server') parser.add_argument('--m-ssh-port', type=int, help='location of master ssh port') parser.add_argument('--limit', type=int, help='global limit of objects to process') parser.add_argument('--master-file', help='location of master file') parser.add_argument('--slave-file', help='location of slave file') parser.add_argument('--card-file') group = parser.add_argument_group() group.add_argument('--debug-abstract', action='store_true', dest='debug_abstract') group.add_argument('--debug-parser', action='store_true', dest='debug_parser') group.add_argument('--debug-update', action='store_true', dest='debug_update') group.add_argument('--debug-flat', action='store_true', dest='debug_flat') group.add_argument('--debug-name', action='store_true', dest='debug_name') group.add_argument('--debug-address', action='store_true', dest='debug_address') group.add_argument('--debug-client', action='store_true', dest='debug_client') group.add_argument('--debug-utils', action='store_true', dest='debug_utils') group.add_argument('--debug-contact', action='store_true', dest='debug_contact') args = parser.parse_args() if args: print args if args.verbosity > 0: Registrar.DEBUG_PROGRESS = True Registrar.DEBUG_ERROR = True if args.verbosity > 1: Registrar.DEBUG_MESSAGE = True if args.quiet: Registrar.DEBUG_PROGRESS = False Registrar.DEBUG_ERROR = False Registrar.DEBUG_MESSAGE = False if args.testmode is not None: testMode = args.testmode if args.download_slave is not None: download_slave = args.download_slave if args.download_master is not None: download_master = args.download_master if args.update_slave is not None: update_slave = args.update_slave if args.update_master is not None: update_master = args.update_master if args.do_filter is not None: do_filter = args.do_filter if args.do_sync is not None: do_sync = args.do_sync if args.do_problematic is not None: do_problematic = args.do_problematic if args.do_post is not None: do_post = args.do_post if args.m_ssh_port: m_ssh_port = args.m_ssh_port if args.m_ssh_host: m_ssh_host = args.m_ssh_host if args.master_file is not None: download_master = False master_file = args.master_file if args.slave_file is not None: download_slave = False slave_file = args.slave_file if args.card_file is not None: cardFile = args.card_file do_filter = True if args.debug_abstract is not None: Registrar.DEBUG_ABSTRACT = args.debug_abstract if args.debug_parser is not None: Registrar.DEBUG_PARSER = args.debug_parser if args.debug_update is not None: Registrar.DEBUG_UPDATE = args.debug_update if args.debug_flat is not None: Registrar.DEBUG_FLAT = args.debug_flat if args.debug_name is not None: Registrar.DEBUG_NAME = args.debug_name if args.debug_address is not None: Registrar.DEBUG_ADDRESS = args.debug_address if args.debug_client is not None: Registrar.DEBUG_CLIENT = args.debug_client if args.debug_utils is not None: Registrar.DEBUG_UTILS = args.debug_utils if args.debug_contact is not None: Registrar.DEBUG_CONTACT = args.dest='debug_contact' global_limit = args.limit # api config with open(yamlPath) as stream: optionNamePrefix = 'test_' if testMode else '' config = yaml.load(stream) ssh_user = config.get(optionNamePrefix+'ssh_user') ssh_pass = config.get(optionNamePrefix+'ssh_pass') ssh_host = config.get(optionNamePrefix+'ssh_host') ssh_port = config.get(optionNamePrefix+'ssh_port', 22) m_ssh_user = config.get(optionNamePrefix+'m_ssh_user') m_ssh_pass = config.get(optionNamePrefix+'m_ssh_pass') m_ssh_host = config.get(optionNamePrefix+'m_ssh_host') m_ssh_port = config.get(optionNamePrefix+'m_ssh_port', 22) remote_bind_host = config.get(optionNamePrefix+'remote_bind_host', '127.0.0.1') remote_bind_port = config.get(optionNamePrefix+'remote_bind_port', 3306) db_user = config.get(optionNamePrefix+'db_user') db_pass = config.get(optionNamePrefix+'db_pass') db_name = config.get(optionNamePrefix+'db_name') db_charset = config.get(optionNamePrefix+'db_charset', 'utf8mb4') wp_srv_offset = config.get(optionNamePrefix+'wp_srv_offset', 0) m_db_user = config.get(optionNamePrefix+'m_db_user') m_db_pass = config.get(optionNamePrefix+'m_db_pass') m_db_name = config.get(optionNamePrefix+'m_db_name') m_db_host = config.get(optionNamePrefix+'m_db_host') m_x_cmd = config.get(optionNamePrefix+'m_x_cmd') m_i_cmd = config.get(optionNamePrefix+'m_i_cmd') tbl_prefix = config.get(optionNamePrefix+'tbl_prefix', '') wp_user = config.get(optionNamePrefix+'wp_user', '') wp_pass = config.get(optionNamePrefix+'wp_pass', '') store_url = config.get(optionNamePrefix+'store_url', '') wc_api_key = config.get(optionNamePrefix+'wc_api_key') wc_api_secret = config.get(optionNamePrefix+'wc_api_secret') remote_export_folder = config.get(optionNamePrefix+'remote_export_folder', '') ### DISPLAY CONFIG ### if Registrar.DEBUG_MESSAGE: if testMode: print "testMode enabled" else: print "testMode disabled" if not download_slave: print "no download_slave" if not download_master: print "no download_master" if not update_master: print "not updating maseter" if not update_slave: print "not updating slave" if not do_filter: print "not doing filter" if not do_sync: print "not doing sync" if not do_post: print "not doing post" ### PROCESS CLASS PARAMS ### FieldGroup.do_post = do_post SyncUpdate.setGlobals( MASTER_NAME, SLAVE_NAME, merge_mode, DEFAULT_LAST_SYNC) TimeUtils.setWpSrvOffset(wp_srv_offset) ### SET UP DIRECTORIES ### for path in (inFolder, outFolder, logFolder, srcFolder, pklFolder): if not os.path.exists(path): os.mkdir(path) fileSuffix = "_test" if testMode else "" fileSuffix += "_filter" if do_filter else "" m_x_filename = "act_x"+fileSuffix+"_"+importName+".csv" m_i_filename = "act_i"+fileSuffix+"_"+importName+".csv" s_x_filename = "wp_x"+fileSuffix+"_"+importName+".csv" remoteExportPath = os.path.join(remote_export_folder, m_x_filename) if download_master: maPath = os.path.join(inFolder, m_x_filename) maEncoding = "utf-8" else: # maPath = os.path.join(inFolder, "act_x_test_2016-05-03_23-01-48.csv") maPath = os.path.join(inFolder, master_file) # maPath = os.path.join(inFolder, "500-act-records-edited.csv") # maPath = os.path.join(inFolder, "500-act-records.csv") maEncoding = "utf8" if download_slave: saPath = os.path.join(inFolder, s_x_filename) saEncoding = "utf8" else: saPath = os.path.join(inFolder, slave_file) # saPath = os.path.join(inFolder, "500-wp-records-edited.csv") saEncoding = "utf8" moPath = os.path.join(outFolder, m_i_filename) repPath = os.path.join(outFolder, "usr_sync_report%s.html" % fileSuffix) WPresCsvPath = os.path.join(outFolder, "sync_report_wp%s.csv" % fileSuffix) masterResCsvPath = os.path.join(outFolder, "sync_report_act%s.csv" % fileSuffix) masterDeltaCsvPath = os.path.join(outFolder, "delta_report_act%s.csv" % fileSuffix) slaveDeltaCsvPath = os.path.join(outFolder, "delta_report_wp%s.csv" % fileSuffix) mFailPath = os.path.join(outFolder, "act_fails%s.csv" % fileSuffix) sFailPath = os.path.join(outFolder, "wp_fails%s.csv" % fileSuffix) sqlPath = os.path.join(srcFolder, "select_userdata_modtime.sql") # pklPath = os.path.join(pklFolder, "parser_pickle.pkl" ) pklPath = os.path.join(pklFolder, "parser_pickle%s.pkl" % fileSuffix ) logPath = os.path.join(logFolder, "log_%s.txt" % importName) zipPath = os.path.join(logFolder, "zip_%s.zip" % importName) ### PROCESS OTHER CONFIG ### assert store_url, "store url must not be blank" xmlrpc_uri = store_url + 'xmlrpc.php' json_uri = store_url + 'wp-json/wp/v2' actFields = ";".join(ColData_User.getACTImportCols()) jsonConnectParams = { 'json_uri': json_uri, 'wp_user': wp_user, 'wp_pass': wp_pass } wcApiParams = { 'api_key':wc_api_key, 'api_secret':wc_api_secret, 'url':store_url } sqlConnectParams = { } actConnectParams = { 'hostname': m_ssh_host, 'port': m_ssh_port, 'username': m_ssh_user, 'password': m_ssh_pass, } actDbParams = { 'db_x_exe':m_x_cmd, 'db_i_exe':m_i_cmd, 'db_name': m_db_name, 'db_host': m_db_host, 'db_user': m_db_user, 'db_pass': m_db_pass, 'fields' : actFields, } if sinceM: actDbParams['since'] = sinceM fsParams = { 'importName': importName, 'remote_export_folder': remote_export_folder, 'inFolder': inFolder, 'outFolder': outFolder } ######################################### # Prepare Filter Data ######################################### print debugUtils.hashify("PREPARE FILTER DATA"), timediff() if do_filter: filterFiles = { 'users': userFile, 'emails': emailFile, 'cards': cardFile, } filterItems = {} for key, filterFile in filterFiles.items(): if filterFile: try: with open(os.path.join(inFolder,filterFile) ) as filterFileObj: filterItems[key] = [\ re.sub(r'\s*([^\s].*[^\s])\s*(?:\n)', r'\1', line)\ for line in filterFileObj\ ] except IOError, e: SanitationUtils.safePrint("could not open %s file [%s] from %s" % ( key, filterFile, unicode(os.getcwd()) )) raise e if sinceM: filterItems['sinceM'] = TimeUtils.wpStrptime(sinceM) if sinceS: filterItems['sinceS'] = TimeUtils.wpStrptime(sinceS)
def __init__(self, cols={}, defaults={}, schema='MY', importName="", \ taxoSubs={}, itemSubs={}, taxoDepth=3, itemDepth=2, metaWidth=2): if self.DEBUG_MRO: self.registerMessage(' ') extra_cols = [ 'WNRC', 'RNRC', 'HTML Description' ] extra_defaults = OrderedDict([ ('Sell', 'S'), ('Tax Code When Sold', 'GST'), ('Sell Price Inclusive', 'X'), ('Income Acct', '41000'), ('Use Desc. On Sale', ''), ('Inactive Item', 'N'), ]) extra_taxoSubs = OrderedDict([ ('', ''), ]) extra_itemSubs = OrderedDict([ ('Body Butter with Shimmer', 'Body Butter w/ Shimmer'), ('Tan Saver Body Wash', 'Body Wash'), ('Full Body Moisturizer', 'Moisturizer'), ('Moisturizing Body Milk', 'Body Milk'), ('Hair Brush', 'Brush'), ('Training Session', 'Session'), ('Skin Trial Pack', "Pack"), ('Trial Pack', "Pack"), ('Starter Package', 'Pack'), ('Sample Pack', "Pack"), ('Evaluation Package', 'Pack'), ('Spare Pot & Hose', 'Pot, Hose'), ('Spare Pots & Hose', 'Pots, Hose'), ('spare pot + Hose', 'pot, hose'), ('spare pots + Hose', 'pots, hose'), ('extraction fans', 'xfan'), ('Low Voltage', 'LV'), ('Double Sided', '2Sided'), ('TechnoTan', 'TT'), ('VuTan', 'VT'), ('EzeBreathe', 'EZB'), ('Sticky Soul', 'SS'), ('My Tan', 'MT'), ('TanSense', 'TS'), ('Tanning Advantage', 'TA'), ('Tanbience', 'TB'), ('Mosaic Minerals', 'MM'), ('Removal', 'Rem.'), ('Remover', 'Rem.'), ('Application', 'App.'), ('Peach & Vanilla', 'P&V'), ('Tamarillo & Papaya', 'T&P'), ('Tamarillo', 'TAM'), ('Lavander & Rosmary', 'L&R'), ('Lavender & Rosemary', 'L&R'), ('Coconut & Lime', 'C&L'), ('Melon & Cucumber', 'M&C'), ('Coconut Cream', 'CC'), ('Black & Silver', 'B&S'), ('Black & Gold', 'B&G'), ('Hot Pink', 'PNK'), ('Hot Lips (Red)', 'RED'), ('Hot Lips Red', 'RED'), ('Hot Lips', 'RED'), ('Silken Chocolate (Bronze)', 'BRZ'), ('Silken Chocolate', 'BRZ'), ('Moon Marvel (Silver)', 'SLV'), ('Dusty Gold', 'GLD'), ('Black', 'BLK'), ('Light Blue', 'LBLU'), ('Dark Blue', 'DBLU'), ('Blue', 'BLU'), ('Green', 'GRN'), ('Pink', 'PNK'), ('White', 'WHI'), ('Grey', 'GRY'), ('Peach', 'PEA'), ('Bronze', 'BRZ'), ('Silver', 'SLV'), ('Gold', 'GLD'), ('Red', 'RED'), ('Cyclone', 'CYC'), ('Classic', 'CLA'), ('Premier', 'PRE'), ('Deluxe', 'DEL'), ('ProMist Cube', 'CUBE'), ('ProMist', 'PRO'), ('Mini Mist', 'MIN'), ('Choc Fudge', 'CFdg.'), ('Choc Mousse', 'Cmou'), ('Ebony', 'Ebny.'), ('Creme Caramel', 'CCarm.'), ('Caramel', 'Carm.'), ('Cappuccino', 'Capp.'), ('Evaluation', 'Eval.'), ('Package', 'Pack'), ('Sample', 'Samp.'), ('sample', 'Samp.'), ('Tan Care', 'TCare'), ('After Care', "ACare"), ('A-Frame', 'AFrm'), ('X-Frame', 'XFrm'), ('Tear Drop Banner', 'TDBnr'), ('Roll Up Banner', 'RUBnr'), ('Hose Fitting', 'Fit.'), ('Magnetic', 'Mag.'), ('Option ', 'Opt.'), ('Style ', 'Sty.'), ('Insert and Frame', 'ins. frm.'), ('Insert Only', 'ins.'), ('Insert', 'ins.'), ('insert', 'ins.'), ('Frame', 'frm.'), ('Foundation', 'Found.'), ('Economy', 'Econ.'), ('Medium-Dark', 'MDark'), ('Medium Dark', 'MDark'), ('Medium', 'Med.'), ('medium', 'med.'), ('Extra Dark', 'XDark'), ('Extra-Dark', 'XDark'), ('Dark', 'Dark'), ('Tanning', 'Tan.'), ('Extra Small', 'XSml.'), ('Small', 'Sml.'), ('Extra Large', 'XLge.'), ('Large', 'Lge.'), ('Ladies', 'Ld.'), ('Mens', 'Mn.'), ('Non Personalized', 'Std.'), ('Personalized', 'Per.'), ('personalized', 'per.'), ('Personalised', 'Per.'), ('personalised', 'per.'), ('Custom Designed', 'Cust.'), ('Refurbished', 'Refurb.'), ('Compressor', 'Cmpr.'), ('Spray Gun', 'Gun'), ('Permanent', 'Perm.'), ('Shimmering', 'Shim.'), ('Screen Printed', 'SP'), ('Embroidered', 'Embr.'), ('Athletic', 'Athl.'), ('Singlet', 'Sing.'), ('Solution', 'Soln.'), ('Flash Tan', 'FTan'), ('Original', 'Orig.'), ('Exfoliating', 'Exfo.'), ('Disposable', 'Disp.'), ('Retractable', 'Ret.'), ('Synthetic', 'SYN'), ('Natural', 'NAT'), ('Bayonet', 'BAY'), ('Hexagonal', 'Hex.'), ('Product Pack', 'PrPck'), ('Complete', 'Compl.'), ('Fanatic', 'Fan.'), ('one', '1'), ('One', '1'), ('two', '2'), ('Two', '2'), ('three', '3'), ('Three', '3'), ('four', '4'), ('Four', '4'), # ('for', '4'), ('five', '5'), ('Five', '5'), ('six', '6'), ('Six', '6'), ('seven', '7'), ('seven', '7'), ('eight', '8'), ('Eight', '8'), ('nine', '9'), ('Nine', '9'), (' Plus', '+'), (' - ', ' '), (' Pack / ', ' x '), ('with', 'w/'), ('With', 'w/'), ('Box of', 'Box/'), (' Fitting for ', ' Fit '), (' Fits ', ' Fit '), (' and ', ' & '), (' And ', ' & '), # (' (2hr)', ''), (' (sachet)', ''), (' (pump bottle)', ''), (' Bottle with Flip Cap', ''), (' (jar)', ''), (' (tube)', ''), (' (spray)', ''), (' \xe2\x80\x94 ', ' '), ]) if not importName: importName = time.strftime("%Y-%m-%d %H:%M:%S") cols = listUtils.combineLists( cols, extra_cols ) defaults = listUtils.combineOrderedDicts( defaults, extra_defaults ) taxoSubs = listUtils.combineOrderedDicts( taxoSubs, extra_taxoSubs ) itemSubs = listUtils.combineOrderedDicts( itemSubs, extra_itemSubs ) if not schema: schema = "MY" super(CSVParse_MYO, self).__init__( cols, defaults, schema, \ taxoSubs, itemSubs, taxoDepth, itemDepth, metaWidth) if self.DEBUG_MYO: self.registerMessage( "csvparse initialized with cols: %s" % SanitationUtils.coerceUnicode(extra_cols))
from utils import SanitationUtils import json from wordpress_json import WordpressJsonWrapper wp = WordpressJsonWrapper('http://www.technotea.com.au/wp-json/wp/v2', 'neil', 'Stretch@6164') # wp = WordpressJsonWrapper('http://minimac.ddns.me:11182/wp-json/wp/v2', 'neil', 'Stretch6164') headers = { "User-Agent": "curl", "Content-Type": "text/json", # ... } # posts = wp.get_posts(post_id=16886, headers=headers) # print json.dumps(posts) # posts = wp.get_user(user_id=9) # print json.dumps(posts) fields_json = '{"client_grade": "Silver", "myob_customer_card_id": "", "direct_brand": "Pending", "myob_card_id": "C000009", "act_role": "RN", "billing_company": null, "first_name":"asdasdadadaadggg" }' fields_json_base64 = SanitationUtils.encodeBase64(fields_json) # fields_json_base64 = "eyJ1c2VyX2xvZ2luIjogImFkbWluIiwgImZpcnN0X25hbWUiOiAibm/wn5GMb2Twn5GMbGUiLCAidXNlcl91cmwiOiAiaHR0cDovL3d3dy5sYXNlcnBoaWxlLmNvbS9hc2QifQ==" posts = wp.update_user(user_id=9, data={'tansync_updated_fields': fields_json_base64}) print json.dumps(posts) posts = wp.get_user(user_id=9) print json.dumps(posts)
insort(mDeltaUpdates, syncUpdate) if syncUpdate.sUpdated and syncUpdate.sDeltas: insort(sDeltaUpdates, syncUpdate) if not syncUpdate: continue if syncUpdate.sUpdated: syncSlaveUpdates = syncUpdate.getSlaveUpdates() if 'E-mail' in syncSlaveUpdates: newEmail = syncSlaveUpdates['E-mail'] if newEmail in saParser.emails: mObjects = [mObject] sObjects = [sObject] + saParser.emails[newEmail] SanitationUtils.safePrint("duplicate emails", mObjects, sObjects) emailConflictMatches.addMatch(Match(mObjects, sObjects)) continue if(not syncUpdate.importantStatic): if(syncUpdate.mUpdated and syncUpdate.sUpdated): if(syncUpdate.sMod): insort(problematicUpdates, syncUpdate) continue elif(syncUpdate.mUpdated and not syncUpdate.sUpdated): insort(nonstaticMUpdates, syncUpdate) if(syncUpdate.sMod): insort(problematicUpdates, syncUpdate) continue elif(syncUpdate.sUpdated and not syncUpdate.mUpdated): insort(nonstaticSUpdates, syncUpdate)