def tabulate(self, cols=None, tablefmt=None):
objs = self.objects
sanitizer = self.getSanitizer(tablefmt)
# sanitizer = (lambda x: str(x)) if tablefmt == 'html' else SanitationUtils.makeSafeOutput
if objs:
if not cols: cols = self.reportCols
header = [self.objList_type]
for col in cols.keys():
header += [col]
table = []
for obj in objs:
row = [obj.identifier]
for col in cols.keys():
# if col == 'Address':
# print repr(str(obj.get(col))), repr(sanitizer(obj.get(col)))
row += [ sanitizer(obj.get(col) )or ""]
try:
SanitationUtils.coerceUnicode(row[-1])
except:
Registrar.registerWarning("can't turn row into unicode: %s" % SanitationUtils.coerceUnicode(row))
table += [row]
# table += [[obj.index] + [ sanitizer(obj.get(col) )or "" for col in cols.keys()]]
# print "table", table
# SanitationUtils.safePrint(table)
# return SanitationUtils.coerceUnicode(tabulate(table, headers=header, tablefmt=tablefmt))
return (tabulate(table, headers=header, tablefmt=tablefmt))
# print repr(table)
# print repr(table.encode('utf8'))
# return table.encode('utf8')
else:
Registrar.registerWarning("cannot tabulate Objlist: there are no objects")
return ""
def append(self, objectData):
assert issubclass(objectData.__class__, ImportShopMixin), \
"object must be subclass of ImportShopMixin not %s : %s" % (
SanitationUtils.coerceUnicode(objectData.__class__),
SanitationUtils.coerceUnicode(objectData)
)
return super(ShopProdList, self).append(objectData)
def execSilentCommandAssert(self, command):
self.assertConnect()
stdin, stdout, stderr = self.service.exec_command(command)
if stdin: pass #gets rid of annoying warnings
possible_errors = stdout.readlines() + stderr.readlines()
for error in possible_errors:
if re.match("^Countries.*", error):
print error
continue
assert not error, "command <%s> returned errors: %s" % (
SanitationUtils.coerceUnicode(command),
SanitationUtils.coerceUnicode(error)
)
def __init__(self, cols={}, defaults={}, schema='MY', importName="", \
taxoSubs={}, itemSubs={}, taxoDepth=3, itemDepth=2, metaWidth=2):
if self.DEBUG_MRO:
self.registerMessage(' ')
extra_cols = [ 'WNRC', 'RNRC', 'HTML Description' ]
extra_defaults = OrderedDict([
('Sell', 'S'),
('Tax Code When Sold', 'GST'),
('Sell Price Inclusive', 'X'),
('Income Acct', '41000'),
('Use Desc. On Sale', ''),
('Inactive Item', 'N'),
])
extra_taxoSubs = OrderedDict([
('', ''),
])
extra_itemSubs = OrderedDict([
('Body Butter with Shimmer', 'Body Butter w/ Shimmer'),
('Tan Saver Body Wash', 'Body Wash'),
('Full Body Moisturizer', 'Moisturizer'),
('Moisturizing Body Milk', 'Body Milk'),
('Hair Brush', 'Brush'),
('Training Session', 'Session'),
('Skin Trial Pack', "Pack"),
('Trial Pack', "Pack"),
('Starter Package', 'Pack'),
('Sample Pack', "Pack"),
('Evaluation Package', 'Pack'),
('Spare Pot & Hose', 'Pot, Hose'),
('Spare Pots & Hose', 'Pots, Hose'),
('spare pot + Hose', 'pot, hose'),
('spare pots + Hose', 'pots, hose'),
('extraction fans', 'xfan'),
('Low Voltage', 'LV'),
('Double Sided', '2Sided'),
('TechnoTan', 'TT'),
('VuTan', 'VT'),
('EzeBreathe', 'EZB'),
('Sticky Soul', 'SS'),
('My Tan', 'MT'),
('TanSense', 'TS'),
('Tanning Advantage', 'TA'),
('Tanbience', 'TB'),
('Mosaic Minerals', 'MM'),
('Removal', 'Rem.'),
('Remover', 'Rem.'),
('Application', 'App.'),
('Peach & Vanilla', 'P&V'),
('Tamarillo & Papaya', 'T&P'),
('Tamarillo', 'TAM'),
('Lavander & Rosmary', 'L&R'),
('Lavender & Rosemary', 'L&R'),
('Coconut & Lime', 'C&L'),
('Melon & Cucumber', 'M&C'),
('Coconut Cream', 'CC'),
('Black & Silver', 'B&S'),
('Black & Gold', 'B&G'),
('Hot Pink', 'PNK'),
('Hot Lips (Red)', 'RED'),
('Hot Lips Red', 'RED'),
('Hot Lips', 'RED'),
('Silken Chocolate (Bronze)', 'BRZ'),
('Silken Chocolate', 'BRZ'),
('Moon Marvel (Silver)', 'SLV'),
('Dusty Gold', 'GLD'),
('Black', 'BLK'),
('Light Blue', 'LBLU'),
('Dark Blue', 'DBLU'),
('Blue', 'BLU'),
('Green', 'GRN'),
('Pink', 'PNK'),
('White', 'WHI'),
('Grey', 'GRY'),
('Peach', 'PEA'),
('Bronze', 'BRZ'),
('Silver', 'SLV'),
('Gold', 'GLD'),
('Red', 'RED'),
('Cyclone', 'CYC'),
('Classic', 'CLA'),
('Premier', 'PRE'),
('Deluxe', 'DEL'),
('ProMist Cube', 'CUBE'),
('ProMist', 'PRO'),
('Mini Mist', 'MIN'),
('Choc Fudge', 'CFdg.'),
('Choc Mousse', 'Cmou'),
('Ebony', 'Ebny.'),
('Creme Caramel', 'CCarm.'),
('Caramel', 'Carm.'),
('Cappuccino', 'Capp.'),
('Evaluation', 'Eval.'),
('Package', 'Pack'),
('Sample', 'Samp.'),
('sample', 'Samp.'),
('Tan Care', 'TCare'),
('After Care', "ACare"),
('A-Frame', 'AFrm'),
('X-Frame', 'XFrm'),
('Tear Drop Banner', 'TDBnr'),
('Roll Up Banner', 'RUBnr'),
('Hose Fitting', 'Fit.'),
('Magnetic', 'Mag.'),
('Option ', 'Opt.'),
('Style ', 'Sty.'),
('Insert and Frame', 'ins. frm.'),
('Insert Only', 'ins.'),
('Insert', 'ins.'),
('insert', 'ins.'),
('Frame', 'frm.'),
('Foundation', 'Found.'),
('Economy', 'Econ.'),
('Medium-Dark', 'MDark'),
('Medium Dark', 'MDark'),
('Medium', 'Med.'),
('medium', 'med.'),
('Extra Dark', 'XDark'),
('Extra-Dark', 'XDark'),
('Dark', 'Dark'),
('Tanning', 'Tan.'),
('Extra Small', 'XSml.'),
('Small', 'Sml.'),
('Extra Large', 'XLge.'),
('Large', 'Lge.'),
('Ladies', 'Ld.'),
('Mens', 'Mn.'),
('Non Personalized', 'Std.'),
('Personalized', 'Per.'),
('personalized', 'per.'),
('Personalised', 'Per.'),
('personalised', 'per.'),
('Custom Designed', 'Cust.'),
('Refurbished', 'Refurb.'),
('Compressor', 'Cmpr.'),
('Spray Gun', 'Gun'),
('Permanent', 'Perm.'),
('Shimmering', 'Shim.'),
('Screen Printed', 'SP'),
('Embroidered', 'Embr.'),
('Athletic', 'Athl.'),
('Singlet', 'Sing.'),
('Solution', 'Soln.'),
('Flash Tan', 'FTan'),
('Original', 'Orig.'),
('Exfoliating', 'Exfo.'),
('Disposable', 'Disp.'),
('Retractable', 'Ret.'),
('Synthetic', 'SYN'),
('Natural', 'NAT'),
('Bayonet', 'BAY'),
('Hexagonal', 'Hex.'),
('Product Pack', 'PrPck'),
('Complete', 'Compl.'),
('Fanatic', 'Fan.'),
('one', '1'),
('One', '1'),
('two', '2'),
('Two', '2'),
('three', '3'),
('Three', '3'),
('four', '4'),
('Four', '4'),
# ('for', '4'),
('five', '5'),
('Five', '5'),
('six', '6'),
('Six', '6'),
('seven', '7'),
('seven', '7'),
('eight', '8'),
('Eight', '8'),
('nine', '9'),
('Nine', '9'),
(' Plus', '+'),
(' - ', ' '),
(' Pack / ', ' x '),
('with', 'w/'),
('With', 'w/'),
('Box of', 'Box/'),
(' Fitting for ', ' Fit '),
(' Fits ', ' Fit '),
(' and ', ' & '),
(' And ', ' & '),
# (' (2hr)', ''),
(' (sachet)', ''),
(' (pump bottle)', ''),
(' Bottle with Flip Cap', ''),
(' (jar)', ''),
(' (tube)', ''),
(' (spray)', ''),
(' \xe2\x80\x94 ', ' '),
])
if not importName: importName = time.strftime("%Y-%m-%d %H:%M:%S")
cols = listUtils.combineLists( cols, extra_cols )
defaults = listUtils.combineOrderedDicts( defaults, extra_defaults )
taxoSubs = listUtils.combineOrderedDicts( taxoSubs, extra_taxoSubs )
itemSubs = listUtils.combineOrderedDicts( itemSubs, extra_itemSubs )
if not schema: schema = "MY"
super(CSVParse_MYO, self).__init__( cols, defaults, schema, \
taxoSubs, itemSubs, taxoDepth, itemDepth, metaWidth)
if self.DEBUG_MYO:
self.registerMessage( "csvparse initialized with cols: %s" %
SanitationUtils.coerceUnicode(extra_cols))
xml_back = SanitationUtils.xmlToUnicode(xml_str)
ascii_str = SanitationUtils.unicodeToAscii(u_str)
ascii_back = SanitationUtils.asciiToUnicode(u_str)
printandrep( "u_str", u_str )
printandrep( "utf8_str", utf8_str, utf8_back )
printandrep( "xml_str", xml_str, xml_back )
printandrep( "ascii_str", ascii_str, ascii_back )
print SanitationUtils.unicodeToUTF8(None)
print SanitationUtils.utf8ToUnicode(None)
print SanitationUtils.unicodeToXml(None)
print SanitationUtils.xmlToUnicode(None)
print SanitationUtils.unicodeToAscii(None)
print SanitationUtils.asciiToUnicode(None)
print SanitationUtils.coerceUnicode(None)
SanitationUtils.safePrint(None)
print converters.to_bytes(SanitationUtils.coerceUnicode("\xf0\x9f\x91\x8c"))
print converters.to_bytes(SanitationUtils.unicodeToXml("\xf0\x9f\x91\x8c".decode("utf8"), True))
print converters.xml_to_unicode("L")
print converters.xml_to_byte_string("L")
print converters.to_bytes(converters.xml_to_unicode("と"))
print converters.to_bytes(converters.xml_to_byte_string("と", input_encoding="ascii"))
print converters.xml_to_unicode("👌", encoding="ascii")
print converters.xml_to_byte_string("👌", input_encoding="ascii")
map_json = '{"E-mail":"*****@*****.**","Web Site":"http:\/\/technotan.com.au","MYOB Customer Card ID":[""],"MYOB Card ID":[""],"First Name":["Neil \ud83d\udc4c\'&>"],"Surname":["Cunliffe-Williams"],"Contact":["Neil Cunliffe-Williams"],"Company":["Laserphile"],"Address 1":["7 Grosvenor Road"],"Address 2":[""],"City":[""],"Postcode":["6053"],"State":["WA"],"Phone":["0416160912"],"Home Address 1":["7 Grosvenor Road"],"Home Address 2":[""],"Home City":["Bayswater"],"Home Postcode":["6053"],"Home Country":["AU"],"Home State":["WA"],"Role":["ADMIN"],"ABN":["32"],"Business Type":[""],"Birth Date":[""],"Mobile Phone":["+61416160912"],"Fax":[""],"Lead Source":[""],"Referred By":[""]}'
UsrSyncClient_WC(wcApiParams) as slaveClient:
# UsrSyncClient_JSON(jsonConnectParams) as slaveClient:
for count, update in enumerate(allUpdates):
if Registrar.DEBUG_PROGRESS:
updateProgressCounter.maybePrintUpdate(count)
# if update.WPID == '1':
# print repr(update.WPID)
# continue
if update_master and update.mUpdated :
try:
update.updateMaster(masterClient)
except Exception, e:
masterFailures.append({
'update':update,
'master':SanitationUtils.coerceUnicode(update.newMObject),
'slave':SanitationUtils.coerceUnicode(update.newSObject),
'mchanges':SanitationUtils.coerceUnicode(update.getMasterUpdates()),
'schanges':SanitationUtils.coerceUnicode(update.getSlaveUpdates()),
'exception':repr(e)
})
SanitationUtils.safePrint("ERROR UPDATING MASTER (%s): %s" % (update.MasterID, repr(e) ) )
# continue
if update_slave and update.sUpdated :
try:
update.updateSlave(slaveClient)
except Exception, e:
slaveFailures.append({
'update':update,
'master':SanitationUtils.coerceUnicode(update.newMObject),
'slave':SanitationUtils.coerceUnicode(update.newSObject),
def analyseRemote(self, parser, since=None, limit=None, filterItems=None):
self.assertConnect()
# srv_offset = self.dbParams.pop('srv_offset','')
self.dbParams['port'] = self.service.local_bind_address[-1]
cursor = pymysql.connect( **self.dbParams ).cursor()
sm_where_clauses = []
if since:
since_t = TimeUtils.wpServerToLocalTime( TimeUtils.wpStrptime(since))
assert since_t, "Time should be valid format, got %s" % since
since_s = TimeUtils.wpTimeToString(since_t)
sm_where_clauses.append( "tu.`time` > '%s'" % since_s )
modtime_cols = [
"tu.`user_id` as `user_id`",
"MAX(tu.`time`) as `Edited in Wordpress`"
]
for tracking_name, aliases in ColData_User.getWPTrackedCols().items():
case_clauses = []
for alias in aliases:
case_clauses.append("LOCATE('\"%s\"', tu.`changed`) > 0" % alias)
modtime_cols.append("MAX(CASE WHEN {case_clauses} THEN tu.`time` ELSE \"\" END) as `{tracking_name}`".format(
case_clauses = " OR ".join(case_clauses),
tracking_name = tracking_name
))
if sm_where_clauses:
sm_where_clause = 'WHERE ' + ' AND '.join(sm_where_clauses)
else:
sm_where_clause = ''
sql_select_modtime = """\
SELECT
{modtime_cols}
FROM
{tbl_tu} tu
{sm_where_clause}
GROUP BY
tu.`user_id`""".format(
modtime_cols = ",\n\t\t".join(modtime_cols),
tbl_tu=self.tbl_prefix+'tansync_updates',
sm_where_clause = sm_where_clause,
)
# print sql_select_modtime
if since:
cursor.execute(sql_select_modtime)
headers = [SanitationUtils.coerceUnicode(i[0]) for i in cursor.description]
results = [[SanitationUtils.coerceUnicode(cell) for cell in row] for row in cursor]
table = [headers] + results
# print tabulate(table, headers='firstrow')
# results = list(cursor)
# if len(results) == 0:
# #nothing to analyse
# return
# else:
# # n rows to analyse
# print "THERE ARE %d ITEMS" % len(results)
wpDbMetaCols = ColData_User.getWPDBCols(meta=True)
wpDbCoreCols = ColData_User.getWPDBCols(meta=False)
userdata_cols = ",\n\t\t".join(filter(None,
[
"u.%s as `%s`" % (key, name)\
for key, name in wpDbCoreCols.items()
] + [
"MAX(CASE WHEN um.meta_key = '%s' THEN um.meta_value ELSE \"\" END) as `%s`" % (key, name) \
for key, name in wpDbMetaCols.items()
]
))
# wpCols = OrderedDict(filter( lambda (k, v): not v.get('wp',{}).get('generated'), ColData_User.getWPCols().items()))
# assert all([
# 'Wordpress ID' in wpCols.keys(),
# wpCols['Wordpress ID'].get('wp', {}).get('key') == 'ID',
# wpCols['Wordpress ID'].get('wp', {}).get('final')
# ]), 'ColData should be configured correctly'
# userdata_cols2 = ",\n\t\t".join(filter(None,[
# ("MAX(CASE WHEN um.meta_key = '%s' THEN um.meta_value ELSE \"\" END) as `%s`" if data['wp'].get('meta') else "u.%s as `%s`") % (data['wp']['key'], col)\
# for col, data in wpCols.items()
# ]))
# print " -> COLS1: ", userdata_cols
# print " -> COLS2: ", userdata_cols2
# print userdata_cols
sql_select_user = """
SELECT
{usr_cols}
FROM
{tbl_u} u
LEFT JOIN {tbl_um} um
ON ( um.`user_id` = u.`ID`)
GROUP BY
u.`ID`""".format(
tbl_u = self.tbl_prefix+'users',
tbl_um = self.tbl_prefix+'usermeta',
usr_cols = userdata_cols,
)
um_on_clauses = []
um_where_clauses = []
um_on_clauses.append('ud.`Wordpress ID` = lu.`user_id`')
if filterItems:
if 'cards' in filterItems:
um_where_clauses.append( "ud.`MYOB Card ID` IN (%s)" % (','.join([
'"%s"' % card for card in filterItems['cards']
])))
if um_on_clauses:
um_on_clause = ' AND '.join([
"(%s)" % clause for clause in um_on_clauses
])
else:
um_on_clause = ''
if um_where_clauses:
um_where_clause = 'WHERE ' + ' AND '.join([
"(%s)" % clause for clause in um_where_clauses
])
else:
um_where_clause = ''
# print sql_select_user
sql_select_user_modtime = """
SELECT *
FROM
(
{sql_ud}
) as ud
{join_type} JOIN
(
{sql_mt}
) as lu
ON {um_on_clause}
{um_where_clause}
{limit_clause};""".format(
sql_ud = sql_select_user,
sql_mt = sql_select_modtime,
join_type = "INNER" if sm_where_clause else "LEFT",
limit_clause = "LIMIT %d" % limit if limit else "",
um_on_clause = um_on_clause,
um_where_clause = um_where_clause
)
if Registrar.DEBUG_CLIENT: Registrar.registerMessage(sql_select_user_modtime)
cursor.execute(sql_select_user_modtime)
headers = [SanitationUtils.coerceUnicode(i[0]) for i in cursor.description]
results = [[SanitationUtils.coerceUnicode(cell) for cell in row] for row in cursor]
rows = [headers] + results
# print rows
if results:
print "there are %d results" % len(results)
parser.analyseRows(rows)
for value in changedMap.values():
for val in value:
SanitationUtils.safePrint(val)
dataMap = json2map(data)
diffMap = listUtils.keysNotIn(dataMap, changedMap.keys()) if isinstance(changedMap, dict) else dataMap
changeDataFmt.append( [
user_id,
c_time,
"C:%s<br/>S:%s" % (map2table(changedMap), map2table(diffMap))
] )
print "creating report..."
with io.open(repPath, 'w+', encoding='utf-8') as resFile:
reporter = HtmlReporter()
group = HtmlReporter.Group('changes', 'Changes')
group.addSection(
HtmlReporter.Section(
'wp_changes',
'Wordpress Changes',
"modifications made to wordpress" + ( "since %s" % since if since else ""),
data = re.sub("<table>","<table class=\"table table-striped\">",
tabulate(changeDataFmt, headers="firstrow", tablefmt="html")
),
length = len(changeDataFmt)
)
)
resFile.write( SanitationUtils.coerceUnicode( reporter.getDocument() ))
