if sinceS:
filterItems['sinceS'] = TimeUtils.wpStrptime(sinceS)
else:
filterItems = None
print filterItems
#########################################
# Download / Generate Slave Parser Object
#########################################
print debugUtils.hashify("Download / Generate Slave Parser Object"), timediff()
saParser = CSVParse_User(
cols = ColData_User.getWPImportCols(),
defaults = ColData_User.getDefaults(),
filterItems = filterItems,
limit = global_limit
)
if download_slave:
SSHTunnelForwarderAddress = (ssh_host, ssh_port)
SSHTunnelForwarderBindAddress = (remote_bind_host, remote_bind_port)
for host in ['SSHTunnelForwarderAddress', 'SSHTunnelForwarderBindAddress']:
try:
check_address(eval(host))
except Exception, e:
assert not e, "Host must be valid: %s [%s = %s]" % (str(e), host, repr(eval(host)))
SSHTunnelForwarderParams = {
'ssh_address_or_host':SSHTunnelForwarderAddress,
'ssh_password':ssh_pass,
'ssh_username':ssh_user,
'remote_bind_address': SSHTunnelForwarderBindAddress,
fstat = sftpClient.stat(exportFilename)
if fstat:
sftpClient.get(exportFilename, maPath)
except Exception, e:
SanitationUtils.safePrint("ERROR IN SFTP: " + str(e))
finally:
sftpClient.close()
except Exception, e:
SanitationUtils.safePrint("ERROR IN SSH: " + str(e))
finally:
sshClient.close()
maParser = CSVParse_User(
cols = colData.getImportCols(),
defaults = colData.getDefaults(),
contact_schema = 'act',
)
maParser.analyseFile(maPath, maEncoding)
def printBasicColumns(users):
# print len(users)
usrList = UsrObjList()
for user in users:
usrList.append(user)
# SanitationUtils.safePrint( "BILLING ADDRESS:", repr(user), user['First Name'], user.get('First Name'), user.name.__unicode__(out_schema="flat"))
cols = colData.getBasicCols()
SanitationUtils.safePrint( usrList.tabulate(
assert all([
'Wordpress ID' in wpCols.keys(),
wpCols['Wordpress ID'].get('wp', {}).get('key') == 'ID',
wpCols['Wordpress ID'].get('wp', {}).get('final')
]), 'ColData should be configured correctly'
userdata_select = ",\n\t\t\t".join([
("MAX(CASE WHEN um.meta_key = '%s' THEN um.meta_value ELSE \"\" END) as `%s`" if data['wp']['meta'] else "u.%s as `%s`") % (data['wp']['key'], col)\
for col, data in wpCols.items()
])
print sqlFile.read() % (userdata_select, '%susers'%tbl_prefix,'%susermeta'%tbl_prefix,'%stansync_updates'%tbl_prefix)
sqlFile.seek(0)
cursor = conn.cursor()
cursor.execute(sqlFile.read() % (userdata_select, '%susers'%tbl_prefix,'%susermeta'%tbl_prefix,'%stansync_updates'%tbl_prefix))
# headers = wpCols.keys() + ['ID', 'user_id', 'updated']
headers = [i[0] for i in cursor.description]
# print headers
saRows = [headers] + list(cursor.fetchall())
# print saRows
saParser = CSVParse_User(
cols = colData.getImportCols(),
defaults = colData.getDefaults()
)
if saRows:
saParser.analyseRows(saRows)
print saParser.tabulate()
