def analyseRemote(self, parser, gid=None, outPath=None, limit=None):
if not outPath:
outPath = '/tmp/' + gid + '.csv'
if Registrar.DEBUG_GDRIVE:
Registrar.registerMessage( "Downloading gid %s to: %s" % (gid, outPath) )
if not self.skip_download:
# try:
# assert os.path.isfile(outPath)
# local_timestamp = os.path.getmtime(outPath)
# except:
# local_timestamp = 0
# local_gmtime = time.gmtime(local_timestamp)
#
# remote_gmtime = get_gm_modtime(gid)
#
# print "local / remote gmtime", local_gmtime, remote_gmtime
# if local_gmtime < remote_gmtime:
content = self.download_file_content_csv(gid)
if content:
with open(outPath, 'w') as outFile:
outFile.write(content)
Registrar.registerMessage( "downloaded %s" % outFile)
parser.analyseFile(outPath, limit=limit)
def tabulate(self, cols=None, tablefmt=None):
objs = self.objects
sanitizer = self.getSanitizer(tablefmt)
# sanitizer = (lambda x: str(x)) if tablefmt == 'html' else SanitationUtils.makeSafeOutput
if objs:
if not cols: cols = self.reportCols
header = [self.objList_type]
for col in cols.keys():
header += [col]
table = []
for obj in objs:
row = [obj.identifier]
for col in cols.keys():
# if col == 'Address':
# print repr(str(obj.get(col))), repr(sanitizer(obj.get(col)))
row += [ sanitizer(obj.get(col) )or ""]
try:
SanitationUtils.coerceUnicode(row[-1])
except:
Registrar.registerWarning("can't turn row into unicode: %s" % SanitationUtils.coerceUnicode(row))
table += [row]
# table += [[obj.index] + [ sanitizer(obj.get(col) )or "" for col in cols.keys()]]
# print "table", table
# SanitationUtils.safePrint(table)
# return SanitationUtils.coerceUnicode(tabulate(table, headers=header, tablefmt=tablefmt))
return (tabulate(table, headers=header, tablefmt=tablefmt))
# print repr(table)
# print repr(table.encode('utf8'))
# return table.encode('utf8')
else:
Registrar.registerWarning("cannot tabulate Objlist: there are no objects")
return ""
def analyseRemote(self, parser, since=None, limit=None):
if since: pass #todo: implement since
resultCount = 0
# apiIterator = self.ApiIterator(self.service, self.endpoint_plural)
apiIterator = self.getIterator(self.endpoint_plural)
progressCounter = None
for page in apiIterator:
if progressCounter is None:
total_items = apiIterator.total_items
if limit:
total_items = min(limit, total_items)
progressCounter = ProgressCounter(total_items)
progressCounter.maybePrintUpdate(resultCount)
# if Registrar.DEBUG_API:
# Registrar.registerMessage('processing page: %s' % str(page))
if self.endpoint_plural in page:
for page_item in page.get(self.endpoint_plural):
parser.analyseWpApiObj(page_item)
resultCount += 1
if limit and resultCount > limit:
if Registrar.DEBUG_API:
Registrar.registerMessage('reached limit, exiting')
return
def clearTransients(self):
if Registrar.DEBUG_MRO:
Registrar.registerMessage(' ')
# super(CSVParse_Shop_Mixin,self).clearTransients()
self.products = OrderedDict()
self.categories = OrderedDict()
self.attributes = OrderedDict()
self.vattributes= OrderedDict()
self.variations = OrderedDict()
self.images = OrderedDict()
self.categories_name = OrderedDict()
def outputFailures(failures, filePath):
with open(filePath, 'w+') as outFile:
for failure in failures:
Registrar.registerError(failure)
dictwriter = unicodecsv.DictWriter(
outFile,
fieldnames = ['update', 'master', 'slave', 'mchanges', 'schanges', 'exception'],
extrasaction = 'ignore',
)
dictwriter.writerows(failures)
print "WROTE FILE: ", filePath
def __init__(self, objects=None, indexer=None):
super(ObjList, self).__init__()
Registrar.__init__(self)
if self.DEBUG_MRO:
self.registerMessage('ObjList')
self.indexer = indexer if indexer else (lambda x: x.index)
self.supported_type = ImportObject
# self._objList_type = 'objects'
# self._objects = []
self.indices = []
if objects:
self.extend(objects)
def registerProduct(self, prodData):
if Registrar.DEBUG_SHOP:
Registrar.registerMessage("registering product %s" % prodData.identifier)
assert prodData.isProduct
self.registerAnything(
prodData,
self.products,
indexer = self.productIndexer,
singular = True,
resolver = self.resolveConflict,
registerName = 'products'
)
def getParserData(self, **kwargs):
if Registrar.DEBUG_MRO:
Registrar.registerMessage(' ')
defaults = {
self.objectContainer.namesumKey:'',
self.objectContainer.codesumKey:'',
self.objectContainer.descsumKey:''
}
# superData = super(CSVParse_Gen_Mixin, self).getParserData(**kwargs)
# superData = {}
# defaults.update(superData)
return defaults
def __init__(self, *args, **kwargs):
if Registrar.DEBUG_MRO:
Registrar.registerMessage('ImportShopMixin')
if Registrar.DEBUG_SHOP:
self.registerMessage("creating shop object; %s %s %s %s" % (
'isProduct' if self.isProduct else '!isProduct',
'isCategory' if self.isCategory else '!isCategory',
'isVariable' if self.isVariable else '!isVariable',
'isVariation' if self.isVariation else '!isVariation'
) )
super(ImportShopMixin, self).__init__(*args, **kwargs)
self.attributes = OrderedDict()
self.images = []
def uploadChanges(self, pkey, data=None):
super(SyncClient_Rest, self).uploadChanges(pkey)
service_endpoint = '%s/%d' % (self.endpoint_plural,pkey)
data = dict([(key, value) for key, value in data.items()])
if self.service.version is not 'wc/v1':
data = {self.endpoint_singular:data}
if Registrar.DEBUG_API:
Registrar.registerMessage("updating %s: %s" % (service_endpoint, data))
response = self.service.put(service_endpoint, data)
assert response.status_code not in [400,401], "API ERROR"
assert response.json(), "json should exist"
assert not isinstance(response.json(), int), "could not convert response to json: %s %s" % (str(response), str(response.json()))
assert 'errors' not in response.json(), "response has errors: %s" % str(response.json()['errors'])
return response
def registerObject(self, objectData):
if Registrar.DEBUG_MRO:
Registrar.registerMessage(' ')
# super(CSVParse_Shop_Mixin, self).registerObject(objectData)
if issubclass(type(objectData), ImportShopProductMixin):
if issubclass(type(objectData), ImportShopProductVariationMixin):
assert \
objectData.isVariation, \
"objectData not variation %s, obj isVariation: %s, cls isVariation; %s" \
% (type(objectData), repr(objectData.isVariation), repr(type(objectData).isVariation))
parentData = objectData.parent
assert parentData and parentData.isVariable
self.registerVariation(parentData, objectData)
else:
self.registerProduct(objectData)
def createItem(self, data):
data = dict([(key, value) for key, value in data.items()])
assert 'name' in data, "name is required to create a category, instead provided with %s" \
% (str(data))
if str(data.get('parent')) == str(-1):
del data['parent']
service_endpoint = self.endpoint_plural
endpoint_singular = self.endpoint_singular
endpoint_singular = re.sub('/','_', endpoint_singular)
if self.service.version is not 'wc/v1':
data = {endpoint_singular:data}
if Registrar.DEBUG_API:
Registrar.registerMessage("creating %s: %s" % (service_endpoint, data))
response = self.service.post(service_endpoint, data)
assert response.status_code not in [400,401], "API ERROR"
assert response.json(), "json should exist"
assert not isinstance(response.json(), int), "could not convert response to json: %s %s" % (str(response), str(response.json()))
assert 'errors' not in response.json(), "response has errors: %s" % str(response.json()['errors'])
return response
def processHeaders(self, response):
headers = response.headers
if Registrar.DEBUG_API:
Registrar.registerMessage("headers: %s" % str(headers))
if self.service.namespace == 'wp-api':
total_pages_key = 'X-WP-TotalPages'
total_items_key = 'X-WP-Total'
else:
total_pages_key = 'X-WC-TotalPages'
total_items_key = 'X-WC-Total'
if total_items_key in headers:
self.total_pages = int(headers.get(total_pages_key,''))
if total_pages_key in headers:
self.total_items = int(headers.get(total_items_key,''))
# if self.progressCounter is None:
# self.progressCounter = ProgressCounter(total=self.total_pages)
# self.stopNextIteration = True
prev_endpoint = self.next_endpoint
self.next_endpoint = None
for rel, link in response.links.items():
if rel == 'next' and link.get('url'):
next_response_url = link['url']
# if Registrar.DEBUG_API:
# Registrar.registerMessage('next_response_url: %s' % str(next_response_url))
self.next_page = int(UrlUtils.get_query_singular(next_response_url, 'page'))
if not self.next_page:
return
assert \
self.next_page <= self.total_pages, \
"next page (%s) should be lte total pages (%s)" \
% (str(self.next_page), str(self.total_pages))
self.next_endpoint = UrlUtils.set_query_singular(prev_endpoint,'page', self.next_page)
# if Registrar.DEBUG_API:
# Registrar.registerMessage('next_endpoint: %s' % str(self.next_endpoint))
if self.next_page:
self.offset = (self.limit * self.next_page) + 1
def __init__(self, *args, **kwargs):
if Registrar.DEBUG_MRO:
Registrar.registerMessage('ImportShopCategoryMixin')
# super(ImportShopCategoryMixin, self).__init__(*args, **kwargs)
self.members = OrderedDict()
def getProducts(self):
e = DeprecationWarning("Use .products instead of .getProducts()")
self.registerError(e)
if Registrar.DEBUG_SHOP:
Registrar.registerMessage("returning products: {}".format(self.products.keys()))
return self.products
def analyseRemote(self, parser, since=None, limit=None, filterItems=None):
self.assertConnect()
# srv_offset = self.dbParams.pop('srv_offset','')
self.dbParams['port'] = self.service.local_bind_address[-1]
cursor = pymysql.connect( **self.dbParams ).cursor()
sm_where_clauses = []
if since:
since_t = TimeUtils.wpServerToLocalTime( TimeUtils.wpStrptime(since))
assert since_t, "Time should be valid format, got %s" % since
since_s = TimeUtils.wpTimeToString(since_t)
sm_where_clauses.append( "tu.`time` > '%s'" % since_s )
modtime_cols = [
"tu.`user_id` as `user_id`",
"MAX(tu.`time`) as `Edited in Wordpress`"
]
for tracking_name, aliases in ColData_User.getWPTrackedCols().items():
case_clauses = []
for alias in aliases:
case_clauses.append("LOCATE('\"%s\"', tu.`changed`) > 0" % alias)
modtime_cols.append("MAX(CASE WHEN {case_clauses} THEN tu.`time` ELSE \"\" END) as `{tracking_name}`".format(
case_clauses = " OR ".join(case_clauses),
tracking_name = tracking_name
))
if sm_where_clauses:
sm_where_clause = 'WHERE ' + ' AND '.join(sm_where_clauses)
else:
sm_where_clause = ''
sql_select_modtime = """\
SELECT
{modtime_cols}
FROM
{tbl_tu} tu
{sm_where_clause}
GROUP BY
tu.`user_id`""".format(
modtime_cols = ",\n\t\t".join(modtime_cols),
tbl_tu=self.tbl_prefix+'tansync_updates',
sm_where_clause = sm_where_clause,
)
# print sql_select_modtime
if since:
cursor.execute(sql_select_modtime)
headers = [SanitationUtils.coerceUnicode(i[0]) for i in cursor.description]
results = [[SanitationUtils.coerceUnicode(cell) for cell in row] for row in cursor]
table = [headers] + results
# print tabulate(table, headers='firstrow')
# results = list(cursor)
# if len(results) == 0:
# #nothing to analyse
# return
# else:
# # n rows to analyse
# print "THERE ARE %d ITEMS" % len(results)
wpDbMetaCols = ColData_User.getWPDBCols(meta=True)
wpDbCoreCols = ColData_User.getWPDBCols(meta=False)
userdata_cols = ",\n\t\t".join(filter(None,
[
"u.%s as `%s`" % (key, name)\
for key, name in wpDbCoreCols.items()
] + [
"MAX(CASE WHEN um.meta_key = '%s' THEN um.meta_value ELSE \"\" END) as `%s`" % (key, name) \
for key, name in wpDbMetaCols.items()
]
))
# wpCols = OrderedDict(filter( lambda (k, v): not v.get('wp',{}).get('generated'), ColData_User.getWPCols().items()))
# assert all([
# 'Wordpress ID' in wpCols.keys(),
# wpCols['Wordpress ID'].get('wp', {}).get('key') == 'ID',
# wpCols['Wordpress ID'].get('wp', {}).get('final')
# ]), 'ColData should be configured correctly'
# userdata_cols2 = ",\n\t\t".join(filter(None,[
# ("MAX(CASE WHEN um.meta_key = '%s' THEN um.meta_value ELSE \"\" END) as `%s`" if data['wp'].get('meta') else "u.%s as `%s`") % (data['wp']['key'], col)\
# for col, data in wpCols.items()
# ]))
# print " -> COLS1: ", userdata_cols
# print " -> COLS2: ", userdata_cols2
# print userdata_cols
sql_select_user = """
SELECT
{usr_cols}
FROM
{tbl_u} u
LEFT JOIN {tbl_um} um
ON ( um.`user_id` = u.`ID`)
GROUP BY
u.`ID`""".format(
tbl_u = self.tbl_prefix+'users',
tbl_um = self.tbl_prefix+'usermeta',
usr_cols = userdata_cols,
)
um_on_clauses = []
um_where_clauses = []
um_on_clauses.append('ud.`Wordpress ID` = lu.`user_id`')
if filterItems:
if 'cards' in filterItems:
um_where_clauses.append( "ud.`MYOB Card ID` IN (%s)" % (','.join([
'"%s"' % card for card in filterItems['cards']
])))
if um_on_clauses:
um_on_clause = ' AND '.join([
"(%s)" % clause for clause in um_on_clauses
])
else:
um_on_clause = ''
if um_where_clauses:
um_where_clause = 'WHERE ' + ' AND '.join([
"(%s)" % clause for clause in um_where_clauses
])
else:
um_where_clause = ''
# print sql_select_user
sql_select_user_modtime = """
SELECT *
FROM
(
{sql_ud}
) as ud
{join_type} JOIN
(
{sql_mt}
) as lu
ON {um_on_clause}
{um_where_clause}
{limit_clause};""".format(
sql_ud = sql_select_user,
sql_mt = sql_select_modtime,
join_type = "INNER" if sm_where_clause else "LEFT",
limit_clause = "LIMIT %d" % limit if limit else "",
um_on_clause = um_on_clause,
um_where_clause = um_where_clause
)
if Registrar.DEBUG_CLIENT: Registrar.registerMessage(sql_select_user_modtime)
cursor.execute(sql_select_user_modtime)
headers = [SanitationUtils.coerceUnicode(i[0]) for i in cursor.description]
results = [[SanitationUtils.coerceUnicode(cell) for cell in row] for row in cursor]
rows = [headers] + results
# print rows
if results:
print "there are %d results" % len(results)
parser.analyseRows(rows)
)
dictwriter.writerows(failures)
print "WROTE FILE: ", filePath
outputFailures(masterFailures, mFailPath)
outputFailures(slaveFailures, sFailPath)
# Registrar.registerError('testing errors')
if __name__ == '__main__':
try:
main()
except SystemExit:
exit()
except:
Registrar.registerError(traceback.format_exc())
with io.open(logPath, 'w+', encoding='utf8') as logFile:
for source, messages in Registrar.getMessageItems(1).items():
print source
logFile.writelines([SanitationUtils.coerceUnicode(source)])
logFile.writelines(
[SanitationUtils.coerceUnicode(message) for message in messages]
)
for message in messages:
pprint( message, indent=4, width=80, depth=2)
#########################################
# email reports
def __init__(self, *args, **kwargs):
if Registrar.DEBUG_MRO:
Registrar.registerMessage('ImportShopProductVariableMixin')
# super(ImportShopProductVariableMixin, self).__init__(*args, **kwargs)
self.variations = OrderedDict()
def next(self):
if Registrar.DEBUG_API:
Registrar.registerMessage('start')
if self.next_endpoint is None:
if Registrar.DEBUG_API:
Registrar.registerMessage('stopping due to no next endpoint')
raise StopIteration()
# get API response
try:
self.prev_response = self.service.get(self.next_endpoint)
except ReadTimeout as e:
# instead of processing this endoint, do the page product by product
if self.limit > 1:
new_limit = 1
if Registrar.DEBUG_API:
Registrar.registerMessage('reducing limit in %s' % self.next_endpoint)
self.next_endpoint = UrlUtils.set_query_singular(
self.next_endpoint,
'filter[limit]',
new_limit
)
self.next_endpoint = UrlUtils.del_query_singular(
self.next_endpoint,
'page'
)
if self.offset:
self.next_endpoint = UrlUtils.set_query_singular(
self.next_endpoint,
'filter[offset]',
self.offset
)
self.limit = new_limit
# endpoint_queries = parse_qs(urlparse(self.next_endpoint).query)
# endpoint_queries = dict([
# (key, value[0]) for key, value in endpoint_queries.items()
# ])
# endpoint_queries['filter[limit]'] = 1
# if self.next_page:
# endpoint_queries['page'] = 10 * self.next_page
# print "endpoint_queries: ", endpoint_queries
# self.next_endpoint = UrlUtils.substitute_query(
# self.next_endpoint,
# urlencode(endpoint_queries)
# )
if Registrar.DEBUG_API:
Registrar.registerMessage('new endpoint %s' % self.next_endpoint)
self.prev_response = self.service.get(self.next_endpoint)
# handle API errors
if self.prev_response.status_code in range(400, 500):
raise ConnectionError('api call failed: %dd with %s' %( self.prev_response.status_code, self.prev_response.text))
# can still 200 and fail
try:
prev_response_json = self.prev_response.json()
except JSONDecodeError:
prev_response_json = {}
e = ConnectionError('api call to %s failed: %s' % (self.next_endpoint, self.prev_response.text))
Registrar.registerError(e)
# if Registrar.DEBUG_API:
# Registrar.registerMessage('first api response: %s' % str(prev_response_json))
if 'errors' in prev_response_json:
raise ConnectionError('first api call returned errors: %s' % (prev_response_json['errors']))
# process API headers
self.processHeaders(self.prev_response)
return prev_response_json