def outputFailures(failures, filePath):
with open(filePath, 'w+') as outFile:
for failure in failures:
Registrar.registerError(failure)
dictwriter = unicodecsv.DictWriter(
outFile,
fieldnames = ['update', 'master', 'slave', 'mchanges', 'schanges', 'exception'],
extrasaction = 'ignore',
)
dictwriter.writerows(failures)
print "WROTE FILE: ", filePath
)
dictwriter.writerows(failures)
print "WROTE FILE: ", filePath
outputFailures(masterFailures, mFailPath)
outputFailures(slaveFailures, sFailPath)
# Registrar.registerError('testing errors')
if __name__ == '__main__':
try:
main()
except SystemExit:
exit()
except:
Registrar.registerError(traceback.format_exc())
with io.open(logPath, 'w+', encoding='utf8') as logFile:
for source, messages in Registrar.getMessageItems(1).items():
print source
logFile.writelines([SanitationUtils.coerceUnicode(source)])
logFile.writelines(
[SanitationUtils.coerceUnicode(message) for message in messages]
)
for message in messages:
pprint( message, indent=4, width=80, depth=2)
#########################################
# email reports
def next(self):
if Registrar.DEBUG_API:
Registrar.registerMessage('start')
if self.next_endpoint is None:
if Registrar.DEBUG_API:
Registrar.registerMessage('stopping due to no next endpoint')
raise StopIteration()
# get API response
try:
self.prev_response = self.service.get(self.next_endpoint)
except ReadTimeout as e:
# instead of processing this endoint, do the page product by product
if self.limit > 1:
new_limit = 1
if Registrar.DEBUG_API:
Registrar.registerMessage('reducing limit in %s' % self.next_endpoint)
self.next_endpoint = UrlUtils.set_query_singular(
self.next_endpoint,
'filter[limit]',
new_limit
)
self.next_endpoint = UrlUtils.del_query_singular(
self.next_endpoint,
'page'
)
if self.offset:
self.next_endpoint = UrlUtils.set_query_singular(
self.next_endpoint,
'filter[offset]',
self.offset
)
self.limit = new_limit
# endpoint_queries = parse_qs(urlparse(self.next_endpoint).query)
# endpoint_queries = dict([
# (key, value[0]) for key, value in endpoint_queries.items()
# ])
# endpoint_queries['filter[limit]'] = 1
# if self.next_page:
# endpoint_queries['page'] = 10 * self.next_page
# print "endpoint_queries: ", endpoint_queries
# self.next_endpoint = UrlUtils.substitute_query(
# self.next_endpoint,
# urlencode(endpoint_queries)
# )
if Registrar.DEBUG_API:
Registrar.registerMessage('new endpoint %s' % self.next_endpoint)
self.prev_response = self.service.get(self.next_endpoint)
# handle API errors
if self.prev_response.status_code in range(400, 500):
raise ConnectionError('api call failed: %dd with %s' %( self.prev_response.status_code, self.prev_response.text))
# can still 200 and fail
try:
prev_response_json = self.prev_response.json()
except JSONDecodeError:
prev_response_json = {}
e = ConnectionError('api call to %s failed: %s' % (self.next_endpoint, self.prev_response.text))
Registrar.registerError(e)
# if Registrar.DEBUG_API:
# Registrar.registerMessage('first api response: %s' % str(prev_response_json))
if 'errors' in prev_response_json:
raise ConnectionError('first api call returned errors: %s' % (prev_response_json['errors']))
# process API headers
self.processHeaders(self.prev_response)
return prev_response_json
