def outputFailures(failures, filePath): with open(filePath, 'w+') as outFile: for failure in failures: Registrar.registerError(failure) dictwriter = unicodecsv.DictWriter( outFile, fieldnames = ['update', 'master', 'slave', 'mchanges', 'schanges', 'exception'], extrasaction = 'ignore', ) dictwriter.writerows(failures) print "WROTE FILE: ", filePath
) dictwriter.writerows(failures) print "WROTE FILE: ", filePath outputFailures(masterFailures, mFailPath) outputFailures(slaveFailures, sFailPath) # Registrar.registerError('testing errors') if __name__ == '__main__': try: main() except SystemExit: exit() except: Registrar.registerError(traceback.format_exc()) with io.open(logPath, 'w+', encoding='utf8') as logFile: for source, messages in Registrar.getMessageItems(1).items(): print source logFile.writelines([SanitationUtils.coerceUnicode(source)]) logFile.writelines( [SanitationUtils.coerceUnicode(message) for message in messages] ) for message in messages: pprint( message, indent=4, width=80, depth=2) ######################################### # email reports
def next(self): if Registrar.DEBUG_API: Registrar.registerMessage('start') if self.next_endpoint is None: if Registrar.DEBUG_API: Registrar.registerMessage('stopping due to no next endpoint') raise StopIteration() # get API response try: self.prev_response = self.service.get(self.next_endpoint) except ReadTimeout as e: # instead of processing this endoint, do the page product by product if self.limit > 1: new_limit = 1 if Registrar.DEBUG_API: Registrar.registerMessage('reducing limit in %s' % self.next_endpoint) self.next_endpoint = UrlUtils.set_query_singular( self.next_endpoint, 'filter[limit]', new_limit ) self.next_endpoint = UrlUtils.del_query_singular( self.next_endpoint, 'page' ) if self.offset: self.next_endpoint = UrlUtils.set_query_singular( self.next_endpoint, 'filter[offset]', self.offset ) self.limit = new_limit # endpoint_queries = parse_qs(urlparse(self.next_endpoint).query) # endpoint_queries = dict([ # (key, value[0]) for key, value in endpoint_queries.items() # ]) # endpoint_queries['filter[limit]'] = 1 # if self.next_page: # endpoint_queries['page'] = 10 * self.next_page # print "endpoint_queries: ", endpoint_queries # self.next_endpoint = UrlUtils.substitute_query( # self.next_endpoint, # urlencode(endpoint_queries) # ) if Registrar.DEBUG_API: Registrar.registerMessage('new endpoint %s' % self.next_endpoint) self.prev_response = self.service.get(self.next_endpoint) # handle API errors if self.prev_response.status_code in range(400, 500): raise ConnectionError('api call failed: %dd with %s' %( self.prev_response.status_code, self.prev_response.text)) # can still 200 and fail try: prev_response_json = self.prev_response.json() except JSONDecodeError: prev_response_json = {} e = ConnectionError('api call to %s failed: %s' % (self.next_endpoint, self.prev_response.text)) Registrar.registerError(e) # if Registrar.DEBUG_API: # Registrar.registerMessage('first api response: %s' % str(prev_response_json)) if 'errors' in prev_response_json: raise ConnectionError('first api call returned errors: %s' % (prev_response_json['errors'])) # process API headers self.processHeaders(self.prev_response) return prev_response_json