def analyseRemote(self, parser, since=None, limit=None): if since: pass #todo: implement since resultCount = 0 # apiIterator = self.ApiIterator(self.service, self.endpoint_plural) apiIterator = self.getIterator(self.endpoint_plural) progressCounter = None for page in apiIterator: if progressCounter is None: total_items = apiIterator.total_items if limit: total_items = min(limit, total_items) progressCounter = ProgressCounter(total_items) progressCounter.maybePrintUpdate(resultCount) # if Registrar.DEBUG_API: # Registrar.registerMessage('processing page: %s' % str(page)) if self.endpoint_plural in page: for page_item in page.get(self.endpoint_plural): parser.analyseWpApiObj(page_item) resultCount += 1 if limit and resultCount > limit: if Registrar.DEBUG_API: Registrar.registerMessage('reached limit, exiting') return
# TODO: further sort emailMatcher print debugUtils.hashify("BEGINNING MERGE (%d)" % len(globalMatches)) print timediff() syncCols = ColData_User.getSyncCols() if Registrar.DEBUG_PROGRESS: syncProgressCounter = ProgressCounter(len(globalMatches)) for count, match in enumerate(globalMatches): if Registrar.DEBUG_PROGRESS: syncProgressCounter.maybePrintUpdate(count) mObject = match.mObjects[0] sObject = match.sObjects[0] syncUpdate = SyncUpdate_Usr(mObject, sObject) syncUpdate.update(syncCols) # SanitationUtils.safePrint( syncUpdate.tabulate(tablefmt = 'simple')) if syncUpdate.mUpdated and syncUpdate.mDeltas: insort(mDeltaUpdates, syncUpdate) if syncUpdate.sUpdated and syncUpdate.sDeltas: insort(sDeltaUpdates, syncUpdate)
class UsrSyncClient_SSH_ACT(SyncClient_Abstract): def __init__(self, connectParams, dbParams, fsParams): self.dbParams = dbParams self.fsParams = fsParams super(UsrSyncClient_SSH_ACT, self).__init__(connectParams) def attemptConnect(self): self.service = paramiko.SSHClient() self.service.set_missing_host_key_policy(paramiko.AutoAddPolicy()) self.service.connect(**self.connectParams) @property def connectionReady(self): return self.service and self.service._transport and self.service._transport.active def execSilentCommandAssert(self, command): self.assertConnect() stdin, stdout, stderr = self.service.exec_command(command) if stdin: pass #gets rid of annoying warnings possible_errors = stdout.readlines() + stderr.readlines() for error in possible_errors: if re.match("^Countries.*", error): print error continue assert not error, "command <%s> returned errors: %s" % ( SanitationUtils.coerceUnicode(command), SanitationUtils.coerceUnicode(error) ) def putFile(self, localPath, remotePath): self.assertConnect() # remoteDir, remoteFileName = os.path.split(remotePath) remoteDir = os.path.split(remotePath)[0] exception = Exception() sftpClient = self.service.open_sftp() if remoteDir: try: sftpClient.stat(remoteDir) except: sftpClient.mkdir(remoteDir) sftpClient.put(localPath, remotePath) fstat = sftpClient.stat(remotePath) sftpClient.close() if not fstat: exception = UserWarning("could not stat remote file") # try: # sftpClient = self.service.open_sftp() # if remoteDir: # try: # sftpClient.stat(remoteDir) # except: # sftpClient.mkdir(remoteDir) # sftpClient.put(localPath, remotePath) # fstat = sftpClient.stat(remotePath) # if not fstat: # exception = UserWarning("could not stat remote file") # except Exception, e: # exception = e # finally: # sftpClient.close() # if not isinstance(exception, Exception): # raise exception def assertRemoteFileExists(self, remotePath, assertion = ""): self.assertConnect() # stdin, stdout, stderr = self.service.exec_command('stat "%s"' % remotePath) stderr = self.service.exec_command('stat "%s"' % remotePath)[2] possible_errors = stderr.readlines() assert not possible_errors, " ".join([assertion, "stat returned possible errors", str(possible_errors)]) @classmethod def printFileProgress(self, completed, total): if not hasattr(self, 'progressCounter'): self.progressCounter = ProgressCounter(total) self.progressCounter.maybePrintUpdate(completed) def getDeleteFile(self, remotePath, localPath): self.assertRemoteFileExists(remotePath) sftpClient = self.service.open_sftp() sftpClient.get(remotePath, localPath, self.printFileProgress) sftpClient.remove(remotePath) sftpClient.close() # exception = None # try: # sftpClient = self.service.open_sftp() # sftpClient.get(remotePath, localPath, self.printFileProgress) # sftpClient.remove(remotePath) # except Exception, e: # exception = e # finally: # sftpClient.close() # if exception: # raise exception def removeRemoteFile(self, remotePath): self.assertRemoteFileExists(remotePath) self.service.exec_command('rm "%s"' % remotePath) def uploadChanges(self, user_pkey, updates=None): if not updates: return # print "UPDATES:", updates self.assertConnect() if 'MYOB Card ID' in updates: del updates['MYOB Card ID'] updates = OrderedDict( [('MYOB Card ID', user_pkey)] \ + updates.items() ) importName = self.fsParams['importName'] outFolder = self.fsParams['outFolder'] remote_export_folder = self.fsParams['remote_export_folder'] fileRoot = 'act_i_' + importName + '_' + user_pkey fileName = fileRoot + '.csv' localPath = os.path.join(outFolder, fileName) remotePath = os.path.join(remote_export_folder, fileName) importedFile = os.path.join(remote_export_folder, fileRoot + '.imported') with open(localPath, 'w+') as outFile: csvdialect = UnicodeCsvDialectUtils.act_out dictwriter = unicodecsv.DictWriter( outFile, dialect=csvdialect, fieldnames=updates.keys(), encoding='utf8', extrasaction='ignore', ) dictwriter.writeheader() dictwriter.writerow(updates) self.putFile( localPath, remotePath) tokens = [ 'cd ' + remote_export_folder + ';', '{db_i_exe} "-d{db_name}" "-h{db_host}" "-u{db_user}" "-p{db_pass}"'.format( **self.dbParams ), ('"%s"' % fileName) if fileName else None ] command = " ".join( token for token in tokens if token) # command = " ".join(filter(None,)) # # command = " ".join(filter(None,[ # 'cd ' + remote_export_folder + ';', # '{db_i_exe} "-d{db_name}" "-h{db_host}" "-u{db_user}" "-p{db_pass}"'.format( # **self.dbParams # ), # ('"%s"' % fileName) if fileName else None # ])) self.execSilentCommandAssert(command) try: self.removeRemoteFile(importedFile) except: raise Exception("import didn't produce a .imported file") def analyseRemote(self, parser, since=None, limit=None): if not since: since = '1970-01-01' if limit: # todo: implement limit # this gets rid of unused argument warnings pass importName = self.fsParams['importName'] remote_export_folder = self.fsParams['remote_export_folder'] fileRoot = 'act_x_' + importName fileName = fileRoot + '.csv' inFolder = self.fsParams['inFolder'] localPath = os.path.join(inFolder, fileName) remotePath = os.path.join(remote_export_folder, fileName) tokens = [ 'cd ' + remote_export_folder + ';', '{db_x_exe} "-d{db_name}" "-h{db_host}" "-u{db_user}" "-p{db_pass}" -c"{fields}"'.format( **self.dbParams ), '-s"%s"' % since, '"%s"' % fileName ] command = " ".join([token for token in tokens if token]) # command = " ".join(filter(None,[ # 'cd ' + remote_export_folder + ';', # '{db_x_exe} "-d{db_name}" "-h{db_host}" "-u{db_user}" "-p{db_pass}" -c"{fields}"'.format( # **self.dbParams # ), # '-s"%s"' % since, # '"%s"' % fileName # ])) print "executing export command..." self.execSilentCommandAssert(command) print "donloading file..." self.getDeleteFile(remotePath, localPath) print "analysing file..." parser.analyseFile(localPath, dialect_suggestion='act_out')