def transfer_files(self, filename, local_path, remote_path, transfer_type, overwrite=True):
local_file_path = os.path.join(local_path, filename)
remote_file_path = remote_path + "/" + filename
if transfer_type==config.SFTP_TRANSFER_UL and not os.path.exists(local_file_path):
print("the uploaded file %s not exist. Please check it before uploading it!" % (local_file_path))
return False
with pysftp.Connection(config.SFTP_HOSTNAME, username=config.SFTP_USERNAME,password=config.SFTP_PASSWORD, cnopts=cnopts) as sftp:
if transfer_type==config.SFTP_TRANSFER_DL and not sftp.exists(remote_file_path):
print("the downloaded file %s not exist. Please check it before downloading it!" % (remote_file_path))
return False
if transfer_type == config.SFTP_TRANSFER_UL:
if not sftp.exists(remote_file_path) or overwrite:
with pysftp.cd(local_path):
with sftp.cd(remote_path):
sftp.put(filename, callback=self.progress_callback, preserve_mtime=False)
if sftp.exists(remote_file_path) and sftp.isfile(remote_file_path):
return True
else:
return False
elif transfer_type == config.SFTP_TRANSFER_DL:
if not os.path.exists(local_file_path) or overwrite:
with pysftp.cd(local_path):
with sftp.cd(remote_path):
sftp.get(filename, callback=self.progress_callback, preserve_mtime=False)
if os.path.exists(local_file_path) and os.path.isfile(local_file_path):
return True
else:
return False
def _put(self, local_path, remote_path):
"""Upload local file to remote path.
Args:
local_path: local file path
remote_path: remote path to upload. /files/ wille be prepended.
"""
if self.sftp_connection is not None:
dir_path = os.path.dirname(os.path.realpath(__file__))
pysftp.cd(dir_path)
remote_path = os.path.dirname(remote_path)
self.sftp_connection.makedirs("/files/" + remote_path)
self.sftp_connection.chdir('/files/' + remote_path)
filename = os.path.basename(local_path)
try:
# upload file to files/ on remote
self.sftp_connection.put(local_path,
filename,
None,
True,
preserve_mtime=True)
except IOError:
print("Log: remote path does not exist : " + str(local_path) +
" -> remote:" + str(filename))
# Marked as duplicate except by pylint but still following the pysftp.put doc.
except OSError:
print("Log: Local path does not exist : " + str(local_path) +
" -> remote:" + str(filename))
def pysftp_method(rdsf_config, remotepath, localpath):
config = json.load(open(rdsf_config, 'rt'))
localdir = os.path.dirname(localpath)
with pysftp.Connection(config['server'],
username=config['user'],
password=config['password']) as sftp:
with pysftp.cd(localdir):
sftp.get(remotepath)
def get_dtime(self, sDTime, eDTime, outDir=outDir, ow=False):
srcDIR = ['%i/%02d/'%(y,m) for y in range(sDTime.year, eDTime.year+1) for m in range(1,13)]
srcDIR = srcDIR[sDTime.month-1 : None if eDTime.month==12 else eDTime.month-12] # tentaive patch to resolve skipping December
#srcDIR = srcDIR[sDTime.month-1 : eDTime.month-12]
srcPATH = []
for srcDir in srcDIR:
if not self.sftp.exists(srcDir):
print 'Not Exists!', srcDir
continue
with self.sftp.cd(srcDir):
srcPATH.extend([os.path.join(srcDir,srcFName) for srcFName in self.sftp.listdir()])
DTIME = [datetime.strptime( self.dtFunc(srcPath),'%Y%m%d') for srcPath in srcPATH]
for dtime, srcPath in map(None, DTIME, srcPATH):
if sDTime <= dtime and eDTime >= dtime: pass
else: continue
subDir = dtime.strftime('%Y/%m')
#saveDir = os.path.join(outDir, self.prjName, self.subPrj, subDir)
saveDir = os.path.join(outDir, self.prjName, self.subPrj, self.version, subDir)
print "*"*50
print "*"*50
print "*"*50
print "*"*50
print saveDir
if not os.path.exists( saveDir ): os.makedirs( saveDir )
#outPath = os.path.join(outDir, self.prjName, self.subPrj, srcPath)
outPath = os.path.join(outDir, self.prjName, self.subPrj, self.version, srcPath)
if os.path.exists(outPath) and ow == False:
print 'Skip (ow:%5s) ::'%ow, outPath
continue
with pysftp.cd(saveDir):
self.sftp.get(srcPath)
print 'Save (ow:%5s) ->'%ow, outPath
def build(self, image, file, extension):
localPath = self.imageBasePath + '/' + file + "." + extension
localName = file + '.' + extension
self.pygame.image.save(image, localPath)
cnopts = pysftp.CnOpts()
cnopts.hostkeys = None
con = pysftp.Connection(
host=self.ftpServer,
username=self.ftpUsername,
password=self.ftpPassword,
cnopts=cnopts
)
with pysftp.cd(self.imageBasePath):
con.chdir(self.ftpFolder)
if not con.exists(self.gameName):
con.mkdir(self.gameName)
con.chdir(self.gameName)
if con.exists(self.gameName):
con.remove(localName)
con.put(localName)
con.close()
return self.ftpBasePath + '/' + self.gameName + '/' + file + '.' + extension
src_dir = path(SOURCE_DIR)
for f in src_dir.files():
if today_str in f.name or 'lastsnap' in f.name:
print("{} | Not moving {}".format(now(), f.name))
continue
#files_to_move.append(f.name)
files_to_move[f.name] = f
print("{} | Adding {} to list".format(now(), f.name))
#files_to_delete.append(f)
with pysftp.Connection(const.NAS_IP,
username=const.NAS_USER,
password=const.NAS_PWD) as sftp:
print("Connected")
with pysftp.cd(SOURCE_DIR):
print("Changed directory")
for i in files_to_move.keys():
sftp.chdir(DEST_DIR)
if not sftp.exists(i):
print("{1} Moving {0}".format(i, now()))
try:
sftp.put(i, preserve_mtime=True)
except FileNotFoundError:
pass
else:
print("{1} Not moving {0} - already exists remotely".format(
i, now()))
print("{1} Removing {0}".format(i, now()))
try:
def uploadNotifcation():
sftp = connectToFTPServer() #login to ftp server
#get brandPersistance.json Path and data
brandPersFile = 'C:\Users\IBM_ADMIN\PycharmProjects\CSVCompChecker\sbrandPersistance.json'
with open(brandPersFile) as data_file:
data = json.load(data_file)
#get brand list
print(sftp.pwd)
propFile = 'C:\Users\IBM_ADMIN\PycharmProjects\CSVCompChecker\config\dct_app_local.properties'
propName = 'test_brandlist'
brandList = loadODSProps.getPropertyValueAsList(propFile, propName)
print brandList
#remotepath
remotepath = '/bacc/upload/usageTest/usage'
#usage pattern to check
usagePattern = "[\\w\\W]*_usage_20\\d*_[\\w\\W]*.csv"
compiledPattern = re.compile(usagePattern)
#get remote path to each brand folder inside ftp
for brand in brandList:
brandPath = remotepath + '/' + brand
print "Current BrandPath: " + brandPath
#cd into there
with sftp.cd(brandPath):
print(sftp.pwd)
#variable to check if brand folder has usage file (set back to false for each brand)
usageFileFound = False
#get list of files inside brand folder
brandFileList = sftp.listdir()
print brandFileList
for file in brandFileList:
if compiledPattern.match(file):
print "We found a match for usage pattern for file: " + file
usageFileFound = True
else:
print "We did NOT find match for usage pattern for file: " + file
#update json file with current day
data["brands"][brand]["currDay"] = int(datetime.date.today().strftime("%w")) - 1;
with open(brandPersFile, "w") as data_file:
data_file.write(json.dumps(data))
currDay = data["brands"][brand]["currDay"]
prevDay = data["brands"][brand]["prevDay"]
if currDay != prevDay:
data["brands"][brand]["checkFalse"] = False #new day so reset to false and write and set prev to curr -> write to file
data["brands"][brand]["prevDay"] = currDay
with open(brandPersFile, "w") as data_file:
data_file.write(json.dumps(data))
if usageFileFound:
data["brands"][brand]["count"] = 0 #reset count -> write to file
with open(brandPersFile, "w") as data_file:
data_file.write(json.dumps(data))
elif not data["brands"][brand]["checkFalse"]:
data["brands"][brand]["checkFalse"] = True #set to true
data["brands"][brand]["count"] += 1 #first time running today and usage file not found so increment count
with open(brandPersFile, "w") as data_file:
data_file.write(json.dumps(data))
iteration = getBrandIteration(brand, data) #get iteration type and check
if data["brands"][brand]["count"] >= 7 and iteration == "weekly":
#then iteration is weekly and we are past deadline
#notify the brand
emailUtil.notifyUsers()
elif data["brands"][brand]["count"] and iteration == "monthly":
#then iteration is monthly and past deadline
#notify the brand
emailUtil.notifyUsers()
# Pattern looks for 4 digits of year, then two digits of month, then two of day
# pattern is anchored in file name with '-'
p = re.compile("-(\d\d\d\d)(\d\d)(\d\d)")
files_to_move = []
files_to_delete = []
src_dir = path(SOURCE_DIR)
for f in src_dir.files():
if today_str in f.name or 'lastsnap' in f.name:
print("{1} Not moving {0}".format(f.name, now()))
continue
files_to_move.append(f.name)
files_to_delete.append(f)
with pysftp.Connection('192.168.0.224', username='******',
password='******') as sftp:
with pysftp.cd(SOURCE_DIR):
for i in files_to_move:
sftp.chdir(DEST_DIR)
if not sftp.exists(i):
print("{1} Moving {0}".format(i, now()))
sftp.put(i, preserve_mtime=True)
else:
print("{1} Not moving {0} - already exists remotely".format(
i, now()))
for f in files_to_delete:
print("{1} Removing {0}".format(f.name, now()))
f.remove()
import pysftp as sftp
"""Получение логов"""
# TODO Реализовать для получения json файлов и любых других
cnopts = sftp.CnOpts()
cnopts.hostkeys = None
with sftp.Connection('192.168.1.33', username='******', password='******', cnopts=cnopts) as sftp:
with sftp.cd('/home/pad/jetty/logs/'): # temporarily chdir to public
remotepath = r'/home/pad/jetty/logs/'
localpath = 'C://work/ttv_autotest/qa_tools/tmp_files/'
sftp.get_d(remotepath, localpath)
def test_cd_local_bad():
'''test pysftp.cd on non-existing local directory'''
with pytest.raises(OSError):
with pysftp.cd('not-there'):
pass
def test_cd_local():
'''test pysftp.cd on local directories'''
original = os.getcwd()
with pysftp.cd('docs'):
assert os.getcwd() == os.path.join(original, 'docs')
assert os.getcwd() == original
