def rename(filename, replacethis, withthis):
"""
Simply renames files
Inputs:
filename input file to rename
replacethis String to be replaced. such as " " (a space)
withthis What to replace the string with. such as "_" (an underscore)
Outputs:
newfilename returns the new name of the renamed file.
"""
if replacethis in filename:
# make sure the filenameexists
exists(filename)
# define a new filename
newfilename = filename.replace(replacethis, withthis)
# rename the file
os.rename(filename,newfilename)
print("renamed" + filename + "to" + newfilename)
return newfilename
else:
return filename
def rename(filename, replace_this, with_this):
"""
renames a file
:param filename: input file to rename
:param replace_this: string to be replaced. such as " " (a space)
:param with_this: what to replace the string with. such as "_" (an underscore)
:return newfilename: the new name of the file.
"""
if replace_this in filename:
# make sure the filename exists
exists(filename)
# define a new filename
newfilename = filename.replace(replace_this, with_this)
# rename the file
os.rename(filename, newfilename)
print("renamed" + filename + "to" + newfilename)
return newfilename
else:
return filename
def list_files(recursive, directory, contains=None, not_contains=None):
"""
Simple file listing function with more versatility than python builtins or arcpy.List
This function sifts through a directory and returns a list of filepaths for all files
meeting the input criteria. Useful for discriminatory iteration or recursive searches.
Could be used to find all tiles with a given datestring such as 'MOD11A1.A2012', or
perhaps all Band 4 tiles from a directory containing landsat 8 data.
:param recursive: 'True' if search should search subfolders within the directory
'False' if search should ignore files in subfolders.
:param directory: The directory in which to search for files meeting the criteria
:param contains: search criteria to limit returned file list. File names must
contain parameters listed here. If no criteria exists use 'False'
:param not_contains: search criteria to limit returned file list. File names must not
contain parameters listed here. If no criteria exists use 'False'
:return filelist: An array of full filepaths meeting the criteria.
"""
# import modules and set up empty lists
filelist = []
templist = []
# ensure input directory actually exists
if not exists(directory):
raise Exception("{0} is not a valid file or folder!".format(directory))
# Ensure single strings are in list format for the loops below
if contains:
contains = enf_list(contains)
if not_contains:
not_contains = enf_list(not_contains)
not_contains.append(
'sr.lock') # make sure lock files don't get counted
else:
not_contains = ['sr.lock'] # make sure lock files don't get counted
# use os.walk commands to search through whole directory if recursive
if recursive:
for root, dirs, files in os.walk(directory):
for basename in files:
filename = os.path.join(root, basename)
# if both conditions exist, add items which meet contains criteria
if contains and not_contains:
for i in contains:
if i in basename:
templist.append(filename)
# if the entire array of 'contains' terms were found, add to list
if len(templist) == len(contains):
filelist.append(filename)
templist = []
# remove items which do not meet the DoesNotcontain criteria
for j in not_contains:
if j in basename:
try:
filelist.remove(filename)
except:
pass
# If both conditions do not exist (one is false)
else:
# determine if a file is good. if it is, add it to the list.
if contains:
for i in contains:
if i in basename:
templist.append(filename)
# if the entire array of 'contains' terms were found, add to list
if len(templist) == len(contains):
filelist.append(filename)
templist = []
# add all files to the list, then remove the bad ones.
elif not_contains:
filelist.append(filename)
for j in not_contains:
if j in basename:
try:
filelist.remove(filename)
except:
pass
else:
filelist.append(filename)
# if neither conditionexists
if not contains and not not_contains:
filelist.append(filename)
# use a simple listdir if recursive is False
else:
# list only files in current directory, not subdir and check criteria
try:
for basename in os.listdir(directory):
filename = os.path.join(directory, basename)
if os.path.isfile(filename):
if contains:
for i in contains:
if i in basename:
templist.append(filename)
# if the entire array of 'contains' terms were found, add to list
if len(templist) == len(contains):
filelist.append(filename)
templist = []
else:
filelist.append(filename)
# Remove any files from the filelist that fail not_contains criteria
if not_contains:
for j in not_contains:
if j in basename:
try:
filelist.remove(filename)
except:
pass
except:
pass
print('Files found which meet all input criteria: {0}'.format(
len(filelist)))
return filelist
def list_files(recursive, Dir, Contains = False, DoesNotContain = False):
"""
Simple file listing function with more versatility than python builtins or arcpy.List
This function sifts through a directory and returns a list of filepaths for all files
meeting the input criteria. Useful for discriminatory iteration or recursive searches.
Could be used to find all tiles with a given datestring such as 'MOD11A1.A2012', or
perhaps all Band 4 tiles from a directory containing landsat 8 data.
Inputs:
recursive 'True' if search should search subfolders within the directory
'False' if search should ignore files in subfolders.
Dir The directory in which to search for files meeting the criteria
Contains search criteria to limit returned file list. File names must
contain parameters listed here. If no criteriaexists use 'False'
DoesNotContain search criteria to limit returned file list. File names must not
contain parameters listed here. If no criteriaexists use 'False'
Outputs:
filelist An array of full filepaths meeting the criteria.
Example Usage:
from dnppy import core
filelist = core.list_files(True,r'E:\Landsat7','B1',['gz','xml','ovr'])
The above statement will find all the Band 1 tifs in a landsat data directory
without including the associated metadata and uncompressed gz files.
"filelist" variable will contain full filepaths to all files found.
"""
# import modules and set up empty lists
filelist = []
templist = []
# ensure input directory actually exists
if not exists(Dir):
raise Exception("{0} is not a valid file or folder!".format(Dir))
# Ensure single strings are in list format for the loops below
if Contains:
Contains = enf_list(Contains)
if DoesNotContain:
DoesNotContain = enf_list(DoesNotContain)
DoesNotContain.append('sr.lock') # make sure lock files don't get counted
else:
DoesNotContain=['sr.lock'] # make sure lock files don't get counted
# use os.walk commands to search through whole directory if recursive
if recursive:
for root,dirs,files in os.walk(Dir):
for basename in files:
filename = os.path.join(root,basename)
# if both conditions exist, add items which meet Contains criteria
if Contains and DoesNotContain:
for i in Contains:
if i in basename:
templist.append(filename)
# if the entire array of 'Contains' terms were found, add to list
if len(templist)==len(Contains):
filelist.append(filename)
templist=[]
# remove items which do not meet the DoesNotcontain criteria
for j in DoesNotContain:
if j in basename:
try: filelist.remove(filename)
except: pass
# If both conditions do not exist (one is false)
else:
# determine if a file is good. if it is, add it to the list.
if Contains:
for i in Contains:
if i in basename:
templist.append(filename)
# if the entire array of 'Contains' terms were found, add to list
if len(templist)==len(Contains):
filelist.append(filename)
templist=[]
# add all files to the list, then remove the bad ones.
elif DoesNotContain:
filelist.append(filename)
for j in DoesNotContain:
if j in basename:
try: filelist.remove(filename)
except: pass
else:
filelist.append(filename)
# if neither conditionexists
if not Contains and not DoesNotContain:
filelist.append(filename)
# use a simple listdir if recursive is False
else:
# list only files in current directory, not subdir and check criteria
try:
for basename in os.listdir(Dir):
filename = os.path.join(Dir,basename)
if os.path.isfile(filename):
if Contains:
for i in Contains:
if i in basename:
templist.append(filename)
# if the entire array of 'Contains' terms were found, add to list
if len(templist)==len(Contains):
filelist.append(filename)
templist=[]
else:
filelist.append(filename)
# Remove any files from the filelist that fail DoesNotContain criteria
if DoesNotContain:
for j in DoesNotContain:
if j in basename:
try: filelist.remove(filename)
except: pass
except: pass
# Print a quick status summary before finishing up if Quiet is False
print('Files found which meet all input criteria: {0}'.format(len(filelist)))
return filelist
def list_files(recursive, directory, contains=None, not_contains=None):
"""
Simple file listing function with more versatility than python builtins or arcpy.List
This function sifts through a directory and returns a list of filepaths for all files
meeting the input criteria. Useful for discriminatory iteration or recursive searches.
Could be used to find all tiles with a given datestring such as 'MOD11A1.A2012', or
perhaps all Band 4 tiles from a directory containing landsat 8 data.
:param recursive: 'True' if search should search subfolders within the directory
'False' if search should ignore files in subfolders.
:param directory: The directory in which to search for files meeting the criteria
:param contains: search criteria to limit returned file list. File names must
contain parameters listed here. If no criteria exists use 'False'
:param not_contains: search criteria to limit returned file list. File names must not
contain parameters listed here. If no criteria exists use 'False'
:return filelist: An array of full filepaths meeting the criteria.
"""
# import modules and set up empty lists
filelist = []
templist = []
# ensure input directory actually exists
if not exists(directory):
raise Exception("{0} is not a valid file or folder!".format(directory))
# Ensure single strings are in list format for the loops below
if contains:
contains = enf_list(contains)
if not_contains:
not_contains = enf_list(not_contains)
not_contains.append("sr.lock") # make sure lock files don't get counted
else:
not_contains = ["sr.lock"] # make sure lock files don't get counted
# use os.walk commands to search through whole directory if recursive
if recursive:
for root, dirs, files in os.walk(directory):
for basename in files:
filename = os.path.join(root, basename)
# if both conditions exist, add items which meet contains criteria
if contains and not_contains:
for i in contains:
if i in basename:
templist.append(filename)
# if the entire array of 'contains' terms were found, add to list
if len(templist) == len(contains):
filelist.append(filename)
templist = []
# remove items which do not meet the DoesNotcontain criteria
for j in not_contains:
if j in basename:
try:
filelist.remove(filename)
except:
pass
# If both conditions do not exist (one is false)
else:
# determine if a file is good. if it is, add it to the list.
if contains:
for i in contains:
if i in basename:
templist.append(filename)
# if the entire array of 'contains' terms were found, add to list
if len(templist) == len(contains):
filelist.append(filename)
templist = []
# add all files to the list, then remove the bad ones.
elif not_contains:
filelist.append(filename)
for j in not_contains:
if j in basename:
try:
filelist.remove(filename)
except:
pass
else:
filelist.append(filename)
# if neither conditionexists
if not contains and not not_contains:
filelist.append(filename)
# use a simple listdir if recursive is False
else:
# list only files in current directory, not subdir and check criteria
try:
for basename in os.listdir(directory):
filename = os.path.join(directory, basename)
if os.path.isfile(filename):
if contains:
for i in contains:
if i in basename:
templist.append(filename)
# if the entire array of 'contains' terms were found, add to list
if len(templist) == len(contains):
filelist.append(filename)
templist = []
else:
filelist.append(filename)
# Remove any files from the filelist that fail not_contains criteria
if not_contains:
for j in not_contains:
if j in basename:
try:
filelist.remove(filename)
except:
pass
except:
pass
print ("Files found which meet all input criteria: {0}".format(len(filelist)))
return filelist
def list_files(recursive, Dir, Contains=False, DoesNotContain=False):
"""
Simple file listing function with more versatility than python builtins or arcpy.List
This function sifts through a directory and returns a list of filepaths for all files
meeting the input criteria. Useful for discriminatory iteration or recursive searches.
Could be used to find all tiles with a given datestring such as 'MOD11A1.A2012', or
perhaps all Band 4 tiles from a directory containing landsat 8 data.
Inputs:
recursive 'True' if search should search subfolders within the directory
'False' if search should ignore files in subfolders.
Dir The directory in which to search for files meeting the criteria
Contains search criteria to limit returned file list. File names must
contain parameters listed here. If no criteriaexists use 'False'
DoesNotContain search criteria to limit returned file list. File names must not
contain parameters listed here. If no criteriaexists use 'False'
Outputs:
filelist An array of full filepaths meeting the criteria.
Example Usage:
from dnppy import core
filelist = core.list_files(True,r'E:\Landsat7','B1',['gz','xml','ovr'])
The above statement will find all the Band 1 tifs in a landsat data directory
without including the associated metadata and uncompressed gz files.
"filelist" variable will contain full filepaths to all files found.
"""
# import modules and set up empty lists
filelist = []
templist = []
# ensure input directory actually exists
if not exists(Dir):
raise Exception("{0} is not a valid file or folder!".format(Dir))
# Ensure single strings are in list format for the loops below
if Contains:
Contains = enf_list(Contains)
if DoesNotContain:
DoesNotContain = enf_list(DoesNotContain)
DoesNotContain.append(
'sr.lock') # make sure lock files don't get counted
else:
DoesNotContain = ['sr.lock'] # make sure lock files don't get counted
# use os.walk commands to search through whole directory if recursive
if recursive:
for root, dirs, files in os.walk(Dir):
for basename in files:
filename = os.path.join(root, basename)
# if both conditions exist, add items which meet Contains criteria
if Contains and DoesNotContain:
for i in Contains:
if i in basename:
templist.append(filename)
# if the entire array of 'Contains' terms were found, add to list
if len(templist) == len(Contains):
filelist.append(filename)
templist = []
# remove items which do not meet the DoesNotcontain criteria
for j in DoesNotContain:
if j in basename:
try:
filelist.remove(filename)
except:
pass
# If both conditions do not exist (one is false)
else:
# determine if a file is good. if it is, add it to the list.
if Contains:
for i in Contains:
if i in basename:
templist.append(filename)
# if the entire array of 'Contains' terms were found, add to list
if len(templist) == len(Contains):
filelist.append(filename)
templist = []
# add all files to the list, then remove the bad ones.
elif DoesNotContain:
filelist.append(filename)
for j in DoesNotContain:
if j in basename:
try:
filelist.remove(filename)
except:
pass
else:
filelist.append(filename)
# if neither conditionexists
if not Contains and not DoesNotContain:
filelist.append(filename)
# use a simple listdir if recursive is False
else:
# list only files in current directory, not subdir and check criteria
try:
for basename in os.listdir(Dir):
filename = os.path.join(Dir, basename)
if os.path.isfile(filename):
if Contains:
for i in Contains:
if i in basename:
templist.append(filename)
# if the entire array of 'Contains' terms were found, add to list
if len(templist) == len(Contains):
filelist.append(filename)
templist = []
else:
filelist.append(filename)
# Remove any files from the filelist that fail DoesNotContain criteria
if DoesNotContain:
for j in DoesNotContain:
if j in basename:
try:
filelist.remove(filename)
except:
pass
except:
pass
# Print a quick status summary before finishing up if Quiet is False
print('Files found which meet all input criteria: {0}'.format(
len(filelist)))
return filelist
sftp_dst = remote.sftpclient(args.destination) if origin_dst == 'remote' else None
root_src = remote.info(args.source).root if origin_src == 'remote' else os.path.abspath(args.source)
root_dst = remote.info(args.destination).root if origin_dst == 'remote' else os.path.abspath(args.destination)
ignore = ignore(root_src + os.sep + 'config.json', sftp_src)
if args.backup:
print 'backuping folder ' + root_dst
zip(root_dst, paths(root_dst, ignore, sftp_dst), args.backup, sftp_dst)
for file_src in paths(root_src, ignore, sftp_src):
file_dst = file_src.replace(root_src, root_dst)
if utils.isdir(file_src, sftp_src):
print 'creating folder ' + file_dst
utils.mkdir(file_dst, sftp_dst)
else:
data_src = copy(file_src, sftp_src)
if exists(file_dst):
data_dst = copy(file_dst, sftp_dst)
md5_data_src = md5(data_src).digest()
md5_data_dst = md5(data_dst).digest()
if md5_data_src == md5_data_dst:
print 'unchanged ' + file_dst
else:
print 'modificing ' + file_dst
paste(file_dst, data_src, sftp_dst)
else:
print 'creating ' + file_dst
paste(file_dst, data_src, sftp_dst)
print 'golem finish'
