root, ext = os.path.splitext(filename)

basename = os.path.basename(filename)

if ext in ['.pyc', '.pyo', '.backup', '.aux'] \

or ext.endswith('~') or ext.endswith('#') or basename.startswith('.'):

return True

if 'CVS' in dirs: dirs.remove('CVS')

if '.svn' in dirs: dirs.remove('.svn')

if '.bzr' in dirs: dirs.remove('.bzr')

if trash_dir in dirs: dirs.remove(trash_dir)

def __init__(self, server_url, skip_dirs = [], export_only=False):

if not server_url.startswith('ftp://'):

server_url = 'ftp://' + server_url

o = urlparse(server_url)

assert o.scheme=='ftp',`o`

self.server = o.hostname

self._username = o.username

self._password = o.password

remote_path = normpath(o[2] or '/')

if remote_path.startswith('//'):

remote_path = remote_path[1:]

assert remote_path.startswith('/'),`remote_path`

self.remote_path = remote_path

self._ftp = None

self.clock_offset = 0

self.skip_dirs = skip_dirs or []

self.export_only = export_only

@property

def server_url(self):

return 'ftp://%s@%s' % (self.username, self.server, self.remote_path)

@property

def username(self):

n = self._username

if n is None:

while 1:

n = raw_input('Enter username for ftp://%s: ' % (self.server))

if n:

break

self._username = n

return n

@property

def password(self):

n = self._password

if n is None:

if gtkpresence:

try:

keyring = gnomekeyring.get_default_keyring_sync()

keyring_info = gnomekeyring.get_info_sync(keyring)

if keyring_info.get_is_locked():

nof_tries = 3

for i in range(nof_tries):

keyring_pass = getpass('Enter password to unlock your default keychain: ')

try:

gnomekeyring.unlock_sync(keyring, keyring_pass)

break

except gnomekeyring.DeniedError:

if i+1==nof_tries:

sys.stderr.write('\nFailed to unlock keychain (%s times)\n' % (i+1))

else:

sys.stderr.write("\nInvalid password, try again...\n")

items = gnomekeyring.find_items_sync(gnomekeyring.ITEM_NETWORK_PASSWORD,

{"server": self.server,

"protocol": "ftp",

"user": self.username})

if len(items) > 0:

n = items[0].secret

except gnomekeyring.DeniedError:

sys.stderr.write("\nGnome keyring error : Access denied ..\n")

except gnomekeyring.NoMatchError:

sys.stderr.write("\nGnome keyring error : No credential for %s..\n" % self.server)

except Exception, msg:

sys.stderr.write("\nGnome keyring error : %s\n" % (msg))

sys.stderr.flush()

if n is None:

n = getpass('Enter password for ftp://%s@%s: ' % (self.username, self.server))

self._password = n

return n

@property

def ftp(self):

c = self._ftp

if c is None:

sys.stdout.write('<connecting to %s..' % (self.server))

sys.stdout.flush()

self._ftp = c = FTP(self.server, self.username, self.password)

self.clock_offset = self.clocksync()

sys.stdout.write('clock offset: %s> ' % (self.clock_offset))

sys.stdout.flush()

return c

def abspath(self, *paths):

filename = os.path.join(*(paths or ('',)))

if not os.path.isabs(filename):

filename = os.path.join(self.remote_path, filename)

return filename

def get_mtime(self, name):

""" Return modification time of the file in ftp server.

If the file does not exist, return None.

If the name refers to a directory, return 0.

To get the local modification time, use

``get_mtime()-clock_offset``.

"""

filename = self.abspath(name)

try:

resp = self.ftp.sendcmd('MDTM ' + filename)

except error_perm, msg:

s = str(msg)

if s.startswith('550 I can only retrieve regular files'):

# filename is directory

return 0

if s.startswith('550 Can\'t check for file existence'):

# file with filename does not exist

return None

raise

assert resp[:3]=='213', `resp, filename`

modtime = int(time.mktime(time.strptime(resp[3:].strip(), '%Y%m%d%H%M%S')))

return modtime

def clocksync(self):

fn = '.ftpsync.clocksync'

rfn = self.abspath(fn)

if not os.path.isfile(fn):

f = open(fn,'w')

f.write('temporary file created by ftpsync.py')

f.close()

self.makedirs(os.path.dirname(rfn))

f = open(fn, 'rb')

self.ftp.storbinary('STOR ' + rfn, f, 1024)

f.close()

local_time1 = time.time()

remote_time = self.get_mtime(rfn)

local_time2 = time.time()

self.ftp.delete(rfn)

os.remove(fn)

if local_time2 < remote_time: # remote is in future

sync_off = int(remote_time - local_time2)

elif local_time1 > remote_time: # remote is in past or equal

sync_off = int(remote_time - local_time1)

else:

sync_off = 0

return sync_off

def get_listing_map(self, path):

filename = self.abspath(path, '.listing')

l = []

self.ftp.retrbinary('RETR ' + filename, l.append, 8*1024)

l = ''.join(l)

func = lambda fn, mtime: (fn, int(mtime))

return dict([func(*line.rsplit(':', 1)) for line in l.splitlines()])

def upload_listing_map(self, path, d):

filename = self.abspath(path, '.listing')

fp = StringIO('\n'.join(['%s:%s' % item for item in d.iteritems()]))

try:

self.ftp.storbinary('STOR ' + filename, fp, 8*1024)

except error_perm:

return 0

return 1

def get_remote_files(self, directory='', verbose=True, listing=True,

update_listing=False):

""" Return a {files:modification times} map of the ftp directory.

If listing is True then .listing files are used to accelerate

getting the map information. When .listing file does not exist,

it will be created.

When uploading a file or a directory then the uploader is

responsible for removing .listing file from the parent

directory of the uploaded file or directory. The .listing file

should also be removed when removing a file, this is not

necessary when removing a directory, though. The above will

ensure that .listing will be kept up-to-date.

Note that for directories the modification times is set to 0,

and for .listing files it is -1, in the returned map. The

modification time is using the clock in ftp server.

The paths to files correspond to absolute paths in the ftp

server.

"""

r = {}

dirs = []

wd = self.abspath(directory)

if wd in self.skip_dirs:

return r

r[wd] = 0 # directories have 0 mtime.

if verbose:

sys.stdout.write('listing directory %r ' % (wd))

sys.stdout.flush()

lst = filter(lambda n: n not in ['.', '..'], self.ftp.nlst(wd))

if verbose:

sys.stdout.write('[%s items]: ' % (len(lst)))

sys.stdout.flush()

if listing:

lstfn = os.path.join(wd, '.listing')

r[lstfn] = -1 # the mtime of .listing file is not used

if '.listing' in lst and not update_listing:

d = self.get_listing_map(wd)

dirnames = set([os.path.dirname(fn) for fn, mtime in d.items() if mtime>0]\

+ [dn for dn, mtime in d.items() if mtime==0])

if verbose:

sys.stdout.write('<checking integrity [%s directories]> ' % (len(dirnames)))

sys.stdout.flush()

for dn in dirnames:

if verbose:

sys.stdout.write('+')

sys.stdout.flush()

fn = os.path.join(dn, '.listing')

if self.get_mtime(fn) is None:

sys.stdout.write(' <missing %r, forcing regeneration of %r>\n' % (fn, lstfn))

sys.stdout.flush()

d = None

break

if d is not None:

if verbose:

sys.stdout.write(' <using %r>\n' % (lstfn))

sys.stdout.flush()

return d

for n in lst:

path = os.path.join(wd, n)

mtime = self.get_mtime(path)

if mtime==0:

dirs.append(path)

if verbose:

sys.stdout.write('d')

sys.stdout.flush()

else:

r[path] = mtime

if verbose:

sys.stdout.write('.')

sys.stdout.flush()

if verbose:

sys.stdout.write('\n')

sys.stdout.flush()

for path in dirs:

r.update(**self.get_remote_files(path, verbose=verbose, listing=listing,

update_listing=update_listing))

if ((not self.export_only) and listing):

if self.upload_listing_map(wd, r):

if verbose:

sys.stdout.write('<uploaded %r>\n' % (lstfn))

sys.stdout.flush()

else:

sys.stderr.write('<failed to upload %r>\n' % (lstfn))

sys.stderr.flush()

return r

def fix_local_mtime(self, filename, local, verbose=True):

fullname = self.abspath(filename)

rmtime = self.get_mtime(fullname)

lmtime = int(os.path.getmtime(local))

mtime = rmtime - self.clock_offset

if verbose:

sys.stdout.write('<adjusting local mtime: %s secs>' % (lmtime - mtime))

sys.stdout.flush()

os.utime(local, (mtime, mtime))

#print local, mtime, rmtime, lmtime, int(os.path.getmtime(local))

def get_files(self, update_listing=False):

""" Return a {files:modification times} map of the ftp

directory where files are relative to remote_path and

modification times are relative to local clock.

"""

if self.remote_path=='/':

n = 0

else:

assert not self.remote_path.endswith('/'), `self.remote_path`

n = len(self.remote_path)

files = {}

for rfn, mtime in self.get_remote_files(update_listing=update_listing).items():

if mtime==0:

#skip directories

continue

if os.path.basename(rfn).startswith('.'):

continue

assert rfn[n:n+1] in ['/',''],``rfn,n``

fn = rfn[n+1:]

files[fn] = mtime - self.clock_offset

#print rfn, mtime, files[fn]

return files

def download(self, filename, target, verbose=True):

if verbose:

sys.stdout.write('downloading %r..' % (filename))

sys.stdout.flush()

fullname = self.abspath(filename)

targetdir = os.path.dirname(target)

if not os.path.exists(targetdir):

os.makedirs(targetdir)

f = open(target, 'wb')

try:

self.ftp.retrbinary('RETR '+fullname, f.write, 8*1024)

except error_perm, msg:

if verbose:

sys.stdout.write('FAILED: %s\n' % (msg))

sys.stdout.flush()

else:

sys.stderr.write('FAILED to download %r: %s\n' % (filename, msg))

sys.stderr.flush()

f.close()

os.remove(target)

return 0

f.close()

self.fix_local_mtime(filename, target)

if verbose:

sys.stdout.write(' ok [%s bytes]\n' % (os.path.getsize(target)))

sys.stdout.flush()

return 1

def makedirs(self, path, rm_local_listing = False, verbose=True, _cache=[]):

fullpath = self.abspath(path)

if fullpath in _cache:

return

_cache.append(fullpath)

parent = os.path.dirname(fullpath)

name = os.path.basename(fullpath)

if parent!='/':

self.makedirs(parent, verbose=verbose)

lst = self.ftp.nlst(parent)

if name and name not in lst:

if verbose:

sys.stdout.write('<creating directory %r>' % (fullpath))

sys.stdout.flush()

self.ftp.mkd(fullpath)

if '.listing' in lst:

listing = os.path.join(parent, '.listing')

if verbose:

sys.stdout.write('<removing %r>' % (listing))

sys.stdout.flush()

self.ftp.delete(listing)

if rm_local_listing:

lst = self.ftp.nlst(fullpath)

if '.listing' in lst:

listing = os.path.join(fullpath, '.listing')

if verbose:

sys.stdout.write('<removing %r>' % (listing))

sys.stdout.flush()

self.ftp.delete(listing)

def upload(self, filename, source, mk_backup=True, verbose=True):

fullname = self.abspath(filename)

if verbose:

sys.stdout.write('uploading %r [%s]..' % (filename, os.path.getsize(source)))

sys.stdout.flush()

self.makedirs(os.path.dirname(fullname), rm_local_listing=True, verbose=verbose)

if mk_backup:

if verbose:

sys.stdout.write('<creating %r>' % (filename+'.backup'))

sys.stdout.flush()

self.ftp.rename(fullname, fullname + '.backup')

f = open(source, 'rb')

if verbose:

sys.stdout.write('<storing>')

try:

self.ftp.storbinary('STOR '+fullname, f, 8*1024)

except error_perm, msg:

if verbose:

sys.stdout.write('FAILED: %s\n' % (msg))

sys.stdout.flush()

else:

sys.stderr.write('FAILED to upload %r: %s\n' % (filename, msg))

sys.stderr.flush()

f.close()

if mk_backup:

if verbose:

sys.stdout.write('<restoring from %r>' % (filename+'.backup'))

sys.stdout.flush()

self.ftp.rename(fullname + '.backup', fullname)

return 0

f.close()

if mk_backup:

if verbose:

sys.stdout.write('<cleaning up %r>' % (filename+'.backup'))

sys.stdout.flush()

self.ftp.delete(fullname + '.backup')

self.fix_local_mtime(filename, source)

if verbose:

sys.stdout.write(' ok\n')

sys.stdout.flush()

return 1

def remove(self, filename, verbose=True):

fullname = self.abspath(filename)

if verbose:

sys.stdout.write('<removing %r..' % (filename))

sys.stdout.flush()

try:

self.ftp.delete(fullname)

except error_perm, msg:

if verbose:

sys.stdout.write('FAILED: %s>' % (msg))

sys.stdout.flush()

else:

sys.stderr.write('FAILED to remove %r: %s\n' % (filename, msg))

sys.stderr.flush()

return

dname = os.path.dirname(fullname)

lst = self.ftp.nlst(dname)

if '.listing' in lst:

listing = os.path.join(dname, '.listing')

if verbose:

sys.stdout.write('<removing %r>' % (listing))

sys.stdout.flush()

self.ftp.delete(listing)

if verbose:

sys.stdout.write('ok>')

sys.stdout.flush()

def remove_directory(self, path, verbose=True):

fullname = self.abspath(path)

if verbose:

sys.stdout.write('<removing %r..' % (path))

sys.stdout.flush()

lst = self.ftp.nlst(fullname)

for n in lst:

if n in ['.', '..']:

continue

fn = os.path.join(fullname, n)

mtime = self.get_mtime(fn)

if mtime==0:

self.remove_directory(fn, verbose=verbose)

elif mtime:

if verbose:

sys.stdout.write('<removing %r>' % (fn))

sys.stdout.flush()

self.ftp.delete(fn)

try:

self.ftp.rmd(fullname)

except error_perm, msg:

if verbose:

sys.stdout.write('FAILED: %s>' % (msg))

sys.stdout.flush()

else:

sys.stderr.write('FAILED to remove %r: %s\n' % (path, msg))

sys.stderr.flush()

return

dname = os.path.dirname(fullname)

lst = self.ftp.nlst(dname)

if '.listing' in lst:

listing = os.path.join(dname, '.listing')

if verbose:

sys.stdout.write('<removing %r>' % (listing))

sys.stdout.flush()

self.ftp.delete(listing)

if verbose:

sys.stdout.write('ok>')

r = {}

wd = os.path.abspath(os.path.normpath(local_root))

n = len(wd)

if n==1: n = 0

for root, dirs, files in os.walk(wd):

fix_dirs(dirs)

for f in files:

if ignore_filename(f):

continue

fn = os.path.join(root, f)

mtime = int(os.path.getmtime(fn))

assert fn[n:n+1] in ['/','','\\'],`fn, n`

r[fn[n+1:]] = mtime

#print fn, mtime

""" Return (download_list, upload_list, new_upload_list).

The ``local_files`` and ``remote_files`` are dictionaries of

``{<filenames>: <modification times>}``.

The ``require_download_list`` contains filenames that needs

to be downloaded (i.e. they are newer than local versions).

The ``require_upload_list`` contains filenames that needs to

be uploaded (i.e. they are newer that remote versions).

The ``mtime_tol`` determines maximal difference of modification times

for considering the local and remote files to have equal

modification times. Default is 5 seconds.

"""

download_list, upload_list, new_upload_list, remove_list = [], [], [], []

remove_dir_list = []

for path in remove_paths:

if path in remote_files:

remove_list.append(path)

elif path in local_files:

remove_list.append(path)

else:

flag = False

for fn, mt in remote_files.items():

if fn.startswith(path):

flag = True

remove_list.append(fn)

remove_dir_list.append(path)

for filename, rmtime in remote_files.items():

if filename in remove_list:

continue

if filename in local_files:

lmtime = local_files[filename]

#print filename, rmtime, lmtime, rmtime-lmtime

if abs(rmtime - lmtime) < mtime_tol:

continue

if rmtime < lmtime:

upload_list.append(filename)

else:

download_list.append(filename)

else:

download_list.append(filename)

for filename, lmtime in local_files.items():

if filename in remove_list:

continue

if filename in remote_files:

continue

new_upload_list.append(filename)

usage = "usage: %prog [options] <remote path> <local path>"

parser = OptionParser(usage=usage,

version="%prog "+__version__,

description='''\

%prog is tool to synchronize remote and local directory trees using

FTP protocol (see http://ftpsync2d.googlecode.com/ for updates). Both

directions are supported. Write access to FTP server is required.

To skip processing remote directories that one does not have read or

write access, use --skip option. When a new file was added to FTP

server by other means than using %prog, use --listing to update

.listing files.

Remote path must be given in the following form:

[ftp://][username[:password]@]hostname[<remote directory>]

'''

)

parser.add_option("-u", "--upload", dest="upload_files",

action="store_true", default=False,

help="enable uploading files")

parser.add_option("-d", "--download", dest="download_files",

action="store_true", default=False,

help="enable downloading files")

parser.add_option("-s", "--skip", dest="skip_path",

default = [], action="append",

help="skip listing specified remote (absolute) path")

parser.add_option("-l", "--listing", dest="update_listing",

action="store_true", default=False,

help="update remote .listing files")

parser.add_option("-r", "--remove", dest="remove_paths",

default=[], action="append",

help="remove specified files and directories")

parser.add_option("-e", "--export", dest="export_only",

default=[], action="store_true",

help="only file export, no .listing files; will override all functions modifying or creating .listing files")

(options, args) = parser.parse_args()

if len(args)!=2:

parser.error("incorrect number of arguments")

export_only = False

if options.export_only:

export_only = True

start_time = time.time()

remote_path, local_path = args

local_files = get_local_files(local_path)

session = FtpSession(remote_path)

session.export_only = export_only

session.skip_dirs.extend(options.skip_path)

remote_files = session.get_files(update_listing=options.update_listing)

download_list, upload_list, new_upload_list, remove_list, remove_dir_list = compute_task(local_files, remote_files, options.remove_paths)

removed_files = 0

for path in remove_list:

lpath = os.path.join(local_path, path)

if os.path.exists(lpath):

print 'moving %r to %r' % (lpath, trash_dir)

target = os.path.join(trash_dir, lpath)

target_dir = os.path.dirname(target)

if not os.path.exists(target_dir):

os.makedirs(target_dir)

if os.path.exists(target):

os.remove(target)

shutil.move(lpath, target)

mtime = session.get_mtime(path)

if mtime:

rpath = session.abspath(path)

print 'moving %r to %r' % (rpath, ftp_trash_dir)

target = os.path.join(ftp_trash_dir, rpath[1:] if rpath.startswith('/') else rpath)

target_dir = os.path.dirname(target)

if not os.path.exists(target_dir):

os.makedirs(target_dir)

status = session.download(path, target)

if status:

session.remove(path)

for path in remove_dir_list:

lpath = os.path.join(local_path, path)

if os.path.exists(lpath):

print 'moving %r to %r' % (lpath, trash_dir)

if not os.path.exists(trash_dir):

os.makedirs(trash_dir)

target = os.path.join(trash_dir, lpath)

if os.path.exists(target):

shutil.rmtree(target)

shutil.move(lpath, target)

mtime = session.get_mtime(path)

if mtime==0:

rpath = session.abspath(path)

print 'moving %r to %r' % (rpath, ftp_trash_dir)

target = os.path.join(ftp_trash_dir, rpath[1:] if rpath.startswith('/') else rpath)

target_dir = os.path.dirname(target)

if not os.path.exists(target_dir):

os.makedirs(target_dir)

session.remove_directory(path)

downloaded_files = 0

if options.download_files:

for filename in download_list:

target = os.path.join(local_path, filename)

status = session.download(filename, target)

if status:

downloaded_files += 1

else:

n = len(download_list)

if n:

print "Skipping downloading",n,"files (see --download option):"

for filename in download_list:

print filename

uploaded_files = 0

if options.upload_files:

for filename in upload_list:

source = os.path.join(local_path, filename)

status = session.upload(filename, source, mk_backup=True)

if status:

uploaded_files += 1

for filename in new_upload_list:

source = os.path.join(local_path, filename)

status = session.upload(filename, source, mk_backup=False)

if status:

uploaded_files += 1

else:

n = len(upload_list) + len(new_upload_list)

if n:

print "Skipping uploading",n,"files (see --upload option):"

for filename in upload_list + new_upload_list:

print filename

if downloaded_files:

print '# downloaded files:', downloaded_files

if uploaded_files:

print '# uploaded files:', uploaded_files