def __init__(self, host, user, password):
try:
self._transport = paramiko.Transport((host, 22))
if password is not None:
self._transport.connect(username=user, password=password)
else:
agent = paramiko.Agent()
agent_keys = agent.get_keys()
if len(agent_keys) == 0:
raise DataSourceError(
"Error performing passwordless login")
success = False
for key in agent_keys:
try:
self._transport.connect(username=user, pkey=key)
success = True
break
except paramiko.SSHException:
pass
if not success:
raise DataSourceError(
"Error performing passwordless login")
self._sftp = paramiko.SFTPClient.from_transport(self._transport)
except Exception, e:
raise DataSourceError(e.__str__())
def copy(self, remote_path, local_path, rest=None):
assert rest == None, "SFTP doesn't support partial downloads"
try:
os.system("mkdir -p `dirname %s`" % local_path)
self._sftp.get(remote_path, local_path)
except Exception, e:
raise DataSourceError(e.__str__())
def __init__(self, host, user, password, dirsMax, compress=True):
try:
SFTPSource.__init__(self, host, user, password)
self._dirsMax = int(dirsMax)
self._compress = compress
except Exception, e:
raise DataSourceError(e.__str__())
def modtime(self, path):
try:
result = self._ftp.sendcmd("MDTM %s" % path)
except ftplib.error_perm:
return None
except ftplib.all_errors, e:
raise DataSourceError(e.__str__())
def __init__(self, host, user, password, tz):
try:
self._ftp = ftplib.FTP(host, timeout=60)
self._ftp.login(user, password)
# 10/4/08 - suddenly passive stopped working, switching to active for now
# 10/6/08 - back to normal
#self._ftp.set_pasv(False)
except ftplib.all_errors, e:
raise DataSourceError(e.__str__())
def copy(self, remote_path, local_path, rest=None):
util.info("Copying from %s to %s" % (remote_path, local_path))
try:
os.system("mkdir -p `dirname %s`" % local_path)
with file(local_path, 'ab') as f:
self._ftp.retrbinary("RETR %s" % remote_path,
f.write,
rest=rest)
except ftplib.all_errors, e:
raise DataSourceError(e.__str__())
def list(self, regex):
try:
result = []
self._ftp.retrlines("LIST", result.append)
if (result[0].find("total ", 0) != -1):
del result[0]
return self._parse_ls(result, regex)
except ftplib.error_perm:
return []
except ftplib.all_errors, e:
raise DataSourceError(e.__str__())
def copy(self, remote_path, local_path, rest=None):
if not self._compress:
SFTPSource.copy(self, remote_path, local_path, rest)
else:
tmpFileName = None
try:
tmpFileName = os.environ["TMP_DIR"] + "/" + "".join(
[random.choice(string.letters) for x in xrange(32)])
SFTPSource.copy(self, remote_path, tmpFileName, rest)
os.system("mkdir -p `dirname %s`" % local_path)
f_in = open(tmpFileName, 'rb')
f_out = gzip.open(local_path, 'wb')
f_out.writelines(f_in)
f_out.close()
f_in.close()
os.unlink(tmpFileName)
except Exception, e:
if os.path.isfile(tmpFileName):
os.unlink(tmpFileName)
raise DataSourceError(str(e))
def list(self, regex):
cwd = self._sftp.getcwd()
try:
daydirs = self.list_dir(r"\d{8}")
daydirs = sorted(daydirs, key=lambda x: x[0],
reverse=True)[0:min(self._dirsMax, len(daydirs))]
daydirs = sorted(daydirs, key=lambda x: x[0])
files = []
wd = self._sftp.getcwd()
for daydir in daydirs:
self.cwd(wd + "/" + daydir[0])
for file in self.list_dir(regex):
files.append((daydir[0] + "/" + file[0], file[1]))
#files.extend(self.list_dir(regex))
files = sorted(files, key=lambda x: x[0])
self.cwd(cwd)
return files
except Exception, e:
raise DataSourceError(str(e))
def put(self, local_path, remote_path):
try:
self._sftp.put(local_path, remote_path)
except Exception, e:
raise DataSourceError(e.__str__())
def modtime(self, path):
try:
result = self._sftp.stat(path).st_mtime
except Exception, e:
raise DataSourceError(e.__str__())
def list(self, regex):
try:
result = self._sftp.listdir_attr()
return self._parse_ls(result, regex)
except Exception, e:
raise DataSourceError(e.__str__())
def cwd(self, remote_dir):
try:
self._sftp.chdir(remote_dir)
except Exception, e:
raise DataSourceError(e.__str__())
try:
result = self._sftp.listdir_attr()
return self._parse_ls(result, regex)
except Exception, e:
raise DataSourceError(e.__str__())
def modtime(self, path):
try:
result = self._sftp.stat(path).st_mtime
except Exception, e:
raise DataSourceError(e.__str__())
try:
return datetime.datetime.utcfromtimestamp(result).replace(
tzinfo=pytz.utc)
except:
raise DataSourceError("Could not parse modtime result")
# FTP's SIZE command seems unreliable (was returning larger values)
# def size(self, path):
# try:
# result = self._ftp.sendcmd("SIZE %s" % path)
# except ftplib.all_errors, e:
# raise DataSourceError(e.__str__())
# try:
# return int(result.split()[1])
# except (ValueError, IndexError):
# raise DataSourceError("Could not parse size result")
def copy(self, remote_path, local_path, rest=None):
assert rest == None, "SFTP doesn't support partial downloads"
try:
def list_dir(self, regex):
try:
return SFTPSource.list(self, regex)
except Exception, e:
raise DataSourceError(e.__str__())
def cwd(self, remote_dir):
try:
self._ftp.cwd(remote_dir)
except ftplib.all_errors, e:
raise DataSourceError(e.__str__())