def send_chunks(self):
if self.final_chunks is None:
return ERR
if self.tls_flag:
if self.auth_flag:
ftp_obj = FTP_TLS(host=self.server,
user=self.creds[0],
passwd=self.creds[1])
else:
ftp_obj = FTP_TLS(host=self.server)
else:
if self.auth_flag:
ftp_obj = FTP(host=self.server,
user=self.creds[0],
passwd=self.creds[1])
else:
ftp_obj = FTP(host=self.server)
try:
ftp_obj.login()
sys.stdout.write("\t[+]\tConnected to server %s.\n" % self.server)
except:
sys.stderr.write("\t[-]\tCould not login to the server.\n")
return ERR
for chunk in self.final_chunks:
ftp_obj.mkd(chunk)
time.sleep(SLEEP)
ftp_obj.quit()
sys.stdout.write("\t[+]\tWrote %s(+1) folders.\n" %
(len(self.final_chunks) - 1))
return OKAY
def _get_ftp(self):
try:
username, password = self._get_auth()
if self._tls:
ftp = FTP_TLS(self._uri, timeout=CONNECT_TIMEOUT)
else:
ftp = FTP(self._uri, timeout=CONNECT_TIMEOUT)
ftp.login(username, password)
if self._tls:
ftp.prot_d()
if not catch(ftp.cwd, self._path):
if self._path.startswith("/"):
ftp.cwd("/")
components = self._path.split("/")
for component in components:
if not catch(ftp.cwd, component):
ftp.mkd(component)
ftp.cwd(component)
return ftp
except Exception:
log.exception()
log.warning(
"[FTP] failed to establish server connection, disabled")
self._disabled = True
return None
def upload_to_ftp(root_dir, upload_path):
ftp = FTP_TLS()
ftp.connect(host, port)
ftp.set_debuglevel(2)
ftp.login("account_id", "account_password")
files, directories = get_list_local(upload_path)
directory_name = os.path.dirname(upload_path)
directory_name = os.path.basename(directory_name)
root_dir = root_dir + directory_name + "/"
directories.insert(0, "")
for directory in directories:
path = directory.replace(upload_path, "").replace("\\", "/")
path = root_dir + path
# for fName in ftp.nlst():
# print(fName)
# if folderName in ftp.nlst():
# continue
try:
ftp.mkd(path)
except:
continue
print("Make directory is " + path)
for file in files:
path = file.replace(upload_path, "").replace("\\", "/")
with open(file, "rb") as localfile:
path = root_dir + path
ftp.storbinary('STOR ' + path, localfile)
print("Upload file is " + path)
print("done upload")
def send_chunks(self):
if self.final_chunks is None:
return ERR
if self.tls_flag:
if self.auth_flag:
ftp_obj = FTP_TLS(host=self.server, user=self.creds[0], passwd=self.creds[1])
else:
ftp_obj = FTP_TLS(host=self.server)
else:
if self.auth_flag:
ftp_obj = FTP(host=self.server, user=self.creds[0], passwd=self.creds[1])
else:
ftp_obj = FTP(host=self.server)
try:
ftp_obj.login()
sys.stdout.write("\t[+]\tConnected to server %s.\n" % self.server)
except:
sys.stderr.write("\t[-]\tCould not login to the server.\n")
return ERR
for chunk in self.final_chunks:
ftp_obj.mkd(chunk)
time.sleep(SLEEP)
ftp_obj.quit()
sys.stdout.write("\t[+]\tWrote %s(+1) folders.\n" % (len(self.final_chunks)-1))
return OKAY
def upload(self,path,type,name=None):
try:
fname = name if name else path.split('\\')[-1]
ftps = FTP_TLS(self.host)
ftps.login(self.user,self.password)
ftps.prot_p()
# force encoding to utf-8, this will let us to work with unicode file names
ftps.encoding = "utf-8"
ftps.cwd(self.user_dir)
ftps.cwd(type)
if type != 'files':
# if type of requested file is screenshot or keylog
# upload it to special folder on ftp
today = date.today().strftime("%b-%d-%Y")
if today not in ftps.nlst():
ftps.mkd(today)
ftps.cwd(today)
ftps.storbinary('STOR %s' % fname, open(path, 'rb'))
return 'Upload Started!'
if fname in ftps.nlst():
return 'File Already on FTP'
ftps.storbinary('STOR %s' %fname, open(path,'rb'))
ftps.close()
return 'Upload Started!'
except Exception as e:
if e.args[0] == 0:
return 'Uploaded to FTP!'
return 'Upload Failed!'
def uploadFilesToFTP(ftpURL, ftpUser, ftpPassword, ftpPath, localPath,
filematch, historyBackupPath):
ftps = FTP_TLS(ftpURL)
ftps.set_debuglevel(1)
ftps.set_pasv(False)
ftps.connect(port=21, timeout=80)
ftps.login(ftpUser, ftpPassword)
ftps.prot_p()
ftps.ccc()
try:
ftps.cwd(ftpPath)
except Exception:
ftps.mkd(ftpPath)
for (localPathDir, _, files) in os.walk(localPath):
newdir = ftpPath
try:
ftps.cwd(newdir)
except Exception:
ftps.mkd(newdir)
LOGGER.info("filematch=" + filematch)
for f in fnmatch.filter(files, filematch):
fileLocalPath = os.path.join(localPathDir, f)
file = open(fileLocalPath, 'rb')
ftps.storbinary('STOR ' + f, file, blocksize=8192)
file.close()
LOGGER.info("Fichero transferido #:# " + fileLocalPath)
sleep(1)
now = datetime.datetime.now()
historyBackupPathYear = os.path.join(historyBackupPath,
str(now.year))
try:
os.stat(historyBackupPathYear)
except:
os.mkdir(historyBackupPathYear)
moveFilesUUID(fileLocalPath, historyBackupPathYear)
ftps.close()
def ftp_login():
global ftp
# Use ftps
if config.get_value("ftp", "protocol") == "ftps":
try:
ftp = FTP_TLS(config.get_value("ftp", "host"))
except:
return ("host_error")
try:
ftp.login(config.get_value("ftp", "username"),
config.get_value("ftp", "password"))
ftp.prot_p()
try:
ftp.cwd(config.get_value("ftp", "path"))
except:
ftp.mkd(config.get_value("ftp", "path"))
ftp.cwd(config.get_value("ftp", "path"))
except:
return ("login_error")
# Use ftp
if config.get_value("ftp", "protocol") == "ftp":
try:
ftp = FTP(config.get_value("ftp", "host"))
except:
return ("host_error")
try:
ftp.login(config.get_value("ftp", "username"),
config.get_value("ftp", "password"))
try:
ftp.cwd(config.get_value("ftp", "path"))
except:
ftp.mkd(config.get_value("ftp", "path"))
ftp.cwd(config.get_value("ftp", "path"))
except:
return ("login_error")
CSV3 = '_Inverter3.csv'
CSV4 = '_Inverter4.csv'
CSV = [CSV1, CSV2, CSV3, CSV4]
foldername = str(datetime.date.today())
PATH_TO_SAVE_CVSDIRECTORIES = '/root/regs/'
client1 = mqtt.Client()
try:
ftps = FTP_TLS('host')
ftps.login('user',
'password') # login anonymously before securing control channel
ftps.prot_p() # switch to secure data connection
ftps.cwd('dir') # execute a cd command to access to directory
if (not (foldername in ftps.nlst())):
ftps.mkd(foldername) #create a new folder with current date
ftps.cwd(str(datetime.date.today())) #access to current date folder
for i in range(0, len(CSV)):
filename = str(PATH_TO_SAVE_CVSDIRECTORIES +
str(datetime.date.today()) + '/' +
str(datetime.date.today()) + CSV[i])
file = open(filename, 'rb')
filename2 = str(datetime.date.today()) + CSV[i]
ftps.storbinary('STOR ' + filename2, file)
file.close()
ftps.quit()
print("End try")
except:
print("except")
ftp.mkd(name)
except error_perm as e:
if not e.args[0].startswith('550'):
raise
print("CWD", name)
ftp.cwd(name)
upload_files(ftp, localpath)
print("CWD", "..")
ftp.cwd("..")
ftp_username = os.environ['FTP_USERNAME']
ftp_password = os.environ['FTP_PASSWORD']
with open('version') as file:
version = file.read().rstrip()
working_directory = os.getcwd()
ftp = FTP_TLS("ftp.keycdn.com")
ftp.login(user=ftp_username, passwd=ftp_password)
ftp.cwd('alted')
ftp.mkd(version)
ftp.cwd(version)
upload_files(ftp, '{}/{}'.format(working_directory, 'static'))
ftp.quit()
def genTiles(task):
time_start = datetime.now()
try:
cpu_count = 1
if thread_count is None:
cpu_count = multiprocessing.cpu_count()
else:
cpu_count = thread_count
s_file = None
if source_file.endswith(".zip"):
src = source_file.replace("\\", "/")
fileinfo = QFileInfo(src)
filename = fileinfo.completeBaseName()
s_file = [src, filename]
else:
return [0, time_start]
#cpu_count = min(6, cpu_count)
#if cpu_count % 2 == 0:
# cpu_count = int(cpu_count / 2)
# converted_source = os.path.join(source_folder, "converted").replace("\\","/")
QgsMessageLog.logMessage(
'Started creating dataset from {name}'.format(name=s_file[1]),
CATEGORY, Qgis.Info)
source_folder = os.path.dirname(source_file)
org_vrt = os.path.join(source_folder,
"OrgSource.vrt").replace("\\", "/")
if os.path.isfile(org_vrt):
os.remove(org_vrt)
org_files = []
c_file = '/vsizip/{archive}/{file}'.format(archive=s_file[0],
file=source_file_name)
QgsMessageLog.logMessage(c_file, CATEGORY, Qgis.Info)
org_files.append(c_file)
ds = gdal.BuildVRT(org_vrt,
org_files,
resolution="highest",
resampleAlg="bilinear")
ds.FlushCache()
ds = None
sleep(0.05)
QgsMessageLog.logMessage('Created original vrt', CATEGORY, Qgis.Info)
vrt = os.path.join(source_folder, "Source.vrt").replace("\\", "/")
if os.path.isfile(vrt):
os.remove(vrt)
pds = gdal.Open(org_vrt)
pds = gdal.Warp(vrt, pds, dstSRS="EPSG:3857")
pds = None
QgsMessageLog.logMessage('Created reporojected vrt', CATEGORY,
Qgis.Info)
terrarium_tile = os.path.join(source_folder,
"TerrariumSource.vrt").replace(
"\\", "/")
if os.path.isfile(terrarium_tile):
os.remove(terrarium_tile)
statistics = []
task.setProgress(0)
ress = calculateStat(c_file)
sleep(0.05)
if ress is not None:
statistics.append(ress)
task.setProgress(12)
else:
return [0, time_start]
ress = None
org_files = None
# Find minV and maxV from statistics
minV = None
maxV = None
for stat in statistics:
if stat.minV is not None and stat.maxV is not None:
if minV is None:
minV = stat.minV
elif stat.minV < minV:
minV = stat.minV
if maxV is None:
maxV = stat.maxV
elif stat.maxV > maxV:
maxV = stat.maxV
if minV is None or maxV is None:
QgsMessageLog.logMessage(
'Error: Minimum and maximum height are None', CATEGORY,
Qgis.Info)
return None
else:
QgsMessageLog.logMessage(
'Minimum and maximum height are {minv} and {maxv}'.format(
minv=minV, maxv=maxV), CATEGORY, Qgis.Info)
statistics = None
color_ramp_file = os.path.join(source_folder,
"TerrariumSource.txt").replace(
"\\", "/")
if os.path.isfile(color_ramp_file):
os.remove(color_ramp_file)
color_ramp = []
altitudes = []
# Create color ramp
for meters in range(minV, maxV):
for fraction in range(0, 256):
altitudes.append(meters + (fraction / 256))
exx = concurrent.futures.ThreadPoolExecutor(max_workers=cpu_count)
ff = exx.map(createRamp, repeat(convert_feet_to_meters), altitudes)
for res in ff:
if res is not None:
color_ramp.append(res)
QgsMessageLog.logMessage('Creating color ramp file', CATEGORY,
Qgis.Info)
#sorted_color_ramp = sorted(color_ramp, key=itemgetter(0))
color_ramp.sort(key=lambda x: x.altitude)
f = open(color_ramp_file, "w")
rt = 0
cf = len(color_ramp)
for ramp in color_ramp:
if manual_nodata_value is not None:
if ramp.altitude == manual_nodata_value:
f.write('{altitude}\t0\t0\t0\t0\n'.format(
altitude=manual_nodata_value))
continue
f.write('{altitude}\t{red}\t{green}\t{blue}\n'.format(
altitude=ramp.altitude,
red=ramp.red,
green=ramp.green,
blue=ramp.blue))
rt += 1
task.setProgress(max(0, min(int(((rt * 8) / cf) + 12), 100)))
f.write('nv\t0\t0\t0\t0')
f.close()
sleep(0.05)
QgsMessageLog.logMessage('Created color ramp file', CATEGORY,
Qgis.Info)
QgsMessageLog.logMessage(
'Rendering vrt to terrarium format. This might take a while',
CATEGORY, Qgis.Info)
dst_ds = gdal.Open(vrt)
ds = gdal.DEMProcessing(terrarium_tile,
dst_ds,
'color-relief',
colorFilename=color_ramp_file,
format="VRT",
addAlpha=True)
sleep(0.05)
dst_ds = None
QgsMessageLog.logMessage('Created vrt in terrarium format', CATEGORY,
Qgis.Info)
input_file = terrarium_tile
QgsMessageLog.logMessage('Input file is {inp}'.format(inp=input_file),
CATEGORY, Qgis.Info)
dst_ds = gdal.Open(input_file, gdal.GA_ReadOnly)
QgsMessageLog.logMessage('Calculating bounds', CATEGORY, Qgis.Info)
diff = (20037508.3427892439067364 +
20037508.3427892439067364) / 2**zoom
x_diff = diff
y_diff = diff
ulx, xres, xskew, uly, yskew, yres = dst_ds.GetGeoTransform()
#lrx = ulx + (dst_ds.RasterXSize * xres)
#lry = uly + (dst_ds.RasterYSize * yres)
info = gdal.Info(dst_ds, format='json')
ulx, uly = info['cornerCoordinates']['upperLeft'][0:2]
lrx, lry = info['cornerCoordinates']['lowerRight'][0:2]
wgs_minx, wgs_minz = info['wgs84Extent']['coordinates'][0][1][0:2]
wgs_maxx, wgs_maxz = info['wgs84Extent']['coordinates'][0][3][0:2]
QgsMessageLog.logMessage(
'The dataset bounds are (in WGS84 [EPSG:4326]), minX: {minX}, minZ: {minZ}, maxX: {maxX}, maxZ: {maxZ}'
.format(minX=str(wgs_minx),
minZ=str(wgs_minz),
maxX=str(wgs_maxx),
maxZ=str(wgs_maxz)), CATEGORY, Qgis.Info)
QgsMessageLog.logMessage(
'Use this info for following step F in Part two: Generating/using your dataset',
CATEGORY, Qgis.Info)
base_x = 0
base_y = 0
start_tile_x = base_x
start_tile_y = base_y
min_x = -20037508.3427892439067364
max_y = 20037508.3427892439067364
x_tiles = 0
y_tiles = 0
# Find min lot
lon = min_x
while lon <= 20037508.3427892439067364:
if lon >= ulx:
break
start_tile_x += 1
min_x = lon
lon += x_diff
# Find max lat
lat = max_y
while lat >= -20037508.3427892550826073:
if lat <= uly:
break
start_tile_y += 1
max_y = lat
lat -= y_diff
# Find how many lon tiles to make
lon = min_x
while lon < lrx:
x_tiles += 1
lon += x_diff
# Find how many lat tiles to make
lat = max_y
while lat >= lry:
y_tiles += 1
lat -= y_diff
if start_tile_x > 0:
start_tile_x -= 1
if start_tile_y > 0:
start_tile_y -= 1
QgsMessageLog.logMessage(
'Start tile: {tx} ({mlon}), {ty} ({mxlat}), Tiles to generate: {xt} (Width: {xtx} Height: {xty})'
.format(tx=start_tile_x,
mlon=min_x,
ty=start_tile_y,
mxlat=max_y,
xt=((x_tiles + 1) * (y_tiles + 1)),
xtx=x_tiles,
xty=y_tiles), CATEGORY, Qgis.Info)
QgsMessageLog.logMessage('Creating output folders', CATEGORY,
Qgis.Info)
mx = start_tile_x + x_tiles
zoom_folder = ''
temp_folder = os.path.join(source_folder, 'TEMP')
if not ftp_upload:
zoom_folder = os.path.join(output_directory,
str(zoom)).replace("\\", "/")
else:
if os.path.isdir(temp_folder):
shutil.rmtree(temp_folder)
os.mkdir(temp_folder)
zoom_folder = os.path.join(temp_folder,
str(zoom)).replace("\\", "/")
if not os.path.isdir(zoom_folder):
os.mkdir(zoom_folder)
for x in range(start_tile_x, mx):
folderx = os.path.join(zoom_folder, str(x)).replace("\\", "/")
if not os.path.isdir(folderx):
os.mkdir(folderx)
zip_file = os.path.join(zoom_folder,
'RenderedDataset.zip').replace("\\", "/")
rd_file = None
if ftp_upload:
if ftp_one_file:
rd_file = zipfile.ZipFile(zip_file, 'w', zipfile.ZIP_DEFLATED)
else:
QgsMessageLog.logMessage(
'Creating output folders on target ftp server', CATEGORY,
Qgis.Info)
ftp = None
if ftp_s:
ftp = FTP_TLS()
else:
ftp = FTP()
ftp.connect(ftp_upload_url, ftp_upload_port)
if ftp_user is None or ftp_password is None:
ftp.login()
else:
ftp.login(user=ftp_user, passwd=ftp_password)
if ftp_upload_folder is not None:
ftp.cwd(ftp_upload_folder)
if not isFTPDir(ftp, str(zoom)):
ftp.mkd(str(zoom))
ftp.cwd(str(zoom))
for x in range(start_tile_x, mx):
ftp.mkd(str(x))
ftp.quit()
sleep(0.01)
QgsMessageLog.logMessage('Created {ct} folders'.format(ct=x_tiles),
CATEGORY, Qgis.Info)
tiled = []
# Tile dataset
sub_min_x = min_x
sub_max_x = min_x + x_diff
for x in range(start_tile_x, start_tile_x + x_tiles):
sub_min_y = max_y - y_diff
sub_max_y = max_y
for y in range(start_tile_y, start_tile_y + y_tiles):
tiled.append(
Tile(dst_ds,
str(x),
str(y),
sub_min_x,
sub_max_y,
sub_max_x,
sub_min_y,
ulx,
xres,
uly,
yres,
querysize=1024))
sub_min_y -= y_diff
sub_max_y -= y_diff
sub_min_x += x_diff
sub_max_x += x_diff
job = Job(zoom_folder, str(zoom), input_file, getBands(dst_ds),
resampling_algorithm, ftp_upload, ftp_one_file, ftp_s,
ftp_upload_url, ftp_upload_port, ftp_upload_folder, ftp_user,
ftp_password)
dst_ds = None
# Tile the dataset
realtiles = 0
if cpu_count == 1:
QgsMessageLog.logMessage(
'Started tilling vrt in singlethread mode', CATEGORY,
Qgis.Info)
rt = 0
cf = len(tiled)
for tile in tiled:
res = tileVrt(job, tile)
if res is not None:
if job.ftpUpload and job.ftpOnefile:
addToZIP(job, res, rd_file)
realtiles += 1
rt += 1
if job.ftpUpload and job.ftpOnefile:
task.setProgress(
max(0, min(int(((rt * 40) / cf) + 20), 100)))
else:
task.setProgress(
max(0, min(int(((rt * 80) / cf) + 20), 100)))
if getattr(localThread, 'ds', None):
del localThread.ds
else:
QgsMessageLog.logMessage(
'Started tilling vrt in multithread mode with {count} threads'.
format(count=cpu_count), CATEGORY, Qgis.Info)
gdal_cache_max = gdal.GetCacheMax()
gdal_cache_max_per_process = max(
1024 * 1024, math.floor(gdal_cache_max / cpu_count))
setCacheMax(gdal_cache_max_per_process)
tpe = concurrent.futures.ThreadPoolExecutor(max_workers=cpu_count)
tm = tpe.map(tileVrt, repeat(job), tiled)
rt = 0
cf = len(tiled)
realtiles = 0
for res in tm:
if res is not None:
if job.ftpUpload and job.ftpOnefile:
addToZIP(job, res, rd_file)
realtiles += 1
rt += 1
if job.ftpUpload and job.ftpOnefile:
task.setProgress(
max(0, min(int(((rt * 40) / cf) + 20), 100)))
else:
task.setProgress(
max(0, min(int(((rt * 80) / cf) + 20), 100)))
setCacheMax(gdal_cache_max)
if job.ftpUpload and job.ftpOnefile:
rd_file.close()
totalSize = os.path.getsize(zip_file)
QgsMessageLog.logMessage(
'Starting uploading renderned archive ({size} GB) out of {count} tiles'
.format(count=realtiles,
size=str(round(totalSize * 9.3132257461548E-10,
4))), CATEGORY, Qgis.Info)
ftp = None
if job.ftpS:
ftp = FTP_TLS()
else:
ftp = FTP()
ftp.connect(job.ftpUrl, job.ftpPort)
if job.ftpUser is None or job.ftpPassword is None:
ftp.login()
else:
ftp.login(user=job.ftpUser, passwd=job.ftpPassword)
if job.ftpFolder is not None:
ftp.cwd(job.ftpFolder)
ftapr = FTPArchiveProgress(int(totalSize), task)
with open(zip_file, 'rb') as rdf_file:
ftp.storbinary('STOR RenderedDataset.zip',
rdf_file,
blocksize=1024,
callback=ftapr.updateProgress)
ftp.quit()
QgsMessageLog.logMessage(
'Uploaded rendered dataset archive RenderedDataset.zip with {count} tiles. You can now unzip it.'
.format(count=realtiles), CATEGORY, Qgis.Info)
elif job.ftpUpload:
QgsMessageLog.logMessage(
'Tiled and uploaded dataset with {count} tiles to ftp server'.
format(count=realtiles), CATEGORY, Qgis.Info)
else:
QgsMessageLog.logMessage(
'Tiled dataset with {count} tiles'.format(count=realtiles),
CATEGORY, Qgis.Info)
# Clean up
if cleanup:
os.remove(org_vrt)
os.remove(vrt)
os.remove(color_ramp_file)
os.remove(terrarium_tile)
if job.ftpUpload:
shutil.rmtree(temp_folder)
QgsMessageLog.logMessage('Cleaned up temp files', CATEGORY,
Qgis.Info)
return [realtiles, time_start]
except Exception as e:
QgsMessageLog.logMessage('Error: ' + str(e), CATEGORY, Qgis.Info)
return None
class FTPClient(object):
"""Class FTPClient
"""
_mh = None
_client = None
_secured = None
_host = None
_port = None
_user = None
_passw = None
_path = None
_verbose = None
_is_connected = None
def __init__(self, secured=False, verbose=False):
"""Class constructor
Called when the object is initialized
Args:
secured (bool): secured FTP
verbose (bool): verbose mode
"""
self._mh = MasterHead.get_head()
self._secured = secured
if (not self._secured):
self._client = FTP()
else:
if (not(version_info[0] == 2 and version_info[1] == 6)):
self._client = FTP_TLS()
else:
raise NotImplementedError(
'Secured mode is not supported for Python 2.6')
self._verbose = verbose
if (self._verbose):
self._client.set_debuglevel(2)
@property
def client(self):
""" FTP client property getter """
return self._client
@property
def secured(self):
""" secured protocol mode property getter """
return self._secured
@property
def host(self):
""" server host property getter """
return self._host
@property
def port(self):
""" server port property getter """
return self._port
@property
def user(self):
""" username property getter """
return self._user
@property
def passw(self):
""" user password property getter """
return self._passw
@property
def path(self):
""" remote path property getter """
return self._path
@property
def verbose(self):
""" verbose mode property getter """
return self._verbose
@property
def is_connected(self):
""" is_connected property getter """
return self._is_connected
def connect(self, host, port=21, user=None, passw=None, path='/', timeout=10):
"""Method connects to server
Args:
host (str): server host
port (int): server port, default protocol port
user (str): username
passw (str): password
path (str): server path
timeout (int): timeout
Returns:
bool: result
Raises:
event: ftp_before_connect
event: ftp_after_connect
"""
try:
message = '{0}/{1}@{2}:{3}{4} timeout:{5}'.format(
user, passw, host, port, path, timeout)
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_connecting', message), self._mh.fromhere())
ev = event.Event(
'ftp_before_connect', host, port, user, passw, path, timeout)
if (self._mh.fire_event(ev) > 0):
host = ev.argv(0)
port = ev.argv(1)
user = ev.argv(2)
passw = ev.argv(3)
path = ev.argv(4)
timeout = ev.argv(5)
self._host = host
self._port = port
self._user = user
self._passw = passw
if (ev.will_run_default()):
self._client.connect(self._host, self._port, timeout=timeout)
if (self._user != None):
self._client.login(self._user, self._passw)
if (self._secured):
self._client.prot_p()
self._is_connected = True
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_connected'), self._mh.fromhere())
if (path != None):
self.change_dir(path)
ev = event.Event('ftp_after_connect')
self._mh.fire_event(ev)
return True
except all_errors as ex:
self._mh.demsg(
'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere())
return False
def disconnect(self):
"""Method disconnects from server
Args:
none
Returns:
bool: result
"""
try:
if (not self._is_connected):
self._mh.demsg('htk_on_warning', self._mh._trn.msg(
'htk_ftp_not_connected'), self._mh.fromhere())
return False
else:
self._client.quit()
self._is_connected = False
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_disconnected'), self._mh.fromhere())
return True
except all_errors as ex:
self._mh.demsg(
'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere())
return False
def list_dir(self):
"""Method lists remote working directory
Args:
none
Returns:
list: names
"""
try:
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_list_dir', self._path), self._mh.fromhere())
if (not self._is_connected):
self._mh.demsg('htk_on_warning', self._mh._trn.msg(
'htk_ftp_not_connected'), self._mh.fromhere())
return False
names = self._client.nlst()
if ('.' in names):
names.remove('.')
if ('..' in names):
names.remove('..')
return names
except all_errors as ex:
self._mh.demsg(
'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere())
return None
def change_dir(self, path):
"""Method changes remote working directory
Args:
path (str): new remote path
Returns:
bool: result
Raises:
event: ftp_before_change_dir
"""
try:
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_change_dir', path), self._mh.fromhere())
if (not self._is_connected):
self._mh.demsg('htk_on_warning', self._mh._trn.msg(
'htk_ftp_not_connected'), self._mh.fromhere())
return False
ev = event.Event('ftp_before_change_dir', path)
if (self._mh.fire_event(ev) > 0):
path = ev.argv(0)
if (ev.will_run_default()):
self._client.cwd(path)
self._path = self._client.pwd()
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_cur_dir', self._path), self._mh.fromhere())
return True
except all_errors as ex:
self._mh.demsg(
'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere())
return False
def download_file(self, remote_path, local_path=None):
"""Method downloads file from server
Args:
remote_path (str): remote path
local_path (str): local path, default ./filename
Returns:
bool: result
Raises:
event: ftp_before_download_file
event: ftp_after_download_file
"""
try:
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_downloading_file', remote_path), self._mh.fromhere())
if (not self._is_connected):
self._mh.demsg('htk_on_warning', self._mh._trn.msg(
'htk_ftp_not_connected'), self._mh.fromhere())
return False
ev = event.Event(
'ftp_before_download_file', remote_path, local_path)
if (self._mh.fire_event(ev) > 0):
remote_path = ev.argv(0)
local_path = ev.argv(1)
if (local_path != None and not path.exists(local_path)):
self._mh.demsg('htk_on_error', self._mh._trn.msg(
'htk_ftp_unknown_dir', local_path), self._mh.fromhere())
return False
filename = remote_path.split('/')[-1]
lpath = filename if (local_path == None) else path.join(
local_path, filename)
if (ev.will_run_default()):
with open(lpath, 'wb') as f:
self._client.retrbinary('RETR ' + remote_path, f.write)
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_file_downloaded'), self._mh.fromhere())
ev = event.Event('ftp_after_download_file')
self._mh.fire_event(ev)
return True
except all_errors as ex:
self._mh.demsg(
'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere())
if (path.exists(lpath)):
remove(lpath)
return False
def upload_file(self, local_path, remote_path=None):
"""Method uploads file to server
Args:
local_path (str): local path
remote_path (str): remote path, default ./filename
Returns:
bool: result
Raises:
event: ftp_before_upload_file
event: ftp_after_upload_file
"""
try:
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_uploading_file', local_path), self._mh.fromhere())
if (not self._is_connected):
self._mh.demsg('htk_on_warning', self._mh._trn.msg(
'htk_ftp_not_connected'), self._mh.fromhere())
return False
ev = event.Event('ftp_before_upload_file', local_path, remote_path)
if (self._mh.fire_event(ev) > 0):
local_path = ev.argv(0)
remote_path = ev.argv(1)
if (not(path.exists(local_path) or path.exists(path.relpath(local_path)))):
self._mh.demsg('htk_on_error', self._mh._trn.msg(
'htk_ftp_unknown_file', local_path), self._mh.fromhere())
return False
filename = local_path.split('/')[-1]
rpath = filename if (remote_path == None) else path.join(
remote_path, filename)
if (ev.will_run_default()):
with open(local_path, 'rb') as f:
self._client.storbinary('STOR ' + rpath, f)
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_file_uploaded'), self._mh.fromhere())
ev = event.Event('ftp_after_upload_file')
self._mh.fire_event(ev)
return True
except all_errors as ex:
self._mh.demsg(
'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere())
return False
def delete_file(self, path):
"""Method deletes file from server
Args:
path (str): remote path
Returns:
bool: result
Raises:
event: ftp_before_delete_file
"""
try:
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_deleting_file', path), self._mh.fromhere())
if (not self._is_connected):
self._mh.demsg('htk_on_warning', self._mh._trn.msg(
'htk_ftp_not_connected'), self._mh.fromhere())
return False
ev = event.Event('ftp_before_delete_file', path)
if (self._mh.fire_event(ev) > 0):
path = ev.argv(0)
if (ev.will_run_default()):
self._client.delete(path)
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_file_deleted'), self._mh.fromhere())
return True
except all_errors as ex:
self._mh.demsg(
'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere())
return False
def make_dir(self, path):
"""Method makes directory on server
Args:
path (str): remote path
Returns:
bool: result
Raises:
event: ftp_before_make_dir
"""
try:
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_making_dir', path), self._mh.fromhere())
if (not self._is_connected):
self._mh.demsg('htk_on_warning', self._mh._trn.msg(
'htk_ftp_not_connected'), self._mh.fromhere())
return False
ev = event.Event('ftp_before_make_dir', path)
if (self._mh.fire_event(ev) > 0):
path = ev.argv(0)
if (ev.will_run_default()):
self._client.mkd(path)
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_dir_made'), self._mh.fromhere())
return True
except all_errors as ex:
self._mh.demsg(
'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere())
return False
def remove_dir(self, path):
"""Method removes directory from server
Args:
path (str): remote path
Returns:
bool: result
Raises:
event: ftp_before_remove_dir
"""
try:
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_removing_dir', path), self._mh.fromhere())
if (not self._is_connected):
self._mh.demsg('htk_on_warning', self._mh._trn.msg(
'htk_ftp_not_connected'), self._mh.fromhere())
return False
ev = event.Event('ftp_before_remove_dir', path)
if (self._mh.fire_event(ev) > 0):
path = ev.argv(0)
if (ev.will_run_default()):
self._client.rmd(path)
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_dir_removed'), self._mh.fromhere())
return True
except all_errors as ex:
self._mh.demsg(
'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere())
return False
class FTPSession(object):
""" Attempt to create some robustness and performance to FTPing """
def __init__(self, server, username, password, tmpdir='/tmp', timeout=60):
"""Build a FTP session """
self.conn = None
self.server = server
self.username = username
self.password = password
self.tmpdir = tmpdir
self.timeout = timeout
def _connect(self):
"""Connect to FTP server """
if self.conn is not None:
return
logging.debug('Creating new connection to server %s', self.server)
not_connected = True
attempt = 1
while not_connected and attempt < 6:
try:
self.conn = FTP_TLS(self.server, timeout=self.timeout)
self.conn.login(self.username, self.password)
self.conn.prot_p()
not_connected = False
except Exception as exp:
logging.debug(exp)
time.sleep(5)
self.close()
attempt += 1
if not_connected is True:
raise Exception("Failed to make FTP connection after 5 tries!")
def _reconnect(self):
""" First attempt to shut down connection and then reconnect """
logging.debug('_reconnect() was called...')
try:
self.conn.quit()
self.conn.close()
except:
pass
finally:
self.conn = None
self._connect()
def _put(self, path, localfn, remotefn):
""" """
self.chdir(path)
sz = os.path.getsize(localfn)
if sz > 14000000000:
# Step 1 Split this big file into 14GB chunks, each file will have
# suffix .aa then .ab then .ac etc
basefn = os.path.basename(localfn)
cmd = "split --bytes=14000M %s %s/%s." % (localfn, self.tmpdir,
basefn)
subprocess.call(cmd, shell=True, stderr=subprocess.PIPE)
files = glob.glob("%s/%s.??" % (self.tmpdir, basefn))
for filename in files:
suffix = filename.split(".")[-1]
self.conn.storbinary('STOR %s.%s' % (remotefn, suffix),
open(filename))
os.unlink(filename)
else:
logging.debug("_put '%s' to '%s'", localfn, remotefn)
self.conn.storbinary('STOR %s' % (remotefn, ), open(localfn))
return True
def close(self):
""" Good bye """
try:
self.conn.quit()
self.conn.close()
except:
pass
finally:
self.conn = None
def chdir(self, path):
if self.pwd() == path.rstrip("/"):
return
self.conn.cwd("/")
for dirname in path.split("/"):
if dirname == '':
continue
bah = []
self.conn.retrlines('NLST', bah.append)
if dirname not in bah:
logging.debug("Creating directory '%s'", dirname)
self.conn.mkd(dirname)
logging.debug("Changing to directory '%s'", dirname)
self.conn.cwd(dirname)
def pwd(self):
""" Low friction function to get connectivity """
self._connect()
pwd = exponential_backoff(self.conn.pwd)
if pwd is None:
self._reconnect()
pwd = exponential_backoff(self.conn.pwd)
logging.debug("pwd() is currently '%s'", pwd)
return pwd
def put_file(self, path, localfn, remotefn):
""" Put the File """
res = exponential_backoff(self._put, path, localfn, remotefn)
if not res:
self._reconnect()
res = exponential_backoff(self._put, path, localfn, remotefn)
if not res:
logging.error("Double Failure to upload filename: '%s'",
localfn)
return False
return True
def put_files(self, path, localfns, remotefns):
""" Put the File """
res = []
for localfn, remotefn in zip(localfns, remotefns):
res.append(self.put_file(path, localfn, remotefn))
return res
#picamera.resolution = (1024, 768) # こちらは554KBで済む
picamera.resolution = (800, 600) # こちらは●●●KBで済む
#picamera.start_preview()
# Camera warm-up time、Whiteバランスをとるための猶予時間。これがないと色が青白くて使い物にならない
time.sleep(2)
file_name = file_name+'.jpg'
picamera.capture(dir_path+'/'+file_name)
print "Now start Upload."
_ftps = FTP_TLS(userID)
_ftps.set_debuglevel(1) # デバッグログをリアルタイムで確認
_ftps.login(archive_server,pw)
_file = open(dir_path+'/'+file_name, 'rb') #target file. 次のステップでアップロード成功したら削除した方がよいのではないか？
#SD Memoryがパンクする恐れがあるので、次のステップでアップロードが成功したらファイルは削除するように、改良しましょう。
_ftps.cwd(put_directory)
try:
_ftps.cwd(dir_name) #target_dir 撮影データは日付付きのフォルダ内に納められる。
except Exception as e:
#ディレクトリがなければ '550 Failed to change directory.' が帰ってくる。
#もし、"550"があれば．．．まだその日のフォルダはないので、新規作成
## if str(e).count('550') : 文字列が含まれるかどうかの判定はこの関数を使ってもよいが、次の書き方が分かりやすい。
if '550' in str(e):
_ftps.mkd(dir_name)
_ftps.cwd(dir_name)
_ftps.storbinary('STOR ' + file_name, _file)
_file.close()
_ftps.quit()
class ServerWatcher(Watcher):
downloadProgress = Signal((int, int,))
uploadProgress = Signal((int, int,))
# Si added:
textStatus = Signal((str,))
fileEvent = Signal((str,))
fileEventCompleted = Signal()
loginCompleted = Signal((bool, str,))
badFilenameFound = Signal((str,))
LOCATION = 'server'
TEST_FILE = 'iqbox.test'
def __init__(self, host, ssl, parent=None):
"""
Initializes parent class and attributes. Decides
whether to use `FTP_TLS` or `FTP` based on the `ssl` param.
:param host: Location of the FTP server
:param ssl: Tells whether the FTP needs to support TLS or not
:param parent: Reference to a `QObject` instance a parent
"""
super(ServerWatcher, self).__init__(parent)
self.interval = 5000
self.localdir = ''
self.deleteQueue = []
self.downloadQueue = []
self.uploadQueue = []
self.warnedNames = []
self.ftp = None
self.useSSL = ssl
self.host = host
self.preemptiveCheck = False
self.preemptiveActions = []
self.testFile = 'iqbox.test'
@property
def currentdir(self):
"""Returns the current working directory at the server"""
return self.ftp.pwd()
def setLocalDir(self, localdir):
"""
Sets the local directory used to stored all
downloaded files. Creates the directory if needed.
:param localdir: Absolute path to local directory
"""
self.localdir = localdir
if not os.path.exists(self.localdir):
os.makedirs(self.localdir)
@pause_timer
@Slot()
def checkout(self):
"""
Recursively checks out all files on the server.
Returns a dictionary of files on the server with their last modified date.
:param download: Indicates whether or not the files should be downloaded
"""
# Check `self.deleteQueue`, `self.uploadQueue` and `self.downloadQueue` queues.
# These tasks are done in queues to make sure all FTP commands
# are done sequentially, in the same thread.
self.deleteAll()
self.uploadAll()
self.downloadAll()
# Handy list to keep track of the checkout process.
# This list contain absolute paths only.
checked_dirs = list()
# Sets '/' as initial directory and initializes `downloading_dir`
self.ftp.cwd('/')
downloading_dir = self.currentdir
check_date = dt.utcnow()
sidirlist = list()
root_cached = False
fileC = 0
while True:
# Gets the list of sub directories and files inside the
# current directory `downloading_dir`.
self.textStatus.emit('Remote scan- Downloading folder list of '+downloading_dir+'...')
if root_cached and downloading_dir == '/':
dir_subdirs = saved_root_dirs
dirfiles = saved_root_files
else:
dir_subdirs = self.getDirs(downloading_dir)
if downloading_dir == '/':
saved_root_dirs = dir_subdirs
# sidirlist.extend(dir_subdirs)
self.textStatus.emit('Remote scan- Downloading files list of '+downloading_dir+'...')
dirfiles = self.getFiles(downloading_dir)
if downloading_dir == '/':
saved_root_files = dirfiles
root_cached = True
# Leading '/' in `downloading_dir` breaks the `os.path.join` call
localdir = os.path.join(self.localdir, downloading_dir[1:])
if not os.path.exists(localdir):
# Creates the directory if it doesn't already exists.
os.makedirs(localdir)
for file_ in dirfiles:
# `serverpath` is the absolute path of the file on the server,
# download it only if it hasn't been already downloaded
serverpath = os.path.join(downloading_dir, file_)
serverpath = QDir.fromNativeSeparators(serverpath)
server_file = File.fromPath(serverpath)
self.textStatus.emit('Scanning remote file... '+serverpath+'...')
# How do we know if we should check this server file?
# We see if the date last checked is the check start time.
if server_file.last_checked_server != check_date:
# Do this process only once per file
# Added by Simon
# Give feedback on scanning of files.
fileC += 1
if fileC % 1 == 2:
self.textStatus.emit('Scanning remote files for changes, '+str(fileC)+' files scanned.')
# STEP: IS THIS THE FIRST TIME WE SAW THE FILE, OR WAS IT ALREADY IN OUR DB?
just_added = not server_file.inserver
# STEP: IF ITS A NEW FILE, ENSURE WE DONT WANT TO SKIP IT
# Example: If it's a temporary file, or a Unix file with a name we don't support.
if just_added:
filename = os.path.basename(serverpath)
if platform.system() == 'Windows':
badName = False
for chr in ['\\', '/', ':', '?', '"', '<', '>', '|']:
if chr in filename:
badName = True
break
if badName:
if filename not in self.warnedNames:
self.warnedNames.append(filename)
self.badFilenameFound.emit(filename)
continue
# STEP: ASSUMING THE FILE DID EXIST IN OUR DB, LETS SAVE THE LAST MODIFICATION DATE
lastmdate = server_file.servermdate
# STEP: SAVE THE MOD DATE TO A VARIABLE
# Now we get the last mod time.
# We expect this to work fine since this file
# was found on the server
servermdate = self.lastModified(serverpath)
# STEP: SET BOOL SHOWING THAT IT WAS ON THE SERVER, SINCE WE KNOW IT IS.
server_file.inserver = True
# STEP: SET THE TIME THE FILE WAS LAST CHECKED TO THE SCAN START TIME
server_file.last_checked_server = check_date
# STEP: SET THE MOD DATE IN THE DATABASE TO THE ONE WE JUST GOT
server_file.servermdate = servermdate
# STEP: SAVE THIS CHANGE TO THE DATABASE
server_file.session.commit()
delta = 0
if server_file.inlocal:
delta = server_file.timeDiff()
# Emit the signals after the attributes has been set and committed
if just_added is True:
self.fileAdded.emit(ServerWatcher.LOCATION, serverpath)
elif server_file.servermdate > lastmdate or delta < -Watcher.TOLERANCE:
self.fileChanged.emit(ServerWatcher.LOCATION, serverpath, False)
#END FOR
self.textStatus.emit('Remote scan- Finding next folder...')
dir_ready = True
for dir_ in dir_subdirs:
# `dirpath` is the absolute path of the subdirectory on the server,
dirpath = QDir.fromNativeSeparators(os.path.join(downloading_dir, dir_))
# `downloading_dir` is ready only when all its subdirectory are on the
# `checked_dirs` list.
if dirpath not in checked_dirs:
# Found one subdirectory that is not on `checked_dirs`,
# will process it in the next iteration.
downloading_dir = dirpath
dir_ready = False
break
if dir_ready is True:
# All subdirectories of `downloading_dir` are already in `checked_dirs`
if downloading_dir == '/':
# All directories ready and at '/', means checkout is complete
# So, exit the main While loop!!
break
else:
# Not at '/'. Current directory is ready so is appended to `checked_dirs`
# Back one directory to find directories that are not in `checked_dirs`
checked_dirs.append(downloading_dir)
downloading_dir = os.path.dirname(downloading_dir)
self.textStatus.emit('Remote scan- Found Folder...')
##### END OF WHILE ################
###################################################################
# Deleted files are the ones whose `last_checked_server` attribute
# didn't get updated in the recursive run.
session = Session()
deleted = session.query(File).filter(File.last_checked_server < check_date).filter(File.inserver == True)
for file_ in deleted:
self.fileDeleted.emit(ServerWatcher.LOCATION, file_.path)
# Wraps up the checkout process, commits to the database.
session.commit()
@Slot()
def onLogin(self, username, passwd):
ok = True
msg = ''
error_msg = 'Login failed.'
try:
if not self.ftp:
self.ftp = FTP_TLS(self.host) if self.useSSL is True else FTP(self.host)
loginResponse = self.ftp.login(username, passwd)
except socket.gaierror:
self.ftp = None
ok = False
msg = 'Server address could not be found.'
except (error_perm, error_reply):
info = traceback.format_exception(*sys.exc_info())
for i in info: sys.stderr.write(i)
ok = False
msg = error_msg
else:
if '230' in loginResponse:
ok = True
else:
ok = False
msg = error_msg
if ok:
# Logged in. Now let's do compability tests.
if not self.testPermissions():
# User doesn't have write permissions, don't bother doing next test.
ok = False
msg = 'It seems like you do not have write access to this server.'
else:
# Permissions test passed, now let's test MFMT for timestamp modification.
if not self.testMFMT():
ok = False
msg = 'This server does not support timestamp modification\n \
need by this application.'
self.loginCompleted.emit(ok, msg)
def getFiles(self, path):
"""
This method simply wraps the `nlst` method with an exception handler,
and returns an empty list in case an exception is caught.
:param path: Relative or absolute path on the server
"""
try:
nlst = self.ftp.nlst(path)
dirs = self.getDirs(path)
# Files are items in nlst that are not in dirs
files = [item for item in nlst if os.path.basename(item) not in dirs]
return files
except:
print 'Exception in ServerWatcher.getDirs'
info = traceback.format_exception(*sys.exc_info())
for i in info: sys.stderr.write(i)
return []
def getDirs(self, path):
"""
Retrieves a list of the directories inside `path`,
uses `retrlines` and the LIST command to retrieve the items.
:param path: Relative or absolute path on the server
"""
dirs = list()
def handleLine(line):
"""
Recieves a line from the LIST command.
This function is meant to be used as callback for the `retrlines` method.
:params line: Line from the LIST command
"""
if line.startswith('d'):
# Only lines starting with 'd' are directories
# Parse the directory out of the line; lines look like:
# 'drwxrwxrwx 1 user group 0 Jun 15 2012 dirname'
dirname = line[55:].strip()
if dirname != '.' and dirname != '..':
# Ignoring '.' and '..' entries
dirs.append(dirname)
try:
self.ftp.retrlines('LIST %s' % path, handleLine)
return dirs
except:
print 'Exception in ServerWatcher.getDirs'
info = traceback.format_exception(*sys.exc_info())
for i in info: sys.stderr.write(i)
return []
@upload_test
def testPermissions(self):
# For interface purposes. upload_test takes care of everything.
return True
@upload_test
def testMFMT(self):
# Absurd date to test whether the change really happened.
time = dt.utcfromtimestamp(100000000)
try:
self.setLastModified(self.testFile, time)
otherTime = self.lastModified(self.testFile)
diff = (time - otherTime).total_seconds()
if abs(diff) < 2:
# Let's give it a 2 seconds tolerance.
mdtm = True
else:
mdtm = False
except (ValueError, error_reply, error_perm):
info = traceback.format_exception(*sys.exc_info())
for i in info: sys.stderr.write(i)
mdtm = False
return mdtm
@Slot(str)
def onDelete(self, filename):
self.deleteQueue.append(filename)
def deleteNext(self):
if len(self.deleteQueue) > 0:
next = self.deleteQueue.pop(0)
self.deleteFile(next)
def deleteAll(self):
for filename in self.deleteQueue:
self.deleteFile(filename)
self.deleteQueue = []
@Slot(str)
def deleteFile(self, filename):
"""
Deletes the file `filename` to the server
:param filename: Absolute or relative path to the file
"""
try:
print 'Deleting %s' % filename
self.ftp.delete(filename)
return True
except (error_reply, error_perm):
print 'Error deleting %s' % filename
return False
self.fileEventCompleted.emit()
@Slot(str)
def onDownload(self, filename):
self.downloadQueue.append(filename)
def downloadNext(self):
if len(self.downloadQueue) > 0:
next = self.downloadQueue.pop(0)
self.downloadFile(next)
def downloadAll(self):
for filename in self.downloadQueue:
self.downloadFile(filename)
self.downloadQueue = []
@Slot(str, str)
def downloadFile(self, filename, localpath=None):
"""
Performs a binary download to the file `filename` located on the server.
`filename` parameter can be either absolute or relative, though it can
fail for relative paths if the current directory is not appropiate.
:param filename: Relative or absolute path to the file
:param localpath: Absolute local path where the file will be saved
"""
def handleChunk(chunk):
"""
Receives chuncks of data downloaded from the server.
This function is meant to be used as callback for the `retrbinary` method.
:params chunk: Chunk of downloaded bytes to be written into the file
"""
# Simply writes the received data into the file `self.downloading`
self.downloading.write(chunk)
self.download_progress += len(chunk)
self.downloadProgress.emit(self.download_size, self.download_progress)
if localpath is None:
localpath = self.localFromServer(filename)
localdir = os.path.dirname(localpath)
if not os.path.exists(localdir):
# Creates the directory if it doesn't already exists.
os.makedirs(localdir)
print 'Downloading: %s to %s' % (filename, localpath)
try:
with open(localpath, 'wb') as f:
# Opens the file at `localname` which will hold the downloaded file.
# Object attributes regarding download status are updated accordingly.
self.fileEvent.emit(filename)
self.downloading = f
self.download_progress = 0
self.download_size = int(self.ftp.sendcmd('SIZE %s' % filename).split(' ')[-1])
self.ftp.retrbinary('RETR %s' % filename, handleChunk)
print 'Download finished'
# Let's set the same modified time to that on the server.
with File.fromPath(filename) as downloadedfile:
mdate = LocalWatcher.lastModified(localpath)
downloadedfile.localmdate = mdate
downloadedfile.servermdate = mdate
self.setLastModified(filename, mdate)
downloaded = True
except (IOError, OSError):
downloaded = False
self.ioError.emit(localpath)
except (error_reply, error_perm) as ftperr:
print 'Error downloading %s, %s' % (filename, ftperr)
downloaded = False
# TODO: Sometimes the file doesn't complete properly.
# in that case we maybe shouldn't call this?
self.fileEventCompleted.emit()
return downloaded
@Slot(str)
def onUpload(self, filename):
self.uploadQueue.append(filename)
def uploadNext(self):
if len(self.uploadQueue) > 0:
next = self.uploadQueue.pop(0)
self.uploadFile(next)
def uploadAll(self):
for filename in self.uploadQueue:
self.uploadFile(filename)
self.uploadQueue = []
@Slot(str)
def uploadFile(self, filename):
"""
Uploads the file `filename` to the server, creating
the needed directories.
:param filename: Absolute or relative path to the file
"""
def handle(buf):
"""This function is meant to be used as callback for the `storbinary` method."""
self.upload_progress += 1024
self.uploadProgress.emit(self.upload_size, self.upload_progress)
# Creates the directory where the file will be uploaded to
self.mkpath(os.path.dirname(filename))
localpath = self.localFromServer(filename)
print 'Uploading %s to %s' % (localpath, filename)
try:
# Uploads file and updates its modified date in the server
# to match the date in the local filesystem.
self.upload_progress = 0
self.upload_size = os.path.getsize(localpath)
self.fileEvent.emit(localpath)
self.ftp.storbinary('STOR %s' % filename,
open(localpath, 'rb'),
1024,
handle)
print 'Upload finished'
with File.fromPath(filename) as uploaded:
modified = uploaded.localmdate
uploaded.servermdate = modified
self.setLastModified(filename, modified)
uploaded = True
except (IOError, OSError):
uploaded = False
self.ioError.emit(localpath)
except (error_reply, error_perm, OSError) as err:
print 'Error uploading %s, %s' % (filename, err)
uploaded = False
# TODO: Sometimes the file doesn't complete properly.
# in that case we maybe shouldn't call this?
self.fileEventCompleted.emit()
return uploaded
def lastModified(self, filename):
"""
Uses the MDTM FTP command to find the last modified timestamp
of the file `filename`.
Returns a `datetime.datetime` object in UTC representing the file's
last modified date and time.
:param filename: Relative or absolute path to the file
"""
timestamp = self.ftp.sendcmd('MDTM %s' % filename)
if '213 ' not in timestamp:
# Second chance was found to be needed in some cases.
timestamp = self.ftp.sendcmd('MDTM %s' % filename)
timestamp = timestamp.split(' ')[-1]
dateformat = '%Y%m%d%H%M%S.%f' if '.' in timestamp else '%Y%m%d%H%M%S'
try:
mtime = dt.strptime(timestamp, dateformat)
except ValueError:
mtime = dt.utcnow()
return mtime
def setLastModified(self, serverpath, newtime):
"""
Uses the MFMT or MDTM FTP commands to set `newtime` as the modified timestamp of the
file `serverpath` on the server.
:param serverpath: Relative or absolute path to the file
:param newtime: datedatime object holding the required time
"""
cmds = ['MFMT', 'MDTM']
for cmd in cmds:
try:
self.ftp.sendcmd(
'%s %s %s' % (cmd, newtime.strftime('%Y%m%d%H%M%S'), serverpath))
return
except (error_perm, error_reply) as e:
if cmd == cmds[len(cmds) - 1]:
# If is the last comand, re-raise the exception, else
# keep trying.
raise e
else:
continue
def mkpath(self, path):
"""
Creates the path `path` on the server by recursively
created folders, if needed.
:param path: Absolute path on the server to be created
"""
try:
self.ftp.cwd(path)
except error_perm:
# `cwd` call failed. Need to create some folders
make_dir = '/'
steps = path.split('/')
for step in steps:
if len(step) == 0:
continue
make_dir += '%s/' % step
try:
self.ftp.mkd(make_dir)
except error_perm:
# Probably already exists
continue
else:
# `cwd` call succeed. No need to create
# any folders
self.ftp.cwd('/')
return
@Slot(str, str)
def added(self, location, serverpath):
super(ServerWatcher, self).added(location, serverpath)
def actionFromPath(serverpath):
f = File()
fileExistsOnServer = True
try:
f.servermdate = self.lastModified(serverpath)
except error_perm:
fileExistsOnServer = False
f.servermdate = 0
f.localmdate = LocalWatcher.lastModified(self.localFromServer(serverpath))
diff = f.timeDiff()
action = None
if abs(diff) > Watcher.TOLERANCE:
if not fileExistsOnServer or diff > 0:
action = FileAction(serverpath, FileAction.UPLOAD, ServerWatcher.LOCATION)
else:
action = FileAction(serverpath, FileAction.DOWNLOAD, LocalWatcher.LOCATION)
return action
if self.preemptiveCheck:
if location == ServerWatcher.LOCATION:
localpath = self.localFromServer(serverpath)
if not os.path.exists(localpath):
action = FileAction(serverpath, FileAction.DOWNLOAD, ServerWatcher.LOCATION)
self.preemptiveActions.append(action)
else:
action = actionFromPath(serverpath)
if action is not None:
self.preemptiveActions.append(action)
elif location == LocalWatcher.LOCATION:
try:
self.ftp.sendcmd('SIZE %s' % serverpath)
except (error_reply, error_perm):
exists = False
else:
exists = True
if not exists:
action = FileAction(serverpath, FileAction.UPLOAD, LocalWatcher.LOCATION)
self.preemptiveActions.append(action)
else:
action = actionFromPath(serverpath)
if action is not None:
self.preemptiveActions.append(action)
@Slot(str, str)
def changed(self, location, serverpath):
super(ServerWatcher, self).changed(location, serverpath)
@Slot(str, str)
def deleted(self, location, serverpath):
super(ServerWatcher, self).deleted(location, serverpath)
with File.fromPath(serverpath) as deleted:
deleted.inserver = False
import sys,json
server = json.loads(sys.argv[1])
try:
if server['secure'] == True:
from ftplib import FTP_TLS
ftp = FTP_TLS()
else:
from ftplib import FTP
ftp = FTP()
ftp.connect(server['host'], server['port'])
ftp.login(server['user'], server['pass'])
try:
ftp.storbinary('STOR %s' % sys.argv[2], open(sys.argv[3], 'rb'))
except:
strsplt = sys.argv[2][1:].split('/')
strsplt = strsplt[:-1]
wdir = "/"+"/".join(strsplt)
ftp.mkd(wdir)
ftp.storbinary('STOR %s' % sys.argv[2], open(sys.argv[3], 'rb'))
ftp.quit()
except:
sys.exit(1)
class ServerWatcher(Watcher):
downloadProgress = Signal((
int,
int,
))
uploadProgress = Signal((
int,
int,
))
# Si added:
textStatus = Signal((str, ))
fileEvent = Signal((str, ))
fileEventCompleted = Signal()
loginCompleted = Signal((
bool,
str,
))
badFilenameFound = Signal((str, ))
LOCATION = 'server'
TEST_FILE = 'iqbox.test'
def __init__(self, host, ssl, parent=None):
"""
Initializes parent class and attributes. Decides
whether to use `FTP_TLS` or `FTP` based on the `ssl` param.
:param host: Location of the FTP server
:param ssl: Tells whether the FTP needs to support TLS or not
:param parent: Reference to a `QObject` instance a parent
"""
super(ServerWatcher, self).__init__(parent)
self.interval = 5000
self.localdir = ''
self.deleteQueue = []
self.downloadQueue = []
self.uploadQueue = []
self.warnedNames = []
self.ftp = None
self.useSSL = ssl
self.host = host
self.preemptiveCheck = False
self.preemptiveActions = []
self.testFile = 'iqbox.test'
@property
def currentdir(self):
"""Returns the current working directory at the server"""
return self.ftp.pwd()
def setLocalDir(self, localdir):
"""
Sets the local directory used to stored all
downloaded files. Creates the directory if needed.
:param localdir: Absolute path to local directory
"""
self.localdir = localdir
if not os.path.exists(self.localdir):
os.makedirs(self.localdir)
@pause_timer
@Slot()
def checkout(self):
"""
Recursively checks out all files on the server.
Returns a dictionary of files on the server with their last modified date.
:param download: Indicates whether or not the files should be downloaded
"""
# Check `self.deleteQueue`, `self.uploadQueue` and `self.downloadQueue` queues.
# These tasks are done in queues to make sure all FTP commands
# are done sequentially, in the same thread.
self.deleteAll()
self.uploadAll()
self.downloadAll()
# Handy list to keep track of the checkout process.
# This list contain absolute paths only.
checked_dirs = list()
# Sets '/' as initial directory and initializes `downloading_dir`
self.ftp.cwd('/')
downloading_dir = self.currentdir
check_date = dt.utcnow()
sidirlist = list()
root_cached = False
fileC = 0
while True:
# Gets the list of sub directories and files inside the
# current directory `downloading_dir`.
self.textStatus.emit('Remote scan- Downloading folder list of ' +
downloading_dir + '...')
if root_cached and downloading_dir == '/':
dir_subdirs = saved_root_dirs
dirfiles = saved_root_files
else:
dir_subdirs = self.getDirs(downloading_dir)
if downloading_dir == '/':
saved_root_dirs = dir_subdirs
# sidirlist.extend(dir_subdirs)
self.textStatus.emit(
'Remote scan- Downloading files list of ' +
downloading_dir + '...')
dirfiles = self.getFiles(downloading_dir)
if downloading_dir == '/':
saved_root_files = dirfiles
root_cached = True
# Leading '/' in `downloading_dir` breaks the `os.path.join` call
localdir = os.path.join(self.localdir, downloading_dir[1:])
if not os.path.exists(localdir):
# Creates the directory if it doesn't already exists.
os.makedirs(localdir)
for file_ in dirfiles:
# `serverpath` is the absolute path of the file on the server,
# download it only if it hasn't been already downloaded
serverpath = os.path.join(downloading_dir, file_)
serverpath = QDir.fromNativeSeparators(serverpath)
server_file = File.fromPath(serverpath)
self.textStatus.emit('Scanning remote file... ' + serverpath +
'...')
# How do we know if we should check this server file?
# We see if the date last checked is the check start time.
if server_file.last_checked_server != check_date:
# Do this process only once per file
# Added by Simon
# Give feedback on scanning of files.
fileC += 1
if fileC % 1 == 2:
self.textStatus.emit(
'Scanning remote files for changes, ' +
str(fileC) + ' files scanned.')
# STEP: IS THIS THE FIRST TIME WE SAW THE FILE, OR WAS IT ALREADY IN OUR DB?
just_added = not server_file.inserver
# STEP: IF ITS A NEW FILE, ENSURE WE DONT WANT TO SKIP IT
# Example: If it's a temporary file, or a Unix file with a name we don't support.
if just_added:
filename = os.path.basename(serverpath)
if platform.system() == 'Windows':
badName = False
for chr in [
'\\', '/', ':', '?', '"', '<', '>', '|'
]:
if chr in filename:
badName = True
break
if badName:
if filename not in self.warnedNames:
self.warnedNames.append(filename)
self.badFilenameFound.emit(filename)
continue
# STEP: ASSUMING THE FILE DID EXIST IN OUR DB, LETS SAVE THE LAST MODIFICATION DATE
lastmdate = server_file.servermdate
# STEP: SAVE THE MOD DATE TO A VARIABLE
# Now we get the last mod time.
# We expect this to work fine since this file
# was found on the server
servermdate = self.lastModified(serverpath)
# STEP: SET BOOL SHOWING THAT IT WAS ON THE SERVER, SINCE WE KNOW IT IS.
server_file.inserver = True
# STEP: SET THE TIME THE FILE WAS LAST CHECKED TO THE SCAN START TIME
server_file.last_checked_server = check_date
# STEP: SET THE MOD DATE IN THE DATABASE TO THE ONE WE JUST GOT
server_file.servermdate = servermdate
# STEP: SAVE THIS CHANGE TO THE DATABASE
server_file.session.commit()
delta = 0
if server_file.inlocal:
delta = server_file.timeDiff()
# Emit the signals after the attributes has been set and committed
if just_added is True:
self.fileAdded.emit(ServerWatcher.LOCATION, serverpath)
elif server_file.servermdate > lastmdate or delta < -Watcher.TOLERANCE:
self.fileChanged.emit(ServerWatcher.LOCATION,
serverpath, False)
#END FOR
self.textStatus.emit('Remote scan- Finding next folder...')
dir_ready = True
for dir_ in dir_subdirs:
# `dirpath` is the absolute path of the subdirectory on the server,
dirpath = QDir.fromNativeSeparators(
os.path.join(downloading_dir, dir_))
# `downloading_dir` is ready only when all its subdirectory are on the
# `checked_dirs` list.
if dirpath not in checked_dirs:
# Found one subdirectory that is not on `checked_dirs`,
# will process it in the next iteration.
downloading_dir = dirpath
dir_ready = False
break
if dir_ready is True:
# All subdirectories of `downloading_dir` are already in `checked_dirs`
if downloading_dir == '/':
# All directories ready and at '/', means checkout is complete
# So, exit the main While loop!!
break
else:
# Not at '/'. Current directory is ready so is appended to `checked_dirs`
# Back one directory to find directories that are not in `checked_dirs`
checked_dirs.append(downloading_dir)
downloading_dir = os.path.dirname(downloading_dir)
self.textStatus.emit('Remote scan- Found Folder...')
##### END OF WHILE ################
###################################################################
# Deleted files are the ones whose `last_checked_server` attribute
# didn't get updated in the recursive run.
session = Session()
deleted = session.query(File).filter(
File.last_checked_server < check_date).filter(
File.inserver == True)
for file_ in deleted:
self.fileDeleted.emit(ServerWatcher.LOCATION, file_.path)
# Wraps up the checkout process, commits to the database.
session.commit()
@Slot()
def onLogin(self, username, passwd):
ok = True
msg = ''
error_msg = 'Login failed.'
try:
if not self.ftp:
self.ftp = FTP_TLS(self.host) if self.useSSL is True else FTP(
self.host)
loginResponse = self.ftp.login(username, passwd)
except socket.gaierror:
self.ftp = None
ok = False
msg = 'Server address could not be found.'
except (error_perm, error_reply):
info = traceback.format_exception(*sys.exc_info())
for i in info:
sys.stderr.write(i)
ok = False
msg = error_msg
else:
if '230' in loginResponse:
ok = True
else:
ok = False
msg = error_msg
if ok:
# Logged in. Now let's do compability tests.
if not self.testPermissions():
# User doesn't have write permissions, don't bother doing next test.
ok = False
msg = 'It seems like you do not have write access to this server.'
else:
# Permissions test passed, now let's test MFMT for timestamp modification.
if not self.testMFMT():
ok = False
msg = 'This server does not support timestamp modification\n \
need by this application.'
self.loginCompleted.emit(ok, msg)
def getFiles(self, path):
"""
This method simply wraps the `nlst` method with an exception handler,
and returns an empty list in case an exception is caught.
:param path: Relative or absolute path on the server
"""
try:
nlst = self.ftp.nlst(path)
dirs = self.getDirs(path)
# Files are items in nlst that are not in dirs
files = [
item for item in nlst if os.path.basename(item) not in dirs
]
return files
except:
print 'Exception in ServerWatcher.getDirs'
info = traceback.format_exception(*sys.exc_info())
for i in info:
sys.stderr.write(i)
return []
def getDirs(self, path):
"""
Retrieves a list of the directories inside `path`,
uses `retrlines` and the LIST command to retrieve the items.
:param path: Relative or absolute path on the server
"""
dirs = list()
def handleLine(line):
"""
Recieves a line from the LIST command.
This function is meant to be used as callback for the `retrlines` method.
:params line: Line from the LIST command
"""
if line.startswith('d'):
# Only lines starting with 'd' are directories
# Parse the directory out of the line; lines look like:
# 'drwxrwxrwx 1 user group 0 Jun 15 2012 dirname'
dirname = line[55:].strip()
if dirname != '.' and dirname != '..':
# Ignoring '.' and '..' entries
dirs.append(dirname)
try:
self.ftp.retrlines('LIST %s' % path, handleLine)
return dirs
except:
print 'Exception in ServerWatcher.getDirs'
info = traceback.format_exception(*sys.exc_info())
for i in info:
sys.stderr.write(i)
return []
@upload_test
def testPermissions(self):
# For interface purposes. upload_test takes care of everything.
return True
@upload_test
def testMFMT(self):
# Absurd date to test whether the change really happened.
time = dt.utcfromtimestamp(100000000)
try:
self.setLastModified(self.testFile, time)
otherTime = self.lastModified(self.testFile)
diff = (time - otherTime).total_seconds()
if abs(diff) < 2:
# Let's give it a 2 seconds tolerance.
mdtm = True
else:
mdtm = False
except (ValueError, error_reply, error_perm):
info = traceback.format_exception(*sys.exc_info())
for i in info:
sys.stderr.write(i)
mdtm = False
return mdtm
@Slot(str)
def onDelete(self, filename):
self.deleteQueue.append(filename)
def deleteNext(self):
if len(self.deleteQueue) > 0:
next = self.deleteQueue.pop(0)
self.deleteFile(next)
def deleteAll(self):
for filename in self.deleteQueue:
self.deleteFile(filename)
self.deleteQueue = []
@Slot(str)
def deleteFile(self, filename):
"""
Deletes the file `filename` to the server
:param filename: Absolute or relative path to the file
"""
try:
print 'Deleting %s' % filename
self.ftp.delete(filename)
return True
except (error_reply, error_perm):
print 'Error deleting %s' % filename
return False
self.fileEventCompleted.emit()
@Slot(str)
def onDownload(self, filename):
self.downloadQueue.append(filename)
def downloadNext(self):
if len(self.downloadQueue) > 0:
next = self.downloadQueue.pop(0)
self.downloadFile(next)
def downloadAll(self):
for filename in self.downloadQueue:
self.downloadFile(filename)
self.downloadQueue = []
@Slot(str, str)
def downloadFile(self, filename, localpath=None):
"""
Performs a binary download to the file `filename` located on the server.
`filename` parameter can be either absolute or relative, though it can
fail for relative paths if the current directory is not appropiate.
:param filename: Relative or absolute path to the file
:param localpath: Absolute local path where the file will be saved
"""
def handleChunk(chunk):
"""
Receives chuncks of data downloaded from the server.
This function is meant to be used as callback for the `retrbinary` method.
:params chunk: Chunk of downloaded bytes to be written into the file
"""
# Simply writes the received data into the file `self.downloading`
self.downloading.write(chunk)
self.download_progress += len(chunk)
self.downloadProgress.emit(self.download_size,
self.download_progress)
if localpath is None:
localpath = self.localFromServer(filename)
localdir = os.path.dirname(localpath)
if not os.path.exists(localdir):
# Creates the directory if it doesn't already exists.
os.makedirs(localdir)
print 'Downloading: %s to %s' % (filename, localpath)
try:
with open(localpath, 'wb') as f:
# Opens the file at `localname` which will hold the downloaded file.
# Object attributes regarding download status are updated accordingly.
self.fileEvent.emit(filename)
self.downloading = f
self.download_progress = 0
self.download_size = int(
self.ftp.sendcmd('SIZE %s' % filename).split(' ')[-1])
self.ftp.retrbinary('RETR %s' % filename, handleChunk)
print 'Download finished'
# Let's set the same modified time to that on the server.
with File.fromPath(filename) as downloadedfile:
mdate = LocalWatcher.lastModified(localpath)
downloadedfile.localmdate = mdate
downloadedfile.servermdate = mdate
self.setLastModified(filename, mdate)
downloaded = True
except (IOError, OSError):
downloaded = False
self.ioError.emit(localpath)
except (error_reply, error_perm) as ftperr:
print 'Error downloading %s, %s' % (filename, ftperr)
downloaded = False
# TODO: Sometimes the file doesn't complete properly.
# in that case we maybe shouldn't call this?
self.fileEventCompleted.emit()
return downloaded
@Slot(str)
def onUpload(self, filename):
self.uploadQueue.append(filename)
def uploadNext(self):
if len(self.uploadQueue) > 0:
next = self.uploadQueue.pop(0)
self.uploadFile(next)
def uploadAll(self):
for filename in self.uploadQueue:
self.uploadFile(filename)
self.uploadQueue = []
@Slot(str)
def uploadFile(self, filename):
"""
Uploads the file `filename` to the server, creating
the needed directories.
:param filename: Absolute or relative path to the file
"""
def handle(buf):
"""This function is meant to be used as callback for the `storbinary` method."""
self.upload_progress += 1024
self.uploadProgress.emit(self.upload_size, self.upload_progress)
# Creates the directory where the file will be uploaded to
self.mkpath(os.path.dirname(filename))
localpath = self.localFromServer(filename)
print 'Uploading %s to %s' % (localpath, filename)
try:
# Uploads file and updates its modified date in the server
# to match the date in the local filesystem.
self.upload_progress = 0
self.upload_size = os.path.getsize(localpath)
self.fileEvent.emit(localpath)
self.ftp.storbinary('STOR %s' % filename, open(localpath, 'rb'),
1024, handle)
print 'Upload finished'
with File.fromPath(filename) as uploaded:
modified = uploaded.localmdate
uploaded.servermdate = modified
self.setLastModified(filename, modified)
uploaded = True
except (IOError, OSError):
uploaded = False
self.ioError.emit(localpath)
except (error_reply, error_perm, OSError) as err:
print 'Error uploading %s, %s' % (filename, err)
uploaded = False
# TODO: Sometimes the file doesn't complete properly.
# in that case we maybe shouldn't call this?
self.fileEventCompleted.emit()
return uploaded
def lastModified(self, filename):
"""
Uses the MDTM FTP command to find the last modified timestamp
of the file `filename`.
Returns a `datetime.datetime` object in UTC representing the file's
last modified date and time.
:param filename: Relative or absolute path to the file
"""
timestamp = self.ftp.sendcmd('MDTM %s' % filename)
if '213 ' not in timestamp:
# Second chance was found to be needed in some cases.
timestamp = self.ftp.sendcmd('MDTM %s' % filename)
timestamp = timestamp.split(' ')[-1]
dateformat = '%Y%m%d%H%M%S.%f' if '.' in timestamp else '%Y%m%d%H%M%S'
try:
mtime = dt.strptime(timestamp, dateformat)
except ValueError:
mtime = dt.utcnow()
return mtime
def setLastModified(self, serverpath, newtime):
"""
Uses the MFMT or MDTM FTP commands to set `newtime` as the modified timestamp of the
file `serverpath` on the server.
:param serverpath: Relative or absolute path to the file
:param newtime: datedatime object holding the required time
"""
cmds = ['MFMT', 'MDTM']
for cmd in cmds:
try:
self.ftp.sendcmd(
'%s %s %s' %
(cmd, newtime.strftime('%Y%m%d%H%M%S'), serverpath))
return
except (error_perm, error_reply) as e:
if cmd == cmds[len(cmds) - 1]:
# If is the last comand, re-raise the exception, else
# keep trying.
raise e
else:
continue
def mkpath(self, path):
"""
Creates the path `path` on the server by recursively
created folders, if needed.
:param path: Absolute path on the server to be created
"""
try:
self.ftp.cwd(path)
except error_perm:
# `cwd` call failed. Need to create some folders
make_dir = '/'
steps = path.split('/')
for step in steps:
if len(step) == 0:
continue
make_dir += '%s/' % step
try:
self.ftp.mkd(make_dir)
except error_perm:
# Probably already exists
continue
else:
# `cwd` call succeed. No need to create
# any folders
self.ftp.cwd('/')
return
@Slot(str, str)
def added(self, location, serverpath):
super(ServerWatcher, self).added(location, serverpath)
def actionFromPath(serverpath):
f = File()
fileExistsOnServer = True
try:
f.servermdate = self.lastModified(serverpath)
except error_perm:
fileExistsOnServer = False
f.servermdate = 0
f.localmdate = LocalWatcher.lastModified(
self.localFromServer(serverpath))
diff = f.timeDiff()
action = None
if abs(diff) > Watcher.TOLERANCE:
if not fileExistsOnServer or diff > 0:
action = FileAction(serverpath, FileAction.UPLOAD,
ServerWatcher.LOCATION)
else:
action = FileAction(serverpath, FileAction.DOWNLOAD,
LocalWatcher.LOCATION)
return action
if self.preemptiveCheck:
if location == ServerWatcher.LOCATION:
localpath = self.localFromServer(serverpath)
if not os.path.exists(localpath):
action = FileAction(serverpath, FileAction.DOWNLOAD,
ServerWatcher.LOCATION)
self.preemptiveActions.append(action)
else:
action = actionFromPath(serverpath)
if action is not None:
self.preemptiveActions.append(action)
elif location == LocalWatcher.LOCATION:
try:
self.ftp.sendcmd('SIZE %s' % serverpath)
except (error_reply, error_perm):
exists = False
else:
exists = True
if not exists:
action = FileAction(serverpath, FileAction.UPLOAD,
LocalWatcher.LOCATION)
self.preemptiveActions.append(action)
else:
action = actionFromPath(serverpath)
if action is not None:
self.preemptiveActions.append(action)
@Slot(str, str)
def changed(self, location, serverpath):
super(ServerWatcher, self).changed(location, serverpath)
@Slot(str, str)
def deleted(self, location, serverpath):
super(ServerWatcher, self).deleted(location, serverpath)
with File.fromPath(serverpath) as deleted:
deleted.inserver = False
# ftps.retrlines('LIST') # list directory content securely
# Open the upload directory
ftps.cwd(FtpUploadDir)
# Open or create 'year' folder
try:
# Open 'year' folder
ftps.cwd(year)
print("Opened '" + year + "' folder")
except:
print("Failed to open '" + year + "' folder.\nCreating new '" + year +
"' folder...")
try:
# Create new 'year' folder
ftps.mkd(year)
# Open 'year' folder
ftps.cwd(year)
print("Created '" + year + "' folder!")
except:
print("Failed to create '" + year + "' folder.")
sys.exit()
# Open or create 'month' folder
try:
# Open 'month' folder
ftps.cwd(month)
print("Opened '" + month + "' folder")
except:
print("Failed to open '" + month + "' folder.\nCreating new '" + month +
"' folder...")
except ftplib.error_perm as e:
print('Ftp fail -> ', e)
ftps.nlst()
files_list = ftps.nlst()
lg.info("List of FTP remote folders:")
for filename in files_list:
lg.debug(f'{filename} ')
if folder in ftps.nlst(
): #check if 'wordpress-todayDate' exist inside '/home/wordpress'
ftps.cwd(folder) #change into "wordpress-todayDate" directory
lg.debug(f'{folder} ')
else:
ftps.mkd(
folder
) #Create a new directory called "wordpress-todayDate" on the server.
lg.debug(f'{folder} 2')
ftps.cwd(folder)
def placeFiles():
for file in os.listdir(wp_backup_path):
localpath = os.path.join(wp_backup_path, file)
if os.path.isfile(localpath):
filesize = os.path.getsize(localpath)
#print("STOR", file, localpath)
with tqdm(unit='blocks',
unit_scale=True,
leave=False,
miniters=1,
class FTPSession(object):
""" Attempt to create some robustness and performance to FTPing """
def __init__(self, server, username, password, tmpdir="/tmp", timeout=60):
"""Build a FTP session """
self.conn = None
self.server = server
self.username = username
self.password = password
self.tmpdir = tmpdir
self.timeout = timeout
def _connect(self):
"""Connect to FTP server """
if self.conn is not None:
return
logging.debug("Creating new connection to server %s", self.server)
not_connected = True
attempt = 1
while not_connected and attempt < 6:
try:
self.conn = FTP_TLS(self.server, timeout=self.timeout)
self.conn.login(self.username, self.password)
self.conn.prot_p()
not_connected = False
except Exception as exp:
logging.debug(exp)
time.sleep(5)
self.close()
attempt += 1
if not_connected is True:
raise Exception("Failed to make FTP connection after 5 tries!")
def _reconnect(self):
""" First attempt to shut down connection and then reconnect """
logging.debug("_reconnect() was called...")
try:
self.conn.quit()
self.conn.close()
except:
pass
finally:
self.conn = None
self._connect()
def _put(self, path, localfn, remotefn):
""" """
self.chdir(path)
sz = os.path.getsize(localfn)
if sz > 14000000000:
# Step 1 Split this big file into 14GB chunks, each file will have
# suffix .aa then .ab then .ac etc
basefn = os.path.basename(localfn)
cmd = "split --bytes=14000M %s %s/%s." % (localfn, self.tmpdir, basefn)
subprocess.call(cmd, shell=True, stderr=subprocess.PIPE)
files = glob.glob("%s/%s.??" % (self.tmpdir, basefn))
for filename in files:
suffix = filename.split(".")[-1]
self.conn.storbinary("STOR %s.%s" % (remotefn, suffix), open(filename))
os.unlink(filename)
else:
logging.debug("_put '%s' to '%s'", localfn, remotefn)
self.conn.storbinary("STOR %s" % (remotefn,), open(localfn))
return True
def close(self):
""" Good bye """
try:
self.conn.quit()
self.conn.close()
except:
pass
finally:
self.conn = None
def chdir(self, path):
if self.pwd() == path.rstrip("/"):
return
self.conn.cwd("/")
for dirname in path.split("/"):
if dirname == "":
continue
bah = []
self.conn.retrlines("NLST", bah.append)
if dirname not in bah:
logging.debug("Creating directory '%s'", dirname)
self.conn.mkd(dirname)
logging.debug("Changing to directory '%s'", dirname)
self.conn.cwd(dirname)
def pwd(self):
""" Low friction function to get connectivity """
self._connect()
pwd = exponential_backoff(self.conn.pwd)
if pwd is None:
self._reconnect()
pwd = exponential_backoff(self.conn.pwd)
logging.debug("pwd() is currently '%s'", pwd)
return pwd
def put_file(self, path, localfn, remotefn):
""" Put the File """
res = exponential_backoff(self._put, path, localfn, remotefn)
if not res:
self._reconnect()
res = exponential_backoff(self._put, path, localfn, remotefn)
if not res:
logging.error("Double Failure to upload filename: '%s'", localfn)
def put_files(self, path, localfns, remotefns):
""" Put the File """
for localfn, remotefn in zip(localfns, remotefns):
self.put_file(path, localfn, remotefn)
images_to_upload = []
if image_list:
# Add first image
images_to_upload.append(image_list[0])
if len(image_list) > 1:
# Add last image (if more than 1 image)
images_to_upload.append(image_list[len(image_list)-1])
# Download oldest & newest image
for image in images_to_upload:
print 'Downloading %s...' % image
ftps.retrbinary('RETR %s' % image, open(image, 'wb').write)
# Check if directory for old images exists, if not create it
if 'old' not in directory_list:
print 'Creating dir "old"...'
ftps.mkd('old')
# Move checked images to old
for image in image_list:
#date_str = image.split('_')[1].split('.')[0]
#img_date = datetime.strptime(date_str, '%Y%m%d-%H%M%S')
print 'Moving %s to "old"...' % image
ftps.rename(image, 'old/%s' % image)
# Disconnect from FTP server
ftps.quit()
def upload_image(filename):
image_file = {'file': open(filename, 'rb')}
print 'Uploading %s to GroupMe image service...' % filename
r = requests.post('%s?access_token=%s' % (GROUPME_IMAGE_URL, TOKEN), files=image_file)
class FTPES:
def __init__(self, host, port=None, username=None, password=None, remote_path=None, absolute_zipfile_path=None):
"""
This is a helper class to manage the FTP commands from the ftplib.
@param host: The host for the connection.
@type host: String
@param port: The post for the connection.
@type port: Integer
@param username: The username for the connection. Leave blank to use "anonymous".
@type username: String
@param password: The password for the connection. Leave empty for none.
@type password: String
@param remote_path: The remote path of the server in which the zip file should be uploaded. If the path does not
exists, it will be created (recursive).
@type remote_path: String
@param absolute_zipfile_path: The absolute LOCAL filepath of the zip file.
@type absolute_zipfile_path: String
"""
self.ftps = None
self._host = host
self._port = port
self._username = username
self._password = password
self._remote_path = remote_path
self._absolute_zipfile_path = absolute_zipfile_path
self._bytesWritten = 0;
self._totalFileSize = os.path.getsize(self._absolute_zipfile_path)
self._uploadCallback = None
self._currentProgress = 0
# make the remote path relative if it isnt absolute or relative yet
if self._remote_path is not None and self._remote_path.startswith(
'.') is False and self._remote_path.startswith('/') is False:
self._remote_path = './' + self._remote_path
if self._username is None:
self._username = '******'
if self._port is None:
self._port = 22
def connect(self):
"""
Try to connect to the FTP server.
Raise an error if something went wrong.
"""
try:
self.ftps = FTP_TLS()
self.ftps.set_pasv(True)
self.ftps.connect(self._host, self._port)
except socket.gaierror:
raise
def login(self):
"""
Try to login in on the FTP server.
Raise an error if something went wrong.
"""
try:
self.ftps.login(self._username, self._password)
self.ftps.prot_p()
except ftplib.error_perm:
raise
def cwd(self):
"""
Try to switch the working directory on the FTP server (if set in the settings).
If the path does not exist, it will be created.
"""
if self._remote_path is not None:
try:
self.ftps.cwd(self._remote_path)
except ftplib.error_perm:
self.create_directory_tree(self._remote_path)
except IOError:
raise
def create_directory_tree(self, current_directory):
"""
Helper function to create the remote path.
@param current_directory: The current working directory.
@type current_directory: String
"""
if current_directory is not "":
try:
self.ftps.cwd(current_directory)
except ftplib.error_perm:
self.create_directory_tree("/".join(current_directory.split("/")[:-1]))
self.ftps.mkd(current_directory)
self.ftps.cwd(current_directory)
except IOError:
raise
def upload(self, callback=None):
"""
The upload function.
@param callback: The callback function for the upload progress.
@type callback: Function
"""
self._uploadCallback = callback
zipfile_to_upload = open(self._absolute_zipfile_path, 'rb')
zipfile_basename = os.path.basename(self._absolute_zipfile_path)
self.ftps.storbinary('STOR %s' % zipfile_basename, zipfile_to_upload, 1024, self.handle_upload_state)
zipfile_to_upload.close()
def handle_upload_state(self, block):
"""
The callback function for the upload progress.
@param block: The StringIO of the current upload state
@type block: StringIO
"""
self._bytesWritten += 1024
progress = math.floor((float(self._bytesWritten) / float(self._totalFileSize)) * 100)
do_update = False
if progress > self._currentProgress:
do_update = True
self._currentProgress = progress
if self._uploadCallback is not None:
self._uploadCallback(progress, self, do_update)
def quit(self):
"""
Try to quit everything and close the session.
"""
self.ftps.quit()
@property
def upload_path(self):
"""
Returns the upload path of the FTP server.
@return: The path of the uploaded file on the FTP server.
@rtype: String
"""
tmp_remote_path = ''
if self._remote_path is not None:
tmp_remote_path = self._remote_path
return os.path.join(tmp_remote_path, os.path.basename(self._absolute_zipfile_path))
bus.write_byte_data(I2C_ADDRESS, 0x09, relay_status)
# GPIO.output(DENYLED, GPIO.LOW)
# FTP
if lpr_filepath != '':
try:
ftp = FTP_TLS('192.168.0.200')
ftp.sendcmd('USER vlada')
ftp.sendcmd('PASS raspberry2018')
timestamp = "%04d%02d%02d" % (datetime.datetime.now().year,
datetime.datetime.now().month,
datetime.datetime.now().day)
try:
ftp.cwd('/PARKING_08_iz/' + timestamp)
except Exception as ex:
ftp.mkd('/PARKING_08_iz/' + timestamp)
ftp.cwd('/PARKING_08_iz/' + timestamp)
lpr_file = open(DIR_PATH + lpr_filepath, 'rb')
print("To ftp: " + timestamp + '/' + ftp_file_case)
ftp.storbinary('STOR ' + ftp_file_case, lpr_file)
ftp.quit()
ftp.close()
except Exception as exc:
displayMsg("FTP Error", "ftp_error.jpg")
print("FTP error")
lpr_filepath = ''
continue
lpr_filepath = ''
class FTPES(object):
"""
The FTPES class manage all ftplib commands.
"""
# pylint: disable=too-many-instance-attributes
# pylint: disable=too-many-arguments
def __init__(self,
host,
port=None,
username=None,
password=None,
remote_path=None,
absolute_zipfile_path=None):
"""
Initializer
@param host: The host for the connection.
@type host: String
@param port: The post for the connection.
@type port: Integer
@param username: The username for the connection. Leave blank to use "anonymous".
@type username: String
@param password: The password for the connection. Leave empty for none.
@type password: String
@param remote_path: The remote path of the server in which the zip file should be
uploaded. If the path does not
exists, it will be created (recursive).
@type remote_path: String
@param absolute_zipfile_path: The absolute LOCAL filepath of the zip file.
@type absolute_zipfile_path: String
"""
self.ftps = None
self._host = host
self._port = port
self._username = username
self._password = password
self._remote_path = remote_path
self._absolute_zipfile_path = absolute_zipfile_path
self._bytes_written = 0
self._total_file_size = os.path.getsize(self._absolute_zipfile_path)
self._upload_callback = None
self._current_progress = 0
# make the remote path relative if it isnt absolute or relative yet
if self._remote_path is not None and self._remote_path.startswith(
'.') is False and self._remote_path.startswith('/') is False:
self._remote_path = './' + self._remote_path
if self._username is None:
self._username = '******'
if self._port is None:
self._port = 22
# pylint: enable=too-many-instance-attributes
# pylint: enable=too-many-arguments
def connect(self):
"""
Try to connect to the FTP server.
Raise an error if something went wrong.
"""
try:
self.ftps = FTP_TLS()
self.ftps.set_pasv(True)
self.ftps.connect(self._host, self._port)
except socket.gaierror:
raise
def login(self):
"""
Try to login in on the FTP server.
Raise an error if something went wrong.
"""
try:
self.ftps.login(self._username, self._password)
self.ftps.prot_p()
except ftplib.error_perm:
raise
def cwd(self):
"""
Try to switch the working directory on the FTP server (if set in the settings).
If the path does not exist, it will be created.
"""
if self._remote_path is not None:
try:
self.ftps.cwd(self._remote_path)
except ftplib.error_perm:
self.create_directory_tree(self._remote_path)
except IOError:
raise
def create_directory_tree(self, current_directory):
"""
Helper function to create the remote path.
@param current_directory: The current working directory.
@type current_directory: String
"""
if current_directory is not "":
try:
self.ftps.cwd(current_directory)
except ftplib.error_perm:
self.create_directory_tree("/".join(
current_directory.split("/")[:-1]))
self.ftps.mkd(current_directory)
self.ftps.cwd(current_directory)
except IOError:
raise
def upload(self, callback=None):
"""
The upload function.
@param callback: The callback function for the upload progress.
@type callback: Function
"""
self._upload_callback = callback
zipfile_to_upload = open(self._absolute_zipfile_path, 'rb')
zipfile_basename = os.path.basename(self._absolute_zipfile_path)
self.ftps.storbinary('STOR %s' % zipfile_basename, zipfile_to_upload,
1024, self.handle_upload_state)
zipfile_to_upload.close()
# pylint: disable=unused-argument
def handle_upload_state(self, block):
"""
The callback function for the upload progress.
@param block: The StringIO of the current upload state
@type block: StringIO
"""
self._bytes_written += 1024
progress = math.floor(
(float(self._bytes_written) / float(self._total_file_size)) * 100)
do_update = False
if progress > self._current_progress:
do_update = True
self._current_progress = progress
if self._upload_callback is not None:
self._upload_callback(progress, self, do_update)
# pylint: enable=unused-argument
def quit(self):
"""
Try to quit everything and close the session.
"""
self.ftps.quit()
@property
def upload_path(self):
"""
Returns the upload path of the FTP server.
@return: The path of the uploaded file on the FTP server.
@rtype: String
"""
tmp_remote_path = ''
if self._remote_path is not None:
tmp_remote_path = self._remote_path
return os.path.join(tmp_remote_path,
os.path.basename(self._absolute_zipfile_path))
images_to_upload = []
if image_list:
# Add first image
images_to_upload.append(image_list[0])
if len(image_list) > 1:
# Add last image (if more than 1 image)
images_to_upload.append(image_list[len(image_list) - 1])
# Download oldest & newest image
for image in images_to_upload:
print 'Downloading %s...' % image
ftps.retrbinary('RETR %s' % image, open(image, 'wb').write)
# Check if directory for old images exists, if not create it
if 'old' not in directory_list:
print 'Creating dir "old"...'
ftps.mkd('old')
# Move checked images to old
for image in image_list:
#date_str = image.split('_')[1].split('.')[0]
#img_date = datetime.strptime(date_str, '%Y%m%d-%H%M%S')
print 'Moving %s to "old"...' % image
ftps.rename(image, 'old/%s' % image)
# Disconnect from FTP server
ftps.quit()
def upload_image(filename):
image_file = {'file': open(filename, 'rb')}
print 'Uploading %s to GroupMe image service...' % filename
