def getnewfileslist(self, dev):
ip = self.ip
usr = self.usr
pas = self.pas
ret_lis = []
data = []
ftphandler = ftplib.FTP()
self.error.resetall()
while self.error.exit == 0:
try:
data[:] = []
ret_lis[:] = []
ftphandler = ftplib.FTP(ip, usr, pas)
ftphandler.cwd("/%s" % dev.rpath)
ftphandler.retrlines('MLSD', data.append)
LOG.info(data)
ret_lis = buildlistfromdata_MLSD(data, dev)
self.error.resetall()
self.error.exit = 1
except ftplib.all_errors, e:
self.error.e = e
self.printerror("FTP")
except socket.error, e:
self.error.e = e
self.printerror("SOCKET")
def buildlistfromdata_MLSD(data, dev):
dm = DatesManager.DatesManager()
datelist = []
filelist = []
last = dev.getlocallastdatesecs()
for line in data:
col = line.split(';')
mod_str = col[1].replace("modify=", "").strip()
mod_sec = dm.str2secs(mod_str)
ld_str = dm.secs2str(dev.last)
filename = col[3].strip()
if mod_sec > dev.last:
filename = col[3].strip()
filesize = col[2].replace("size=", "").strip()
filelist.append("%s;%s;%s" % (filename, filesize, mod_str))
datelist.append(mod_str)
################################################################################
# Debug info
GLOBAL.NUM_FILES += 1
if mod_sec > dev.last:
GLOBAL.NEW_FILES += 1
LOG.info(
"DEVICE=%-6s : NAME=%-22s : MOD=%-14s - REF=%-14s : NOW=%-14s ::: OLD=%-5s ::: AGE=%d"
% (dev.name, filename, mod_str, ld_str, GLOBAL.NOW_STR,
(mod_sec <= dev.last), int(GLOBAL.NOW_SEC - mod_sec)))
################################################################################
return zip(datelist, filelist)
def getallfiles_MLSD(self, lsrv, ut):
LOG.info("-" * GLOBAL.NCHAR)
# files_all = FilesManager()
for lsrv in lsrv:
arr = lsrv.split(';')
dev = DevManager.DevManager(arr[3].strip(),
arr[4].strip(),
arr[0].strip(),
ut,
setMLSD=True)
ftp = FtpManager.FtpManager(arr[0].strip(), arr[1].strip(),
arr[2].strip())
# ftp.error.resetall()
files = FilesManager()
lst = ftp.getnewfileslist(dev)
if ftp.error.exit != 0:
files.addfilesfromlist_MLSD(lst, dev)
if files.count > 0:
ftp = getftp_MLSD(ftp, files)
self.extendlist(files.list)
ftp.error.resetall()
else:
LOG.error("ERROR getting the list of files ::: Device=%s" %
dev.name)
LOG.info("-" * GLOBAL.NCHAR)
def keepperiod(period, start_time):
time_consumed = datetime.datetime.utcnow() - start_time
time_to_wait = (datetime.timedelta(seconds=period) -
time_consumed).total_seconds()
LOG.info("Last cycle duration was: %s secs." %
time_consumed.total_seconds())
LOG.info("Waiting for %s secs." % time_to_wait)
if time_to_wait > 0:
time.sleep(time_to_wait)
def createMonitor(self, monitor_json):
monitor_config = {
"vargroupname": "COUNTERS",
"cadence": 1024,
"numshots": 400
}
monitor_url = self._url + "/monitors"
response = self._postrequest(monitor_url, monitor_config)
LOG.info("calling buildmonitor")
return response
def movezipfiles(files):
files_new = FilesManager.FilesManager()
if files.count > 0:
for f in files.list:
LOG.info("ZIP=%s, UP=%s" % (f['zfname'], f['ufname']))
f['srv'] = SETTINGS.RSRV1.split(';')[0]
# LOG.info("%s, %s" % (f['zfname'],f['ufname']))
os.system("mv %s %s" % (f['zfname'], f['ufname']))
files_new.addelement(f)
return files_new
def buildlistfromdata(data, dev):
#LOG.info("COMPARING MODIFIED FILES")
dm = DatesManager.DatesManager()
datelist = []
filelist = []
i = 0
# Emparejamos la lista y si no coinciden las longitudes rellenamos con default
nfiles_data = len(data)
nfiles_dev = len(dev.list)
# En caso de que haya mayor numero de ficheros en el servidor que en el fichero local
# significaria que se ha anadido alguno por lo que pongo un default
while nfiles_data > nfiles_dev:
LOG.info("New file in the server")
dev.list.append('default;0000000000000')
nfiles_dev = len(dev.list)
# En caso de que haya mas ficheros en local que en el servidor (algun cambio de tcu o tcu nueva)
# solo se compararan los que tenga el servidor ya que parte de la lista data sera null y el zip truncara
for line_data, line_dev in zip(data, dev.list):
if line_dev == None:
line_dev.list.append(
'default;0000000000000'
) # Se añade a dev un default file hasta completar la lista
col_data = line_data.split(';')
col_dev = line_dev.split(';')
# Guardamos la segunda columna para quedarnos con la fecha de modificacion de lo descargado
mod_str_col = col_data[1].strip()
# Guardamos la segunda columna de lo que teníamos en la lista de dev
mod_str_dev = col_dev[1].strip()
# Se guarda el nombre del fichero descargado
filename = col_data[0].strip()
# Si las fechas de modificacion de ambas listas no coincidense anade el fichero a la lista
if mod_str_col != mod_str_dev:
filelist.append(
"%s;%s" %
(filename, mod_str_col)) # Se anade el nombre del fichero
datelist.append(mod_str_col) # Se anade la fecha de mod
################################################################################
# Debug info
GLOBAL.NUM_FILES += 1
if mod_str_col != mod_str_dev:
GLOBAL.NEW_FILES += 1
#LOG.info("DEVICE=%-6s : NAME=%-22s : MOD=%-14s - PREV=%-14s : NOW=%-14s ::: CHANGE=%-5s " %
# (dev.name, filename, mod_str_col, mod_str_dev, GLOBAL.NOW_STR, (mod_str_col != mod_str_dev),
# ))
#################################################################################
i += 1
return zip(datelist, filelist)
def getuic559(Rest, file559, first, DEVICE_LID, DEVICE_UID):
LOG.info("-" * GLOBAL.NCHAR)
LOG.info("GETTING UIC from %s:%s" % (Rest._ip, Rest._port))
LOG.info("-" * GLOBAL.NCHAR)
# Se importan configuraciones del rest segun el device que se pida
try:
write = 0 # Variable (1,0) que indica si hay que escribir el uic para no volver a escribir lo mismo
if (first):
# The first time we ask for the last 200 events
response = Rest.getUic()
else:
# Once we have the last id we ask from that id to newer events
response = Rest.getUicfromid(DEVICE_UID)
DEVICE_NEW_LID = response.headers['X-Cafpa-LowerEventId']
DEVICE_NEW_UID = response.headers['X-Cafpa-UpperEventId']
if (DEVICE_NEW_UID > DEVICE_UID):
write = 1
else:
LOG.info("No new events")
if response.status_code == 200 and write == 1:
with open(file559, "wb") as f:
f.write(response.text)
f.close()
return response.status_code, False, DEVICE_NEW_LID, DEVICE_NEW_UID
else:
return response.status_code, False, DEVICE_NEW_LID, DEVICE_NEW_UID
except requests.exceptions.RequestException as e:
LOG.warning("ERROR REQUESTS: %s" % str(e))
return 0, False, DEVICE_LID, DEVICE_UID
def getftp(ftp, files):
dm = DatesManager.DatesManager()
for f in files.list:
ftp.error.resetall()
while ftp.error.exit == 0:
ftp.connect()
ftp.getbinary(f)
if ftp.error.exit == 1:
LOG.info("File %s downloaded" % (f['name']))
GLOBAL.DOW_FILES += 1
LOG.info("Date of %s changed" % (f['name']))
# END WHILE
# END FOR
ftp.disconnect()
return ftp
def getbinary(self, f):
if self.connected:
try:
self.ftp.cwd("/%s" % f['rpath'])
file_hand = open(f['flname'], "wb")
self.ftp.retrbinary("RETR " + f['name'], file_hand.write)
file_hand.close()
self.error.resetall()
self.error.exit = 1
except ftplib.all_errors, e:
self.error.e = e
LOG.info("ERROR with file: %s" % f['name'])
self.copy(printerror("FTP", self))
except socket.error, e:
self.error.e = e
LOG.info("ERROR with file: %s" % f['name'])
self.copy(printerror("SOCKET", self))
def printerror(self, ref):
s = 0
n = 0
if ref == "FTP":
n = self.error.nerr_ftp
s = 1
elif ref == "SOCKET":
n = self.error.nerr_soc
s = 2
elif ref == "GENERAL":
n = 1000
s = 0
elif ref == "FTP.CLOSE":
n = 1000
s = 0
self.error.exit = 0
if (n > SETTINGS.MAX_RETRIES) and (str(self.error.e) !=
self.error.msg):
self.error.msg = str(self.error.e)
LOG.error("%s ERROR: %s" % (ref, str(self.error.e)))
self.error.exit = -1
if n >= 1000:
self.error.msg = ""
self.error.exit = -1
if ref == "FTP":
self.error.nerr_ftp += 1
self.error.exit = -1
elif ref == "SOCKET":
self.error.nerr_soc += 1
self.error.exit = -1
elif ref == "GENERAL":
self.error.nerr_gen = 1000
self.error.exit = -1
elif ref == "FTP.CLOSE":
self.error.nerr_cls = 1000
self.error.exit = -1
else:
LOG.info("FTP error no managed")
self.error.nerr_cls = 1000
self.error.exit = -1
LOG.error("%s ERROR: %s" % (ref, str(self.error.e)))
time.sleep(s)
def putftp(self, files, rpath):
files_up = FilesManager.FilesManager()
for f in files.list:
LOG.info("uploading file: %s" % (f['ufname']))
f['rpath'] = rpath
self.error.resetall()
while self.error.exit == 0:
self.connect()
self.putbinary(f)
if self.error.exit == 1:
LOG.info("File %s uploaded" % (f['ufname']))
GLOBAL.UP_FILES += 1
files_up.addelement(f)
# END WHILE
# END FOR
self.disconnect()
return files_up
def getftp_MLSD(ftp, files):
dm = DatesManager.DatesManager()
for f in files.list:
ftp.error.resetall()
while ftp.error.exit == 0:
ftp.connect()
ftp.getbinary(f)
if ftp.error.exit == 1:
LOG.info("File %s downloaded" % (f['name']))
GLOBAL.DOW_FILES += 1
seconds = dm.str2secs(str(f['mod']))
os.utime(f['flname'], (seconds, seconds))
LOG.info("Date of %s changed" % (f['name']))
# END WHILE
# END FOR
ftp.disconnect()
return ftp
def ZipandMovetoUpload(
filename,
filenamefull,
ORIGIN_PATH,
ZIPDESTINATION_PATH,
):
LOG.info("ZIPPING FILE %s" % (filenamefull))
zip_fname = ("%s.zip" % filenamefull).replace(ORIGIN_PATH,
ZIPDESTINATION_PATH)
zf = zipfile.ZipFile(zip_fname, 'a')
zf.write(filenamefull, filename, compress_type=zipfile.ZIP_DEFLATED)
zf.close()
# ------------------------------------------------------------------------------
# Mover ZIP HMI a cola de envío
# ------------------------------------------------------------------------------
LOG.info("MOVING %s TO UPLOAD FOLDER" % (filenamefull))
os.system(
"mv %s %s" %
(zip_fname, zip_fname.replace(SETTINGS.ZPATH559, SETTINGS.UPATH)))
# ------------------------------------------------------------------------------
# BORRAR UIC559 Propio sin comprimir
# ------------------------------------------------------------------------------
try:
LOG.info("DELETING UNCOMPRESSED FILES: %s" % filenamefull)
os.remove(filenamefull)
except OSError:
pass
def backupuploadedfiles(self):
if self.count > 0:
new_list = []
for tup in self.list:
try:
os.system("mv %s %s" %
(tup['ufname'], tup['ufname'].replace(
SETTINGS.UPATH, SETTINGS.UEDPATH)))
size = 0
while True:
size = getarchivesize(SETTINGS.UEDPATH)
if size > SETTINGS.MAX_SIZE:
oldest = str(oldestfile(SETTINGS.UEDPATH)[0])
os.remove("%s" % oldest)
LOG.info("%s deleted" % oldest)
else:
break
LOG.info("%s size = %s" % (SETTINGS.UEDPATH, size))
LOG.info("-" * GLOBAL.NCHAR)
except OSError, e:
LOG.error("%s" % str(e))
new_list.append(tup)
self.list = list(new_list)
self.count = len(self.list)
def getallfiles(lsrv, ut):
LOG.info("-" * GLOBAL.NCHAR)
files_all = FilesManager()
for lsrv in lsrv:
arr = lsrv.split(';')
dev = DevManager.DevManager(arr[3].strip(), arr[4].strip(),
arr[0].strip(), ut)
ftp = FtpManager.FtpManager(arr[0].strip(), arr[1].strip(),
arr[2].strip())
ftp.error.resetall()
files = FilesManager.FilesManager()
(ftp, lst) = getnewfileslist(ftp, dev)
if ftp.error.exit != 0:
files.addfilesfromlist(lst, dev)
if files.count > 0:
ftp = getftp(ftp, files)
files_all.extendlist(files.list)
ftp.error.resetall()
else:
LOG.error("ERROR getting the list of files ::: Device=%s" %
dev.name)
LOG.info("-" * GLOBAL.NCHAR)
return files_all
def putallfiles(self, rsrv):
LOG.info("-" * GLOBAL.NCHAR)
srv = rsrv.split(';')
rpath = srv[3].strip()
ftp = FtpManager.FtpManager(srv[0].strip(), srv[1].strip(),
srv[2].strip())
# ftp.error.resetall()
if self.count > 0:
# ftp, lst = putftp(ftp, self, rpath)
self.list = ftp.putftp(self, rpath).list
self.count = len(self.list)
if ftp.error.exit != 1:
LOG.error("ERROR uploading files")
else:
LOG.info("There are no files to upload")
LOG.info("-" * GLOBAL.NCHAR)
def _getrequest(self, url):
LOG.info("GET REQUEST %s" % url)
response = requests.get(url,
timeout=self._timeout,
headers=self._headers)
return response
def _postrequest(self, url, data):
LOG.info("POST REQUEST %s , json-> %s" % (url, data))
response = requests.post(url,
data=json.dumps(data),
headers=self._headers)
return response
file_hand = open(f['flname'], "wb")
self.ftp.retrbinary("RETR " + f['name'], file_hand.write)
file_hand.close()
self.error.resetall()
self.error.exit = 1
except ftplib.all_errors, e:
self.error.e = e
LOG.info("ERROR with file: %s" % f['name'])
self.copy(printerror("FTP", self))
except socket.error, e:
self.error.e = e
LOG.info("ERROR with file: %s" % f['name'])
self.copy(printerror("SOCKET", self))
except Exception, e:
self.error.e = e
LOG.info("ERROR with file: %s" % f['name'])
self.copy(printerror("GENERAL", self))
# Carga en srv remoto los ficheros incluidos
def putftp(self, files, rpath):
files_up = FilesManager.FilesManager()
for f in files.list:
LOG.info("uploading file: %s" % (f['ufname']))
f['rpath'] = rpath
self.error.resetall()
while self.error.exit == 0:
self.connect()
self.putbinary(f)
if self.error.exit == 1:
LOG.info("File %s uploaded" % (f['ufname']))
GLOBAL.UP_FILES += 1
