def upload(self, src, dst):
log.log('Uploading {} -> {} '.format(
src.split('/')[-1],
dst.split('/')[-1]),
self.name,
nl=False)
fp = open(src, 'rb')
command = 'STOR {}'.format(dst)
try:
self.ftp.storbinary(command, fp, blocksize=1024)
except Exception as e:
log.error('{}'.format(e), self.name)
return 0
try:
self.ftp.sendcmd('SITE CHMOD 755 {}'.format(dst))
except Exception as e:
log.error('{}'.format(e), self.name)
return 0
fp.close()
log.success('OK', tag='')
return 1
def setup(self):
self.context = zmq.Context()
self.receiver = self.context.socket(zmq.PULL)
self.receiver.bind(self.address)
time.sleep(0.5)
log.success('ReceiverData from {}'.format(self.address))
def connect(self):
'''
'''
log.log('Connecting to ftp://{}'.format(self.server), self.name)
try:
self.ftp = ftplib.FTP(self.server, timeout=1)
except ftplib.all_errors:
log.error('Server connection fail: {}'.format(self.server),
self.name)
if self.ftp is not None:
self.ftp.close()
self.ftp = None
self.ready = False
return
try:
self.ftp.login(self.username, self.password)
except ftplib.all_errors:
log.error('The given username y/o password are incorrect',
self.name)
if self.ftp is not None:
self.ftp.close()
self.ftp = None
self.ready = False
return
log.success('Connection success', self.name)
self.ready = True
return
def close(self):
BaseClass.close(self)
log.success(
'Done...(Time:{:4.2f} secs)'.format(time.time() -
self.start_time),
self.name)
def setFile(self):
'''
Create new cedar file object
'''
self.mnemonic = MNEMONICS[self.kinst] #TODO get mnemonic from madrigal
date = datetime.datetime.utcfromtimestamp(self.dataOut.utctime)
filename = '{}{}{}'.format(self.mnemonic,
date.strftime('%Y%m%d_%H%M%S'), self.ext)
self.fullname = os.path.join(self.path, filename)
if os.path.isfile(self.fullname):
log.warning(
'Destination file {} already exists, previous file deleted.'.
format(self.fullname), 'MADWriter')
os.remove(self.fullname)
try:
log.success('Creating file: {}'.format(self.fullname), 'MADWriter')
if not os.path.exists(self.path):
os.makedirs(self.path)
self.fp = madrigal.cedar.MadrigalCedarFile(self.fullname, True)
except ValueError as e:
log.error(
'Impossible to create a cedar object with "madrigal.cedar.MadrigalCedarFile"',
'MADWriter')
return
return 1
def setNextFile(self):
file_id = self.fileIndex
if file_id == len(self.fileList):
log.success('No more files in the folder', self.name)
self.flagNoMoreFiles = 1
return 0
log.success('Opening {}'.format(self.fileList[file_id]), self.name)
filename = os.path.join(self.path, self.fileList[file_id])
dirname, name = os.path.split(filename)
self.siteFile = name.split('.')[0]
if self.filename is not None:
self.fp.close()
self.filename = filename
self.fp = open(self.filename, 'rb')
self.header_file = numpy.fromfile(self.fp, FILE_HEADER_STRUCTURE, 1)
yy = self.header_file['year'] - 1900 * (self.header_file['year'] >
3000)
self.year = int(yy + 1900 * (yy < 1000))
self.doy = int(self.header_file['doy'])
self.dH = round(self.header_file['dh'], 2)
self.ipp = round(self.header_file['ipp'], 2)
self.sizeOfFile = os.path.getsize(self.filename)
self.counter_records = 0
self.flagIsNewFile = 0
self.fileIndex += 1
return 1
def readFile(self):
'''
You must indicate if you are reading in Online or Offline mode and load the
The parameters for this file reading mode.
Then you must do 2 actions:
1. Get the BLTR FileHeader.
2. Start reading the first block.
'''
if self.fileSelector < len(self.filenameList):
log.success(
'Opening file: {}'.format(
self.filenameList[self.fileSelector]), self.name)
self.fp = open(self.filenameList[self.fileSelector])
self.fheader = FileHeaderBLTR(self.fp)
self.rheader = RecordHeaderBLTR(self.fp)
self.nFDTdataRecors = self.fheader.nFDTdataRecors
self.fileSelector += 1
self.BlockCounter = 0
return 1
else:
self.flagNoMoreFiles = True
self.dataOut.flagNoData = True
return 0
def run(self):
log.success('\nStarting Project {} [id={}]'.format(self.name, self.id),
tag='')
self.started = True
self.start_time = time.time()
self.createObjects()
self.runProcs()
log.success(
'{} Done (Time: {:4.2f}s)'.format(self.name,
time.time() - self.start_time),
'')
def setNextFile(self):
'''
Open next files for the current datetime
'''
cursor = self.cursor
if not self.online_mode:
if cursor == len(self.dates):
if self.online:
cursor = 0
self.dt = self.dates[cursor]
self.online_mode = True
if not self.search_files_online():
log.success('No more files', 'PXReader')
return 0
else:
log.success('No more files', 'PXReader')
self.flagNoMoreFiles = 1
return 0
else:
if not self.search_files_online():
return 0
cursor = self.cursor
self.data = {}
self.header = []
for fullname in self.files[self.dates[cursor]]:
log.log('Opening: {}'.format(fullname), 'PXReader')
if os.path.splitext(fullname)[-1] == '.tgz':
tar = tarfile.open(fullname, 'r:gz')
tar.extractall('/tmp')
files = [
os.path.join('/tmp', member.name)
for member in tar.getmembers()
]
else:
files = [fullname]
for filename in files:
if self.filename is not None:
self.fp.close()
self.filename = filename
self.filedate = self.dates[cursor]
self.fp = Dataset(self.filename, 'r')
self.parseFile()
self.counter_records += 1
self.cursor += 1
return 1
def cmdlist(nextcommand):
if nextcommand is None:
log.error('Missing argument, available arguments: procs, operations', '')
elif nextcommand == 'procs':
procs = getProcs()
log.success(
'Current ProcessingUnits are:\n {}'.format('\n '.join(procs)), '')
elif nextcommand == 'operations':
operations = getOperations()
log.success('Current Operations are:\n {}'.format(
'\n '.join(operations)), '')
else:
log.error('Wrong argument', '')
def search(nextcommand):
if nextcommand is None:
log.error('There is no Operation/ProcessingUnit to search', '')
else:
try:
args = getArgs(nextcommand)
doc = getDoc(nextcommand)
log.success('{}\n{}\n\nparameters:\n {}'.format(
nextcommand, doc, ', '.join(args)), ''
)
except Exception as e:
log.error('Module `{}` does not exists'.format(nextcommand), '')
allModules = getAll()
similar = [t[0] for t in process.extract(nextcommand, allModules, limit=12) if t[1]>80]
log.success('Possible modules are: {}'.format(', '.join(similar)), '')
def parseHeader(self):
'''
'''
self.output = {}
self.version = '2'
s_parameters = None
if self.ext == '.txt':
self.parameters = [
s.strip().lower()
for s in self.fp.readline().decode().strip().split(' ') if s
]
elif self.ext == '.hdf5':
self.metadata = self.fp['Metadata']
if '_record_layout' in self.metadata:
s_parameters = [
s[0].lower().decode()
for s in self.metadata['Independent Spatial Parameters']
]
self.version = '3'
self.parameters = [
s[0].lower().decode() for s in self.metadata['Data Parameters']
]
log.success('Parameters found: {}'.format(self.parameters),
'MADReader')
if s_parameters:
log.success('Spatial parameters found: {}'.format(s_parameters),
'MADReader')
for param in list(self.oneDDict.keys()):
if param.lower() not in self.parameters:
log.warning(
'Parameter {} not found will be ignored'.format(param),
'MADReader')
self.oneDDict.pop(param, None)
for param, value in list(self.twoDDict.items()):
if param.lower() not in self.parameters:
log.warning(
'Parameter {} not found, it will be ignored'.format(param),
'MADReader')
self.twoDDict.pop(param, None)
continue
if isinstance(value, list):
if value[0] not in self.output:
self.output[value[0]] = []
self.output[value[0]].append([])
def generate():
inputs = basicInputs()
if inputs['multiprocess'] == 1:
current = templates.voltage.format(**inputs)
elif inputs['multiprocess'] == 2:
current = templates.spectra.format(**inputs)
elif inputs['multiprocess'] == 3:
current = templates.voltagespectra.format(**inputs)
scriptname = '{}_{}.py'.format(PREFIX, inputs['name'])
script = open(scriptname, 'w')
try:
script.write(current)
log.success('Script {} generated'.format(scriptname))
except Exception as e:
log.error('I cannot create the file. Do you have writing permissions?')
def getData(self):
'''
Storing data from databuffer to dataOut object
'''
if self.flagNoMoreFiles:
self.dataOut.flagNoData = True
log.success('No file left to process', self.name)
return 0
if not self.readNextBlock():
self.dataOut.flagNoData = True
return 0
self.set_output()
return 1
def setHeader(self):
'''
Create an add catalog and header to cedar file
'''
log.success('Closing file {}'.format(self.fullname), 'MADWriter')
if self.ext == '.dat':
self.fp.write()
else:
self.fp.dump()
self.fp.close()
header = madrigal.cedar.CatalogHeaderCreator(self.fullname)
header.createCatalog(**self.catalog)
header.createHeader(**self.header)
header.write()
def search_files(self, path, startDate, endDate, ext):
'''
Searching for BLTR rawdata file in path
Creating a list of file to proces included in [startDate,endDate]
Input:
path - Path to find BLTR rawdata files
startDate - Select file from this date
enDate - Select file until this date
ext - Extension of the file to read
'''
log.success('Searching files in {} '.format(path), self.name)
fileList0 = glob.glob1(path, '{}*{}'.format(self.format.upper(), ext))
fileList0.sort()
self.fileList = []
self.dateFileList = []
for thisFile in fileList0:
year = thisFile[2:4]
if not isNumber(year):
continue
month = thisFile[4:6]
if not isNumber(month):
continue
day = thisFile[6:8]
if not isNumber(day):
continue
year, month, day = int(year), int(month), int(day)
dateFile = datetime.date(year + 2000, month, day)
if (startDate > dateFile) or (endDate < dateFile):
continue
self.fileList.append(thisFile)
self.dateFileList.append(dateFile)
return
def createObjects(self):
'''
Instancia de unidades de procesamiento.
'''
className = eval(self.name)
kwargs = self.getKwargs()
procUnitObj = className()
procUnitObj.name = self.name
log.success('creating process...', self.name)
for conf in self.operations:
opObj = conf.createObject()
log.success(
'adding operation: {}, type:{}'.format(conf.name, conf.type),
self.name)
procUnitObj.addOperation(conf, opObj)
self.object = procUnitObj
def __setup(self, **kwargs):
'''
Initialize variables
'''
self.figures = []
self.axes = []
self.cb_axes = []
self.localtime = kwargs.pop('localtime', True)
self.show = kwargs.get('show', True)
self.save = kwargs.get('save', False)
self.save_period = kwargs.get('save_period', 0)
self.colormap = kwargs.get('colormap', self.colormap)
self.colormap_coh = kwargs.get('colormap_coh', 'jet')
self.colormap_phase = kwargs.get('colormap_phase', 'RdBu_r')
self.colormaps = kwargs.get('colormaps', None)
self.bgcolor = kwargs.get('bgcolor', self.bgcolor)
self.showprofile = kwargs.get('showprofile', False)
self.title = kwargs.get('wintitle', self.CODE.upper())
self.cb_label = kwargs.get('cb_label', None)
self.cb_labels = kwargs.get('cb_labels', None)
self.labels = kwargs.get('labels', None)
self.xaxis = kwargs.get('xaxis', 'frequency')
self.zmin = kwargs.get('zmin', None)
self.zmax = kwargs.get('zmax', None)
self.zlimits = kwargs.get('zlimits', None)
self.xmin = kwargs.get('xmin', None)
self.xmax = kwargs.get('xmax', None)
self.xrange = kwargs.get('xrange', 12)
self.xscale = kwargs.get('xscale', None)
self.ymin = kwargs.get('ymin', None)
self.ymax = kwargs.get('ymax', None)
self.yscale = kwargs.get('yscale', None)
self.xlabel = kwargs.get('xlabel', None)
self.attr_time = kwargs.get('attr_time', 'utctime')
self.attr_data = kwargs.get('attr_data', 'data_param')
self.decimation = kwargs.get('decimation', None)
self.showSNR = kwargs.get('showSNR', False)
self.oneFigure = kwargs.get('oneFigure', True)
self.width = kwargs.get('width', None)
self.height = kwargs.get('height', None)
self.colorbar = kwargs.get('colorbar', True)
self.factors = kwargs.get('factors', [1, 1, 1, 1, 1, 1, 1, 1])
self.channels = kwargs.get('channels', None)
self.titles = kwargs.get('titles', [])
self.polar = False
self.type = kwargs.get('type', 'iq')
self.grid = kwargs.get('grid', False)
self.pause = kwargs.get('pause', False)
self.save_code = kwargs.get('save_code', self.CODE)
self.throttle = kwargs.get('throttle', 0)
self.exp_code = kwargs.get('exp_code', None)
self.server = kwargs.get('server', False)
self.sender_period = kwargs.get('sender_period', 60)
self.tag = kwargs.get('tag', '')
self.height_index = kwargs.get('height_index', None)
self.__throttle_plot = apply_throttle(self.throttle)
code = self.attr_data if self.attr_data else self.CODE
self.data = PlotterData(self.CODE, self.exp_code, self.localtime)
if self.server:
if not self.server.startswith('tcp://'):
self.server = 'tcp://{}'.format(self.server)
log.success('Sending to server: {}'.format(self.server), self.name)
def sendEmail(self, email_from, email_to, subject='Error running ...', message="", subtitle="", filename="", html_format=True):
if not email_to:
return 0
if not self.__emailServer:
return 0
log.success('Sending email to {}...'.format(email_to), 'System')
msg = MIMEMultipart()
msg['Subject'] = subject
msg['From'] = "SChain API (v{}) <{}>".format(schainpy.__version__, email_from)
msg['Reply-to'] = email_from
msg['To'] = email_to
# That is what u see if dont have an email reader:
msg.preamble = 'SChainPy'
if html_format:
message = "<h1> %s </h1>" %subject + "<h3>" + subtitle.replace("\n", "</h3><h3>\n") + "</h3>" + message.replace("\n", "<br>\n")
message = "<html>\n" + message + '</html>'
# This is the textual part:
part = MIMEText(message, "html")
else:
message = subject + "\n" + subtitle + "\n" + message
part = MIMEText(message)
msg.attach(part)
if filename and os.path.isfile(filename):
# This is the binary part(The Attachment):
part = MIMEApplication(open(filename,"rb").read())
part.add_header('Content-Disposition',
'attachment',
filename=os.path.basename(filename))
msg.attach(part)
# Create an instance in SMTP server
try:
smtp = smtplib.SMTP(self.__emailServer)
except:
log.error('Could not connect to server {}'.format(self.__emailServer), 'System')
return 0
# Start the server:
# smtp.ehlo()
if self.__emailPass:
smtp.login(self.__emailFromAddress, self.__emailPass)
# Send the email
try:
smtp.sendmail(msg['From'], msg['To'], msg.as_string())
except:
log.error('Could not send the email to {}'.format(msg['To']), 'System')
smtp.quit()
return 0
smtp.quit()
log.success('Email sent ', 'System')
return 1