def call(self, **kwargs):
'''
'''
try:
if self.dataIn is not None and self.dataIn.flagNoData and not self.dataIn.error:
return self.dataIn.isReady()
elif self.dataIn is None or not self.dataIn.error:
self.run(**kwargs)
elif self.dataIn.error:
self.dataOut.error = self.dataIn.error
self.dataOut.flagNoData = True
except:
err = traceback.format_exc()
if 'SchainWarning' in err:
log.warning(
err.split('SchainWarning:')[-1].split('\n')[0].strip(),
self.name)
elif 'SchainError' in err:
log.error(
err.split('SchainError:')[-1].split('\n')[0].strip(),
self.name)
else:
log.error(err, self.name)
self.dataOut.error = True
for op, optype, opkwargs in self.operations:
if optype == 'other' and not self.dataOut.flagNoData:
self.dataOut = op.run(self.dataOut, **opkwargs)
elif optype == 'external' and not self.dataOut.flagNoData:
op.queue.put(self.dataOut)
elif optype == 'external' and self.dataOut.error:
op.queue.put(self.dataOut)
return 'Error' if self.dataOut.error else self.dataOut.isReady()
def readXml(self, filename):
abs_file = os.path.abspath(filename)
self.configurations = {}
try:
self.xml = ElementTree().parse(abs_file)
except:
log.error('Error reading %s, verify file format' % filename)
return 0
self.id = self.xml.get('id')
self.name = self.xml.get('name')
self.description = self.xml.get('description')
for element in self.xml:
if element.tag == 'ReadUnit':
conf = ReadUnitConf()
conf.readXml(element, self.id, self.err_queue)
self.configurations[conf.id] = conf
elif element.tag == 'ProcUnit':
conf = ProcUnitConf()
input_proc = self.configurations[element.get('inputId')]
conf.readXml(element, self.id, self.err_queue)
self.configurations[conf.id] = conf
self.filename = abs_file
return 1
def upload(self, src, dst):
log.log('Uploading {} -> {} '.format(
src.split('/')[-1],
dst.split('/')[-1]),
self.name,
nl=False)
fp = open(src, 'rb')
command = 'STOR {}'.format(dst)
try:
self.ftp.storbinary(command, fp, blocksize=1024)
except Exception as e:
log.error('{}'.format(e), self.name)
return 0
try:
self.ftp.sendcmd('SITE CHMOD 755 {}'.format(dst))
except Exception as e:
log.error('{}'.format(e), self.name)
return 0
fp.close()
log.success('OK', tag='')
return 1
def connect(self):
'''
'''
log.log('Connecting to ftp://{}'.format(self.server), self.name)
try:
self.ftp = ftplib.FTP(self.server, timeout=1)
except ftplib.all_errors:
log.error('Server connection fail: {}'.format(self.server),
self.name)
if self.ftp is not None:
self.ftp.close()
self.ftp = None
self.ready = False
return
try:
self.ftp.login(self.username, self.password)
except ftplib.all_errors:
log.error('The given username y/o password are incorrect',
self.name)
if self.ftp is not None:
self.ftp.close()
self.ftp = None
self.ready = False
return
log.success('Connection success', self.name)
self.ready = True
return
def setFile(self):
'''
Create new cedar file object
'''
self.mnemonic = MNEMONICS[self.kinst] #TODO get mnemonic from madrigal
date = datetime.datetime.utcfromtimestamp(self.dataOut.utctime)
filename = '{}{}{}'.format(self.mnemonic,
date.strftime('%Y%m%d_%H%M%S'), self.ext)
self.fullname = os.path.join(self.path, filename)
if os.path.isfile(self.fullname):
log.warning(
'Destination file {} already exists, previous file deleted.'.
format(self.fullname), 'MADWriter')
os.remove(self.fullname)
try:
log.success('Creating file: {}'.format(self.fullname), 'MADWriter')
if not os.path.exists(self.path):
os.makedirs(self.path)
self.fp = madrigal.cedar.MadrigalCedarFile(self.fullname, True)
except ValueError as e:
log.error(
'Impossible to create a cedar object with "madrigal.cedar.MadrigalCedarFile"',
'MADWriter')
return
return 1
def cmdlist(nextcommand):
if nextcommand is None:
log.error('Missing argument, available arguments: procs, operations', '')
elif nextcommand == 'procs':
procs = getProcs()
log.success(
'Current ProcessingUnits are:\n {}'.format('\n '.join(procs)), '')
elif nextcommand == 'operations':
operations = getOperations()
log.success('Current Operations are:\n {}'.format(
'\n '.join(operations)), '')
else:
log.error('Wrong argument', '')
def get_path():
'''
Return schainpy path
'''
try:
root = __file__
if os.path.islink(root):
root = os.path.realpath(root)
return os.path.dirname(os.path.abspath(root))
except:
log.error('I am sorry, but something is wrong... __file__ not found')
def search(nextcommand):
if nextcommand is None:
log.error('There is no Operation/ProcessingUnit to search', '')
else:
try:
args = getArgs(nextcommand)
doc = getDoc(nextcommand)
log.success('{}\n{}\n\nparameters:\n {}'.format(
nextcommand, doc, ', '.join(args)), ''
)
except Exception as e:
log.error('Module `{}` does not exists'.format(nextcommand), '')
allModules = getAll()
similar = [t[0] for t in process.extract(nextcommand, allModules, limit=12) if t[1]>80]
log.success('Possible modules are: {}'.format(', '.join(similar)), '')
def generate():
inputs = basicInputs()
if inputs['multiprocess'] == 1:
current = templates.voltage.format(**inputs)
elif inputs['multiprocess'] == 2:
current = templates.spectra.format(**inputs)
elif inputs['multiprocess'] == 3:
current = templates.voltagespectra.format(**inputs)
scriptname = '{}_{}.py'.format(PREFIX, inputs['name'])
script = open(scriptname, 'w')
try:
script.write(current)
log.success('Script {} generated'.format(scriptname))
except Exception as e:
log.error('I cannot create the file. Do you have writing permissions?')
def run(self):
while True:
dataOut = self.queue.get()
if not dataOut.error:
try:
BaseClass.run(self, dataOut, **self.kwargs)
except:
err = traceback.format_exc()
log.error(err, self.name)
else:
break
self.close()
def getData(self):
'''
Storing data from databuffer to dataOut object
'''
if self.flagNoMoreFiles:
self.dataOut.flagNoData = True
log.error('No file left to process', 'PXReader')
return 0
if not self.readNextFile():
self.dataOut.flagNoData = True
return 0
self.set_output()
return 1
def verifyFile(self, filename):
flag = True
try:
fp = open(filename, 'rb')
except IOError:
log.error("File {} can't be opened".format(filename), self.name)
return False
if self.online and self.waitDataBlock(0):
pass
basicHeaderObj = BasicHeader(LOCALTIME)
systemHeaderObj = SystemHeader()
radarControllerHeaderObj = RadarControllerHeader()
processingHeaderObj = ProcessingHeader()
if not (basicHeaderObj.read(fp)):
flag = False
if not (systemHeaderObj.read(fp)):
flag = False
if not (radarControllerHeaderObj.read(fp)):
flag = False
if not (processingHeaderObj.read(fp)):
flag = False
if not self.online:
dt1 = basicHeaderObj.datatime
pos = self.fileSize - processingHeaderObj.blockSize - 24
if pos < 0:
flag = False
log.error('Invalid size for file: {}'.format(self.filename),
self.name)
else:
fp.seek(pos)
if not (basicHeaderObj.read(fp)):
flag = False
dt2 = basicHeaderObj.datatime
if not self.isDateTimeInRange(dt1, self.startDate, self.endDate, self.startTime, self.endTime) and not \
self.isDateTimeInRange(dt2, self.startDate, self.endDate, self.startTime, self.endTime):
flag = False
fp.close()
return flag
def main(command, nextcommand, version):
"""COMMAND LINE INTERFACE FOR SIGNAL CHAIN - JICAMARCA RADIO OBSERVATORY V3.0\n
Available commands:\n
xml: runs a schain XML generated file\n
run: runs any python script'\n
generate: generates a template schain script\n
list: return a list of available procs and operations\n
search: return avilable operations, procs or arguments of the given
operation/proc\n"""
if command == 'xml':
runFromXML(nextcommand)
elif command == 'generate':
generate()
elif command == 'test':
test()
elif command == 'run':
runschain(nextcommand)
elif command == 'search':
search(nextcommand)
elif command == 'list':
cmdlist(nextcommand)
else:
log.error('Command {} is not defined'.format(command))
def runschain(nextcommand):
if nextcommand is None:
currentfiles = glob.glob('./{}_*.py'.format(PREFIX))
numberfiles = len(currentfiles)
if numberfiles > 1:
log.error('There is more than one file to run')
elif numberfiles == 1:
subprocess.call(['python ' + currentfiles[0]], shell=True)
else:
log.error('There is no file to run')
else:
try:
subprocess.call(['python ' + nextcommand], shell=True)
except Exception as e:
log.error("I cannot run the file. Does it exists?")
def sendEmail(self, email_from, email_to, subject='Error running ...', message="", subtitle="", filename="", html_format=True):
if not email_to:
return 0
if not self.__emailServer:
return 0
log.success('Sending email to {}...'.format(email_to), 'System')
msg = MIMEMultipart()
msg['Subject'] = subject
msg['From'] = "SChain API (v{}) <{}>".format(schainpy.__version__, email_from)
msg['Reply-to'] = email_from
msg['To'] = email_to
# That is what u see if dont have an email reader:
msg.preamble = 'SChainPy'
if html_format:
message = "<h1> %s </h1>" %subject + "<h3>" + subtitle.replace("\n", "</h3><h3>\n") + "</h3>" + message.replace("\n", "<br>\n")
message = "<html>\n" + message + '</html>'
# This is the textual part:
part = MIMEText(message, "html")
else:
message = subject + "\n" + subtitle + "\n" + message
part = MIMEText(message)
msg.attach(part)
if filename and os.path.isfile(filename):
# This is the binary part(The Attachment):
part = MIMEApplication(open(filename,"rb").read())
part.add_header('Content-Disposition',
'attachment',
filename=os.path.basename(filename))
msg.attach(part)
# Create an instance in SMTP server
try:
smtp = smtplib.SMTP(self.__emailServer)
except:
log.error('Could not connect to server {}'.format(self.__emailServer), 'System')
return 0
# Start the server:
# smtp.ehlo()
if self.__emailPass:
smtp.login(self.__emailFromAddress, self.__emailPass)
# Send the email
try:
smtp.sendmail(msg['From'], msg['To'], msg.as_string())
except:
log.error('Could not send the email to {}'.format(msg['To']), 'System')
smtp.quit()
return 0
smtp.quit()
log.success('Email sent ', 'System')
return 1
def _monitor(self, queue, ctx):
import socket
procs = 0
err_msg = ''
while True:
msg = queue.get()
if '#_start_#' in msg:
procs += 1
elif '#_end_#' in msg:
procs -= 1
else:
err_msg = msg
if procs == 0 or 'Traceback' in err_msg:
break
time.sleep(0.1)
if '|' in err_msg:
name, err = err_msg.split('|')
if 'SchainWarning' in err:
log.warning(
err.split('SchainWarning:')[-1].split('\n')[0].strip(),
name)
elif 'SchainError' in err:
log.error(
err.split('SchainError:')[-1].split('\n')[0].strip(), name)
else:
log.error(err, name)
else:
name, err = self.name, err_msg
time.sleep(1)
ctx.term()
message = ''.join(err)
if err_msg:
subject = 'SChain v%s: Error running %s\n' % (schainpy.__version__,
self.name)
subtitle = 'Hostname: %s\n' % socket.gethostbyname(
socket.gethostname())
subtitle += 'Working directory: %s\n' % os.path.abspath('./')
subtitle += 'Configuration file: %s\n' % self.filename
subtitle += 'Time: %s\n' % str(datetime.datetime.now())
readUnitConfObj = self.getReadUnit()
if readUnitConfObj:
subtitle += '\nInput parameters:\n'
subtitle += '[Data path = %s]\n' % readUnitConfObj.parameters[
'path']
subtitle += '[Start date = %s]\n' % readUnitConfObj.parameters[
'startDate']
subtitle += '[End date = %s]\n' % readUnitConfObj.parameters[
'endDate']
subtitle += '[Start time = %s]\n' % readUnitConfObj.parameters[
'startTime']
subtitle += '[End time = %s]\n' % readUnitConfObj.parameters[
'endTime']
a = Alarm(modes=self.alarm,
email=self.email,
message=message,
subject=subject,
subtitle=subtitle,
filename=self.filename)
a.start()