def call(self, **kwargs): ''' ''' try: if self.dataIn is not None and self.dataIn.flagNoData and not self.dataIn.error: return self.dataIn.isReady() elif self.dataIn is None or not self.dataIn.error: self.run(**kwargs) elif self.dataIn.error: self.dataOut.error = self.dataIn.error self.dataOut.flagNoData = True except: err = traceback.format_exc() if 'SchainWarning' in err: log.warning( err.split('SchainWarning:')[-1].split('\n')[0].strip(), self.name) elif 'SchainError' in err: log.error( err.split('SchainError:')[-1].split('\n')[0].strip(), self.name) else: log.error(err, self.name) self.dataOut.error = True for op, optype, opkwargs in self.operations: if optype == 'other' and not self.dataOut.flagNoData: self.dataOut = op.run(self.dataOut, **opkwargs) elif optype == 'external' and not self.dataOut.flagNoData: op.queue.put(self.dataOut) elif optype == 'external' and self.dataOut.error: op.queue.put(self.dataOut) return 'Error' if self.dataOut.error else self.dataOut.isReady()
def readXml(self, filename): abs_file = os.path.abspath(filename) self.configurations = {} try: self.xml = ElementTree().parse(abs_file) except: log.error('Error reading %s, verify file format' % filename) return 0 self.id = self.xml.get('id') self.name = self.xml.get('name') self.description = self.xml.get('description') for element in self.xml: if element.tag == 'ReadUnit': conf = ReadUnitConf() conf.readXml(element, self.id, self.err_queue) self.configurations[conf.id] = conf elif element.tag == 'ProcUnit': conf = ProcUnitConf() input_proc = self.configurations[element.get('inputId')] conf.readXml(element, self.id, self.err_queue) self.configurations[conf.id] = conf self.filename = abs_file return 1
def upload(self, src, dst): log.log('Uploading {} -> {} '.format( src.split('/')[-1], dst.split('/')[-1]), self.name, nl=False) fp = open(src, 'rb') command = 'STOR {}'.format(dst) try: self.ftp.storbinary(command, fp, blocksize=1024) except Exception as e: log.error('{}'.format(e), self.name) return 0 try: self.ftp.sendcmd('SITE CHMOD 755 {}'.format(dst)) except Exception as e: log.error('{}'.format(e), self.name) return 0 fp.close() log.success('OK', tag='') return 1
def connect(self): ''' ''' log.log('Connecting to ftp://{}'.format(self.server), self.name) try: self.ftp = ftplib.FTP(self.server, timeout=1) except ftplib.all_errors: log.error('Server connection fail: {}'.format(self.server), self.name) if self.ftp is not None: self.ftp.close() self.ftp = None self.ready = False return try: self.ftp.login(self.username, self.password) except ftplib.all_errors: log.error('The given username y/o password are incorrect', self.name) if self.ftp is not None: self.ftp.close() self.ftp = None self.ready = False return log.success('Connection success', self.name) self.ready = True return
def setFile(self): ''' Create new cedar file object ''' self.mnemonic = MNEMONICS[self.kinst] #TODO get mnemonic from madrigal date = datetime.datetime.utcfromtimestamp(self.dataOut.utctime) filename = '{}{}{}'.format(self.mnemonic, date.strftime('%Y%m%d_%H%M%S'), self.ext) self.fullname = os.path.join(self.path, filename) if os.path.isfile(self.fullname): log.warning( 'Destination file {} already exists, previous file deleted.'. format(self.fullname), 'MADWriter') os.remove(self.fullname) try: log.success('Creating file: {}'.format(self.fullname), 'MADWriter') if not os.path.exists(self.path): os.makedirs(self.path) self.fp = madrigal.cedar.MadrigalCedarFile(self.fullname, True) except ValueError as e: log.error( 'Impossible to create a cedar object with "madrigal.cedar.MadrigalCedarFile"', 'MADWriter') return return 1
def cmdlist(nextcommand): if nextcommand is None: log.error('Missing argument, available arguments: procs, operations', '') elif nextcommand == 'procs': procs = getProcs() log.success( 'Current ProcessingUnits are:\n {}'.format('\n '.join(procs)), '') elif nextcommand == 'operations': operations = getOperations() log.success('Current Operations are:\n {}'.format( '\n '.join(operations)), '') else: log.error('Wrong argument', '')
def get_path(): ''' Return schainpy path ''' try: root = __file__ if os.path.islink(root): root = os.path.realpath(root) return os.path.dirname(os.path.abspath(root)) except: log.error('I am sorry, but something is wrong... __file__ not found')
def search(nextcommand): if nextcommand is None: log.error('There is no Operation/ProcessingUnit to search', '') else: try: args = getArgs(nextcommand) doc = getDoc(nextcommand) log.success('{}\n{}\n\nparameters:\n {}'.format( nextcommand, doc, ', '.join(args)), '' ) except Exception as e: log.error('Module `{}` does not exists'.format(nextcommand), '') allModules = getAll() similar = [t[0] for t in process.extract(nextcommand, allModules, limit=12) if t[1]>80] log.success('Possible modules are: {}'.format(', '.join(similar)), '')
def generate(): inputs = basicInputs() if inputs['multiprocess'] == 1: current = templates.voltage.format(**inputs) elif inputs['multiprocess'] == 2: current = templates.spectra.format(**inputs) elif inputs['multiprocess'] == 3: current = templates.voltagespectra.format(**inputs) scriptname = '{}_{}.py'.format(PREFIX, inputs['name']) script = open(scriptname, 'w') try: script.write(current) log.success('Script {} generated'.format(scriptname)) except Exception as e: log.error('I cannot create the file. Do you have writing permissions?')
def run(self): while True: dataOut = self.queue.get() if not dataOut.error: try: BaseClass.run(self, dataOut, **self.kwargs) except: err = traceback.format_exc() log.error(err, self.name) else: break self.close()
def getData(self): ''' Storing data from databuffer to dataOut object ''' if self.flagNoMoreFiles: self.dataOut.flagNoData = True log.error('No file left to process', 'PXReader') return 0 if not self.readNextFile(): self.dataOut.flagNoData = True return 0 self.set_output() return 1
def verifyFile(self, filename): flag = True try: fp = open(filename, 'rb') except IOError: log.error("File {} can't be opened".format(filename), self.name) return False if self.online and self.waitDataBlock(0): pass basicHeaderObj = BasicHeader(LOCALTIME) systemHeaderObj = SystemHeader() radarControllerHeaderObj = RadarControllerHeader() processingHeaderObj = ProcessingHeader() if not (basicHeaderObj.read(fp)): flag = False if not (systemHeaderObj.read(fp)): flag = False if not (radarControllerHeaderObj.read(fp)): flag = False if not (processingHeaderObj.read(fp)): flag = False if not self.online: dt1 = basicHeaderObj.datatime pos = self.fileSize - processingHeaderObj.blockSize - 24 if pos < 0: flag = False log.error('Invalid size for file: {}'.format(self.filename), self.name) else: fp.seek(pos) if not (basicHeaderObj.read(fp)): flag = False dt2 = basicHeaderObj.datatime if not self.isDateTimeInRange(dt1, self.startDate, self.endDate, self.startTime, self.endTime) and not \ self.isDateTimeInRange(dt2, self.startDate, self.endDate, self.startTime, self.endTime): flag = False fp.close() return flag
def main(command, nextcommand, version): """COMMAND LINE INTERFACE FOR SIGNAL CHAIN - JICAMARCA RADIO OBSERVATORY V3.0\n Available commands:\n xml: runs a schain XML generated file\n run: runs any python script'\n generate: generates a template schain script\n list: return a list of available procs and operations\n search: return avilable operations, procs or arguments of the given operation/proc\n""" if command == 'xml': runFromXML(nextcommand) elif command == 'generate': generate() elif command == 'test': test() elif command == 'run': runschain(nextcommand) elif command == 'search': search(nextcommand) elif command == 'list': cmdlist(nextcommand) else: log.error('Command {} is not defined'.format(command))
def runschain(nextcommand): if nextcommand is None: currentfiles = glob.glob('./{}_*.py'.format(PREFIX)) numberfiles = len(currentfiles) if numberfiles > 1: log.error('There is more than one file to run') elif numberfiles == 1: subprocess.call(['python ' + currentfiles[0]], shell=True) else: log.error('There is no file to run') else: try: subprocess.call(['python ' + nextcommand], shell=True) except Exception as e: log.error("I cannot run the file. Does it exists?")
def sendEmail(self, email_from, email_to, subject='Error running ...', message="", subtitle="", filename="", html_format=True): if not email_to: return 0 if not self.__emailServer: return 0 log.success('Sending email to {}...'.format(email_to), 'System') msg = MIMEMultipart() msg['Subject'] = subject msg['From'] = "SChain API (v{}) <{}>".format(schainpy.__version__, email_from) msg['Reply-to'] = email_from msg['To'] = email_to # That is what u see if dont have an email reader: msg.preamble = 'SChainPy' if html_format: message = "<h1> %s </h1>" %subject + "<h3>" + subtitle.replace("\n", "</h3><h3>\n") + "</h3>" + message.replace("\n", "<br>\n") message = "<html>\n" + message + '</html>' # This is the textual part: part = MIMEText(message, "html") else: message = subject + "\n" + subtitle + "\n" + message part = MIMEText(message) msg.attach(part) if filename and os.path.isfile(filename): # This is the binary part(The Attachment): part = MIMEApplication(open(filename,"rb").read()) part.add_header('Content-Disposition', 'attachment', filename=os.path.basename(filename)) msg.attach(part) # Create an instance in SMTP server try: smtp = smtplib.SMTP(self.__emailServer) except: log.error('Could not connect to server {}'.format(self.__emailServer), 'System') return 0 # Start the server: # smtp.ehlo() if self.__emailPass: smtp.login(self.__emailFromAddress, self.__emailPass) # Send the email try: smtp.sendmail(msg['From'], msg['To'], msg.as_string()) except: log.error('Could not send the email to {}'.format(msg['To']), 'System') smtp.quit() return 0 smtp.quit() log.success('Email sent ', 'System') return 1
def _monitor(self, queue, ctx): import socket procs = 0 err_msg = '' while True: msg = queue.get() if '#_start_#' in msg: procs += 1 elif '#_end_#' in msg: procs -= 1 else: err_msg = msg if procs == 0 or 'Traceback' in err_msg: break time.sleep(0.1) if '|' in err_msg: name, err = err_msg.split('|') if 'SchainWarning' in err: log.warning( err.split('SchainWarning:')[-1].split('\n')[0].strip(), name) elif 'SchainError' in err: log.error( err.split('SchainError:')[-1].split('\n')[0].strip(), name) else: log.error(err, name) else: name, err = self.name, err_msg time.sleep(1) ctx.term() message = ''.join(err) if err_msg: subject = 'SChain v%s: Error running %s\n' % (schainpy.__version__, self.name) subtitle = 'Hostname: %s\n' % socket.gethostbyname( socket.gethostname()) subtitle += 'Working directory: %s\n' % os.path.abspath('./') subtitle += 'Configuration file: %s\n' % self.filename subtitle += 'Time: %s\n' % str(datetime.datetime.now()) readUnitConfObj = self.getReadUnit() if readUnitConfObj: subtitle += '\nInput parameters:\n' subtitle += '[Data path = %s]\n' % readUnitConfObj.parameters[ 'path'] subtitle += '[Start date = %s]\n' % readUnitConfObj.parameters[ 'startDate'] subtitle += '[End date = %s]\n' % readUnitConfObj.parameters[ 'endDate'] subtitle += '[Start time = %s]\n' % readUnitConfObj.parameters[ 'startTime'] subtitle += '[End time = %s]\n' % readUnitConfObj.parameters[ 'endTime'] a = Alarm(modes=self.alarm, email=self.email, message=message, subject=subject, subtitle=subtitle, filename=self.filename) a.start()