예제 #1
0
 def __iter__(self):
     t1 = now()
     report('starting query: ')
     q = self.query
     t2 = now()
     report('finished query ({0}s)'.format(t2-t1))
     return q
예제 #2
0
 def __init__(self, path=None, settings=None,
              items=[], fill=None,
              force=False, wrap_exit=True,
              **kargs):
     """ fill+path determine self.query """
     if wrap_exit:
         self.__call__ = wrap_kbi(self.__call__)
     settings._engine.get_server() # HACK: ensure we're initialized
     self.record = defaultdict(lambda: 0)
     if self.requires_path:
        if not path or not ope(path):
            raise FileDoesntExist(str(path))
     self.path = path and abspath(path)
     self.conf = settings
     self.force = force
     if fill:
         if path is not None:
             raise SystemExit('if you use --fill you cant '
                              'use a path (and vice versa)')
     self.fill = fill
     if items:
         report('instantiated {0} with size {1} item-list'.format(
             self,len(items)))
         self.__iter__ = lambda himself: ([i.id, i] for i in items)
     self.record_invocation()
예제 #3
0
    def sgEmail(self):
        log.sgLog('发送测试结果\n')
        testDate = report().testDate()
        filename = report().fileName()

        msg = MIMEMultipart()
        html = open(filename, 'rb').read()
        part1 = MIMEText(html, 'html', 'utf-8')
        msg.attach(part1)

        msg['to'] = 'email address'
        msg['from'] = 'email address'
        msg['subject'] = '前台自动化测试报告' + testDate

        try:
            server = smtplib.SMTP()
            server.connect('smtp.gmail.com:587')
            server.ehlo()
            server.starttls()
            server.ehlo()
            server.login('username', 'password')
            server.sendmail(msg['from'], msg['to'], msg.as_string())
            server.quit()
            print '发送成功'
        except Exception, e:
            print str(e)
예제 #4
0
파일: server.py 프로젝트: sjmf/reportgen
def report_worker(sid):
    try:
        job = get_job(sid) 

        log.info("=============  STARTING WORKER  ==============")
        log.debug(job)
        from ast import literal_eval
        job['series'] = literal_eval(job['series'])  # From string
        # Expand paths to full location on filesystem 
        output_filename = os.path.join(
            app.config['UPLOAD_FOLDER'], 
            next(tempfile._get_candidate_names()) + '.pdf')

        # Make list of input datafiles
        input_datafiles = [
            os.path.join(app.config['UPLOAD_FOLDER'], f['temporary_name'])
            for f in get_files(sid)
        ]

        report.report(input_datafiles, output_filename, 
                      **{**job, 'pdf': True, 'htm': False})

        log.info("=============  WORKER FINISHED  ==============")

        # Update finished job 
        upd_job(sid, 'generated_pdf', output_filename)
        upd_job(sid, 'status', 'done')

    except Exception as e:
        log.error("Exception occurred in worker thread")
        log.error(sys.exc_info()[0])

        upd_job(sid, 'status', 'error')
        upd_job(sid, 'generated_pdf', None)
        raise e
예제 #5
0
파일: raven.py 프로젝트: pyne/magic-cones
def send(trans):
    """Sends an email to the list."""
    me = '*****@*****.**'
    you = '*****@*****.**'
    today = datetime.date.today()

    # Create message container - the correct MIME type is multipart/alternative.
    msg = MIMEMultipart('alternative')
    msg['Subject'] = 'Magic Cones Report {0}'.format(today)
    msg['From'] = me
    msg['To'] = you

    # create content
    text = TEXT.format(report=report(trans), rules=rules())
    html = HTML.format(report=report(trans, html=True), rules=rules(html=True))
    part1 = MIMEText(text, 'plain')
    part2 = MIMEText(html, 'html')
    msg.attach(part1)
    msg.attach(part2)

    # Send the message
    pw = passwd(me)
    s = smtplib.SMTP('smtp.gmail.com', 587)
    #s.ehlo()
    s.starttls()
    #s.ehlo()
    s.login(me, pw)
    s.sendmail(me, you, msg.as_string())
    s.quit()
예제 #6
0
    def callback(self, item=None, **kargs):
        report(item._id)

        if item._id not in self.collisions['fname']:
            self.seek_fname_collision(item)
        if item._id not in self.collisions['md5']:
            self.seek_md5_collision(item)
예제 #7
0
def run( d , dname, full ):

	partitionTestResult = TestResult()
	if full == True:
		partitionTestResult.set_total_points(4)
	else:
		partitionTestResult.set_total_points(1)
	partitionScore = 0
	print("Validating that {} has a separate partition...".format(d))
	try:

		#Input:
		#>>> mount | grep `d`
		#Expected output:
		#>>> tmpfs on `d` type tmpfs (rw,nosuid,nodev,noexec,relatime)

		fsTest1 = subprocess.Popen(('mount'), stdout=subprocess.PIPE)
		try:
			fstTest1Output = subprocess.check_output(('grep', d), stdin=fsTest1.stdout)
			partitionScore += 1
			print("......Passed!")
			if full == True:
				partitionScore += output_verification(fstTest1Output, d, dname)
			print partitionScore
		except subprocess.CalledProcessError as e:
			report.report("(X)...{} does not exist in a separate partition.".format(d))
			mit(d, dname)
			

	except OSError:
		report.report("(!)...Tools do not support the use of the mount command.".format(fs))


	partitionTestResult.set_points(partitionScore)
	return partitionTestResult
예제 #8
0
 def delete_file(self, key=None, item=None):
     report('deleting file')
     if key and item:
         self.record['errors'] += 1
         self.record['error'] = 'cant pass key and item to delete_file.'
         return
     if not (key or item):
         self.record['errors'] += 1
         self.record['error'] = 'need either key or item'
         return
     if not key:
         if item is None:
             self.record['errors'] += 1
             self.record['error'] = 'item is none'
             return
         key = item and item.id
     if not key:
         self.record['errors'] += 1
         self.record['error'] = 'item is none'
     report('deleting file',key)
     if not ope(key):
         self.record['errors'] += 1
         self.record['error']='file does not exist.'
     os.remove(key) # TODO: use unipath
     self.record['files_deleted'] += 1
     self.delete_record(key)
예제 #9
0
def update_db(db, _id, dct, schema=None):
    """  stupid.. have to delete and restore instead of update? """

    if not schema:
        report('SCHEMA NOT PROVIDED!!!!!!!')
        report('updating db',[db, _id, dct])
        doc = db[_id]
        report('before',doc.items())

        for x in dct:
            doc[x] = dct[x]

        # TODO: use db.update(doc) ?
        #db[doc.id] = doc

        report('after', doc)
        report('updated "{id}" with new values for keys'.format(id=_id), dct.keys())
    else:
        doc = schema.load(db, _id)
        for x in dct:
            val = dct[x]
            fieldtype = getattr(schema, x).__class__
            if fieldtype==ListField:
                val = demjson.decode(val)
            elif fieldtype==StringField:
                pass
            else:
                raise Exception, 'NIY:'+str(fieldtype)
            setattr(doc, x, val)
        doc.store(db)
예제 #10
0
 def mapData(self):
   # pracujeme s: http://sigma.nkp.cz/F/?func=file&file_name=find-b&local_base=ADR
   report("INFO: SiglaMapper.mapData")
   siglaURI = False
   sigla = self.doc.getXPath("//varfield[@id='040']/subfield[@label='a']")
   if not sigla == []:
     sigla = sigla[0]
     # Check whether the sigla is already cached.
     if sigla in cache.siglaCache.keys():
       siglaURI = cache.siglaCache[sigla]
     else:
       doc = self.searchAlephBase("http://sigma.nkp.cz", "ADR", "SIG", sigla)
       if doc:
         xpath = "present/record/doc_number"
         docNum = doc.getXPath(xpath)
         if not docNum == []:
           docNum = docNum[0].lstrip("0")
           siglaURI = "http://sigma.nkp.cz/X?op=doc-num&base=ADR&doc-num=" + docNum
           cache.siglaCache[sigla] = siglaURI
   if siglaURI:
     return [(
       self.representationURI,
       rdflibWrapper.namespaces["dc"]["creator"], 
       rdflib.URIRef(siglaURI)
     )]
   else:
     return False
예제 #11
0
 def delete_all(self,really=False):
     if not really:
         report('seriously?  well ok, but pass `really=True`')
     else:
         for x in self:
             report('deleting: '+str(x))
             del self[x]
예제 #12
0
def partition(d, dname):

    partitionTestResult = TestResult()
    partitionTestResult.set_total_points(1)
    partitionScore = 0
    print("Validating that {} has a separate partition...".format(d))
    try:
        fsTest1 = subprocess.Popen(('mount'), stdout=subprocess.PIPE)
        try:
            fstTest1Output = subprocess.check_output(('grep', d),
                                                     stdin=fsTest1.stdout)
            partitionScore += 1
            print("......Passed!")
        except subprocess.CalledProcessError as e:
            report.report(
                "(X)...{} does not exist in a separate partition.".format(d))
            report.mitigation(
                "      Mitigation: run systemctl unmask {}.mount".format(
                    dname))
            report.mitigation(
                "                      systemctl enable {}.mount".format(
                    dname))
            print("......Failed!")

    except OSError:
        report.report(
            "(!)...Tools do not support the use of the mount command.".format(
                fs))

    partitionTestResult.set_points(partitionScore)
    return partitionTestResult
예제 #13
0
파일: test.py 프로젝트: sjmf/reportgen
def report_test():
    os.makedirs(TEST_PATH, exist_ok=True)
    print_memory('pre')

    for i, line in enumerate(DATAFILES):

        objgraph.show_growth(limit=3)

        input_files, location = line[0], line[1]
        report_name = TEST_PATH + '/test_' + str(i) + '.pdf'

        log.info(input_files)
        log.info(location)
        log.info(report_name)

        try:
            report.report(input_files, report_name, location=location)

            log.info("Report saved to {}".format(report_name))

        except Exception as e:
            log.exception("Exception occurred when reporting (test failed):")

        print_memory(i)

    plot_object_counts()
예제 #14
0
def main():
    # # # Generate instances
    # instances = instance_helper.generate_instances(config.n, config.m, config.q, config.num_instances, loc=config.loc)
    # instance_helper.convert_instances_pickle_to_mat(loc=config.loc, num_instances=config.num_instances)

    # # # Optimization
    # load instances
    instances = instance_helper.load_instances(
        loc=config.loc, num_instances=config.num_instances)
    # calculate honest scores, if the honest file does not exist
    honestScore(instances)

    print(f'loaded {len(instances)} instances')
    # MMM
    alg_greedy.run_MMM(instances, loc=config.loc)  # MMM
    # CGS
    alg_cyclic.loadCGS(instances)  # load CGS
    # MMM-CGS
    alg_cyclicgreedy.run_MMM_CGS(
        instances, loc=config.loc,
        cycloc=f'{config.loc}/CGS_assignments')  # MMM-CGS
    # ILP, usually takes a long time for large scale optimization
    #    opt_ilp.run_ilp(instances, loc=config.loc) # ILP

    # report with ilp
    #    report.report(loc=config.loc, algs=['random', 'MMM', 'CGS', 'MMMCGS', 'ilp'])
    # report without ilpS
    report.report(loc=config.loc, algs=['random', 'MMM', 'CGS', 'MMMCGS'])
예제 #15
0
def main():
    while True:
        print_menu()
        choice = int(input("Enter your choice [1-6]:"))
        if choice == 1:
            clear()
            add()
        elif choice == 2:
            clear()
            list_all()
        elif choice == 3:
            clear()
            report()
        elif choice == 4:
            clear()
            update()
        elif choice == 5:
            clear()
            delete()
        elif choice == 6:
            print("Exit")
            conn.close()
            exit(1)
            break
        else:
            clear()
            print("Wrong option selection.")
예제 #16
0
 def record_collision(self, reason, item_ids, item=None):
     self.collisions[reason] += item_ids
     item_ids = sorted(item_ids)
     event = Event(reason=reason, item_ids=item_ids,
                   details=dict(md5=item.md5))
     event.store(self.events_db)
     report(' - by {0}: found {1} events'.format(
         reason, len(item_ids)))
예제 #17
0
 def getParsedDoc(self, url):
   """Na zadané URL nebo urllib2.Request vrátí naparsovaný XML dokument.""" 
   report("INFO: Mapper.getParsedDoc on URL %s" % (url))
   result = urllib2.urlopen(url)
   doc = result.read()
   result.close()
   doc = Record(libxml2.parseDoc(doc))
   return doc
예제 #18
0
 def validateURI(self, uri):
   """Zjišťuje, zdali je zadané URI dostupné."""
   report("INFO: validating URI %s" % (uri))
   try:
     urllib2.urlopen(urllib2.Request(uri, None, {"Accept" : "application/rdf+xml"})) # Validation on marccodes.heroku.com doesn't work without the Accept HTTP header
     return True
   except urllib2.HTTPError:
     return False
def show(project):
    build(project)
    proot = opj(src_root, project)
    report("serving "+proot)
    def f():
        webbrowser.open(URL)
    threading.Thread(target=f).start()
    local("cd {0} && poole --serve".format(proot))
예제 #20
0
 def run_and_collect(self, cmd):
     """ for gathering the output from file(1) and md5(1) etc """
     cmd = cmd.replace('`','\`')
     try:
         return os.popen(cmd).read().strip()
     except IOError,e:
         report("IOError: " + str(e))
         return None
예제 #21
0
def get_or_create(DB_NAME):
    from ixle.settings import Settings
    server = Settings().server
    if DB_NAME not in server:
        report("creating database {0} on {1} ".format(DB_NAME, server))
        server.create(DB_NAME)
    db = server[DB_NAME]
    return db
예제 #22
0
def call_agent_on_dir(agent_nick, dirname):
    kls = get_agent_by_name(agent_nick)
    agent_obj = kls(path=dirname, settings=conf())
    result = agent_obj()
    if result is None:
        report('WARNING: got None-result from agent, '
               'should have been self.record.')
        result = {}
    return agent_obj, result
예제 #23
0
def reportProjectData(confFile,globalConfFile):
#java -Xmx2024m  -cp "C:\Program Files\Weka-3-7\weka.jar" weka.core.Instances 13_Appended.arff
    vers, gitPath,bugsPath, workingDir = utilsConf.configure(confFile)
    versPath, dbadd= Mkdirs(workingDir, vers)
    LocalGitPath=os.path.join(workingDir,"repo")
    reportCsv=os.path.join(workingDir,"report.csv")
    lastVer=os.path.join(dbadd,vers[-1]+".db")
    testsDB=os.path.join(workingDir,"testsBugsMethods.db")
    report.report(reportCsv,LocalGitPath,lastVer,testsDB)
예제 #24
0
 def collapse(self):
     assert self.unipath.isdir()
     parent = self.unipath.parent
     for path in self.unipath.listdir():
         subitem = Item.objects.get(path=path)
         fname = subitem.fname
         new_path = opj(parent, fname)
         report("moving: ", path, new_path)
         subitem._move(new_path)
예제 #25
0
 def seek_md5_collision(self, item):
     if not item.md5:
         report(' - md5 not set, calling subagent');
         self.md5er.callback(item)
     reason = 'md5'
     results = self.find_matches(item, 'md5')
     if not len(results): return
     item_ids = [row.value['_id'] for row in results] + [item._id]
     self.record_collision(reason, item_ids, item)
예제 #26
0
def item2text(item):
    report('reading file')
    if ope(item.abspath):
        with open(item.abspath, 'r') as fhandle:
            if item.fext in 'txt'.split():
                return fhandle
            elif item.fext=='pdf':
                import StringIO
                body = StringIO.StringIO(getPDFContent(item.abspath))
                return body
 def insertSKOSRelations(self, terms, predicate):
     if not terms == []:
         for term in terms:
             try:
                 termTranslated = self.pshTranslateDict[term]
                 self.results.append(
                     (self.resourceURI, rdflibWrapper.namespaces["skos"][predicate], rdflib.URIRef(termTranslated))
                 )
             except KeyError:
                 report("[ERROR] term %s doesn't have a translation." % (term))
예제 #28
0
 def finish_displayhook(self):
     """Finish up all displayhook activities."""
     try:
         super(SmashDisplayHook, self).finish_displayhook()
     except AttributeError, e:
         # occasionally throws
         # IOStream instance has no attribute 'flush'
         #
         # I think this is a race condition on embedded shells
         report(str(e))
예제 #29
0
 def main(self):
     abspath = self['_']
     if abspath and ope(abspath):
         return '<font style="color:green;">exists ✓</font>'
     msg = '(this file is not available.  is the drive mounted?)'
     report(msg.replace('this file','"{0}"'.format(abspath)))
     return ''.join([
         '<small><font style="color:red;margin-left:15px;">',
         "<img src=/static/img/icons/red-x.png width=15px>"+msg,
         '</font></small>'])
예제 #30
0
 def record_invocation(self):
     from ixle.schema import Event
     report("writing event for my birthday")
     e = Event(
         #reason="birthday::"+self.__class__.__name__
         reason = "birthday",
         details = dict(
             agent=self.__class__.__name__,
             path=self.path)
         )
     e.save()
예제 #31
0
 def __call__(self):
     report('moving record', self)
     old_doc = Item.load(self.db, self.old_key)
     new_doc = Item.load(self.db, self.new_key) or Item()
     for attr_name in old_doc._fields:
         setattr(new_doc, attr_name, getattr(old_doc, attr_name))
     new_doc._id = self.new_key
     new_doc.fname = new_doc.unipath.name
     report('saving record', new_doc)
     new_doc.store(self.db)
     return new_doc
예제 #32
0
 def mapExtractedValues(self, predicate, valueDict, extractedValues):
   report("INFO: Fixfield008Mapper.mapExtractedValues")
   for extractedValue in extractedValues:
     try:
       self.results.append((
         self.representationURI,
         predicate, 
         valueDict[extractedValue]
       ))
     except KeyError:
       report("[ERROR] Fixfield008Mapper.mapExtractedValues KeyError for key %s" % (extractedValue))
예제 #33
0
 def is_blacklisted(self, key):
     from ixle.schema import DSetting
     blacklist = getattr(self, '_blacklist_cache', None)
     if blacklist is None:
         report('no cache found.. recomputing blacklist setting')
         # ?TODO: move this type of thing to DSetting.initialize()
         blacklist_setting, created = DSetting.objects.get_or_create(
             name='file_name_blacklist',
             defaults = dict(value='[]'))
         self._blacklist_cache = blacklist_setting.decode()
         return self.is_blacklisted(key)
     return ops(key)[-1] in blacklist
예제 #34
0
 def mount_dir(self):
     from ixle import settings
     try:
         mounts_dir = settings.Settings()['ixle']['mount_dir']
     except KeyError:
         raise Exception, 'settings[ixle][mount_dir] is not defined'
     mountpoint = opj(expanduser(mounts_dir), self.nickname)
     mountpoint = unipath.FSPath(mountpoint)
     if not mountpoint.exists():
         mountpoint.mkdir(parents=True)
         report("mountpoint did not exist.  created it: "+mountpoint)
     return mountpoint
예제 #35
0
 def query(self):
     report.console.draw_line()
     if self._query_override() is not None:
         q = self._query_override()
     elif self.path:
         q = self._query_from_path()
     elif self.fill:
         q = self._query_from_fill()
     else:
         q = Item.objects.all()
         report("chose query: (everything)")
     return q
예제 #36
0
 def delete_dir(self, path):
     ctx = self.get_ctx()
     _from = ctx['_from']
     assert ctx['num_fs_files']==0, 'can only remove empty dirs currently'
     if _from in 'both db'.split():
         for item in ctx['items']:
             report("removing from db: ",item)
             item.delete()
     if _from in 'both fs'.split():
         import os
         report('removing (empty) dir',self['_'])
         os.rmdir(self['_'])
예제 #37
0
def main():
    parser = argparse.ArgumentParser(description='Machine Learning')
    parser.add_argument('-s', '--seed', type=int, default=2020,
                        metavar='S', help='random seed (default: 2020)')
    parser.add_argument('--test', action='store_true',
                        default=False, help='training or testing mode')
    args = parser.parse_args()
    logging.basicConfig(format='%(asctime)s %(message)s', datefmt='%H:%M:%S.')
    logger = logging.getLogger()
    logger.setLevel(logging.INFO)

    y_true = dict()
    y_pred = dict()
    names = ["DT_Entropy", "DT_Gini", "RF_Entropy", "RF_Gini"]
    classifiers = [
        DecisionTreeClassifier(random_state=args.seed, criterion='entropy'),
        DecisionTreeClassifier(random_state=args.seed, criterion='gini'),
        RandomForestClassifier(random_state=args.seed, n_estimators=10,
                               n_jobs=-1, criterion='entropy'),
        RandomForestClassifier(random_state=args.seed, n_estimators=10,
                               n_jobs=-1, criterion='gini'),
    ]
    hyperparam = [
        {'splitter': ['best', 'random'],
            'max_features': ['sqrt', 'log2', None]},
        {'splitter': ['best', 'random'],
            'max_features': ['sqrt', 'log2', None]},
        {},
        {},
    ]
    colors = ['blue', 'orange', 'green', 'red',
              'purple', 'brown', 'pink', 'gray']

    X_train, y_train, X_test, y_test = prepare_data(args.seed)
    logger.info(str(X_train.shape) + ' ' + str(X_test.shape))

    for name, est, hyper in zip(names, classifiers, hyperparam):
        logger.info(name + '...')
        if not args.test:
            clf = GridSearchCV(est, hyper, cv=5, n_jobs=-1)
            clf.fit(X_train, y_train)
            y_true[name],  y_pred[name] = y_test, clf.predict(X_test)
            logger.info('____Accuracy: %0.4f' %
                        metrics.accuracy_score(y_true[name], y_pred[name]))
            print(clf.best_estimator_)
            pickle.dump(clf, open('model/kdd99_' + str(name) + '.sav', 'wb'))
        else:
            clf = pickle.load(open('model/kdd99_' + str(name) + '.sav', 'rb'))
            y_true[name], y_pred[name] = y_test, clf.predict(X_test)

    report('kdd99', names, y_true, y_pred)
    draw_roc('kdd99', names, colors, y_true, y_pred)
예제 #38
0
def search(filename, initial_i, initial_j, goal_i, goal_j):
    '''    
    <filename> is the file that contains the enviornment.
    the rest of the arguments are the initial and goal coordinates.
    
    Returns a list of tuples. Each tuple is a pair of integers.
    The first tuple represents the (i, j) coordinate of the start
    state, and the last tuple represents the goal coordinates. The
    tuple in between are coordinates in order that represents an
    optimal path from the start state to the goal.
    '''

    # If path was not already computed, compute
    retList = remember.getPath((initial_i, initial_j), (goal_i, goal_j))
    if not retList:

        # Define the start and goal node
        startState = world.State(initial_i, initial_j)
        goalState = world.State(goal_i, goal_j)
        startNode = astar.Node(startState, config.heuristicWeight)
        goalNode = astar.Node(goalState, config.heuristicWeight)
        startNode.heuristicFucntion = goalNode.heuristicFucntion = config.heuristicFunction(
            goalNode)

        # Get environment from file
        environment = readMaze(filename)

        # Perform A* search
        foundGoal = astar.aStar(startNode, goalNode, environment,
                                config.heuristicWeight)

        # Construct the path
        retList = []
        while foundGoal:
            fromAction = foundGoal.fromAction
            i = foundGoal.state.i
            j = foundGoal.state.j
            retList.append((i, j))
            if fromAction:
                foundGoal = fromAction.fromNode
            else:
                foundGoal = None
        retList.reverse()

        # Save the path
        report.report(filename, retList)
        remember.sendPath(retList)

    return retList
예제 #39
0
파일: job.py 프로젝트: zQuantz/OscraP
def main():

	logger.info(f"SCRAPER,JOB,INITIATED,{DATE},")

	init()

	tickers = _connector.get_equity_tickers(N_USD)
	checkpoint = len(tickers) / BATCH_SIZE
	checkpoint = int(checkpoint / 4)

	faults_summary = {
		"options" : {},
		"analysis" : {},
		"keystats" : {},
		"ohlc" : {}
	}

	db_flags, db_stats = [], []

	###############################################################################################

	for batch_id, batch in enumerate(range(BATCH_SIZE, len(tickers) + BATCH_SIZE, BATCH_SIZE)):

		ticker_batch = tickers[batch - BATCH_SIZE : batch]

		results = batch_main(batch_id, ticker_batch)
		b_fault_summary, b_db_flag, b_db_stats = results

		for key in b_fault_summary:
			for ticker in b_fault_summary[key]:
				faults_summary[key][ticker] = b_fault_summary[key][ticker]

		db_flags.append(b_db_flag)
		db_stats.append(b_db_stats)

		success, failure = get_job_success_rates(tickers[ : BATCH_SIZE * (1 + batch_id)])
		send_metrics(success, failure)

		# if batch_id % checkpoint == 0 and batch_id != 0:
		# 	report("Partial", success, failure, faults_summary, db_flags, db_stats)

	###############################################################################################

	success, failure = get_job_success_rates(tickers)
	report("Full", success, failure, faults_summary, db_flags, db_stats)

	store()

	logger.info(f"SCRAPER,JOB,TERMINATED,{DATE},")
예제 #40
0
def main():
    aReport = report.report()
    aReport.execute(reportName="0VerView",
                    table="0VerView",
                    excelFile="./0VerView.xlsx")
    aReport.execute(reportName="2019v1Member",
                    table="2019v1Member",
                    excelFile="./2019v1Member.xlsx")
    aReport.execute(reportName="2019v1MemberClass",
                    table="2019v1MemberClass",
                    excelFile="./2019v1MemberClass.xlsx")
    aReport.execute(reportName="2019v1MemberClassCount",
                    table="2019v1MemberClassCount",
                    excelFile="./2019v1MemberClassCount.xlsx")
    aReport.execute(reportName="2019v1RegisteredWaitingPayment",
                    table="2019v1RegisteredWaitingPayment",
                    excelFile="./2019v1RegisteredWaitingPayment.xlsx")
    aReport.execute(reportName="2019v1UserClassSelection",
                    table="2019v1UserClassSelection",
                    excelFile="./2019v1UserClassSelection.xlsx")
    aReport.execute(reportName="2019v1UserClassCount",
                    table="2019v1UserClassCount",
                    excelFile="./2019v1UserClassCount.xlsx")
    aReport.execute(reportName="2019v1AllClassSelection",
                    table="2019v1AllClassSelection",
                    excelFile="./2019v1AllClassSelection.xlsx")

    aReport.execute(reportName="v1Donations",
                    table="v1Donations",
                    excelFile="./v1Donations.xlsx")
예제 #41
0
def run():
    logger = logging.getLogger(__name__)
    messages = []
    alarm = 0
    terminating = 0
    time.time()
    for x in range(0, 8):
        running_time = datetime.datetime.now(
            tz=pytz.timezone('Australia/Sydney')).strftime('%Y-%m-%d %H:%M')
        now = time.time() - 36000
        record = worker_config.get_record(str(x))
        if record is not 0:
            last_update_time_diff = int(now) - record.lastUpdateTime
            last_index = record.lastIndex
            r_object = report(x, last_update_time_diff, last_index,
                              record.isActive, True)
            if record.isActive:
                if (last_update_time_diff > 125):
                    r_object.is_healthy = False
                    alarm = 1
                    if (last_update_time_diff > 300):
                        terminating = 1
                        r_object.terminating = True
                        logger.info("Terminating work:{}".format(x))
                        port = cluster.delete_tasks(str(x))
                        r_object.port = port
            logging.info("{}\t{}\t{}\t{}\t{}\t{}".format(
                x, last_update_time_diff, last_index, r_object.is_active,
                r_object.is_healthy, r_object.terminating))
            messages.append(r_object)


# if(alarm==1):
    email_handler.process_item(messages, running_time, alarm, terminating)
예제 #42
0
def param_init(log_filename):

    global bio_ner_train_data, bio_ner_test_data, bio_ner_model, bio_ner_report

    bio_ner_train_data = data(
        RAW_DATA_PATH, PROCESSED_DATA_PATH, WORD_VEC_FILE, CHAR_VEC_FILE,
        POS_VEC_FILE, ADD_FEAT_VEC_FILE, CLASS_VEC_FILE,
        ADD_FEAT_FILE_SUBSTRING, CHAR_ONE_HOT_FILE_SUBSTRING,
        WORD_CONTEXT_LENGTH, NB_UNIQUE_CHARS, CHAR_VECTOR_SIZE,
        WORD_VECTOR_SIZE, NB_CLASSES, MAX_WORD_LENGTH, ADD_FEAT_VEC_SIZE,
        LIMITED_ADD_FEAT_VEC_SIZE, POS_TAG_VECTOR_SIZE)

    bio_ner_test_data = data(
        RAW_DATA_PATH, PROCESSED_DATA_PATH, TEST_word_vec_file_substring,
        CHAR_VEC_FILE, POS_VEC_FILE, ADD_FEAT_VEC_FILE,
        TEST_entity_vec_file_substring, TEST_add_feat_vec_file_substring,
        TEST_char_vec_file_substring, WORD_CONTEXT_LENGTH, NB_UNIQUE_CHARS,
        CHAR_VECTOR_SIZE, WORD_VECTOR_SIZE, NB_CLASSES, MAX_WORD_LENGTH,
        ADD_FEAT_VEC_SIZE, LIMITED_ADD_FEAT_VEC_SIZE, POS_TAG_VECTOR_SIZE)

    bio_ner_model = model(WORD_VECTOR_SIZE, WORD_CONTEXT_LENGTH,
                          NB_UNIQUE_CHARS, CHAR_VECTOR_SIZE, MAX_WORD_LENGTH,
                          EMBEDDING_OP_DIM, MAX_FEATURES, NB_CLASSES,
                          CHAR_FEATURE_OUTPUT, HIDDEN_SIZE, ADD_FEAT_VEC_SIZE,
                          POS_TAG_VECTOR_SIZE)

    bio_ner_report = report(REPORT_PATH, log_filename)
예제 #43
0
def testeAssembly(jar, testDir, nasmDir, hackDir, gui, verbose):

    pwd = os.path.dirname(os.path.abspath(__file__))

    # global path
    os.path.abspath(nasm)
    os.path.abspath(hack)

    print("==== Assembling Files ==========================")
    clearbin(hack)

    errAssembler, logAssembler = assembler(jar, nasm, hack, True)

    if errAssembler == 0:
        print("==== Simulating ================================")
        clearTestDir(testDir)
        if simulateFromTestDir(testDir, hackDir, gui, verbose) < 0:
            sys.exit(1)

        # testAssembling files
        print("==== Testando ==================================")
        error, log = compareFromTestDir(testDir)

        if error < -1:
            sys.exit(-1)

    # report error
    print("==== Reporting results =========================")
    r = report(log, 'H')
    error = r.assemblyTeste(log)
    r.send()
예제 #44
0
    def run(self):

        if not len(self.queue):
            return 1

        filenames = [q.filename for q in self.queue]
        results = report(filenames)

        for item in self.queue:

            logging.info('\n  Optimizing: ' + str(item))

            item.createBackup()

            try:
                for tool in self.getOptimizer(item).tools:
                    logging.info('    Running ' + tool.__name__)
                    tool(item).execute()

            except:
                self.failure = 1
                logging.error('  Error while compressing \'' + str(item) +
                              '\'')

            else:
                item.removeBackup()
                results.reportItem(item)

        results.reportTotals(filenames)

        return self.failure
예제 #45
0
def Reports2020():
    #aReport.execute(table = "2020Membership")
    #aReport.execute(table = "2020MembershipFromChina")
    # datetime object containing current date and time
    now = datetime.now()

    print("now =", now)
    dt_string = now.strftime("%m%d%H%M")
    print("date and time =", dt_string)

    aReport = report.report()
    #if not os.path.isdir("reports"):
    #    os.mkdir("reports")

    excel_filename = "2020全体会员." + dt_string + ".xlsx"
    aReport.execute(table="2020AllMembers", excelFile=excel_filename)
    aReport.execute(table="vwUserInfo", excelFile="./UserInfo.xlsx")
    #aReport.execute(table="2020CMemberClass")
    tables = [
        "2020CMemberClass", "2020CM春季-楷书班-九成宫", "2020CM春季-小楷班-灵飞经",
        "2020CM春季-行书班-赵孟頫", "2020CM春季-隶书班", "2020CM春季-隶书班-曹全碑",
        "2020CM春季-行草班-米芾", "2020CM春季-楷书班-赵孟頫", "2020CM春季-硬笔班", "2020CM春季-小楷班",
        "2020CM春季-篆刻班", "2020CM春季-篆书班-石鼓文", "2020CM春季-楷书班-多宝塔"
    ]

    excel_filename = "2020春季各班选课." + dt_string + ".xlsx"
    aReport.executeMulti(excelFile=excel_filename, tables=tables)
예제 #46
0
def Reports2020Fall():
    #aReport.execute(table = "2020Membership")
    #aReport.execute(table = "2020MembershipFromChina")
    # datetime object containing current date and time
    now = datetime.now()

    print("now =", now)
    dt_string = now.strftime("%m%d%H%M")
    print("date and time =", dt_string)

    aReport = report.report()
    #if not os.path.isdir("reports"):
    #    os.mkdir("reports")

    excel_filename = "2020全体会员." + dt_string + ".xlsx"
    aReport.execute(table="2020AllMembers", excelFile=excel_filename)
    aReport.execute(table="vwUserInfo", excelFile="./UserInfo.xlsx")
    #aReport.execute(table="2020FMemberClass", excelFile = "./2020FMemberClass." + dt_string + ".xlsx")

    tables = [
        "2020FMemberClass", "2020FM秋季-楷书班", "2020FM秋季-隶书班", "2020FM秋季-楷书班-赵孟頫",
        "2020FM秋季-集王圣教序", "2020FM秋季-兰亭集序", "2020FM秋季-行草班-文徵明",
        "2020FM秋季-行草班-文徵明(旁听)", "2020FM秋季-小楷班", "2020FM秋季-篆书班", "2020FM秋季-篆刻班",
        "2020FM秋季-硬笔班", "2020FM秋季-软硬班-跋保母", "2020FM秋季-章草兴趣班"
    ]

    excel_filename = "2020秋季各班选课." + dt_string + ".xlsx"
    aReport.executeMulti(excelFile=excel_filename, tables=tables)
예제 #47
0
def run():
    logger = logging.getLogger(__name__)
    messages = []
    alarm = 0
    terminating = 0
    time.time()

    print("WorkerID\tLastRun\tPMIDs\tActive\tHealthy\tTerminating")
    for x in range(0,8):
        running_time = datetime.datetime.now(tz=pytz.timezone('Australia/Sydney')).strftime('%Y-%m-%d %H:%M')
        now = time.time() -36000
        record = worker_config.get_record(str(x))
        if record is not 0:
            last_update_time_diff = int(now) - record.lastUpdateTime
            last_index = record.lastIndex
            r_object = report(x, last_update_time_diff, last_index, record.isActive, True)
            if record.isActive:
                if(last_update_time_diff>125):
                    r_object.is_healthy = False
                    alarm = 1
                    if(last_update_time_diff>300):
                        terminating =1
                        r_object.terminating = True
                        logger.info("Terminating work:{}".format(x))
                        # port = cluster.delete_tasks(str(x))
                        port = ""
                        r_object.port = port
            print("{}\t{}\t{}\t{}\t{}\t{}".format(x, last_update_time_diff, last_index,r_object.is_active, r_object.is_healthy, r_object.terminating))
예제 #48
0
def Reports2021Spring():
    #aReport.execute(table = "2020Membership")
    #aReport.execute(table = "2020MembershipFromChina")
    # datetime object containing current date and time
    now = datetime.now()

    print("now =", now)
    dt_string = now.strftime("%m%d%H%M")
    print("date and time =", dt_string)

    aReport = report.report()
    #if not os.path.isdir("reports"):
    #    os.mkdir("reports")

    excel_filename = "2021全体会员." + dt_string + ".xlsx"
    aReport.execute(table="2021AllMembers", excelFile=excel_filename)
    aReport.execute(table="vwUserInfo", excelFile="./UserInfo.xlsx")
    #return

    tables = [
        "2021CMemberClass", "2021CM春季-楷书班-九成宫", "2021CM春季-隶书班",
        "2021CM春季-楷书班-赵孟頫", "2021CM春季-行草班-二王尺牍", "2021CM春季-行草班-二王尺牍(旁听)",
        "2021CM春季-小楷班", "2021CM春季-诗词班", "2021CM春季-篆刻班", "2021CM春季-硬笔班",
        "2021CM春季-软硬班-跋保母", "2021CM春季-兰亭互助班", "2021CM春季-多宝塔互助班",
        "2021CM春季-书法与艺术研讨班"
    ]

    excel_filename = "2021春季各班选课." + dt_string + ".xlsx"
    aReport.executeMulti(excelFile=excel_filename, tables=tables)
예제 #49
0
def controlHeating():
    thermostat1 = Thermostat()
    if report.report() > 0:  # someone at home -> set manual temperature
        thermostat1.set_mode(auto_mode=0)  # manual mode
        manualTemp = thermostat1.getManualTemp()
        thermostat1.temperature(manualTemp)
    else:  # nobody at home -> auto mode (was predicted and programmed in the mornimg)
        thermostat1.set_mode(auto_mode=1)  # auto mode on
예제 #50
0
def main():
    parsed = parser()
    args = parsed.parse_args()
    user = args.user

    if args.ip:
        ipLst = getIP()
    else:
        try:
            discover = mdnsDiscover()
            ipLst = discover.getIPAddr()
        except:
            choice = input(
                'No BeagleBoards were found. \nWould you like to enter an IP address manually? (yes or no) '
            )
            choice = strtobool(choice)
            if choice:
                ipLst = getIP()
            else:
                quit()

    reports = []
    for i in range(len(ipLst)):
        ip = ipLst[i]
        com = SSHComs(ip, user)
        re = report(ipLst[i])
        reports.append(re)
        print('Scanning device', (i + 1), 'of', len(ipLst), '\n')

        if com.defPass:
            reports[i].isPassDef()
        if com.passEnabled:
            reports[i].isPassEnabled()
        software = com.In('dpkg --list')

        services = [
            'nginx', 'cloud9.socket', 'nodered.socket', 'bonescript.socket',
            'bonescript-autorun', 'hostapd', 'bb-bbai-tether'
        ]
        enabledServices = []
        for serv in services:
            service = com.In(f'systemctl is-enabled {serv}')
            if service.find('enabled') > -1:
                enabledServices.append(serv)
                reports[i].disServices(enabledServices)

        netstatData = com.In('netstat -lntu')
        listening = netstatParse(netstatData)
        if len(listening) > 0:
            reports[i].listeningPorts(listening)

        if args.cve:
            vulnSoft = checkForVulnSoft(software)
            reports[i].vulnSoft(vulnSoft)

        com.endSession()
    writeReport(reports)
예제 #51
0
def main():
    args = parser.parse_args()

    bounds, ghosts, pacman, goal = mapPositions(args.layout)

    print('Barreiras:', bounds)
    print('Fantasmas:', ghosts)
    print('Pacman:', pacman)
    print('Gol:', goal)
    print()

    #Problema e algoritmos
    problem = PacmanProblem(obstacles=bounds | ghosts,
                            initial=pacman,
                            goal=goal)
    gfsProblem = greedy_best_first_search(problem)
    astarProblem = astar_search(problem)
    bfsProblem = breadth_first_graph_search(problem)
    dfsProblem = depth_first_graph_search(problem)
    print('Greedy Best First Search:')
    print('Caminho:', gfsProblem.path())
    print('Gol:', gfsProblem)
    print('A* Search:')
    print('Caminho:', astarProblem.path())
    print('Gol:', astarProblem)
    print('Breadth-First Search:')
    print('Caminho:', bfsProblem.path())
    print('Gol:', dfsProblem)
    print('Depth-First Search:')
    print('Caminho:', dfsProblem.path())
    print('Gol:', dfsProblem)
    print()
    print('Gerando saídas...')
    generateOutput(gfsProblem.path(), args.layout, 'gfs')
    generateOutput(astarProblem.path(), args.layout, 'astar')
    generateOutput(dfsProblem.path(), args.layout, 'bfs')
    generateOutput(dfsProblem.path(), args.layout, 'dfs')

    print()
    print('Desempenho:')
    report([
        greedy_best_first_search, astar_search, breadth_first_graph_search,
        depth_first_graph_search
    ], [problem])
예제 #52
0
def synthesis(prepare_res, analysis_res, params):
    from math import sqrt

    separator, filename, orig_filename, labels, dw = prepare_res
    labels = [n for n in labels if n not in options.discard]

    if filename != orig_filename:
        os.unlink(filename)

    # aggregate typing and statistics
    res = {}
    res['num_broken_lines'] = 0
    res['num_lines'] = 0
    res['lines_per_slice'] = []
    for sliceno, tmp in enumerate(analysis_res):
        res['num_broken_lines'] += tmp['num_broken_lines']
        res['num_lines'] += tmp['num_lines']
        res['lines_per_slice'].append(tmp['num_lines'])
        dw.set_lines(sliceno, tmp['num_lines'])

    blob.save(res, 'import')

    # write report
    r = report.report()
    if not res['num_lines']:
        r.println('No lines read - empty file!')
        r.close()
        return
    r.println('Number of rows read\n')
    r.println('  slice                            lines')
    for sliceno, nlines in enumerate(res['lines_per_slice']):
        if res['num_lines']:
            r.println('    %2d                         %9d  (%6.2f%%)' %
                      (sliceno, nlines, 100 * nlines / res['num_lines']))
        else:
            r.println('    %2d                         %9d           ' %
                      (sliceno, nlines, 100 * nlines / res['num_lines']))
    r.println('  total                        %9d' % (res['num_lines'], ))
    stdev = sqrt(
        sum((x - res['num_lines'] / params.slices)**2
            for x in res['lines_per_slice']) / params.slices)
    r.println('\n  hash stdev                   %9d  (%6.2f%%)' %
              (stdev, round(100 * stdev / res['num_lines'])))
    r.line()

    r.println('Number of columns              %9d' % len(labels, ))
    r.close()

    if res['num_broken_lines'] and not options.allow_bad:
        raise Exception('%d bad lines without options.allow_bad' %
                        (res['num_broken_lines'], ))
예제 #53
0
 def format_report(self):
     r = report(self._start_date, self._end_date)
     try:
         r.stock_selection_analysis(self._stock_pool_info)
     except Exception as e:
         print(e)
     try:
         r.trade_analysis(self._trade_info, self._buy_info, self._sell_info)
     except Exception as e:
         print(e)
     try:
         r.fund_analysis(self._starting_fund, self._fund_record)
     except Exception as e:
         print(e)
예제 #54
0
파일: main.py 프로젝트: SF73/Diamfab
 def export(self):
     if self.sample is None:
         QMessageBox.warning(self, "Error", "No open sample")
         return
     try:
         dlg = pdfDialog()
         if (dlg.exec_() == QDialog.Accepted):
             name, subs, layers = dlg.getValues()
             path = QFileDialog.getSaveFileName(self, "Save file", "",
                                                '(*.pdf)')[0]
             if path:
                 report(path=path,
                        name=name,
                        samp=self.sample,
                        substrate=subs,
                        layers=layers)
     except PermissionError as e:
         logger.exception("Can't export PDF : File is probably open")
         QMessageBox.critical(self, "Error",
                              "Can't export PDF : File is probably open")
     except Exception as e:
         logger.exception("Can't export PDF")
         QMessageBox.critical(self, "Error", "Can't export PDF")
예제 #55
0
    def submit(self):
        text_buf = self.error_submit_text.get_buffer()
        content = text_buf.get_text(text_buf.get_start_iter(), text_buf.get_end_iter())
        signature = report.createAlertSignature("selinux-policy", 
                                                "setroubleshoot", 
                                                self.alert.get_hash(), 
                                                self.summary, 
                                                content,
                                                package=self.alert.get_policy_rpm())
 
        try:
            rc = report.report(signature, report.io.GTKIO.GTKIO(self.parent.accounts))
        except ProtocolError, e:
            FailDialog(e)
예제 #56
0
def Reports2019():
    aReport = report.report()
    aReport.execute(reportName="2019v1Member",
                    table="2019v1Member",
                    excelFile="./2019v1Member.xlsx")
    aReport.execute(reportName="2019v1RegisteredWaitingPayment",
                    table="2019v1RegisteredWaitingPayment",
                    excelFile="./2019v1RegisteredWaitingPayment.xlsx")
    #aReport.execute(reportName = "2019v1UserClassSelection", table="2019v1UserClassSelection", excelFile = "./2019v1UserClassSelection.xlsx")
    #aReport.execute(reportName = "2019v1MemberClass", table="2019v1MemberClass", excelFile = "./2019v1MemberClass.xlsx")
    #aReport.execute(reportName = "2019v1MemberClassCount", table="2019v1MemberClassCount", excelFile = "./2019v1MemberClassCount.xlsx")
    aReport.execute(reportName="2019v1AllClassSelection",
                    table="2019v1AllClassSelection",
                    excelFile="./2019v1AllClassSelection.xlsx")
예제 #57
0
def run( fs ):

	mountingTestResult = TestResult()
	mountingTestResult.set_total_points(1)
	passedTest = True

	print("Validating that {} support is disabled...".format(fs))

	#In order to run the tests, a try catch block is set up to ensure the needed commands
	#are available on the system.

	try:

		#Input:
		#>>> modprobe -n -v `fs`
		#Expected output:
		#>>> install /bin/true

		fsTest1 = subprocess.check_output(('modprobe', '-n', '-v', fs))
		if "install /bin/true" not in fsTest1:
			report.report("(X)...Support for mounting {} is not disabled.".format(fs))
			passedTest = False

		#Input:
		#>>> lsmod | grep `fs`
		#Expected output:
		#<NONE>

		fsTest2 = subprocess.Popen(('lsmod'), stdout=subprocess.PIPE)

		#With grep piping, a try catch block is needed to guarantee that if the grep
		#returns no results, the process will not fail.
		try:
			fsTest2Output = subprocess.check_output(('grep', fs), stdin=fsTest2.stdout)
			passedTest = False
			print("(X) ... A module exists in /proc/modules for {}.")
		except subprocess.CalledProcessError as e:
			if str(e) != "Command '('grep', '{}')' returned non-zero exit status 1".format(fs):
				passedTest = False
			
	except OSError as e:                    #Catch if any of our commands fail
		report.error("(!)...Tools do not support running a scan for {}\n".format(fs))
		mountingTestResult.set_error(True)
		mountingTestResult.set_error_status("      {}".format(e))
		return mountingTestResult

	#If passedTest has been set by any of the checks, the test fails
	if passedTest == True:
		report.report("......Passed!")
		mountingTestResult.set_points(1)
	else:
		report.mitigation("      Mitigation: run install {} /bin/true".format(fs))
		report.report("......Failed!")

	
	#Send up the result
	return mountingTestResult
예제 #58
0
파일: server.py 프로젝트: sjmf/reportgen
def report_worker(sid):
    try:
        job = get_job(sid)

        log.info("=============  STARTING WORKER  ==============")
        log.debug(job)
        from ast import literal_eval
        job['series'] = literal_eval(job['series'])  # From string
        # Expand paths to full location on filesystem
        output_filename = os.path.join(
            app.config['UPLOAD_FOLDER'],
            next(tempfile._get_candidate_names()) + '.pdf')

        # Make list of input datafiles
        input_datafiles = [
            os.path.join(app.config['UPLOAD_FOLDER'], f['temporary_name'])
            for f in get_files(sid)
        ]

        report.report(input_datafiles, output_filename, **{
            **job, 'pdf': True,
            'htm': False
        })

        log.info("=============  WORKER FINISHED  ==============")

        # Update finished job
        upd_job(sid, 'generated_pdf', output_filename)
        upd_job(sid, 'status', 'done')

    except Exception as e:
        log.error("Exception occurred in worker thread")
        log.error(sys.exc_info()[0])

        upd_job(sid, 'status', 'error')
        upd_job(sid, 'generated_pdf', None)
        raise e
예제 #59
0
def report(cmdline):
    survey = model.survey.Survey.load(cmdline['project'])
    import report

    if cmdline['format'] == 'short':
        small = 1
    else:
        small = 0

    if cmdline['all_filters']:
        return report.stats(survey, cmdline['filter'], cmdline['output'],
                            cmdline['papersize'], small, cmdline['suppress'])
    else:
        return report.report(survey, cmdline['filter'], cmdline['output'],
                             cmdline['papersize'], small, cmdline['suppress'])