def test_default_value_load_mutable(self): configure_main(with_config_logging=False) server = model.Database(immutable=False) self.assertEqual(server.addr, '127.0.0.1') self.assertEqual(server.port, 3306) server.addr = '192.168.0.1' server.port = 8888 self.assertEqual(server.addr, '192.168.0.1') self.assertEqual(server.port, 8888)
def run(self): self.db = model.Database('hieroch.db') self.cio = cio.cio() self.cio.print_status(0, "Hieroch.") self.store_id = None self.today = datetime.date.today() self.origin_no = model.origin_by_name('offline') self.loop() self.db.save()
def assert_main_output(self, threads, expected=None, db=None, client=MockedClient): if expected is None: expected = self.get_expected_builds() if db is None: db = model.Database(':memory:') make_db.main(db, {self.JOBS_DIR: {}}, threads, True, client) result = {path: (started, finished, db.test_results_for_build(path)) for _rowid, path, started, finished in db.get_builds()} self.assertEqual(result, expected) return db
def test_file_path_value_load_with_two_files(self): configure_main(config_path=['test.ini', 'test2.ini'], config_arg=None) server = model.Server() self.assertEqual(server.addr, '127.0.0.1') self.assertEqual(server.port, 80) self.assertEqual(server.udp, True) self.assertEqual(server.val_f, 0.5) self.assertEqual(server.val_d, {'a': 10, 'b': 20, 'c': 30}) self.assertEqual(server.val_l, [1, 2, 3, 4, 5, 6]) self.assertEqual(server.home, Path('../../..')) with self.assertRaises(AttributeError): self.conf.addr = '1.2.3.4' # not editable server = model.Database(immutable=True) self.assertEqual(server.addr, '192.168.0.1') self.assertEqual(server.port, 3333)
def test_ifconf_config_path(self): configure_main(config_path='test_all.ini', config_arg=None) server = model.Server() self.assertEqual(server.addr, '127.0.0.1') self.assertEqual(server.port, 80) self.assertEqual(server.udp, True) self.assertEqual(server.val_f, 0.5) self.assertEqual(server.val_d, {'a': 10, 'b': 20, 'c': 30}) self.assertEqual(server.val_l, [9, 9, 9]) self.assertEqual(server.home, Path('../../..')) with self.assertRaises(AttributeError): self.conf.addr = '1.2.3.4' # not editable server = model.Database(immutable=True) self.assertEqual(server.addr, '192.168.0.1') self.assertEqual(server.port, 3333)
def main(): args = get_args() db = model.Database(args.database) if not args.results_only: print("\nFacebook login:"******"") driver = lib.create_driver(not args.no_headless) try: lib.login_facebook(driver) get_groups.get_groups(driver) get_events.get_all_events(driver, args.debug) get_times.get_all_times(driver, args.debug) finally: driver.close() parse_results.parse_results(args.folder) print("\nDone...", flush=True)
#!/usr/bin/python3 import itertools from bottle import route, run, view, static_file, redirect, request, response import model import datetime import sys max_votes = model.max_votes db = model.Database(sys.argv[1] if len(sys.argv) > 1 else 'db.db') started = False def auth(method): def wrapper(*args, **kwargs): user = request.get_cookie("user") if user: kwargs['user'] = user return method(*args, **kwargs) else: redirect('/who') return wrapper @route('/admin') @view('admin') def admin(): return dict(started=started)
def main(): database = model.Database() main_menu(database) database.disconnect()
def test_main(self): # It's easier to run a full integration test with stubbed-out # external interfaces and validate the trace than it is to test # each individual piece. # The components are mostly tested in make_*_test.py. db = model.Database(':memory:') fake_sub = FakeSub([ FakePullResponse([ FakeReceivedMessage( 'b', FakePubSubMessage( 'no_data', { 'eventType': 'OBJECT_FINALIZE', 'objectId': 'logs/fake/123/finished.json', 'bucketId': 'kubernetes-jenkins' })) ]), FakePullResponse([]), FakePullResponse([ FakeReceivedMessage( 'c', FakePubSubMessage( 'no_data', { 'eventType': 'OBJECT_FINALIZE', 'objectId': 'logs/fake/123/finished.json', 'bucketId': 'kubernetes-jenkins' })) ]), FakePullResponse([]), FakePullResponse([ FakeReceivedMessage( 'd', FakePubSubMessage( 'no_data', { 'eventType': 'OBJECT_FINALIZE', 'objectId': 'logs/fake/124/started.json', 'bucketId': 'kubernetes-jenkins' })) ]), FakePullResponse([]), ]) fake_client = FakeClient() fake_table = FakeTable('day', stream.load_schema(FakeSchemaField)) fake_sub_path = 'projects/{project_id}/subscriptions/{sub}' tables = {'day': (fake_table, 'incr')} stream.main(db, fake_sub, fake_sub_path, fake_client, tables, self.fake_buckets, make_db_test.MockedClient, [1, 0, 0, 0].pop) # uncomment if the trace changes # import pprint; pprint.pprint(fake_sub.trace) # import pprint; pprint.pprint(fake_client.trace) # self.maxDiff = 3000 now = make_db_test.MockedClient.NOW self.assertEqual( fake_sub.trace, [['pull', fake_sub_path, False], ['pull', fake_sub_path, True], ['modify-ack', fake_sub_path, ['b'], 180], ['ack', fake_sub_path, ['b']], ['pull', fake_sub_path, False], ['pull', fake_sub_path, True], ['modify-ack', fake_sub_path, ['c'], 180], ['ack', fake_sub_path, ['c']], ['pull', fake_sub_path, False], ['pull', fake_sub_path, True], ['ack', fake_sub_path, ['d']]]) self.assertEqual(fake_client.trace, [[ 'insert-rows', ([{ 'elapsed': 5, 'finished': now, 'job': 'fake', 'number': 123, 'passed': True, 'path': 'gs://kubernetes-jenkins/logs/fake/123', 'result': 'SUCCESS', 'started': now - 5, 'test': [{ 'name': 'Foo', 'time': 3.0 }, { 'failed': True, 'failure_text': 'stacktrace', 'name': 'Bad', 'time': 4.0 }], 'tests_failed': 1, 'tests_run': 2 }], ), { 'skip_invalid_rows': True } ]])
required=True, ) parser.add_argument( '--threads', help='number of concurrent threads to download results with', default=32, type=int, ) parser.add_argument('--junit', action='store_true', help='Download JUnit results from each build') parser.add_argument( '--buildlimit', help='maximum number of runs within each job to pull, \ all jobs will be collected if unset or 0', default=sys.maxsize, type=int, ) return parser.parse_args(argv) if __name__ == '__main__': OPTIONS = get_options(sys.argv[1:]) main( model.Database(), yaml.safe_load(open(OPTIONS.buckets)), OPTIONS.threads, OPTIONS.junit, OPTIONS.buildlimit, )
bucket += '/' get_builds(db, bucket, metadata, threads, client_class) if get_junit: download_junit(db, threads, client_class) def get_options(argv): """Process command line arguments.""" parser = argparse.ArgumentParser() parser.add_argument( '--buckets', help='YAML file with GCS bucket locations', required=True, ) parser.add_argument( '--threads', help='number of concurrent threads to download results with', default=32, type=int, ) parser.add_argument('--junit', action='store_true', help='Download JUnit results from each build') return parser.parse_args(argv) if __name__ == '__main__': OPTIONS = get_options(sys.argv[1:]) main(model.Database(), yaml.load(open(OPTIONS.buckets)), OPTIONS.threads, OPTIONS.junit)
def setUp(self): self.db = model.Database(':memory:')
def test_file_path_value_load_test2(self): configure_main(config_path='test2.ini', config_arg=None) server = model.Database(immutable=True) self.assertEqual(server.addr, '192.168.0.1') self.assertEqual(server.port, 3333)
bucket += '/' get_builds(db, bucket, metadata, threads, client_class) if get_junit: download_junit(db, threads, client_class) def get_options(argv): """Process command line arguments.""" parser = argparse.ArgumentParser() parser.add_argument( '--buckets', help='YAML file with GCS bucket locations', required=True, ) parser.add_argument( '--threads', help='number of concurrent threads to download results with', default=32, type=int, ) parser.add_argument('--junit', action='store_true', help='Download JUnit results from each build') return parser.parse_args(argv) if __name__ == '__main__': OPTIONS = get_options(sys.argv[1:]) main(model.Database(), yaml.safe_load(open(OPTIONS.buckets)), OPTIONS.threads, OPTIONS.junit)
def get_reactors(self): db = model.Database() con = db.get_country_codes() dict = {} reactors = [] coorurl = 'https://en.wikipedia.org/wiki/List_of_nuclear_power_stations' coorr = requests.get(coorurl) coorsoup = BeautifulSoup(coorr.content, 'lxml') coortable = coorsoup.find_all('table') coordf = pd.read_html(str(coortable), header=0) coorjsondata = json.loads(coordf[1].to_json(orient='records')) for i in range(0, len(con)): if con[i][0] is not None: url = 'https://pris.iaea.org/PRIS/CountryStatistics/CountryDetails.aspx?current=' + con[ i][0].replace(" ", "") r = requests.get(url) soup = BeautifulSoup(r.content, 'lxml') table = soup.find_all('table') df = pd.read_html(str(table), header=0) jsondata = json.loads(df[3].to_json(orient='index')) dict[con[i][0].replace(" ", "")] = jsondata # manipulate json data and add country id to the reactors from the countries table print(coorjsondata) # for j in range(0, len(coorjsondata)): # for i in con[0]: # print(coorjsondata[j]) # print(dict[i]['0']) # if coorjsondata[j]['Power station'].upper() in i[1].upper(): # lat = j['Location'].upper().split("/")[1].split(" ")[1] # long = j['Location'].upper().split("/")[1].split(" ")[2] # i['lat'] = long # i['long'] = lat # print(i) for i in range(0, len(con)): if con[i][0] is not None: for j in range(0, len(dict[con[i][0].replace(" ", "")])): dict[con[i][0].replace(" ", "")][str(j)]['ID'] = con[i][1] name = dict[con[i][0].replace(" ", "")][str(j)]['Name'] reactorType = dict[con[i][0].replace(" ", "")][str(j)]['Type'] status = dict[con[i][0].replace(" ", "")][str(j)]['Status'] location = dict[con[i][0].replace(" ", "")][str(j)]['Location'] rup = dict[con[i][0].replace( " ", "")][str(j)]['Reference Unit Power [MW]'] gec = dict[con[i][0].replace( " ", "")][str(j)]['Gross Electrical Capacity [MW]'] fgc = dict[con[i][0].replace( " ", "")][str(j)]['First Grid Connection'] id = dict[con[i][0].replace(" ", "")][str(j)]['ID'] for z in coorjsondata: # print(name.upper(), z['Power station'].upper()) if z['Power station'].upper() in name.upper(): # print(name.upper(), z['Power station'].upper()) lat = z['Location'].split("/")[1].split(" ")[1] long = z['Location'].split("/")[1].split(" ")[2] if "S" in lat: lat = float( re.findall("[0-9]*\.[0-9]*", lat)[0]) * -1 else: lat = float( re.findall("[0-9]*\.[0-9]*", lat)[0]) if "W" in long: long = float( re.findall("[0-9]*\.[0-9]*", long)[0]) * -1 else: long = float( re.findall("[0-9]*\.[0-9]*", long)[0]) print(lat, long) break else: lat = None long = None # print(name, reactorType, status, location, rup, gec, fgc, id, lat, long) reactors.append( rea.Reactor(name, reactorType, status, location, rup, gec, fgc, id, lat, long)) return reactors
parser = argparse.ArgumentParser() parser.add_argument( '--poll', required=True, help='Follow GCS changes from project/topic/subscription', ) parser.add_argument( '--dataset', help='BigQuery dataset (e.g. k8s-gubernator:build)' ) parser.add_argument( '--tables', nargs='+', default=[], help='Upload rows to table:days [e.g. --tables day:1 week:7 all:0]', ) parser.add_argument( '--stop_at', type=int, help='Terminate when this hour (0-23) rolls around (in local time).' ) return parser.parse_args(argv) if __name__ == '__main__': OPTIONS = get_options(sys.argv[1:]) main(model.Database(), *load_sub(OPTIONS.poll), *load_tables(OPTIONS.dataset, OPTIONS.tables), stop=StopWhen(OPTIONS.stop_at))
def get_recipe(): with model.Database(DB) as rdb: result = rdb.select_recipe(recipe_id) recipe = model.Recipe(result[0], result[1], result[2], result[3], result[4]) return jsonify(recipe.dictionary)
def test_file_path_value_load_test3_override(self): configure_main(config_path=['test3.ini', 'test2.ini'], config_arg=None) server = model.Database(immutable=True) self.assertEqual(server.addr, '192.168.0.100') self.assertEqual(server.port, 4444)
def main(): database = model.Database() main_menu(database)
bucket += '/' get_builds(db, bucket, metadata, threads, client_class) if get_junit: download_junit(db, threads, client_class) def get_options(argv): """Process command line arguments.""" parser = argparse.ArgumentParser() parser.add_argument( '--buckets', help='YAML file with GCS bucket locations', required=True, ) parser.add_argument( '--threads', help='number of concurrent threads to download results with', default=32, type=int, ) parser.add_argument('--junit', action='store_true', help='Download JUnit results from each build') return parser.parse_args(argv) if __name__ == '__main__': OPTIONS = get_options(sys.argv[1:]) main(model.Database('build.db'), yaml.load(open(OPTIONS.buckets)), OPTIONS.threads, OPTIONS.junit)
import model logging.basicConfig( level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') logger = logging.getLogger(__name__) app = Flask(__name__) app.config['MAX_CONTENT_LENGTH'] = 20 * 1024 * 1024 limiter = Limiter( app, key_func=get_remote_address, #根据访问者的IP记录访问次数 default_limits=["200 per day", "50 per hour"] #默认限制,一天最多访问200次,一小时最多访问50次 ) UPLOAD_PATH = 'files' #where papers stored minganlist = ['peace', 'and', 'love'] db = model.Database() @app.route('/') def method_name(): hot_paper = db.get_hot_paper() return render_template('homepage.html', articles=hot_paper) @app.route('/articles/<id>') def article_detail(id): ip = request.remote_addr db.visit_paper(ip, id) lst = db.get_paper(id) lst2 = db.get_comments(id) return render_template('article_detail.html', articles=lst, comments=lst2)
def test_main(self): # It's easier to run a full integration test with stubbed-out # external interfaces and validate the trace than it is to test # each individual piece. # The components are mostly tested in make_*_test.py. db = model.Database(':memory:') fakesub = FakeSub([ [ ('a', Attrs({'eventType': 'OBJECT_DELETE'})), ], [ ('b', Attrs({ 'eventType': 'OBJECT_FINALIZE', 'objectId': 'logs/fake/123/finished.json', 'bucketId': 'kubernetes-jenkins' })), ], [], [ ('c', Attrs({ 'eventType': 'OBJECT_FINALIZE', 'objectId': 'logs/fake/123/finished.json', 'bucketId': 'kubernetes-jenkins' })), ], [], [ ('d', Attrs({ 'eventType': 'OBJECT_FINALIZE', 'objectId': 'logs/fake/124/started.json' })), ], [], ]) faketable = FakeTable('day', stream.load_schema(FakeSchemaField), fakesub.trace) tables = {'day': (faketable, 'incr')} stream.main(db, fakesub, tables, make_db_test.MockedClient, [1, 0, 0, 0].pop) # uncomment if the trace changes # import pprint; pprint.pprint(fakesub.trace) # self.maxDiff = 3000 now = make_db_test.MockedClient.NOW self.assertEqual( fakesub.trace, [['pull', False], ['pull', True], ['pull', True], ['ack', ['a']], ['modify-ack', ['b'], 180], ['ack', ['b']], [ 'insert-data', ([[ 5, now - 5, now, True, u'SUCCESS', None, u'gs://kubernetes-jenkins/logs/fake/123', u'fake', 123, [], [{ 'name': 'Foo', 'time': 3.0 }, { 'failed': True, 'failure_text': 'stacktrace', 'name': 'Bad', 'time': 4.0 }], 2, 1, None ]], [1]), { 'skip_invalid_rows': True } ], ['pull', False], ['pull', True], ['modify-ack', ['c'], 180], ['ack', ['c']], ['pull', False], ['pull', True], ['ack', ['d']]])
db.reset_emitted(incremental_table) builds = db.get_builds(min_started=min_started, incremental_table=incremental_table) rows_emitted = set() for rowid, path, started, finished in builds: try: results = db.test_results_for_build(path) row = row_for_build(path, started, finished, results) json.dump(row, outfile, sort_keys=True) outfile.write('\n') rows_emitted.add(rowid) except IOError: return except: logging.exception('error on %s', path) if rows_emitted: gen = db.insert_emitted(rows_emitted, incremental_table=incremental_table) print >> sys.stderr, 'incremental progress gen #%d' % gen else: print >> sys.stderr, 'no rows emitted' if __name__ == '__main__': db = model.Database('build.db') opts = parse_args(sys.argv[1:]) main(db, opts, sys.stdout)
parser.add_argument( '--poll', required=True, help='Follow GCS changes from project/topic/subscription', ) parser.add_argument( '--dataset', help='BigQuery dataset (e.g. k8s-gubernator:build)' ) parser.add_argument( '--tables', nargs='+', default=[], help='Upload rows to table:days [e.g. --tables day:1 week:7 all:0]', ) parser.add_argument( '--stop_at', type=int, help='Terminate when this hour (0-23) rolls around (in local time).' ) return parser.parse_args(argv) if __name__ == '__main__': OPTIONS = get_options(sys.argv[1:]) main(model.Database('build.db'), load_sub(OPTIONS.poll), load_tables(OPTIONS.dataset, OPTIONS.tables), stop=StopWhen(OPTIONS.stop_at))
def __init__(self): self.root = tkinter.Tk() # main window of application self.app_view = view.AppView(self.root) # creates app's view self.app_model = model.Database() # creates app's model self.app_view.insert_button.config(command=self.add_record) # bind method to button self.app_view.delete_button.config(command=self.delete_record) # bind method to button
db.reset_emitted(incremental_table) builds = list(db.get_builds_from_paths(opts.paths, incremental_table)) else: builds = db.get_builds(min_started=min_started, incremental_table=incremental_table) rows_emitted = set() for rowid, row in make_rows(db, builds): size = json_size(row) if size > MAX_ROW_SIZE: print('row for %s exceeds maximum for bigquery %d > %d' % (row['path'], size, MAX_ROW_SIZE)) continue json.dump(row, outfile, sort_keys=True) outfile.write('\n') rows_emitted.add(rowid) if rows_emitted: gen = db.insert_emitted(rows_emitted, incremental_table=incremental_table) print('incremental progress gen #%d' % gen, file=sys.stderr) else: print('no rows emitted', file=sys.stderr) return 0 if __name__ == '__main__': DB = model.Database() OPTIONS = parse_args(sys.argv[1:]) sys.exit(main(DB, OPTIONS, sys.stdout))
import os import re import psycopg2 import model from flask import Flask, request, jsonify from flask_cors import CORS app = Flask(__name__) CORS(app) DATABASE_URL = os.environ['DATABASE_URL'] conn = psycopg2.connect(DATABASE_URL, sslmode='require') cursor = conn.cursor() db = model.Database(cursor) @app.route('/news', methods=['GET', 'POST']) def news(): if request.method == 'GET': amount = int(request.args.get('amount')) category = int(request.args.get('category')) return jsonify(db.many_news(amount, category)) elif request.method == 'POST': content = request.get_json(cache=False) db.add_news( content['title'], get_image(content['content']), content['category'], content['content'],