def insert_names_into_db(batch_size: int = 10000, limit: int = 200000): with open('name.basics.tsv') as names_tsv: names_tsv.readline() reader = csv.reader(names_tsv, delimiter='\t') counter = 0 names = [] number_in_batch = 0 for row in tqdm(reader): if number_in_batch > batch_size: number_in_batch = 0 db_session.add_all(names) db_session.commit() names = [] counter += batch_size print(f'added {counter} names') if limit and counter >= limit: return row = list(map(map_null, row)) name = Name( nconst=row[0], primaryName=row[1], birthYear=row[2], deathYear=row[3], primaryProfession=row[4], ) if row[5]: titles = Title.query.filter(Title.tconst.in_( row[5].split(','))) name.knownForTitles = list(titles) names.append(name) number_in_batch += 1 db_session.commit()
def generate_fake_data(cups, samples): db_session.add_all([CupModel(id=cup) for cup in range(cups)]) db_session.commit() timestamp = datetime.datetime.now() - datetime.timedelta(minutes=10) for cup in range(cups): generate_fake_cup(cup, timestamp, samples)
def insert_titles_into_db(batch_size: int = 10000, limit: int = 200000): with open('title.basics.tsv') as titles_tsv: titles_tsv.readline() reader = csv.reader(titles_tsv, delimiter='\t') counter = 0 titles = [] number_in_batch = 0 for row in tqdm(reader): if number_in_batch > batch_size: number_in_batch = 0 db_session.add_all(titles) db_session.commit() titles = [] counter += batch_size print(f'added {counter} titles') if limit and counter >= limit: return row = list(map(map_null, row)) title = Title(tconst=row[0], titleType=row[1], primaryTitle=row[2], originalTitle=row[3], isAdult=bool(int(row[4])), startYear=row[5], endYear=row[6], runtimeMinutes=row[7], genres=row[8]) titles.append(title) number_in_batch += 1 db_session.commit()
def assign_tasks(cls): books = Book.gets() tasks = [] weight_2 = [] weight_3 = [] for book in books: book_source = BookSource.get_by_bid(book.id) if book_source is None: print '%d has no book source info' % book.id continue if book.latest_chapter: task = cls(book.id, book.latest_chapter.title, book_source.source_site, book_source.source_url) tasks.append(task) if book.weight != 1: if book.weight == 2: weight_2.append(task) elif book.weight == 3: weight_3.append(task) # 处理权重 p_half = len(tasks) / 2 p_third_one = len(tasks) / 3 p_third_two = len(tasks) / 3 * 2 tasks[p_half:p_half] = weight_2 tasks[p_third_one:p_third_one] = weight_3 tasks[p_third_two:p_third_two] = weight_3 db_session.add_all(tasks) db_session.commit()
def generate_fake_cup(cup, timestamp, samples): values = (0, 0, 0, 0) for _ in range(samples): delta = [random.randint(0, 20) for _ in range(4)] values = [sum(x) for x in zip(values, delta)] timestamp += datetime.timedelta(seconds=5) db_session.add_all([ GreyscaleModel( value=values[0], timestamp=timestamp, cup_id=cup, ), HumidityModel( value=values[1], timestamp=timestamp, cup_id=cup, ), LiquidModel( value=values[2], timestamp=timestamp, cup_id=cup, ), TemperatureModel( value=values[3], timestamp=timestamp, cup_id=cup, ) ]) db_session.commit()
def assignmentToCoders(article_ids, coder_ids, pass_number='1'): """ Assigns each article to all coders in list. """ if pass_number == '1': model = ArticleQueue elif pass_number == '2': model = SecondPassQueue elif pass_number == 'ec': model = EventCreatorQueue ## add to queues to_add = [] for coder_id in coder_ids: existing = [x.article_id for x in db_session.query(model).all()] for a in article_ids: if a not in existing: if pass_number == '1': item = ArticleQueue(article_id=a, coder_id=coder_id) elif pass_number == '2': item = SecondPassQueue(article_id=a, coder_id=coder_id) elif pass_number == 'ec': item = EventCreatorQueue(article_id=a, coder_id=coder_id) to_add.append(item) db_session.add_all(to_add) db_session.commit() return len(to_add)
def assignmentToCoders(article_ids, coder_ids, pass_number): """ Assigns each article to all coders in list. """ if pass_number == '1': model = ArticleQueue elif pass_number == '2': model = SecondPassQueue elif pass_number == 'ec': model = EventCreatorQueue ## add to queues to_add = [] for c in coder_ids: for a in article_ids: if pass_number == '1': item = ArticleQueue(article_id=a, coder_id=c) elif pass_number == '2': item = SecondPassQueue(article_id=a, coder_id=c) elif pass_number == 'ec': item = EventCreatorQueue(article_id=a, coder_id=c) to_add.append(item) db_session.add_all(to_add) db_session.commit() return len(to_add)
def test_input_data(self): u1 = User(name="Paul", email="*****@*****.**", password="******") u2 = User(name="Joe", email="*****@*****.**", password="******") s1 = Alert(name="guitars", string="guitars -steel", status=1, last_update=datetime.date(2013, 9, 25), user=u1) s2 = Alert(name="banjo", string="banjo -100", status=1, last_update=datetime.date(2013, 9, 30), user=u1) s3 = Alert(name="judo stuff", string="gee", status=1, last_update=datetime.date(2013, 5, 13), user=u2) s4 = Alert(name="bmx bike", string="bmx -hoffman", status=0, last_update=datetime.date(2011, 3, 5), user=u2) r1 = Result(title="Guitar for Sale - $100", link="http://craigslist.com/12341234", alert=s1) r2 = Result(title="Shitty Guitar for Sale - $50", link="http://craigslist.com/54325432", alert=s1) r3 = Result(title="Even shittier Guitar for Sale - $30", link="http://craigslist.com/4672345", alert=s1) db_session.add_all([u1, u2, s1, s2, s3, s4, r1, r2, r3]) db_session.commit() print "****Data Has Been Commited to DB****"
def test_input_data(self): # Add users u1 = User(name="Paul", email="*****@*****.**", password="******") u2 = User(name="Joe", email="*****@*****.**", password="******") # Add beers b1 = Beer(name="Chaisoin", style = "Saison"); b2 = Beer(name="Orca Porter", style = "Porter"); # Add brewers to beers brewer1 = Brewer(user=u1, beer=b1) brewer2 = Brewer(user=u2, beer=b1) brewer3 = Brewer(user=u1, beer=b2) # Add a Recipe to a beer recipe1 = Recipe(steep_time=60, boil_time=60, brew_date=date(2014, 10, 31), beer=b1) malt1 = Malt(variety="Pale Malt LME", weight=5, recipe=recipe1) malt2 = Malt(variety="Great Western 2-row", weight=4.5, recipe=recipe1) hop1 = Hop(variety="Magnum", addition_time=60, weight=1, recipe=recipe1) hop2 = Hop(variety="Cascade", addition_time=0, weight=1, recipe=recipe1) db_session.add_all([u1, u2, b1, b2, brewer1, brewer2, brewer3]) db_session.add_all([recipe1, malt1, malt2, hop1, hop2]) db_session.commit() print "****Data Has Been Commited to DB****"
def setUp(self): init_db() self.client = Client(schema) user = User(name="Foo", password="******") post = Post(author=user, title="Test", content="Test Content") comment = Comment(author=user, post=post, content="Comment Test") db_session.add_all([user, post, comment]) db_session.commit()
def resetVariableOptions(): """ Load current dropdowns from file. """ db_session.query(VarOption).delete() print("Adding variables...") dds = [] with open("dropdowns.csv") as csvfile: reader = csv.reader(csvfile) for row in reader: dds.append(VarOption(variable=row[0], option=row[1])) db_session.add_all(dds) db_session.commit()
def resetVariableOptions(): """ Load current dropdowns from file. """ db_session.query(VarOption).delete() print("Adding variables...") dds = [] with open("dropdowns.csv") as csvfile: reader = csv.reader(csvfile) for row in reader: dds.append( VarOption(variable = row[0], option = row[1]) ) db_session.add_all(dds) db_session.commit()
def addArticles(filename, db_name): articles = [] with open(filename) as csvfile: reader = csv.reader(csvfile) for row in reader: title = row[0] db_id = row[1] if title == 'TITLE': continue articles.append( ArticleMetadata(filename = db_id, db_id = db_id, title = title, db_name = db_name) ) db_session.add_all(articles) db_session.commit()
def addArticlesExample(db_name = 'test'): """ Add articles from example directory. """ print("Adding example articles...") articles = [] for f in glob.iglob(jsonConfig['DOC_ROOT'] + "*.txt"): filename = f.split('/')[-1] lines = open(f, 'r').read().split("\n") title = lines[0].replace("TITLE: ", "") articles.append( ArticleMetadata(filename = filename, title = unicode(title, "utf-8", errors = "ignore"), db_name = db_name) ) db_session.add_all(articles) db_session.commit()
def create_user(): init_db() # Создаем 2 категории games = Category(name='Games') films = Category(name='Films') db_session.add_all([games, films]) # Создаем товары SOME_TEXT = 'some_text' skyrim = Product(name='Skyrim', price=10.1, category=games, description=SOME_TEXT) fallout = Product(name='Fallout', price=5.5, category=games, description=SOME_TEXT) matrix = Product(name='Matrix', price=1.2, category=films, description=SOME_TEXT) db_session.add_all([skyrim, fallout, matrix]) # commit db_session.commit()
def setup_workdir(request_data: dict) -> Artifact: """ Creates new dir for simulation and places simulation configuration file in it :param request_data: Flask request with configuration :return: tuple with path to workdir and saved configuration """ conf_name = get_conf_name(request_data['configurationName']) conf = request_data['configuration'] start_time = datetime.datetime.utcnow() simulation = Simulation(started_utc=start_time, name="test_simulation", current_step="CLI", status='ONGOING') db_session.add(simulation) db_session.flush() step = SimulationStep(started_utc=start_time, origin="CLI", simulation_id=simulation.id, status='ONGOING') db_session.add(step) db_session.flush() workdir_path = create_workdir(simulation.id) conf_path = '{}/{}'.format(workdir_path, conf_name) simulation.workdir = workdir_path simulation.current_step_id = step.id with open(conf_path, 'w+') as f: json.dump(conf, f, indent=2) configuration = Artifact(size_kb=os.path.getsize(conf_path), path=conf_path, created_utc=start_time, step_id=step.id, name=conf_name, file_type='JSON', simulation_id=simulation.id) simulation.artifacts.append(configuration) step.artifacts.append(configuration) simulation.steps.append(step) db_session.begin() db_session.add_all([configuration, step, simulation]) db_session.flush() db_session.commit() return configuration
def execute_state_1(db, tablemd, schema): "annotype -> [(column, extractorname)]" print "state 1\t", tablemd.tablename extractors = find_location_extractors(db, tablemd, schema) annos = [] for anntype, pairs in extractors.iteritems(): for colname, extractor in pairs: annos.append(Annotation(colname, anntype, extractor, tablemd)) tablemd.hasloc = len(annos) > 0 tablemd.state = 2 db_session.add_all(annos) db_session.add(tablemd) db_session.commit() return tablemd.state
def test_get_configuration(self): """test the get_configuration function""" string_config = model.Configuration( "String_Config", "sample", "string", "admin" ) bool_config = model.Configuration("Bool_Config", "True", "bool", "admin") integer_config = model.Configuration("Integer_Config", "12", "integer", "admin") db_session.add_all([string_config, bool_config, integer_config]) db_session.commit() self.assertEqual(util.get_configuration("String_Config"), "sample") self.assertEqual(util.get_configuration("Bool_Config"), True) self.assertEqual(util.get_configuration("Integer_Config"), 12)
def addArticlesExample(db_name = 'test'): """ Add articles from example directory. """ print("Adding example articles...") articles = [] for f in glob.iglob(config.DOC_ROOT + "*.txt"): print(f) filename = f.split('/')[-1] lines = open(f, 'r').read().split("\n") title = lines[0].replace("TITLE: ", "") articles.append( ArticleMetadata(filename = filename, title = title, db_name = db_name) ) db_session.add_all(articles) db_session.commit()
def addQueueExample(): print("Adding example queues...") articles = db_session.query(ArticleMetadata).all() random.shuffle(articles) users = db_session.query(User).filter(User.authlevel == 1).all() aq = [] ## assign articles randomly to core team members for funsies for a in articles: for u in users: aq.append(ArticleQueue(article_id=a.id, coder_id=u.id)) db_session.add_all(aq) db_session.commit()
def addQueueExample(): print("Adding example queues...") articles = db_session.query(ArticleMetadata).all() random.shuffle(articles) users = db_session.query(User).filter(User.authlevel == 1).all() aq = [] ## assign articles randomly to core team members for funsies for a in articles: for u in users: aq.append( ArticleQueue(article_id = a.id, coder_id = u.id) ) db_session.add_all(aq) db_session.commit()
def addArticlesExample(db_name='test'): """ Add articles from example directory. """ print("Adding example articles...") articles = [] for f in glob.iglob(jsonConfig['DOC_ROOT'] + "*.txt"): filename = f.split('/')[-1] lines = open(f, 'r').read().split("\n") title = lines[0].replace("TITLE: ", "") articles.append( ArticleMetadata(filename=filename, title=unicode(title, "utf-8", errors="ignore"), db_name=db_name)) db_session.add_all(articles) db_session.commit()
def RecordFightToDB(fighter1,fighter2,fight,Winner): #Identify winner and save as foreign key if Winner == fighter1.name: fight.winner = fighter1 try: fighter1.streak += 1 except(TypeError): print "No streak found. Current fight mode did not give one." try: fighter2.streak -= 1 except(TypeError): print "No streak found. Current fight mode did not give one." elif Winner == fighter2.name: fight.winner = fighter2 try: fighter2.streak += 1 except(TypeError): print "No streak found. Current fight mode did not give one." try: fighter1.streak -= 1 except(TypeError): print "No streak found. Current fight mode did not give one." else: print ("Unable to match declared winner with a fighter.") return fight.time = datetime.datetime.now() #SQLAlchemey is similar to git, one must add then commit. #Unlike others once added the pk is created db_session.add(fight) #determine ratios and averages avgBetRatio1, winRate1 = GetFighterStats(fighter1) avgBetRatio2, winRate2 = GetFighterStats(fighter2) #Save fighter values. fighter1.winRate = winRate1 fighter1.betRatio = avgBetRatio1 fighter2.winRate = winRate2 fighter2.betRatio = avgBetRatio2 db_session.add_all([fighter1,fighter2]) db_session.commit()
def install(): from database import engine from datamodel import Base, User, PrefType, PrefWeight Base.metadata.create_all(engine) #Add initial admin user# user = User(user_name='harvimt',user_type='admin') sess.add(user) #add pref types sess.add_all(( PrefType('Time', 'rank', 5.0), PrefType('Faculty', 'weight', 2.0), PrefType('Theme', 'weight', 1.0) )) sess.commit() #and pref weights sess.add_all(( PrefWeight(1,0,0.00), PrefWeight(1,1,0.25), PrefWeight(1,2,0.50), PrefWeight(1,3,0.75), PrefWeight(1,4,1.00), PrefWeight(1,5,1.25), PrefWeight(1,6,1.50), PrefWeight(2,0,0.00), PrefWeight(2,1,0.33), PrefWeight(2,2,0.66), PrefWeight(2,3,1.00), PrefWeight(2,4,1.33), PrefWeight(2,5,1.67), PrefWeight(2,6,2.00), PrefWeight(3,0,0.00), PrefWeight(3,1,0.17), PrefWeight(3,2,0.33), PrefWeight(3,3,0.50), PrefWeight(3,4,0.67), PrefWeight(3,5,0.83), PrefWeight(3,6,1.00) )) sess.commit()
def setup_contest(): roles = {x.name: x for x in model.UserRole.query.all()} test_contestant = model.User( "testuser", "Test User", "pass", user_roles=[roles["defendant"]] ) test_executioner = model.User( "testexec", "Test Executioner", "epass", user_roles=[roles["executioner"]] ) test_contest = model.Contest( "test_contest", util.str_to_dt("2017-02-05T22:04:00Z"), util.str_to_dt("2030-01-01T11:11:00Z"), True, ) io_problem_type = model.ProblemType.query.filter_by(name="input-output").one() test_problem = model.Problem( io_problem_type, "fizzbuzz", "FizzBuzz", "## FizzBuzz\nPerform fizzbuzz up to the given number", "3", "1\n2\nFizz", "15", "1\n2\nFizz\n4\nBuzz\nFizz\n7\n8\n9\nBuzz\n11\nFizz\n13\n14\nFizzBuzz\n", ) test_contest.problems.append(test_problem) test_contest.users.append(test_contestant) python = model.Language.query.filter_by(name="python").one() test_run = model.Run( test_contestant, test_contest, python, test_problem, util.str_to_dt("2017-02-05T23:00:00Z"), 'import sys\nn=raw_input()\nfor i in range(1, n+1): print("Fizz"*(i%3==0)+"Buzz"*(i%5==0) or i)', test_problem.secret_input, test_problem.secret_output, True, ) db_session.add_all( [test_executioner, test_contestant, test_contest, test_problem, test_run] ) db_session.commit()
def seed_db(): db_session.execute('''CREATE TABLE data (a int, b int, c int, x int , y int , z int) ''') db_session.execute('''INSERT INTO data VALUES (1,2,3,4,5,6) ''') db_session.execute('''INSERT INTO data VALUES (2,2,3,6,5,10) ''') db_session.execute('''INSERT INTO data VALUES (3,3,3,9,5,12) ''') db_session.execute('''INSERT INTO data VALUES (4,3,2,3,5,14) ''') db_session.execute('''INSERT INTO data VALUES (5,4,2,5,5,16) ''') db_session.execute('''INSERT INTO data VALUES (6,4,2,1,5,18) ''') db_session.execute('''INSERT INTO data VALUES (7,5,1,2,5,16) ''') db_session.execute('''INSERT INTO data VALUES (8,5,1,8,5,16) ''') db_session.execute('''INSERT INTO data VALUES (9,6,1,3,5,12) ''') db_session.execute('''INSERT INTO data VALUES (10,7,1,5,5,10) ''') db_session.execute('''INSERT INTO data VALUES (11,8,0,8,5,6) ''') db_session.add_all([p1,p2,p3,p4,p5,p6]) db_session.commit()
def seed_db(): db_session.execute('''CREATE TABLE data (a int, b int, c int, x int , y int , z int) ''') db_session.execute('''INSERT INTO data VALUES (1,2,3,4,5,6) ''') db_session.execute('''INSERT INTO data VALUES (2,2,3,6,5,10) ''') db_session.execute('''INSERT INTO data VALUES (3,3,3,9,5,12) ''') db_session.execute('''INSERT INTO data VALUES (4,3,2,3,5,14) ''') db_session.execute('''INSERT INTO data VALUES (5,4,2,5,5,16) ''') db_session.execute('''INSERT INTO data VALUES (6,4,2,1,5,18) ''') db_session.execute('''INSERT INTO data VALUES (7,5,1,2,5,16) ''') db_session.execute('''INSERT INTO data VALUES (8,5,1,8,5,16) ''') db_session.execute('''INSERT INTO data VALUES (9,6,1,3,5,12) ''') db_session.execute('''INSERT INTO data VALUES (10,7,1,5,5,10) ''') db_session.execute('''INSERT INTO data VALUES (11,8,0,8,5,6) ''') db_session.add_all([p1, p2, p3, p4, p5, p6]) db_session.commit()
def mutate(self, info, users): new_users = [] for user in users: new_user = UserModel(**input_to_dictionary(user, ["interests"])) if user.interests is not None: new_user.interests.extend([ UserInterest(interest=interest) for interest in user.interests ]) new_users.append(new_user) db_session.add_all(new_users) db_session.commit() ok = True return CreateUsers(ok=ok)
def mutate(self, info, adverts): new_adverts = [] for advert in adverts: new_advert = AdvertModel( **input_to_dictionary(advert, ["text_contents"])) if advert.text_contents is not None: new_advert.text_contents.extend([ AdvertText(text_content=text) for text in advert.text_contents ]) new_adverts.append(new_advert) db_session.add_all(new_adverts) db_session.commit() ok = True return CreateAdverts(ok=ok)
def populate(): #u = User('Sean','Johnson','*****@*****.**','pass') #db_session.add(u) db_session.add_all([ User('demo','demo','*****@*****.**','pass'), Accounts('Savings','*****@*****.**'), Accounts('Checking','*****@*****.**'), Accounts('Cash','*****@*****.**'), Categories('Emergency','*****@*****.**'), Categories('Vacation','*****@*****.**'), Categories('Car Repair','*****@*****.**'), Categories('Car Registration','*****@*****.**'), Categories('Car Replace','*****@*****.**'), Categories('Christmas','*****@*****.**'), Categories('Birthday','*****@*****.**'), Goals('*****@*****.**','Emergency','2013-01-01','Minimum Balance', 1000.00), Goals('*****@*****.**','Car Replace','2018-07-01','New truck in 5 yrs', 8000.00), Goals('*****@*****.**','Car Registration','2014-07-01','Annual Registration', 125.00), Goals('*****@*****.**','Car Repair','2013-09-01','Keep car running w/o debt', 500.00), Goals('*****@*****.**','Vacation','2014-08-30','Caribbean or Bust', 4000.00), Transactions('*****@*****.**','Savings','Emergency','Minimum Balance','2013-07-01', 'Its a start', 100.00), Transactions('*****@*****.**','Checking','Car Replace','New truck','2013-07-01','4x4',300.00), Transactions('*****@*****.**','Cash','Birthday','','2013-07-01','Suprise',150.00), Transactions('*****@*****.**','Savings','Vacation','Caribbean or Bust','2013-07-01', 'Its a start',100.00), Transactions('*****@*****.**','Savings','Emergency','','2013-07-01', 'Darn',-200.00), Transactions('*****@*****.**','Savings','Emergency','','2013-07-01', 'Building Emerg fund again',300.00), Transactions('*****@*****.**','Savings','Vacation','Caribbean or Bust' ,'2013-07-01', 'cant wait',300.00), Transactions('*****@*****.**','Savings','Emergency','Minimum Balance','2013-07-01', 'Its a start',100.00), Transactions('*****@*****.**','Checking','Car Replace','','2013-07-01', 'New truck',900.00), Transactions('*****@*****.**','Cash','Birthday','','2013-07-01', 'Suprise', 150.00), Transactions('*****@*****.**','Savings','Car Registration','Annual Registration','2013-07-01', 'Its a start',100.00), Transactions('*****@*****.**','Savings','Emergency','','2013-07-01', 'Darn',-200.00), Transactions('*****@*****.**','Savings','Emergency','Minimum Balance','2013-07-01', 'Building Emerg fund again',300.00), Transactions('*****@*****.**','Savings','Car Repair','Keep car running w/o debt','2013-07-01','cant wait',300.00), ]) db_session.commit()
def addUsers(filename): users = [] with open(filename) as csvfile: reader = csv.reader(csvfile) for row in reader: users.append( User(username=row[0], password=row[1], authlevel=row[2])) added = db_session.add_all(users) db_session.commit() return added
def CommitStreakData(fight, string1, string2): try: streak1, streak2 = FindStreak(string1, string2) fighter1 = db_session.query(Fighter).get(fight.fighter1_id) fighter2 = db_session.query(Fighter).get(fight.fighter2_id) fighter1.streak = streak1 fighter2.streak = streak2 db_session.add_all([fighter1,fighter2]) db_session.commit() except(TypeError): print ("Error has occured when parsing the player string. " + "Possibly no streak was presented. This happens in certain modes.") return
def relation2mysql(time_dir): file_path = csv_dir + "relation/%s/relation.csv" % (time_dir) f = open(file_path, "r") data = csv.reader(f) nrows = 0 relationList = [] print u"----> 开始存储关系数据..." cnt_suc = 0 for row in data: nrows += 1 #print len(row) #print "%s / %s " % (row[0], row[1]) try: pdf_path = str(row[0].strip()) pdf_page = int(row[1].strip()) mark_id = int(row[2].strip()) statement = db_session.query(VerStatement).filter( VerStatement.pdf_path == pdf_path, VerStatement.pdf_no == pdf_page, VerStatement.mark_id == mark_id).first() if statement == None: print "pdf_path=%s, pdf_page=%d, mark_id=%d" % ( pdf_path, pdf_page, mark_id) assert (statement != None) # 必须存在对应的entity stat_id = statement.id relation_no = row[5].strip() relation_no = int(relation_no) content = "".join([seg.strip() for seg in row[6:]]) newRelation = RelationMark(content, relation_no, stat_id) relationList.append(newRelation) cnt_suc += 1 except: print u"数据行%d 存在异常" % nrows print traceback.format_exc() db_session.add_all(relationList) db_session.commit() del relationList[:] print u"----> 关系数据存储完毕,总共存储数据%d行,数据异常共%d行" % (cnt_suc, nrows - cnt_suc)
def add_unit_entry(table): #평균테이블에 단위를 넣어주기 위한 함수. 만들고싶은 테이블의 형태는 # unit avg1 avg2 avg3 # 3분 # 5분 # 10분 #이러한 형태이고, unit은 미리 넣어줘야함. 각각 평균값들은 계속해서 업데이트됨. queries = db_session.query(table) id = 0 for i in queries: id = i.id #만약 기존에 생성되어있다면 진행하지 않음. if id < 3 and id >= 0: db_session.add_all([ # u 1 2 3 table('3minute', 0, 0, 0), # 3분 0 0 0 table('5minute', 0, 0, 0), # 5분 0 0 0 table('10minute', 0, 0, 0) # 10분 0 0 0 ]) db_session.commit()
def book_entry(): isbn = request.form['isbn'] librarian = current_user cip = CIP.query.get(isbn) if cip is None: book_name = request.form['book_name'] author = request.form['author'] publisher = request.form['publisher'] publish_year_month = datetime.strptime(request.form['publish_date'], '%Y-%m').date() cip = CIP(isbn=isbn, book_name=book_name, author=author, publisher=publisher, publish_year_month=publish_year_month, librarian=librarian) location = request.form['location'] book_ids = request.form.getlist('book_id') status = BookStatus.available if location == '图书流通室' else BookStatus.unborrowable books = [Book(id=book_id, cip=cip, location=location, status=status, librarian=librarian) for book_id in book_ids] db_session.add_all(books) db_session.commit() return jsonify({'success': True})
def initial_data(): system_user = SystemUser() system_user.username = '******' system_user.password = '******' system_user.admin = True db_session.add(system_user) db_session.add_all([ Department(id="01", name=u"LIMA"), Department(id="02", name=u"CALLAO"), Department(id="03", name=u"AMAZONAS"), Department(id="04", name=u"ANCASH"), Department(id="05", name=u"APURIMAC"), Department(id="06", name=u"AREQUIPA"), Department(id="07", name=u"AYACUCHO"), Department(id="08", name=u"CAJAMARCA"), Department(id="09", name=u"CUSCO"), Department(id="10", name=u"HUANCAVELICA"), Department(id="11", name=u"HUANUCO"), Department(id="12", name=u"ICA"), Department(id="13", name=u"JUNIN"), Department(id="14", name=u"LA LIBERTAD"), Department(id="15", name=u"LAMBAYEQUE"), Department(id="16", name=u"LORETO"), Department(id="17", name=u"MADRE DE DIOS"), Department(id="18", name=u"MOQUEGUA"), Department(id="19", name=u"PASCO"), Department(id="20", name=u"PIURA"), Department(id="21", name=u"PUNO"), Department(id="22", name=u"SAN MARTIN"), Department(id="23", name=u"TACNA"), Department(id="24", name=u"TUMBES"), Department(id="25", name=u"UCAYALI"), ]) db_session.commit() exit(0)
def entity2mysql(time_dir): file_path = csv_dir + "entity/%s/entity.csv" % (time_dir) f = open(file_path, "r") data = csv.reader(f) nrows = 0 entityList = [] print u"----> 开始存储实体数据..." duplicate_cnt = 0 for row in data: nrows += 1 pdf_path = str(row[0].strip()) pdf_page = int(row[1].strip()) mark_id = int(row[2].strip()) mark_user = str(row[3].strip()) statement = db_session.query(VerStatement).filter( VerStatement.pdf_path == pdf_path, VerStatement.pdf_no == pdf_page, VerStatement.mark_id == mark_id).first() if statement != None: ## 一般情况下不该存在同名元组 duplicate_cnt += 1 continue statement = VerStatement(pdf_no=pdf_page, pdf_path=pdf_path, mark_id=mark_id, mark_user=mark_user) db_session.add(statement) db_session.commit() stat_id = statement.id u""" 以硬性规则对标注做一定的补全 """ content = "".join([seg.strip() for seg in row[4:]]) content = content_completion(content) newEntity = EntityMark(content, stat_id) entityList.append(newEntity) db_session.add_all(entityList) db_session.commit() del entityList[:] print u"----> 实体数据存储完毕,总共存储数据%d行, 冲突id共%d行" % (nrows, duplicate_cnt)
def reports_step(self, simulation_id: int) -> None: start_time = datetime.datetime.utcnow() simulation: Simulation = db_session.query(Simulation).get(simulation_id) step: SimulationStep = SimulationStep(started_utc=start_time, origin="REPORT", simulation_id=simulation.id, status='ONGOING') db_session.add(step) db_session.flush() workdir: str = simulation.workdir output_path: str = workdir + "/plots/" if not os.path.exists(output_path): os.makedirs(output_path) simulation.current_step = "REPORT" simulation.current_step_id = step.id db_session.begin() db_session.add_all([simulation, step]) db_session.commit() result = chain( all_clones_plot_stats.s(workdir), all_clones_mullerplot.s(), noise_plot_stats.s(), major_clones_plot_stats.s(), major_clones_mullerplot.s(), mullerplot.s(), mutation_histogram.s(), simulation_report.s(), build_cell_model.s(), save_results_and_cleanup.s(simulation_id, step.id, workdir), ).apply_async() return result
def init_db(): Base.metadata.drop_all(bind=engine) Base.metadata.create_all(bind=engine) wall_question = Question(text='What was on your wall as a kid?') db_session.add(wall_question) pet_question = Question(text='What was your favorite pet?') db_session.add(pet_question) matt = Comrade(name='Matt', email='*****@*****.**') db_session.add(matt) julian = Comrade(name='Julian', email='*****@*****.**') db_session.add(julian) wall_1 = Round(name='wall_round_one', date=date.today(), question=wall_question) wall_2 = Round(name='wall_round_two', date=date.today() + one_week, question=wall_question) pet_1 = Round(name='pet_round_one', date=date.today() + 2 * one_week, question=pet_question) pet_2 = Round(name='pet_round_two', date=date.today() + 3 * one_week, question=pet_question) db_session.add_all([wall_1, wall_2, pet_1, pet_2]) db_session.commit() db_session.execute(comrade_to_round.insert().values([ (matt.uuid, wall_1.uuid), (julian.uuid, wall_2.uuid), (julian.uuid, pet_1.uuid), (matt.uuid, pet_2.uuid) ])) db_session.commit()
def save_results_and_cleanup(plots, simulation_id: int, step_id: int, workdir: str): output_path: str = workdir + "/plots/" plots: List[Artifact] = index_plots(output_path, simulation_id, step_id) reports: List[Artifact] = index_reports(workdir + "/reports/", simulation_id, step_id) simulation: Simulation = db_session.query(Simulation).get(simulation_id) step: SimulationStep = db_session.query(SimulationStep).get(step_id) db_session.flush() db_session.begin() end_time = datetime.datetime.utcnow() simulation.finished_utc = end_time simulation.status = 'SUCCESS' step.finished_utc = end_time step.status = 'SUCCESS' db_session.add_all(plots) db_session.add_all(reports) db_session.add_all([simulation, step]) db_session.commit() cleanup(workdir) return workdir
def addCode(pn): aid = int(request.args.get('article')) var = request.args.get('variable') val = request.args.get('value') ev = request.args.get('event') text = request.args.get('text') pn = int(pn) aqs = [] now = dt.datetime.now(tz = central).replace(tzinfo = None) if False: pass elif pn == 1: model = CodeFirstPass ## store highlighted text on first pass if text: p = CodeFirstPass(aid, var, val, current_user.id, text) else: p = CodeFirstPass(aid, var, val, current_user.id) ## update datetime on every edit aq = db_session.query(ArticleQueue).filter_by(article_id = aid, coder_id = current_user.id).first() aq.coded1_dt = now aqs.append(aq) elif pn == 2: model = CodeSecondPass p = CodeSecondPass(aid, ev, var, val, current_user.id) for aq in db_session.query(SecondPassQueue).filter_by(article_id = aid, coder_id = current_user.id).all(): aq.coded_dt = now aqs.append(aq) else: return make_response("Invalid model", 404) ## variables which only have one value per article if var in sv: if pn == 1: a = db_session.query(model).filter_by( article_id = aid, variable = var, coder_id = current_user.id ).all() else: ## for second pass, filter for distinct event a = db_session.query(model).filter_by( article_id = aid, variable = var, event_id = ev, coder_id = current_user.id ).all() ## if there's more then one, delete them if len(a) > 0: for o in a: db_session.delete(o); db_session.commit() ## if this is a 2nd comment pass comment and it is null, skip it if var == 'comments' and pn == 2 and val == '': return jsonify(result={"status": 200}) # try: db_session.add(p) db_session.add_all(aqs) db_session.commit() return jsonify(result={"status": 200})
def add_data(): # Add User: user1 = User(name="superuser", email="*****@*****.**", password="******", phone="+16509119111") # Add the Sensor Units: unit1 = SensorUnit(name="katies-kitchen", user=user1) unit2 = SensorUnit(name="pauls-house", user=user1) # Add the Sensors: sensor1 = Sensor(name="2882E92D6008F", dt_created=datetime(2014, 12, 27, 0, 0, 0, 0), active=1, sensorunit=unit1) sensor2 = Sensor(name="28FFEB6760144E7", dt_created=datetime(2015, 1, 25, 0, 0, 0, 0), active=1, sensorunit=unit1) sensor3 = Sensor(name="28FF1F51601441F", dt_created=datetime(2015, 1, 18, 0, 0, 0, 0), active=1, sensorunit=unit2) sensor4 = Sensor(name="28FFC81D60144E2", dt_created=datetime(2015, 1, 18, 0, 0, 0, 0), active=1, sensorunit=unit2) # Add Fermentation Temperature Profiles: profile1 = Profile(name="Test Profile", dt_created=datetime(2014, 12, 27, 14, 0, 0, 0), num_sensors=1, active=0, dt_stopped=datetime(2015, 1, 12, 1, 50, 0, 0), description="""This profile is a test run of the sensor and was montiroing the ambient temperature in the room, not an active fermentation. This gives a good baseline for what the ambient temperature will look like, so I can compare the fluctuations in ambient due to space conditioning, with the fluctuations when it is hooked up to an actual fermentation.""") st1 = SensorTable(sensor=sensor1, profile=profile1, label="Ambient") profile2 = Profile(name="Single Hop Cascade", dt_created=datetime(2015, 1, 12, 1, 52, 0, 0), num_sensors=1, active=0, dt_stopped=datetime(2015, 1, 24, 19, 30, 0, 0), description="""Currently the temperature sensor is hooked up to a one gallon batch of a single hop pale ale. The sensor was attached to the outside of the fermentation carboy, and I used a piece of packing foam to insulate the sensor from the ambient conditions. Based on the fluctuations observed during the test run, it appears that the insulation is working as intended.""") st2 = SensorTable(sensor=sensor1, profile=profile2, label="Fermentation") profile3 = Profile(name="Single Hop Sorachi", dt_created=datetime(2015, 1, 25, 1, 30, 0, 0), num_sensors=2, active=1, dt_stopped=datetime.now(), description="""Currently the temperature sensors are hooked up to a one gallon batch of Single Hop (Sorachi Ace) Pale Ale. I brewed two single hop batches in the same session (the other with Amarillo), and hooked up the fementation sensor to the Sorachi carboy. The other sensor is monitoring ambient temperature.""") st3 = SensorTable(sensor=sensor1, profile=profile3, label="Ambient") st4 = SensorTable(sensor=sensor2, profile=profile3, label="Fermentation") profile4 = Profile(name="Imperial Stout", dt_created=datetime(2015, 1, 18, 13, 0, 0, 0), num_sensors=2, active=0, dt_stopped=datetime(2015, 2, 1, 2, 34, 0, 0), description="""Currently the temperature sensors are hooked up to a five gallon batch of an Imperial Stout. One sensor is attached to the fermentation bucket, with some foam insulation on the back, and the other temperature sensor is recording the ambient conditions.""") st5 = SensorTable(sensor=sensor3, profile=profile4, label="Ambient") # need to check this st6 = SensorTable(sensor=sensor4, profile=profile4, label="Fermentation") # need to check this profile5 = Profile(name="Honey Ale", dt_created=datetime(2015, 2, 1, 2, 35, 0, 0), num_sensors=2, active=0, dt_stopped=datetime.now(), description="""Currently the temperature sensors are hooked up to a five gallon batch of an honey ale. One sensor is attached to the fermentation bucket, with some foam insulation on the back, and the other temperature sensor is recording the ambient conditions.""") st7 = SensorTable(sensor=sensor3, profile=profile5, label="Ambient") # need to check this st8 = SensorTable(sensor=sensor4, profile=profile5, label="Fermentation") # need to check this # Add some Notes for the Profiles: note1 = ProfileNote(dt_created=datetime(2015, 1, 12, 2, 30, 0, 0), dt_applied=datetime(2015, 1, 12, 2, 30, 0, 0), note="""Pitched the Yeast way too hot, probably should have cooled the wort a few more degrees.""", profile_note=profile2) note2 = ProfileNote(dt_created=datetime(2015, 1, 18, 2, 0, 0, 0), dt_applied=datetime(2015, 1, 18, 2, 0, 0, 0), note="""Updated the Arduino/Python code, but wasn't able to update this sensor unit remotely, so the unit went offline for a bit.""", profile_note=profile2) db_session.add_all([unit1, unit2, sensor1, sensor2, sensor3, sensor4, profile1, profile2, profile3, profile4, st1, st2, st3, st4, st5, st6, st7, st8, note1, note2]) db_session.commit()
def run_state_machine(db, tablemd, schema): """ Execute a state machine to analyze and extract location data from a single table """ tablename = tablemd.tablename print tablename cols = schema.columns.keys() if tablemd.state == 0 or ('_latlon' not in cols and '_address' not in cols): # check if table exists in metadata table print '\tstate 0' if '_latlon' not in cols and '_address' not in cols: # create shadow columns db.execute(ALTERQUERY % tablename) tablemd.state = 1 db_session.add(tablemd) db_session.commit() if tablemd.state == 1: # check if shadow cols exist print '\tstate 1' # loc -> [(column, extractor)] extractors = find_location_extractors(db, tablemd, schema) annos = [] for anntype, pairs in extractors.iteritems(): for colname, extractor in pairs: annos.append(Annotation(colname, anntype, extractor, tablemd)) tablemd.hasloc = len(annos) > 0 tablemd.state = 2 db_session.add_all(annos) db_session.add(tablemd) db_session.commit() return tablemd.state if tablemd.state == 2: # check if table needs to be analyzed for annotations print '\tstate 2' needs_geocoding = populate_shadow_cols(db, tablemd, schema) tablemd.state = 3 if needs_geocoding else 5 db_session.add(tablemd) db_session.commit() return tablemd.state if tablemd.state == 3 or tablemd.state == 4: # check if table has annotations and no geocode/shapes print '\tstate 3/4' maxid = geocode_table(db, tablemd) try: if maxid == db.execute('select max(id) from %s' %tablename).fetchone()[0]: tablemd.state = 5 db_session.add(tablemd) db_session.commit() except Exception as e: raise return tablemd.state # check if user added any new annotations since # last geocode if tablemd.state == 5: print '\tstate 5' return tablemd.state return tablemd.state
"""Populate the database for testing""" from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker from database import init_db, Base, db_session as session from models import User, Category, Item init_db() # Create users darth_vader = User(name="Darth Vader", email="*****@*****.**") emperor_palpatine = User( name="Emperor Palpatine", email="*****@*****.**") revan = User(name="Revan", email="*****@*****.**") session.add_all([darth_vader, emperor_palpatine, revan]) session.commit() # Create categories # Reference: http://starwars.wikia.com/ starfighter = Category(name="Starfighter", description="Small one- or " "two-man vessels typically used for dogfighting.") capital_ship = Category(name="Capital Ship", description="Largest of " "starships, often used as a warship.") scout_vessel = Category(name="Scout Vessel", description="Small, fast " "ships used to scout an area before a main " "force arrives.") blaster_pistol = Category(name="Blaster Pistol", description="Small " "handheld blasters.") blaster_rifle = Category(name="Blaster Rifle", description="Heavy duty " "blaster variants, much more powerful than pistols.")