def insert_names_into_db(batch_size: int = 10000, limit: int = 200000):
with open('name.basics.tsv') as names_tsv:
names_tsv.readline()
reader = csv.reader(names_tsv, delimiter='\t')
counter = 0
names = []
number_in_batch = 0
for row in tqdm(reader):
if number_in_batch > batch_size:
number_in_batch = 0
db_session.add_all(names)
db_session.commit()
names = []
counter += batch_size
print(f'added {counter} names')
if limit and counter >= limit:
return
row = list(map(map_null, row))
name = Name(
nconst=row[0],
primaryName=row[1],
birthYear=row[2],
deathYear=row[3],
primaryProfession=row[4],
)
if row[5]:
titles = Title.query.filter(Title.tconst.in_(
row[5].split(',')))
name.knownForTitles = list(titles)
names.append(name)
number_in_batch += 1
db_session.commit()
def generate_fake_data(cups, samples):
db_session.add_all([CupModel(id=cup) for cup in range(cups)])
db_session.commit()
timestamp = datetime.datetime.now() - datetime.timedelta(minutes=10)
for cup in range(cups):
generate_fake_cup(cup, timestamp, samples)
def insert_titles_into_db(batch_size: int = 10000, limit: int = 200000):
with open('title.basics.tsv') as titles_tsv:
titles_tsv.readline()
reader = csv.reader(titles_tsv, delimiter='\t')
counter = 0
titles = []
number_in_batch = 0
for row in tqdm(reader):
if number_in_batch > batch_size:
number_in_batch = 0
db_session.add_all(titles)
db_session.commit()
titles = []
counter += batch_size
print(f'added {counter} titles')
if limit and counter >= limit:
return
row = list(map(map_null, row))
title = Title(tconst=row[0],
titleType=row[1],
primaryTitle=row[2],
originalTitle=row[3],
isAdult=bool(int(row[4])),
startYear=row[5],
endYear=row[6],
runtimeMinutes=row[7],
genres=row[8])
titles.append(title)
number_in_batch += 1
db_session.commit()
def assign_tasks(cls):
books = Book.gets()
tasks = []
weight_2 = []
weight_3 = []
for book in books:
book_source = BookSource.get_by_bid(book.id)
if book_source is None:
print '%d has no book source info' % book.id
continue
if book.latest_chapter:
task = cls(book.id, book.latest_chapter.title,
book_source.source_site, book_source.source_url)
tasks.append(task)
if book.weight != 1:
if book.weight == 2:
weight_2.append(task)
elif book.weight == 3:
weight_3.append(task)
# 处理权重
p_half = len(tasks) / 2
p_third_one = len(tasks) / 3
p_third_two = len(tasks) / 3 * 2
tasks[p_half:p_half] = weight_2
tasks[p_third_one:p_third_one] = weight_3
tasks[p_third_two:p_third_two] = weight_3
db_session.add_all(tasks)
db_session.commit()
def generate_fake_cup(cup, timestamp, samples):
values = (0, 0, 0, 0)
for _ in range(samples):
delta = [random.randint(0, 20) for _ in range(4)]
values = [sum(x) for x in zip(values, delta)]
timestamp += datetime.timedelta(seconds=5)
db_session.add_all([
GreyscaleModel(
value=values[0],
timestamp=timestamp,
cup_id=cup,
),
HumidityModel(
value=values[1],
timestamp=timestamp,
cup_id=cup,
),
LiquidModel(
value=values[2],
timestamp=timestamp,
cup_id=cup,
),
TemperatureModel(
value=values[3],
timestamp=timestamp,
cup_id=cup,
)
])
db_session.commit()
def assignmentToCoders(article_ids, coder_ids, pass_number='1'):
""" Assigns each article to all coders in list. """
if pass_number == '1':
model = ArticleQueue
elif pass_number == '2':
model = SecondPassQueue
elif pass_number == 'ec':
model = EventCreatorQueue
## add to queues
to_add = []
for coder_id in coder_ids:
existing = [x.article_id for x in db_session.query(model).all()]
for a in article_ids:
if a not in existing:
if pass_number == '1':
item = ArticleQueue(article_id=a, coder_id=coder_id)
elif pass_number == '2':
item = SecondPassQueue(article_id=a, coder_id=coder_id)
elif pass_number == 'ec':
item = EventCreatorQueue(article_id=a, coder_id=coder_id)
to_add.append(item)
db_session.add_all(to_add)
db_session.commit()
return len(to_add)
def assignmentToCoders(article_ids, coder_ids, pass_number):
""" Assigns each article to all coders in list. """
if pass_number == '1':
model = ArticleQueue
elif pass_number == '2':
model = SecondPassQueue
elif pass_number == 'ec':
model = EventCreatorQueue
## add to queues
to_add = []
for c in coder_ids:
for a in article_ids:
if pass_number == '1':
item = ArticleQueue(article_id=a, coder_id=c)
elif pass_number == '2':
item = SecondPassQueue(article_id=a, coder_id=c)
elif pass_number == 'ec':
item = EventCreatorQueue(article_id=a, coder_id=c)
to_add.append(item)
db_session.add_all(to_add)
db_session.commit()
return len(to_add)
def test_input_data(self):
u1 = User(name="Paul", email="*****@*****.**",
password="******")
u2 = User(name="Joe", email="*****@*****.**",
password="******")
s1 = Alert(name="guitars", string="guitars -steel", status=1,
last_update=datetime.date(2013, 9, 25), user=u1)
s2 = Alert(name="banjo", string="banjo -100", status=1,
last_update=datetime.date(2013, 9, 30), user=u1)
s3 = Alert(name="judo stuff", string="gee", status=1,
last_update=datetime.date(2013, 5, 13), user=u2)
s4 = Alert(name="bmx bike", string="bmx -hoffman", status=0,
last_update=datetime.date(2011, 3, 5), user=u2)
r1 = Result(title="Guitar for Sale - $100",
link="http://craigslist.com/12341234", alert=s1)
r2 = Result(title="Shitty Guitar for Sale - $50",
link="http://craigslist.com/54325432", alert=s1)
r3 = Result(title="Even shittier Guitar for Sale - $30",
link="http://craigslist.com/4672345", alert=s1)
db_session.add_all([u1, u2, s1, s2, s3, s4, r1, r2, r3])
db_session.commit()
print "****Data Has Been Commited to DB****"
def test_input_data(self):
# Add users
u1 = User(name="Paul", email="*****@*****.**",
password="******")
u2 = User(name="Joe", email="*****@*****.**",
password="******")
# Add beers
b1 = Beer(name="Chaisoin", style = "Saison");
b2 = Beer(name="Orca Porter", style = "Porter");
# Add brewers to beers
brewer1 = Brewer(user=u1, beer=b1)
brewer2 = Brewer(user=u2, beer=b1)
brewer3 = Brewer(user=u1, beer=b2)
# Add a Recipe to a beer
recipe1 = Recipe(steep_time=60, boil_time=60, brew_date=date(2014, 10, 31),
beer=b1)
malt1 = Malt(variety="Pale Malt LME", weight=5, recipe=recipe1)
malt2 = Malt(variety="Great Western 2-row", weight=4.5, recipe=recipe1)
hop1 = Hop(variety="Magnum", addition_time=60, weight=1, recipe=recipe1)
hop2 = Hop(variety="Cascade", addition_time=0, weight=1, recipe=recipe1)
db_session.add_all([u1, u2, b1, b2, brewer1, brewer2, brewer3])
db_session.add_all([recipe1, malt1, malt2, hop1, hop2])
db_session.commit()
print "****Data Has Been Commited to DB****"
def assign_tasks(cls):
books = Book.gets()
tasks = []
weight_2 = []
weight_3 = []
for book in books:
book_source = BookSource.get_by_bid(book.id)
if book_source is None:
print '%d has no book source info' % book.id
continue
if book.latest_chapter:
task = cls(book.id, book.latest_chapter.title,
book_source.source_site, book_source.source_url)
tasks.append(task)
if book.weight != 1:
if book.weight == 2:
weight_2.append(task)
elif book.weight == 3:
weight_3.append(task)
# 处理权重
p_half = len(tasks) / 2
p_third_one = len(tasks) / 3
p_third_two = len(tasks) / 3 * 2
tasks[p_half:p_half] = weight_2
tasks[p_third_one:p_third_one] = weight_3
tasks[p_third_two:p_third_two] = weight_3
db_session.add_all(tasks)
db_session.commit()
def setUp(self):
init_db()
self.client = Client(schema)
user = User(name="Foo", password="******")
post = Post(author=user, title="Test", content="Test Content")
comment = Comment(author=user, post=post, content="Comment Test")
db_session.add_all([user, post, comment])
db_session.commit()
def resetVariableOptions():
""" Load current dropdowns from file. """
db_session.query(VarOption).delete()
print("Adding variables...")
dds = []
with open("dropdowns.csv") as csvfile:
reader = csv.reader(csvfile)
for row in reader:
dds.append(VarOption(variable=row[0], option=row[1]))
db_session.add_all(dds)
db_session.commit()
def resetVariableOptions():
""" Load current dropdowns from file. """
db_session.query(VarOption).delete()
print("Adding variables...")
dds = []
with open("dropdowns.csv") as csvfile:
reader = csv.reader(csvfile)
for row in reader:
dds.append( VarOption(variable = row[0], option = row[1]) )
db_session.add_all(dds)
db_session.commit()
def addArticles(filename, db_name):
articles = []
with open(filename) as csvfile:
reader = csv.reader(csvfile)
for row in reader:
title = row[0]
db_id = row[1]
if title == 'TITLE':
continue
articles.append( ArticleMetadata(filename = db_id, db_id = db_id, title = title, db_name = db_name) )
db_session.add_all(articles)
db_session.commit()
def addArticlesExample(db_name = 'test'):
""" Add articles from example directory. """
print("Adding example articles...")
articles = []
for f in glob.iglob(jsonConfig['DOC_ROOT'] + "*.txt"):
filename = f.split('/')[-1]
lines = open(f, 'r').read().split("\n")
title = lines[0].replace("TITLE: ", "")
articles.append( ArticleMetadata(filename = filename, title = unicode(title, "utf-8", errors = "ignore"), db_name = db_name) )
db_session.add_all(articles)
db_session.commit()
def create_user():
init_db()
# Создаем 2 категории
games = Category(name='Games')
films = Category(name='Films')
db_session.add_all([games, films])
# Создаем товары
SOME_TEXT = 'some_text'
skyrim = Product(name='Skyrim', price=10.1, category=games, description=SOME_TEXT)
fallout = Product(name='Fallout', price=5.5, category=games, description=SOME_TEXT)
matrix = Product(name='Matrix', price=1.2, category=films, description=SOME_TEXT)
db_session.add_all([skyrim, fallout, matrix])
# commit
db_session.commit()
def setup_workdir(request_data: dict) -> Artifact:
"""
Creates new dir for simulation and places simulation configuration file in it
:param request_data: Flask request with configuration
:return: tuple with path to workdir and saved configuration
"""
conf_name = get_conf_name(request_data['configurationName'])
conf = request_data['configuration']
start_time = datetime.datetime.utcnow()
simulation = Simulation(started_utc=start_time,
name="test_simulation",
current_step="CLI",
status='ONGOING')
db_session.add(simulation)
db_session.flush()
step = SimulationStep(started_utc=start_time,
origin="CLI",
simulation_id=simulation.id,
status='ONGOING')
db_session.add(step)
db_session.flush()
workdir_path = create_workdir(simulation.id)
conf_path = '{}/{}'.format(workdir_path, conf_name)
simulation.workdir = workdir_path
simulation.current_step_id = step.id
with open(conf_path, 'w+') as f:
json.dump(conf, f, indent=2)
configuration = Artifact(size_kb=os.path.getsize(conf_path),
path=conf_path,
created_utc=start_time,
step_id=step.id,
name=conf_name,
file_type='JSON',
simulation_id=simulation.id)
simulation.artifacts.append(configuration)
step.artifacts.append(configuration)
simulation.steps.append(step)
db_session.begin()
db_session.add_all([configuration, step, simulation])
db_session.flush()
db_session.commit()
return configuration
def execute_state_1(db, tablemd, schema):
"annotype -> [(column, extractorname)]"
print "state 1\t", tablemd.tablename
extractors = find_location_extractors(db, tablemd, schema)
annos = []
for anntype, pairs in extractors.iteritems():
for colname, extractor in pairs:
annos.append(Annotation(colname, anntype, extractor, tablemd))
tablemd.hasloc = len(annos) > 0
tablemd.state = 2
db_session.add_all(annos)
db_session.add(tablemd)
db_session.commit()
return tablemd.state
def test_get_configuration(self):
"""test the get_configuration function"""
string_config = model.Configuration(
"String_Config", "sample", "string", "admin"
)
bool_config = model.Configuration("Bool_Config", "True", "bool", "admin")
integer_config = model.Configuration("Integer_Config", "12", "integer", "admin")
db_session.add_all([string_config, bool_config, integer_config])
db_session.commit()
self.assertEqual(util.get_configuration("String_Config"), "sample")
self.assertEqual(util.get_configuration("Bool_Config"), True)
self.assertEqual(util.get_configuration("Integer_Config"), 12)
def addArticlesExample(db_name = 'test'):
""" Add articles from example directory. """
print("Adding example articles...")
articles = []
for f in glob.iglob(config.DOC_ROOT + "*.txt"):
print(f)
filename = f.split('/')[-1]
lines = open(f, 'r').read().split("\n")
title = lines[0].replace("TITLE: ", "")
articles.append( ArticleMetadata(filename = filename, title = title, db_name = db_name) )
db_session.add_all(articles)
db_session.commit()
def addQueueExample():
print("Adding example queues...")
articles = db_session.query(ArticleMetadata).all()
random.shuffle(articles)
users = db_session.query(User).filter(User.authlevel == 1).all()
aq = []
## assign articles randomly to core team members for funsies
for a in articles:
for u in users:
aq.append(ArticleQueue(article_id=a.id, coder_id=u.id))
db_session.add_all(aq)
db_session.commit()
def addQueueExample():
print("Adding example queues...")
articles = db_session.query(ArticleMetadata).all()
random.shuffle(articles)
users = db_session.query(User).filter(User.authlevel == 1).all()
aq = []
## assign articles randomly to core team members for funsies
for a in articles:
for u in users:
aq.append( ArticleQueue(article_id = a.id, coder_id = u.id) )
db_session.add_all(aq)
db_session.commit()
def addArticlesExample(db_name='test'):
""" Add articles from example directory. """
print("Adding example articles...")
articles = []
for f in glob.iglob(jsonConfig['DOC_ROOT'] + "*.txt"):
filename = f.split('/')[-1]
lines = open(f, 'r').read().split("\n")
title = lines[0].replace("TITLE: ", "")
articles.append(
ArticleMetadata(filename=filename,
title=unicode(title, "utf-8", errors="ignore"),
db_name=db_name))
db_session.add_all(articles)
db_session.commit()
def RecordFightToDB(fighter1,fighter2,fight,Winner):
#Identify winner and save as foreign key
if Winner == fighter1.name:
fight.winner = fighter1
try:
fighter1.streak += 1
except(TypeError):
print "No streak found. Current fight mode did not give one."
try:
fighter2.streak -= 1
except(TypeError):
print "No streak found. Current fight mode did not give one."
elif Winner == fighter2.name:
fight.winner = fighter2
try:
fighter2.streak += 1
except(TypeError):
print "No streak found. Current fight mode did not give one."
try:
fighter1.streak -= 1
except(TypeError):
print "No streak found. Current fight mode did not give one."
else:
print ("Unable to match declared winner with a fighter.")
return
fight.time = datetime.datetime.now()
#SQLAlchemey is similar to git, one must add then commit.
#Unlike others once added the pk is created
db_session.add(fight)
#determine ratios and averages
avgBetRatio1, winRate1 = GetFighterStats(fighter1)
avgBetRatio2, winRate2 = GetFighterStats(fighter2)
#Save fighter values.
fighter1.winRate = winRate1
fighter1.betRatio = avgBetRatio1
fighter2.winRate = winRate2
fighter2.betRatio = avgBetRatio2
db_session.add_all([fighter1,fighter2])
db_session.commit()
def install():
from database import engine
from datamodel import Base, User, PrefType, PrefWeight
Base.metadata.create_all(engine)
#Add initial admin user#
user = User(user_name='harvimt',user_type='admin')
sess.add(user)
#add pref types
sess.add_all((
PrefType('Time', 'rank', 5.0),
PrefType('Faculty', 'weight', 2.0),
PrefType('Theme', 'weight', 1.0)
))
sess.commit()
#and pref weights
sess.add_all((
PrefWeight(1,0,0.00),
PrefWeight(1,1,0.25),
PrefWeight(1,2,0.50),
PrefWeight(1,3,0.75),
PrefWeight(1,4,1.00),
PrefWeight(1,5,1.25),
PrefWeight(1,6,1.50),
PrefWeight(2,0,0.00),
PrefWeight(2,1,0.33),
PrefWeight(2,2,0.66),
PrefWeight(2,3,1.00),
PrefWeight(2,4,1.33),
PrefWeight(2,5,1.67),
PrefWeight(2,6,2.00),
PrefWeight(3,0,0.00),
PrefWeight(3,1,0.17),
PrefWeight(3,2,0.33),
PrefWeight(3,3,0.50),
PrefWeight(3,4,0.67),
PrefWeight(3,5,0.83),
PrefWeight(3,6,1.00)
))
sess.commit()
def setup_contest():
roles = {x.name: x for x in model.UserRole.query.all()}
test_contestant = model.User(
"testuser", "Test User", "pass", user_roles=[roles["defendant"]]
)
test_executioner = model.User(
"testexec", "Test Executioner", "epass", user_roles=[roles["executioner"]]
)
test_contest = model.Contest(
"test_contest",
util.str_to_dt("2017-02-05T22:04:00Z"),
util.str_to_dt("2030-01-01T11:11:00Z"),
True,
)
io_problem_type = model.ProblemType.query.filter_by(name="input-output").one()
test_problem = model.Problem(
io_problem_type,
"fizzbuzz",
"FizzBuzz",
"## FizzBuzz\nPerform fizzbuzz up to the given number",
"3",
"1\n2\nFizz",
"15",
"1\n2\nFizz\n4\nBuzz\nFizz\n7\n8\n9\nBuzz\n11\nFizz\n13\n14\nFizzBuzz\n",
)
test_contest.problems.append(test_problem)
test_contest.users.append(test_contestant)
python = model.Language.query.filter_by(name="python").one()
test_run = model.Run(
test_contestant,
test_contest,
python,
test_problem,
util.str_to_dt("2017-02-05T23:00:00Z"),
'import sys\nn=raw_input()\nfor i in range(1, n+1): print("Fizz"*(i%3==0)+"Buzz"*(i%5==0) or i)',
test_problem.secret_input,
test_problem.secret_output,
True,
)
db_session.add_all(
[test_executioner, test_contestant, test_contest, test_problem, test_run]
)
db_session.commit()
def seed_db():
db_session.execute('''CREATE TABLE data
(a int, b int, c int,
x int , y int , z int) ''')
db_session.execute('''INSERT INTO data VALUES (1,2,3,4,5,6) ''')
db_session.execute('''INSERT INTO data VALUES (2,2,3,6,5,10) ''')
db_session.execute('''INSERT INTO data VALUES (3,3,3,9,5,12) ''')
db_session.execute('''INSERT INTO data VALUES (4,3,2,3,5,14) ''')
db_session.execute('''INSERT INTO data VALUES (5,4,2,5,5,16) ''')
db_session.execute('''INSERT INTO data VALUES (6,4,2,1,5,18) ''')
db_session.execute('''INSERT INTO data VALUES (7,5,1,2,5,16) ''')
db_session.execute('''INSERT INTO data VALUES (8,5,1,8,5,16) ''')
db_session.execute('''INSERT INTO data VALUES (9,6,1,3,5,12) ''')
db_session.execute('''INSERT INTO data VALUES (10,7,1,5,5,10) ''')
db_session.execute('''INSERT INTO data VALUES (11,8,0,8,5,6) ''')
db_session.add_all([p1,p2,p3,p4,p5,p6])
db_session.commit()
def seed_db():
db_session.execute('''CREATE TABLE data
(a int, b int, c int,
x int , y int , z int) ''')
db_session.execute('''INSERT INTO data VALUES (1,2,3,4,5,6) ''')
db_session.execute('''INSERT INTO data VALUES (2,2,3,6,5,10) ''')
db_session.execute('''INSERT INTO data VALUES (3,3,3,9,5,12) ''')
db_session.execute('''INSERT INTO data VALUES (4,3,2,3,5,14) ''')
db_session.execute('''INSERT INTO data VALUES (5,4,2,5,5,16) ''')
db_session.execute('''INSERT INTO data VALUES (6,4,2,1,5,18) ''')
db_session.execute('''INSERT INTO data VALUES (7,5,1,2,5,16) ''')
db_session.execute('''INSERT INTO data VALUES (8,5,1,8,5,16) ''')
db_session.execute('''INSERT INTO data VALUES (9,6,1,3,5,12) ''')
db_session.execute('''INSERT INTO data VALUES (10,7,1,5,5,10) ''')
db_session.execute('''INSERT INTO data VALUES (11,8,0,8,5,6) ''')
db_session.add_all([p1, p2, p3, p4, p5, p6])
db_session.commit()
def mutate(self, info, users):
new_users = []
for user in users:
new_user = UserModel(**input_to_dictionary(user, ["interests"]))
if user.interests is not None:
new_user.interests.extend([
UserInterest(interest=interest)
for interest in user.interests
])
new_users.append(new_user)
db_session.add_all(new_users)
db_session.commit()
ok = True
return CreateUsers(ok=ok)
def mutate(self, info, adverts):
new_adverts = []
for advert in adverts:
new_advert = AdvertModel(
**input_to_dictionary(advert, ["text_contents"]))
if advert.text_contents is not None:
new_advert.text_contents.extend([
AdvertText(text_content=text)
for text in advert.text_contents
])
new_adverts.append(new_advert)
db_session.add_all(new_adverts)
db_session.commit()
ok = True
return CreateAdverts(ok=ok)
def populate():
#u = User('Sean','Johnson','*****@*****.**','pass')
#db_session.add(u)
db_session.add_all([
User('demo','demo','*****@*****.**','pass'),
Accounts('Savings','*****@*****.**'),
Accounts('Checking','*****@*****.**'),
Accounts('Cash','*****@*****.**'),
Categories('Emergency','*****@*****.**'),
Categories('Vacation','*****@*****.**'),
Categories('Car Repair','*****@*****.**'),
Categories('Car Registration','*****@*****.**'),
Categories('Car Replace','*****@*****.**'),
Categories('Christmas','*****@*****.**'),
Categories('Birthday','*****@*****.**'),
Goals('*****@*****.**','Emergency','2013-01-01','Minimum Balance', 1000.00),
Goals('*****@*****.**','Car Replace','2018-07-01','New truck in 5 yrs', 8000.00),
Goals('*****@*****.**','Car Registration','2014-07-01','Annual Registration', 125.00),
Goals('*****@*****.**','Car Repair','2013-09-01','Keep car running w/o debt', 500.00),
Goals('*****@*****.**','Vacation','2014-08-30','Caribbean or Bust', 4000.00),
Transactions('*****@*****.**','Savings','Emergency','Minimum Balance','2013-07-01', 'Its a start', 100.00),
Transactions('*****@*****.**','Checking','Car Replace','New truck','2013-07-01','4x4',300.00),
Transactions('*****@*****.**','Cash','Birthday','','2013-07-01','Suprise',150.00),
Transactions('*****@*****.**','Savings','Vacation','Caribbean or Bust','2013-07-01', 'Its a start',100.00),
Transactions('*****@*****.**','Savings','Emergency','','2013-07-01', 'Darn',-200.00),
Transactions('*****@*****.**','Savings','Emergency','','2013-07-01', 'Building Emerg fund again',300.00),
Transactions('*****@*****.**','Savings','Vacation','Caribbean or Bust' ,'2013-07-01', 'cant wait',300.00),
Transactions('*****@*****.**','Savings','Emergency','Minimum Balance','2013-07-01', 'Its a start',100.00),
Transactions('*****@*****.**','Checking','Car Replace','','2013-07-01', 'New truck',900.00),
Transactions('*****@*****.**','Cash','Birthday','','2013-07-01', 'Suprise', 150.00),
Transactions('*****@*****.**','Savings','Car Registration','Annual Registration','2013-07-01', 'Its a start',100.00),
Transactions('*****@*****.**','Savings','Emergency','','2013-07-01', 'Darn',-200.00),
Transactions('*****@*****.**','Savings','Emergency','Minimum Balance','2013-07-01', 'Building Emerg fund again',300.00),
Transactions('*****@*****.**','Savings','Car Repair','Keep car running w/o debt','2013-07-01','cant wait',300.00),
])
db_session.commit()
def addUsers(filename):
users = []
with open(filename) as csvfile:
reader = csv.reader(csvfile)
for row in reader:
users.append(
User(username=row[0], password=row[1], authlevel=row[2]))
added = db_session.add_all(users)
db_session.commit()
return added
def CommitStreakData(fight, string1, string2):
try:
streak1, streak2 = FindStreak(string1, string2)
fighter1 = db_session.query(Fighter).get(fight.fighter1_id)
fighter2 = db_session.query(Fighter).get(fight.fighter2_id)
fighter1.streak = streak1
fighter2.streak = streak2
db_session.add_all([fighter1,fighter2])
db_session.commit()
except(TypeError):
print ("Error has occured when parsing the player string. " +
"Possibly no streak was presented. This happens in certain modes.")
return
def relation2mysql(time_dir):
file_path = csv_dir + "relation/%s/relation.csv" % (time_dir)
f = open(file_path, "r")
data = csv.reader(f)
nrows = 0
relationList = []
print u"----> 开始存储关系数据..."
cnt_suc = 0
for row in data:
nrows += 1
#print len(row)
#print "%s / %s " % (row[0], row[1])
try:
pdf_path = str(row[0].strip())
pdf_page = int(row[1].strip())
mark_id = int(row[2].strip())
statement = db_session.query(VerStatement).filter(
VerStatement.pdf_path == pdf_path,
VerStatement.pdf_no == pdf_page,
VerStatement.mark_id == mark_id).first()
if statement == None:
print "pdf_path=%s, pdf_page=%d, mark_id=%d" % (
pdf_path, pdf_page, mark_id)
assert (statement != None) # 必须存在对应的entity
stat_id = statement.id
relation_no = row[5].strip()
relation_no = int(relation_no)
content = "".join([seg.strip() for seg in row[6:]])
newRelation = RelationMark(content, relation_no, stat_id)
relationList.append(newRelation)
cnt_suc += 1
except:
print u"数据行%d 存在异常" % nrows
print traceback.format_exc()
db_session.add_all(relationList)
db_session.commit()
del relationList[:]
print u"----> 关系数据存储完毕，总共存储数据%d行，数据异常共%d行" % (cnt_suc, nrows - cnt_suc)
def add_unit_entry(table):
#평균테이블에 단위를 넣어주기 위한 함수. 만들고싶은 테이블의 형태는
# unit avg1 avg2 avg3
# 3분
# 5분
# 10분
#이러한 형태이고, unit은 미리 넣어줘야함. 각각 평균값들은 계속해서 업데이트됨.
queries = db_session.query(table)
id = 0
for i in queries:
id = i.id
#만약 기존에 생성되어있다면 진행하지 않음.
if id < 3 and id >= 0:
db_session.add_all([ # u 1 2 3
table('3minute', 0, 0, 0), # 3분 0 0 0
table('5minute', 0, 0, 0), # 5분 0 0 0
table('10minute', 0, 0, 0) # 10분 0 0 0
])
db_session.commit()
def book_entry():
isbn = request.form['isbn']
librarian = current_user
cip = CIP.query.get(isbn)
if cip is None:
book_name = request.form['book_name']
author = request.form['author']
publisher = request.form['publisher']
publish_year_month = datetime.strptime(request.form['publish_date'], '%Y-%m').date()
cip = CIP(isbn=isbn, book_name=book_name, author=author, publisher=publisher, publish_year_month=publish_year_month, librarian=librarian)
location = request.form['location']
book_ids = request.form.getlist('book_id')
status = BookStatus.available if location == '图书流通室' else BookStatus.unborrowable
books = [Book(id=book_id, cip=cip, location=location, status=status, librarian=librarian) for book_id in book_ids]
db_session.add_all(books)
db_session.commit()
return jsonify({'success': True})
def initial_data():
system_user = SystemUser()
system_user.username = '******'
system_user.password = '******'
system_user.admin = True
db_session.add(system_user)
db_session.add_all([
Department(id="01", name=u"LIMA"),
Department(id="02", name=u"CALLAO"),
Department(id="03", name=u"AMAZONAS"),
Department(id="04", name=u"ANCASH"),
Department(id="05", name=u"APURIMAC"),
Department(id="06", name=u"AREQUIPA"),
Department(id="07", name=u"AYACUCHO"),
Department(id="08", name=u"CAJAMARCA"),
Department(id="09", name=u"CUSCO"),
Department(id="10", name=u"HUANCAVELICA"),
Department(id="11", name=u"HUANUCO"),
Department(id="12", name=u"ICA"),
Department(id="13", name=u"JUNIN"),
Department(id="14", name=u"LA LIBERTAD"),
Department(id="15", name=u"LAMBAYEQUE"),
Department(id="16", name=u"LORETO"),
Department(id="17", name=u"MADRE DE DIOS"),
Department(id="18", name=u"MOQUEGUA"),
Department(id="19", name=u"PASCO"),
Department(id="20", name=u"PIURA"),
Department(id="21", name=u"PUNO"),
Department(id="22", name=u"SAN MARTIN"),
Department(id="23", name=u"TACNA"),
Department(id="24", name=u"TUMBES"),
Department(id="25", name=u"UCAYALI"),
])
db_session.commit()
exit(0)
def entity2mysql(time_dir):
file_path = csv_dir + "entity/%s/entity.csv" % (time_dir)
f = open(file_path, "r")
data = csv.reader(f)
nrows = 0
entityList = []
print u"----> 开始存储实体数据..."
duplicate_cnt = 0
for row in data:
nrows += 1
pdf_path = str(row[0].strip())
pdf_page = int(row[1].strip())
mark_id = int(row[2].strip())
mark_user = str(row[3].strip())
statement = db_session.query(VerStatement).filter(
VerStatement.pdf_path == pdf_path, VerStatement.pdf_no == pdf_page,
VerStatement.mark_id == mark_id).first()
if statement != None: ## 一般情况下不该存在同名元组
duplicate_cnt += 1
continue
statement = VerStatement(pdf_no=pdf_page,
pdf_path=pdf_path,
mark_id=mark_id,
mark_user=mark_user)
db_session.add(statement)
db_session.commit()
stat_id = statement.id
u""" 以硬性规则对标注做一定的补全 """
content = "".join([seg.strip() for seg in row[4:]])
content = content_completion(content)
newEntity = EntityMark(content, stat_id)
entityList.append(newEntity)
db_session.add_all(entityList)
db_session.commit()
del entityList[:]
print u"----> 实体数据存储完毕，总共存储数据%d行, 冲突id共%d行" % (nrows, duplicate_cnt)
def test_input_data(self):
u1 = User(name="Paul", email="*****@*****.**", password="******")
u2 = User(name="Joe", email="*****@*****.**", password="******")
s1 = Alert(name="guitars",
string="guitars -steel",
status=1,
last_update=datetime.date(2013, 9, 25),
user=u1)
s2 = Alert(name="banjo",
string="banjo -100",
status=1,
last_update=datetime.date(2013, 9, 30),
user=u1)
s3 = Alert(name="judo stuff",
string="gee",
status=1,
last_update=datetime.date(2013, 5, 13),
user=u2)
s4 = Alert(name="bmx bike",
string="bmx -hoffman",
status=0,
last_update=datetime.date(2011, 3, 5),
user=u2)
r1 = Result(title="Guitar for Sale - $100",
link="http://craigslist.com/12341234",
alert=s1)
r2 = Result(title="Shitty Guitar for Sale - $50",
link="http://craigslist.com/54325432",
alert=s1)
r3 = Result(title="Even shittier Guitar for Sale - $30",
link="http://craigslist.com/4672345",
alert=s1)
db_session.add_all([u1, u2, s1, s2, s3, s4, r1, r2, r3])
db_session.commit()
print "****Data Has Been Commited to DB****"
def reports_step(self, simulation_id: int) -> None:
start_time = datetime.datetime.utcnow()
simulation: Simulation = db_session.query(Simulation).get(simulation_id)
step: SimulationStep = SimulationStep(started_utc=start_time,
origin="REPORT",
simulation_id=simulation.id,
status='ONGOING')
db_session.add(step)
db_session.flush()
workdir: str = simulation.workdir
output_path: str = workdir + "/plots/"
if not os.path.exists(output_path):
os.makedirs(output_path)
simulation.current_step = "REPORT"
simulation.current_step_id = step.id
db_session.begin()
db_session.add_all([simulation, step])
db_session.commit()
result = chain(
all_clones_plot_stats.s(workdir),
all_clones_mullerplot.s(),
noise_plot_stats.s(),
major_clones_plot_stats.s(),
major_clones_mullerplot.s(),
mullerplot.s(),
mutation_histogram.s(),
simulation_report.s(),
build_cell_model.s(),
save_results_and_cleanup.s(simulation_id, step.id, workdir),
).apply_async()
return result
def init_db():
Base.metadata.drop_all(bind=engine)
Base.metadata.create_all(bind=engine)
wall_question = Question(text='What was on your wall as a kid?')
db_session.add(wall_question)
pet_question = Question(text='What was your favorite pet?')
db_session.add(pet_question)
matt = Comrade(name='Matt', email='*****@*****.**')
db_session.add(matt)
julian = Comrade(name='Julian', email='*****@*****.**')
db_session.add(julian)
wall_1 = Round(name='wall_round_one',
date=date.today(),
question=wall_question)
wall_2 = Round(name='wall_round_two',
date=date.today() + one_week,
question=wall_question)
pet_1 = Round(name='pet_round_one',
date=date.today() + 2 * one_week,
question=pet_question)
pet_2 = Round(name='pet_round_two',
date=date.today() + 3 * one_week,
question=pet_question)
db_session.add_all([wall_1, wall_2, pet_1, pet_2])
db_session.commit()
db_session.execute(comrade_to_round.insert().values([
(matt.uuid, wall_1.uuid), (julian.uuid, wall_2.uuid),
(julian.uuid, pet_1.uuid), (matt.uuid, pet_2.uuid)
]))
db_session.commit()
def save_results_and_cleanup(plots, simulation_id: int, step_id: int,
workdir: str):
output_path: str = workdir + "/plots/"
plots: List[Artifact] = index_plots(output_path, simulation_id, step_id)
reports: List[Artifact] = index_reports(workdir + "/reports/",
simulation_id, step_id)
simulation: Simulation = db_session.query(Simulation).get(simulation_id)
step: SimulationStep = db_session.query(SimulationStep).get(step_id)
db_session.flush()
db_session.begin()
end_time = datetime.datetime.utcnow()
simulation.finished_utc = end_time
simulation.status = 'SUCCESS'
step.finished_utc = end_time
step.status = 'SUCCESS'
db_session.add_all(plots)
db_session.add_all(reports)
db_session.add_all([simulation, step])
db_session.commit()
cleanup(workdir)
return workdir
def addCode(pn):
aid = int(request.args.get('article'))
var = request.args.get('variable')
val = request.args.get('value')
ev = request.args.get('event')
text = request.args.get('text')
pn = int(pn)
aqs = []
now = dt.datetime.now(tz = central).replace(tzinfo = None)
if False:
pass
elif pn == 1:
model = CodeFirstPass
## store highlighted text on first pass
if text:
p = CodeFirstPass(aid, var, val, current_user.id, text)
else:
p = CodeFirstPass(aid, var, val, current_user.id)
## update datetime on every edit
aq = db_session.query(ArticleQueue).filter_by(article_id = aid, coder_id = current_user.id).first()
aq.coded1_dt = now
aqs.append(aq)
elif pn == 2:
model = CodeSecondPass
p = CodeSecondPass(aid, ev, var, val, current_user.id)
for aq in db_session.query(SecondPassQueue).filter_by(article_id = aid, coder_id = current_user.id).all():
aq.coded_dt = now
aqs.append(aq)
else:
return make_response("Invalid model", 404)
## variables which only have one value per article
if var in sv:
if pn == 1:
a = db_session.query(model).filter_by(
article_id = aid,
variable = var,
coder_id = current_user.id
).all()
else:
## for second pass, filter for distinct event
a = db_session.query(model).filter_by(
article_id = aid,
variable = var,
event_id = ev,
coder_id = current_user.id
).all()
## if there's more then one, delete them
if len(a) > 0:
for o in a:
db_session.delete(o);
db_session.commit()
## if this is a 2nd comment pass comment and it is null, skip it
if var == 'comments' and pn == 2 and val == '':
return jsonify(result={"status": 200})
# try:
db_session.add(p)
db_session.add_all(aqs)
db_session.commit()
return jsonify(result={"status": 200})
def add_data():
# Add User:
user1 = User(name="superuser", email="*****@*****.**", password="******",
phone="+16509119111")
# Add the Sensor Units:
unit1 = SensorUnit(name="katies-kitchen", user=user1)
unit2 = SensorUnit(name="pauls-house", user=user1)
# Add the Sensors:
sensor1 = Sensor(name="2882E92D6008F",
dt_created=datetime(2014, 12, 27, 0, 0, 0, 0),
active=1, sensorunit=unit1)
sensor2 = Sensor(name="28FFEB6760144E7",
dt_created=datetime(2015, 1, 25, 0, 0, 0, 0),
active=1, sensorunit=unit1)
sensor3 = Sensor(name="28FF1F51601441F",
dt_created=datetime(2015, 1, 18, 0, 0, 0, 0),
active=1, sensorunit=unit2)
sensor4 = Sensor(name="28FFC81D60144E2",
dt_created=datetime(2015, 1, 18, 0, 0, 0, 0),
active=1, sensorunit=unit2)
# Add Fermentation Temperature Profiles:
profile1 = Profile(name="Test Profile",
dt_created=datetime(2014, 12, 27, 14, 0, 0, 0),
num_sensors=1, active=0,
dt_stopped=datetime(2015, 1, 12, 1, 50, 0, 0),
description="""This profile is a test run of the sensor and was montiroing
the ambient temperature in the room, not an active
fermentation. This gives a good baseline for what the ambient
temperature will look like, so I can compare the fluctuations
in ambient due to space conditioning, with the fluctuations
when it is hooked up to an actual fermentation.""")
st1 = SensorTable(sensor=sensor1, profile=profile1, label="Ambient")
profile2 = Profile(name="Single Hop Cascade",
dt_created=datetime(2015, 1, 12, 1, 52, 0, 0),
num_sensors=1, active=0,
dt_stopped=datetime(2015, 1, 24, 19, 30, 0, 0),
description="""Currently the temperature sensor is hooked up to a one
gallon batch of a single hop pale ale. The sensor was attached
to the outside of the fermentation carboy, and I used a piece
of packing foam to insulate the sensor from the ambient
conditions. Based on the fluctuations observed during the test
run, it appears that the insulation is working as intended.""")
st2 = SensorTable(sensor=sensor1, profile=profile2, label="Fermentation")
profile3 = Profile(name="Single Hop Sorachi",
dt_created=datetime(2015, 1, 25, 1, 30, 0, 0),
num_sensors=2, active=1,
dt_stopped=datetime.now(),
description="""Currently the temperature sensors are hooked up to a one
gallon batch of Single Hop (Sorachi Ace) Pale Ale. I brewed two
single hop batches in the same session (the other with
Amarillo), and hooked up the fementation sensor to the Sorachi
carboy. The other sensor is monitoring ambient temperature.""")
st3 = SensorTable(sensor=sensor1, profile=profile3, label="Ambient")
st4 = SensorTable(sensor=sensor2, profile=profile3, label="Fermentation")
profile4 = Profile(name="Imperial Stout",
dt_created=datetime(2015, 1, 18, 13, 0, 0, 0),
num_sensors=2, active=0,
dt_stopped=datetime(2015, 2, 1, 2, 34, 0, 0),
description="""Currently the temperature sensors are hooked up to a five
gallon batch of an Imperial Stout. One sensor is attached to
the fermentation bucket, with some foam insulation on the back,
and the other temperature sensor is recording the ambient
conditions.""")
st5 = SensorTable(sensor=sensor3, profile=profile4, label="Ambient") # need to check this
st6 = SensorTable(sensor=sensor4, profile=profile4, label="Fermentation") # need to check this
profile5 = Profile(name="Honey Ale",
dt_created=datetime(2015, 2, 1, 2, 35, 0, 0),
num_sensors=2, active=0,
dt_stopped=datetime.now(),
description="""Currently the temperature sensors are hooked up to a five
gallon batch of an honey ale. One sensor is attached to the
fermentation bucket, with some foam insulation on the back, and
the other temperature sensor is recording the ambient
conditions.""")
st7 = SensorTable(sensor=sensor3, profile=profile5, label="Ambient") # need to check this
st8 = SensorTable(sensor=sensor4, profile=profile5, label="Fermentation") # need to check this
# Add some Notes for the Profiles:
note1 = ProfileNote(dt_created=datetime(2015, 1, 12, 2, 30, 0, 0),
dt_applied=datetime(2015, 1, 12, 2, 30, 0, 0),
note="""Pitched the Yeast way too hot, probably should have cooled the wort
a few more degrees.""",
profile_note=profile2)
note2 = ProfileNote(dt_created=datetime(2015, 1, 18, 2, 0, 0, 0),
dt_applied=datetime(2015, 1, 18, 2, 0, 0, 0),
note="""Updated the Arduino/Python code, but wasn't able to update this
sensor unit remotely, so the unit went offline for a bit.""",
profile_note=profile2)
db_session.add_all([unit1, unit2, sensor1, sensor2, sensor3, sensor4,
profile1, profile2, profile3, profile4, st1, st2, st3,
st4, st5, st6, st7, st8, note1, note2])
db_session.commit()
def run_state_machine(db, tablemd, schema):
"""
Execute a state machine to analyze and extract location data from
a single table
"""
tablename = tablemd.tablename
print tablename
cols = schema.columns.keys()
if tablemd.state == 0 or ('_latlon' not in cols and '_address' not in cols):
# check if table exists in metadata table
print '\tstate 0'
if '_latlon' not in cols and '_address' not in cols:
# create shadow columns
db.execute(ALTERQUERY % tablename)
tablemd.state = 1
db_session.add(tablemd)
db_session.commit()
if tablemd.state == 1:
# check if shadow cols exist
print '\tstate 1'
# loc -> [(column, extractor)]
extractors = find_location_extractors(db, tablemd, schema)
annos = []
for anntype, pairs in extractors.iteritems():
for colname, extractor in pairs:
annos.append(Annotation(colname, anntype, extractor, tablemd))
tablemd.hasloc = len(annos) > 0
tablemd.state = 2
db_session.add_all(annos)
db_session.add(tablemd)
db_session.commit()
return tablemd.state
if tablemd.state == 2:
# check if table needs to be analyzed for annotations
print '\tstate 2'
needs_geocoding = populate_shadow_cols(db, tablemd, schema)
tablemd.state = 3 if needs_geocoding else 5
db_session.add(tablemd)
db_session.commit()
return tablemd.state
if tablemd.state == 3 or tablemd.state == 4:
# check if table has annotations and no geocode/shapes
print '\tstate 3/4'
maxid = geocode_table(db, tablemd)
try:
if maxid == db.execute('select max(id) from %s' %tablename).fetchone()[0]:
tablemd.state = 5
db_session.add(tablemd)
db_session.commit()
except Exception as e:
raise
return tablemd.state
# check if user added any new annotations since
# last geocode
if tablemd.state == 5:
print '\tstate 5'
return tablemd.state
return tablemd.state
"""Populate the database for testing"""
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from database import init_db, Base, db_session as session
from models import User, Category, Item
init_db()
# Create users
darth_vader = User(name="Darth Vader", email="*****@*****.**")
emperor_palpatine = User(
name="Emperor Palpatine", email="*****@*****.**")
revan = User(name="Revan", email="*****@*****.**")
session.add_all([darth_vader, emperor_palpatine, revan])
session.commit()
# Create categories
# Reference: http://starwars.wikia.com/
starfighter = Category(name="Starfighter", description="Small one- or "
"two-man vessels typically used for dogfighting.")
capital_ship = Category(name="Capital Ship", description="Largest of "
"starships, often used as a warship.")
scout_vessel = Category(name="Scout Vessel", description="Small, fast "
"ships used to scout an area before a main "
"force arrives.")
blaster_pistol = Category(name="Blaster Pistol", description="Small "
"handheld blasters.")
blaster_rifle = Category(name="Blaster Rifle", description="Heavy duty "
"blaster variants, much more powerful than pistols.")
