def save_objects(session, data, region, date):
position, track_name, artist_name, url = data
artist = get_or_create(session, Artist, name=artist_name)
track_vals = {
'title': track_name,
'artist': artist
}
track = get_or_create(session, Track, **track_vals)
link_vals = {
'url': url,
'track': track
}
link = get_or_create(session, Link, **link_vals)
chart_vals = {
'type': 'viral50',
'region': region,
'position': position,
'streams': None,
'date': date,
'track': track
}
chart = get_or_create(session, Chart, **chart_vals)
def build_db():
'''
Build a small db with some example entries.
'''
names = [
'Oleg', 'Mark', 'William', 'James', 'Harry', 'Amelia', 'Oliver', 'Jack', 'John'
]
books = [
'MySQL for Python', 'MySQL Cookbook', 'Essential SQLAlchemy', 'Learning Python',
'Python for Data Analysis', 'Core Python Applications Programming', 'Python Algorithms'
'Think Python', 'Computer Games with Python', 'MongoDb in action'
]
user = {'name': 'user', 'passwd': 'like-books'}
counter = session.query(Author).count()
if not counter:
for i in range(len(names)):
author = get_or_create(Author, name=names[i])
author.books.append(get_or_create(Book, title=books[i]))
author.books.append(get_or_create(Book, title=random.choice(books)))
session.commit()
session.add(User(nickname=user['name'], password=user['passwd']))
session.commit()
def _get_written_question_bulletin():
for i in range(48, 54):
soup = read_or_dl("http://www.lachambre.be/kvvcr/showpage.cfm?section=/qrva&language=fr&rightmenu=right?legislat=52&cfm=/site/wwwcfm/qrva/qrvaList.cfm?legislat=%i" % i, "bulletin list %i" % i)
for b in soup('table')[4]('tr')[1:]:
try:
if i == 53:
get_or_create(WrittenQuestionBulletin,
legislature="53",
lachambre_id=b('td')[0]('a')[-1].text.split()[-1],
date=b('td')[2].text,
publication_date=b('td')[3].text,
url=b('td')[1].a["href"],
pdf_url=b('td')[0].a["href"],
)
else:
get_or_create(WrittenQuestionBulletin,
legislature=str(i),
lachambre_id=b('td')[0]('a')[-1].text.split()[-1],
publication_date=b('td')[2].text,
url=b('td')[1].a["href"] if b('td')[1].a else None,
pdf_url=b('td')[0].a["href"],
)
logger.debug("%s" % b('td')[0]('a')[-1].text.split()[-1])
except TypeError:
continue
def scrape():
for a, url in enumerate(
('http://www.lachambre.be/kvvcr/showpage.cfm?section=none&language=fr&cfm=/site/wwwcfm/rajv/rajvlist.cfm?lastreports=y',
'http://www.lachambre.be/kvvcr/showpage.cfm?section=none&language=fr&cfm=/site/wwwcfm/rajv/rajvlist.cfm?lastreports=n'
)):
soup, suppe = read_or_dl_with_nl(url, "annual repports %i" % a)
for i, j in zip(
soup.find('div', id="story")('table')[1]('tr',
recursive=False)[::5],
suppe.find('div',
id="story")('table')[1]('tr',
recursive=False)[::5]):
get_or_create(
AnnualReport,
title={
"fr": i('td')[2].text,
"nl": j('td')[2].text
},
date=i('td')[0].text,
law_and_article={
"fr": i('td')[4].text,
"nl": j('td')[4].text
},
periodicity=re.sub("[^0-9]", "",
i('td')[5].text),
pdf_url=i('td')[1].a["href"] if i('td')[1].a else "",
)
def _save_a_written_question(link):
soupsoup, suppesuppe = read_or_dl_with_nl(LACHAMBRE_PREFIX + link.a["href"], "written question %s" % re.search("dossierID=([0-9A-Z-]+).xml", link.a["href"]).groups()[0])
data = AccessControlDict(((x.td.text, x('td')[1]) for x in soupsoup.find('table', 'txt')('tr') if x.td.text))
data_nl = AccessControlDict(((x.td.text, x('td')[1]) for x in suppesuppe.find('table', 'txt')('tr') if x.td.text))
get_or_create(WrittenQuestion,
_id="lachambre_id",
lachambre_id=re.search("dossierID=([0-9A-Z-]+).xml", link.a["href"]).groups()[0],
title={"fr": data["Titre"].text, "nl": data_nl["Titel"].text},
departement={"fr": data[u"Département"].text, "nl": data_nl[u"Departement"].text},
sub_departement={"fr": data[u"Sous-département"].text, "nl": data_nl[u"Sub-departement"].text},
deposition_date=data[u"Date de dépôt"].text,
delay_date=dico_get_text(data, u"Date de délai"),
publication_date=dico_get_text(data, "Date publication"),
# TODO: link to the actual deputy
author=data[u"Auteur"].text,
language=data[u"Langue"].text,
question_status={"fr": dico_get_text(data, "Statut question"), "nl": dico_get_text(data_nl, "Status vraag")},
status={"fr": dico_get_text(data, "Statut"), "nl": dico_get_text(data_nl, "Status")},
question={"fr": u"%s" % data["Question"], "nl": "%s" % data_nl["Vraag"]},
answer={"fr": dico_get_text(data, u"Réponse"), "nl": dico_get_text(data_nl, u"Antwoord")},
publication_reponse_pdf_url=get_href_else_blank(data, u"Publication réponse"),
publication_question_pdf_url=get_href_else_blank(data, u"Publication question"),
publication_reponse=get_text_else_blank(data, u"Publication réponse"),
publication_question=get_text_else_blank(data, u"Publication question"),
eurovoc_descriptors={"fr": get_items_list_else_empty_list(data, "Descripteurs Eurovoc"),
"nl": get_items_list_else_empty_list(data_nl, "Eurovoc-descriptoren")},
eurovoc_candidats_descriptors={"fr": get_items_list_else_empty_list(data, "Candidats-descripteurs Eurovoc"),
"nl": get_items_list_else_empty_list(data_nl, "Eurovoc kandidaat-descriptoren")},
keywords={"fr": get_items_list_else_empty_list(data, u"Mots-clés libres"),
"nl": get_items_list_else_empty_list(data_nl, u"Vrije trefwoorden")},
url=link.a["href"],
)
data.die_if_got_not_accessed_keys()
def fill_database():
admin_role = get_or_create(db.session, Role, name='admin')
dj_role = get_or_create(db.session, Role, name='dj')
default_radio = get_or_create(db.session, Radio, name='default')
admin_user = get_or_create(db.session, User, login=config.admin_login)
admin_user.password = pwd_context.encrypt(config.admin_password)
admin_user.roles = [admin_role, dj_role]
admin_user.radio = default_radio
admin_user.is_active = True
admin_user.comment = "admin z config.py, zawsze posiada haslo z config.py"
def scrape():
for a, url in enumerate(('http://www.lachambre.be/kvvcr/showpage.cfm?section=none&language=fr&cfm=/site/wwwcfm/rajv/rajvlist.cfm?lastreports=y',
'http://www.lachambre.be/kvvcr/showpage.cfm?section=none&language=fr&cfm=/site/wwwcfm/rajv/rajvlist.cfm?lastreports=n')):
soup, suppe = read_or_dl_with_nl(url, "annual repports %i" % a)
for i, j in zip(soup.find('div', id="story")('table')[1].tbody('tr', recursive=False)[::5], suppe.find('div', id="story")('table')[1].tbody('tr', recursive=False)[::5]):
get_or_create(AnnualReport,
title={"fr": i('td')[2].text, "nl": j('td')[2].text},
date=i('td')[0].text,
law_and_article={"fr": i('td')[4].text, "nl": j('td')[4].text},
periodicity=re.sub("[^0-9]", "", i('td')[5].text),
pdf_url=i('td')[1].a["href"] if i('td')[1].a else "",
)
def post(self):
form = AddAuthorForm()
if form.validate():
# if session.query(Author).filter_by(name=form.name.data).count() == 0:
author = get_or_create(Author, name=form.name.data)
for title in str_separator(form.title.data):
author.books.append(get_or_create(Book, title=title))
session.commit()
return redirect(url_for('add_author'))
else:
msg = 'Author <strong class=badge>{0}</strong> already \
exists!'.format(form.name.data)
flash(msg)
return render_template('add_author.haml', form=form)
def test_get_or_create_already_exists(self, m):
from utils import get_or_create
quiz_id = 5
quiz_title = "Final Exam"
course_id = 1
prebuilt_quiz = Quiz(
canvas_id=quiz_id,
course_id=course_id,
title=quiz_title
)
views.db.session.add(prebuilt_quiz)
views.db.session.commit()
quiz, created = get_or_create(
views.db.session,
Quiz,
canvas_id=quiz_id,
course_id=course_id
)
self.assertFalse(created)
self.assertIsInstance(quiz, Quiz)
self.assertEqual(quiz.canvas_id, quiz_id)
self.assertEqual(quiz.course_id, course_id)
self.assertEqual(quiz.title, quiz_title)
def _create_objects(self, object_names):
"""
Create objects from a list of object names.
:param object_names: list, List of object names
:return:
"""
for obj_name in object_names:
if self.data.get(obj_name):
data = self.data[obj_name]
multiple = False
if isinstance(data, list) or isinstance(data, dict):
multiple = True
table_name = TABLE_NAMES[obj_name]
object_class = get_class_by_table_name(table_name)
if object_class:
if multiple:
# inject product id into data dictionary for objects that need it as a field.
if obj_name in self.product_config[PRODUCT_DEPENDENT]:
for key, value in data.items():
value['product_id'] = self.objects['product_id']
self.objects[obj_name] = get_or_create_multiple(object_class, data=data)
db_session.add_all(self.objects[obj_name])
else:
self.objects[obj_name], _ = get_or_create(object_class, name=self.data[obj_name])
db_session.add(self.objects[obj_name])
def _get_deputie_commissions(soup, deputy):
# here we will walk in a list of h4 .. h5 .. div+ .. h5 .. div+
# look at the bottom of each deputies' page
membership = soup.find('td', rowspan="1")
item = membership.h4
role = None
deputy.commissions = []
while item.nextSibling:
if hasattr(item, 'tag'):
if item.name == 'h5':
role = item.text[6:-1]
elif item.name == 'div':
logger.debug("linking deputy to commission %s" % item.a.text)
commission = get_or_create(Commission, url=item.a['href'], lachambre_id=int(re.search("com=(\d+)", item.a["href"]).groups()[0]))
deputy.commissions.append(get_or_create(CommissionMembership, commission=commission, role=role))
item = item.nextSibling
def deputies_list(reset=False):
soup = read_or_dl(
"http://www.lachambre.be/kvvcr/showpage.cfm?section=/depute&language=fr&rightmenu=right_depute&cfm=/site/wwwcfm/depute/cvlist.cfm",
"deputies", reset)
for dep in soup('table')[4]('tr'):
items = dep('td')
full_name = re.sub(' +', ' ', items[0].a.text).strip()
url = items[0].a['href']
party = get_or_create(Party,
name=items[1].a.text,
url=dict(items[1].a.attrs)['href'])
email = items[2].a.text
website = items[3].a['href'] if items[3].a else None
# yes, one deputies key contains a O instead of an 0, I'm not joking
lachambre_id = re.search('key=([0-9O]+)', url).groups()[0]
if not Deputy.objects.filter(lachambre_id=lachambre_id):
deputy = Deputy.objects.create(
full_name=full_name,
party=party,
url=url,
websites=[website] if website else [],
lachambre_id=lachambre_id,
emails=[email])
logger.debug(
'adding new deputy %s %s %s %s %s' %
(lachambre_id.encode("Utf-8"),
full_name.encode("Utf-8"), party, email.encode("Utf-8"),
website.encode("Utf-8") if website else ''))
logger.info("[NEW] deputy: %s" % deputy)
def scrape():
soup, suppe = read_or_dl_with_nl(
"http://www.lachambre.be/kvvcr/showpage.cfm?section=/comm/commissions&language=fr&cfm=/site/wwwcfm/comm/LstCom.cfm&rightmenu=right_cricra",
"commissions list")
_type = ""
for i, j in zip(soup("div", id="story")[1], suppe("div", id="story")[1]):
if not isinstance(i, NavigableString) and (i.h4 or i.a):
if i.h4:
_type = i.h4.text
_type_nl = j.h4.text
elif i.a:
commission = get_or_create(Commission,
lachambre_id=int(
re.search(
"com=(\d+)",
i.a["href"]).groups()[0]))
commission.type["fr"] = _type
commission.type["nl"] = _type_nl
commission.name["fr"] = i.a.text
commission.name["nl"] = j.a.text
commission.url = i.a["href"]
commission.save()
for com in list(Commission.objects.all()):
handle_commission(com)
def handle_commission(commission):
soup, suppe = read_or_dl_with_nl(LACHAMBRE_PREFIX + commission.url,
"commission %s" % commission.lachambre_id)
commission.full_name["fr"] = soup.h1.text
commission.full_name["nl"] = suppe.h1.text
commission.deputies = []
seats = {"fr": {}, "nl": {}}
for i, j in zip(soup('p'), suppe('p')):
role = i.b.text[:-1]
role_nl = j.b.text[:-1]
for dep in i('a'):
deputy = Deputy.objects.get(lachambre_id=re.search(
"key=([O0-9]+)", dep["href"]).groups()[0])
membership = get_or_create(CommissionMembership,
deputy=deputy,
commission=commission)
membership.role = role
membership.save()
commission.deputies.append(membership.id)
seats["fr"][role] = map(
lambda x: (x[0], len(x[1].split(','))),
zip(map(lambda x: x.text[:-1],
i('b')[1:]),
str(i).split("<br />")[1:]))
seats["nl"][role_nl] = map(
lambda x: (x[0], len(x[1].split(','))),
zip(map(lambda x: x.text[:-1],
i('b')[1:]),
str(i).split("<br />")[1:]))
commission.seats = seats
commission.save()
def post(self):
form = AddBookForm()
if form.validate():
if session.query(Book).filter_by(title=form.title.data).count() == 0:
print form.title.data
book = get_or_create(Book, title=form.title.data)
for name in str_separator(form.name.data):
book.authors.append(get_or_create(Author, name=name))
session.commit()
msg = 'Book <strong class=badge>{0}</strong> added!'.format(form.title.data)
flash(msg)
return redirect(url_for('add_book'))
else:
msg = 'Book <strong class=badge>{0}</strong> already \
exists!'.format(form.title.data)
flash(msg)
return render_template('add_book.haml', form=form)
def add_participant(self, first_name, last_name, email, session):
participant = get_or_create(session,
Student,
first_name=first_name,
last_name=last_name,
email=email)
if participant not in self.participants:
self.participants.append(participant)
def _extract_features(task, view, sequence, func_name, func, left, right, do_update):
function = get_or_create(
Function,
keys=dict(
task=task,
name=func_name,
left=left,
right=right
)
)
labels = Labels.select().where(
Labels.task == task,
Labels.sequence == sequence
)
data = Data.select().where(
Data.view == view,
Data.sequence == sequence
)
df = pd.DataFrame(
data=[dd.x for dd in data],
index=[dd.t for dd in data],
columns=view.columns
)
df.index.name = 't'
rows = []
for label in labels:
inds = ((df.index > label.t + left) &
(df.index <= label.t + right))
seq = df[inds]
f_of_x = func(seq)
f_of_x = parse_feature_type(f_of_x)
assert np.all(np.isfinite(f_of_x)), 'Extracted features contain NaNs, with\ndf=\n{}\n\nf(df) = {}'.format(
df[inds], f_of_x
)
if do_update:
Features.update(
x=f_of_x
).where(
Features.label == label,
Features.function == function
).execute()
else:
rows.append(dict(
function=function,
label=label,
x=f_of_x
))
if not do_update:
Features.insert_many(rows).execute()
def register_split_definition(self, task, partition_name, split_num, split, tag=None):
assert isinstance(split, dict)
tag = tag or ''
partition = get_or_create(
Partition,
keys=dict(
task=task,
name=partition_name
)
)
get_or_create(
SplitDefinition,
keys=dict(
tag=tag,
split_num=split_num,
partition=partition
),
values=split
)
def _get_deputie_commissions(soup, deputy):
# here we will walk in a list of h4 .. h5 .. div+ .. h5 .. div+
# look at the bottom of each deputies' page
membership = soup.find('td', rowspan="1")
item = membership.h4
role = None
deputy.commissions = []
while item.nextSibling:
if hasattr(item, 'tag'):
if item.name == 'h5':
role = item.text[6:-1]
elif item.name == 'div':
logger.debug("linking deputy to commission %s" % item.a.text)
commission = get_or_create(
Commission,
url=item.a['href'],
lachambre_id=int(
re.search("com=(\d+)", item.a["href"]).groups()[0]))
deputy.commissions.append(
get_or_create(CommissionMembership,
commission=commission,
role=role))
item = item.nextSibling
def log(json_data):
logging.info('Received JSON: %s' % json_data)
payload = json.loads(json_data)
nickname = payload.get('nickname')
server = payload.get('server', 'unknown')
channel = payload.get('channel')
message_type = payload.get('message_type')
message_content = payload.get('message_content')
timestamp = parse_timestamp(payload.get('timestamp'))
user = get_or_create(User, server=server, nickname=nickname)
user.last_seen_at = datetime.utcnow()
user.put()
channel = get_or_create(Channel, server=server, channel=channel)
# store the message
msg = Message(user=user, channel=channel, message_type=message_type,
message_content=message_content, json=json_data,
timestamp=timestamp)
msg.put()
return msg
def push_unsent_payloads(self):
# we need to create our own session here as we're running in a new thread
local_session = self.Session()
while True: # Infinite loop while parent process is working
# check if internet access
if active_internet_connection(self.ipaddress):
unsent_sensordata = local_session.query(SensorData).filter_by(sent=False)
for sensordata in unsent_sensordata:
for consumer in self.payload_consumers:
if not isinstance(consumer, PushDataBase):
consumer_obj = consumer()
else:
consumer_obj = consumer
# get SensorDataPushState entry for this payload. If it doesn't exist. Create it
consumer_push_state, created = get_or_create(local_session, SensorDataPushState,
defaults={'timestamp': str(datetime.utcnow()),
'attempts': 0,
'aborted': False},
sensordata_id=sensordata.id,
consumer=str(consumer_obj.name))
# check to see if has been sent already. it's it's just been created then the answer is no
if created or (not consumer_push_state.aborted and not consumer_push_state.sent):
# we need to check that it was successful.
push_success = consumer_obj.push(sensordata)
consumer_push_state.attempts += 1 # add one to the number of attempts
consumer_push_state.timestamp = str(datetime.utcnow())
if push_success:
consumer_push_state.sent = True
else:
if consumer_push_state.attempts > consumer_obj.max_retries:
logging.error('%s push aborted after too many retries' % consumer_push_state)
consumer_push_state.aborted = True
# check to see if there are any more consumers of this data left to push.
# if not then we can now mark it as sent
unsent_count = local_session.query(func.count(SensorDataPushState.id))\
.filter_by(sensordata_id=sensordata.id,
sent=False,
aborted=False).scalar()
if unsent_count == 0:
sensordata.sent = True
local_session.commit()
def test_get_or_create_created(self, m):
from utils import get_or_create
quiz_id = 5
course_id = 1
quiz, created = get_or_create(
views.db.session,
Quiz,
canvas_id=quiz_id,
course_id=course_id,
)
self.assertTrue(created)
self.assertIsInstance(quiz, Quiz)
self.assertEqual(quiz.canvas_id, quiz_id)
self.assertEqual(quiz.course_id, course_id)
def _get_deputy_analysis(url, deputy, type, reset=False):
soupsoup = read_or_dl(LACHAMBRE_PREFIX + lame_url(url), '%s %s' % (deputy.full_name, type), reset)
setattr(deputy, "analysis_%s_url" % type, url)
setattr(deputy, "analysis_%s_list" % type, [])
for i in soupsoup('table')[3]('tr', valign="top"):
logger.debug("add", type, i.tr('td')[1].text.strip())
dico = table2dic(i.table('td'))
logger.debug("%s" % dico)
getattr(deputy, "analysis_%s_list" % type).\
append(get_or_create(Analysis,
_id="lachambre_id",
lachambre_id=re.search("dossierID=([0-9A-Za-z-]+)", i.a["href"]).groups()[0],
title=dico["Titre"],
descriptor=dico["Descripteurs"],
url=i.a['href'],
type=type))
def extract_features(connection, task, view, func, func_name, left, right, force):
assert left < right;
task = connection.tasks[task]
view = connection.views[view]
function = get_or_create(
Function,
keys=dict(
task=task,
name=func_name,
left=left,
right=right
)
)
num_features = Features.select().where(
Features.function == function
).count()
num_labels = Labels.select().where(
Labels.task == task
).count()
if not force:
if num_features == num_labels:
return
do_update = num_features == num_labels
if num_features != num_labels:
Features.delete().where(
Features.function == function
).execute()
for _, sequence in connection.itersequences():
print task, view, sequence, function
_extract_features(
task=task,
view=view,
sequence=sequence,
func_name=func_name,
func=func,
left=left,
right=right,
do_update=do_update
)
def _get_deputy_analysis(url, deputy, type, reset=False):
soupsoup = read_or_dl(LACHAMBRE_PREFIX + lame_url(url),
'%s %s' % (deputy.full_name, type), reset)
setattr(deputy, "analysis_%s_url" % type, url)
setattr(deputy, "analysis_%s_list" % type, [])
for i in soupsoup('table')[3]('tr', valign="top"):
logger.debug("add", type, i.tr('td')[1].text.strip())
dico = table2dic(i.table('td'))
logger.debug("%s" % dico)
getattr(deputy, "analysis_%s_list" % type).\
append(get_or_create(Analysis,
_id="lachambre_id",
lachambre_id=re.search("dossierID=([0-9A-Za-z-]+)", i.a["href"]).groups()[0],
title=dico["Titre"],
descriptor=dico["Descripteurs"],
url=i.a['href'],
type=type))
def _get_competences(dico, dico_nl, document):
# FIXME: meh, DRY
if dico.get(u"Compétence") and dico_nl.get(u"Bevoegdheid"):
document.timeline = []
for (_date, _title), (_, _title_nl) in zip([clean_text(x).split(u" \xa0 ", 1) for x in dico[u"Compétence"]["head"].contents[::2]],
[clean_text(x).split(u" \xa0 ", 1) for x in dico_nl[u"Bevoegdheid"]["head"].contents[::2]]):
logger.debug("append time line %s %s %s" % (_date, _title, _title_nl))
document.timeline.append(DocumentTimeLine.objects.create(title={"fr": _title, "nl": _title_nl}, date=_date))
elif dico.get(u"Compétence"):
document.timeline = []
for (_date, _title) in [clean_text(x).split(u" \xa0 ", 1) for x in dico[u"Compétence"]["head"].contents[::2]]:
logger.debug("append time line %s %s %s" % (_date, _title, ""))
document.timeline.append(DocumentTimeLine.objects.create(title={"fr": _title, "nl": ""}, date=_date))
elif dico_nl.get(u"Bevoegdheid"):
document.timeline = []
for (_date, _title_nl) in [clean_text(x).split(u" \xa0 ", 1) for x in dico_nl[u"Bevoegdheid"]["head"].contents[::2]]:
logger.debug("append time line %s %s %s" % (_date, "", _title_nl))
document.timeline.append(DocumentTimeLine.objects.create(title={"fr": "", "nl": _title_nl}, date=_date))
if dico.get("Analyse des interventions"):
document.analysis = get_or_create(Analysis, _id="lachambre_id", lachambre_id=dico["Analyse des interventions"]["head"].a.text, url=dico["Analyse des interventions"]["head"].a["href"])
def _get_competences(dico, dico_nl, document):
# FIXME: meh, DRY
if dico.get(u"Compétence") and dico_nl.get(u"Bevoegdheid"):
document.timeline = []
for (_date, _title), (_, _title_nl) in zip([clean_text(x).split(u" \xa0 ", 1) for x in dico[u"Compétence"]["head"].contents[::2]],
[clean_text(x).split(u" \xa0 ", 1) for x in dico_nl[u"Bevoegdheid"]["head"].contents[::2]]):
logger.debug("append time line %s %s %s" % (_date, _title, _title_nl))
document.timeline.append(DocumentTimeLine.objects.create(title={"fr": _title, "nl": _title_nl}, date=_date))
elif dico.get(u"Compétence"):
document.timeline = []
for (_date, _title) in [clean_text(x).split(u" \xa0 ", 1) for x in dico[u"Compétence"]["head"].contents[::2]]:
logger.debug("append time line %s %s %s" % (_date, _title, ""))
document.timeline.append(DocumentTimeLine.objects.create(title={"fr": _title, "nl": ""}, date=_date))
elif dico_nl.get(u"Bevoegdheid"):
document.timeline = []
for (_date, _title_nl) in [clean_text(x).split(u" \xa0 ", 1) for x in dico_nl[u"Bevoegdheid"]["head"].contents[::2]]:
logger.debug("append time line %s %s %s" % (_date, "", _title_nl))
document.timeline.append(DocumentTimeLine.objects.create(title={"fr": "", "nl": _title_nl}, date=_date))
if dico.get("Analyse des interventions"):
document.analysis = get_or_create(Analysis, _id="lachambre_id", lachambre_id=dico["Analyse des interventions"]["head"].a.text, url=dico["Analyse des interventions"]["head"].a["href"])
def handle_commission(commission):
soup, suppe = read_or_dl_with_nl(LACHAMBRE_PREFIX + commission.url, "commission %s" % commission.lachambre_id)
commission.full_name["fr"] = soup.h1.text
commission.full_name["nl"] = suppe.h1.text
commission.deputies = []
seats = {"fr": {}, "nl": {}}
for i, j in zip(soup('p'), suppe('p')):
role = i.b.text[:-1]
role_nl = j.b.text[:-1]
for dep in i('a'):
deputy = Deputy.objects.get(lachambre_id=re.search("key=([O0-9]+)", dep["href"]).groups()[0])
membership = get_or_create(CommissionMembership, deputy=deputy, commission=commission)
membership.role = role
membership.save()
commission.deputies.append(membership.id)
seats["fr"][role] = map(lambda x: (x[0], len(x[1].split(','))), zip(map(lambda x: x.text[:-1], i('b')[1:]), str(i).split("<br />")[1:]))
seats["nl"][role_nl] = map(lambda x: (x[0], len(x[1].split(','))), zip(map(lambda x: x.text[:-1], i('b')[1:]), str(i).split("<br />")[1:]))
commission.seats = seats
commission.save()
def scrape():
soup, suppe = read_or_dl_with_nl("http://www.lachambre.be/kvvcr/showpage.cfm?section=/comm/commissions&language=fr&cfm=/site/wwwcfm/comm/LstCom.cfm&rightmenu=right_cricra", "commissions list")
_type = ""
for i, j in zip(soup("div", id="story")[1], suppe("div", id="story")[1]):
if not isinstance(i, NavigableString) and (i.h4 or i.a):
if i.h4:
_type = i.h4.text
_type_nl = j.h4.text
elif i.a:
commission = get_or_create(Commission, lachambre_id=int(re.search("com=(\d+)", i.a["href"]).groups()[0]))
commission.type["fr"] = _type
commission.type["nl"] = _type_nl
commission.name["fr"] = i.a.text
commission.name["nl"] = j.a.text
commission.url = i.a["href"]
commission.save()
for com in list(Commission.objects.all()):
handle_commission(com)
def register_partition(self, task, partition_name, key_func, force=False):
assert isinstance(task, Task)
partition = get_or_create(
Partition,
keys=dict(
task=task,
name=partition_name
)
)
label_query = Labels.select().where(
Labels.task == task
)
group_query = GroupDefinition.select().where(
GroupDefinition.partition == partition
)
update = group_query.count() == label_query.count()
if update and not force:
return
if update:
print 'updating'
for label in label_query:
GroupDefinition.update(
fold=key_func(label)
).where(
SplitDefinition.partition == partition,
SplitDefinition.label == label
).execute()
else:
print 'saving'
rows = [dict(
partition=partition,
label=label,
key=key_func(label)
) for label in label_query]
GroupDefinition.insert_many(rows).execute()
def deputies_list(reset=False):
soup = read_or_dl("http://www.lachambre.be/kvvcr/showpage.cfm?section=/depute&language=fr&rightmenu=right_depute&cfm=/site/wwwcfm/depute/cvlist.cfm", "deputies", reset)
for dep in soup('table')[4]('tr'):
items = dep('td')
full_name = re.sub(' +', ' ', items[0].a.text).strip()
url = items[0].a['href']
party = get_or_create(Party, name=items[1].a.text, url=dict(items[1].a.attrs)['href'])
email = items[2].a.text
website = items[3].a['href'] if items[3].a else None
# yes, one deputies key contains a O instead of an 0, I'm not joking
lachambre_id = re.search('key=([0-9O]+)', url).groups()[0]
if not Deputy.objects.filter(lachambre_id=lachambre_id):
deputy = Deputy.objects.create(full_name=full_name,
party=party,
url=url,
websites=[website] if website else [],
lachambre_id=lachambre_id,
emails=[email])
logger.debug('adding new deputy %s %s %s %s %s' % (lachambre_id.encode("Utf-8"), full_name.encode("Utf-8"), party, email.encode("Utf-8"), website.encode("Utf-8") if website else ''))
logger.info("[NEW] deputy: %s" % deputy)
def _get_deputy_questions(url, deputy, type, reset=False):
soupsoup = read_or_dl(LACHAMBRE_PREFIX + lame_url(url), '%s %s' % (deputy.full_name, type), reset)
setattr(deputy, "questions_%s_url" % type, url)
setattr(deputy, "questions_%s_list" % type, [])
for i in soupsoup('table')[3]('tr', valign="top"):
logger.debug("add", type, i.tr('td')[1].text.strip())
dico = table2dic(i.table('td'))
logger.debug("%s" % dico)
getattr(deputy, "questions_%s_list" % type).\
append(get_or_create(Question,
_id="lachambre_id",
title=dico["Titre"],
lachambre_id=re.search("dossierID=([0-9A-Za-z-]+)", i.a["href"]).groups()[0],
reunion_type=dico.get(u"Réunion"),
reunion_date=dico.get("Date discussion"),
session_id=dico.get("Session"),
pdf_url=dico.get(u"Compte rendu intégral", {"href": None})["href"],
eurovoc_descriptors=map(lambda x: x.strip(), dico.get("Descripteurs Eurovoc", "").split('|')),
keywords=map(lambda x: x.strip(), dico.get(u"Mots-clés libres", "").split("|")),
url=i.a['href'],
type=type))
def _create_base_product(self):
"""
Creates base product without relationships that require its existence formerly. Needs objects from
INDEPENDENT object names to exist first. Product is then flushed to the database but not committed.
:return: product, instance of a product
"""
product_class = get_class_by_table_name(f'{self.product_type}_product')
if product_class:
# build kwargs for product
product_kwargs = {}
for field in self.product_config[BASE_FIELDS]:
if field in self.product_config[SINGLE_RELATIONS]:
product_kwargs[field] = self.objects[field]
else:
product_kwargs[field] = self.data[field]
product, _ = get_or_create(product_class, **product_kwargs)
db_session.add(product)
db_session.flush()
self.objects['product_id'] = product.id
return product
raise ValueError('Unknown product class')
def post(self, name):
print name
form = EditAuthorForm()
author = get_or_404(Author, name=name)
if form.validate():
if author.name != form.name.data:
author.name = form.name.data
try:
session.commit()
except:
session.rollback()
msg = 'Author <strong class=badge>{0}</strong> already \
exists!'.format(form.name.data)
flash(msg)
# return render_template('edit_author.haml', form=form, name=name)
return self.get(name)
new_books = str_separator(form.title.data)
old_books = get_books(author, format='list')
# get what to delete from exists books list and what to add, to bring
# to edited field books
rm_lst, append_lst = diff_list(old_books, new_books)
print(rm_lst, append_lst)
# remove not related books
for title in rm_lst:
b = get_or_404(Book, title=title)
author.books.remove(b)
if len(b.authors) == 1:
session.delete(b)
# relate new books with exist author
for title in append_lst:
b = get_or_create(Book, title=title)
author.books.append(b)
session.commit()
return redirect(url_for('index'))
return render_template('edit_author.haml', form=form, name=name)
def post(self, title):
book = get_or_404(Book, title=title)
form = EditBookForm()
if form.validate():
if book.title != form.title.data:
book.title = form.title.data
try:
session.commit()
except:
session.rollback()
msg = 'Book <strong class=badge>{0}</strong> already \
exists!'.format(form.title.data)
flash(msg)
# return render_template('edit_book.haml', form=form, title=title)
return self.get(title)
# Get current and changed list of authors.
new_authors = str_separator(form.name.data)
old_authors = get_authors(book, format='list')
# We get authors' names which have to be deleted from an original list
# and also authors' names which have to be added.
rm_lst, append_lst = diff_list(old_authors, new_authors)
# remove not related authors
for name in rm_lst:
a = get_or_404(Author, name=name)
book.authors.remove(a)
if len(a.books) == 1:
session.delete(a)
# relate new authors with exist book
for name in append_lst:
a = get_or_create(Author, name=name)
book.authors.append(a)
session.commit()
return redirect(url_for('index'))
return render_template('edit_book.haml', form=form, title=title)
def _get_deputy_questions(url, deputy, type, reset=False):
soupsoup = read_or_dl(LACHAMBRE_PREFIX + lame_url(url),
'%s %s' % (deputy.full_name, type), reset)
setattr(deputy, "questions_%s_url" % type, url)
setattr(deputy, "questions_%s_list" % type, [])
for i in soupsoup('table')[3]('tr', valign="top"):
logger.debug("add", type, i.tr('td')[1].text.strip())
dico = table2dic(i.table('td'))
logger.debug("%s" % dico)
getattr(deputy, "questions_%s_list" % type).\
append(get_or_create(Question,
_id="lachambre_id",
title=dico["Titre"],
lachambre_id=re.search("dossierID=([0-9A-Za-z-]+)", i.a["href"]).groups()[0],
reunion_type=dico.get(u"Réunion"),
reunion_date=dico.get("Date discussion"),
session_id=dico.get("Session"),
pdf_url=dico.get(u"Compte rendu intégral", {"href": None})["href"],
eurovoc_descriptors=map(lambda x: x.strip(), dico.get("Descripteurs Eurovoc", "").split('|')),
keywords=map(lambda x: x.strip(), dico.get(u"Mots-clés libres", "").split("|")),
url=i.a['href'],
type=type))
def login():
credentials = request.get_json(force=True)
email = credentials['email']
app_id = credentials['app_id']
password = credentials['password']
user = db.session.query(UserAccount).filter_by(email=email).first()
if user and user.check_password(password):
if db.session.query(Application).filter_by(id=app_id).first():
token = generate_token()
user_app, created = get_or_create(UserApplication, application_id=app_id,
user_id=user.id)
if not created:
user_app.last_login = datetime.datetime.now()
user_app.token = token
else:
user_app.token = token
db.session.add(user_app)
db.session.commit()
return jsonify({"code": 0, "token": token})
else:
return jsonify({"code": 2, "token": None})
else:
return jsonify({"code": 2, "token": None})
def add_grade():
"""
Запрос на добавление оценки студенту по дисциплине
:param student_id: - int - идентификатор студента
:param class_id: - int - идентификатор дисциплины
:param score: - int - оценка(от 0 до 5)
:return:
Если дисциплинна уже проставленна, оценка заменятеся
"""
data = request.get_json(force=True)
student = Student.query.get_or_404(data['student_id'])
c_class = Class.query.get_or_404(data['class_id'])
grade = get_or_create(
db.session,
Grade,
student_id=student.id,
class_id=c_class.id
)
grade.score = data['score']
db.session.commit()
return jsonify({"status": "ok"})
def __init__(self, connection_path, meta_path):
self.connection_params = json.load(open(connection_path, 'r'))
db.init(
self.connection_params['database'],
**self.connection_params['credentials']
)
db.create_tables(all_tables, safe=True)
self.db = db
self.meta = Metadata(meta_path)
# Load the dataset object
self.dataset, created = Dataset.get_or_create(
name=self.meta.name
)
# Load the tasks
self.tasks = {}
for task in self.meta.tasks:
self.tasks[task['name']] = get_or_create(
Task,
keys=dict(
dataset=self.dataset,
name=task['name']
),
values=dict(
columns=task['columns'],
num_columns=len(task['columns'])
)
)
# Load the views
self.views = {}
for view in self.meta.views:
self.views[view['name']] = get_or_create(
View,
keys=dict(
dataset=self.dataset,
name=view['name']
),
values=dict(
columns=view['columns'],
num_columns=len(view['columns'])
)
)
# Load the sequences
self.sequences = {}
for sequence in self.meta.sequences:
self.sequences[sequence['name']] = get_or_create(
Sequence,
keys=dict(
dataset=self.dataset,
name=sequence['name']
),
values=dict(
t_min=sequence['t_min'],
t_max=sequence['t_max']
)
)
# Load the partitions
self.splits = {}
self.partitions = {}
self.classifiers = {}
self.feature_unions = {}
for task_name, task in self.tasks.iteritems():
pp = {pp.name: pp for pp in task.partitions}
self.partitions[task_name] = pp
self.partitions[task] = pp
ss = {pp.name: list(pp.splits.execute()) for pp in task.partitions}
self.splits[task_name] = ss
self.splits[task] = ss
cc = {}
for classifier in task.classifiers:
cc[classifier.name] = classifier
self.classifiers[task] = cc
self.classifiers[task_name] = cc
ff = {}
for feature_union in task.feature_unions:
ff[feature_union.name] = feature_union
self.feature_unions[task] = ff
self.feature_unions[task_name] = ff
def setTags(self, tags):
self.tags = [get_or_create(Tag, title=tag) for tag in tags]
def refresh_background(course_id):
"""
Look up existing extensions and apply them to new quizzes.
:param course_id: The Canvas ID of the Course.
:type course_id: int
:rtype: dict
:returns: A dictionary containing two parts:
- success `bool` False if there was an error, True otherwise.
- message `str` A long description of success or failure.
"""
job = get_current_job()
update_job(job, 0, 'Starting...', 'started')
with app.app_context():
course, created = get_or_create(
db.session,
Course,
canvas_id=course_id
)
try:
course_name = get_course(course_id).get(
'name',
'<UNNAMED COURSE>'
)
course.course_name = course_name
db.session.commit()
except requests.exceptions.HTTPError:
update_job(
job,
0,
'Course not found.',
'failed',
error=True
)
logger.exception('Unable to find course #{}'.format(course_id))
return job.meta
# quiz stuff
quizzes = missing_quizzes(course_id)
num_quizzes = len(quizzes)
if num_quizzes < 1:
update_job(
job,
100,
'Complete. No quizzes required updates.',
'complete',
error=False
)
return job.meta
percent_user_map = defaultdict(list)
inactive_list = []
update_job(job, 0, 'Getting past extensions.', 'processing', False)
for extension in course.extensions:
# If extension is inactive, ignore.
if not extension.active:
inactive_list.append(extension.user.sortable_name)
logger.debug('Extension #{} is inactive.'.format(
extension.id
))
continue
user_canvas_id = User.query.filter_by(
id=extension.user_id
).first().canvas_id
# Check if user is in course. If not, deactivate extension.
try:
canvas_user = get_user(course_id, user_canvas_id)
# Skip user if not a student. Fixes an edge case where a
# student that previously recieved an extension changes roles.
enrolls = canvas_user.get('enrollments', [])
type_list = [e['type'] for e in enrolls if e['enrollment_state'] == 'active']
if not any(t == 'StudentEnrollment' for t in type_list):
logger.info((
"User #{} was found in course #{}, but is not an "
"active student. Deactivating extension #{}. Roles "
"found: {}"
).format(
user_canvas_id,
course_id,
extension.id,
", ".join(type_list) if len(enrolls) > 0 else None
))
extension.active = False
db.session.commit()
inactive_list.append(extension.user.sortable_name)
continue
except requests.exceptions.HTTPError:
log_str = (
'User #{} not in course #{}. Deactivating extension #{}.'
)
logger.info(
log_str.format(user_canvas_id, course_id, extension.id)
)
extension.active = False
db.session.commit()
inactive_list.append(extension.user.sortable_name)
continue
percent_user_map[extension.percent].append(user_canvas_id)
if len(percent_user_map) < 1:
msg_str = 'No active extensions were found.<br>'
if len(inactive_list) > 0:
msg_str += ' Extensions for the following students are inactive:<br>{}'
msg_str = msg_str.format("<br>".join(inactive_list))
update_job(
job,
100,
msg_str,
'complete',
error=False
)
return job.meta
for index, quiz in enumerate(quizzes):
quiz_id = quiz.get('id', None)
quiz_title = quiz.get('title', '[UNTITLED QUIZ]')
comp_perc = int(((float(index)) / float(num_quizzes)) * 100)
refreshing_str = 'Refreshing quiz #{} - {} [{} of {}]'
update_job(
job,
comp_perc,
refreshing_str.format(
quiz_id,
quiz_title,
index + 1,
num_quizzes
),
'processing',
error=False
)
for percent, user_list in percent_user_map.iteritems():
extension_response = extend_quiz(
course_id,
quiz,
percent,
user_list
)
if extension_response.get('success', False) is True:
# add/update quiz
quiz_obj, created = get_or_create(
db.session,
Quiz,
canvas_id=quiz_id,
course_id=course.id
)
quiz_obj.title = quiz_title
db.session.commit()
else:
error_message = 'Some quizzes couldn\'t be updated. '
error_message += extension_response.get('message', '')
update_job(
job,
comp_perc,
error_message,
'failed',
error=True,
)
return job.meta
msg = '{} quizzes have been updated.'.format(len(quizzes))
update_job(job, 100, msg, 'complete', error=False)
return job.meta
def get_new_documents():
for document_page in read_or_dl("http://www.lachambre.be/kvvcr/showpage.cfm?section=/flwb&language=fr&rightmenu=right&cfm=ListDocument.cfm", "all documents")('div', **{'class': re.compile("linklist_[01]")}):
soup, suppe = read_or_dl_with_nl(LACHAMBRE_PREFIX + document_page.a["href"], "document %s" % document_page.a.text)
for soup, suppe in zip(soup('table')[4]('tr', valign="top"), suppe('table')[4]('tr', valign="top")):
get_or_create(Document, _id="lachambre_id", title={"fr": soup('div')[1].text, "nl": suppe('div')[1].text}, lachambre_id=soup.div.text, url=soup.a["href"])
def update_background(course_id, extension_dict):
"""
Update time on selected students' quizzes to a specified percentage.
:param course_id: The Canvas ID of the Course to update in
:type course_id: int
:param extension_dict: A dictionary that includes the percent of
time and a list of canvas user ids.
Example:
{
'percent': '300',
'user_ids': [
'0123456',
'1234567',
'9867543',
'5555555'
]
}
:type extension_dict: dict
"""
job = get_current_job()
update_job(job, 0, 'Starting...', 'started')
with app.app_context():
if not extension_dict:
update_job(
job,
0,
'Invalid Request',
'failed',
error=True
)
logger.warning('Invalid Request: {}'.format(extension_dict))
return job.meta
try:
course_json = get_course(course_id)
except requests.exceptions.HTTPError:
update_job(
job,
0,
'Course not found.',
'failed',
error=True
)
logger.exception('Unable to find course #{}'.format(course_id))
return job.meta
course_name = course_json.get('name', '<UNNAMED COURSE>')
user_ids = extension_dict.get('user_ids', [])
percent = extension_dict.get('percent', None)
if not percent:
update_job(
job,
0,
'`percent` field required.',
'failed',
error=True
)
logger.warning('Percent field not provided. Request: {}'.format(
extension_dict
))
return job.meta
course, created = get_or_create(db.session, Course, canvas_id=course_id)
course.course_name = course_name
db.session.commit()
for user_id in user_ids:
try:
canvas_user = get_user(course_id, user_id)
sortable_name = canvas_user.get('sortable_name', '<MISSING NAME>')
sis_id = canvas_user.get('sis_user_id')
except requests.exceptions.HTTPError:
# Unable to find user. Log and skip them.
logger.warning(
"Unable to find user #{} in course #{}".format(
user_id,
course_id
)
)
continue
user, created = get_or_create(db.session, User, canvas_id=user_id)
user.sortable_name = sortable_name
user.sis_id = sis_id
db.session.commit()
# create/update extension
extension, created = get_or_create(
db.session,
Extension,
course_id=course.id,
user_id=user.id
)
extension.percent = percent
db.session.commit()
quizzes = get_quizzes(course_id)
num_quizzes = len(quizzes)
quiz_time_list = []
unchanged_quiz_time_list = []
if num_quizzes < 1:
update_job(
job,
0,
'Sorry, there are no quizzes for this course.',
'failed',
error=True
)
logger.warning(
"No quizzes found for course {}. Unable to update.".format(
course_id
)
)
return job.meta
for index, quiz in enumerate(quizzes):
quiz_id = quiz.get('id', None)
quiz_title = quiz.get('title', "[UNTITLED QUIZ]")
comp_perc = int(((float(index)) / float(num_quizzes)) * 100)
updating_str = 'Updating quiz #{} - {} [{} of {}]'
update_job(
job,
comp_perc,
updating_str.format(quiz_id, quiz_title, index + 1, num_quizzes),
'processing',
error=False
)
extension_response = extend_quiz(course_id, quiz, percent, user_ids)
if extension_response.get('success', False) is True:
# add/update quiz
quiz_obj, created = get_or_create(
db.session,
Quiz,
canvas_id=quiz_id,
course_id=course.id
)
quiz_obj.title = quiz_title
db.session.commit()
added_time = extension_response.get('added_time', None)
if added_time is not None:
quiz_time_list.append({
"title": quiz_title,
"added_time": added_time
})
else:
unchanged_quiz_time_list.append({"title": quiz_title})
else:
update_job(
job,
comp_perc,
extension_response.get(
'message',
'An unknown error occured.'
),
'failed',
error=True
)
logger.error("Extension failed: {}".format(extension_response))
return job.meta
msg_str = (
'Success! {} {} been updated for {} student(s) to have {}% time. '
'{} {} no time limit and were left unchanged.'
)
message = msg_str.format(
len(quiz_time_list),
"quizzes have" if len(quiz_time_list) != 1 else "quiz has",
len(user_ids),
percent,
len(unchanged_quiz_time_list),
"quizzes have" if len(unchanged_quiz_time_list) != 1 else "quiz has"
)
update_job(job, 100, message, 'complete', error=False)
job.meta['quiz_list'] = quiz_time_list
job.meta['unchanged_list'] = unchanged_quiz_time_list
job.save()
return job.meta
def _createTags(self, tags):
return [get_or_create(Tag, title=tag) for tag in tags]
forks = determine_capacity(v2)
if forks is None:
raise RuntimeException("Capactiy not determinable")
forks = forks - 1
print("Forks set to {}".format(forks))
cancel_all(v2.jobs)
cancel_all(v2.workflow_jobs)
delete_all(v2.jobs)
delete_all(v2.workflow_jobs)
delete_all(v2.workflow_job_templates)
delete_all(v2.inventory)
delete_all(v2.job_templates)
org = get_or_create(v2.organizations, name='cmurders murder inc.')
inv = get_or_create(org.related.inventories,
name='cmurders murdertown',
organization=org,
variables=inventory_vars)
for i in xrange(0, forks + 5):
h = get_or_create(inv.related.hosts,
name='high-capacity-host-{}'.format(i),
inventory=inv)
proj = get_or_create(
v2.projects,
name='Murder Tools',
organization=org,
scm_url='https://github.com/chrismeyersfsu/ansible-examples.git',
def get_new_documents():
for document_page in read_or_dl("http://www.lachambre.be/kvvcr/showpage.cfm?section=/flwb&language=fr&rightmenu=right&cfm=ListDocument.cfm", "all documents")('div', **{'class': re.compile("linklist_[01]")}):
soup, suppe = read_or_dl_with_nl(LACHAMBRE_PREFIX + document_page.a["href"], "document %s" % document_page.a.text)
for soup, suppe in zip(soup('table')[4]('tr', valign="top"), suppe('table')[4]('tr', valign="top")):
get_or_create(Document, _id="lachambre_id", title={"fr": soup('div')[1].text, "nl": suppe('div')[1].text}, lachambre_id=soup.div.text, url=soup.a["href"])
import time
import sys, os
sys.path.append(os.path.join(os.getcwd(), '..', '..'))
from utils import get_or_create, delete_all
host_vars = """ansible_connection: local"""
sleep_interval_vars = """---
sleep_min: 240
sleep_max: 480
"""
delete_all(v2.jobs)
org = get_or_create(v2.organizations, name='cmurders murder inc.')
inv = get_or_create(org.related.inventories, name='cmurders murdertown', organization=org)
h = get_or_create(inv.related.hosts, name='cmurder', variables=host_vars, inventory=inv)
proj = get_or_create(v2.projects, name='Murder Tools', organization=org, scm_url='https://github.com/chrismeyersfsu/ansible-examples.git', wait=True)
jt = get_or_create(v2.job_templates, name="Kill Number 1", organization=org, inventory=inv, project=proj, extra_vars=sleep_interval_vars, playbook='sleep.yml', allow_simultaneous=True)
wfjt = get_or_create(v2.workflow_job_templates, name="Mass Murder", organization=org)
parallel_count = 100
current_node_count = wfjt.related.workflow_nodes.get()['count']
if current_node_count < parallel_count:
for i in xrange(0, parallel_count):
wfjt.related.workflow_nodes.post(dict(unified_job_template=jt.id))
wfj = None
#wfj = wfjt.launch().wait_until_completed()
def on_event():
"""Handles an event from Hangouts Chat."""
engine = create_engine(DATABASE_URL)
if not engine.dialect.has_table(engine, USER_TABLE_NAME):
Base.metadata.create_all(engine)
Base.metadata.bind = engine
DBSession = sessionmaker(bind=engine)
session = DBSession()
event = request.get_json()
email = event['message']['sender']['email']
text = event['message']['text']
if event['type'] == 'ADDED_TO_SPACE' and event['space']['type'] == 'ROOM':
text = 'Thanks for adding me to "{0}"!'.format(
event['space']['displayName'])
elif event['type'] == 'MESSAGE':
message = event['message']['text']
message = message.replace("@visualtime", "")
if message.startswith('/login'):
if len(message.split()) >= 2:
password = message.split()[1]
get_or_create(session, User, email=email, password=password)
text = 'Usuario {0} ha sido iniciado correctamente'.format(
email)
if message.startswith('/info'):
users = session.query(User).filter(User.email == email)
if users.count() == 0:
text = "Oppp!! usa /login <password> para iniciar tu usuario"
else:
user = users.first()
visualtime_client = VisualTimeHelper(user.email, user.password)
visualtime_client.login()
visualtime_info = visualtime_client.get_output_time()
text = 'Hola {0}: \n' \
'Llevas trabajando: {1} \n' \
'Hoy te vas a casa a las: {2} \n' \
'El porcentaje de avance de tu jornada es: {3} % \n' \
'La jornada de hoy tiene: {4} horas \n' \
'Ahora mismo te encuentras: {5}'.format(user.email,
visualtime_info['working_time'],
visualtime_info['output_time'].split()[1],
visualtime_info['percent'],
visualtime_info['day'],
"trabajando" if int(visualtime_info['direction']) == 1 else "descansando")
elif message.startswith("/push"):
users = session.query(User).filter(User.email == email)
if users.count() == 0:
text = u"Oppp!! usa /login <password> para iniciar tu usuario"
else:
user = users.first()
visualtime_client = VisualTimeHelper(user.email, user.password)
visualtime_client.login()
visualtime_info = visualtime_client.push()
if visualtime_info:
text = u"Ok!! Fichaje creado con éxito."
else:
text = u"Upps! Hay un error creado el fichaje."
elif message.startswith("/logout"):
users = session.query(User).filter(User.email == email)
if users.count() == 0:
text = u"Oppp!! usa /login <password> para iniciar tu usuario"
else:
users.delete(synchronize_session=False)
text = u"Ha cerrado sesión con éxito."
else:
return
return jsonify({'text': text})
forks = determine_capacity(v2)
if forks is None:
raise RuntimeException("Capactiy not determinable")
forks = forks - 1
print("Forks set to {}".format(forks))
cancel_all(v2.jobs)
cancel_all(v2.workflow_jobs)
delete_all(v2.jobs)
delete_all(v2.workflow_jobs)
delete_all(v2.workflow_job_templates)
delete_all(v2.inventory)
delete_all(v2.job_templates)
org = get_or_create(v2.organizations, name='cmurders murder inc.')
inv = get_or_create(org.related.inventories, name='cmurders murdertown', organization=org, variables=inventory_vars)
for i in xrange(0, forks+5):
h = get_or_create(inv.related.hosts, name='high-capacity-host-{}'.format(i), inventory=inv)
proj = get_or_create(v2.projects, name='Murder Tools', organization=org, scm_url='https://github.com/chrismeyersfsu/ansible-examples.git', wait=True)
jt = get_or_create(v2.job_templates, name="High Capacity JT - {} forks".format(forks), organization=org, inventory=inv, project=proj, extra_vars=sleep_interval_vars, playbook='sleep.yml', allow_simultaneous=True, forks=forks)
wfjt = get_or_create(v2.workflow_job_templates, name="High Capacity Workflow", organization=org)
parallel_count = 100
current_node_count = wfjt.related.workflow_nodes.get()['count']
if current_node_count < parallel_count:
for i in xrange(0, parallel_count):
def setTags(self, tags):
self.tags = [get_or_create(Tag, title = tag) for tag in tags]
def _createTags(self, tags):
return [get_or_create(Tag, title = tag) for tag in tags]
