def disable():
users = User.objects(Q(timestamp__lt=int(time())))
for user in list(users):
Session.objects(data__login=user.login).update(
**dict(data__rate=0, data__timestamp=2147483647))
users.update(**dict(rate=0, timestamp=2147483647))
return dumps(dict(code=200))
def login(username, master_password):
"""
Posts username and authentication hash to /login.
Decrypts vault on successful login.
Returns new Session.
"""
print('Logging in...')
url = api_base_url + '/login'
auth_hash = get_auth_hash(username, master_password)
body = {
'username': username,
'auth_hash': base64.b64encode(auth_hash).decode(),
}
response = requests.post(url, data=json.dumps(body))
if response.status_code != requests.codes.ok:
print('Login attempt failed with status code: ' +
str(response.status_code))
exit()
response_body = response.json()
cipher_text = base64.b64decode(response_body['vault'].encode())
key = get_key(username, master_password)
vault_str = decrypt_vault(cipher_text, key)
vault_dict = json.loads(vault_str)
session = Session(username, auth_hash, key)
session.dict_to_vault(vault_dict)
return session
def insert_tasks(PL, task_file):
pipeline = Pipeline(PL.name, PL.log_dir)
logging.debug("Pipeline is: {}".format(pipeline))
task_list = PL.prepare_managed_tasks()
logging.debug("Task list is: {}".format([x['name'] for x in task_list]))
# we need to be able to translate the dependencies as stored in the task
# list (list of other task names that a particular task depends on)
# into a list of Job object references that have already been added to the
# session. We will build up a dictionary of task['name'] : Job as we
# insert them
deps_to_job = {}
print(" Inserting tasks into {}".format(task_file))
logging.info("Inserting tasks into {}".format(task_file))
try:
for task in task_list:
print(" -> {}".format(task['name']))
try:
dependencies = [deps_to_job[d] for d in task['dependencies']]
except KeyError as e:
logging.exception("Key error processing dependencies")
msg = "Task {} depends on a task that hasn't been been " \
"processed ({}). Check your Pipeline XML".format(
task['name'], e.args[0])
raise Exception(msg)
job = Job(pipeline, task['name'], task['threads'],
task['stdout_path'], task['stderr_path'],
task['script_path'], task['epilogue_path'], task['mem'],
task['email_list'], task['mail_options'],
task['batch_env'], dependencies, task['queue'],
task['walltime'])
deps_to_job[task['name']] = job
logging.debug("Adding job {} (log dir: {}) to session".format(
job.job_name, job.pipeline.log_directory))
Session.add(job)
except Exception as e:
logging.exception("Error inserting tasks into database")
print("Error inserting tasks into database: {}".format(e),
file=sys.stderr)
sys.exit(6)
# only commit the session if we were able to add all the jobs to the session
# without catching an Exception
Session.commit()
logging.info(" {} tasks have been inserted into task file {}; "
"(log dir: {})".format(len(task_list), task_file, PL.log_dir))
return len(task_list)
def get(self):
cutoff = datetime.utcnow() - MAX_AGE
sessions = Session.query(Session.created_at < cutoff).fetch(
200, keys_only=True)
if len(sessions) > 0:
ndb.delete_multi(sessions)
logging.info('%s sessions cleared' % len(sessions))
def run():
systemLog = Log()
threads = []
sessions = {}
proxy = Proxy().random_proxy()
while 1:
payments = Payment().get_payments()
for p in payments:
payment = Payment()
payment = payment.set_payment(p)
name = payment.get_name()
type = payment.get_type()
if payment.get_status() == 0:
continue
connector_module = 'connector.http.' + name.lower()
if type == 'enterprise':
connector_module += '_enterprise'
name += 'Enterprise'
module = importlib.import_module(connector_module)
class_ = getattr(module, name)
sessions[name] = Session(proxy)
connector = class_(payment, sessions[name], proxy)
if sessions[name].is_changing_proxy() == 1:
systemLog.log('Need to change ' + name + ' proxy!', 'debug')
proxy = Proxy().random_proxy()
thread_name = "Thread " + str(
payment.get_id()) + " " + name + " " + type
thread = ThreadConnector(thread_name.upper(), connector, 5)
thread.start()
threads.append(thread)
for t in threads:
t.join(60)
def wrapper(*args, **kwargs):
"""Wrap the route."""
database = kwargs["database"]
session_id = str(bottle.request.get_cookie("session_id"))
session = Session(sessions.find_session(database, session_id))
if not session.is_valid():
cls.abort(
401,
"%s-access to %s denied: session %s not authenticated",
context, session_id)
authorized_users = latest_reports_overview(database).get("editors")
if not session.is_authorized(authorized_users):
cls.abort(403,
"%s-access to %s denied: session %s not authorized",
context, session_id)
return callback(*args, **kwargs)
def signup(username, master_password):
"""
Posts username and authentication hash to /signup.
Calls update vault with empty vault on successful signup.
Returns new Session.
"""
print('Signing up...')
url = api_base_url + '/signup'
auth_hash = get_auth_hash(username, master_password)
body = {
'username': username,
'auth_hash': base64.b64encode(auth_hash).decode(),
}
response = requests.post(url, data=json.dumps(body))
if response.status_code != requests.codes.ok:
print('Signup attempt failed with status code: ' +
str(response.status_code))
exit()
key = get_key(username, master_password)
vault = []
session = Session(username, auth_hash, key, vault)
update_vault(session)
return session
def put(self):
""" User Log in """
# Validation
if not request.is_json:
return {"result": -1, "msg": "Missing JSON in request"}, 400
id = request.json.get('id', None)
password = request.json.get('password', None)
if not id:
return {"result": -1, "msg": "Missing id parameter"}, 400
if not password:
return {"result": -1, "msg": "Missing password parameter"}, 400
user = User.query.filter_by(id=id).first()
if not user or user.signout_dt:
return {"result": -1, "msg": "User not found"}, 400
if not bcrypt.checkpw(password.encode('utf-8'),
user.password.encode('utf-8')):
return {"result": -1, "msg": "Bad password"}, 400
# Make session info
session_id = str(uuid.uuid4()).replace('-', '')[:20]
client_ip = request.environ.get('HTTP_X_REAL_IP',
request.remote_addr) # get Client IP
session = Session(session_id=session_id, id=id, client_ip=client_ip)
db.session.add(session)
db.session.commit()
return {"result": 0, "session": session_id}, 200
def update_existing_session(self, conn, session_id, tag):
cherrypy.log('Hello from update_existing_session', severity=logging.DEBUG)
temp = execute_fetchone(
conn,
FetchController.update_existing_session_query,
{ 'tag': tag, 'ses_id': session_id })
conn.commit()
cherrypy.log(pformat(temp), severity=logging.DEBUG)
return Session.from_db_row(temp['result'])
def wrapper(*args, **kwargs):
token = request.headers.get('Authorization')
if token:
user_session = Session.find_one_by_text_token(token)
if user_session:
if provide_user:
kwargs['user'] = user_session.user
if provide_session:
kwargs['session'] = user_session
return func(*args, **kwargs)
raise AccessDenied
def login(self, username, password):
user = self.users_dao.find_by_username(username)
if user is None:
raise Exception('El usuario y/o contraseña son incorrectos, por favor intenta de nuevo.')
crypto = Crypto()
if not crypto.verify_encrypted_password(password, user.password):
raise Exception('El usuario y/o contraseña son incorrectos, por favor intenta de nuevo.')
session_uuid = str(uuid.uuid4())
session = Session(username, session_uuid)
self.session_dao.save_session(session)
return session_uuid
def read_session(path):
group_id = ""
info = []
session_array = []
wb = xlrd.open_workbook(path)
sheet = wb.sheet_by_index(0)
sheet.cell_value(0, 0)
data = [sheet.row_values(row_num) for row_num in range(sheet.nrows)]
for item in data:
new_item = excel_tools.remove_empty_element_in_array(item)
if new_item:
info.append(new_item)
for item in info:
index = info.index(item)
if index == 2:
if len(item) > 0:
group_id = excel_tools.format_group_id(item[0])
if index > 3:
if len(item) >= 5:
audience = excel_tools.format_audience_name(
item[5]) if item[5] else "-"
teacher_name = excel_tools.remove_repetition_in_str(item[4])
else:
audience = "-"
teacher_name = "-"
date = excel_tools.format_date_from_excel(wb, item[0])
time_value = item[1]
if type(time_value) is not str:
time = excel_tools.format_time(item[1])
else:
time = time_value
name = excel_tools.remove_repetition_in_str(item[3])
session_item = Session(group_id=group_id,
date=date,
time=time,
type=item[2],
name=name,
teacher_name=teacher_name,
audience=audience)
if session_item:
session_array.append(session_item)
if session_array:
return session_array
else:
return None
def update_session_tag(self, conn, session_type, session_key, tag):
cherrypy.log('Hello from update_session_tag', severity=logging.DEBUG)
temp = execute_fetchone(
conn,
UpdateTagController.update_session_tag_query,
{
'tag': tag,
'ses_type': session_type,
'ses_key': session_key
})
conn.commit()
cherrypy.log(pformat(temp), severity=logging.DEBUG)
return Session.from_db_row(temp['result'])
def get_verified_data(jws, expected=None, session_token=None):
headers = json_parse(get_jws_part(jws, 0))
raw_username = headers['kikUsr']
username = raw_username.lower()
hostname = headers['kikCrdDm'].split('/')[0].lower()
payload = get_jws_part(jws, 1)
if expected is not None and payload != expected:
logging.info('jws, payload does not match expected value')
raise Exception('payload does not match expected value')
try:
data = json_parse(payload)
except:
data = None
try:
session = ndb.Key(urlsafe=session_token).get()
except Exception as e:
session = None
if session is None or not isinstance(
session, Session
) or session.username != username or session.hostname != hostname:
session = None
session_token = None
if username not in TEST_USERS:
verify_jws(jws, raw_username, hostname,
(headers.get('kikDbg') and DEBUG))
elif not DEBUG:
logging.info('jws, chrome user detected')
raise Exception('chrome user detected')
try:
session = Session(username=username, hostname=hostname)
session.put()
session_token = session.key.urlsafe()
except:
pass
return username, hostname, data, session_token
def activate_user(admin, username, timestamp):
user = User.objects(login=username).first()
if user.rate:
user.timestamp += int(timestamp)
else:
user.timestamp = int(time()) + int(timestamp)
user.rate = 1
user.save()
Session.objects(data__login=user.login).update(
**dict(data__timestamp=user.timestamp, data__rate=user.rate))
if app.config['SEND_ACTIVATE']:
unit = get_unit_time(timestamp)
send(
app.config['SMTP_LOGIN'],
'Активация профиля на {0}'.format(app.config['TITLE']),
str(
render_template('mail/activate.html',
admin=admin,
username=username,
unit=unit['key'],
time=unit['value'],
date=datetime.fromtimestamp(int(
time())).strftime('%Y-%m-%d %H:%M:%S'))))
class MainApp(App):
session = Session()
admin_user = User("admin", "12345")
admin_role = Administrator()
admin_assign = Assignment(admin_user, admin_role)
rootpath = "/home/gagos/unb/git/das/das_ep0/objects/organization"
rootdir = Directory(rootpath, None, admin_role)
admin_privilege = Privilege(admin_role, rootdir)
admin_privilege.add_command('OpenObject')
rootdir.update_directory()
def build(self):
return ScreenManagement()
def get_verified_data(jws, expected=None, session_token=None):
headers = json_parse(get_jws_part(jws, 0))
raw_username = headers['kikUsr']
username = raw_username.lower()
hostname = headers['kikCrdDm'].split('/')[0].lower()
payload = get_jws_part(jws, 1)
if expected is not None and payload != expected:
logging.info('jws, payload does not match expected value')
raise Exception('payload does not match expected value')
try:
data = json_parse(payload)
except:
data = None
try:
session = ndb.Key(urlsafe=session_token).get()
except Exception:
session = None
if session is None or not isinstance(session, Session) \
or session.username != username or session.hostname != hostname:
session_token = None
if username not in TEST_USERS:
verify_jws(jws, raw_username, hostname, (headers.get('kikDbg') and DEBUG))
elif not DEBUG:
logging.info('jws, chrome user detected')
raise Exception('chrome user detected')
try:
session = Session(username=username, hostname=hostname)
session.put()
session_token = session.key.urlsafe()
except:
pass
return username, hostname, data, session_token
def recreate_session(self, conn, session_id, session_key, session_type, tag):
cherrypy.log('Hello from recreate_session', severity=logging.DEBUG)
session_string = self.make_session_string()
execute_only(conn, FetchController.delete_session_query, { 'ses_id': session_id })
temp = execute_fetchone(
conn,
FetchController.new_session_query,
{
'ses_key': session_key,
'ses_str': session_string,
'ses_type': session_type,
'tag': tag
})
conn.commit()
cherrypy.log(pformat(temp), severity=logging.DEBUG)
return Session.from_db_row(temp['result'])
def get(self):
try:
client_id = self.request.GET['client_id']
except:
self.redirect("/error?msg=missing client_id for new session")
return
try:
client = ndb.Key(urlsafe=client_id).get()
except:
self.redirect("/error?msg=client was not found")
return
session = Session()
session.dni = client.dni
session.subject = "Revisión"
session.comments = ""
session.proposal = ""
session.put()
self.redirect("/modifySession?session_id=" + session.key.urlsafe() + "&client_id=" + client_id)
def data_to_list_class(data, to_class):
list_answer = []
for el in data:
if to_class == "user":
list_answer.append(User(data=el))
elif to_class == "lesson":
list_answer.append(Lesson(data=el))
elif to_class == "teacher":
list_answer.append(Teacher(data=el))
elif to_class == "event":
list_answer.append(Event(data=el))
elif to_class == "faculty":
list_answer.append(Faculty(data=el))
elif to_class == "session":
list_answer.append(Session(data=el))
elif to_class == "qualification":
list_answer.append(Qualification(data=el))
return list_answer
def get(self):
user = users.get_current_user()
try:
client_id = self.request.GET['client_id']
except:
self.redirect(
"/error?msg=missing client_id for sessions management")
return
if user:
if User.chk_allowed(user.email()):
user_name = user.nickname()
access_link = users.create_logout_url("/")
try:
client = ndb.Key(urlsafe=client_id).get()
except:
self.redirect(
"/error?msg=client key was not found for sessions management"
)
return
sessions = Session.query(
Session.dni == client.dni).order(-Session.added)
template_values = {
"user_name": user_name,
"access_link": access_link,
"client": client,
"sessions": sessions,
"info": AppInfo,
}
jinja = jinja2.get_jinja2(app=self.app)
self.response.write(
jinja.render_template("sessionsManagement.html",
**template_values))
else:
self.redirect("/nouser")
else:
self.redirect("/")
def __init__(self, state_directory):
super().__init__()
# db's up and going
self.state_db = SqlCache(state_directory, 'state', init_state)
self.domains_db = SqlCache(state_directory, 'domains', init_domains)
self.descriptions_db = SqlCache(state_directory, 'descriptions', init_descriptions)
# initialise sessions, containers and ip allocations
binary_sessions = self.state_db.query("SELECT rid,cbor FROM sessions")
self.sessions = {rid: Session.from_binary(rid, binary) for rid, binary in binary_sessions}
self.containers = TaggedCollection()
self.allocations = set()
for session in self.sessions.values():
for uuid, container in session.dependent_containers.items():
self.containers.add(container)
self.allocations.add(container.ip)
# fetch the forwarding table
forwarding = self.state_db.query("SELECT key,value FROM forwarding")
for forward in forwarding:
self.long_term_forwards[forward[0]] = forward[1]
# volumes
self.volumes = Volume.all()
# domain ownership
self.domains = {}
self.global_domains = {}
domains = self.domains_db.query("SELECT domain,token,attempted,user,global FROM domains")
for domain, token, attempted, user, gbl in domains:
dom_obj = Domain(domain, token, user, attempted, gbl)
if user not in self.domains:
self.domains[user] = {} # map domain name to domain object
self.domains[user][domain] = dom_obj
if gbl:
self.global_domains[domain] = dom_obj
# descriptions
descriptions = self.descriptions_db.query("SELECT full_id,cbor FROM descriptions")
self.descriptions = {d[0]: cbor.loads(d[1]) for d in descriptions}
def get_session(self, session_url=None, session_id=None):
"""
Load session details for the given detail page URL or numeric ID
"""
# Read either session_id or session_url from the opposite
if session_id is not None:
session_url = self.urls['SESSION_DETAIL_PRINT_PATTERN'] % session_id
elif session_url is not None:
parsed = parse.search(self.urls['SESSION_DETAIL_PARSE_PATTERN'],
session_url)
session_id = parsed['session_id']
logging.info("Getting session %d from %s", session_id, session_url)
session = Session(numeric_id=session_id)
time.sleep(self.config.WAIT_TIME)
response = self.user_agent.open(session_url)
# forms for later attachment download
mechanize_forms = mechanize.ParseResponse(response,
backwards_compat=False)
# seek(0) is necessary to reset response pointer.
response.seek(0)
html = response.read()
html = html.replace(' ', ' ')
parser = etree.HTMLParser()
dom = etree.parse(StringIO(html), parser)
# check for page errors
try:
page_title = dom.xpath('//h1')[0].text
if 'Fehlermeldung' in page_title:
logging.info("Page %s cannot be accessed due to server error",
session_url)
if self.options.verbose:
print "Page %s cannot be accessed due to server error" % session_url
return
if 'Berechtigungsfehler' in page_title:
logging.info("Page %s cannot be accessed due to permissions",
session_url)
if self.options.verbose:
print "Page %s cannot be accessed due to permissions" % session_url
return
except:
pass
try:
error_h3 = dom.xpath('//h3[@class="smc_h3"]')[0].text.strip()
if 'Keine Daten gefunden' in error_h3:
logging.info("Page %s does not contain any agenda items",
session_url)
if self.options.verbose:
print "Page %s does not contain agenda items" % session_url
return
except:
pass
session.original_url = session_url
# Session title
try:
session.title = dom.xpath(
self.xpath['SESSION_DETAIL_TITLE'])[0].text
except:
logging.critical(
'Cannot find session title element using XPath SESSION_DETAIL_TITLE'
)
raise TemplateError(
'Cannot find session title element using XPath SESSION_DETAIL_TITLE'
)
# Committe link
try:
links = dom.xpath(self.xpath['SESSION_DETAIL_COMMITTEE_LINK'])
for link in links:
href = link.get('href')
parsed = parse.search(
self.urls['COMMITTEE_DETAIL_PARSE_PATTERN'], href)
if parsed is not None:
session.committee_id = parsed['committee_id']
except:
logging.critical(
'Cannot find link to committee detail page using SESSION_DETAIL_COMMITTEE_LINK_XPATH'
)
raise TemplateError(
'Cannot find link to committee detail page using SESSION_DETAIL_COMMITTEE_LINK_XPATH'
)
# Session identifier, date, address etc
tds = dom.xpath(self.xpath['SESSION_DETAIL_IDENTIFIER_TD'])
if len(tds) == 0:
logging.critical(
'Cannot find table fields using SESSION_DETAIL_IDENTIFIER_TD_XPATH'
)
raise TemplateError(
'Cannot find table fields using SESSION_DETAIL_IDENTIFIER_TD_XPATH'
)
else:
for n in range(0, len(tds)):
try:
tdcontent = tds[n].text.strip()
nextcontent = tds[n + 1].text.strip()
except:
continue
if tdcontent == 'Sitzung:':
session.identifier = nextcontent
elif tdcontent == 'Gremium:':
session.committee_name = nextcontent
elif tdcontent == 'Datum:':
datestring = nextcontent
if tds[n + 2].text == 'Zeit:':
if (n + 3) in tds and tds[n + 3].text is not None:
datestring + ' ' + tds[n + 3].text
session.date_start = datestring
elif tdcontent == 'Raum:':
session.address = " ".join(tds[n + 1].xpath('./text()'))
elif tdcontent == 'Bezeichnung:':
session.description = nextcontent
if not hasattr(session, 'identifier'):
logging.critical(
'Cannot find session identifier using XPath SESSION_DETAIL_IDENTIFIER_TD'
)
raise TemplateError(
'Cannot find session identifier using XPath SESSION_DETAIL_IDENTIFIER_TD'
)
# Agendaitems
found_attachments = []
rows = dom.xpath(self.xpath['SESSION_DETAIL_AGENDA_ROWS'])
if len(rows) == 0:
logging.critical(
'Cannot find agenda using XPath SESSION_DETAIL_AGENDA_ROWS')
raise TemplateError(
'Cannot find agenda using XPath SESSION_DETAIL_AGENDA_ROWS')
else:
agendaitems = {}
agendaitem_id = None
public = True
for row in rows:
row_id = row.get('id')
row_classes = row.get('class').split(' ')
fields = row.xpath('td')
number = fields[0].xpath('./text()')
if len(number) > 0:
number = number[0]
if number == []:
number = None
#print "number: %s" % number
if row_id is not None:
# Agendaitem main row
agendaitem_id = row_id.rsplit('_', 1)[1]
agendaitems[agendaitem_id] = {}
agendaitems[agendaitem_id]['id'] = int(agendaitem_id)
if number is not None:
agendaitems[agendaitem_id]['number'] = number
agendaitems[agendaitem_id]['subject'] = "; ".join(
fields[1].xpath('./text()'))
agendaitems[agendaitem_id]['public'] = public
# submission links
links = row.xpath(
self.
xpath['SESSION_DETAIL_AGENDA_ROWS_SUBMISSION_LINK'])
submissions = []
for link in links:
href = link.get('href')
if href is None:
continue
parsed = parse.search(
self.urls['SUBMISSION_DETAIL_PARSE_PATTERN'], href)
if parsed is not None:
submission = Submission(numeric_id=int(
parsed['submission_id']),
identifier=link.text)
submissions.append(submission)
# Add submission to submission queue
if hasattr(self, 'submission_queue'):
self.submission_queue.add(
int(parsed['submission_id']))
if len(submissions):
agendaitems[agendaitem_id]['submissions'] = submissions
"""
Note: we don't scrape agendaitem-related attachments for now,
based on the assumption that they are all found via submission
detail pages. All we do here is get a list of attachment IDs
in found_attachments
"""
#attachments = []
forms = row.xpath('.//form')
for form in forms:
for hidden_field in form.xpath('input'):
if hidden_field.get('name') != 'DT':
continue
attachment_id = hidden_field.get('value')
#attachments.append(attachment_id)
found_attachments.append(attachment_id)
#if len(attachments):
# agendaitems[agendaitem_id]['attachments'] = attachments
elif 'smc_tophz' in row_classes:
# additional (optional row for agendaitem)
label = fields[1].text
value = fields[2].text
if label is not None and value is not None:
label = label.strip()
value = value.strip()
#print (label, value)
if label in ['Ergebnis:', 'Beschluss:']:
if value in self.config.RESULT_STRINGS:
agendaitems[agendaitem_id][
'result'] = self.config.RESULT_STRINGS[
value]
else:
logging.warn(
"String '%s' not found in configured RESULT_STRINGS",
value)
if self.options.verbose:
print "WARNING: String '%s' not found in RESULT_STRINGS\n" % value
agendaitems[agendaitem_id]['result'] = value
elif label == 'Bemerkung:':
agendaitems[agendaitem_id]['result_note'] = value
elif label == 'Abstimmung:':
agendaitems[agendaitem_id]['voting'] = value
else:
logging.critical(
"Agendaitem info label '%s' is unknown", label)
raise ValueError(
'Agendaitem info label "%s" is unknown' %
label)
elif 'smcrowh' in row_classes:
# Subheading (public / nonpublic part)
if fields[
0].text is not None and "Nicht öffentlich" in fields[
0].text.encode('utf-8'):
public = False
#print json.dumps(agendaitems, indent=2)
session.agendaitems = agendaitems.values()
# session-related attachments
containers = dom.xpath(self.xpath['SESSION_DETAIL_ATTACHMENTS'])
for container in containers:
classes = container.get('class')
if classes is None:
continue
classes = classes.split(' ')
if self.xpath[
'SESSION_DETAIL_ATTACHMENTS_CONTAINER_CLASSNAME'] not in classes:
continue
attachments = []
rows = container.xpath('.//tr')
for row in rows:
forms = row.xpath('.//form')
for form in forms:
#print "Form: ", form
name = " ".join(row.xpath('./td/text()')).strip()
for hidden_field in form.xpath('input'):
if hidden_field.get('name') != 'DT':
continue
attachment_id = hidden_field.get('value')
# make sure to add only those which aren't agendaitem-related
if attachment_id not in found_attachments:
attachment = Attachment(identifier=attachment_id,
name=name)
# Traversing the whole mechanize response to submit this form
for mform in mechanize_forms:
#print "Form found: '%s'" % mform
for control in mform.controls:
if control.name == 'DT' and control.value == attachment_id:
#print "Found matching form: ", control.name, control.value
attachment = self.get_attachment_file(
attachment, mform)
attachments.append(attachment)
found_attachments.append(attachment_id)
if len(attachments):
session.attachments = attachments
oid = self.db.save_session(session)
if self.options.verbose:
logging.info("Session %d stored with _id %s", session_id, oid)
def get_session(self, session_url=None, session_id=None):
"""
Load session details for the given detail page URL or numeric ID
"""
# Read either session_id or session_url from the opposite
if session_id is not None:
session_url = self.urls['SESSION_DETAIL_PRINT_PATTERN'] % session_id
elif session_url is not None:
parsed = parse.search(self.urls['SESSION_DETAIL_PARSE_PATTERN'], session_url)
session_id = parsed['session_id']
logging.info("Getting session %d from %s", session_id, session_url)
session = Session(numeric_id=session_id)
time.sleep(self.config.WAIT_TIME)
response = self.user_agent.open(session_url)
# forms for later attachment download
mechanize_forms = mechanize.ParseResponse(response, backwards_compat=False)
# seek(0) is necessary to reset response pointer.
response.seek(0)
html = response.read()
html = html.replace(' ', ' ')
parser = etree.HTMLParser()
dom = etree.parse(StringIO(html), parser)
# check for page errors
try:
page_title = dom.xpath('//h1')[0].text
if 'Fehlermeldung' in page_title:
logging.info("Page %s cannot be accessed due to server error", session_url)
if self.options.verbose:
print "Page %s cannot be accessed due to server error" % session_url
return
if 'Berechtigungsfehler' in page_title:
logging.info("Page %s cannot be accessed due to permissions", session_url)
if self.options.verbose:
print "Page %s cannot be accessed due to permissions" % session_url
return
except:
pass
try:
error_h3 = dom.xpath('//h3[@class="smc_h3"]')[0].text.strip()
if 'Keine Daten gefunden' in error_h3:
logging.info("Page %s does not contain any agenda items", session_url)
if self.options.verbose:
print "Page %s does not contain agenda items" % session_url
return
except:
pass
session.original_url = session_url
# Session title
try:
session.title = dom.xpath(self.xpath['SESSION_DETAIL_TITLE'])[0].text
except:
logging.critical('Cannot find session title element using XPath SESSION_DETAIL_TITLE')
raise TemplateError('Cannot find session title element using XPath SESSION_DETAIL_TITLE')
# Committe link
try:
links = dom.xpath(self.xpath['SESSION_DETAIL_COMMITTEE_LINK'])
for link in links:
href = link.get('href')
parsed = parse.search(self.urls['COMMITTEE_DETAIL_PARSE_PATTERN'], href)
if parsed is not None:
session.committee_id = parsed['committee_id']
except:
logging.critical('Cannot find link to committee detail page using SESSION_DETAIL_COMMITTEE_LINK_XPATH')
raise TemplateError('Cannot find link to committee detail page using SESSION_DETAIL_COMMITTEE_LINK_XPATH')
# Session identifier, date, address etc
tds = dom.xpath(self.xpath['SESSION_DETAIL_IDENTIFIER_TD'])
if len(tds) == 0:
logging.critical('Cannot find table fields using SESSION_DETAIL_IDENTIFIER_TD_XPATH')
raise TemplateError('Cannot find table fields using SESSION_DETAIL_IDENTIFIER_TD_XPATH')
else:
for n in range(0, len(tds)):
try:
tdcontent = tds[n].text.strip()
nextcontent = tds[n + 1].text.strip()
except:
continue
if tdcontent == 'Sitzung:':
session.identifier = nextcontent
elif tdcontent == 'Gremium:':
session.committee_name = nextcontent
elif tdcontent == 'Datum:':
datestring = nextcontent
if tds[n + 2].text == 'Zeit:':
if (n + 3) in tds and tds[n + 3].text is not None:
datestring + ' ' + tds[n + 3].text
session.date_start = datestring
elif tdcontent == 'Raum:':
session.address = " ".join(tds[n + 1].xpath('./text()'))
elif tdcontent == 'Bezeichnung:':
session.description = nextcontent
if not hasattr(session, 'identifier'):
logging.critical('Cannot find session identifier using XPath SESSION_DETAIL_IDENTIFIER_TD')
raise TemplateError('Cannot find session identifier using XPath SESSION_DETAIL_IDENTIFIER_TD')
# Agendaitems
found_attachments = []
rows = dom.xpath(self.xpath['SESSION_DETAIL_AGENDA_ROWS'])
if len(rows) == 0:
logging.critical('Cannot find agenda using XPath SESSION_DETAIL_AGENDA_ROWS')
raise TemplateError('Cannot find agenda using XPath SESSION_DETAIL_AGENDA_ROWS')
else:
agendaitems = {}
agendaitem_id = None
public = True
for row in rows:
row_id = row.get('id')
row_classes = row.get('class').split(' ')
fields = row.xpath('td')
number = fields[0].xpath('./text()')
if len(number) > 0:
number = number[0]
if number == []:
number = None
#print "number: %s" % number
if row_id is not None:
# Agendaitem main row
agendaitem_id = row_id.rsplit('_', 1)[1]
agendaitems[agendaitem_id] = {}
agendaitems[agendaitem_id]['id'] = int(agendaitem_id)
if number is not None:
agendaitems[agendaitem_id]['number'] = number
agendaitems[agendaitem_id]['subject'] = "; ".join(fields[1].xpath('./text()'))
agendaitems[agendaitem_id]['public'] = public
# submission links
links = row.xpath(self.xpath['SESSION_DETAIL_AGENDA_ROWS_SUBMISSION_LINK'])
submissions = []
for link in links:
href = link.get('href')
if href is None:
continue
parsed = parse.search(self.urls['SUBMISSION_DETAIL_PARSE_PATTERN'], href)
if parsed is not None:
submission = Submission(numeric_id=int(parsed['submission_id']),
identifier=link.text)
submissions.append(submission)
# Add submission to submission queue
if hasattr(self, 'submission_queue'):
self.submission_queue.add(int(parsed['submission_id']))
if len(submissions):
agendaitems[agendaitem_id]['submissions'] = submissions
"""
Note: we don't scrape agendaitem-related attachments for now,
based on the assumption that they are all found via submission
detail pages. All we do here is get a list of attachment IDs
in found_attachments
"""
#attachments = []
forms = row.xpath('.//form')
for form in forms:
for hidden_field in form.xpath('input'):
if hidden_field.get('name') != 'DT':
continue
attachment_id = hidden_field.get('value')
#attachments.append(attachment_id)
found_attachments.append(attachment_id)
#if len(attachments):
# agendaitems[agendaitem_id]['attachments'] = attachments
elif 'smc_tophz' in row_classes:
# additional (optional row for agendaitem)
label = fields[1].text
value = fields[2].text
if label is not None and value is not None:
label = label.strip()
value = value.strip()
#print (label, value)
if label in ['Ergebnis:', 'Beschluss:']:
if value in self.config.RESULT_STRINGS:
agendaitems[agendaitem_id]['result'] = self.config.RESULT_STRINGS[value]
else:
logging.warn("String '%s' not found in configured RESULT_STRINGS", value)
if self.options.verbose:
print "WARNING: String '%s' not found in RESULT_STRINGS\n" % value
agendaitems[agendaitem_id]['result'] = value
elif label == 'Bemerkung:':
agendaitems[agendaitem_id]['result_note'] = value
elif label == 'Abstimmung:':
agendaitems[agendaitem_id]['voting'] = value
else:
logging.critical("Agendaitem info label '%s' is unknown", label)
raise ValueError('Agendaitem info label "%s" is unknown' % label)
elif 'smcrowh' in row_classes:
# Subheading (public / nonpublic part)
if fields[0].text is not None and "Nicht öffentlich" in fields[0].text.encode('utf-8'):
public = False
#print json.dumps(agendaitems, indent=2)
session.agendaitems = agendaitems.values()
# session-related attachments
containers = dom.xpath(self.xpath['SESSION_DETAIL_ATTACHMENTS'])
for container in containers:
classes = container.get('class')
if classes is None:
continue
classes = classes.split(' ')
if self.xpath['SESSION_DETAIL_ATTACHMENTS_CONTAINER_CLASSNAME'] not in classes:
continue
attachments = []
rows = container.xpath('.//tr')
for row in rows:
forms = row.xpath('.//form')
for form in forms:
#print "Form: ", form
name = " ".join(row.xpath('./td/text()')).strip()
for hidden_field in form.xpath('input'):
if hidden_field.get('name') != 'DT':
continue
attachment_id = hidden_field.get('value')
# make sure to add only those which aren't agendaitem-related
if attachment_id not in found_attachments:
attachment = Attachment(
identifier=attachment_id,
name=name
)
# Traversing the whole mechanize response to submit this form
for mform in mechanize_forms:
#print "Form found: '%s'" % mform
for control in mform.controls:
if control.name == 'DT' and control.value == attachment_id:
#print "Found matching form: ", control.name, control.value
attachment = self.get_attachment_file(attachment, mform)
attachments.append(attachment)
found_attachments.append(attachment_id)
if len(attachments):
session.attachments = attachments
oid = self.db.save_session(session)
if self.options.verbose:
logging.info("Session %d stored with _id %s", session_id, oid)
player_changes = Player()
player_changes.load_from_db("*****@*****.**")
player_changes.nickname = "Lesha"
player_changes.save_to_db()
print("{}Player was updated in db.\n".format(player_changes))
player_changes.delete_from_db()
print("{}Player was deleted from db.\n".format(player_changes))
########################################################################################################################
player_session = Session(player_id=1, finish_time=datetime.datetime(2017, 2, 14, 00, 00),
elapse_time=datetime.datetime(1, 1, 1),
created=datetime.datetime.now(), updated=datetime.datetime.now())
print("{}Information about player session.\n".format(player_session))
player_session.save_to_db()
print("{}Player session was inserted to db.\n".format(player_session))
player_session_info = Session()
player_session_info.load_from_db(1)
player_session_change = Session()
player_session_change.load_from_db(1)
player_session_change.finish_time = datetime.datetime(2016, 2, 14, 00, 00)
player_session_change.elapse_time = datetime.datetime(1,1,2)
def main():
parser = argparse.ArgumentParser(description="Freedom Fighting Mode.")
parser.add_argument("--debug-input",
action="store_true",
help="Toggle debugging of the user input.")
parser.add_argument("--debug-output",
action="store_true",
help="Toggle debugging of the terminal output.")
parser.add_argument("--config",
"-c",
help="The harness' configuration file.",
default=os.path.join(os.path.dirname(__file__),
"ffm.conf"))
parser.add_argument("--stdout", help="Redirect stdout to the target file.")
args = parser.parse_args()
context.debug_input = args.debug_input
context.debug_output = args.debug_output
context.stdout = open(args.stdout,
"wb") if args.stdout is not None else sys.stdout
# Print the banner
print(random.choice(BANNERS) + "\n")
print(
"FFM enabled. Type !list to see available commands and exit to quit.")
# Check that the configuration file exists and is sane.
if not os.path.exists(args.config):
print(
"Could not find %s. Please provide it with the --config option." %
args.config)
return
context.config = configparser.ConfigParser(allow_no_value=True,
inline_comment_prefixes=("#",
";"))
context.config.read(args.config)
context.terminal_driver = DefaultInputDriver()
stdin_fd = sys.stdin.fileno()
old_settings = termios.tcgetattr(stdin_fd)
old_handler = signal.signal(signal.SIGWINCH, update_window_size)
tty.setraw(context.stdin)
context.active_session = Session()
context.sessions.append(context.active_session)
update_window_size() # Set the correct window size in the PTY.
try:
while context.active_session and context.active_session.bash.poll(
) is None:
try:
r, w, e = select.select(
[sys.stdin, context.active_session.master], [], [], 1)
if sys.stdin in r:
typed_char = os.read(sys.stdin.fileno(), 1)
try:
context.active_session.input_driver.handle_input(
typed_char)
except RuntimeError as e:
os.write(context.stdout.fileno(),
b"\r\n%s\r\n" % str(e).encode("UTF-8"))
elif context.active_session.master in r:
read = os.read(context.active_session.master, 2048)
if context.debug_output:
for c in read:
os.write(context.stdout.fileno(),
("%02X " % c).encode("UTF-8"))
# Store the last line for future use
last = read.split(b"\n")[-1]
# Debian terminals update the window title with this escape sequence. Ignore it.
last = re.sub(b"\x1b]0;.*?\x07", b"", last)
# Kali terminals add color to the prompt. Strip it.
last = re.sub(b"\x1b\[[0-?]*[ -/]*[@-~]", b"", last)
if re.match(PROMPT_REGEXP,
last.decode("UTF-8", errors='ignore'),
re.UNICODE):
# TODO: keep the colors in the saved prompt. This will require all
# references of len(last_line) to be updated to ignore escape sequences.
context.active_session.input_driver.last_line = last.decode(
"UTF-8")
else:
context.active_session.input_driver.last_line = ''
# Pass the output to the output driver for display after applying output processors.
(proceed,
output) = apply_processors(read, OUTPUT_PROCESSOR_LIST)
if proceed:
context.active_session.output_driver.handle_bytes(
output)
except select.error as e:
if "[Errno 4]" in str(
e
): # Interrupted system call. May be raised if SIGWINCH is received.
continue
else:
raise
# Pretty printing for unimplemented opcodes: no need for a full trace. Probably remove that in the future.
except RuntimeError as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
filename = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
print("\r\n%s (%s, line %d)\r" %
(str(e), filename, exc_tb.tb_lineno))
return
# Bash has finished running
print("FFM disabled.\r")
finally:
termios.tcsetattr(stdin_fd, termios.TCSADRAIN, old_settings)
signal.signal(signal.SIGWINCH, old_handler)
def main():
parser = argparse.ArgumentParser(description="Freedom Fighting Mode.")
parser.add_argument("--debug-input", action="store_true", help="Toggle debugging of the user input.")
parser.add_argument("--debug-output", action="store_true", help="Toggle debugging of the terminal output.")
parser.add_argument("--stdout", help="Redirect stdout to the target file.")
args = parser.parse_args()
context.debug_input = args.debug_input
context.debug_output = args.debug_output
context.stdout = open(args.stdout, "wb") if args.stdout is not None else sys.stdout
# Print the banner
print(random.choice(BANNERS) + "\n")
print("FFM enabled. Type !list to see available commands and exit to quit.")
context.terminal_driver = DefaultInputDriver()
stdin_fd = sys.stdin.fileno()
old_settings = termios.tcgetattr(stdin_fd)
old_handler = signal.signal(signal.SIGWINCH, update_window_size)
tty.setraw(context.stdin)
context.active_session = Session()
context.sessions.append(context.active_session)
update_window_size() # Set the correct window size in the PTY.
try:
while context.active_session and context.active_session.bash.poll() is None:
try:
r, w, e = select.select([sys.stdin, context.active_session.master], [], [], 1)
if sys.stdin in r:
typed_char = os.read(sys.stdin.fileno(), 1)
try:
context.active_session.input_driver.handle_input(typed_char)
except RuntimeError as e:
os.write(context.stdout.fileno(), b"\r\n%s\r\n" % str(e).encode("UTF-8"))
elif context.active_session.master in r:
read = os.read(context.active_session.master, 2048)
if context.debug_output:
for c in read:
os.write(context.stdout.fileno(), ("%02X " % c).encode("UTF-8"))
# Store the last line for future use
# Only work on the last line
last = read.split(b"\n")[-1]
if len(last) < 150 and b"\x07" in last: # TODO: bug when cat-ing a binary file!
# This is motivated by my Debian shell's prompt containing weird \x07 bytes separating
# two prompt occurrences.
context.active_session.input_driver.last_line = last.split(b"\x07")[-1].decode("UTF-8", errors='ignore')
elif re.match(PROMPT_REGEXP, last.decode("UTF-8", errors='ignore'), re.UNICODE):
context.active_session.input_driver.last_line = last.decode("UTF-8")
else:
context.active_session.input_driver.last_line = ''
# Pass the output to the output driver for display after applying output processors.
(proceed, output) = apply_processors(read, OUTPUT_PROCESSOR_LIST)
if proceed:
context.active_session.output_driver.handle_bytes(output)
except select.error as e:
if "[Errno 4]" in str(e): # Interrupted system call. May be raised if SIGWINCH is received.
continue
else:
raise
# Pretty printing for unimplemented opcodes: no need for a full trace. Probably remove that in the future.
except RuntimeError as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
filename = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
print("\r\n%s (%s, line %d)\r" % (str(e), filename, exc_tb.tb_lineno))
return
# Bash has finished running
print("FFM disabled.\r")
finally:
termios.tcsetattr(stdin_fd, termios.TCSADRAIN, old_settings)
signal.signal(signal.SIGWINCH, old_handler)
def config_injector(binder):
binder.bind(Config, Config('server-config.cfg'))
binder.bind(Session, Session())
# import logging
from controller.logincontroller import LoginController
from controller.downloadcontroller import NalandaDownloaderController
from model.googleoauth import GoogleSession
from model.session import Session
from model.nalanda import NalandaSession
from view.authenticationwindow import AuthenticationWindow
from view.nalandadownloader import NalandaDownloader
# logging.basicConfig(format='%(asctime)s:%(name)s:LINE %(lineno)d:%(levelname)s:%(message)s', level=logging.DEBUG)
if __name__ == '__main__':
session = Session()
nalanda_session = NalandaSession(session=session)
google_session = GoogleSession(session=session,
nalanda_session=nalanda_session)
login_window = AuthenticationWindow()
login_controller = LoginController(login_window=login_window,
google_session=google_session)
nalanda_downloader = NalandaDownloader()
nalanda_controller = NalandaDownloaderController(
nalanda_downloader=nalanda_downloader, nalanda_session=nalanda_session)
def get(self):
cutoff = datetime.utcnow() - MAX_AGE
sessions = Session.query(Session.created_at < cutoff).fetch(200, keys_only=True)
if len(sessions) > 0:
ndb.delete_multi(sessions)
logging.info('%s sessions cleared' % len(sessions))
