def disable(): users = User.objects(Q(timestamp__lt=int(time()))) for user in list(users): Session.objects(data__login=user.login).update( **dict(data__rate=0, data__timestamp=2147483647)) users.update(**dict(rate=0, timestamp=2147483647)) return dumps(dict(code=200))
def login(username, master_password): """ Posts username and authentication hash to /login. Decrypts vault on successful login. Returns new Session. """ print('Logging in...') url = api_base_url + '/login' auth_hash = get_auth_hash(username, master_password) body = { 'username': username, 'auth_hash': base64.b64encode(auth_hash).decode(), } response = requests.post(url, data=json.dumps(body)) if response.status_code != requests.codes.ok: print('Login attempt failed with status code: ' + str(response.status_code)) exit() response_body = response.json() cipher_text = base64.b64decode(response_body['vault'].encode()) key = get_key(username, master_password) vault_str = decrypt_vault(cipher_text, key) vault_dict = json.loads(vault_str) session = Session(username, auth_hash, key) session.dict_to_vault(vault_dict) return session
def insert_tasks(PL, task_file): pipeline = Pipeline(PL.name, PL.log_dir) logging.debug("Pipeline is: {}".format(pipeline)) task_list = PL.prepare_managed_tasks() logging.debug("Task list is: {}".format([x['name'] for x in task_list])) # we need to be able to translate the dependencies as stored in the task # list (list of other task names that a particular task depends on) # into a list of Job object references that have already been added to the # session. We will build up a dictionary of task['name'] : Job as we # insert them deps_to_job = {} print(" Inserting tasks into {}".format(task_file)) logging.info("Inserting tasks into {}".format(task_file)) try: for task in task_list: print(" -> {}".format(task['name'])) try: dependencies = [deps_to_job[d] for d in task['dependencies']] except KeyError as e: logging.exception("Key error processing dependencies") msg = "Task {} depends on a task that hasn't been been " \ "processed ({}). Check your Pipeline XML".format( task['name'], e.args[0]) raise Exception(msg) job = Job(pipeline, task['name'], task['threads'], task['stdout_path'], task['stderr_path'], task['script_path'], task['epilogue_path'], task['mem'], task['email_list'], task['mail_options'], task['batch_env'], dependencies, task['queue'], task['walltime']) deps_to_job[task['name']] = job logging.debug("Adding job {} (log dir: {}) to session".format( job.job_name, job.pipeline.log_directory)) Session.add(job) except Exception as e: logging.exception("Error inserting tasks into database") print("Error inserting tasks into database: {}".format(e), file=sys.stderr) sys.exit(6) # only commit the session if we were able to add all the jobs to the session # without catching an Exception Session.commit() logging.info(" {} tasks have been inserted into task file {}; " "(log dir: {})".format(len(task_list), task_file, PL.log_dir)) return len(task_list)
def get(self): cutoff = datetime.utcnow() - MAX_AGE sessions = Session.query(Session.created_at < cutoff).fetch( 200, keys_only=True) if len(sessions) > 0: ndb.delete_multi(sessions) logging.info('%s sessions cleared' % len(sessions))
def run(): systemLog = Log() threads = [] sessions = {} proxy = Proxy().random_proxy() while 1: payments = Payment().get_payments() for p in payments: payment = Payment() payment = payment.set_payment(p) name = payment.get_name() type = payment.get_type() if payment.get_status() == 0: continue connector_module = 'connector.http.' + name.lower() if type == 'enterprise': connector_module += '_enterprise' name += 'Enterprise' module = importlib.import_module(connector_module) class_ = getattr(module, name) sessions[name] = Session(proxy) connector = class_(payment, sessions[name], proxy) if sessions[name].is_changing_proxy() == 1: systemLog.log('Need to change ' + name + ' proxy!', 'debug') proxy = Proxy().random_proxy() thread_name = "Thread " + str( payment.get_id()) + " " + name + " " + type thread = ThreadConnector(thread_name.upper(), connector, 5) thread.start() threads.append(thread) for t in threads: t.join(60)
def wrapper(*args, **kwargs): """Wrap the route.""" database = kwargs["database"] session_id = str(bottle.request.get_cookie("session_id")) session = Session(sessions.find_session(database, session_id)) if not session.is_valid(): cls.abort( 401, "%s-access to %s denied: session %s not authenticated", context, session_id) authorized_users = latest_reports_overview(database).get("editors") if not session.is_authorized(authorized_users): cls.abort(403, "%s-access to %s denied: session %s not authorized", context, session_id) return callback(*args, **kwargs)
def signup(username, master_password): """ Posts username and authentication hash to /signup. Calls update vault with empty vault on successful signup. Returns new Session. """ print('Signing up...') url = api_base_url + '/signup' auth_hash = get_auth_hash(username, master_password) body = { 'username': username, 'auth_hash': base64.b64encode(auth_hash).decode(), } response = requests.post(url, data=json.dumps(body)) if response.status_code != requests.codes.ok: print('Signup attempt failed with status code: ' + str(response.status_code)) exit() key = get_key(username, master_password) vault = [] session = Session(username, auth_hash, key, vault) update_vault(session) return session
def put(self): """ User Log in """ # Validation if not request.is_json: return {"result": -1, "msg": "Missing JSON in request"}, 400 id = request.json.get('id', None) password = request.json.get('password', None) if not id: return {"result": -1, "msg": "Missing id parameter"}, 400 if not password: return {"result": -1, "msg": "Missing password parameter"}, 400 user = User.query.filter_by(id=id).first() if not user or user.signout_dt: return {"result": -1, "msg": "User not found"}, 400 if not bcrypt.checkpw(password.encode('utf-8'), user.password.encode('utf-8')): return {"result": -1, "msg": "Bad password"}, 400 # Make session info session_id = str(uuid.uuid4()).replace('-', '')[:20] client_ip = request.environ.get('HTTP_X_REAL_IP', request.remote_addr) # get Client IP session = Session(session_id=session_id, id=id, client_ip=client_ip) db.session.add(session) db.session.commit() return {"result": 0, "session": session_id}, 200
def update_existing_session(self, conn, session_id, tag): cherrypy.log('Hello from update_existing_session', severity=logging.DEBUG) temp = execute_fetchone( conn, FetchController.update_existing_session_query, { 'tag': tag, 'ses_id': session_id }) conn.commit() cherrypy.log(pformat(temp), severity=logging.DEBUG) return Session.from_db_row(temp['result'])
def wrapper(*args, **kwargs): token = request.headers.get('Authorization') if token: user_session = Session.find_one_by_text_token(token) if user_session: if provide_user: kwargs['user'] = user_session.user if provide_session: kwargs['session'] = user_session return func(*args, **kwargs) raise AccessDenied
def login(self, username, password): user = self.users_dao.find_by_username(username) if user is None: raise Exception('El usuario y/o contraseña son incorrectos, por favor intenta de nuevo.') crypto = Crypto() if not crypto.verify_encrypted_password(password, user.password): raise Exception('El usuario y/o contraseña son incorrectos, por favor intenta de nuevo.') session_uuid = str(uuid.uuid4()) session = Session(username, session_uuid) self.session_dao.save_session(session) return session_uuid
def read_session(path): group_id = "" info = [] session_array = [] wb = xlrd.open_workbook(path) sheet = wb.sheet_by_index(0) sheet.cell_value(0, 0) data = [sheet.row_values(row_num) for row_num in range(sheet.nrows)] for item in data: new_item = excel_tools.remove_empty_element_in_array(item) if new_item: info.append(new_item) for item in info: index = info.index(item) if index == 2: if len(item) > 0: group_id = excel_tools.format_group_id(item[0]) if index > 3: if len(item) >= 5: audience = excel_tools.format_audience_name( item[5]) if item[5] else "-" teacher_name = excel_tools.remove_repetition_in_str(item[4]) else: audience = "-" teacher_name = "-" date = excel_tools.format_date_from_excel(wb, item[0]) time_value = item[1] if type(time_value) is not str: time = excel_tools.format_time(item[1]) else: time = time_value name = excel_tools.remove_repetition_in_str(item[3]) session_item = Session(group_id=group_id, date=date, time=time, type=item[2], name=name, teacher_name=teacher_name, audience=audience) if session_item: session_array.append(session_item) if session_array: return session_array else: return None
def update_session_tag(self, conn, session_type, session_key, tag): cherrypy.log('Hello from update_session_tag', severity=logging.DEBUG) temp = execute_fetchone( conn, UpdateTagController.update_session_tag_query, { 'tag': tag, 'ses_type': session_type, 'ses_key': session_key }) conn.commit() cherrypy.log(pformat(temp), severity=logging.DEBUG) return Session.from_db_row(temp['result'])
def get_verified_data(jws, expected=None, session_token=None): headers = json_parse(get_jws_part(jws, 0)) raw_username = headers['kikUsr'] username = raw_username.lower() hostname = headers['kikCrdDm'].split('/')[0].lower() payload = get_jws_part(jws, 1) if expected is not None and payload != expected: logging.info('jws, payload does not match expected value') raise Exception('payload does not match expected value') try: data = json_parse(payload) except: data = None try: session = ndb.Key(urlsafe=session_token).get() except Exception as e: session = None if session is None or not isinstance( session, Session ) or session.username != username or session.hostname != hostname: session = None session_token = None if username not in TEST_USERS: verify_jws(jws, raw_username, hostname, (headers.get('kikDbg') and DEBUG)) elif not DEBUG: logging.info('jws, chrome user detected') raise Exception('chrome user detected') try: session = Session(username=username, hostname=hostname) session.put() session_token = session.key.urlsafe() except: pass return username, hostname, data, session_token
def activate_user(admin, username, timestamp): user = User.objects(login=username).first() if user.rate: user.timestamp += int(timestamp) else: user.timestamp = int(time()) + int(timestamp) user.rate = 1 user.save() Session.objects(data__login=user.login).update( **dict(data__timestamp=user.timestamp, data__rate=user.rate)) if app.config['SEND_ACTIVATE']: unit = get_unit_time(timestamp) send( app.config['SMTP_LOGIN'], 'Активация профиля на {0}'.format(app.config['TITLE']), str( render_template('mail/activate.html', admin=admin, username=username, unit=unit['key'], time=unit['value'], date=datetime.fromtimestamp(int( time())).strftime('%Y-%m-%d %H:%M:%S'))))
class MainApp(App): session = Session() admin_user = User("admin", "12345") admin_role = Administrator() admin_assign = Assignment(admin_user, admin_role) rootpath = "/home/gagos/unb/git/das/das_ep0/objects/organization" rootdir = Directory(rootpath, None, admin_role) admin_privilege = Privilege(admin_role, rootdir) admin_privilege.add_command('OpenObject') rootdir.update_directory() def build(self): return ScreenManagement()
def get_verified_data(jws, expected=None, session_token=None): headers = json_parse(get_jws_part(jws, 0)) raw_username = headers['kikUsr'] username = raw_username.lower() hostname = headers['kikCrdDm'].split('/')[0].lower() payload = get_jws_part(jws, 1) if expected is not None and payload != expected: logging.info('jws, payload does not match expected value') raise Exception('payload does not match expected value') try: data = json_parse(payload) except: data = None try: session = ndb.Key(urlsafe=session_token).get() except Exception: session = None if session is None or not isinstance(session, Session) \ or session.username != username or session.hostname != hostname: session_token = None if username not in TEST_USERS: verify_jws(jws, raw_username, hostname, (headers.get('kikDbg') and DEBUG)) elif not DEBUG: logging.info('jws, chrome user detected') raise Exception('chrome user detected') try: session = Session(username=username, hostname=hostname) session.put() session_token = session.key.urlsafe() except: pass return username, hostname, data, session_token
def recreate_session(self, conn, session_id, session_key, session_type, tag): cherrypy.log('Hello from recreate_session', severity=logging.DEBUG) session_string = self.make_session_string() execute_only(conn, FetchController.delete_session_query, { 'ses_id': session_id }) temp = execute_fetchone( conn, FetchController.new_session_query, { 'ses_key': session_key, 'ses_str': session_string, 'ses_type': session_type, 'tag': tag }) conn.commit() cherrypy.log(pformat(temp), severity=logging.DEBUG) return Session.from_db_row(temp['result'])
def get(self): try: client_id = self.request.GET['client_id'] except: self.redirect("/error?msg=missing client_id for new session") return try: client = ndb.Key(urlsafe=client_id).get() except: self.redirect("/error?msg=client was not found") return session = Session() session.dni = client.dni session.subject = "Revisión" session.comments = "" session.proposal = "" session.put() self.redirect("/modifySession?session_id=" + session.key.urlsafe() + "&client_id=" + client_id)
def data_to_list_class(data, to_class): list_answer = [] for el in data: if to_class == "user": list_answer.append(User(data=el)) elif to_class == "lesson": list_answer.append(Lesson(data=el)) elif to_class == "teacher": list_answer.append(Teacher(data=el)) elif to_class == "event": list_answer.append(Event(data=el)) elif to_class == "faculty": list_answer.append(Faculty(data=el)) elif to_class == "session": list_answer.append(Session(data=el)) elif to_class == "qualification": list_answer.append(Qualification(data=el)) return list_answer
def get(self): user = users.get_current_user() try: client_id = self.request.GET['client_id'] except: self.redirect( "/error?msg=missing client_id for sessions management") return if user: if User.chk_allowed(user.email()): user_name = user.nickname() access_link = users.create_logout_url("/") try: client = ndb.Key(urlsafe=client_id).get() except: self.redirect( "/error?msg=client key was not found for sessions management" ) return sessions = Session.query( Session.dni == client.dni).order(-Session.added) template_values = { "user_name": user_name, "access_link": access_link, "client": client, "sessions": sessions, "info": AppInfo, } jinja = jinja2.get_jinja2(app=self.app) self.response.write( jinja.render_template("sessionsManagement.html", **template_values)) else: self.redirect("/nouser") else: self.redirect("/")
def __init__(self, state_directory): super().__init__() # db's up and going self.state_db = SqlCache(state_directory, 'state', init_state) self.domains_db = SqlCache(state_directory, 'domains', init_domains) self.descriptions_db = SqlCache(state_directory, 'descriptions', init_descriptions) # initialise sessions, containers and ip allocations binary_sessions = self.state_db.query("SELECT rid,cbor FROM sessions") self.sessions = {rid: Session.from_binary(rid, binary) for rid, binary in binary_sessions} self.containers = TaggedCollection() self.allocations = set() for session in self.sessions.values(): for uuid, container in session.dependent_containers.items(): self.containers.add(container) self.allocations.add(container.ip) # fetch the forwarding table forwarding = self.state_db.query("SELECT key,value FROM forwarding") for forward in forwarding: self.long_term_forwards[forward[0]] = forward[1] # volumes self.volumes = Volume.all() # domain ownership self.domains = {} self.global_domains = {} domains = self.domains_db.query("SELECT domain,token,attempted,user,global FROM domains") for domain, token, attempted, user, gbl in domains: dom_obj = Domain(domain, token, user, attempted, gbl) if user not in self.domains: self.domains[user] = {} # map domain name to domain object self.domains[user][domain] = dom_obj if gbl: self.global_domains[domain] = dom_obj # descriptions descriptions = self.descriptions_db.query("SELECT full_id,cbor FROM descriptions") self.descriptions = {d[0]: cbor.loads(d[1]) for d in descriptions}
def get_session(self, session_url=None, session_id=None): """ Load session details for the given detail page URL or numeric ID """ # Read either session_id or session_url from the opposite if session_id is not None: session_url = self.urls['SESSION_DETAIL_PRINT_PATTERN'] % session_id elif session_url is not None: parsed = parse.search(self.urls['SESSION_DETAIL_PARSE_PATTERN'], session_url) session_id = parsed['session_id'] logging.info("Getting session %d from %s", session_id, session_url) session = Session(numeric_id=session_id) time.sleep(self.config.WAIT_TIME) response = self.user_agent.open(session_url) # forms for later attachment download mechanize_forms = mechanize.ParseResponse(response, backwards_compat=False) # seek(0) is necessary to reset response pointer. response.seek(0) html = response.read() html = html.replace(' ', ' ') parser = etree.HTMLParser() dom = etree.parse(StringIO(html), parser) # check for page errors try: page_title = dom.xpath('//h1')[0].text if 'Fehlermeldung' in page_title: logging.info("Page %s cannot be accessed due to server error", session_url) if self.options.verbose: print "Page %s cannot be accessed due to server error" % session_url return if 'Berechtigungsfehler' in page_title: logging.info("Page %s cannot be accessed due to permissions", session_url) if self.options.verbose: print "Page %s cannot be accessed due to permissions" % session_url return except: pass try: error_h3 = dom.xpath('//h3[@class="smc_h3"]')[0].text.strip() if 'Keine Daten gefunden' in error_h3: logging.info("Page %s does not contain any agenda items", session_url) if self.options.verbose: print "Page %s does not contain agenda items" % session_url return except: pass session.original_url = session_url # Session title try: session.title = dom.xpath( self.xpath['SESSION_DETAIL_TITLE'])[0].text except: logging.critical( 'Cannot find session title element using XPath SESSION_DETAIL_TITLE' ) raise TemplateError( 'Cannot find session title element using XPath SESSION_DETAIL_TITLE' ) # Committe link try: links = dom.xpath(self.xpath['SESSION_DETAIL_COMMITTEE_LINK']) for link in links: href = link.get('href') parsed = parse.search( self.urls['COMMITTEE_DETAIL_PARSE_PATTERN'], href) if parsed is not None: session.committee_id = parsed['committee_id'] except: logging.critical( 'Cannot find link to committee detail page using SESSION_DETAIL_COMMITTEE_LINK_XPATH' ) raise TemplateError( 'Cannot find link to committee detail page using SESSION_DETAIL_COMMITTEE_LINK_XPATH' ) # Session identifier, date, address etc tds = dom.xpath(self.xpath['SESSION_DETAIL_IDENTIFIER_TD']) if len(tds) == 0: logging.critical( 'Cannot find table fields using SESSION_DETAIL_IDENTIFIER_TD_XPATH' ) raise TemplateError( 'Cannot find table fields using SESSION_DETAIL_IDENTIFIER_TD_XPATH' ) else: for n in range(0, len(tds)): try: tdcontent = tds[n].text.strip() nextcontent = tds[n + 1].text.strip() except: continue if tdcontent == 'Sitzung:': session.identifier = nextcontent elif tdcontent == 'Gremium:': session.committee_name = nextcontent elif tdcontent == 'Datum:': datestring = nextcontent if tds[n + 2].text == 'Zeit:': if (n + 3) in tds and tds[n + 3].text is not None: datestring + ' ' + tds[n + 3].text session.date_start = datestring elif tdcontent == 'Raum:': session.address = " ".join(tds[n + 1].xpath('./text()')) elif tdcontent == 'Bezeichnung:': session.description = nextcontent if not hasattr(session, 'identifier'): logging.critical( 'Cannot find session identifier using XPath SESSION_DETAIL_IDENTIFIER_TD' ) raise TemplateError( 'Cannot find session identifier using XPath SESSION_DETAIL_IDENTIFIER_TD' ) # Agendaitems found_attachments = [] rows = dom.xpath(self.xpath['SESSION_DETAIL_AGENDA_ROWS']) if len(rows) == 0: logging.critical( 'Cannot find agenda using XPath SESSION_DETAIL_AGENDA_ROWS') raise TemplateError( 'Cannot find agenda using XPath SESSION_DETAIL_AGENDA_ROWS') else: agendaitems = {} agendaitem_id = None public = True for row in rows: row_id = row.get('id') row_classes = row.get('class').split(' ') fields = row.xpath('td') number = fields[0].xpath('./text()') if len(number) > 0: number = number[0] if number == []: number = None #print "number: %s" % number if row_id is not None: # Agendaitem main row agendaitem_id = row_id.rsplit('_', 1)[1] agendaitems[agendaitem_id] = {} agendaitems[agendaitem_id]['id'] = int(agendaitem_id) if number is not None: agendaitems[agendaitem_id]['number'] = number agendaitems[agendaitem_id]['subject'] = "; ".join( fields[1].xpath('./text()')) agendaitems[agendaitem_id]['public'] = public # submission links links = row.xpath( self. xpath['SESSION_DETAIL_AGENDA_ROWS_SUBMISSION_LINK']) submissions = [] for link in links: href = link.get('href') if href is None: continue parsed = parse.search( self.urls['SUBMISSION_DETAIL_PARSE_PATTERN'], href) if parsed is not None: submission = Submission(numeric_id=int( parsed['submission_id']), identifier=link.text) submissions.append(submission) # Add submission to submission queue if hasattr(self, 'submission_queue'): self.submission_queue.add( int(parsed['submission_id'])) if len(submissions): agendaitems[agendaitem_id]['submissions'] = submissions """ Note: we don't scrape agendaitem-related attachments for now, based on the assumption that they are all found via submission detail pages. All we do here is get a list of attachment IDs in found_attachments """ #attachments = [] forms = row.xpath('.//form') for form in forms: for hidden_field in form.xpath('input'): if hidden_field.get('name') != 'DT': continue attachment_id = hidden_field.get('value') #attachments.append(attachment_id) found_attachments.append(attachment_id) #if len(attachments): # agendaitems[agendaitem_id]['attachments'] = attachments elif 'smc_tophz' in row_classes: # additional (optional row for agendaitem) label = fields[1].text value = fields[2].text if label is not None and value is not None: label = label.strip() value = value.strip() #print (label, value) if label in ['Ergebnis:', 'Beschluss:']: if value in self.config.RESULT_STRINGS: agendaitems[agendaitem_id][ 'result'] = self.config.RESULT_STRINGS[ value] else: logging.warn( "String '%s' not found in configured RESULT_STRINGS", value) if self.options.verbose: print "WARNING: String '%s' not found in RESULT_STRINGS\n" % value agendaitems[agendaitem_id]['result'] = value elif label == 'Bemerkung:': agendaitems[agendaitem_id]['result_note'] = value elif label == 'Abstimmung:': agendaitems[agendaitem_id]['voting'] = value else: logging.critical( "Agendaitem info label '%s' is unknown", label) raise ValueError( 'Agendaitem info label "%s" is unknown' % label) elif 'smcrowh' in row_classes: # Subheading (public / nonpublic part) if fields[ 0].text is not None and "Nicht öffentlich" in fields[ 0].text.encode('utf-8'): public = False #print json.dumps(agendaitems, indent=2) session.agendaitems = agendaitems.values() # session-related attachments containers = dom.xpath(self.xpath['SESSION_DETAIL_ATTACHMENTS']) for container in containers: classes = container.get('class') if classes is None: continue classes = classes.split(' ') if self.xpath[ 'SESSION_DETAIL_ATTACHMENTS_CONTAINER_CLASSNAME'] not in classes: continue attachments = [] rows = container.xpath('.//tr') for row in rows: forms = row.xpath('.//form') for form in forms: #print "Form: ", form name = " ".join(row.xpath('./td/text()')).strip() for hidden_field in form.xpath('input'): if hidden_field.get('name') != 'DT': continue attachment_id = hidden_field.get('value') # make sure to add only those which aren't agendaitem-related if attachment_id not in found_attachments: attachment = Attachment(identifier=attachment_id, name=name) # Traversing the whole mechanize response to submit this form for mform in mechanize_forms: #print "Form found: '%s'" % mform for control in mform.controls: if control.name == 'DT' and control.value == attachment_id: #print "Found matching form: ", control.name, control.value attachment = self.get_attachment_file( attachment, mform) attachments.append(attachment) found_attachments.append(attachment_id) if len(attachments): session.attachments = attachments oid = self.db.save_session(session) if self.options.verbose: logging.info("Session %d stored with _id %s", session_id, oid)
def get_session(self, session_url=None, session_id=None): """ Load session details for the given detail page URL or numeric ID """ # Read either session_id or session_url from the opposite if session_id is not None: session_url = self.urls['SESSION_DETAIL_PRINT_PATTERN'] % session_id elif session_url is not None: parsed = parse.search(self.urls['SESSION_DETAIL_PARSE_PATTERN'], session_url) session_id = parsed['session_id'] logging.info("Getting session %d from %s", session_id, session_url) session = Session(numeric_id=session_id) time.sleep(self.config.WAIT_TIME) response = self.user_agent.open(session_url) # forms for later attachment download mechanize_forms = mechanize.ParseResponse(response, backwards_compat=False) # seek(0) is necessary to reset response pointer. response.seek(0) html = response.read() html = html.replace(' ', ' ') parser = etree.HTMLParser() dom = etree.parse(StringIO(html), parser) # check for page errors try: page_title = dom.xpath('//h1')[0].text if 'Fehlermeldung' in page_title: logging.info("Page %s cannot be accessed due to server error", session_url) if self.options.verbose: print "Page %s cannot be accessed due to server error" % session_url return if 'Berechtigungsfehler' in page_title: logging.info("Page %s cannot be accessed due to permissions", session_url) if self.options.verbose: print "Page %s cannot be accessed due to permissions" % session_url return except: pass try: error_h3 = dom.xpath('//h3[@class="smc_h3"]')[0].text.strip() if 'Keine Daten gefunden' in error_h3: logging.info("Page %s does not contain any agenda items", session_url) if self.options.verbose: print "Page %s does not contain agenda items" % session_url return except: pass session.original_url = session_url # Session title try: session.title = dom.xpath(self.xpath['SESSION_DETAIL_TITLE'])[0].text except: logging.critical('Cannot find session title element using XPath SESSION_DETAIL_TITLE') raise TemplateError('Cannot find session title element using XPath SESSION_DETAIL_TITLE') # Committe link try: links = dom.xpath(self.xpath['SESSION_DETAIL_COMMITTEE_LINK']) for link in links: href = link.get('href') parsed = parse.search(self.urls['COMMITTEE_DETAIL_PARSE_PATTERN'], href) if parsed is not None: session.committee_id = parsed['committee_id'] except: logging.critical('Cannot find link to committee detail page using SESSION_DETAIL_COMMITTEE_LINK_XPATH') raise TemplateError('Cannot find link to committee detail page using SESSION_DETAIL_COMMITTEE_LINK_XPATH') # Session identifier, date, address etc tds = dom.xpath(self.xpath['SESSION_DETAIL_IDENTIFIER_TD']) if len(tds) == 0: logging.critical('Cannot find table fields using SESSION_DETAIL_IDENTIFIER_TD_XPATH') raise TemplateError('Cannot find table fields using SESSION_DETAIL_IDENTIFIER_TD_XPATH') else: for n in range(0, len(tds)): try: tdcontent = tds[n].text.strip() nextcontent = tds[n + 1].text.strip() except: continue if tdcontent == 'Sitzung:': session.identifier = nextcontent elif tdcontent == 'Gremium:': session.committee_name = nextcontent elif tdcontent == 'Datum:': datestring = nextcontent if tds[n + 2].text == 'Zeit:': if (n + 3) in tds and tds[n + 3].text is not None: datestring + ' ' + tds[n + 3].text session.date_start = datestring elif tdcontent == 'Raum:': session.address = " ".join(tds[n + 1].xpath('./text()')) elif tdcontent == 'Bezeichnung:': session.description = nextcontent if not hasattr(session, 'identifier'): logging.critical('Cannot find session identifier using XPath SESSION_DETAIL_IDENTIFIER_TD') raise TemplateError('Cannot find session identifier using XPath SESSION_DETAIL_IDENTIFIER_TD') # Agendaitems found_attachments = [] rows = dom.xpath(self.xpath['SESSION_DETAIL_AGENDA_ROWS']) if len(rows) == 0: logging.critical('Cannot find agenda using XPath SESSION_DETAIL_AGENDA_ROWS') raise TemplateError('Cannot find agenda using XPath SESSION_DETAIL_AGENDA_ROWS') else: agendaitems = {} agendaitem_id = None public = True for row in rows: row_id = row.get('id') row_classes = row.get('class').split(' ') fields = row.xpath('td') number = fields[0].xpath('./text()') if len(number) > 0: number = number[0] if number == []: number = None #print "number: %s" % number if row_id is not None: # Agendaitem main row agendaitem_id = row_id.rsplit('_', 1)[1] agendaitems[agendaitem_id] = {} agendaitems[agendaitem_id]['id'] = int(agendaitem_id) if number is not None: agendaitems[agendaitem_id]['number'] = number agendaitems[agendaitem_id]['subject'] = "; ".join(fields[1].xpath('./text()')) agendaitems[agendaitem_id]['public'] = public # submission links links = row.xpath(self.xpath['SESSION_DETAIL_AGENDA_ROWS_SUBMISSION_LINK']) submissions = [] for link in links: href = link.get('href') if href is None: continue parsed = parse.search(self.urls['SUBMISSION_DETAIL_PARSE_PATTERN'], href) if parsed is not None: submission = Submission(numeric_id=int(parsed['submission_id']), identifier=link.text) submissions.append(submission) # Add submission to submission queue if hasattr(self, 'submission_queue'): self.submission_queue.add(int(parsed['submission_id'])) if len(submissions): agendaitems[agendaitem_id]['submissions'] = submissions """ Note: we don't scrape agendaitem-related attachments for now, based on the assumption that they are all found via submission detail pages. All we do here is get a list of attachment IDs in found_attachments """ #attachments = [] forms = row.xpath('.//form') for form in forms: for hidden_field in form.xpath('input'): if hidden_field.get('name') != 'DT': continue attachment_id = hidden_field.get('value') #attachments.append(attachment_id) found_attachments.append(attachment_id) #if len(attachments): # agendaitems[agendaitem_id]['attachments'] = attachments elif 'smc_tophz' in row_classes: # additional (optional row for agendaitem) label = fields[1].text value = fields[2].text if label is not None and value is not None: label = label.strip() value = value.strip() #print (label, value) if label in ['Ergebnis:', 'Beschluss:']: if value in self.config.RESULT_STRINGS: agendaitems[agendaitem_id]['result'] = self.config.RESULT_STRINGS[value] else: logging.warn("String '%s' not found in configured RESULT_STRINGS", value) if self.options.verbose: print "WARNING: String '%s' not found in RESULT_STRINGS\n" % value agendaitems[agendaitem_id]['result'] = value elif label == 'Bemerkung:': agendaitems[agendaitem_id]['result_note'] = value elif label == 'Abstimmung:': agendaitems[agendaitem_id]['voting'] = value else: logging.critical("Agendaitem info label '%s' is unknown", label) raise ValueError('Agendaitem info label "%s" is unknown' % label) elif 'smcrowh' in row_classes: # Subheading (public / nonpublic part) if fields[0].text is not None and "Nicht öffentlich" in fields[0].text.encode('utf-8'): public = False #print json.dumps(agendaitems, indent=2) session.agendaitems = agendaitems.values() # session-related attachments containers = dom.xpath(self.xpath['SESSION_DETAIL_ATTACHMENTS']) for container in containers: classes = container.get('class') if classes is None: continue classes = classes.split(' ') if self.xpath['SESSION_DETAIL_ATTACHMENTS_CONTAINER_CLASSNAME'] not in classes: continue attachments = [] rows = container.xpath('.//tr') for row in rows: forms = row.xpath('.//form') for form in forms: #print "Form: ", form name = " ".join(row.xpath('./td/text()')).strip() for hidden_field in form.xpath('input'): if hidden_field.get('name') != 'DT': continue attachment_id = hidden_field.get('value') # make sure to add only those which aren't agendaitem-related if attachment_id not in found_attachments: attachment = Attachment( identifier=attachment_id, name=name ) # Traversing the whole mechanize response to submit this form for mform in mechanize_forms: #print "Form found: '%s'" % mform for control in mform.controls: if control.name == 'DT' and control.value == attachment_id: #print "Found matching form: ", control.name, control.value attachment = self.get_attachment_file(attachment, mform) attachments.append(attachment) found_attachments.append(attachment_id) if len(attachments): session.attachments = attachments oid = self.db.save_session(session) if self.options.verbose: logging.info("Session %d stored with _id %s", session_id, oid)
player_changes = Player() player_changes.load_from_db("*****@*****.**") player_changes.nickname = "Lesha" player_changes.save_to_db() print("{}Player was updated in db.\n".format(player_changes)) player_changes.delete_from_db() print("{}Player was deleted from db.\n".format(player_changes)) ######################################################################################################################## player_session = Session(player_id=1, finish_time=datetime.datetime(2017, 2, 14, 00, 00), elapse_time=datetime.datetime(1, 1, 1), created=datetime.datetime.now(), updated=datetime.datetime.now()) print("{}Information about player session.\n".format(player_session)) player_session.save_to_db() print("{}Player session was inserted to db.\n".format(player_session)) player_session_info = Session() player_session_info.load_from_db(1) player_session_change = Session() player_session_change.load_from_db(1) player_session_change.finish_time = datetime.datetime(2016, 2, 14, 00, 00) player_session_change.elapse_time = datetime.datetime(1,1,2)
def main(): parser = argparse.ArgumentParser(description="Freedom Fighting Mode.") parser.add_argument("--debug-input", action="store_true", help="Toggle debugging of the user input.") parser.add_argument("--debug-output", action="store_true", help="Toggle debugging of the terminal output.") parser.add_argument("--config", "-c", help="The harness' configuration file.", default=os.path.join(os.path.dirname(__file__), "ffm.conf")) parser.add_argument("--stdout", help="Redirect stdout to the target file.") args = parser.parse_args() context.debug_input = args.debug_input context.debug_output = args.debug_output context.stdout = open(args.stdout, "wb") if args.stdout is not None else sys.stdout # Print the banner print(random.choice(BANNERS) + "\n") print( "FFM enabled. Type !list to see available commands and exit to quit.") # Check that the configuration file exists and is sane. if not os.path.exists(args.config): print( "Could not find %s. Please provide it with the --config option." % args.config) return context.config = configparser.ConfigParser(allow_no_value=True, inline_comment_prefixes=("#", ";")) context.config.read(args.config) context.terminal_driver = DefaultInputDriver() stdin_fd = sys.stdin.fileno() old_settings = termios.tcgetattr(stdin_fd) old_handler = signal.signal(signal.SIGWINCH, update_window_size) tty.setraw(context.stdin) context.active_session = Session() context.sessions.append(context.active_session) update_window_size() # Set the correct window size in the PTY. try: while context.active_session and context.active_session.bash.poll( ) is None: try: r, w, e = select.select( [sys.stdin, context.active_session.master], [], [], 1) if sys.stdin in r: typed_char = os.read(sys.stdin.fileno(), 1) try: context.active_session.input_driver.handle_input( typed_char) except RuntimeError as e: os.write(context.stdout.fileno(), b"\r\n%s\r\n" % str(e).encode("UTF-8")) elif context.active_session.master in r: read = os.read(context.active_session.master, 2048) if context.debug_output: for c in read: os.write(context.stdout.fileno(), ("%02X " % c).encode("UTF-8")) # Store the last line for future use last = read.split(b"\n")[-1] # Debian terminals update the window title with this escape sequence. Ignore it. last = re.sub(b"\x1b]0;.*?\x07", b"", last) # Kali terminals add color to the prompt. Strip it. last = re.sub(b"\x1b\[[0-?]*[ -/]*[@-~]", b"", last) if re.match(PROMPT_REGEXP, last.decode("UTF-8", errors='ignore'), re.UNICODE): # TODO: keep the colors in the saved prompt. This will require all # references of len(last_line) to be updated to ignore escape sequences. context.active_session.input_driver.last_line = last.decode( "UTF-8") else: context.active_session.input_driver.last_line = '' # Pass the output to the output driver for display after applying output processors. (proceed, output) = apply_processors(read, OUTPUT_PROCESSOR_LIST) if proceed: context.active_session.output_driver.handle_bytes( output) except select.error as e: if "[Errno 4]" in str( e ): # Interrupted system call. May be raised if SIGWINCH is received. continue else: raise # Pretty printing for unimplemented opcodes: no need for a full trace. Probably remove that in the future. except RuntimeError as e: exc_type, exc_obj, exc_tb = sys.exc_info() filename = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] print("\r\n%s (%s, line %d)\r" % (str(e), filename, exc_tb.tb_lineno)) return # Bash has finished running print("FFM disabled.\r") finally: termios.tcsetattr(stdin_fd, termios.TCSADRAIN, old_settings) signal.signal(signal.SIGWINCH, old_handler)
def main(): parser = argparse.ArgumentParser(description="Freedom Fighting Mode.") parser.add_argument("--debug-input", action="store_true", help="Toggle debugging of the user input.") parser.add_argument("--debug-output", action="store_true", help="Toggle debugging of the terminal output.") parser.add_argument("--stdout", help="Redirect stdout to the target file.") args = parser.parse_args() context.debug_input = args.debug_input context.debug_output = args.debug_output context.stdout = open(args.stdout, "wb") if args.stdout is not None else sys.stdout # Print the banner print(random.choice(BANNERS) + "\n") print("FFM enabled. Type !list to see available commands and exit to quit.") context.terminal_driver = DefaultInputDriver() stdin_fd = sys.stdin.fileno() old_settings = termios.tcgetattr(stdin_fd) old_handler = signal.signal(signal.SIGWINCH, update_window_size) tty.setraw(context.stdin) context.active_session = Session() context.sessions.append(context.active_session) update_window_size() # Set the correct window size in the PTY. try: while context.active_session and context.active_session.bash.poll() is None: try: r, w, e = select.select([sys.stdin, context.active_session.master], [], [], 1) if sys.stdin in r: typed_char = os.read(sys.stdin.fileno(), 1) try: context.active_session.input_driver.handle_input(typed_char) except RuntimeError as e: os.write(context.stdout.fileno(), b"\r\n%s\r\n" % str(e).encode("UTF-8")) elif context.active_session.master in r: read = os.read(context.active_session.master, 2048) if context.debug_output: for c in read: os.write(context.stdout.fileno(), ("%02X " % c).encode("UTF-8")) # Store the last line for future use # Only work on the last line last = read.split(b"\n")[-1] if len(last) < 150 and b"\x07" in last: # TODO: bug when cat-ing a binary file! # This is motivated by my Debian shell's prompt containing weird \x07 bytes separating # two prompt occurrences. context.active_session.input_driver.last_line = last.split(b"\x07")[-1].decode("UTF-8", errors='ignore') elif re.match(PROMPT_REGEXP, last.decode("UTF-8", errors='ignore'), re.UNICODE): context.active_session.input_driver.last_line = last.decode("UTF-8") else: context.active_session.input_driver.last_line = '' # Pass the output to the output driver for display after applying output processors. (proceed, output) = apply_processors(read, OUTPUT_PROCESSOR_LIST) if proceed: context.active_session.output_driver.handle_bytes(output) except select.error as e: if "[Errno 4]" in str(e): # Interrupted system call. May be raised if SIGWINCH is received. continue else: raise # Pretty printing for unimplemented opcodes: no need for a full trace. Probably remove that in the future. except RuntimeError as e: exc_type, exc_obj, exc_tb = sys.exc_info() filename = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] print("\r\n%s (%s, line %d)\r" % (str(e), filename, exc_tb.tb_lineno)) return # Bash has finished running print("FFM disabled.\r") finally: termios.tcsetattr(stdin_fd, termios.TCSADRAIN, old_settings) signal.signal(signal.SIGWINCH, old_handler)
def config_injector(binder): binder.bind(Config, Config('server-config.cfg')) binder.bind(Session, Session())
# import logging from controller.logincontroller import LoginController from controller.downloadcontroller import NalandaDownloaderController from model.googleoauth import GoogleSession from model.session import Session from model.nalanda import NalandaSession from view.authenticationwindow import AuthenticationWindow from view.nalandadownloader import NalandaDownloader # logging.basicConfig(format='%(asctime)s:%(name)s:LINE %(lineno)d:%(levelname)s:%(message)s', level=logging.DEBUG) if __name__ == '__main__': session = Session() nalanda_session = NalandaSession(session=session) google_session = GoogleSession(session=session, nalanda_session=nalanda_session) login_window = AuthenticationWindow() login_controller = LoginController(login_window=login_window, google_session=google_session) nalanda_downloader = NalandaDownloader() nalanda_controller = NalandaDownloaderController( nalanda_downloader=nalanda_downloader, nalanda_session=nalanda_session)
def get(self): cutoff = datetime.utcnow() - MAX_AGE sessions = Session.query(Session.created_at < cutoff).fetch(200, keys_only=True) if len(sessions) > 0: ndb.delete_multi(sessions) logging.info('%s sessions cleared' % len(sessions))