def filemonitoring(self): log = logging.getLogger(__name__) if self._request.params: # TODO: add this information to the file md5_enabled = True if 'withmd5' in self._request.params and self._request.params['withmd5'] == '0' else False all_files = self._request.params.getall('file') complete_file, filenames, folders = self._get_monitored_files(self._request.params['folder'] + '/') with transaction.manager: for f in all_files: if f in complete_file: log.debug('Skipping file {0}, because it is already monitored'.format(f)) continue (path, filename) = os.path.split(f) dbobj = MonitoredFile(path, filename, f) DBSession.add(dbobj) DBSession.commit() files_not_mentioned = [c for c in complete_file if c not in all_files] # TODO: decide on this log.info('TODO: Still have to decide whether files which are not selected should be deleted or not.' 'Affected files would be: {0}'.format(files_not_mentioned)) else: log.info('Got an empty request, going to redirect to start page') subreq = Request.blank('/') return self._request.invoke_subrequest(subreq) subreq = Request.blank(self._request.route_path('filebrowser'), POST=dict(folder=self._request.params['folder'], currentfolder=self._request.params['currentfolder'], pathdescription='abs')) return self._request.invoke_subrequest(subreq)
def write_jobs_to_database(jobs): """ Writes the retrieved jobs to the database :param jobs: list of all jobs :return: """ log = logging.getLogger(__name__) with transaction.manager: for (classname, classmembers) in jobs.items(): if len(classmembers) == 0: continue for element in classmembers: if 'error' in element: continue if 'JobID' not in element: log.debug('Row didn''t match specified criteria {0}'.format(element)) continue if not re.search('[0-9]*', element['JobID']): log.debug('Row didn''t match specified criteria {0}'.format(element)) continue dbrow = DBSession.query(Job).filter(Job.id == element['JobID']).all() json_str = jsonpickle.encode(element) if len(dbrow) == 0: j = Job(element['JobID'], json_str) DBSession.add(j) elif len(dbrow) == 1: dbrow[0].jobinfo = json_str else: log.error('More than one entry for jobid: {0}'.format(json_str)) DBSession.commit()
def on_set_base_info(cls, company_id, company_name=None, country=None, city=None, address=None, timezone=None, currency=None, note=None): """ 设置公司基础信息 """ session = DBSession() com = session.query(cls).filter(cls.company_id == company_id).first() if com: if company_name: com.company_name = company_name if country: com.country = country if city: com.city = city if address: com.address = address if timezone: com.timezone = timezone if currency: com.currency = currency if note: com.note = note try: session.add(com) session.commit() except Exception as why: session.rollback() message = "companies.on_set_base_info: {0}".format(why) logging.info(message) return -1 return 0
def job_output(self): log = logging.getLogger(__name__) jobid = self._request.matchdict['jobid'] log.info('Request job output for id:{0}'.format(jobid)) ssh_holder = self._request.registry.settings['ssh_holder'] ssh_jobmanager = SSHBasedJobManager(ssh_holder) jobs = self._get_current_jobs(ssh_jobmanager, self._coltitles) joboutput = ssh_jobmanager.get_job_output(jobid) if 'error' in joboutput and joboutput['error'] is None: db_joboutput = JobOutput(id=jobid, output=jsonpickle.encode(joboutput)) DBSession.add(db_joboutput) DBSession.commit() if 'error' in joboutput and joboutput['error'] is not None: jobresult = joboutput['error'] else: jobresult = joboutput['content'] jobresult = ''.join(jobresult) if 'type' not in joboutput: joboutput['type'] = 'type field missing' return {'project': self._projectname, 'jobs': jobs, 'output': dict(jobid=jobid, output=jobresult, type=joboutput['type'])}
def _cleanup(self): now = time() session = DBSession() session.query(Data).filter( now - Data.timestamp > TIMEOUT_THRESHOLD).delete() session.commit() session.close()
def getUser(request): username = request.headers["X-Webauth-User"] #now get the uuid uuid = "" print "UserName:"******"UUID:", uuid if uuid == "": user = DBSession.query(Owners).filter_by(username=username).all() else: user = DBSession.query(Owners).filter_by(uuid=uuid).all() if len(user) == 0: cur = Owners(username, uuid) DBSession.add(cur) DBSession.commit() user = DBSession.query(Owners).filter_by(username=username).all() print "Made" if len(user) == 1: print "Found" cur = user[0] # if user == "zemon1": # user = "******" return cur
def crawl_plat_first_letter(shuju_date="2020-01-062020-01-12"): """ 平台成交数据 https://shuju.wdzj.com/platdata-1.html """ url = "https://shuju.wdzj.com/plat-data-custom.html" form_data = { "type": 0, "shujuDate": shuju_date } response = requests.post(url, data=form_data) status = response.status_code if status != 200: print("crawl failed. (status is not 200)") raise CrawlFailed('crawl failed') plats_data = response.json() for plat_data in plats_data: plat_id = plat_data.get('platId') wdzj_id = plat_data.get('wdzjPlatId') first_letter = plat_data.get('firstLetter') session = DBSession() if wdzj_id != 0: product = session.query(Product).filter_by(plat_id=plat_id).first() product.first_letter = first_letter session.commit() session.close()
def sentiment_data(self, url, common_list): """ Does the sentiment analysis with wit.ai and saves the sentiment data into the database. """ query_str = urllib.parse.quote_plus(' '.join(common_list)[:280]) resp = yield self.fetch_url( 'https://api.wit.ai/message?v=20180703&q={}'.format(query_str), headers={ 'Authorization': 'Bearer {WIT_AUTH_KEY}'.format(WIT_AUTH_KEY=WIT_AUTH_KEY) }) ents = json.loads(resp['body'])['entities'] sentiment = ents and ents.get('sentiment') if sentiment: sentiment = sentiment[0]['value'] s_obj = DBSession.query(UrlSentiment).filter( UrlSentiment.sentiment == sentiment) if s_obj.count() == 0: s_obj = UrlSentiment(url=url, sentiment=sentiment) DBSession.add(s_obj) DBSession.commit() raise gen.Return(s_obj) raise gen.Return(False)
class LagouPipeline(object): def __init__(self): self.session = DBSession() self.pool = redis.Redis(host='raspberrypi', port=6379, db=2) def process_item(self, item, spider): if isinstance(item, LagouItem): if self.session.query(Jobs).filter( Jobs.positionId == item['positionId'], Jobs.companyId == item['companyId']).first(): pass else: obj = Jobs( companyId=item['companyId'], positionId=item['positionId'], jobNature=item['jobNature'], companyName=item['companyName'], financeStage=item['financeStage'], companyFullName=item['companyFullName'], companySize=item['companySize'], industryField=item['industryField'], positionName=item['positionName'], city=item['city'], createTime=item['createTime'], salary_low=item['salary_low'], salary_high=item['salary_high'], workYear=item['workYear'], education=item['education'], positionAdvantage=item['positionAdvantage'], district=item['district'], # uid=item['uid'], companyLabelList=item['companyLabelList'], ) self.session.add(obj) try: self.session.commit() except Exception, e: print e self.session.rollback() # self.session.close() elif isinstance(item, CompanyItem): # 公司信息存入mysql数据库 ''' obj=Company( companyId=item['companyId'], companyName=item['companyFullName'] ) self.session.add(obj) try: self.session.commit() except Exception, e: print e self.session.rollback() ''' # 公司的数据存入redis self.pool.set(item['companyId'], item['companyFullName'])
def create_blog_api(): user = g.get('user', None) if user is None: return redirect(url_for('login')) else: if user.admin: name = request.form['name'].encode('utf8') summary = request.form['summary'].encode('utf8') content = request.form['content'].encode('utf8') user_id = request.form['user_id'].encode('utf8') user_name = request.form['user_name'].encode('utf8') user_image = request.form['user_image'].encode('utf8') sess = DBSession() blog = Blogs(user_id=user_id, user_name=user_name, user_image=user_image, name=name, summary=summary, content=content) sess.add(blog) sess.commit() sess.close() return 'ok' else: return redirect(url_for('login'))
def delete(self, catid): session = DBSession() category = session.query(models.Category).filter_by(id=catid).first() for item in category.items: session.delete(item) session.delete(category) session.commit()
def crawl_plat_overview(first_letter): # url = "https://www.wdzj.com/dangan/pjs/" url = "https://www.wdzj.com/dangan/{first_letter}/".format( first_letter=first_letter ) print("crawl plat {}".format(first_letter)) response = requests.get(url, headers=HEADERS) if response.status_code != 200: print('crawl failed: code: {}, url: {}'.format(response.status_code, url)) return # raise CrawlFailed('crawl failed!') encode_response(response) html = etree.HTML(response.text) try: plat_name = html.xpath("//div[@class='title']/h1|h2")[0].text print("plat name: {}".format(plat_name)) # 注册资金(实缴资金) 银行存管 投标保障 # box = html.xpath("//div[@class='zzfwbox'] | //div[@class='bgbox-bt zzfwbox']") try: zczj = html.xpath("//div[@class='zzfwbox']/dl[1]/dd[1]//div[@class='r']")[0].text.strip().split() except IndexError: zczj = html.xpath("// div[ @class ='bgbox-bt zzfwbox'] // dl[1] / dd[1] // div[@ class ='r']")[0].text.strip().split() # // div[ @class ='bgbox-bt zzfwbox'] // dl[1] / dd[1] // div[@ class ='r'] if len(zczj) == 2: zczj_value, sjzj = zczj sjzj_value = sjzj.strip('()').split(':')[1] else: zczj_value = zczj sjzj_value = '-' try: yhcg_value = html.xpath("//div[@class='zzfwbox']/dl[1]/dd[2]//div[@class='r']")[0].text.strip() except IndexError: yhcg_value = html.xpath("//div[ @class ='bgbox-bt zzfwbox']//dl[1]/dd[2]//div[@class='r']")[0].text.strip() try: tbbz_value = html.xpath("//div[@class='zzfwbox']/dl[2]/dd[3]//div[@class='r']")[0].text.strip() except IndexError: tbbz_value = html.xpath("//div[ @class ='bgbox-bt zzfwbox']//dl[2]/dd[3]//div[@class='r']")[0].text.strip() plat_overview = dict( plat_name=plat_name, zhucezijin=zczj_value, shijiaozijin=sjzj_value, yinhangcunguan=yhcg_value, toubiaobaozhang=tbbz_value ) except AttributeError as ex: print('crawl failed: ex: {}, url: {}'.format(str(ex), url)) raise except IndexError as ex: print('crawl failed: ex: {}, url: {}'.format(str(ex), url)) raise new_plat_overview = PlatOverview(**plat_overview) session = DBSession() session.add(new_plat_overview) session.commit() session.close()
def clear_db(): """On teardown, remove all the db stuff""" db = DBSession() db.query(Payment).delete() db.query(Trade).delete() db.query(Product).delete() db.query(LogTrade).delete() db.commit()
def save_data(self, data): session = DBSession() temp = data.copy() temp['timestamp'] = time() new_record = Data(**temp) session.add(new_record) session.commit() session.close()
def update(self, grid, data): session = DBSession() session.query(Person).filter(Person.grid == grid).update(data) try: session.commit() except Exception as e: log.debug("Ocorreu um erro ao salvar as configurações !\n" + e) session.rollback()
def write_job_archive_config(configuration): """ Writes a new job archive configuration to the database :param configuration: :return: """ with transaction.manager: DBSession.add(ArchiveUserConfiguration(jsonpickle.encode(configuration))) DBSession.commit()
def subtract_money(player_out, amount): out_player = Player.query.filter(Player.id == int(player_out)).first() movement_out = Movement(player=out_player, amount=amount, move_type=Movement.TYPES['OUT']) DBSession.add(movement_out) out_player.update_balance(operation_type=Movement.TYPES['OUT'], amount=amount) DBSession.flush() DBSession.commit()
def add_money(player_in, amount): in_player = Player.query.filter(Player.id == int(player_in)).first() movement_in = Movement(player=in_player, amount=amount, move_type=Movement.TYPES['IN']) DBSession.add(movement_in) in_player.update_balance(operation_type=Movement.TYPES['IN'], amount=amount) DBSession.flush() DBSession.commit()
def update_patient(id): content = request.get_json(silent=True) session = DBSession() patient = session.query(Patient).filter_by(id=id).one() for key, value in content.iteritems(): setattr(patient, key, value) stmt = session.add(patient) session.commit() session.close() return jsonify()
def create_user(request): data = json.loads(request.body) new_user = User( uid=data['username'] ) new_user.created_at = datetime.now() if DBSession.query(User).filter(User.name == new_user.name).count() > 0: raise HTTPUnauthorized() DBSession.add(new_user) DBSession.commit() return {"success": True}
def post(self, catid): session = DBSession() args = parser.parse_args() if 'name' not in args.keys(): abort(400, message="`name` missing") if len(args['name']) == 0: abort(400, message="`name` empty") category = session.query(models.Category).filter_by(id=catid).first() item = models.Item(category=category, name=args['name']) session.add(item) session.commit()
def add_players_command(num_of_players): '''num_of_players: Numero de jogadores''' for p in range(int(num_of_players)): player_name = 'Jogador {}'.format(p + 1) player = Player(player_name=player_name) DBSession.add(player) player = Player(player_name='Banqueiro', balance=float(1000000.00)) DBSession.add(player) DBSession.flush() DBSession.commit()
def eventDelete(request): eventId = int(request.matchdict['event_id']) event = DBSession.query(Event).filter(Event.id == eventId).first() if event == None: return {'status' : 'Event Not Found'} DBSession.delete(event) DBSession.commit() return {'status' : 'Deleted'}
def change_avatar(p_id=None, avatar=None): try: res = {"success": False, "msg": ""} session = DBSession() user_profile = session.query(UserProfile).filter_by(id=p_id).first() user_profile.avatar = avatar session.commit() res["success"] = True return res except Exception as e: res["msg"] = str(e) return res
def save_options_to_db(self): try: transaction.begin() Options.reset() dbsession = DBSession() options = Options(self.options['timeon'], self.options['timeoff'], self.options['speed']) dbsession.add(options) dbsession.commit() transaction.commit() return True except: return False
def put(self, catid): session = DBSession() category = session.query(models.Category).filter_by(id=catid).first() args = parser.parse_args() if args['name'] and len(args['name']): category.name = args['name'] if args['order'] and len(args['order']): neworder = args['order'] other = session.query(models.Category).filter_by(order=neworder).first() if other: other.order = category.order category.order = neworder session.commit()
def delete(self, grid): session = DBSession() session.query(Person).filter(Person.grid == grid).update({'flag': 'D'}) try: session.commit() log.info("Comando executado com sucesso !") log.info(session) return session except Exception as e: log.debug("Ocorreu um erro ao salvar as configurações !\n" + e.__str__()) session.rollback() return None
def save_values_to_db(self): try: transaction.begin() Texts.reset() for k,v in self.items.iteritems(): dbsession = DBSession() texts = Texts(v['text'], v['color'], v['effect'], v['effect_time']) dbsession.add(texts) dbsession.commit() transaction.commit() return True except: return False
def on_remove_admin_user(cls, company_id): """ 删除公司管理员用户 """ session = DBSession() com = session.query(cls).filter(cls.company_id == company_id).first() if com: com.admin_user_id = None try: session.add(com) session.commit() except Exception as why: message = "companies.on_remove_admin_user: {0}".format(why) logging.info(message) return -1 return 0
def add_label(self): log = logging.getLogger(__name__) file = self.request.matchdict['file'] action = self.request.matchdict['action'] assert (action == 'comment') try: DBSession.add(FileLabelModel(file, self.request.POST['label'])) DBSession.commit() except BaseException as e: log.error(e.message) subreq = self.request.copy() subreq.method = 'GET' return self.request.invoke_subrequest(subreq)
def delete_blog_api(): user = g.get('user', None) blog_id = request.args.get('id', '') if user is None: return redirect(url_for('login')) else: if user.admin: sess = DBSession() blog = sess.query(Blogs).filter(Blogs.id == blog_id).delete() sess.commit() sess.close() else: return redirect(url_for('login')) return 'ok'
def add(cls, name, url): ret = True session = DBSession() record = Action(name, url) session.add(record) try: session.commit() session.close() logging.info('add action success<name=%s, url=%s>' % (name, url)) except IntegrityError as error: logging.error(str(error)) ret = False return ret
def create_patient(): content = request.get_json(silent=True) session = DBSession() session.expire_on_commit = False patient = Patient() if content and ("mac_address" in content): patient.mac_address = content["mac_address"].upper() stmt = session.add(patient) print(patient.id) session.commit() id = patient.id print(patient.id) session.close() return jsonify(patient.id)
def populate(comps='comps-f16', do_dependencies=True): from yum.comps import Comps session = DBSession() c = Comps() c.add('comps/%s.xml' % comps) for group in c.groups: g = Group(id=group.groupid, name=group.name, description=group.description) session.add(g) for package in group.packages: p = session.query(Package).filter_by( name=to_unicode(package)).first() if not p: p = Package(name=package) session.add(p) p.group = g session.flush() root = Root(name=u'Fedora') session.add(root) session.flush() for category in c.categories: c = Category(id=category.categoryid, name=category.name, description=category.description) session.add(c) root.categories.append(c) for group in category.groups: g = session.query(Group).filter_by( group_id=to_unicode(group)).first() if not g: print "Cannot find group: %s" % group else: g.category = c session.flush() if do_dependencies: for package in session.query(Package).all(): add_dependencies(package, session) session.commit()
def add(): init_db() c1 = Customer('Anna', 'Smith', '*****@*****.**', datetime.datetime(1990, 5, 1)) c2 = Customer('John', 'Brown', '*****@*****.**', datetime.datetime(1992, 5, 1)) print(c1) a1 = Account(c1) a2 = Account(c2) db = DBSession().get() db.add(c1) db.add(c2) db.add(a1) db.add(a2) db.commit()
def vote_article(self, artId): data = { "userId": self.userId, "accessToken": self.accessToken, "artId": artId } r = requests.post(VOTE_ARTICLE_API, data, verify=False) result = r.json() vote = VoteRecord(artId=artId, result=result['res'], message=result['resMsg']) session = DBSession() session.add(vote) session.commit() session.close()
def create_db_object(obj, search_for=None, **kwargs): # type: (Base, dict, dict) -> Base new_obj = None if search_for: new_obj = find_db_object(obj, **search_for) # print(f"searching for {obj}: {search_for}\n{bool(new_obj)}") if not new_obj: # print(f"Creating new {obj}") new_obj = obj(**kwargs) DBSession.add(new_obj) DBSession.commit() return new_obj
def on_set_admin_user(cls, company_id, user_id): """ 设置公司管理用户 """ session = DBSession() com = session.query(cls).filter(cls.company_id == company_id).first() if com: com.admin_user_id = user_id try: session.add(com) session.commit() except Exception as why: session.rollback() message = "companies.on_set_admin_user: {0}".format(why) logging.info(message) return -1 return 0
def create_message(request): data = json.loads(request.body) cur_user = data['username'] user_id = DBSession.query(User).filter_by(uid=cur_user).one().id data["owner_id"] = user_id target_query = DBSession.query(User).filter( User.uid == cur_user) if target_query.count() > 0: target = target_query.first() if target.uid == cur_user or cur_user.admin: m = Message.from_dict(data) #m = Message() DBSession.add(m) DBSession.commit() return m.to_dict() raise HTTPUnauthorized
def post(self): session = DBSession() args = parser.parse_args() if 'name' not in args.keys(): abort(400, message="`name` missing") if len(args['name']) == 0: abort(400, message="`name` empty") query = session.query(models.Category).order_by(models.Category.order.desc()) if query.count(): order = query.first().order + 1 else: order = 0 category = models.Category(name=args['name'], order=order) session.add(category) session.commit()
def on_set_business_hours(cls, company_id, opening_hours, closing_hours): """ 设置公司营业时间 """ session = DBSession() com = session.query(cls).filter(cls.company_id == company_id).first() if com: com.opening_hours = opening_hours com.closing_hours = closing_hours try: session.add(com) session.commit() except Exception as why: session.rollback() message = "companies.on_set_business_hours: {0}".format(why) logging.info(message) return -1 return 0
def update(self): jobs = self.get_all_jobs() JobDatabaseWrapper.write_jobs_to_database(jobs) for item, elements in jobs.items(): for elem in elements: jobdetail = self.get_job_details(elem['JobID']) output = self.get_job_output(elem['JobID']) job = DBSession.query(Job).filter(Job.id == jobdetail['JobID']).first() if job is not None: job.jobdetails = jsonpickle.encode(jobdetail) DBSession.commit() joboutput = JobOutput(elem['JobID'], jsonpickle.encode(output)) DBSession.add(joboutput) DBSession.commit() return jobs
def addLocationDB(locName, roomNum, locOwner): exists = DBSession.query(Location).filter_by(owner=locOwner.id, name=locName, roomNumber=roomNum).all() if len(exists) == 0: tempLoc = Location(owner.id, name, roomNum) DBSession.add(cur) DBSession.commit() loc = DBSession.query(Location).filter_by(id=tempLoc.id).all() print "Made Loc" if len(exists) == 1: print "Found Loc" loc = exists[0] return loc
def save_to_log(from_who='user', message_type=None, message=None, comment_text='', msg_text=''): """Сохранить в лог. Внимательно передавать from_who from_who - 'bot', 'user', 'system'. От кого сообщение message - тип message. Сообщение от пользователя. comment_text - дополнительный текст. msg_text - текст сообщения. Использовать для сохранения ответа бота на message пользователя Примеры. save_to_log('user', message) - сохранить сообщение от пользователя. save_to_log('system', comment_text=err_text) - сохранить сообщение от системы. Например, об ошибке. save_to_log('bot', message=message_from_user, msg_text=bot_msg_text) - сохранить сообщение от бота пользоателю. """ if from_who not in ('bot', 'user', 'system'): comment_text += ' ' + from_who from_who = 'need_help' operation = None tid = None session = DBSession() if message: tid = message.from_user.id if from_who == 'user': if message.content_type == 'text': msg_text = message.text if message.content_type == 'contact': msg_text = str(message.contact) operation = session.query(Operation).filter_by(telegramid=tid).first() if operation is None: operation = Operation() log = Log(datetime=datetime.datetime.now(), from_who=from_who, user_id=tid, msg_text=msg_text, msg_type=message_type, operation=operation.current_operation, status=operation.operation_status, additional_info=operation.additional_info_db, function=inspect.stack()[1][3], comment=comment_text) session.add(log) session.commit() session.close()
def crawl_problem_plats(): """ 问题平台 https://shuju.wdzj.com/problem-1.html """ url = "https://shuju.wdzj.com/problem-list-all.html" params = {"year": ""} response = requests.get(url, params=params, headers=HEADERS) json_data = response.json() problem_plats = json_data.get('problemList') for problem_plat in problem_plats: session = DBSession() plat_id = problem_plat.get('platId') wdzj_id = problem_plat.get('wdzjPlatId') plat_name = problem_plat.get('platName') if wdzj_id != 0: session.execute( """ INSERT INTO products (plat_id, wdzj_id, name) select '{plat_id}', '{wdzj_id}', '{plat_name}' WHERE not EXISTS (SELECT * FROM products WHERE plat_id = '{plat_id}'); """.format( plat_id=plat_id, wdzj_id=wdzj_id, plat_name=plat_name ) ) new_problem_plat = ProblemPlat( plat_id=problem_plat.get('platId'), # plat_id wdzj_id=problem_plat.get('wdzjPlatId'), # wdzj_id plat_name=problem_plat.get('platName'), # plat_name area=problem_plat.get('area'), # 地区 oneline_time=problem_plat.get('onlineTime'), # 上线时间 problem_date=problem_plat.get('problemTime'), # 问题时间 event_type=problem_plat.get('type'), # 事件类型 people_num=problem_plat.get('peopleNumber'), status1=problem_plat.get('status1'), # 保留字段status1 status2=problem_plat.get('status2') # 保留字段status2 ) session.add(new_problem_plat) session.commit() session.close()
def start_game_command(num_of_players): '''num_of_players: Numero de jogadores''' click.secho("Iniciando novo jogo com {} jogadores\n".format(num_of_players), fg='yellow', bold=True) initialisedb() with click.progressbar(range(int(num_of_players))) as players: for p in players: player_name = 'Jogador {}'.format(p + 1) click.secho(' Inserindo jogador {0}'.format(player_name), fg='blue') player = Player(player_name=player_name) DBSession.add(player) player = Player(player_name='Banqueiro', balance=float(1000000.0)) DBSession.add(player) DBSession.flush() DBSession.commit() click.secho('[OK]', fg='green', bold=True)
def topicUnfollow(request): currentUser = int(authenticated_userid(request)) topicId = int(request.matchdict['topic_id']) followed = DBSession.query(UserFollowedTopic).\ filter(and_(UserFollowedTopic.topic_id == topicId,UserFollowedTopic.user_id == currentUser)).\ first() if followed == None: return {'status' : 'Not Following'} DBSession.delete(followed) DBSession.flush() DBSession.commit() return {'status' : 'Unfollowed'}
def process_item(self, item, spider): session = DBSession() defaults = { 'Immobilientyp': u'Büros, Gewerbeflächen', 'Boersen_ID': 21, 'Vermarktungstyp': u'kaufen', 'Land': u'Deutschland'} item = defaults.update(item) try: quoka_item = Quoka(**defaults) session.add(quoka_item) session.commit() except Exception as e: print e return item
def job_details(self): log = logging.getLogger(__name__) details_jobid = self._request.matchdict['jobid'] log.info('Request details for jobid {0}'.format(details_jobid)) ssh_holder = self._request.registry.settings['ssh_holder'] ssh_jobmanager = SSHBasedJobManager(ssh_holder) jobs = self._get_current_jobs(ssh_jobmanager, self._coltitles) return_details = ssh_jobmanager.get_job_details(details_jobid) # write details to the database job = DBSession.query(Job).filter(Job.id == details_jobid).first() if job is not None: job.jobdetails = jsonpickle.encode(return_details) DBSession.commit() return {'project': self._projectname, 'jobs': jobs, 'details': return_details}
def delete(cls, id): ret = True session = DBSession() actions = session.query(Action).filter(Action.id == id).all() if actions: try: session.delete(actions[0]) session.commit() session.close() except IntegrityError as error: logging.error(error) ret = False else: logging.error('Action not found') ret = False return ret
def getUserAndBlackjack(uName, uGuid=""): curUser = None curBlk = None #if Guid is empty use the username if uGuid == "": userExist = DBSession.query(Users).filter_by(userName = uName).all() else: userExist = DBSession.query(Users).filter_by(userGuid = uGuid).all() #If the user doesnt exist make one if len(userExist) == 0: curUser = Users(uName, userGuid=uGuid) DBSession.add(curUser) DBSession.commit() userExist = DBSession.query(Users).filter_by(userName = uName).all() print "Made" #If they do, retrieve it if len(userExist) == 1: # or curUser != None: print "Found" curUser = userExist[0] userBlk = DBSession.query(Blackjack).filter_by(userId = curUser.id).all() #If a game doesnt exist make one if len(userBlk) == 0: #Make a game curBlk = Blackjack(curUser.id) #add the game DBSession.add(curBlk) #commit it DBSession.commit() curBlk = DBSession.query(Blackjack).filter_by(userId = curUser.id).first() print "Made" elif len(userBlk) == 1: print "Found" curBlk = userBlk[0] if curUser == None or curBlk == None: print "Had to recurse, Should never need this" return getUserAndBlackjack(uName, uGuid) return [curUser, curBlk]
def populate(comps='comps-f16', do_dependencies=True): from yum.comps import Comps session = DBSession() c = Comps() c.add('comps/%s.xml' % comps) for group in c.groups: g = Group(id=group.groupid, name=group.name, description=group.description) session.add(g) for package in group.packages: p = session.query(Package).filter_by(name=to_unicode(package)).first() if not p: p = Package(name=package) session.add(p) p.group = g session.flush() root = Root(name=u'Fedora') session.add(root) session.flush() for category in c.categories: c = Category(id=category.categoryid, name=category.name, description=category.description) session.add(c) root.categories.append(c) for group in category.groups: g = session.query(Group).filter_by(group_id=to_unicode(group)).first() if not g: print "Cannot find group: %s" % group else: g.category = c session.flush() if do_dependencies: for package in session.query(Package).all(): add_dependencies(package, session) session.commit()
def on_set_state(cls, company_id, state): """ 设置公司状态 """ if state not in ("00", "01", "02"): message = "companies.on_set_state: wrong state: {0}".format(state) logging.info(message) return -1 session = DBSession() com = session.query(cls).filter(cls.company_id == company_id).first() if com: com.state = state try: session.add(com) session.commit() except Exception as why: message = "companies.on_set_state: {0}".format(why) logging.info(message) return -1 return 0
def add_description(self): log = logging.getLogger(__name__) file = self.request.matchdict['file'] action = self.request.matchdict['action'] assert (action == 'comment') new_description_entry = dict(person=self.request.authenticated_userid, description=self.request.POST['description'], time=datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), file=file) new_description_str = jsonpickle.encode(new_description_entry) try: DBSession.add(FileDescriptionModel(file, new_description_str, self.request.authenticated_userid)) DBSession.commit() except BaseException as e: log.error(str(e)) subreq = self.request.copy() subreq.method = 'GET' return self.request.invoke_subrequest(subreq)
def save_variable_environment(self): log = logging.getLogger(__name__) try: conf_name = self._request.params['name'] conf_variable = self._request.params['data'] job_name = self._request.params['scriptname'] job_path = self._request.params['scriptpath'] user_id = None with transaction.manager: # TODO: if the conf_name is already query = DBSession.query(JobConfiguration).filter(JobConfiguration.configuration_name == conf_name).first() if query is None: DBSession.add(JobConfiguration(conf_name, job_name, job_path, conf_variable, user_id)) else: query.conf_variable = conf_variable DBSession.commit() return dict(error=None) except BaseException as e: log.error(str(e)) raise e
def on_set_state(cls, card_id, state): """ 设置公司状态 """ if state not in ("00", "01", "02"): message = "bank_cards.on_set_state: wrong state: {0}".format(state) logging.info(message) return -1 session = DBSession() card = session.query(cls).filter(cls.card_id == card_id).first() if card: card.state = state try: session.add(card) session.commit() except Exception as why: session.rollback() message = "bank_cards.on_set_state: {0}".format(why) logging.info(message) return -1 return 0
def fillShoe(black): deck = makeDeck() print len(deck)#, deck if black.shoe == "[]": print "Filled" shoe = [] decksInShoe = 6 for i in range(0, decksInShoe): shoe.extend(deck) random.shuffle(shoe) #black.shoe = json.dumps(shoe) black.shoe = shoe DBSession.commit() else: print "Looked up" shoe = black.shoe return DBSession.query(Blackjack).filter_by(id = black.id).first()
def bootstrap_database(): # # Create default admin user # password = "" dbsession = DBSession() if options.setup.lower().startswith('dev'): admin_user = '******' password = '******' else: admin_user = unicode(raw_input(PROMPT + "Admin username: "******"New Admin ") sys.stdout.flush() password1 = getpass.getpass() sys.stdout.write(PROMPT + "Confirm New Admin ") sys.stdout.flush() password2 = getpass.getpass() if password1 == password2 and 12 <= len(password1): password = password1 else: print(WARN + 'Error: Passwords did not match, or were less than 12 chars') sys.exit() user = User(name=admin_user, password=password) dbsession.add(user) dbsession.flush() admin_permission = Permission(name=ADMIN_PERMISSION, user_id=user.id) user.permissions.append(admin_permission) dbsession.add(admin_permission) dbsession.add(user) # # Commit it all # dbsession.commit()
def _initdb(csvfilename, drop_db=False): session = DBSession() from models import Name, Addrobj # if drop_db: # Base.metadata.drop_all(engine) Base.metadata.create_all(engine) reader = csv.reader(open(csvfilename)) # next(reader, None) # skip the headers i = 0 for line in reader: line = [s.replace("'", "''") for s in line] ( actstatus, aoguid, aoid, aolevel, areacode, autocode, centstatus, citycode, code, currstatus, enddate, formalname, ifnsfl, ifnsul, nextid, offname, okato, oktmo, operstatus, parentguid, placecode, plaincode, postalcode, previd, regioncode, shortname, startdate, streetcode, terrifnsfl, terrifnsul, updatedate, ctarcode, extrcode, sextcode, livestatus, normdoc ) = line try: name = Name( name=formalname, name_tsvect=formalname, name_tsquery=formalname) ts_query1 = name.name_tsquery session.add(name) session.commit() except: session.rollback() try: name = Name( name=offname, name_tsvect=offname, name_tsquery=offname) ts_query2 = name.name_tsquery session.add(name) session.commit() except: session.rollback() # Different names can be translated to one name_tsquery # so we search name_tsquery to prevent storing of small differences # in the DB names = session.query(Name).filter(Name.name_tsquery.in_([ ts_query1, ts_query2 ])) names = names.all() place = Addrobj( actstatus=actstatus, aoguid=aoguid, aoid=aoid, aolevel=aolevel, areacode=areacode, autocode=autocode, centstatus=centstatus, citycode=citycode, code=code, currstatus=currstatus, formalname=formalname, ifnsfl=ifnsfl, ifnsul=ifnsul, nextid=nextid, offname=offname, operstatus=operstatus, parentguid=parentguid, placecode=placecode, plaincode=plaincode, postalcode=postalcode, previd=previd, regioncode=regioncode, shortname=shortname, streetcode=streetcode, terrifnsfl=terrifnsfl, terrifnsul=terrifnsul, ctarcode=ctarcode, extrcode=extrcode, sextcode=sextcode, livestatus=livestatus, names=names) session.add(place) session.commit() i += 1 if i % 10000 == 0: print i
def crawl_question(url, cookie, scheduler): #crawl question page #like host: http://www.zhihu.com/question/123456 print 'start to question from url: ', url question_res = requests.get(url, cookies=cookie) with open('pages/question' + re.search(r'\d+', url).group() + '.html', 'wb') as question_file: question_file.write(question_res.content) session = DBSession() if question_res.status_code == 200: question_dom = BeautifulSoup(question_res.content) #find more question if question_dom.find("div", id="zh-question-related-questions"): all_related_ques = question_dom.find("div", id="zh-question-related-questions").find_all("a", class_="question_link") for ques in all_related_ques: ''' new_url = ques.get('href') if ques.get('href').startswith('http') \ else 'http://www.zhihu.com/' + ques.get('href') ''' if ques.get('href').startswith('http'): new_url = ques.get('href') elif ques.get('href').startswith('/'): new_url = 'http://www.zhihu.com' + ques.get('href') else: new_url = 'http://www.zhihu.com/' + ques.get('href') scheduler.add(new_url) #crawl data q_id = re.search(r'\d+', url).group() q_title = question_dom.find("div", id="zh-question-title").find('h2').contents[0].encode('utf-8') q_detail = '' if question_dom.find('div', id='zh-question-detail').find('textarea'): q_detail = question_dom.find('div', id='zh-question-detail').find('textarea').get_text().encode('utf-8') else: q_detail = question_dom.find('div', id='zh-question-detail').get_text().encode('utf-8') q_author = 'default' question = Question(q_id=q_id, title=q_title, author=q_author, content=q_detail) #crawl answer data #a_id author votes content last_modify answer_list = question_dom.find_all("div", class_="zm-item-answer") for answer in answer_list: a_id = answer.get('data-aid') author_h3 = answer.find("h3", class_="zm-item-answer-author-wrap") if author_h3.find_all('a') and len(author_h3.find_all('a'))>1: author = author_h3.find_all('a')[1].text.encode('utf-8') else: #匿名用户 author = author_h3.text.encode('utf-8') votes_div = answer.find("div", class_="zm-votebar") try: votes_span = votes_div.find_all("span") except: print votes_div sys.exit(1) votes = votes_span[1].text if len(votes_span)>1 else 0 ''' content = answer.find("div", class_="zm-editable-content").get_text().encode('utf-8') ''' content_div = answer.find("div", class_='zm-editable-content') if content_div is None: content = "None" else: content = content_div.get_text().encode('utf-8') #last modify date try: last_modify_date = answer.find("a", class_='answer-date-link').text.split()[1] if last_modify_date.index(':') != -1: last_modify_date = datetime.now().strftime("%Y-%m-%d") except: last_modify_date = "2015-06-17" last_modify = datetime.strptime(last_modify_date, "%Y-%m-%d") answer_info = Answer(a_id=a_id, author=author, votes=votes, content=content, last_modify=last_modify) session.add(answer_info) question.answers.append(answer_info) session.add(question) session.commit() session.close() #process_question(question) else: print "Error: ", str(question_res.status_code)