def filemonitoring(self):
log = logging.getLogger(__name__)
if self._request.params:
# TODO: add this information to the file
md5_enabled = True if 'withmd5' in self._request.params and self._request.params['withmd5'] == '0' else False
all_files = self._request.params.getall('file')
complete_file, filenames, folders = self._get_monitored_files(self._request.params['folder'] + '/')
with transaction.manager:
for f in all_files:
if f in complete_file:
log.debug('Skipping file {0}, because it is already monitored'.format(f))
continue
(path, filename) = os.path.split(f)
dbobj = MonitoredFile(path, filename, f)
DBSession.add(dbobj)
DBSession.commit()
files_not_mentioned = [c for c in complete_file if c not in all_files]
# TODO: decide on this
log.info('TODO: Still have to decide whether files which are not selected should be deleted or not.'
'Affected files would be: {0}'.format(files_not_mentioned))
else:
log.info('Got an empty request, going to redirect to start page')
subreq = Request.blank('/')
return self._request.invoke_subrequest(subreq)
subreq = Request.blank(self._request.route_path('filebrowser'),
POST=dict(folder=self._request.params['folder'],
currentfolder=self._request.params['currentfolder'],
pathdescription='abs'))
return self._request.invoke_subrequest(subreq)
def write_jobs_to_database(jobs):
"""
Writes the retrieved jobs to the database
:param jobs: list of all jobs
:return:
"""
log = logging.getLogger(__name__)
with transaction.manager:
for (classname, classmembers) in jobs.items():
if len(classmembers) == 0:
continue
for element in classmembers:
if 'error' in element:
continue
if 'JobID' not in element:
log.debug('Row didn''t match specified criteria {0}'.format(element))
continue
if not re.search('[0-9]*', element['JobID']):
log.debug('Row didn''t match specified criteria {0}'.format(element))
continue
dbrow = DBSession.query(Job).filter(Job.id == element['JobID']).all()
json_str = jsonpickle.encode(element)
if len(dbrow) == 0:
j = Job(element['JobID'], json_str)
DBSession.add(j)
elif len(dbrow) == 1:
dbrow[0].jobinfo = json_str
else:
log.error('More than one entry for jobid: {0}'.format(json_str))
DBSession.commit()
def on_set_base_info(cls, company_id, company_name=None, country=None, city=None, address=None, timezone=None,
currency=None, note=None):
""" 设置公司基础信息 """
session = DBSession()
com = session.query(cls).filter(cls.company_id == company_id).first()
if com:
if company_name:
com.company_name = company_name
if country:
com.country = country
if city:
com.city = city
if address:
com.address = address
if timezone:
com.timezone = timezone
if currency:
com.currency = currency
if note:
com.note = note
try:
session.add(com)
session.commit()
except Exception as why:
session.rollback()
message = "companies.on_set_base_info: {0}".format(why)
logging.info(message)
return -1
return 0
def job_output(self):
log = logging.getLogger(__name__)
jobid = self._request.matchdict['jobid']
log.info('Request job output for id:{0}'.format(jobid))
ssh_holder = self._request.registry.settings['ssh_holder']
ssh_jobmanager = SSHBasedJobManager(ssh_holder)
jobs = self._get_current_jobs(ssh_jobmanager, self._coltitles)
joboutput = ssh_jobmanager.get_job_output(jobid)
if 'error' in joboutput and joboutput['error'] is None:
db_joboutput = JobOutput(id=jobid, output=jsonpickle.encode(joboutput))
DBSession.add(db_joboutput)
DBSession.commit()
if 'error' in joboutput and joboutput['error'] is not None:
jobresult = joboutput['error']
else:
jobresult = joboutput['content']
jobresult = ''.join(jobresult)
if 'type' not in joboutput:
joboutput['type'] = 'type field missing'
return {'project': self._projectname,
'jobs': jobs,
'output': dict(jobid=jobid, output=jobresult, type=joboutput['type'])}
def _cleanup(self):
now = time()
session = DBSession()
session.query(Data).filter(
now - Data.timestamp > TIMEOUT_THRESHOLD).delete()
session.commit()
session.close()
def getUser(request):
username = request.headers["X-Webauth-User"]
#now get the uuid
uuid = ""
print "UserName:"******"UUID:", uuid
if uuid == "":
user = DBSession.query(Owners).filter_by(username=username).all()
else:
user = DBSession.query(Owners).filter_by(uuid=uuid).all()
if len(user) == 0:
cur = Owners(username, uuid)
DBSession.add(cur)
DBSession.commit()
user = DBSession.query(Owners).filter_by(username=username).all()
print "Made"
if len(user) == 1:
print "Found"
cur = user[0]
# if user == "zemon1":
# user = "******"
return cur
def crawl_plat_first_letter(shuju_date="2020-01-062020-01-12"):
"""
平台成交数据 https://shuju.wdzj.com/platdata-1.html
"""
url = "https://shuju.wdzj.com/plat-data-custom.html"
form_data = {
"type": 0,
"shujuDate": shuju_date
}
response = requests.post(url, data=form_data)
status = response.status_code
if status != 200:
print("crawl failed. (status is not 200)")
raise CrawlFailed('crawl failed')
plats_data = response.json()
for plat_data in plats_data:
plat_id = plat_data.get('platId')
wdzj_id = plat_data.get('wdzjPlatId')
first_letter = plat_data.get('firstLetter')
session = DBSession()
if wdzj_id != 0:
product = session.query(Product).filter_by(plat_id=plat_id).first()
product.first_letter = first_letter
session.commit()
session.close()
def sentiment_data(self, url, common_list):
"""
Does the sentiment analysis with wit.ai
and saves the sentiment data into the database.
"""
query_str = urllib.parse.quote_plus(' '.join(common_list)[:280])
resp = yield self.fetch_url(
'https://api.wit.ai/message?v=20180703&q={}'.format(query_str),
headers={
'Authorization':
'Bearer {WIT_AUTH_KEY}'.format(WIT_AUTH_KEY=WIT_AUTH_KEY)
})
ents = json.loads(resp['body'])['entities']
sentiment = ents and ents.get('sentiment')
if sentiment:
sentiment = sentiment[0]['value']
s_obj = DBSession.query(UrlSentiment).filter(
UrlSentiment.sentiment == sentiment)
if s_obj.count() == 0:
s_obj = UrlSentiment(url=url, sentiment=sentiment)
DBSession.add(s_obj)
DBSession.commit()
raise gen.Return(s_obj)
raise gen.Return(False)
class LagouPipeline(object):
def __init__(self):
self.session = DBSession()
self.pool = redis.Redis(host='raspberrypi', port=6379, db=2)
def process_item(self, item, spider):
if isinstance(item, LagouItem):
if self.session.query(Jobs).filter(
Jobs.positionId == item['positionId'],
Jobs.companyId == item['companyId']).first():
pass
else:
obj = Jobs(
companyId=item['companyId'],
positionId=item['positionId'],
jobNature=item['jobNature'],
companyName=item['companyName'],
financeStage=item['financeStage'],
companyFullName=item['companyFullName'],
companySize=item['companySize'],
industryField=item['industryField'],
positionName=item['positionName'],
city=item['city'],
createTime=item['createTime'],
salary_low=item['salary_low'],
salary_high=item['salary_high'],
workYear=item['workYear'],
education=item['education'],
positionAdvantage=item['positionAdvantage'],
district=item['district'],
# uid=item['uid'],
companyLabelList=item['companyLabelList'],
)
self.session.add(obj)
try:
self.session.commit()
except Exception, e:
print e
self.session.rollback()
# self.session.close()
elif isinstance(item, CompanyItem):
# 公司信息存入mysql数据库
'''
obj=Company(
companyId=item['companyId'],
companyName=item['companyFullName']
)
self.session.add(obj)
try:
self.session.commit()
except Exception, e:
print e
self.session.rollback()
'''
# 公司的数据存入redis
self.pool.set(item['companyId'], item['companyFullName'])
def create_blog_api():
user = g.get('user', None)
if user is None:
return redirect(url_for('login'))
else:
if user.admin:
name = request.form['name'].encode('utf8')
summary = request.form['summary'].encode('utf8')
content = request.form['content'].encode('utf8')
user_id = request.form['user_id'].encode('utf8')
user_name = request.form['user_name'].encode('utf8')
user_image = request.form['user_image'].encode('utf8')
sess = DBSession()
blog = Blogs(user_id=user_id,
user_name=user_name,
user_image=user_image,
name=name,
summary=summary,
content=content)
sess.add(blog)
sess.commit()
sess.close()
return 'ok'
else:
return redirect(url_for('login'))
def delete(self, catid):
session = DBSession()
category = session.query(models.Category).filter_by(id=catid).first()
for item in category.items:
session.delete(item)
session.delete(category)
session.commit()
def crawl_plat_overview(first_letter):
# url = "https://www.wdzj.com/dangan/pjs/"
url = "https://www.wdzj.com/dangan/{first_letter}/".format(
first_letter=first_letter
)
print("crawl plat {}".format(first_letter))
response = requests.get(url, headers=HEADERS)
if response.status_code != 200:
print('crawl failed: code: {}, url: {}'.format(response.status_code, url))
return
# raise CrawlFailed('crawl failed!')
encode_response(response)
html = etree.HTML(response.text)
try:
plat_name = html.xpath("//div[@class='title']/h1|h2")[0].text
print("plat name: {}".format(plat_name))
# 注册资金(实缴资金) 银行存管 投标保障
# box = html.xpath("//div[@class='zzfwbox'] | //div[@class='bgbox-bt zzfwbox']")
try:
zczj = html.xpath("//div[@class='zzfwbox']/dl[1]/dd[1]//div[@class='r']")[0].text.strip().split()
except IndexError:
zczj = html.xpath("// div[ @class ='bgbox-bt zzfwbox'] // dl[1] / dd[1] // div[@ class ='r']")[0].text.strip().split()
# // div[ @class ='bgbox-bt zzfwbox'] // dl[1] / dd[1] // div[@ class ='r']
if len(zczj) == 2:
zczj_value, sjzj = zczj
sjzj_value = sjzj.strip('（）').split('：')[1]
else:
zczj_value = zczj
sjzj_value = '-'
try:
yhcg_value = html.xpath("//div[@class='zzfwbox']/dl[1]/dd[2]//div[@class='r']")[0].text.strip()
except IndexError:
yhcg_value = html.xpath("//div[ @class ='bgbox-bt zzfwbox']//dl[1]/dd[2]//div[@class='r']")[0].text.strip()
try:
tbbz_value = html.xpath("//div[@class='zzfwbox']/dl[2]/dd[3]//div[@class='r']")[0].text.strip()
except IndexError:
tbbz_value = html.xpath("//div[ @class ='bgbox-bt zzfwbox']//dl[2]/dd[3]//div[@class='r']")[0].text.strip()
plat_overview = dict(
plat_name=plat_name,
zhucezijin=zczj_value,
shijiaozijin=sjzj_value,
yinhangcunguan=yhcg_value,
toubiaobaozhang=tbbz_value
)
except AttributeError as ex:
print('crawl failed: ex: {}, url: {}'.format(str(ex), url))
raise
except IndexError as ex:
print('crawl failed: ex: {}, url: {}'.format(str(ex), url))
raise
new_plat_overview = PlatOverview(**plat_overview)
session = DBSession()
session.add(new_plat_overview)
session.commit()
session.close()
def clear_db():
"""On teardown, remove all the db stuff"""
db = DBSession()
db.query(Payment).delete()
db.query(Trade).delete()
db.query(Product).delete()
db.query(LogTrade).delete()
db.commit()
def save_data(self, data):
session = DBSession()
temp = data.copy()
temp['timestamp'] = time()
new_record = Data(**temp)
session.add(new_record)
session.commit()
session.close()
def update(self, grid, data):
session = DBSession()
session.query(Person).filter(Person.grid == grid).update(data)
try:
session.commit()
except Exception as e:
log.debug("Ocorreu um erro ao salvar as configurações !\n" + e)
session.rollback()
def write_job_archive_config(configuration):
"""
Writes a new job archive configuration to the database
:param configuration:
:return:
"""
with transaction.manager:
DBSession.add(ArchiveUserConfiguration(jsonpickle.encode(configuration)))
DBSession.commit()
def subtract_money(player_out, amount):
out_player = Player.query.filter(Player.id == int(player_out)).first()
movement_out = Movement(player=out_player, amount=amount, move_type=Movement.TYPES['OUT'])
DBSession.add(movement_out)
out_player.update_balance(operation_type=Movement.TYPES['OUT'], amount=amount)
DBSession.flush()
DBSession.commit()
def add_money(player_in, amount):
in_player = Player.query.filter(Player.id == int(player_in)).first()
movement_in = Movement(player=in_player, amount=amount, move_type=Movement.TYPES['IN'])
DBSession.add(movement_in)
in_player.update_balance(operation_type=Movement.TYPES['IN'], amount=amount)
DBSession.flush()
DBSession.commit()
def update_patient(id):
content = request.get_json(silent=True)
session = DBSession()
patient = session.query(Patient).filter_by(id=id).one()
for key, value in content.iteritems():
setattr(patient, key, value)
stmt = session.add(patient)
session.commit()
session.close()
return jsonify()
def create_user(request):
data = json.loads(request.body)
new_user = User(
uid=data['username']
)
new_user.created_at = datetime.now()
if DBSession.query(User).filter(User.name == new_user.name).count() > 0:
raise HTTPUnauthorized()
DBSession.add(new_user)
DBSession.commit()
return {"success": True}
def post(self, catid):
session = DBSession()
args = parser.parse_args()
if 'name' not in args.keys():
abort(400, message="`name` missing")
if len(args['name']) == 0:
abort(400, message="`name` empty")
category = session.query(models.Category).filter_by(id=catid).first()
item = models.Item(category=category, name=args['name'])
session.add(item)
session.commit()
def add_players_command(num_of_players):
'''num_of_players: Numero de jogadores'''
for p in range(int(num_of_players)):
player_name = 'Jogador {}'.format(p + 1)
player = Player(player_name=player_name)
DBSession.add(player)
player = Player(player_name='Banqueiro', balance=float(1000000.00))
DBSession.add(player)
DBSession.flush()
DBSession.commit()
def eventDelete(request):
eventId = int(request.matchdict['event_id'])
event = DBSession.query(Event).filter(Event.id == eventId).first()
if event == None:
return {'status' : 'Event Not Found'}
DBSession.delete(event)
DBSession.commit()
return {'status' : 'Deleted'}
def change_avatar(p_id=None, avatar=None):
try:
res = {"success": False, "msg": ""}
session = DBSession()
user_profile = session.query(UserProfile).filter_by(id=p_id).first()
user_profile.avatar = avatar
session.commit()
res["success"] = True
return res
except Exception as e:
res["msg"] = str(e)
return res
def save_options_to_db(self):
try:
transaction.begin()
Options.reset()
dbsession = DBSession()
options = Options(self.options['timeon'], self.options['timeoff'], self.options['speed'])
dbsession.add(options)
dbsession.commit()
transaction.commit()
return True
except:
return False
def put(self, catid):
session = DBSession()
category = session.query(models.Category).filter_by(id=catid).first()
args = parser.parse_args()
if args['name'] and len(args['name']):
category.name = args['name']
if args['order'] and len(args['order']):
neworder = args['order']
other = session.query(models.Category).filter_by(order=neworder).first()
if other:
other.order = category.order
category.order = neworder
session.commit()
def delete(self, grid):
session = DBSession()
session.query(Person).filter(Person.grid == grid).update({'flag': 'D'})
try:
session.commit()
log.info("Comando executado com sucesso !")
log.info(session)
return session
except Exception as e:
log.debug("Ocorreu um erro ao salvar as configurações !\n" +
e.__str__())
session.rollback()
return None
def save_values_to_db(self):
try:
transaction.begin()
Texts.reset()
for k,v in self.items.iteritems():
dbsession = DBSession()
texts = Texts(v['text'], v['color'], v['effect'], v['effect_time'])
dbsession.add(texts)
dbsession.commit()
transaction.commit()
return True
except:
return False
def on_remove_admin_user(cls, company_id):
""" 删除公司管理员用户 """
session = DBSession()
com = session.query(cls).filter(cls.company_id == company_id).first()
if com:
com.admin_user_id = None
try:
session.add(com)
session.commit()
except Exception as why:
message = "companies.on_remove_admin_user: {0}".format(why)
logging.info(message)
return -1
return 0
def add_label(self):
log = logging.getLogger(__name__)
file = self.request.matchdict['file']
action = self.request.matchdict['action']
assert (action == 'comment')
try:
DBSession.add(FileLabelModel(file, self.request.POST['label']))
DBSession.commit()
except BaseException as e:
log.error(e.message)
subreq = self.request.copy()
subreq.method = 'GET'
return self.request.invoke_subrequest(subreq)
def delete_blog_api():
user = g.get('user', None)
blog_id = request.args.get('id', '')
if user is None:
return redirect(url_for('login'))
else:
if user.admin:
sess = DBSession()
blog = sess.query(Blogs).filter(Blogs.id == blog_id).delete()
sess.commit()
sess.close()
else:
return redirect(url_for('login'))
return 'ok'
def add(cls, name, url):
ret = True
session = DBSession()
record = Action(name, url)
session.add(record)
try:
session.commit()
session.close()
logging.info('add action success<name=%s, url=%s>' % (name, url))
except IntegrityError as error:
logging.error(str(error))
ret = False
return ret
def create_patient():
content = request.get_json(silent=True)
session = DBSession()
session.expire_on_commit = False
patient = Patient()
if content and ("mac_address" in content):
patient.mac_address = content["mac_address"].upper()
stmt = session.add(patient)
print(patient.id)
session.commit()
id = patient.id
print(patient.id)
session.close()
return jsonify(patient.id)
def populate(comps='comps-f16', do_dependencies=True):
from yum.comps import Comps
session = DBSession()
c = Comps()
c.add('comps/%s.xml' % comps)
for group in c.groups:
g = Group(id=group.groupid,
name=group.name,
description=group.description)
session.add(g)
for package in group.packages:
p = session.query(Package).filter_by(
name=to_unicode(package)).first()
if not p:
p = Package(name=package)
session.add(p)
p.group = g
session.flush()
root = Root(name=u'Fedora')
session.add(root)
session.flush()
for category in c.categories:
c = Category(id=category.categoryid,
name=category.name,
description=category.description)
session.add(c)
root.categories.append(c)
for group in category.groups:
g = session.query(Group).filter_by(
group_id=to_unicode(group)).first()
if not g:
print "Cannot find group: %s" % group
else:
g.category = c
session.flush()
if do_dependencies:
for package in session.query(Package).all():
add_dependencies(package, session)
session.commit()
def add():
init_db()
c1 = Customer('Anna', 'Smith', '*****@*****.**',
datetime.datetime(1990, 5, 1))
c2 = Customer('John', 'Brown', '*****@*****.**',
datetime.datetime(1992, 5, 1))
print(c1)
a1 = Account(c1)
a2 = Account(c2)
db = DBSession().get()
db.add(c1)
db.add(c2)
db.add(a1)
db.add(a2)
db.commit()
def vote_article(self, artId):
data = {
"userId": self.userId,
"accessToken": self.accessToken,
"artId": artId
}
r = requests.post(VOTE_ARTICLE_API, data, verify=False)
result = r.json()
vote = VoteRecord(artId=artId,
result=result['res'],
message=result['resMsg'])
session = DBSession()
session.add(vote)
session.commit()
session.close()
def create_db_object(obj, search_for=None, **kwargs):
# type: (Base, dict, dict) -> Base
new_obj = None
if search_for:
new_obj = find_db_object(obj, **search_for)
# print(f"searching for {obj}: {search_for}\n{bool(new_obj)}")
if not new_obj:
# print(f"Creating new {obj}")
new_obj = obj(**kwargs)
DBSession.add(new_obj)
DBSession.commit()
return new_obj
def on_set_admin_user(cls, company_id, user_id):
""" 设置公司管理用户 """
session = DBSession()
com = session.query(cls).filter(cls.company_id == company_id).first()
if com:
com.admin_user_id = user_id
try:
session.add(com)
session.commit()
except Exception as why:
session.rollback()
message = "companies.on_set_admin_user: {0}".format(why)
logging.info(message)
return -1
return 0
def create_message(request):
data = json.loads(request.body)
cur_user = data['username']
user_id = DBSession.query(User).filter_by(uid=cur_user).one().id
data["owner_id"] = user_id
target_query = DBSession.query(User).filter(
User.uid == cur_user)
if target_query.count() > 0:
target = target_query.first()
if target.uid == cur_user or cur_user.admin:
m = Message.from_dict(data)
#m = Message()
DBSession.add(m)
DBSession.commit()
return m.to_dict()
raise HTTPUnauthorized
def post(self):
session = DBSession()
args = parser.parse_args()
if 'name' not in args.keys():
abort(400, message="`name` missing")
if len(args['name']) == 0:
abort(400, message="`name` empty")
query = session.query(models.Category).order_by(models.Category.order.desc())
if query.count():
order = query.first().order + 1
else:
order = 0
category = models.Category(name=args['name'], order=order)
session.add(category)
session.commit()
def on_set_business_hours(cls, company_id, opening_hours, closing_hours):
""" 设置公司营业时间 """
session = DBSession()
com = session.query(cls).filter(cls.company_id == company_id).first()
if com:
com.opening_hours = opening_hours
com.closing_hours = closing_hours
try:
session.add(com)
session.commit()
except Exception as why:
session.rollback()
message = "companies.on_set_business_hours: {0}".format(why)
logging.info(message)
return -1
return 0
def update(self):
jobs = self.get_all_jobs()
JobDatabaseWrapper.write_jobs_to_database(jobs)
for item, elements in jobs.items():
for elem in elements:
jobdetail = self.get_job_details(elem['JobID'])
output = self.get_job_output(elem['JobID'])
job = DBSession.query(Job).filter(Job.id == jobdetail['JobID']).first()
if job is not None:
job.jobdetails = jsonpickle.encode(jobdetail)
DBSession.commit()
joboutput = JobOutput(elem['JobID'], jsonpickle.encode(output))
DBSession.add(joboutput)
DBSession.commit()
return jobs
def addLocationDB(locName, roomNum, locOwner):
exists = DBSession.query(Location).filter_by(owner=locOwner.id,
name=locName,
roomNumber=roomNum).all()
if len(exists) == 0:
tempLoc = Location(owner.id, name, roomNum)
DBSession.add(cur)
DBSession.commit()
loc = DBSession.query(Location).filter_by(id=tempLoc.id).all()
print "Made Loc"
if len(exists) == 1:
print "Found Loc"
loc = exists[0]
return loc
def save_to_log(from_who='user',
message_type=None,
message=None,
comment_text='',
msg_text=''):
"""Сохранить в лог. Внимательно передавать from_who
from_who - 'bot', 'user', 'system'. От кого сообщение
message - тип message. Сообщение от пользователя.
comment_text - дополнительный текст.
msg_text - текст сообщения. Использовать для сохранения ответа бота на message пользователя
Примеры.
save_to_log('user', message) - сохранить сообщение от пользователя.
save_to_log('system', comment_text=err_text) - сохранить сообщение от системы. Например, об ошибке.
save_to_log('bot', message=message_from_user, msg_text=bot_msg_text) - сохранить сообщение от бота пользоателю.
"""
if from_who not in ('bot', 'user', 'system'):
comment_text += ' ' + from_who
from_who = 'need_help'
operation = None
tid = None
session = DBSession()
if message:
tid = message.from_user.id
if from_who == 'user':
if message.content_type == 'text':
msg_text = message.text
if message.content_type == 'contact':
msg_text = str(message.contact)
operation = session.query(Operation).filter_by(telegramid=tid).first()
if operation is None: operation = Operation()
log = Log(datetime=datetime.datetime.now(),
from_who=from_who,
user_id=tid,
msg_text=msg_text,
msg_type=message_type,
operation=operation.current_operation,
status=operation.operation_status,
additional_info=operation.additional_info_db,
function=inspect.stack()[1][3],
comment=comment_text)
session.add(log)
session.commit()
session.close()
def crawl_problem_plats():
"""
问题平台 https://shuju.wdzj.com/problem-1.html
"""
url = "https://shuju.wdzj.com/problem-list-all.html"
params = {"year": ""}
response = requests.get(url, params=params, headers=HEADERS)
json_data = response.json()
problem_plats = json_data.get('problemList')
for problem_plat in problem_plats:
session = DBSession()
plat_id = problem_plat.get('platId')
wdzj_id = problem_plat.get('wdzjPlatId')
plat_name = problem_plat.get('platName')
if wdzj_id != 0:
session.execute(
"""
INSERT INTO products
(plat_id, wdzj_id, name)
select
'{plat_id}', '{wdzj_id}', '{plat_name}'
WHERE not EXISTS (SELECT *
FROM products
WHERE plat_id = '{plat_id}');
""".format(
plat_id=plat_id, wdzj_id=wdzj_id, plat_name=plat_name
)
)
new_problem_plat = ProblemPlat(
plat_id=problem_plat.get('platId'), # plat_id
wdzj_id=problem_plat.get('wdzjPlatId'), # wdzj_id
plat_name=problem_plat.get('platName'), # plat_name
area=problem_plat.get('area'), # 地区
oneline_time=problem_plat.get('onlineTime'), # 上线时间
problem_date=problem_plat.get('problemTime'), # 问题时间
event_type=problem_plat.get('type'), # 事件类型
people_num=problem_plat.get('peopleNumber'),
status1=problem_plat.get('status1'), # 保留字段status1
status2=problem_plat.get('status2') # 保留字段status2
)
session.add(new_problem_plat)
session.commit()
session.close()
def start_game_command(num_of_players):
'''num_of_players: Numero de jogadores'''
click.secho("Iniciando novo jogo com {} jogadores\n".format(num_of_players), fg='yellow', bold=True)
initialisedb()
with click.progressbar(range(int(num_of_players))) as players:
for p in players:
player_name = 'Jogador {}'.format(p + 1)
click.secho(' Inserindo jogador {0}'.format(player_name), fg='blue')
player = Player(player_name=player_name)
DBSession.add(player)
player = Player(player_name='Banqueiro', balance=float(1000000.0))
DBSession.add(player)
DBSession.flush()
DBSession.commit()
click.secho('[OK]', fg='green', bold=True)
def topicUnfollow(request):
currentUser = int(authenticated_userid(request))
topicId = int(request.matchdict['topic_id'])
followed = DBSession.query(UserFollowedTopic).\
filter(and_(UserFollowedTopic.topic_id == topicId,UserFollowedTopic.user_id == currentUser)).\
first()
if followed == None:
return {'status' : 'Not Following'}
DBSession.delete(followed)
DBSession.flush()
DBSession.commit()
return {'status' : 'Unfollowed'}
def process_item(self, item, spider):
session = DBSession()
defaults = {
'Immobilientyp': u'Büros, Gewerbeflächen',
'Boersen_ID': 21,
'Vermarktungstyp': u'kaufen',
'Land': u'Deutschland'}
item = defaults.update(item)
try:
quoka_item = Quoka(**defaults)
session.add(quoka_item)
session.commit()
except Exception as e:
print e
return item
def job_details(self):
log = logging.getLogger(__name__)
details_jobid = self._request.matchdict['jobid']
log.info('Request details for jobid {0}'.format(details_jobid))
ssh_holder = self._request.registry.settings['ssh_holder']
ssh_jobmanager = SSHBasedJobManager(ssh_holder)
jobs = self._get_current_jobs(ssh_jobmanager, self._coltitles)
return_details = ssh_jobmanager.get_job_details(details_jobid)
# write details to the database
job = DBSession.query(Job).filter(Job.id == details_jobid).first()
if job is not None:
job.jobdetails = jsonpickle.encode(return_details)
DBSession.commit()
return {'project': self._projectname,
'jobs': jobs,
'details': return_details}
def delete(cls, id):
ret = True
session = DBSession()
actions = session.query(Action).filter(Action.id == id).all()
if actions:
try:
session.delete(actions[0])
session.commit()
session.close()
except IntegrityError as error:
logging.error(error)
ret = False
else:
logging.error('Action not found')
ret = False
return ret
def getUserAndBlackjack(uName, uGuid=""):
curUser = None
curBlk = None
#if Guid is empty use the username
if uGuid == "":
userExist = DBSession.query(Users).filter_by(userName = uName).all()
else:
userExist = DBSession.query(Users).filter_by(userGuid = uGuid).all()
#If the user doesnt exist make one
if len(userExist) == 0:
curUser = Users(uName, userGuid=uGuid)
DBSession.add(curUser)
DBSession.commit()
userExist = DBSession.query(Users).filter_by(userName = uName).all()
print "Made"
#If they do, retrieve it
if len(userExist) == 1: # or curUser != None:
print "Found"
curUser = userExist[0]
userBlk = DBSession.query(Blackjack).filter_by(userId = curUser.id).all()
#If a game doesnt exist make one
if len(userBlk) == 0:
#Make a game
curBlk = Blackjack(curUser.id)
#add the game
DBSession.add(curBlk)
#commit it
DBSession.commit()
curBlk = DBSession.query(Blackjack).filter_by(userId = curUser.id).first()
print "Made"
elif len(userBlk) == 1:
print "Found"
curBlk = userBlk[0]
if curUser == None or curBlk == None:
print "Had to recurse, Should never need this"
return getUserAndBlackjack(uName, uGuid)
return [curUser, curBlk]
def populate(comps='comps-f16', do_dependencies=True):
from yum.comps import Comps
session = DBSession()
c = Comps()
c.add('comps/%s.xml' % comps)
for group in c.groups:
g = Group(id=group.groupid, name=group.name, description=group.description)
session.add(g)
for package in group.packages:
p = session.query(Package).filter_by(name=to_unicode(package)).first()
if not p:
p = Package(name=package)
session.add(p)
p.group = g
session.flush()
root = Root(name=u'Fedora')
session.add(root)
session.flush()
for category in c.categories:
c = Category(id=category.categoryid, name=category.name,
description=category.description)
session.add(c)
root.categories.append(c)
for group in category.groups:
g = session.query(Group).filter_by(group_id=to_unicode(group)).first()
if not g:
print "Cannot find group: %s" % group
else:
g.category = c
session.flush()
if do_dependencies:
for package in session.query(Package).all():
add_dependencies(package, session)
session.commit()
def on_set_state(cls, company_id, state):
""" 设置公司状态 """
if state not in ("00", "01", "02"):
message = "companies.on_set_state: wrong state: {0}".format(state)
logging.info(message)
return -1
session = DBSession()
com = session.query(cls).filter(cls.company_id == company_id).first()
if com:
com.state = state
try:
session.add(com)
session.commit()
except Exception as why:
message = "companies.on_set_state: {0}".format(why)
logging.info(message)
return -1
return 0
def add_description(self):
log = logging.getLogger(__name__)
file = self.request.matchdict['file']
action = self.request.matchdict['action']
assert (action == 'comment')
new_description_entry = dict(person=self.request.authenticated_userid,
description=self.request.POST['description'],
time=datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
file=file)
new_description_str = jsonpickle.encode(new_description_entry)
try:
DBSession.add(FileDescriptionModel(file, new_description_str, self.request.authenticated_userid))
DBSession.commit()
except BaseException as e:
log.error(str(e))
subreq = self.request.copy()
subreq.method = 'GET'
return self.request.invoke_subrequest(subreq)
def save_variable_environment(self):
log = logging.getLogger(__name__)
try:
conf_name = self._request.params['name']
conf_variable = self._request.params['data']
job_name = self._request.params['scriptname']
job_path = self._request.params['scriptpath']
user_id = None
with transaction.manager:
# TODO: if the conf_name is already
query = DBSession.query(JobConfiguration).filter(JobConfiguration.configuration_name == conf_name).first()
if query is None:
DBSession.add(JobConfiguration(conf_name, job_name, job_path, conf_variable, user_id))
else:
query.conf_variable = conf_variable
DBSession.commit()
return dict(error=None)
except BaseException as e:
log.error(str(e))
raise e
def on_set_state(cls, card_id, state):
""" 设置公司状态 """
if state not in ("00", "01", "02"):
message = "bank_cards.on_set_state: wrong state: {0}".format(state)
logging.info(message)
return -1
session = DBSession()
card = session.query(cls).filter(cls.card_id == card_id).first()
if card:
card.state = state
try:
session.add(card)
session.commit()
except Exception as why:
session.rollback()
message = "bank_cards.on_set_state: {0}".format(why)
logging.info(message)
return -1
return 0
def fillShoe(black):
deck = makeDeck()
print len(deck)#, deck
if black.shoe == "[]":
print "Filled"
shoe = []
decksInShoe = 6
for i in range(0, decksInShoe):
shoe.extend(deck)
random.shuffle(shoe)
#black.shoe = json.dumps(shoe)
black.shoe = shoe
DBSession.commit()
else:
print "Looked up"
shoe = black.shoe
return DBSession.query(Blackjack).filter_by(id = black.id).first()
def bootstrap_database():
#
# Create default admin user
#
password = ""
dbsession = DBSession()
if options.setup.lower().startswith('dev'):
admin_user = '******'
password = '******'
else:
admin_user = unicode(raw_input(PROMPT + "Admin username: "******"New Admin ")
sys.stdout.flush()
password1 = getpass.getpass()
sys.stdout.write(PROMPT + "Confirm New Admin ")
sys.stdout.flush()
password2 = getpass.getpass()
if password1 == password2 and 12 <= len(password1):
password = password1
else:
print(WARN +
'Error: Passwords did not match, or were less than 12 chars')
sys.exit()
user = User(name=admin_user, password=password)
dbsession.add(user)
dbsession.flush()
admin_permission = Permission(name=ADMIN_PERMISSION, user_id=user.id)
user.permissions.append(admin_permission)
dbsession.add(admin_permission)
dbsession.add(user)
#
# Commit it all
#
dbsession.commit()
def _initdb(csvfilename, drop_db=False):
session = DBSession()
from models import Name, Addrobj
# if drop_db:
# Base.metadata.drop_all(engine)
Base.metadata.create_all(engine)
reader = csv.reader(open(csvfilename))
# next(reader, None) # skip the headers
i = 0
for line in reader:
line = [s.replace("'", "''") for s in line]
(
actstatus,
aoguid,
aoid,
aolevel,
areacode,
autocode,
centstatus,
citycode,
code,
currstatus,
enddate,
formalname,
ifnsfl,
ifnsul,
nextid,
offname,
okato,
oktmo,
operstatus,
parentguid,
placecode,
plaincode,
postalcode,
previd,
regioncode,
shortname,
startdate,
streetcode,
terrifnsfl,
terrifnsul,
updatedate,
ctarcode,
extrcode,
sextcode,
livestatus,
normdoc
) = line
try:
name = Name(
name=formalname,
name_tsvect=formalname,
name_tsquery=formalname)
ts_query1 = name.name_tsquery
session.add(name)
session.commit()
except:
session.rollback()
try:
name = Name(
name=offname,
name_tsvect=offname,
name_tsquery=offname)
ts_query2 = name.name_tsquery
session.add(name)
session.commit()
except:
session.rollback()
# Different names can be translated to one name_tsquery
# so we search name_tsquery to prevent storing of small differences
# in the DB
names = session.query(Name).filter(Name.name_tsquery.in_([
ts_query1, ts_query2
]))
names = names.all()
place = Addrobj(
actstatus=actstatus,
aoguid=aoguid,
aoid=aoid,
aolevel=aolevel,
areacode=areacode,
autocode=autocode,
centstatus=centstatus,
citycode=citycode,
code=code,
currstatus=currstatus,
formalname=formalname,
ifnsfl=ifnsfl,
ifnsul=ifnsul,
nextid=nextid,
offname=offname,
operstatus=operstatus,
parentguid=parentguid,
placecode=placecode,
plaincode=plaincode,
postalcode=postalcode,
previd=previd,
regioncode=regioncode,
shortname=shortname,
streetcode=streetcode,
terrifnsfl=terrifnsfl,
terrifnsul=terrifnsul,
ctarcode=ctarcode,
extrcode=extrcode,
sextcode=sextcode,
livestatus=livestatus,
names=names)
session.add(place)
session.commit()
i += 1
if i % 10000 == 0:
print i
def crawl_question(url, cookie, scheduler):
#crawl question page
#like host: http://www.zhihu.com/question/123456
print 'start to question from url: ', url
question_res = requests.get(url, cookies=cookie)
with open('pages/question' + re.search(r'\d+', url).group() + '.html', 'wb') as question_file:
question_file.write(question_res.content)
session = DBSession()
if question_res.status_code == 200:
question_dom = BeautifulSoup(question_res.content)
#find more question
if question_dom.find("div", id="zh-question-related-questions"):
all_related_ques = question_dom.find("div",
id="zh-question-related-questions").find_all("a", class_="question_link")
for ques in all_related_ques:
'''
new_url = ques.get('href') if ques.get('href').startswith('http') \
else 'http://www.zhihu.com/' + ques.get('href')
'''
if ques.get('href').startswith('http'):
new_url = ques.get('href')
elif ques.get('href').startswith('/'):
new_url = 'http://www.zhihu.com' + ques.get('href')
else:
new_url = 'http://www.zhihu.com/' + ques.get('href')
scheduler.add(new_url)
#crawl data
q_id = re.search(r'\d+', url).group()
q_title = question_dom.find("div",
id="zh-question-title").find('h2').contents[0].encode('utf-8')
q_detail = ''
if question_dom.find('div',
id='zh-question-detail').find('textarea'):
q_detail = question_dom.find('div',
id='zh-question-detail').find('textarea').get_text().encode('utf-8')
else:
q_detail = question_dom.find('div',
id='zh-question-detail').get_text().encode('utf-8')
q_author = 'default'
question = Question(q_id=q_id, title=q_title, author=q_author, content=q_detail)
#crawl answer data
#a_id author votes content last_modify
answer_list = question_dom.find_all("div", class_="zm-item-answer")
for answer in answer_list:
a_id = answer.get('data-aid')
author_h3 = answer.find("h3", class_="zm-item-answer-author-wrap")
if author_h3.find_all('a') and len(author_h3.find_all('a'))>1:
author = author_h3.find_all('a')[1].text.encode('utf-8')
else:
#匿名用户
author = author_h3.text.encode('utf-8')
votes_div = answer.find("div", class_="zm-votebar")
try:
votes_span = votes_div.find_all("span")
except:
print votes_div
sys.exit(1)
votes = votes_span[1].text if len(votes_span)>1 else 0
'''
content = answer.find("div",
class_="zm-editable-content").get_text().encode('utf-8')
'''
content_div = answer.find("div", class_='zm-editable-content')
if content_div is None:
content = "None"
else:
content = content_div.get_text().encode('utf-8')
#last modify date
try:
last_modify_date = answer.find("a", class_='answer-date-link').text.split()[1]
if last_modify_date.index(':') != -1:
last_modify_date = datetime.now().strftime("%Y-%m-%d")
except:
last_modify_date = "2015-06-17"
last_modify = datetime.strptime(last_modify_date, "%Y-%m-%d")
answer_info = Answer(a_id=a_id, author=author,
votes=votes, content=content, last_modify=last_modify)
session.add(answer_info)
question.answers.append(answer_info)
session.add(question)
session.commit()
session.close()
#process_question(question)
else:
print "Error: ", str(question_res.status_code)
