def storeDates(date):#date为datetime格式的日期
Date = leancloud.Object.extend("HistoryDate")
re = Date()
re.set('date', date)
re.save()
print (date)
print ("已经存好啦~~")
def get_time_stamp():
# list = []
# if list:
# print('============1')
# else:
# print('--------------2')
ct = time.time()
local_time = time.localtime(ct)
today = time.strftime("%Y-%m-%d %H:%M:%S", local_time)
print(today)
re = redis.StrictRedis(host='localhost', port=6379, db=0)
task_five_key = "FLASH_TASK_FIVE_" + str(2)
task_five_value = "FLASH_TASK_FIVE_:%s:%s" % (2, today)
data = re.get(task_five_key)
print(data)
if data:
time_value = str(data[-19:])
print("=========", time_value)
else:
re.set(task_five_key, task_five_value)
print(task_five_value)
# if today.strftime('%Y-%m-%d %H:%M:%S') == time_value:
# print('==========================')
today = datetime.date.today()
yesterday = today - datetime.timedelta(days=1)
print(type(yesterday.strftime('%Y-%m-%d')))
print(type(yesterday.strftime('%Y-%m-%d')))
def storeContent(module,title,author,url,imageList,day,itemindex):
#query = Resource.query
#query_list = query.equal_to('identity',i["_id"]).find()
#if query_list==[]:
Resource = leancloud.Object.extend(module)
# type有七种类型，分别存为7个Class格式
re = Resource()
#re.set('identity', i["_id"])
#把string转化为python的时间格式
#%f是微秒格式
publishtime = datetime.datetime.strptime(day, '%Y-%m-%dT%H:%M:%S.%fZ')
re.set('resourcePublished', publishtime)
re.set('title', title)
#re.set('module', module)
re.set('url', url)
# Android类的who字段有一个值为null，因此leanCloud默认存为Object类型,应当自己设置为string类型
re.set('author', author)
if imageList != []:
imageList = changeImg(imageList, day, module,itemindex) # 把图片转为阿里云存储格式
re.set('image', imageList)
re.save()
def build_soap_call(method, arguments, encoding=SOAP_ENCODING,
envelope_attrib=None, typed=None):
""" Builds a soap call.
@param method: method for the soap call. If set to None, the method element
will be omitted and arguments will be added directly to the body (error
message)
@param arguments: arguments for the call
@param encoding: encoding for the call
@param envelope_attrib: envelope attribute
@param typed: True if typed
@type method: string or None
@type arguments: dict or ElementTree.Element
@type encoding: string
@type envelope_attrib: list
@type typed: boolean or None
@return: soap call
@rtype: string
"""
envelope = ElementTree.Element("s:Envelope")
if envelope_attrib:
for n in envelope_attrib:
envelope.attrib.update({n[0]: n[1]})
else:
envelope.attrib.update({'s:encodingStyle':
"http://schemas.xmlsoap.org/soap/encoding/"})
envelope.attrib.update({'xmlns:s':
"http://schemas.xmlsoap.org/soap/envelope/"})
body = ElementTree.SubElement(envelope, "s:Body")
if method:
re = ElementTree.SubElement(body, method)
if encoding:
re.set("%sencodingStyle" % NS_SOAP_ENV, encoding)
else:
re = body
# append the arguments
if isinstance(arguments, dict):
type_map = {str: 'xsd:string',
unicode: 'xsd:string',
int: 'xsd:int',
long: 'xsd:int',
float: 'xsd:float',
bool: 'xsd:boolean'}
for arg_name, arg_val in arguments.iteritems():
arg_type = type_map[type(arg_val)]
if arg_type == 'xsd:string' and type(arg_val) == unicode:
# arg_val = arg_val.encode('utf-8')
# NOTE - if this conditional is allowed to convert to utf-8, the conversion to utf-8 in the
# return call will convert the utf-8 a second time
pass
if arg_type == 'xsd:int' or arg_type == 'xsd:float':
arg_val = str(arg_val)
if arg_type == 'xsd:boolean':
arg_val = '1' if arg_val else '0'
e = ElementTree.SubElement(re, arg_name)
if typed and arg_type:
if not isinstance(type, ElementTree.QName):
arg_type = ElementTree.QName(
"http://www.w3.org/1999/XMLSchema", arg_type)
e.set('%stype' % NS_XSI, arg_type)
e.text = arg_val
else:
re.append(arguments)
preamble = """<?xml version="1.0" encoding="utf-8"?>"""
return '%s%s' % (preamble, ElementTree.tostring(envelope, 'utf-8'))
def build_soap_call_file(method, arguments, encoding=SOAP_ENCODING,
envelope_attrib=None, typed=None):
""" Builds a soap call.
@param method: method for the soap call. If set to None, the method element
will be omitted and arguments will be added directly to the body (error
message)
@param arguments: arguments for the call
@param encoding: encoding for the call
@param envelope_attrib: envelope attribute
@param typed: True if typed
@type method: string or None
@type arguments: dict or ElementTree.Element
@type encoding: string
@type envelope_attrib: list
@type typed: boolean or None
@return: soap call
@rtype: string
"""
envelope = ElementTree.Element("s:Envelope")
if envelope_attrib:
for n in envelope_attrib:
envelope.attrib.update({n[0]: n[1]})
else:
# envelope.attrib.update({'s:encodingStyle':
# "http://schemas.xmlsoap.org/soap/encoding/"})
envelope.attrib.update({'xmlns:s':
"http://schemas.xmlsoap.org/soap/envelope/"})
'''
<s:Header>
<credentials xmlns="http://www.sonos.com/Services/1.1">
<deviceProvider>Sonos</deviceProvider>
</credentials>
</s:Header>
'''
header = ElementTree.SubElement(envelope, "s:Header")
credentials = ElementTree.SubElement(header, 'credentials')
credentials.attrib.update({'xmlns': "http://www.sonos.com/Services/1.1"})
deviceProvider = ElementTree.SubElement(credentials, "deviceProvider")
deviceProvider.text = 'Sonos'
body = ElementTree.SubElement(envelope, "s:Body")
if method:
# RadioTime does not appear to cater for a namespace prefix on the method name
# (note that it returns the default response for a call it can't process, so for getMetadata it returns the root metadata)
if method.startswith('{') and method.rfind('}') > 1:
ns, method_name = method[1:].split('}')
else:
ns = None
method_name = method
re = ElementTree.SubElement(body, method_name)
if ns:
re.attrib.update({'xmlns': ns})
if encoding:
re.set("%sencodingStyle" % NS_SOAP_ENV, encoding)
else:
re = body
# print "~~~~~~~~~~~~~~~~~~~~~~~~"
# print "~~~~~~~~~~~~~~~~~~~~~~~~"
# print "method"
# print method
# print "~~~~~~~~~~~~~~~~~~~~~~~~"
# print "~~~~~~~~~~~~~~~~~~~~~~~~"
# print "arguments"
# print arguments
if isinstance(arguments, dict):
type_map = {str: 'xsd:string',
unicode: 'xsd:string',
int: 'xsd:int',
long: 'xsd:int',
float: 'xsd:float',
bool: 'xsd:boolean'}
# if method == '{http://www.sonos.com/Services/1.1}getMetadata':
# order = ['id', 'index', 'count', 'recursive']
# else:
# order = arguments.keys()
# for arg_name in order:
for arg_name, arg_val in arguments.iteritems():
if arg_name in arguments:
arg_val = arguments[arg_name]
arg_type = type_map[type(arg_val)]
if arg_type == 'xsd:string' and type(arg_val) == unicode:
# arg_val = arg_val.encode('utf-8')
# NOTE - if this conditional is allowed to convert to utf-8, the conversion to utf-8 in the
# return call will convert the utf-8 a second time
pass
if arg_type == 'xsd:int' or arg_type == 'xsd:float':
arg_val = str(arg_val)
if arg_type == 'xsd:boolean':
arg_val = '1' if arg_val else '0'
e = ElementTree.SubElement(re, arg_name)
if typed and arg_type:
if not isinstance(type, ElementTree.QName):
arg_type = ElementTree.QName(
"http://www.w3.org/1999/XMLSchema", arg_type)
e.set('%stype' % NS_XSI, arg_type)
e.text = arg_val
else:
re.append(arguments)
# preamble = """<?xml version="1.0" encoding="utf-8"?>"""
preamble = ""
#TODO: CHANGE THIS BACK?
return '%s%s' % (preamble, ElementTree.tostring(envelope, 'utf-8'))
def search_modern(self, bv, dv, gramm_cat="praet", end_year=1899):
"""Search the modern subcorpus for the contents of a <derivedVerb>.
Parameters
----------
bv (ET.Element): a base verb element (parent of dv)
dv (ET.Element): a derived verb element (child of bv)
gramm_cat (str): grammatical category to search for
end_year (int): limit searches to sources created prior to this year
"""
qu = dv.find(u'query')
if qu is None:
qu = ET.SubElement(dv, u'query')
if qu.get(u'successful') == u'no':
rs = qu.find(u'results')
if rs is None:
rs = ET.SubElement(qu, u'results')
base_verb = bv.get(u'simplex')
pfx_status = dv.get(u'prefixed')
sfx_status = dv.get(u'suffixed')
## get prefix information
pfxe = dv.find(u'prefix')
pfx_name = pfxe.get(u'prefixName')
pfx = pfxe.text
## get suffix information
sfxe = dv.find(u'suffix')
sfx_name = sfxe.get(u'suffixName')
sfx = sfxe.text
## get full verb information
fve = dv.find(u'fullVerb')
full_verb = fve.text
query = RNCQueryModern(
lex1=full_verb.encode('utf-8'),
gramm1=gramm_cat.encode('utf-8'),
end_year=u"{}".format(end_year).encode('utf-8')
)
search = RNCSearch(
rnc_query=query,
subcorpus=u"modern".encode('utf-8'),
pfx_val=pfx_status.encode('utf-8'),
prefix=pfx_name.encode('utf-8'),
sfx_val=sfx_status.encode('utf-8'),
## the next line raises an AttributeError if sfx is None
#suffix=sfx.encode('utf-8'),
suffix=sfx,
lem=full_verb.encode('utf-8'),
gramm_cat=gramm_cat.encode('utf-8'),
base_verb=base_verb.encode('utf-8')
)
search.scrape_pages()
rt = self.add_results(url=search.address)
rs.set(u"expectedDocuments", u"{}".format(rt[0]))
rs.set(u"expectedContexts", u"{}".format(rt[1]))
for d in search.all_search_results:
for i in range(d[13]):
re = ET.SubElement(rs, u'result')
re.set(u'pageIndex', u"{}".format(d[14]))
sn = ET.SubElement(re, u'sourceName')
sn.text = u"{}".format(d[9])
sn.set(u'begDate', u"{}".format(d[10]))
sn.set(u'centerDate', u"{}".format(d[11]))
sn.set(u'endDate', u"{}".format(d[12]))
q = dv.find(u'query')
q.set(u'successful', u'yes')
def build_soap_call(method, arguments, encoding=SOAP_ENCODING,
envelope_attrib=None, typed=None):
""" Builds a soap call.
@param method: method for the soap call. If set to None, the method element
will be omitted and arguments will be added directly to the body (error
message)
@param arguments: arguments for the call
@param encoding: encoding for the call
@param envelope_attrib: envelope attribute
@param typed: True if typed
@type method: string or None
@type arguments: dict or ElementTree.Element
@type encoding: string
@type envelope_attrib: list
@type typed: boolean or None
@return: soap call
@rtype: string
"""
envelope = ElementTree.Element("s:Envelope")
if envelope_attrib:
for n in envelope_attrib:
envelope.attrib.update({n[0]: n[1]})
else:
envelope.attrib.update({'s:encodingStyle':
"http://schemas.xmlsoap.org/soap/encoding/"})
envelope.attrib.update({'xmlns:s':
"http://schemas.xmlsoap.org/soap/envelope/"})
body = ElementTree.SubElement(envelope, "s:Body")
if method:
re = ElementTree.SubElement(body, method)
if encoding:
re.set("%sencodingStyle" % NS_SOAP_ENV, encoding)
else:
re = body
# append the arguments
if isinstance(arguments, dict):
type_map = {str: 'xsd:string',
unicode: 'xsd:string',
int: 'xsd:int',
long: 'xsd:int',
float: 'xsd:float',
bool: 'xsd:boolean'}
for arg_name, arg_val in arguments.iteritems():
arg_type = type_map[type(arg_val)]
if arg_type == 'xsd:string' and type(arg_val) == unicode:
# arg_val = arg_val.encode('utf-8')
# NOTE - if this conditional is allowed to convert to utf-8, the conversion to utf-8 in the
# return call will convert the utf-8 a second time
pass
if arg_type == 'xsd:int' or arg_type == 'xsd:float':
arg_val = str(arg_val)
if arg_type == 'xsd:boolean':
arg_val = '1' if arg_val else '0'
e = ElementTree.SubElement(re, arg_name)
if typed and arg_type:
if not isinstance(type, ElementTree.QName):
arg_type = ElementTree.QName(
"http://www.w3.org/1999/XMLSchema", arg_type)
e.set('%stype' % NS_XSI, arg_type)
e.text = arg_val
else:
re.append(arguments)
preamble = """<?xml version="1.0" encoding="utf-8"?>"""
return '%s%s' % (preamble, ElementTree.tostring(envelope, 'utf-8'))
def build_soap_call_file(method, arguments, encoding=SOAP_ENCODING,
envelope_attrib=None, typed=None):
""" Builds a soap call.
@param method: method for the soap call. If set to None, the method element
will be omitted and arguments will be added directly to the body (error
message)
@param arguments: arguments for the call
@param encoding: encoding for the call
@param envelope_attrib: envelope attribute
@param typed: True if typed
@type method: string or None
@type arguments: dict or ElementTree.Element
@type encoding: string
@type envelope_attrib: list
@type typed: boolean or None
@return: soap call
@rtype: string
"""
envelope = ElementTree.Element("s:Envelope")
if envelope_attrib:
for n in envelope_attrib:
envelope.attrib.update({n[0]: n[1]})
else:
# envelope.attrib.update({'s:encodingStyle':
# "http://schemas.xmlsoap.org/soap/encoding/"})
envelope.attrib.update({'xmlns:s':
"http://schemas.xmlsoap.org/soap/envelope/"})
'''
<s:Header>
<credentials xmlns="http://www.sonos.com/Services/1.1">
<deviceProvider>Sonos</deviceProvider>
</credentials>
</s:Header>
'''
header = ElementTree.SubElement(envelope, "s:Header")
credentials = ElementTree.SubElement(header, 'credentials')
credentials.attrib.update({'xmlns': "http://www.sonos.com/Services/1.1"})
deviceProvider = ElementTree.SubElement(credentials, "deviceProvider")
deviceProvider.text = 'Sonos'
body = ElementTree.SubElement(envelope, "s:Body")
if method:
# RadioTime does not appear to cater for a namespace prefix on the method name
# (note that it returns the default response for a call it can't process, so for getMetadata it returns the root metadata)
if method.startswith('{') and method.rfind('}') > 1:
ns, method_name = method[1:].split('}')
else:
ns = None
method_name = method
re = ElementTree.SubElement(body, method_name)
if ns:
re.attrib.update({'xmlns': ns})
if encoding:
re.set("%sencodingStyle" % NS_SOAP_ENV, encoding)
else:
re = body
# print "~~~~~~~~~~~~~~~~~~~~~~~~"
# print "~~~~~~~~~~~~~~~~~~~~~~~~"
# print "method"
# print method
# print "~~~~~~~~~~~~~~~~~~~~~~~~"
# print "~~~~~~~~~~~~~~~~~~~~~~~~"
# print "arguments"
# print arguments
if isinstance(arguments, dict):
type_map = {str: 'xsd:string',
unicode: 'xsd:string',
int: 'xsd:int',
long: 'xsd:int',
float: 'xsd:float',
bool: 'xsd:boolean'}
# if method == '{http://www.sonos.com/Services/1.1}getMetadata':
# order = ['id', 'index', 'count', 'recursive']
# else:
# order = arguments.keys()
# for arg_name in order:
for arg_name, arg_val in arguments.iteritems():
if arg_name in arguments:
arg_val = arguments[arg_name]
arg_type = type_map[type(arg_val)]
if arg_type == 'xsd:string' and type(arg_val) == unicode:
# arg_val = arg_val.encode('utf-8')
# NOTE - if this conditional is allowed to convert to utf-8, the conversion to utf-8 in the
# return call will convert the utf-8 a second time
pass
if arg_type == 'xsd:int' or arg_type == 'xsd:float':
arg_val = str(arg_val)
if arg_type == 'xsd:boolean':
arg_val = '1' if arg_val else '0'
e = ElementTree.SubElement(re, arg_name)
if typed and arg_type:
if not isinstance(type, ElementTree.QName):
arg_type = ElementTree.QName(
"http://www.w3.org/1999/XMLSchema", arg_type)
e.set('%stype' % NS_XSI, arg_type)
e.text = arg_val
else:
re.append(arguments)
# preamble = """<?xml version="1.0" encoding="utf-8"?>"""
preamble = ""
#TODO: CHANGE THIS BACK?
return '%s%s' % (preamble, ElementTree.tostring(envelope, 'utf-8'))
def wanted():
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:47.0) Gecko/20100101 Firefox/47.0',
}
url="http://waibao.io/projects"
r = requests.get(url, headers=headers, timeout=40)
time.sleep(3)
soup =BeautifulSoup(r.text,"lxml")
items = soup.find_all(class_="card")#每个项目
for item in items:
url = "http://waibao.io"+item.a["href"]
title = item.h4.string
kind = item.find(class_="card-meta").contents[2].strip().strip("/").strip()#类型
price = item.span.get_text().strip()
detail = item.find(class_="card-body").string.strip()
status = item.find(class_="card-footer").get_text().strip()
#存储
Resource = leancloud.Object.extend("Wanted")
# type有七种类型，分别存为7个Class格式
re = Resource()
# re.set('identity', i["_id"])
# 把string转化为python的时间格式
# %f是微秒格式
re.set('url', url)
re.set('title', title)
re.set('kind', kind)
re.set('price', price)
re.set('detail', detail)
re.set('status', status)
re.save()
