def save_item(item, name):
if not item:
return
try:
cursor = _conn.cursor()
#record last_crawl time
item['add_time'] = get_epoch_datetime()
item['update_time'] = get_epoch_datetime()
_upsert_item(cursor, item)
_conn.commit()
except Exception, e:
log_error(e)
def adapt(self, item):
super(BeiHangItemAdapter, self).adapt(item)
if item.has_key('content'):
self._get_detail(item)
item['content'] = _adapt_content_str(item['content'])
item['post_time'] = get_epoch_datetime()
return item
def report_link(source, catetory, link, description=''):
try:
cursor = _conn.cursor()
insert_sql = "INSERT INTO %s (source, category, link, description, create_time) VALUES('%s', '%s', '%s', '%s', %s)" % \
(_link_monitor_table, source, catetory, link, description, get_epoch_datetime())
cursor.execute(insert_sql)
_conn.commit()
except MySQLdb.Error, e:
log_error(e)
def adapt(self, item):
super(USTBItemAdapter, self).adapt(item)
if item.has_key('address'):
item['address'] = self._get_address(item['address'])
if item.has_key('begin_time'):
item['begin_time'], item['end_time'] = self._get_time(item['begin_time'])
if item.has_key('content'):
item['content'] = _adapt_content_str(item['content'])
item['post_time'] = get_epoch_datetime()
return item
def adapt(self, item):
super(self.__class__, self).adapt(item)
if item.has_key('address'):
item['address'] = item['address'].strip()
if item.has_key('begin_time'):
item['begin_time'] = _adapt_datetime_str(item.get('begin_time').replace(u'\xa0\xa0', ' '))
if item.has_key('post_time'):
item['post_time'] = self._get_post_time(item.get('post_time'))
else:
item['post_time'] = get_epoch_datetime()
if item.has_key('content'):
item['content'] = _adapt_content_str(item.get('content'))
item['end_time'] = 0
return item
def _adapt_datetime_str(datetime_str):
if not datetime_str:
return None;
date = datetime.strptime(datetime_str, "%Y-%m-%d %H:%M")
return get_epoch_datetime(date)
