def robust_date_parser(d):
"""
Robust wrapper around some date parsing libs, making a best effort to return
a single 8601 date from the input string. No range checking is performed, and
any date other than the first occuring will be ignored.
We use timelib for its ability to make at least some sense of invalid dates,
e.g. 2012/02/31 -> 2012/03/03
We rely only on dateutil.parser for picking out dates from nearly arbitrary
strings (fuzzy=True), but at the cost of being forgiving of invalid dates
in those kinds of strings.
Returns None if it fails
"""
dd = dateparser.to_iso8601(d)
if dd is None or out_of_range(d):
try:
dd = dateutil_parse(d, fuzzy=True, default=DEFAULT_DATETIME)
if dd.year == DEFAULT_DATETIME.year:
dd = None
except Exception:
try:
dd = timelib.strtodatetime(d, now=DEFAULT_DATETIME_SECS)
except ValueError:
pass
except Exception as e:
logger.error(e)
if dd:
ddiso = dd.isoformat()
return ddiso[:ddiso.index('T')]
return dd
def robust_date_parser(d):
"""
Robust wrapper around some date parsing libs, making a best effort to return
a single 8601 date from the input string. No range checking is performed, and
any date other than the first occuring will be ignored.
We use timelib for its ability to make at least some sense of invalid dates,
e.g. 2012/02/31 -> 2012/03/03
We rely only on dateutil.parser for picking out dates from nearly arbitrary
strings (fuzzy=True), but at the cost of being forgiving of invalid dates
in those kinds of strings.
Returns None if it fails
"""
dd = dateparser.to_iso8601(d)
if dd is None or out_of_range(d):
try:
dd = dateutil_parse(d, fuzzy=True, default=DEFAULT_DATETIME)
if dd.year == DEFAULT_DATETIME.year:
dd = None
except Exception:
try:
dd = timelib.strtodatetime(d, now=DEFAULT_DATETIME_SECS)
except ValueError:
pass
except Exception as e:
logger.error(e)
if dd:
ddiso = dd.isoformat()
return ddiso[:ddiso.index('T')]
return dd
def date_cast(date):
'''
Casts any object into a datetime.datetime object
date
any datetime, time string representation...
'''
if date is None:
return datetime.datetime.now()
elif isinstance(date, datetime.datetime):
return date
# fuzzy date
try:
if isinstance(date, salt._compat.string_types):
try:
if HAS_TIMELIB:
return timelib.strtodatetime(date)
except ValueError:
pass
# not parsed yet, obviously a timestamp?
if date.isdigit():
date = int(date)
else:
date = float(date)
return datetime.datetime.fromtimestamp(date)
except Exception as e:
if HAS_TIMELIB:
raise ValueError('Unable to parse {0}'.format(date))
raise RuntimeError('Unable to parse {0}.'
' Consider installing timelib'.format(date))
def parse_tweet(self, tweet):
item = JsonItem()
item['_id'] = hashlib.md5(tweet['id_str']).hexdigest()
item['url'] = 'https://twitter.com/' + tweet['user']['screen_name'] + '/' + '/status/' + tweet['id_str']
item['create_time'] = int(time.mktime(timelib.strtodatetime(tweet['created_at']).timetuple()))
item['raw'] = tweet
return item
def main(arguments):
client = FioBank(token=arguments['--token'])
if '<month>' in arguments:
month = strtodatetime(arguments['<month>'])
month_range = calendar.monthrange(month.year, month.month)
month_start = datetime.date(month.year, month.month, month_range[0])
month_end = datetime.date(month.year, month.month, month_range[1])
transactions = client.period(month_start, month_end)
else:
transactions = client.period(arguments['<from>'], arguments['<to>'])
print table_header()
for transaction in transactions:
try:
transaction['recipient_message']
except KeyError:
transaction['recipient_message'] = ''
try:
transaction['variable_symbol']
except KeyError:
transaction['variable_symbol'] = ''
print "||%s\t||%s\t||%s\t||%s\t||%s\t||" % (transaction['date'], transaction['user_identifiaction'], transaction['recipient_message'], transaction['variable_symbol'], transaction['amount'])
def date_cast(date):
'''
Casts any object into a datetime.datetime object
date
any datetime, time string representation...
'''
if date is None:
return datetime.datetime.now()
elif isinstance(date, datetime.datetime):
return date
# fuzzy date
try:
if isinstance(date, salt._compat.string_types):
try:
if HAS_TIMELIB:
return timelib.strtodatetime(date)
except ValueError:
pass
# not parsed yet, obviously a timestamp?
if date.isdigit():
date = int(date)
else:
date = float(date)
return datetime.datetime.fromtimestamp(date)
except Exception as e:
if HAS_TIMELIB:
raise ValueError('Unable to parse {0}'.format(date))
raise RuntimeError('Unable to parse {0}.'
' Consider installing timelib'.format(date))
def modifier_date(value):
try:
value = value.replace('_', ' ')
if is_year(value):
return datetime.datetime(int(value), 1, 1)
dt = timelib.strtodatetime(value.encode('utf8'))
return datetime.datetime(dt.year, dt.month, dt.day)
except Exception:
raise SearchError("Invalid date format: '%s'" % value)
def get_isoduration(date_str):
"""convert the given date_str string into an iso 8601 duration"""
iso_duration = None
if not date_str:
return None
# first, is it already a valid isoduration?
try:
isodate.parse_duration(date_str)
return date_str
except isodate.ISO8601Error, e:
# if not, try to parse it
try:
delta = timelib.strtodatetime(date_str) - timelib.strtodatetime("now")
iso_duration = isodate.duration_isoformat(delta)
except Exception, e:
log.msg(e.message, level=log.WARNING)
return None
def parse_tweet(self, tweet):
item = JsonItem()
item['_id'] = hashlib.md5(tweet['id_str']).hexdigest()
item['url'] = 'https://twitter.com/' + tweet['user'][
'screen_name'] + '/' + '/status/' + tweet['id_str']
item['create_time'] = int(
time.mktime(
timelib.strtodatetime(tweet['created_at']).timetuple()))
item['raw'] = tweet
return item
def get_isoduration(date_str):
"""convert the given date_str string into an iso 8601 duration"""
iso_duration = None
if not date_str:
return None
#first, is it already a valid isoduration?
try:
isodate.parse_duration(date_str)
return date_str
except isodate.ISO8601Error, e:
# if not, try to parse it
try:
delta = (timelib.strtodatetime(date_str) -
timelib.strtodatetime('now'))
iso_duration = isodate.duration_isoformat(delta)
except Exception, e:
log.msg(e.message, level=log.WARNING)
return None
def parse_time(when_str):
import timelib
from dateutil.tz import tzlocal
when = timelib.strtodatetime(when_str)
# Timezone information gets ignored, and the resulting datetime is
# timezone-unaware. However the compstate needs timezone data to be
# present.
# Assume that the user wants their current timezone.
when = when.replace(tzinfo=tzlocal())
return when
def total_by_source(days_back=None):
q = [
{"$group": {'_id': "$source", 'totalItems': {"$sum": 1}}},
{"$sort": {'_id': 1}}
]
if days_back:
days_back = timelib.strtodatetime("%d days ago" % days_back)
q.insert(0, {"$match": {'ts': {"$gte": days_back}}})
totals = conn_coll.aggregate(q)
if not totals:
return False
return totals['result']
def sina_status_parser(status):
if status is None:
return
_status = dict()
if status.get("deleted", None):
_status["wid"] = status["id"]
_status["deleted"] = 1
else:
_status["wid"] = status["id"]
_status["text"] = status["text"]
_status["created_at"] = timelib.strtodatetime(status["created_at"])
_status["url"] = "http://weibo.com/" + str(status["user"]["id"]) + "/" + base62.get_url(status["mid"])
_status["original_pic"] = status.get("original_pic", "")
_status["reposts_count"] = status.get("reposts_count", 0)
_status["comments_count"] = status.get("comments_count", 0)
return _status
def robust_date_parser(d):
"""
Robust wrapper around some date parsing libs, making a best effort to return
a single 8601 date from the input string. No range checking is performed, and
any date other than the first occuring will be ignored.
We use timelib for its ability to make at least some sense of invalid dates,
e.g. 2012/02/31 -> 2012/03/03
We rely only on dateutil.parser for picking out dates from nearly arbitrary
strings (fuzzy=True), but at the cost of being forgiving of invalid dates
in those kinds of strings.
Returns None if it fails
"""
# Function for a formatted date string, since datetime.datetime.strftime()
# only works with years >= 1900.
return_date = lambda d: "%d-%02d-%02d" % (d.year, d.month, d.day)
# Check for EDTF timestamp first, because it is simple.
if edtf_date_and_time.match(d):
try:
dateinfo = dateutil_parse(d)
return return_date(dateinfo)
except TypeError:
# not parseable by dateutil_parse()
dateinfo = None
isodate = dateparser.to_iso8601(d)
if isodate is None or out_of_range(d):
try:
dateinfo = dateutil_parse(d, fuzzy=True, default=DEFAULT_DATETIME)
if dateinfo.year == DEFAULT_DATETIME.year:
dateinfo = None
except Exception:
try:
dateinfo = timelib.strtodatetime(d, now=DEFAULT_DATETIME_SECS)
except ValueError:
dateinfo = None
except Exception as e:
logger.error("Exception %s in %s" % (e, __name__))
if dateinfo:
return return_date(dateinfo)
return isodate
def robust_date_parser(d):
"""
Robust wrapper around some date parsing libs, making a best effort to
return a single 8601 date from the input string. No range checking is
performed, and any date other than the first occuring will be ignored.
We use timelib for its ability to make at least some sense of invalid
dates, e.g. 2012/02/31 -> 2012/03/03
We rely only on dateutil.parser for picking out dates from nearly arbitrary
strings (fuzzy=True), but at the cost of being forgiving of invalid dates
in those kinds of strings.
Returns None if it fails
"""
# Function for a formatted date string, since datetime.datetime.strftime()
# only works with years >= 1900.
return_date = lambda d: "%d-%02d-%02d" % (d.year, d.month, d.day)
# Check for EDTF timestamp first, because it is simple.
if edtf_date_and_time.match(d):
try:
dateinfo = dateutil_parse(d)
return return_date(dateinfo)
except TypeError:
# not parseable by dateutil_parse()
dateinfo = None
isodate = dateparser.to_iso8601(d)
if isodate is None or out_of_range(d):
try:
dateinfo = dateutil_parse(d, fuzzy=True, default=DEFAULT_DATETIME)
if dateinfo.year == DEFAULT_DATETIME.year:
dateinfo = None
except Exception:
try:
dateinfo = timelib.strtodatetime(d, now=DEFAULT_DATETIME_SECS)
except ValueError:
dateinfo = None
except Exception as e:
logger.error("Exception %s in %s" % (e, __name__))
if dateinfo:
return return_date(dateinfo)
return isodate
def get_isodate(date_str):
"""convert the given date_str string into an iso 8601 date"""
iso_date = None
if not date_str:
return None
# first, is it already a valid isodate?
try:
isodate.parse_date(date_str)
return date_str
except isodate.ISO8601Error, e:
# if not, try to parse it
try:
iso_date = isodate.date_isoformat(timelib.strtodatetime(date_str))
except Exception, e:
log.msg(e.message, level=log.WARNING)
return None
def get_isodate(date_str):
"""convert the given date_str string into an iso 8601 date"""
iso_date = None
if not date_str:
return None
#first, is it already a valid isodate?
try:
isodate.parse_date(date_str)
return date_str
except isodate.ISO8601Error, e:
# if not, try to parse it
try:
iso_date = isodate.date_isoformat(timelib.strtodatetime(date_str))
except Exception, e:
log.msg(e.message, level=log.WARNING)
return None
def sina_user_parser(user):
if user is None:
return
if user:
status_user = {
"uid": user["id"],
"screen_name": user["screen_name"],
"profile_image_url": user.get("profile_image_url", ""),
"followers_count": user["followers_count"],
"friends_count": user["friends_count"],
"statuses_count": user["statuses_count"],
"created_at": timelib.strtodatetime(user["created_at"]),
"verified": 1 if user.get("verified", 0) else 0,
"verified_type": user.get("verified_type", 0),
"gender": 1 if user.get("gender", "m") == "m" else 0,
"verified_reason": user.get("verified_reason", ""),
}
return status_user
def to_python(self, value):
"""
Validates that the input can be converted to a date. Returns a Python
datetime.date object.
"""
if value in validators.EMPTY_VALUES:
return None
if isinstance(value, datetime.datetime):
return value.date()
if isinstance(value, datetime.date):
return value
try:
result = timelib.strtodatetime(value).date()
except ValueError:
raise ValidationError(self.error_messages['invalid'])
if result.year < 1900:
# strftime can't handle years before 1900
raise ValidationError(self.error_messages['invalid'])
return result
def to_python(self, value): """ Validates that the input can be converted to a date. Returns a Python datetime.date object. """ if value in validators.EMPTY_VALUES: return None if isinstance(value, datetime.datetime): return value.date() if isinstance(value, datetime.date): return value try: result = timelib.strtodatetime(value).date() except ValueError: raise ValidationError(self.error_messages['invalid']) if result.year < 1900: # strftime can't handle years before 1900 raise ValidationError(self.error_messages['invalid']) return result
def followers_crawler(uid):
try:
uids = []
for i in xrange(500):
url = '%s%s?%s' % (
HTTP_PREFIX, FOLLOWERS_INFO, 'uid=%s&count=%d&cursor=%d' % (uid, FOLLOWERS_PAGE_SIZE, i * 200))
# print url
resp = _http_call(url)
followers = resp.get('users', None)
if followers:
logger.info('用户粉丝数量： ' + str(len(followers)))
for follower in followers:
user_params = (follower['id'],
follower['screen_name'],
follower['name'],
follower['province'],
follower['city'],
follower['location'],
follower['description'],
follower['profile_image_url'],
follower['domain'],
follower['gender'],
follower['followers_count'],
follower['friends_count'],
follower['statuses_count'],
follower['favourites_count'],
timelib.strtodatetime(follower['created_at']),
1 if follower['verified'] else 0,
follower['verified_type'],
follower['verified_reason'],
follower['bi_followers_count'],
uid
)
uids.append(follower['id'])
cursor.execute(FOLLOWERS_INSERT_SQL, user_params)
else:
break
return uids
except Exception:
logger.error(traceback.format_exc())
def total_by_source(days_back=None):
q = [{
"$group": {
'_id': "$source",
'totalItems': {
"$sum": 1
}
}
}, {
"$sort": {
'_id': 1
}
}]
if days_back:
days_back = timelib.strtodatetime("%d days ago" % days_back)
q.insert(0, {"$match": {'ts': {"$gte": days_back}}})
totals = conn_coll.aggregate(q)
if not totals:
return False
return totals['result']
def date_cast(date):
'''
Casts any object into a datetime.datetime object
date
any datetime, time string representation...
'''
if date is None:
return datetime.datetime.now()
elif isinstance(date, datetime.datetime):
return date
# fuzzy date
try:
if isinstance(date, six.string_types):
try:
if HAS_TIMELIB:
# py3: yes, timelib.strtodatetime wants bytes, not str :/
return timelib.strtodatetime(
salt.utils.stringutils.to_bytes(date))
except ValueError:
pass
# not parsed yet, obviously a timestamp?
if date.isdigit():
date = int(date)
else:
date = float(date)
return datetime.datetime.fromtimestamp(date)
except Exception: # pylint: disable=broad-except
if HAS_TIMELIB:
raise ValueError('Unable to parse {0}'.format(date))
raise RuntimeError(
'Unable to parse {0}. Consider installing timelib'.format(date))
def __init__(self, shop_id):
self.shop_id = shop_id
self.start = strtodatetime("2016-05-01 00:00:00".encode('utf-8'))
self.end = strtodatetime("2016-05-31 23:59:59".encode('utf-8'))
self.commission_rate = 0.15;
def sina_comments_parser(status):
_status = dict()
_status["id"] = status.id
_status["current_id"] = status.id
_status["_id"] = long(status.id)
_status["text"] = status.text
_status["status_type"] = 2
_status["created_at"] = int(when.parse2Timestamp(timelib.strtodatetime(status.created_at)))
_status_user = dict()
_status_user["profile_image_url"] = status.user.profile_image_url
_status_user["id"] = status.user.id
_status_user["screen_name"] = status.user.screen_name
_status_user["url"] = "http://weibo.com/u/" + str(status.user.id)
_status["user"] = _status_user
commented_status = status.get("status", None)
if commented_status:
_commented_status = dict()
if commented_status.get("deleted", None):
_commented_status["id"] = commented_status.id
_commented_status["_id"] = long(commented_status.id)
_commented_status["text"] = commented_status.text
_commented_status["deleted"] = 1
else:
_commented_status["id"] = commented_status.id
_commented_status["_id"] = long(commented_status.id)
_commented_status["text"] = commented_status.text
_commented_status["created_at"] = int(
when.parse2Timestamp(timelib.strtodatetime(commented_status.created_at))
)
_commented_status["url"] = (
"http://weibo.com/" + str(commented_status.user.id) + "/" + base62.get_url(commented_status.mid)
)
_commented_status["thumbnail_pic"] = commented_status.get("thumbnail_pic", "")
_commented_status["bmiddle_pic"] = commented_status.get("bmiddle_pic", "")
_commented_status["reposts_count"] = commented_status.get("reposts_count", 0)
_commented_status["comments_count"] = commented_status.get("comments_count", 0)
_commented_status_user = dict()
_commented_status_user["profile_image_url"] = commented_status.user.profile_image_url
_commented_status_user["id"] = commented_status.user.id
_commented_status_user["screen_name"] = commented_status.user.screen_name
_commented_status_user["url"] = "http://weibo.com/" + str(commented_status.user.id)
_commented_status["user"] = _commented_status_user
_status["retweeted_status_id"] = _commented_status["_id"]
_status["retweeted_status"] = _commented_status
reply_comment = status.get("reply_comment", None)
if reply_comment:
_reply_comment = dict()
if _reply_comment.get("deleted", None):
_reply_comment["id"] = reply_comment.id
_reply_comment["_id"] = long(reply_comment.id)
_reply_comment["text"] = reply_comment.text
_reply_comment["deleted"] = 1
else:
_reply_comment["id"] = reply_comment.id
_reply_comment["_id"] = long(reply_comment.id)
_reply_comment["text"] = reply_comment.text
_reply_comment["created_at"] = int(when.parse2Timestamp(timelib.strtodatetime(reply_comment.created_at)))
_reply_comment_user = dict()
_reply_comment_user["profile_image_url"] = reply_comment.user.profile_image_url
_reply_comment_user["id"] = reply_comment.user.id
_reply_comment_user["screen_name"] = reply_comment.user.screen_name
_reply_comment_user["url"] = "http://weibo.com/" + str(reply_comment.user.id)
_reply_comment["user"] = _reply_comment_user
_status["reply_comment"] = _reply_comment
_status["reply_comment_id"] = _reply_comment["_id"]
return _status
# fout.write('''<li class="supporters">''' + str(issue['satisfied_supporter_count']) + ' von ' + str(issue['supporter_count']) + ' (' + str(math.ceil((issue['policy_initiative_quorum_num'] / issue['policy_initiative_quorum_den']) * 100 )) + '''% benötigt)</li>''')
fout.write("""<li class="status">""" + lqfb_status[issue["issue_state"]] + """</li>""")
fout.write("""<li class="tags">""" + issue["area_name"] + """</li>""")
t = time.strptime(issue["current_draft_created"], "%Y-%m-%d %H:%M:%S")
fout.write("""<li class="created">""" + prettydate(t) + """ angelegt</li>""")
t = time.strptime(issue["issue_fully_frozen"], "%Y-%m-%d %H:%M:%S")
fout.write("""<li class="empty">""" + prettydate(t) + """ in Abstimmung</li>""")
# there can be problems with initiatives that are voted < 1 day
try:
s = issue["issue_fully_frozen"] + " + " + issue["issue_voting_time"]
t = time.strptime(str(timelib.strtodatetime(bytes(s, "utf-8"))), "%Y-%m-%d %H:%M:%S")
fout.write("""<li class="empty">""" + prettydate(t) + """ geschlossen</li>""")
except:
s = issue["issue_fully_frozen"]
t = time.strptime(str(timelib.strtodatetime(bytes(s, "utf-8"))), "%Y-%m-%d %H:%M:%S")
fout.write("""<li class="empty">""" + prettydate(t) + """ geschlossen</li>""")
fout.write("</ul>")
fout.write(
"""</a>
</li>"""
)
json_object = json.loads(
urllib.request.urlopen("http://opendata.piratenpartei-mv.de/lqfb/mv/status/frozen").read().decode("utf8")
)
extended = ''
keywords = ''
excerpt = ''
elif line_number in [1, 4, 5, 6]:
pass
elif line_number == 2:
#title
title = line[7:-1]
elif line_number == 3 and line[8:] == 'Draft':
status = 'draft'
elif line_number == 7:
basename = line[10:-1]
elif line_number == 9:
try:
date = timelib.strtodatetime(
line[6:]).strftime("%Y-%m-%d %I:%M:%S")
except:
date = '0000-00-00 00:00:00'
elif line_number > 11:
if line == '-----\n':
section = section + 1
elif section == 0:
if line != 'BODY:\n':
body = body + '\n' + line
elif section == 1:
if line != 'EXTENDED BODY:\n':
extended = extended + '\n' + line
elif section == 2:
if line != 'EXCERPT:\n':
excerpt = excerpt + '\n' + line
def tag_countup(node, context):
return timediff(datetime.datetime.utcnow() - timelib.strtodatetime(node.attribute))
def tag_countdown(node, context):
return timediff(timelib.strtodatetime(node.attribute) - datetime.datetime.utcnow())
extended = ''
keywords =''
excerpt = ''
elif line_number in [1,4,5,6]:
pass
elif line_number == 2:
#title
title = line[7:-1]
elif line_number == 3 and line[8:] == 'Draft':
status = 'draft'
elif line_number == 7:
basename = line[10:-1]
elif line_number == 9:
try:
date = timelib.strtodatetime(line[6:]).strftime("%Y-%m-%d %I:%M:%S")
except:
date = '0000-00-00 00:00:00'
elif line_number > 11:
if line == '-----\n' :
section = section + 1
elif section == 0:
if line != 'BODY:\n':
body = body + '\n' + line
elif section == 1:
if line != 'EXTENDED BODY:\n':
extended = extended + '\n' + line
elif section == 2:
if line != 'EXCERPT:\n':
excerpt = excerpt + '\n' + line