def robust_date_parser(d): """ Robust wrapper around some date parsing libs, making a best effort to return a single 8601 date from the input string. No range checking is performed, and any date other than the first occuring will be ignored. We use timelib for its ability to make at least some sense of invalid dates, e.g. 2012/02/31 -> 2012/03/03 We rely only on dateutil.parser for picking out dates from nearly arbitrary strings (fuzzy=True), but at the cost of being forgiving of invalid dates in those kinds of strings. Returns None if it fails """ dd = dateparser.to_iso8601(d) if dd is None or out_of_range(d): try: dd = dateutil_parse(d, fuzzy=True, default=DEFAULT_DATETIME) if dd.year == DEFAULT_DATETIME.year: dd = None except Exception: try: dd = timelib.strtodatetime(d, now=DEFAULT_DATETIME_SECS) except ValueError: pass except Exception as e: logger.error(e) if dd: ddiso = dd.isoformat() return ddiso[:ddiso.index('T')] return dd
def date_cast(date): ''' Casts any object into a datetime.datetime object date any datetime, time string representation... ''' if date is None: return datetime.datetime.now() elif isinstance(date, datetime.datetime): return date # fuzzy date try: if isinstance(date, salt._compat.string_types): try: if HAS_TIMELIB: return timelib.strtodatetime(date) except ValueError: pass # not parsed yet, obviously a timestamp? if date.isdigit(): date = int(date) else: date = float(date) return datetime.datetime.fromtimestamp(date) except Exception as e: if HAS_TIMELIB: raise ValueError('Unable to parse {0}'.format(date)) raise RuntimeError('Unable to parse {0}.' ' Consider installing timelib'.format(date))
def parse_tweet(self, tweet): item = JsonItem() item['_id'] = hashlib.md5(tweet['id_str']).hexdigest() item['url'] = 'https://twitter.com/' + tweet['user']['screen_name'] + '/' + '/status/' + tweet['id_str'] item['create_time'] = int(time.mktime(timelib.strtodatetime(tweet['created_at']).timetuple())) item['raw'] = tweet return item
def main(arguments): client = FioBank(token=arguments['--token']) if '<month>' in arguments: month = strtodatetime(arguments['<month>']) month_range = calendar.monthrange(month.year, month.month) month_start = datetime.date(month.year, month.month, month_range[0]) month_end = datetime.date(month.year, month.month, month_range[1]) transactions = client.period(month_start, month_end) else: transactions = client.period(arguments['<from>'], arguments['<to>']) print table_header() for transaction in transactions: try: transaction['recipient_message'] except KeyError: transaction['recipient_message'] = '' try: transaction['variable_symbol'] except KeyError: transaction['variable_symbol'] = '' print "||%s\t||%s\t||%s\t||%s\t||%s\t||" % (transaction['date'], transaction['user_identifiaction'], transaction['recipient_message'], transaction['variable_symbol'], transaction['amount'])
def modifier_date(value): try: value = value.replace('_', ' ') if is_year(value): return datetime.datetime(int(value), 1, 1) dt = timelib.strtodatetime(value.encode('utf8')) return datetime.datetime(dt.year, dt.month, dt.day) except Exception: raise SearchError("Invalid date format: '%s'" % value)
def get_isoduration(date_str): """convert the given date_str string into an iso 8601 duration""" iso_duration = None if not date_str: return None # first, is it already a valid isoduration? try: isodate.parse_duration(date_str) return date_str except isodate.ISO8601Error, e: # if not, try to parse it try: delta = timelib.strtodatetime(date_str) - timelib.strtodatetime("now") iso_duration = isodate.duration_isoformat(delta) except Exception, e: log.msg(e.message, level=log.WARNING) return None
def parse_tweet(self, tweet): item = JsonItem() item['_id'] = hashlib.md5(tweet['id_str']).hexdigest() item['url'] = 'https://twitter.com/' + tweet['user'][ 'screen_name'] + '/' + '/status/' + tweet['id_str'] item['create_time'] = int( time.mktime( timelib.strtodatetime(tweet['created_at']).timetuple())) item['raw'] = tweet return item
def get_isoduration(date_str): """convert the given date_str string into an iso 8601 duration""" iso_duration = None if not date_str: return None #first, is it already a valid isoduration? try: isodate.parse_duration(date_str) return date_str except isodate.ISO8601Error, e: # if not, try to parse it try: delta = (timelib.strtodatetime(date_str) - timelib.strtodatetime('now')) iso_duration = isodate.duration_isoformat(delta) except Exception, e: log.msg(e.message, level=log.WARNING) return None
def parse_time(when_str): import timelib from dateutil.tz import tzlocal when = timelib.strtodatetime(when_str) # Timezone information gets ignored, and the resulting datetime is # timezone-unaware. However the compstate needs timezone data to be # present. # Assume that the user wants their current timezone. when = when.replace(tzinfo=tzlocal()) return when
def total_by_source(days_back=None): q = [ {"$group": {'_id': "$source", 'totalItems': {"$sum": 1}}}, {"$sort": {'_id': 1}} ] if days_back: days_back = timelib.strtodatetime("%d days ago" % days_back) q.insert(0, {"$match": {'ts': {"$gte": days_back}}}) totals = conn_coll.aggregate(q) if not totals: return False return totals['result']
def sina_status_parser(status): if status is None: return _status = dict() if status.get("deleted", None): _status["wid"] = status["id"] _status["deleted"] = 1 else: _status["wid"] = status["id"] _status["text"] = status["text"] _status["created_at"] = timelib.strtodatetime(status["created_at"]) _status["url"] = "http://weibo.com/" + str(status["user"]["id"]) + "/" + base62.get_url(status["mid"]) _status["original_pic"] = status.get("original_pic", "") _status["reposts_count"] = status.get("reposts_count", 0) _status["comments_count"] = status.get("comments_count", 0) return _status
def robust_date_parser(d): """ Robust wrapper around some date parsing libs, making a best effort to return a single 8601 date from the input string. No range checking is performed, and any date other than the first occuring will be ignored. We use timelib for its ability to make at least some sense of invalid dates, e.g. 2012/02/31 -> 2012/03/03 We rely only on dateutil.parser for picking out dates from nearly arbitrary strings (fuzzy=True), but at the cost of being forgiving of invalid dates in those kinds of strings. Returns None if it fails """ # Function for a formatted date string, since datetime.datetime.strftime() # only works with years >= 1900. return_date = lambda d: "%d-%02d-%02d" % (d.year, d.month, d.day) # Check for EDTF timestamp first, because it is simple. if edtf_date_and_time.match(d): try: dateinfo = dateutil_parse(d) return return_date(dateinfo) except TypeError: # not parseable by dateutil_parse() dateinfo = None isodate = dateparser.to_iso8601(d) if isodate is None or out_of_range(d): try: dateinfo = dateutil_parse(d, fuzzy=True, default=DEFAULT_DATETIME) if dateinfo.year == DEFAULT_DATETIME.year: dateinfo = None except Exception: try: dateinfo = timelib.strtodatetime(d, now=DEFAULT_DATETIME_SECS) except ValueError: dateinfo = None except Exception as e: logger.error("Exception %s in %s" % (e, __name__)) if dateinfo: return return_date(dateinfo) return isodate
def get_isodate(date_str): """convert the given date_str string into an iso 8601 date""" iso_date = None if not date_str: return None # first, is it already a valid isodate? try: isodate.parse_date(date_str) return date_str except isodate.ISO8601Error, e: # if not, try to parse it try: iso_date = isodate.date_isoformat(timelib.strtodatetime(date_str)) except Exception, e: log.msg(e.message, level=log.WARNING) return None
def get_isodate(date_str): """convert the given date_str string into an iso 8601 date""" iso_date = None if not date_str: return None #first, is it already a valid isodate? try: isodate.parse_date(date_str) return date_str except isodate.ISO8601Error, e: # if not, try to parse it try: iso_date = isodate.date_isoformat(timelib.strtodatetime(date_str)) except Exception, e: log.msg(e.message, level=log.WARNING) return None
def sina_user_parser(user): if user is None: return if user: status_user = { "uid": user["id"], "screen_name": user["screen_name"], "profile_image_url": user.get("profile_image_url", ""), "followers_count": user["followers_count"], "friends_count": user["friends_count"], "statuses_count": user["statuses_count"], "created_at": timelib.strtodatetime(user["created_at"]), "verified": 1 if user.get("verified", 0) else 0, "verified_type": user.get("verified_type", 0), "gender": 1 if user.get("gender", "m") == "m" else 0, "verified_reason": user.get("verified_reason", ""), } return status_user
def to_python(self, value): """ Validates that the input can be converted to a date. Returns a Python datetime.date object. """ if value in validators.EMPTY_VALUES: return None if isinstance(value, datetime.datetime): return value.date() if isinstance(value, datetime.date): return value try: result = timelib.strtodatetime(value).date() except ValueError: raise ValidationError(self.error_messages['invalid']) if result.year < 1900: # strftime can't handle years before 1900 raise ValidationError(self.error_messages['invalid']) return result
def followers_crawler(uid): try: uids = [] for i in xrange(500): url = '%s%s?%s' % ( HTTP_PREFIX, FOLLOWERS_INFO, 'uid=%s&count=%d&cursor=%d' % (uid, FOLLOWERS_PAGE_SIZE, i * 200)) # print url resp = _http_call(url) followers = resp.get('users', None) if followers: logger.info('用户粉丝数量: ' + str(len(followers))) for follower in followers: user_params = (follower['id'], follower['screen_name'], follower['name'], follower['province'], follower['city'], follower['location'], follower['description'], follower['profile_image_url'], follower['domain'], follower['gender'], follower['followers_count'], follower['friends_count'], follower['statuses_count'], follower['favourites_count'], timelib.strtodatetime(follower['created_at']), 1 if follower['verified'] else 0, follower['verified_type'], follower['verified_reason'], follower['bi_followers_count'], uid ) uids.append(follower['id']) cursor.execute(FOLLOWERS_INSERT_SQL, user_params) else: break return uids except Exception: logger.error(traceback.format_exc())
def total_by_source(days_back=None): q = [{ "$group": { '_id': "$source", 'totalItems': { "$sum": 1 } } }, { "$sort": { '_id': 1 } }] if days_back: days_back = timelib.strtodatetime("%d days ago" % days_back) q.insert(0, {"$match": {'ts': {"$gte": days_back}}}) totals = conn_coll.aggregate(q) if not totals: return False return totals['result']
def date_cast(date): ''' Casts any object into a datetime.datetime object date any datetime, time string representation... ''' if date is None: return datetime.datetime.now() elif isinstance(date, datetime.datetime): return date # fuzzy date try: if isinstance(date, six.string_types): try: if HAS_TIMELIB: # py3: yes, timelib.strtodatetime wants bytes, not str :/ return timelib.strtodatetime( salt.utils.stringutils.to_bytes(date)) except ValueError: pass # not parsed yet, obviously a timestamp? if date.isdigit(): date = int(date) else: date = float(date) return datetime.datetime.fromtimestamp(date) except Exception: # pylint: disable=broad-except if HAS_TIMELIB: raise ValueError('Unable to parse {0}'.format(date)) raise RuntimeError( 'Unable to parse {0}. Consider installing timelib'.format(date))
def __init__(self, shop_id): self.shop_id = shop_id self.start = strtodatetime("2016-05-01 00:00:00".encode('utf-8')) self.end = strtodatetime("2016-05-31 23:59:59".encode('utf-8')) self.commission_rate = 0.15;
def sina_comments_parser(status): _status = dict() _status["id"] = status.id _status["current_id"] = status.id _status["_id"] = long(status.id) _status["text"] = status.text _status["status_type"] = 2 _status["created_at"] = int(when.parse2Timestamp(timelib.strtodatetime(status.created_at))) _status_user = dict() _status_user["profile_image_url"] = status.user.profile_image_url _status_user["id"] = status.user.id _status_user["screen_name"] = status.user.screen_name _status_user["url"] = "http://weibo.com/u/" + str(status.user.id) _status["user"] = _status_user commented_status = status.get("status", None) if commented_status: _commented_status = dict() if commented_status.get("deleted", None): _commented_status["id"] = commented_status.id _commented_status["_id"] = long(commented_status.id) _commented_status["text"] = commented_status.text _commented_status["deleted"] = 1 else: _commented_status["id"] = commented_status.id _commented_status["_id"] = long(commented_status.id) _commented_status["text"] = commented_status.text _commented_status["created_at"] = int( when.parse2Timestamp(timelib.strtodatetime(commented_status.created_at)) ) _commented_status["url"] = ( "http://weibo.com/" + str(commented_status.user.id) + "/" + base62.get_url(commented_status.mid) ) _commented_status["thumbnail_pic"] = commented_status.get("thumbnail_pic", "") _commented_status["bmiddle_pic"] = commented_status.get("bmiddle_pic", "") _commented_status["reposts_count"] = commented_status.get("reposts_count", 0) _commented_status["comments_count"] = commented_status.get("comments_count", 0) _commented_status_user = dict() _commented_status_user["profile_image_url"] = commented_status.user.profile_image_url _commented_status_user["id"] = commented_status.user.id _commented_status_user["screen_name"] = commented_status.user.screen_name _commented_status_user["url"] = "http://weibo.com/" + str(commented_status.user.id) _commented_status["user"] = _commented_status_user _status["retweeted_status_id"] = _commented_status["_id"] _status["retweeted_status"] = _commented_status reply_comment = status.get("reply_comment", None) if reply_comment: _reply_comment = dict() if _reply_comment.get("deleted", None): _reply_comment["id"] = reply_comment.id _reply_comment["_id"] = long(reply_comment.id) _reply_comment["text"] = reply_comment.text _reply_comment["deleted"] = 1 else: _reply_comment["id"] = reply_comment.id _reply_comment["_id"] = long(reply_comment.id) _reply_comment["text"] = reply_comment.text _reply_comment["created_at"] = int(when.parse2Timestamp(timelib.strtodatetime(reply_comment.created_at))) _reply_comment_user = dict() _reply_comment_user["profile_image_url"] = reply_comment.user.profile_image_url _reply_comment_user["id"] = reply_comment.user.id _reply_comment_user["screen_name"] = reply_comment.user.screen_name _reply_comment_user["url"] = "http://weibo.com/" + str(reply_comment.user.id) _reply_comment["user"] = _reply_comment_user _status["reply_comment"] = _reply_comment _status["reply_comment_id"] = _reply_comment["_id"] return _status
# fout.write('''<li class="supporters">''' + str(issue['satisfied_supporter_count']) + ' von ' + str(issue['supporter_count']) + ' (' + str(math.ceil((issue['policy_initiative_quorum_num'] / issue['policy_initiative_quorum_den']) * 100 )) + '''% benötigt)</li>''') fout.write("""<li class="status">""" + lqfb_status[issue["issue_state"]] + """</li>""") fout.write("""<li class="tags">""" + issue["area_name"] + """</li>""") t = time.strptime(issue["current_draft_created"], "%Y-%m-%d %H:%M:%S") fout.write("""<li class="created">""" + prettydate(t) + """ angelegt</li>""") t = time.strptime(issue["issue_fully_frozen"], "%Y-%m-%d %H:%M:%S") fout.write("""<li class="empty">""" + prettydate(t) + """ in Abstimmung</li>""") # there can be problems with initiatives that are voted < 1 day try: s = issue["issue_fully_frozen"] + " + " + issue["issue_voting_time"] t = time.strptime(str(timelib.strtodatetime(bytes(s, "utf-8"))), "%Y-%m-%d %H:%M:%S") fout.write("""<li class="empty">""" + prettydate(t) + """ geschlossen</li>""") except: s = issue["issue_fully_frozen"] t = time.strptime(str(timelib.strtodatetime(bytes(s, "utf-8"))), "%Y-%m-%d %H:%M:%S") fout.write("""<li class="empty">""" + prettydate(t) + """ geschlossen</li>""") fout.write("</ul>") fout.write( """</a> </li>""" ) json_object = json.loads( urllib.request.urlopen("http://opendata.piratenpartei-mv.de/lqfb/mv/status/frozen").read().decode("utf8") )
extended = '' keywords = '' excerpt = '' elif line_number in [1, 4, 5, 6]: pass elif line_number == 2: #title title = line[7:-1] elif line_number == 3 and line[8:] == 'Draft': status = 'draft' elif line_number == 7: basename = line[10:-1] elif line_number == 9: try: date = timelib.strtodatetime( line[6:]).strftime("%Y-%m-%d %I:%M:%S") except: date = '0000-00-00 00:00:00' elif line_number > 11: if line == '-----\n': section = section + 1 elif section == 0: if line != 'BODY:\n': body = body + '\n' + line elif section == 1: if line != 'EXTENDED BODY:\n': extended = extended + '\n' + line elif section == 2: if line != 'EXCERPT:\n': excerpt = excerpt + '\n' + line
def tag_countup(node, context): return timediff(datetime.datetime.utcnow() - timelib.strtodatetime(node.attribute))
def tag_countdown(node, context): return timediff(timelib.strtodatetime(node.attribute) - datetime.datetime.utcnow())
extended = '' keywords ='' excerpt = '' elif line_number in [1,4,5,6]: pass elif line_number == 2: #title title = line[7:-1] elif line_number == 3 and line[8:] == 'Draft': status = 'draft' elif line_number == 7: basename = line[10:-1] elif line_number == 9: try: date = timelib.strtodatetime(line[6:]).strftime("%Y-%m-%d %I:%M:%S") except: date = '0000-00-00 00:00:00' elif line_number > 11: if line == '-----\n' : section = section + 1 elif section == 0: if line != 'BODY:\n': body = body + '\n' + line elif section == 1: if line != 'EXTENDED BODY:\n': extended = extended + '\n' + line elif section == 2: if line != 'EXCERPT:\n': excerpt = excerpt + '\n' + line