def acquire_lock(self, key, lease_time=3600, **kwargs) -> Lock: lockfile = os.path.join(self.basepath, f'rssalertbot-{key}.lock') # create our release callback def release(): self.release_lock(key) try: self.locks[key] = zc.lockfile.LockFile(lockfile, content_template='{pid};{hostname}') log.debug(f"Acquired lock '{key}' on {lockfile}") return Lock(release, expires = pendulum.now().add(3600)) except zc.lockfile.LockError: # we've already got a lockfile here, so we should check the age of it # - if it's older than the lease time, we can re-acquire, otherwise # nope! stats = os.stat(lockfile) expires = pendulum.from_timestamp(stats.st_mtime) age = pendulum.now() - expires if age > lease_time: log.debug(f"Acquired lock '{key}' on {lockfile}") return Lock(release, expires) # no you can't have this lock log.debug(f"Lock '{key}' denied") raise LockNotAcquired()
def get_durations(log_records): """Calculate visit duration per day per IP""" timezone = cherrypy.engine.publish( "registry:local_timezone" ).pop() maximums = defaultdict(int) minimums = defaultdict(int) for row in log_records: timestamp = row["unix_timestamp"] formatted_timestamp = pendulum.from_timestamp( timestamp ).in_timezone(timezone).format('YYYY-MM-DD') lookup_key = (row["ip"], formatted_timestamp) if lookup_key not in maximums or timestamp > maximums[lookup_key]: maximums[lookup_key] = timestamp continue if lookup_key not in minimums or timestamp < minimums[lookup_key]: minimums[lookup_key] = timestamp durations = { lookup_key: pendulum.duration( seconds=(maximums[lookup_key] - minimums[lookup_key]) ) for lookup_key in maximums if minimums[lookup_key] > 0 } return durations
def save_date(self, feed, date: pendulum.DateTime): """ Save the date for the current event. """ # just in case someone didn't follow the type hints if isinstance(date, datetime.datetime): date = pendulum.from_timestamp(date.timestamp()) datafile = os.path.join(self.basepath, f'last.{feed}.dat') with open(datafile, 'w') as f: f.write(str(date.in_tz('UTC')))
def get(asset, base): log.debug('checking feeds for %s/%s at %s' % (asset, base, NAME)) headers = {'content-type': 'application/json', 'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:22.0) Gecko/20100101 Firefox/22.0'} data = requests.get('http://api.zb.com/data/v1/ticker?market={}_{}'.format(asset.lower(), base.lower()), headers=headers).json() t = data['ticker'] return FeedPrice(float(t['last']), asset, base, volume=float(t['vol']), last_updated=pendulum.from_timestamp(float(data['date']) / 1000), provider=NAME)
def add(self, url=None, title=None, comments=None, tags=None, added=None): """Store a bookmarked URL and its metadata.""" bookmark = self.find(url=url) if bookmark: sql = """UPDATE bookmarks SET title=?, tags=?, comments=?, updated=CURRENT_TIMESTAMP, deleted=NULL WHERE rowid=?""" values = ( title, tags, comments, bookmark["rowid"] ) self._update(sql, [values]) return True sql = """INSERT INTO bookmarks (domain, url, added, added_date, title, tags, comments) VALUES (?, ?, ?, ?, ?, ?, ?)""" domain_and_url = self.domain_and_url(url) if added and added.isnumeric(): numeric_timestamp = int(added) add_date = pendulum.from_timestamp(numeric_timestamp) else: add_date = pendulum.now() values = ( domain_and_url[0], domain_and_url[1], add_date.format('YYYY-MM-DD HH:mm:ss'), add_date.to_date_string(), title, tags, comments ) self._insert(sql, [values]) cherrypy.engine.publish( "scheduler:add", 2, "bookmarks:add:fulltext" ) return True
def get_deltas(log_records): """Calculate elapsed time intervals between records""" deltas = [] for index, row in enumerate(log_records): try: current_timestamp = pendulum.from_timestamp( row["unix_timestamp"] ) previous_timestamp = pendulum.from_timestamp( log_records[index + 1]["unix_timestamp"] ) delta = current_timestamp.diff_for_humans( previous_timestamp, True ) except (KeyError, IndexError): delta = 0 deltas.append(delta) return deltas
def _get_dayname(ts, idx, *, tz=None): """ Returns the day name given a Unix timestamp, day index and (optionally) a timezone. """ if pendulum is not None: p = pendulum.from_timestamp(ts, tz=tz) return p.format('dddd') else: # Fallback if idx == 0: return 'Today' elif idx == 1: return 'Tomorrow' else: return 'Day_%d' % idx
def last_run(self) -> pendulum.Pendulum: from share.models import CeleryProviderTask if self._last_run is not None: if isinstance(self._last_run, int): last_run = pendulum.from_timestamp(self._last_run) else: last_run = pendulum.parse(self._last_run) else: logger.debug('Finding last successful job') last_run = CeleryProviderTask.objects.filter( app_label=self.config.label, status=CeleryProviderTask.STATUS.succeeded, ).order_by( '-timestamp' ).values_list('timestamp', flat=True).first() if last_run: last_run = pendulum.instance(last_run) else: last_run = pendulum.from_timestamp(0) logger.info('Found last job %s', last_run) logger.info('Using last run of %s', last_run) return last_run
def get_all(): feeds = requests.get('https://api.coinmarketcap.com/v1/ticker/').json() result = FeedSet() for f in feeds: try: price = float(f['price_usd']) volume = float(f['24h_volume_usd']) / price if f.get('24h_volume_usd') else None result.append(FeedPrice(price, f['symbol'], 'USD', volume=volume, last_updated=pendulum.from_timestamp(int(f['last_updated'])))) except TypeError as e: # catches: TypeError: float() argument must be a string or a number, not 'NoneType' # on: f['price_usd'] log.debug('Could not get USD price for feed: {}'.format(json.dumps(f, indent=4))) log.exception(e) pass return result
def ago_filter(value): """Calculate a human-readable time delta between a date in the past and now. If the date provided as an integer, it is treated as a unix timestamp. """ date = value if isinstance(value, int): date = pendulum.from_timestamp(value) zone = cherrypy.engine.publish( "registry:local_timezone" ).pop() return date.in_timezone(zone).diff_for_humans()
def localtime_filter(value, timezone=None): """Switch a datetime to the local timezone, then format it""" if not value: return "" if not timezone: timezone = cherrypy.engine.publish( "registry:local_timezone" ).pop() if isinstance(value, (int, float)): value = pendulum.from_timestamp(value) else: value = pendulum.instance(value) return value.in_timezone(timezone)
def get(cur, base): log.debug('checking feeds for %s/%s at %s' % (cur, base, NAME)) try: secret_key = core.config['credentials']['bitcoinaverage']['secret_key'] public_key = core.config['credentials']['bitcoinaverage']['public_key'] except KeyError: raise KeyError('config.yaml does not specify both "credentials.bitcoinaverage.secret_key" and ' '"credentials.bitcoinaverage.public_key" variables') client = RestfulClient(secret_key=secret_key, public_key=public_key) r = client.ticker_short_local()[cur + base] return FeedPrice(float(r['last']), cur, base, last_updated=pendulum.from_timestamp(r['timestamp']), provider=NAME)
async def process(self, timeout=60): """ Fetch and process this feed. Args: timeout (int): HTTP timeout """ self.log.info(f"Begining processing feed {self.name}, previous date {self.previous_date}") new_date = pendulum.datetime(1970, 1, 1, tz='UTC') now = pendulum.now('UTC') for entry in await self.fetch_and_parse(timeout): pubdate = dateutil.parser.parse(entry.published, tzinfos=rssalertbot.BOGUS_TIMEZONES) entry.published = pendulum.from_timestamp(pubdate.timestamp()) # also save a prettified string format entry.datestring = self.format_timestamp_local(entry.published) # store the date from the first entry if entry.published > new_date: new_date = entry.published # skip anything that's stale if entry.published <= now.subtract(days=1): continue # and anything before the previous date if entry.published <= self.previous_date: continue self.log.debug(f"Found new entry {entry.published}") # alert on it await self.alert(entry) if not new_date: new_date = now self.save_date(new_date) self.log.info(f"End processing feed {self.name}, previous date {new_date}")
def handle(self, sender, target, tokens, bot): if len(tokens) < 2: self.send_help(sender, bot) return nick = tokens[1] config = self.get_config(bot) nick_record = config.get(nick.lower()) msgs = [] if nick_record is None: msgs.append('I have not seen {} in {}'.format(nick, bot.c.get('irc:channel'))) else: diff = pendulum.from_timestamp(nick_record['time']).diff_for_humans() msgs.append('I saw {} in {} {}'.format(nick, bot.c.get('irc:channel'), diff)) msgs.append(nick_record['text']) if bot.is_irc_channel(target): response_target = target else: response_target = sender for msg in msgs: bot.send_privmsg(response_target, msg)
def _deserialize(cls, encoded_var: Any) -> Any: """Helper function of depth first search for deserialization.""" # JSON primitives (except for dict) are not encoded. if cls._is_primitive(encoded_var): return encoded_var elif isinstance(encoded_var, list): return [cls._deserialize(v) for v in encoded_var] if not isinstance(encoded_var, dict): raise ValueError( f"The encoded_var should be dict and is {type(encoded_var)}") var = encoded_var[Encoding.VAR] type_ = encoded_var[Encoding.TYPE] if type_ == DAT.DICT: return {k: cls._deserialize(v) for k, v in var.items()} elif type_ == DAT.DAG: return SerializedDAG.deserialize_dag(var) elif type_ == DAT.OP: return SerializedBaseOperator.deserialize_operator(var) elif type_ == DAT.DATETIME: return pendulum.from_timestamp(var) elif type_ == DAT.POD: if not HAS_KUBERNETES: raise RuntimeError( "Cannot deserialize POD objects without kubernetes libraries installed!" ) pod = PodGenerator.deserialize_model_dict(var) return pod elif type_ == DAT.TIMEDELTA: return datetime.timedelta(seconds=var) elif type_ == DAT.TIMEZONE: return decode_timezone(var) elif type_ == DAT.RELATIVEDELTA: return decode_relativedelta(var) elif type_ == DAT.SET: return {cls._deserialize(v) for v in var} elif type_ == DAT.TUPLE: return tuple(cls._deserialize(v) for v in var) else: raise TypeError(f'Invalid type {type_!s} in deserialization.')
def _deserialize(cls, encoded_var: Any) -> Any: # pylint: disable=too-many-return-statements """Helper function of depth first search for deserialization.""" # JSON primitives (except for dict) are not encoded. if cls._is_primitive(encoded_var): return encoded_var elif isinstance(encoded_var, list): return [cls._deserialize(v) for v in encoded_var] if not isinstance(encoded_var, dict): raise ValueError( f"The encoded_var should be dict and is {type(encoded_var)}") var = encoded_var[Encoding.VAR] type_ = encoded_var[Encoding.TYPE] if type_ == DAT.DICT: return {k: cls._deserialize(v) for k, v in var.items()} elif type_ == DAT.DAG: return SerializedDAG.deserialize_dag(var) elif type_ == DAT.OP: return SerializedBaseOperator.deserialize_operator(var) elif type_ == DAT.DATETIME: return pendulum.from_timestamp(var) elif type_ == DAT.POD: pod = PodGenerator.deserialize_model_dict(var) return pod elif type_ == DAT.TIMEDELTA: return datetime.timedelta(seconds=var) elif type_ == DAT.TIMEZONE: return Timezone(var) elif type_ == DAT.RELATIVEDELTA: if 'weekday' in var: var['weekday'] = relativedelta.weekday( *var['weekday']) # type: ignore return relativedelta.relativedelta(**var) elif type_ == DAT.SET: return {cls._deserialize(v) for v in var} elif type_ == DAT.TUPLE: return tuple([cls._deserialize(v) for v in var]) else: raise TypeError( 'Invalid type {!s} in deserialization.'.format(type_))
def calculate_workout_body_part(self, user_id): last_user_workouts = self.db.get_last_user_workouts(user_id) if last_user_workouts.__len__() > 1: recent_workout = last_user_workouts[0] previous_workout = last_user_workouts[1] today = pendulum.now() week_start = today.start_of('week') tz = pendulum.timezone('Europe/Paris') previous_workout_date = pendulum.from_timestamp( previous_workout["workout_date"]) previous_workout_date = tz.convert(previous_workout_date) if week_start <= previous_workout_date: recent_workout_body_part = self.db.select_workout_by_id( recent_workout['workout_id'])['workout'][0]["body_part"] previous_workout_body_part = self.db.select_workout_by_id( previous_workout['workout_id'])['workout'][0]["body_part"] body_parts_array = numpy.arange(4) body_parts_array = body_parts_array[ body_parts_array != recent_workout_body_part] body_parts_array = body_parts_array[ body_parts_array != previous_workout_body_part] if body_parts_array.__len__() == 3: return random.choice(body_parts_array) if body_parts_array.__len__() == 2: if recent_workout_body_part != 0 and previous_workout_body_part != 0: return body_parts_array[body_parts_array != 0][0] else: return random.choice(body_parts_array) else: return 0 else: return 0
def parse_title(page): """Makes a pretty string containing page info.""" bold = "\x02{}\x0F" if Showmore.page_is_scp(page): if page['scp_num']: if page['title']: title_preview = "{}: {}".format(page['scp_num'].upper(), bold.format(page['title'])) else: title_preview = bold.format(page['scp_num'].upper()) else: title_preview = bold.format(page['title']) else: title_preview = bold.format(page['title']) return "{} · {} · {} · {} · {}".format( title_preview, "by " + " & ".join(page['authors']), ("+" if page['rating'] >= 0 else "") + str(page['rating']), pd.from_timestamp(page['date_posted']).diff_for_humans(), "https://scp-wiki.wikidot.com/" + page['fullname'], )
def dmf_list(self): """List resources in the current workspace. *Arguments*: none. """ self._init_required('list') lines = [ '| ID | Name(s) | Type | Modified | Description | ', '| -- | ------- | ---- | -------- | ----------- |' ] for rsrc in self._dmf.find(): msince = pendulum.from_timestamp(rsrc.v['modified']) \ .diff_for_humans() rowstr = '| {id} | {names} | {type} | {mdate} | {desc} |'.format( id=rsrc.id, names=','.join(rsrc.v['aliases']), type=rsrc.type, mdate=msince, desc=rsrc.v['desc']) lines.append(rowstr) listing = '\n'.join(lines) self._dmf_markdown(listing) return True
def __init__(self, session, deployment_id, log_group_names=None, out_file=sys.stderr): self.deployment_id = deployment_id self.status = None self.log_group_names = log_group_names or [] self._client = session.client('codedeploy') self._list_deployment_targets = \ self._client.get_paginator('list_deployment_targets').paginate self._out_file = out_file self._deploy_info = None self._target_ids = None self._target_lifecycle_events = {} self._targets = None self._log_watcher = LogWatcher(session, out_file=out_file) self._last_update_time = pendulum.from_timestamp(0) self._complete_time = None
def get_external_schedule_execution( recon_repo, instance_ref, schedule_name, scheduled_execution_timestamp, scheduled_execution_timezone, ): check.inst_param( recon_repo, "recon_repo", ReconstructableRepository, ) definition = recon_repo.get_definition() schedule_def = definition.get_schedule_def(schedule_name) with DagsterInstance.from_ref(instance_ref) as instance: scheduled_execution_time = ( pendulum.from_timestamp( scheduled_execution_timestamp, tz=scheduled_execution_timezone, ) if scheduled_execution_timestamp else None ) schedule_context = ScheduleExecutionContext(instance, scheduled_execution_time) try: with user_code_error_boundary( ScheduleExecutionError, lambda: "Error occurred during the execution function for schedule " "{schedule_name}".format(schedule_name=schedule_def.name), ): return ExternalScheduleExecutionData.from_execution_data( schedule_def.get_execution_data(schedule_context) ) except ScheduleExecutionError: return ExternalScheduleExecutionErrorData( serializable_error_info_from_exc_info(sys.exc_info()) )
def take_and_send_measurements(sense, connection_status, connection_code, mqttc, mqtt_topic, mqtt_topic_prefix, device_identifier): time_epochmillis = int(time.time() * 1000) now = pendulum.from_timestamp(time_epochmillis / 1000.0, UTC) # see https://pythonhosted.org/sense-hat/api/#environmental-sensors # degrees Celsius temperature_value_raw = sense.get_temperature() # scale the measurement which comes from inside the case of the Raspberry Pi # in order to obtain measurements similar to usual factory floor temperatures temperature_value = (temperature_value_raw * 4.0) - 138.0 # temperature_value = 22.22 # RH percentage relative_humidity_value = sense.get_humidity() # relative_humidity_value = 44.44 # Millibars pressure_value = sense.get_pressure() # pressure_value = 1111.11 send_measurements(sense, connection_status, connection_code, mqttc, mqtt_topic, mqtt_topic_prefix, device_identifier, time_epochmillis, now, temperature_value, relative_humidity_value, pressure_value)
def testCheckHeartbeat(loggingMixin, mocker, inputTimestamps, expectedTimestamps, expectedDeadReceivers): """ Test checking the heartbeat of given subsystems. """ # Setup subsystems # Normally, we could just get the subsystems by calling `el.keys() for el in inputTimestamps`. # Hoewver, specifing the ``mockInputTimestamps`` and ``expectedTimestamps`` properly depends on the # order in which the values are insert, which in turn depends on the order in which the subsystems # are interated over. Py2 dict keys don't preserve order, so these tests will sometimes fail. By # specifying the order in the ``subsystems`` list, we don't have to get the keys from the dict. subsystems = ["EMC", "HLT"] monitor.parameters["subsystemList"] = subsystems # Setup input values # Converts into [emc1, hlt1, emc2, hlt2] so that the return values are in the appropriate order. mockInputTimestamps = [el[subsystem] for el in inputTimestamps for subsystem in subsystems] # Spot check the third element. assert mockInputTimestamps[2] == inputTimestamps[1]["EMC"] # Sanity check to ensure that we haven't missed any subsystems assert set(subsystems) == set(inputTimestamps[1].keys()) # Setup expected values if isinstance(expectedTimestamps, int): logger.debug("Using fixed offset.") offset = expectedTimestamps expectedSource = inputTimestamps else: logger.debug("Using expected timestamps specified in the parametrization.") offset = 0 expectedSource = expectedTimestamps expectedTimestamps = [pendulum.from_timestamp(el[subsystem] + offset, tz = "Europe/Zurich") for el in expectedSource for subsystem in subsystems] # Setup mocks using expected input and output values. mHeartbeat = mocker.MagicMock(side_effect = mockInputTimestamps) mocker.patch("overwatch.receiver.monitor.getHeartbeat", mHeartbeat) mPendulumNow = mocker.MagicMock(side_effect = expectedTimestamps) mocker.patch("overwatch.receiver.monitor.pendulum.now", mPendulumNow) deadReceivers = set() # Make the check as many times as specified. for edr in expectedDeadReceivers: deadReceivers = monitor.checkHeartbeat(deadReceivers) # The assertion here lets us test each step of checking the heartbeat. assert deadReceivers == edr
def execution_time_iterator(self, start_timestamp): check.float_param(start_timestamp, "start_timestamp") timezone_str = ( self.execution_timezone if self.execution_timezone else pendulum.now().timezone.name ) start_datetime = pendulum.from_timestamp(start_timestamp, tz=timezone_str) date_iter = croniter(self.cron_schedule, start_datetime) # Go back one iteration so that the next iteration is the first time that is >= start_datetime # and matches the cron schedule date_iter.get_prev(datetime.datetime) while True: next_date = pendulum.instance(date_iter.get_next(datetime.datetime)).in_tz(timezone_str) # During DST transitions, croniter returns datetimes that don't actually match the # cron schedule, so add a guard here if croniter.match(self.cron_schedule, next_date): yield next_date
def cancel(self, now=False): """ Cancel a subscription """ cancel = self._processor.cancel(self.plan_id, now=now) if cancel: if now: # delete it now subscription = self._get_subscription() subscription.ends_at = pendulum.now() subscription.trial_ends_at = None subscription.save() return True else: # update the ended at date subscription = self._get_subscription() subscription.ends_at = pendulum.from_timestamp( cancel['current_period_end']) subscription.save() return True return False
def test_timespan(session_or_test, get_real_object): obj = session_or_test timespan = get_real_object(obj).timespan assert timespan is not None span_start = timespan.lower assert span_start is not None assert timespan.upper is None assert abs(timespan.lower - pendulum.from_timestamp(obj.start_time).astimezone()) < _MIN_THRESHOLD duration = 10 flux.current_timeline.sleep(duration) # pylint: disable=no-member obj.report_end() timespan = get_real_object(obj).timespan assert timespan is not None assert timespan.upper is not None assert timespan.lower == span_start assert abs((timespan.upper - timespan.lower) - datetime.timedelta(seconds=duration)) < _MIN_THRESHOLD
def _add_calculated_fields(status: CarStatus, initial_status: CarStatus, configuration: Configuration): start_time = pendulum.from_timestamp(configuration.startTime.timestamp(), tz='utc') now = pendulum.now(tz='utc') end_time = start_time.add(hours=configuration.hours) status.start_time = start_time status.end_time = end_time status.start_odometer = initial_status.odometer status.distance = status.odometer - initial_status.odometer if status.odometer and initial_status.odometer else 0 status.time_since_start = pendulum.period( start_time, now, True) if now >= start_time else pendulum.period( now, now, True) status.time_to_end = pendulum.period( now, end_time, True) if now <= end_time else pendulum.period( now, now, True) if _car_laps_list: current_lap = _car_laps_list[-1] status.lap = current_lap.id status.lap_distance = current_lap.distance
def _save_data(self, d): self._update_last_time(d) if not d: log.error("bad data: %s" % (d)) return ERR_BAD_PARAM timestamp = d.get("timestamp") if not timestamp: log.error("bad data without timestamp: %s" % (d)) return ERR_BAD_PARAM cndt = self.cntz.convert(pendulum.from_timestamp(int(timestamp))) datestr = cndt.format("YYYYMMDD") fname = "%s_%s" % (d.get("ip_from"), d.get("ip_to")) dpath = os.path.join(settings.PINGDATA.data_dir, datestr) if not os.path.exists(dpath): log.info("make new dir: %s" % (dpath)) os.makedirs(dpath) fpath = os.path.join(dpath, fname) with open(fpath, "a") as f: f.write(json_.dumps(d)) f.write("\n") return 0 return ERR_FAIL_TO_SAVE
def form_valid(self, form): now = pendulum.now() twenty_four_hours_ago = pendulum.now().subtract(hours=4) participation_metrics_request_datetime = self.request.session.get( 'participation_metrics_request_datetime', 0) if pendulum.from_timestamp( participation_metrics_request_datetime).between( now, twenty_four_hours_ago): messages.add_message( self.request, messages.WARNING, _('Sorry! You can request the participation metrics report once in 24 hours.' )) else: self.request.session[ 'participation_metrics_request_datetime'] = now.int_timestamp form.generate_metrics() messages.add_message( self.request, messages.INFO, _('The participation metrics report will be emailed to you in a couple of hours' )) return super(ParticipationMetricsFormView, self).form_valid(form)
def decode_literal(node, prim): core_type, value = next(iter(node.items())) if prim in ['int', 'nat']: return int(value) if prim == 'timestamp': if core_type == 'int': return pendulum.from_timestamp(int(value)) else: return pendulum.parse(value) if prim == 'mutez': return Decimal(value) / 10**6 if prim == 'bool': return value == 'True' if prim == 'address' and core_type == 'bytes': prefix = { '0000': b'tz1', '0001': b'tz2', '0002': b'tz3' } # TODO: check it's ttr return base58_encode(bytes.fromhex(value[4:]), prefix[value[:4]]).decode() return value
async def get_updates(self, moment: DateTime) -> List[Update]: result = [] self.log(f"Requesting feed from {self.INDEX_URL}...") response, page = await fetch(self.INDEX_URL) rss = feedparser.parse(page) for entry in rss.entries: if "published_parsed" in entry: parsed_timestamp = entry.published_parsed else: parsed_timestamp = entry.updated_parsed published_on = pendulum.from_timestamp(mktime(parsed_timestamp)) if published_on < moment: continue update = await self.process_entry(entry, published_on) if update is not None: result.append(update) return result
def emit(self, record) -> None: # type: ignore # if we shouldn't log to cloud, don't emit if not prefect.context.config.logging.log_to_cloud: return try: from prefect.client import Client if self.client is None: self.client = Client() # type: ignore assert isinstance(self.client, Client) # mypy assert record_dict = record.__dict__.copy() # remove potentially non-json serializable formatting args record_dict.pop("args", None) log = dict() log["flow_run_id"] = prefect.context.get("flow_run_id", None) log["task_run_id"] = prefect.context.get("task_run_id", None) log["timestamp"] = pendulum.from_timestamp( record_dict.pop("created", time.time()) ).isoformat() log["name"] = record_dict.pop("name", None) log["message"] = record_dict.pop("message", None) log["level"] = record_dict.pop("levelname", None) if record_dict.get("exc_text") is not None: log["message"] += "\n" + record_dict.pop("exc_text", "") record_dict.pop("exc_info", None) log["info"] = record_dict self.put(log) except Exception as exc: message = "Failed to write log with error: {}".format(str(exc)) self.logger.critical(message) self.put(self._make_error_log(message))
def populate_db(pair, sample_distance): now = NTPClient.request('time.google.com') pivot_dt = '1007681680000' while 1: print(pivot_dt) # Build url url = params['http_api_url'] + 'candles/trade:{sample_distance}:{pair}/' \ 'hist?limit={limit}&start={pivot_dt}&sort=1' \ .format(sample_distance=sample_distance, pair=pair, limit=10000, pivot_dt=pivot_dt) print(url) # Request API json_response = requests.get(url) response = json.loads(json_response.text) time.sleep(3) if 'error' in response: # Check rate limit if response[1] == ERROR_CODE_RATE_LIMIT: print('Error: reached the limit number of requests. Wait 120 seconds...') time.sleep(120) continue # Check platform status elif response[1] == ERROR_CODE_START_MAINTENANCE: print('Error: platform is in maintenance. Forced to stop all requests.') break else: # Get last timestamp of request (in second, so divided by 1000) print(response) last_dt = int(response[::-1][0][0]) // 1000 print('2') last_dt = pendulum.from_timestamp(last_dt) # Put it as new start datetime (in millisecond, so multiplied by 1000) if pivot_dt == last_dt.int_timestamp * 1000: break pivot_dt = last_dt.int_timestamp * 1000 client.write_api().write(record=serialize_points(response), bucket=pair)
def emit(self, record) -> None: # type: ignore # if we shouldn't log to cloud, don't emit if not prefect.context.config.logging.log_to_cloud: return try: from prefect.client import Client if self.client is None: self.client = Client() # type: ignore assert isinstance(self.client, Client) # mypy assert record_dict = record.__dict__.copy() flow_run_id = prefect.context.get("flow_run_id", None) task_run_id = prefect.context.get("task_run_id", None) timestamp = pendulum.from_timestamp( record_dict.get("created", time.time())) name = record_dict.get("name", None) message = record_dict.get("message", None) level = record_dict.get("levelname", None) if record_dict.get("exc_text") is not None: message += "\n" + record_dict["exc_text"] record_dict.pop("exc_info", None) self.client.write_run_log( flow_run_id=flow_run_id, task_run_id=task_run_id, timestamp=timestamp, name=name, message=message, level=level, info=record_dict, ) except Exception as exc: self.logger.critical("Failed to write log with error: {}".format( str(exc)))
def command(irc_c, msg, cmd): if defer.check(cmd, 'Secretary_Helen'): return cmd.expandargs(["first f", "count c"]) # have to account for .seen -f name if 'first' in cmd: cmd.args['root'].extend(cmd.args['first']) if 'count' in cmd: cmd.args['root'].extend(cmd.args['count']) if len(cmd.args['root']) < 1: raise CommandError("Specify a user and I'll tell you when I last " "saw them") nick = cmd.args['root'][0] messages = DB.get_messages_from_user(nick, msg.raw_channel) if len(messages) == 0: raise MyFaultError("I've never seen {} in this channel." .format(nick)) if 'count' in cmd: msg.reply("I've seen {} {} times in this channel." .format(nick, len(messages))) return if 'first' in cmd: message = messages[0] response = "I first saw {} {} saying: {}" else: if nick == msg.sender: msg.reply("I can see you right now, {}.".format(msg.sender)) return message = messages[-1] response = "I last saw {} {} saying: {}" response = response.format( nick if nick == message['sender'] else "{} as {}".format(nick, message['sender']), pd.from_timestamp(message['timestamp']).diff_for_humans(), gib.obfuscate(message['message'], DB.get_channel_members(msg.raw_channel))) msg.reply(response)
def get_all(): feeds = requests.get('https://api.coinmarketcap.com/v1/ticker/').json() result = FeedSet() for f in feeds: try: price = float(f['price_usd']) volume = float(f['24h_volume_usd']) / price if f.get( '24h_volume_usd') else None result.append( FeedPrice(price, f['symbol'], 'USD', volume=volume, last_updated=pendulum.from_timestamp( int(f['last_updated'])))) except TypeError as e: # catches: TypeError: float() argument must be a string or a number, not 'NoneType' # on: f['price_usd'] log.debug('Could not get USD price for feed: {}'.format( json.dumps(f, indent=4))) log.exception(e) pass return result
def _process_metrics_interval(self, interval): """ this method wraps up a completed metrics interval and really does too much: 1. append to the interval_queue to keep track to be able to able to looks back numerous data points 2. display the metrics around this interval 3. check if there is any alerts in the last 120 seconds of metric intervals """ self.interval_queue.append(interval) # display some stats interval.display_stats() alert_window_hits = sum([i.hits for i in self.interval_queue]) alert_dt = pendulum.from_timestamp(interval.end_ts) if (alert_window_hits / self.alert_window) > self.avg_hits_threshold: self._set_alert(True, alert_dt, alert_window_hits) else: self._set_alert(False, alert_dt, alert_window_hits) # popping off older intervals beyond the alert window to clean up num_intervals_lookback = int(self.alert_window / self.refresh_rate) if len(self.interval_queue) == num_intervals_lookback: self.interval_queue.popleft()
def init_from_logfile(self) -> None: '''Read plot ID and job start time from logfile. Return true if we find all the info as expected, false otherwise''' assert self.logfile # Try reading for a while; it can take a while for the job to get started as it scans # existing plot dirs (especially if they are NFS). found_id = False found_log = False for attempt_number in range(3): with open(self.logfile, 'r') as f: with contextlib.suppress(UnicodeDecodeError): for line in f: m = re.match('^ID: ([0-9a-f]*)', line) if m: self.plot_id = m.group(1) found_id = True m = re.match(r'^Starting phase 1/4:.*\.\.\. (.*)', line) if m: # Mon Nov 2 08:39:53 2020 self.start_time = parse_chia_plot_time(m.group(1)) found_log = True break # Stop reading lines in file if found_id and found_log: break # Stop trying else: time.sleep(1) # Sleep and try again # If we couldn't find the line in the logfile, the job is probably just getting started # (and being slow about it). In this case, use the last metadata change as the start time. # TODO: we never come back to this; e.g. plot_id may remain uninitialized. # TODO: should we just use the process start time instead? if not found_log: self.start_time = pendulum.from_timestamp(os.path.getctime(self.logfile)) # Load things from logfile that are dynamic self.update_from_logfile()
def _deserialize(cls, encoded_var): # pylint: disable=too-many-return-statements """Helper function of depth first search for deserialization.""" from airflow.serialization.serialized_dag import SerializedDAG from airflow.serialization.serialized_baseoperator import SerializedBaseOperator # JSON primitives (except for dict) are not encoded. if cls._is_primitive(encoded_var): return encoded_var elif isinstance(encoded_var, list): return [cls._deserialize(v) for v in encoded_var] assert isinstance(encoded_var, dict) var = encoded_var[Encoding.VAR] type_ = encoded_var[Encoding.TYPE] if type_ == DAT.DICT: return {k: cls._deserialize(v) for k, v in var.items()} elif type_ == DAT.DAG: return SerializedDAG.deserialize_dag(var) elif type_ == DAT.OP: return SerializedBaseOperator.deserialize_operator(var) elif type_ == DAT.DATETIME: return pendulum.from_timestamp(var) elif type_ == DAT.TIMEDELTA: return datetime.timedelta(seconds=var) elif type_ == DAT.TIMEZONE: return pendulum.timezone(var) elif type_ == DAT.RELATIVEDELTA: if 'weekday' in var: var['weekday'] = relativedelta.weekday(*var['weekday']) return relativedelta.relativedelta(**var) elif type_ == DAT.SET: return {cls._deserialize(v) for v in var} elif type_ == DAT.TUPLE: return tuple([cls._deserialize(v) for v in var]) else: raise TypeError( 'Invalid type {!s} in deserialization.'.format(type_))
def generate_identity(self, age=(20, 30), addr_code=None): ''' 6位数字地址码 8位数字出生日期码 3位数字顺序码 同一天出生的顺序号 奇数表示男 偶数表示女 1位校验码 ''' if addr_code: acode = random.choice(addr_code.split()) locations = [ loc for loc in self.LOCATION if loc[0].startswith(acode) ] else: locations = self.LOCATION area = random.choice(locations) now = pendulum.now() tstart = now.subtract(years=age[1]) tstop = now.subtract(years=age[0]) rts = random.randint(tstart.int_timestamp, tstop.int_timestamp) birthday = pendulum.from_timestamp(rts).format('YYYYMMDD') code = str(random.randint(0, 999)).zfill(3) pre = area[0] + birthday + code vcode = self.identity_verify_code(pre) return pre + vcode, area[1]
def water_usage(request, **kwargs): response_obj = { "units": "l", "data": [], "labels": [] } period = kwargs['period'] dt = pendulum.from_timestamp(kwargs['timestamp']) if period == 'day': objects = SensorUsage.get_usage_by_day(dt, "W") for obj in objects: response_obj["data"].append(obj.value) response_obj["labels"].append(f"{obj.datetime.hour}:{obj.datetime.minute}") return JsonResponse(response_obj) elif period == "month": observations = SensorUsage.get_usage_by_month(dt, "W") results = { "data": {}, } for observation in observations: if observation.datetime.day not in results["labels"]: results["labels"][observation.datetime.day] = observation.value else: results["labels"][observation.datetime.day] += observation.value else: return JsonResponse({ "error": "unknown period given, only day or month is supported" }) return JsonResponse({ "Success": False, "Error": "Unknown error" })
def add_war_logs(self, document): added_war_logs = [] for war_log in document: # Add timestamps for update self.add_timestamps(war_log) # Add readable created dates created_date = pendulum.from_timestamp(war_log["createdDate"], tz="UTC") local_created_date = created_date.in_timezone("America/Denver") war_log[CREATED_DATE_UTC] = created_date.to_datetime_string() war_log[ CREATED_DATE_LOCAL] = local_created_date.to_datetime_string() try: results = self.war_logs.insert_one(war_log) if results.acknowledged: added_war_logs.append(war_log) print(results) except DuplicateKeyError: pass return added_war_logs
def get_form(atx, form_id, start_date, token=None, next_page=False): """Calls the API client get_form_responses() method with requisite params. The API limits responses to a max of 1000 per call, but it supports paging through responses using a `before` or `after` param. https://developer.typeform.com/responses/walkthroughs/#use-query-parameters-to-retrieve-specific-data By using a `sort` with `submitted_at` in ascending order in the initial call we can then use the `after` param that has the greatest `submitted_at` value in the initial response for the next call. :param atx: The Typeform API client. :param form_id: The form id to which responses are to be retrieved. :param start_date: Date to use in the `since` param of the API call. :param token: The token to use for the `after` param of the API call. :param next_page: Boolean if retreiving more records using the `after` param for paginating through responses. :return: API client response. """ after = f'after token: {token} ' if token else '' LOGGER.info('Forms query - form: {} start_date: {} {}'.format( form_id, pendulum.from_timestamp(start_date).strftime("%Y-%m-%d %H:%M"), after)) sort = None if next_page else 'submitted_at,asc' # sorting isn't suppored when using `after` return atx.client.get_form_responses(form_id, params={ 'since': start_date, 'page_size': MAX_RESPONSES_PAGE_SIZE, 'sort': sort, 'after': token, })
def route_list_users_recommendation(user_id: str): """Endpoint for getting recommended users.""" user = User.objects.get_or_404(id=user_id) user.identify(request) recommendation = user.get_recommendation() last_recommended_at = pendulum.from_timestamp( recommendation.last_recommended_at, tz="Asia/Seoul") is_today_recommended = last_recommended_at.date() == pendulum.today().date() if is_today_recommended and len(recommendation.user_ids) >= 2: user_ids = recommendation.user_ids else: user_ids = user.list_recommended_user_ids() user_ids.extend(recommendation.user_ids) recommendation.user_ids = user_ids recommendation.last_recommended_at = pendulum.now().int_timestamp recommendation.save() users = User.list(id__in=user_ids[:MAXIMUM_RECOMMENDATION_SHOW_COUNT]).as_pymongo() users = sort_order_by_ids(user_ids, users) response = encode(list(users)) return Response(response, mimetype="application/json")
def check_for_payments(iota, t_hash, addr) -> Dict: payments = {} receiver_addr = AddressNoChecksum()._apply(TryteString(addr)) t_bytes = bytes(t_hash) t_trytes = str(iota.get_trytes([t_bytes])["trytes"][0]) transaction = Transaction.from_tryte_string(t_trytes) t_age = now() - from_timestamp(transaction.attachment_timestamp / 1000) if transaction.address == receiver_addr and t_age.in_minutes() < 60: logger.warning( f"[{from_timestamp(transaction.timestamp)}] Payment of {transaction.value}i found on receiving address {addr[:8]}..." ) try: data = extract_json(transaction) username, topic, password = parse_payload(data) payments["username"] = username payments["topic"] = topic payments["password"] = password payments["expires_after"] = 10 * (transaction.value // VALUE_PER_TEN_SECONDS) payments["t_hash"] = transaction.hash payments["t_value"] = transaction.value except Exception as e: logger.exception(e) return payments
############################################# uri = 'https://api.forecast.io/forecast/{api_key}/{lat},{lng},{time}'.format( api_key=FORECAST_API_KEY, time=local_time.isoformat(), lat=lat, lng=lng) resp = requests.get(uri) weather = json.loads(resp.text) ############################################# # First record to establish some daily data ############################################# if initial: # These are all UTC. sunrise = pendulum.from_timestamp(weather['daily']['data'][0]['sunriseTime']) sunset = pendulum.from_timestamp(weather['daily']['data'][0]['sunsetTime']) day = Day(sunrise=sunrise, sunset=sunset, date=utc_time.date()) country = Country(name=location['country']) state = State(name=location['state']) #mt = State(name='Montana', country=us) city = City(name=location['city']) #msla = City(name='Missoula', state=mt) import ipdb ipdb.set_trace() initial = False
def test_create_from_timestamp_with_timezone(): d = pendulum.from_timestamp(0, timezone("America/Toronto")) assert d.timezone_name == "America/Toronto" assert_datetime(d, 1969, 12, 31, 19, 0, 0)
def check_date(timestr: str) -> date: if not timestr: return pendulum.from_timestamp(0, "local") return pendulum.from_format(timestr, "MMMM D, YYYY", "local")
def check_date(data: Dict[str, Union[str, int]], key: str) -> Union[DateTime, None]: timestamp = data.get(key) if timestamp: return pendulum.from_timestamp(timestamp, "UTC") else: return None
def shape_forecast(forecast): """Reduce an API response object to wanted values""" result = defaultdict() timezone = forecast.get("timezone") daily_block = forecast.get("daily", {}) days = daily_block.get("data", [{}, {}]) today = days[0] currently = forecast.get("currently", {}) hourly = forecast.get("hourly", {}) result["current_summary"] = currently.get("summary") result["current_temperature"] = math.ceil( currently.get("temperature") ) result["current_time"] = pendulum.from_timestamp( currently.get("time"), tz=timezone ) result["current_humidity"] = currently.get("humidity", 0) result["summary"] = today.get("summary") result["temperature"] = math.ceil(today.get("temperature", 0)) result["sunrise"] = pendulum.from_timestamp( today.get("sunriseTime"), tz=timezone ) result["sunset"] = pendulum.from_timestamp( today.get("sunsetTime"), tz=timezone ) result["humidity"] = currently.get("humidity", 0) result["high"] = math.ceil(today.get("temperatureHigh")) result["high_at"] = pendulum.from_timestamp( today.get("temperatureHighTime"), tz=timezone ) result["low"] = math.ceil(today.get("temperatureLow")) result["low_at"] = pendulum.from_timestamp( today.get("temperatureLowTime"), tz=timezone ) result["precip_prob"] = currently.get("precipProbability", 0) * 100 result["precip_type"] = currently.get("precipType") if "data" in hourly: result["hourly"] = [] for hour in hourly["data"][0:24]: hour_clone = copy.copy(hour) hour_clone["time"] = pendulum.from_timestamp( hour_clone["time"], tz=timezone ) result["hourly"].append(hour_clone) return result
def test_create_from_timestamp_returns_pendulum(): d = pendulum.from_timestamp(pendulum.datetime(1975, 5, 21, 22, 32, 5).timestamp()) assert_datetime(d, 1975, 5, 21, 22, 32, 5) assert d.timezone_name == "UTC"