def main(argv): if len(argv) == 1: print 'Command missing' sys.exit(1) elif argv[1] == 'update_jira': date = None if len(argv) >= 3: date = date_parse(argv[2]).date() update_jira(date) elif argv[1] == 'send_email': end_date = datetime.datetime.now().date() email = config['email']['default_to'] start_date = end_date - datetime.timedelta(days=1) while start_date.isoweekday() > 5: start_date -= datetime.timedelta(days=1) if len(argv) > 2: start_date = date_parse(argv[2]).date() if len(argv) > 3: end_date = date_parse(argv[3]).date() if len(argv) > 4: email = argv[3] send_email(start_date, end_date, email) elif argv[1] == 'get_companies': print get_companies()
def output(self): try: parts = os.path.basename(self.src).split("-") ext = parts[-1] # XXX this parsing stuff is insane... if ext.startswith("probe") or \ ext.startswith("backend"): date = date_parse('-'.join(parts[-5:-2])) asn = parts[-2] test_name = '-'.join(parts[:-5]) elif parts[0].startswith("report"): date = date_parse('-'.join(parts[-3:-1]+parts[-1].split(".")[:1])) asn = "ASX" test_name = '-'.join(parts[1:-3]) ext = "probe."+'.'.join(parts[-1].split(".")[1:]) else: date = date_parse('-'.join(parts[-4:-1])) asn = parts[-1].split(".")[0] ext = "probe."+'.'.join(parts[-1].split(".")[1:]) test_name = '-'.join(parts[:-4]) # To facilitate sorting and splitting around "-" we convert the # date to be something like: 20150101T000015Z timestamp = date.strftime("%Y%m%dT%H%M%SZ") filename = "{date}-{asn}-{test_name}-{df_version}-{ext}".format( date=timestamp, asn=asn, test_name=test_name, df_version="v1", ext=ext.replace(".gz", "").replace(".yamloo", ".yaml") ) uri = os.path.join(self.dst, date.strftime("%Y-%m-%d"), filename) return S3Target(uri) except Exception: return S3Target(os.path.join(self.dst, "failed", os.path.basename(self.src)))
def verify_certificate(environ, verify_key, validity_start_key, validity_end_key): """ Checks if the client certificate is valid. Start and end data is optional, as not all SSL mods give that information. :param environ: The WSGI environment. :param verify_key: The key for the value in the environment where it was stored if the certificate is valid or not. :param validity_start_key: The key for the value in the environment with the encoded datetime that indicates the start of the validity range. :param validity_end_key: The key for the value in the environment with the encoded datetime that indicates the end of the validity range. """ verified = environ.get(verify_key) validity_start = environ.get(validity_start_key) validity_end = environ.get(validity_end_key) if verified != 'SUCCESS': return False if validity_start is None or validity_end is None: return True validity_start = date_parse(validity_start) validity_end = date_parse(validity_end) if validity_start.tzinfo != _TZ_UTC or validity_end.tzinfo != _TZ_UTC: # Can't consider other timezones return False now = datetime.utcnow().replace(tzinfo=_TZ_UTC) return validity_start <= now <= validity_end
def parse_playlist_form(fp): show_date = None doc = html_parse(fp) inputs = {} for input in doc.getroot().body.xpath('//input[@type="text"]'): if input.attrib['name'] == 'date': show_date = date_parse(input.attrib['value']) elif PATTERN.match(input.attrib['name']): field,num = PATTERN.match(input.attrib['name']).groups() if field == 'timestamp': try: time = date_parse(input.attrib['value']) real_date = date_parse(show_date.strftime('%Y-%m-%d') + ' ' + time.strftime('%H:%M:%S')) d = inputs.setdefault(int(num),{}) d['unix_time'] = mktime(real_date.timetuple()) d['date_str'] = real_date.strftime('%Y-%m-%d') d['time_str'] = real_date.strftime('%H:%M:%S') except: pass # 'auto' probably else: try: value = input.attrib['value'] inputs.setdefault(int(num),{})[field] = value except KeyError: pass show = [] for k in sorted(inputs): d = dict((key,inputs[k].get(key,None)) for key in KEEPER_KEYS) if filter(None,d.values()): show.append(d) return (show[0]['date_str'],show)
def is_date(string): try: date_parse(string) if string == "at" or string =="on" or string == "." or string == ",": return False return True except ValueError: return False
def get_entries_in_month(month_to_check, entries): time_periods_entries = [] month_as_date = date_parse(month_to_check) for entry in entries: entry_start = date_parse(entry['start_date']) if month_as_date.month == entry_start.month and month_as_date.year == entry_start.year: time_periods_entries.append(entry) return time_periods_entries
def parse(cls, raw): try: start_date = date_parse(raw['start_date']) end_date = date_parse(raw['end_date']) except (AttributeError, ValueError, TypeError): raise ValueError("'start_date' or 'end_date' invalid.") if start_date >= end_date: raise ValueError("'start_date' must be < 'end_date'.") return cls(start_date, end_date)
def load_resources(bucket, prefix, region, account_config, accounts, assume, start, end, resources, store, db, verbose, debug): """load resources into resource database.""" logging.basicConfig(level=(verbose and logging.DEBUG or logging.INFO)) logging.getLogger('botocore').setLevel(logging.WARNING) logging.getLogger('s3transfer').setLevel(logging.WARNING) start = date_parse(start) end = date_parse(end) if not resources: resources = ['NetworkInterface', 'Instance', 'LoadBalancer'] account_map = {} data = yaml.safe_load(account_config.read()) for a in data.get('accounts', ()): if accounts and (a['name'] in accounts or a['account_id'] in accounts): account_map[a['account_id']] = a elif not accounts: account_map[a['account_id']] = a account_ids = list(account_map) executor = ProcessPoolExecutor if debug: from c7n.executor import MainThreadExecutor MainThreadExecutor.c7n_async = False executor = MainThreadExecutor stats = Counter() t = time.time() with executor(max_workers=multiprocessing.cpu_count()) as w: futures = {} for a in account_ids: for r in resources: futures[w.submit( process_account_resources, a, bucket, prefix, region, store, start, end, r)] = (a, r) indexer = RESOURCE_FILE_INDEXERS[r] for f in as_completed(futures): a, r = futures[f] if f.exception(): log.error("account:%s error:%s", a, f.exception()) continue files, dl_stats = f.result() idx_stats = indexer(db, resource_config_iter(files)) log.info( "loaded account:%s files:%d bytes:%s events:%d resources:%d idx-time:%d dl-time:%d", account_map[a]['name'], len(files), human_size(dl_stats['DownloadSize'] + dl_stats['CacheSize']), idx_stats['Records'], idx_stats['RowCount'], idx_stats['IndexTime'], dl_stats['FetchTime']) stats.update(dl_stats) stats.update(idx_stats) log.info("Loaded %d resources across %d accounts in %0.2f", stats['RowCount'], len(account_ids), time.time() - t)
def process(self, user_handle, **kwargs): """ Determine the edit rate of user(s). The parameter *user_handle* can be either a string or an integer or a list of these types. When the *user_handle* type is integer it is interpreted as a user id, and as a user_name for string input. If a list of users is passed to the *process* method then a dict object with edit rates keyed by user handles is returned. - Parameters: - **user_handle** - String or Integer (optionally lists). Value or list of values representing user handle(s). - Return: - Dictionary. key(string): user handle, value(Float): edit counts """ # Extract edit count for given parameters edit_rate = list() ec_kwargs = deepcopy(self.__dict__) e = ec.EditCount(**ec_kwargs).process(user_handle, **kwargs) # Compute time difference between datetime objects and get the # integer number of seconds if self.group == umpt.REGISTRATION: time_diff_sec = self.t * 3600.0 elif self.group == umpt.INPUT: try: start_ts_obj = date_parse( format_mediawiki_timestamp(self.datetime_start)) end_ts_obj = date_parse( format_mediawiki_timestamp(self.datetime_end)) except (AttributeError, ValueError): raise um.UserMetricError() time_diff_sec = (end_ts_obj - start_ts_obj).total_seconds() else: raise um.UserMetricError('group parameter not specified.') # Normalize the time interval based on the measure if self.time_unit == self.TIME_UNIT_TYPE.DAY: time_diff = time_diff_sec / (24 * 60 * 60) elif self.time_unit == self.TIME_UNIT_TYPE.HOUR: time_diff = time_diff_sec / (60 * 60) else: time_diff = time_diff_sec # Build the list of edit rate metrics for i in e.__iter__(): new_i = i[:] # Make a copy of the edit count element new_i.append(new_i[1] / (time_diff * self.time_unit_count)) new_i.append(time_diff) edit_rate.append(new_i) self._results = edit_rate return self
def process_events(self, events): enrollments = [] for event in events['Events']: section_data = event['Section'] course_data = section_data['Course'] section = Section() section.term = Term(quarter=course_data['Quarter'], year=course_data['Year']) section.curriculum_abbr = course_data['CurriculumAbbreviation'] section.course_number = course_data['CourseNumber'] section.section_id = section_data['SectionID'] section.is_primary_section = True section.linked_section_urls = [] if ('PrimarySection' in event and 'Course' in event['PrimarySection']): primary_course = event['PrimarySection']['Course'] if primary_course: section.is_primary_section = False section.primary_section_curriculum_abbr = \ primary_course['CurriculumAbbreviation'] section.primary_section_course_number = \ primary_course['CourseNumber'] section.primary_section_id = \ event['PrimarySection']['SectionID'] try: data = { 'Section': section, 'Role': EnrollmentModel.STUDENT_ROLE, 'UWRegID': event['Person']['UWRegID'], 'Status': self._enrollment_status(event, section), 'LastModified': date_parse(event['LastModified']), 'InstructorUWRegID': event['Instructor']['UWRegID'] if ( 'Instructor' in event and event['Instructor'] and 'UWRegID' in event['Instructor']) else None } if 'Auditor' in event and event['Auditor']: data['Role'] = EnrollmentModel.AUDITOR_ROLE if 'RequestDate' in event: data['RequestDate'] = date_parse(event['RequestDate']) enrollments.append(data) except UnhandledActionCodeException: self._log.warning("%s UNKNOWN %s for %s at %s" % ( log_prefix, event['Action']['Code'], event['Person']['UWRegID'], event['LastModified'])) pass self.load_enrollments(enrollments)
def _process_help(args): # Unpack args state = args[1] users = args[0] thread_args = um.UserMetric._unpack_params(state) # Log progress if thread_args.log_: logging.debug(__name__ + '::Computing live account. (PID = %s)' % getpid()) # Extract edit button click from edit_page_tracking table (namespace, # article title, timestamp) of first click and registration timestamps # (join on logging table) # # Query will return: (user id, time of registration, time of first # edit button click) query_args = namedtuple('QueryArgs', 'namespace')(thread_args.namespace) query_results = query_mod.live_account_query(users, thread_args.project, query_args) # Iterate over results to determine boolean indicating whether # account is "live" results = {str(user): -1 for user in users} user_reg = query_mod.user_registration_date_logging( users, thread_args.project, None) # uid: diff_time user_reg = {str(r[0]): (datetime.now() - date_parse(r[1])). total_seconds() / 3600 for r in user_reg} # Flag all users alive longer than t hours as "not invalid" for user in results: if user in user_reg and user_reg[user] >= thread_args.t: results[user] = 0 for row in query_results: user = str(row[0]) try: # get the difference in hours diff = (date_parse(row[2]) - date_parse(row[1])).total_seconds() diff /= 3600 except Exception: continue if diff <= thread_args.t: results[user] = 1 else: results[user] = 0 return [(str(key), results[key]) for key in results]
def to_datetime(self, v, format=None): if v == "NULL": return None elif format is None: return date_parse(v) #much slower than strptime, avoid for large jobs else: try: return datetime.datetime.strptime(v, format) except ValueError: if self._log: self._log.warning( "Using the slower date parse: for value '%s' as format '%s' has failed", v, format) return date_parse(v)
def __get_file_names(start_dt, end_dt=None): if not end_dt: start_dt = date_parse(start_dt) return [start_dt.strftime("%Y-%m-%d")], __get_time_range(start_dt, start_dt) start_dt = date_parse(start_dt) end_dt = date_parse(end_dt) dates = __daterange(start_dt, end_dt) time_tuple = __get_time_range(start_dt, end_dt) return [d.strftime("%Y-%m-%d") for d in dates], time_tuple
def test_implicit_date_filters(self): john = Person.objects.get(name="John") # Mark was created at least one second after John. mark = Person.objects.get(name="Mark") from rest_framework import serializers from rest_framework.renderers import JSONRenderer class PersonSerializer(serializers.ModelSerializer): class Meta: model = Person # Figure out what the date strings should look like based on the # serializer output. data = PersonSerializer(john).data date_str = JSONRenderer().render(data['date_joined']).decode('utf-8').strip('"') # Adjust for imprecise rendering of time datetime_str = JSONRenderer().render(date_parse(data['datetime_joined']) + datetime.timedelta(seconds=0.6)).decode('utf-8').strip('"') # Adjust for imprecise rendering of time dt = datetime.datetime.combine(datetime.date.today(), date_parse(data['time_joined']).time()) + datetime.timedelta(seconds=0.6) time_str = JSONRenderer().render(dt.time()).decode('utf-8').strip('"') # DateField GET = { 'date_joined__lte': date_str, } f = AllLookupsPersonDateFilter(GET, queryset=Person.objects.all()) self.assertEqual(len(list(f)), 2) p = list(f)[0] # DateTimeField GET = { 'datetime_joined__lte': datetime_str, } f = AllLookupsPersonDateFilter(GET, queryset=Person.objects.all()) self.assertEqual(len(list(f)), 1) p = list(f)[0] self.assertEqual(p.name, "John") # TimeField GET = { 'time_joined__lte': time_str, } f = AllLookupsPersonDateFilter(GET, queryset=Person.objects.all()) self.assertEqual(len(list(f)), 1) p = list(f)[0] self.assertEqual(p.name, "John")
def get_dates(start, end, tz): mytz = tz and zoneinfo.gettz(tz) or tzutc() start = date_parse(start).replace(tzinfo=mytz) if end: end = date_parse(end).replace(tzinfo=mytz) else: end = datetime.now().replace(tzinfo=mytz) if tz: start = start.astimezone(tzutc()) if end: end = end.astimezone(tzutc()) if start > end: start, end = end, start return start, end
def _row(self, item): photopage_url = None for url in item["urls"]["url"]: if url["type"] == "photopage": photopage_url = url["_content"] return (item["id"], # date posted is gmt epoch time, convert it to the same format as date taken # detail as https://www.flickr.com/services/api/misc.dates.html date_parse(time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(float(item["dates"]["posted"])))).replace(tzinfo=tzutc()), date_parse(item["dates"]["taken"]), item["license"], item["safety_level"], item.get("originalformat"), item["owner"]["nsid"], item["owner"]["username"], item["title"]["_content"].replace('\n', ' '), item["description"]["_content"].replace('\n', ' '), item["media"], photopage_url)
def getHours(): session = Session() try: start = date_parse(request.args.get("start","1979-04-01T07:00:00.000Z")) except: start = date_parse("1979-04-01T07:00:00.000Z") end = request.args.get("stop",None) try: if end:end = date_parse(end) except: end = None print start user_id = current_user.id return HoursDict(TimeLogged.times_between(start,end,user_id=user_id))
def image_to_dict(image, detail=True): d = dict(id=image['id'], name=image['name']) if detail: d['updated'] = utils.isoformat(date_parse(image['updated_at'])) d['created'] = utils.isoformat(date_parse(image['created_at'])) d['status'] = 'DELETED' if image['deleted_at'] else 'ACTIVE' d['progress'] = 100 if image['status'] == 'available' else 0 d['user_id'] = image['owner'] d['tenant_id'] = image['owner'] d['links'] = util.image_to_links(image["id"]) if image["properties"]: d['metadata'] = image['properties'] else: d['metadata'] = {} return d
def when(self, when): if 'time' in when: self.start = self.end = time_parse(when['time']) self.all_day = False elif 'start_time' in when: self.start = time_parse(when['start_time']) self.end = time_parse(when['end_time']) self.all_day = False elif 'date' in when: self.start = self.end = date_parse(when['date']) self.all_day = True elif 'start_date' in when: self.start = date_parse(when['start_date']) self.end = date_parse(when['end_date']) self.all_day = True
def image_to_dict(image, detail=True): d = dict(id=image["id"], name=image["name"]) if detail: d["updated"] = utils.isoformat(date_parse(image["updated_at"])) d["created"] = utils.isoformat(date_parse(image["created_at"])) d["status"] = "DELETED" if image["deleted_at"] else "ACTIVE" d["progress"] = 100 if image["status"] == "available" else 0 d["user_id"] = image["owner"] d["tenant_id"] = image["owner"] d["links"] = util.image_to_links(image["id"]) if image["properties"]: d["metadata"] = image["properties"] else: d["metadata"] = {} return d
def generate_selector(data): first_time = date_parse(data[1][0]) last_time = date_parse(data[-1][0]) time_length = last_time-first_time-SUBSET_SIZE subset_start = first_time +\ datetime.timedelta( seconds=random.random() * time_length.total_seconds()) if subset_start < first_time: subset_start = first_time subset_end = subset_start+SUBSET_SIZE def selector(timestamp): return (subset_start <= timestamp) & (timestamp <= subset_end) return selector
def _get_timeseries(date_start, date_end, interval): """ Generates a series of timestamps given a start date, end date, and interval """ # Ensure the dates are string representations date_start = format_mediawiki_timestamp(date_start) date_end = format_mediawiki_timestamp(date_end) c = date_parse(date_start) + datetime.timedelta(hours=-int(interval)) e = date_parse(date_end) while c < e: c += datetime.timedelta(hours=int(interval)) yield c
def format_response(request): """ Populates data for response to metrics requests. Parameters ~~~~~~~~~~ request : RequestMeta RequestMeta object that stores request data. """ args = ParameterMapping.map(request) metric_class = get_metric_type(request.metric) metric_obj = metric_class(**args) # Prepare metrics output for json response response = OrderedDict() response['type'] = get_request_type(request) response['header'] = metric_obj.header() # Get metric object params for key in metric_obj.__dict__: if not search(r'^_.*', key) and str(key) not in response: response[str(key)] = metric_obj.__dict__[key] response['cohort'] = str(request.cohort_expr) response['cohort_last_generated'] = str(request.cohort_gen_timestamp) response['time_of_response'] = datetime.now().strftime(DATETIME_STR_FORMAT) response['aggregator'] = str(request.aggregator) response['metric'] = str(request.metric) response['interval_hours'] = request.interval if request.group: response['group'] = REVERSE_GROUP_MAP[int(request.group)] else: # @TODO get access to the metric default for this attribute response['group'] = 'default' response['datetime_start'] = date_parse(metric_obj.datetime_start).\ strftime(DATETIME_STR_FORMAT) response['datetime_end'] = date_parse(metric_obj.datetime_end).\ strftime(DATETIME_STR_FORMAT) response['data'] = OrderedDict() return response, metric_class, metric_obj
def insert_text_data(data_source, source_url, text, time_posted, session): """Adds the base entry for a text data source to the database and returns the newly created model Keyword arguments: data_source -- An enum indicating source. The enum is located in interns.models.models.AllowedSources source_url -- A string indicating the url the text was pulled from text -- the raw text data pulled from the url time_posted -- either a datetime object or a datetime string session -- active db session """ if not isinstance(time_posted, datetime): time_posted = date_parse(time_posted) eleanor_logger.debug('Inserting text data into postgres') TextModel = models.TextSource(source_key=data_source, source_url=source_url, written_text=text, time_posted=time_posted) session.add(TextModel) return TextModel
def process_json(self, pjson): """ Process a single proposal into a JSON. """ proposal = {} for kp, kj in self.copy_keys.items(): if kj in pjson: proposal[kp] = pjson[kj] proposal["region_name"] = "Cambridge, MA" proposal["source"] = "data.cambridgema.gov" tz = pytz.timezone("US/Eastern") updated_datestr = pjson.get("decisiondate", pjson["applicationdate"]) updated_naive = date_parse(updated_datestr) proposal["updated_date"] = tz.localize(updated_naive) proposal["complete"] = self.match_complete(pjson["status"]) proposal["description"] = pjson.get("reason_for_petition_other", "") if "location" in pjson and not pjson["location"]["needs_recoding"]: location = pjson["location"] try: human_address = json.loads(location["human_address"]) proposal["address"] = human_address["address"] proposal["long"] = float(location["longitude"]) proposal["lat"] = float(location["latitude"]) except: proposal["location"] = None proposal["attributes"] = [(pk, pjson.get(k)) for pk, k in self.remap_attributes.items()] return proposal
def _date_or_none(thing): if isinstance(thing, dt.date): return thing elif isinstance(thing, dt.datetime): return thing.date() else: return default(lambda: date_parse(thing))
def refresh_next_events(self, lookahead_time: timedelta) -> List[RoutineEvent]: """Refreshes the list of scheduled events. Parses summary events that are returned by the calendar fetcher, in the time range between utc now and utc now + lookahead_time. """ # TODO: should we handle events with the same id but that were rescheduled? events = [] for event in self.gcal.fetch(lookahead_time): summary = event['summary'] parse_match = self.re_extract.search(summary) if not parse_match: logging.warning( 'Event "%s" does not match a routine description, skipping!' ) continue start_date = date_parse(event['start']['dateTime']) routine_name = parse_match.group(1) routine_event = RoutineEvent(event['id'], routine_name, start_date) events.append(routine_event) return events
def get_git_info(): """ Get the abbreviated commit version (not provided by get_git_info()) Returns a dictionary """ LAST_COMMIT = ['git', 'log', '-1'] COMMANDS = { 'short_commit': ['git', 'rev-parse', '--short', 'HEAD'], 'commit': ['git', 'rev-parse', 'HEAD'], 'tag': ['git', 'describe', '--tags'], 'author': LAST_COMMIT + ["--pretty=format:%an"], 'author_email': LAST_COMMIT + ["--pretty=format:%ae"], 'committer': LAST_COMMIT + ["--pretty=format:%cn"], 'committer_email': LAST_COMMIT + ["--pretty=format:%ce"], # %cd is the commit date 'date_ISO': LAST_COMMIT + ['--pretty=format:%cd'], 'message': LAST_COMMIT + ["--pretty=format:%B"], 'raw': LAST_COMMIT, 'root_dir': ['git', 'rev-parse', '--show-toplevel'] } # always return a date, even in case of failure r = {'status': False, 'date': None} try: for var, command in COMMANDS.items(): # NOTE: The 'text' argument is clearer, # but for Python < 3.7, only `universal_newlines` # is accepted try: r[var] = subprocess.check_output( command, universal_newlines=True, stderr=subprocess.DEVNULL).strip() # keep first part if var == 'tag': r[var] = r[var].split('-')[0] elif var == 'date_ISO': r['date'] = date_parse(r[var]) r['status'] = True except subprocess.CalledProcessError as e: if e.returncode == 128: # generally means "unexpected error" # git status (no repo), # git tag (no tag) r[var] = '' else: # should be 1, type whatever that is r[var] = "# Cannot execute '%s': %s" % (command, e) except Exception as e: # any other error, it's probably meaningless at this point r[var] = "# Unexpected error '%s': %s" % (command, e) # convert return r except FileNotFoundError as e: # not git command return r.update({ 'status': False, 'diagnosis': 'Git command not found', 'error': str(e) })
def insert_non_retweet_data(tweet_data): """Takes the passed in JSON tweet_data and inserts into the database""" eleanor_logger.debug('Inserting tweet data') with GetDBSession() as db_session: tweetTextModel = insert_text_data( models.AllowedSources.twitter.name, tweet_data['url'], tweet_data['tweet_text'], date_parse(tweet_data['tweet_created']), db_session) tweetModel = twitter_models.TwitterSource( tweeter_user_name=tweet_data['user_name'], tweet_id=tweet_data['tweet_id'], is_retweet=False) tweetTextModel.twitter_source = tweetModel add_user_mentions(tweet_data, tweetModel) add_hashtags(tweet_data, tweetModel) add_urls(tweet_data, tweetModel) try: db_session.commit() except IntegrityError as e: if 'duplicate key value' in e.message: # We've already captured this so, moving on eleanor_logger.info( 'Duplicate tweet is already in the database, skipping') else: eleanor_logger.critical( ('A database error occurred while attempting ' 'to insert tweet %s'), e) except Exception as e: # Something real bad happened eleanor_logger.critical( ('An error has occurred while inserting a tweet into ' 'the database %s'), e)
def request_age(request): if isinstance(request, Request): created = request.statehistory[0].when else: created = request.find('history').get('when') created = date_parse(created) return datetime.utcnow() - created
def return_point_timestamp(dev_id, unit, period, measurement=None, channel=None): dbcon = InfluxDBClient( INFLUXDB_HOST, INFLUXDB_PORT, INFLUXDB_USER, INFLUXDB_PASSWORD, INFLUXDB_DATABASE) query_str = query_string( unit, dev_id, measure=measurement, channel=channel, value='LAST', past_sec=period) if query_str == 1: return [None, None] try: raw_data = dbcon.query(query_str).raw number = len(raw_data['series'][0]['values']) time_raw = raw_data['series'][0]['values'][number - 1][0] value = raw_data['series'][0]['values'][number - 1][1] value = '{:.3f}'.format(float(value)) # Convert date-time to epoch (potential bottleneck for data) dt = date_parse(time_raw) timestamp = calendar.timegm(dt.timetuple()) * 1000 return [timestamp, value] except KeyError: return [None, None] except Exception: return [None, None]
def check_update_needed(db_table_object, repository_name, pushed_at): """ Returns True if there is a need to clone the github repository """ logger.info( f"This is the repo name from check_update <<{repository_name}>> and db_table <<{db_table_object}>>" ) result = get_single_repository(db_table_object, repository_name) logger.info(result) if not result: logger.info("result not found") return True else: logger.info("result found") logger.info(f"This is the result {result}") epoch = date_parse(pushed_at).timestamp( ) ##the pushed_at timetsamp available in the repo right now logger.info( f"Comparing {int(epoch)} and {result['downloaded_at']} for {repository_name}" ) if int(epoch) > int(result["downloaded_at"]): return True return False
def fetch_data(url): """Helper method to fetch data from url.""" page_tree = get_tree(url) if page_tree is None: return date = \ page_tree.xpath("//div[@class='meta-data']/div/h3/a/text()")[0] date = date_parse(date) if YEAR: if date.year < YEAR: sys.exit(0) image_link = page_tree.xpath("//img[@id='main-comic']/@src")[0] image_link = "http:{}".format(image_link) \ if image_link.startswith("//") else \ image_link permalink = page_tree.xpath("//input[@id='permalink']/@value")[0] author = page_tree.xpath( "//small[@class='author-credit-name']/text()" )[0].strip("by ") comic_number = permalink.strip("/").split("/")[-1] return { "number": comic_number, "image": image_link, "permalink": permalink, "metadata": { "date": (date.year, date.month, date.day), "author": author } }
def pg_sanitize_value(value, pg_datatype, max_length): '''attempt to sanitze the value to be better suitable to cast to the desired datatype in postgres. in case of failures to parse the value it gets returned as it is''' if value is not None: if pg_datatype in ('date', 'timestamptz', 'timestamp'): try: return value.isoformat() except AttributeError: try: return date_parse(value).isoformat() except: pass # let postgresql try its best at parsing :( elif pg_datatype in ('char', 'text', 'varchar'): # truncate texts when there is an charater limit in the db. Cast to string # to make in work for values send as int/float/... if max_length is not None: return str(value)[:max_length] elif pg_datatype in ('bytea', 'geometry'): return Binary(value) elif pg_datatype == 'json': # serialize to json to use value with postgresql json type return Json(value) return value
def __parse_config(self, filtered_config, db_config): self._isolines_provider = filtered_config[ self.ISOLINES_PROVIDER_KEY].lower() if not self._isolines_provider: self._isolines_provider = self.DEFAULT_PROVIDER self._geocoder_provider = filtered_config[ self.GEOCODER_PROVIDER_KEY].lower() self._period_end_date = date_parse( filtered_config[self.PERIOD_END_DATE]) self._isolines_quota = self._get_effective_monthly_quota( self.QUOTA_KEY) if filtered_config[self.SOFT_LIMIT_KEY].lower() == 'true': self._soft_isolines_limit = True else: self._soft_isolines_limit = False if self._isolines_provider == self.HEREMAPS_PROVIDER: self._heremaps_app_id = db_config.heremaps_isolines_app_id self._heremaps_app_code = db_config.heremaps_isolines_app_code self._heremaps_apikey = db_config.heremaps_isolines_apikey self._heremaps_use_apikey = db_config.heremaps_isolines_use_apikey self._heremaps_service_params = db_config.heremaps_isolines_service_params elif self._isolines_provider == self.MAPZEN_PROVIDER: self._mapzen_matrix_api_key = self._db_config.mapzen_matrix_api_key self._mapzen_matrix_service_params = db_config.mapzen_matrix_service_params self._mapzen_isochrones_service_params = db_config.mapzen_isochrones_service_params elif self._isolines_provider == self.MAPBOX_PROVIDER: self._mapbox_matrix_api_keys = self._db_config.mapbox_matrix_api_keys self._mapbox_matrix_service_params = db_config.mapbox_matrix_service_params self._mapbox_isochrones_service_params = db_config.mapbox_isochrones_service_params elif self._isolines_provider == self.TOMTOM_PROVIDER: self._tomtom_isolinesx_api_keys = self._db_config.tomtom_isolines_api_keys self._tomtom_isolines_service_params = db_config.tomtom_isolines_service_params
def future_event(self): today = date.today() if isinstance(self.date, str): return today <= date_parse(self.date).date() return today <= self.date
def list_images(request, detail=False): # Normal Response Codes: 200, 203 # Error Response Codes: computeFault (400, 500), # serviceUnavailable (503), # unauthorized (401), # badRequest (400), # overLimit (413) log.debug('list_images detail=%s', detail) since = utils.isoparse(request.GET.get('changes-since')) with backend.PlanktonBackend(request.user_uniq) as b: images = b.list_images() if since: updated_since = lambda img: date_parse(img["updated_at"]) >= since images = ifilter(updated_since, images) if not images: return HttpResponse(status=304) images = sorted(images, key=lambda x: x['id']) reply = [image_to_dict(image, detail) for image in images] if request.serialization == 'xml': data = render_to_string('list_images.xml', dict(images=reply, detail=detail)) else: data = json.dumps(dict(images=reply)) return HttpResponse(data, status=200)
def request_when_staged(request, project, first=False): when = None for history in request.statehistory: if project in history.comment: when = history.when return date_parse(when)
def last_data(sensor_measure, sensor_id, sensor_period): """Return the most recent time and value from influxdb""" current_app.config['INFLUXDB_USER'] = INFLUXDB_USER current_app.config['INFLUXDB_PASSWORD'] = INFLUXDB_PASSWORD current_app.config['INFLUXDB_DATABASE'] = INFLUXDB_DATABASE dbcon = influx_db.connection try: raw_data = dbcon.query("""SELECT last(value) FROM {} WHERE device_id='{}' AND time > now() - {}m """.format(sensor_measure, sensor_id, sensor_period)).raw number = len(raw_data['series'][0]['values']) time_raw = raw_data['series'][0]['values'][number - 1][0] value = raw_data['series'][0]['values'][number - 1][1] value = '{:.3f}'.format(float(value)) # Convert date-time to epoch (potential bottleneck for data) dt = date_parse(time_raw) timestamp = calendar.timegm(dt.timetuple()) * 1000 live_data = '[{},{}]'.format(timestamp, value) return Response(live_data, mimetype='text/json') except KeyError: logger.debug("No Data returned form influxdb") return '', 204 except Exception as e: logger.exception("URL for 'last_data' raised and error: " "{err}".format(err=e)) return '', 204
def framework_agreement(framework_slug): framework = get_framework(data_api_client, framework_slug, allowed_statuses=['standstill', 'live']) supplier_framework = return_supplier_framework_info_if_on_framework_or_abort(data_api_client, framework_slug) if supplier_framework['agreementReturned']: date_formatter = DateFormatter(current_app.config['DM_TIMEZONE']) supplier_framework['agreementReturnedAt'] = date_formatter.datetimeformat( date_parse(supplier_framework['agreementReturnedAt']) ) # if there's a frameworkAgreementVersion key, it means we're on G-Cloud 8 or higher if framework.get('frameworkAgreementVersion'): drafts, complete_drafts = get_drafts(data_api_client, framework_slug) lots_with_completed_drafts = [ lot for lot in framework['lots'] if count_drafts_by_lot(complete_drafts, lot['slug']) ] return render_template( 'frameworks/contract_start.html', signature_page_filename=SIGNATURE_PAGE_FILENAME, framework=framework, lots=[{ 'name': lot['name'], 'has_completed_draft': (lot in lots_with_completed_drafts) } for lot in framework['lots']], supplier_framework=supplier_framework, ), 200 return render_template_with_csrf( "frameworks/agreement.html", framework=framework, supplier_framework=supplier_framework, agreement_filename=AGREEMENT_FILENAME )
def _format_mediawiki_timestamp(self, timestamp_repr): """ Convert to mediawiki timestamps """ if hasattr(timestamp_repr, 'strftime'): return timestamp_repr.strftime(MEDIAWIKI_TIMESTAMP_FORMAT) else: return date_parse(timestamp_repr).strftime( MEDIAWIKI_TIMESTAMP_FORMAT)
def return_point_timestamp(dev_id, unit, period, measurement=None, channel=None): current_app.config['INFLUXDB_USER'] = INFLUXDB_USER current_app.config['INFLUXDB_PASSWORD'] = INFLUXDB_PASSWORD current_app.config['INFLUXDB_DATABASE'] = INFLUXDB_DATABASE current_app.config['INFLUXDB_TIMEOUT'] = 5 dbcon = influx_db.connection query_str = query_string( unit, dev_id, measure=measurement, channel=channel, value='LAST', past_sec=period) if query_str == 1: return [None, None] try: raw_data = dbcon.query(query_str).raw number = len(raw_data['series'][0]['values']) time_raw = raw_data['series'][0]['values'][number - 1][0] value = raw_data['series'][0]['values'][number - 1][1] value = '{:.3f}'.format(float(value)) # Convert date-time to epoch (potential bottleneck for data) dt = date_parse(time_raw) timestamp = calendar.timegm(dt.timetuple()) * 1000 return [timestamp, value] except KeyError: return [None, None] except Exception: return [None, None]
def _printable_cert(self, docs): archive = docs['archive'] cipher = archive.meta.cipher if hasattr(cipher, 'mode'): cipher = cipher.mode created = archive.meta.created try: created = date_parse(created) except: pass expires = created + date_delta(years=30) md5 = b2a_hex(docs['auth'].md5).upper() key = b2a_hex(docs['cert'].key).upper() hk = pairs(fours(pairs(iter(key))), " . ") return resource_string(__name__, "data/certificate.html").format( json=as_json(docs), aid=docs['signature'].aid, keyB=next(hk), keyC=next(hk), keyD=next(hk), keyE=next(hk), name=archive.meta.name, email=archive.meta.email, uploaded=created.strftime("%c"), expires=expires.strftime("%c"), title=archive.title, desc=archive.description, md5=" . ".join(fours(pairs(iter(md5)))), fmt=archive.meta.format, cipher=cipher)
def setUp(self): data = load_data('report.json') self.records = data['asg']['records'] self.headers = data['asg']['headers'] self.rows = data['asg']['rows'] for rec in self.records.values(): rec['CustodianDate'] = date_parse(rec['CustodianDate'])
def insert_text_data(data_source, source_url, text, time_posted, session): """Adds the base entry for a text data source to the database and returns the newly created model Keyword arguments: data_source -- An enum indicating source. The enum is located in interns.models.models.AllowedSources source_url -- A string indicating the url the text was pulled from text -- the raw text data pulled from the url time_posted -- either a datetime object or a datetime string session -- active db session """ if not isinstance(time_posted, datetime): time_posted = date_parse(time_posted) eleanor_logger.debug('Inserting text data into postgres') TextModel = models.TextSource( source_key=data_source, source_url=source_url, written_text=text, time_posted=time_posted ) session.add(TextModel) return TextModel
def search_count_of_user_tweets_on_day(username, date, search_term): """When given a username, datetime, and search_term return the number of times search term was tweeted by username on the day of datetime""" return_data = {} date = date_parse(date) start = datetime(year=date.year, month=date.month, day=date.day) end = start + timedelta(days=1) with GetDBSession() as db_session: user_query = db_session.query( twitter_models.TwitterSource ).filter( twitter_models.TwitterSource.tweeter_user_name == username ).join( twitter_models.TwitterSource.text_source, aliased=True ).filter( and_( and_( models.TextSource.time_posted > start, models.TextSource.time_posted < end ), models.TextSource.written_text.contains(search_term) ) ) return_data[username] = { search_term: user_query.count(), 'date': start.strftime('%Y-%m-%d') } return return_data
def ensure_latest_package(package_name: str, pip_args=[], md_file: str = 'VERSION.md', update_pkg_pip_args=[]): """Toss this in main to perform a check that the user is always running latest pip_args will be passed as a list to add things like trusted-host or extra-index-url. Example: pip_args = ['--extra-index-url', 'https://artifactory.com/api/pypi/eg/simple', '--trusted-host', 'artifactory.com'] :param package_name: :param pip_args: :param md_file: Display this file if the package was updated :param update_pkg_pip_args: pip args pass to update_package on out of date package e.g. --extra-index-url :return: """ config = Config('~/.double_click/package_versions.ini') if not config.has_section(package_name): config.add_section(package_name) last_checked = None else: last_checked = config.get(package_name, 'last_checked') if not last_checked or date_parse(last_checked) < dt.utcnow() - timedelta( hours=1): try: latest_version = None current_version = pkg_resources.get_distribution( package_name).version version_output = update_package(f'{package_name}==DoesNotExist', pip_args=update_pkg_pip_args) version_output = [ line for line in version_output.split('\n') if 'DoesNotExist (from versions:' in line ] if version_output: versions = re.findall(r'([0-9][^,)]+)', version_output[0]) latest_version = versions[-1] if latest_version and current_version != latest_version: update_pkg_pip_args = update_pkg_pip_args if update_pkg_pip_args else pip_args update_package(package_name, pip_args=update_pkg_pip_args) config.set(package_name, 'last_checked', str(dt.utcnow())) update_msg = f'#An update to {package_name} was retrieved that prevented your command from running.' if md_file: display_version(package_name, md_file) update_msg += f'\nPlease review changes and re-run your command.' else: update_msg += f'\nPlease re-run your command.' echo(update_msg) sys.exit(0) except pkg_resources.DistributionNotFound: # This should only occur during testing echo(f'{package_name} not found') config.set(package_name, 'last_checked', str(dt.utcnow())) config.save()
def parse_query(query): """ Parses the query for a chart and returns the relevant parameters that should be used to make the query on the database. @return: a tuple with all the relevant query information """ # Date range start_time = datetime.min if query.stats_to is not None and query.stats_to != "": start_time = date_parse(query.stats_to) end_time = datetime.now().replace(minute=0, second=0, microsecond=0) if query.stats_from is not None and query.stats_from != "": end_time = date_parse(query.stats_from) date_range = [start_time, end_time] # Exercises filter exercise_list = [ name.strip() for name in query.filter_exercises.split(",") if name.strip() != "" ] # Tags filter tag_list = [ tag.strip() for tag in query.filter_tags.split(",") if tag.strip() != "" ] # Min-max grades min_grade = 0 if query.min_submission_grade is not None and query.min_submission_grade != "": percentage = query.min_submission_grade if percentage.endswith('%'): percentage = percentage[:-1] min_grade = float(percentage) max_grade = 100 if query.max_submission_grade is not None and query.max_submission_grade != "": percentage = query.max_submission_grade if percentage.endswith('%'): percentage = percentage[:-1] max_grade = float(percentage) grade_bounds = (min_grade, max_grade) return (query.chart_type, date_range, exercise_list, tag_list, grade_bounds, query.submissions_filter)
def _process_barcelona_rows(self, table): """Process rows for barcelona port. Decode all rows as unicode strings since tabula outputs byte strings by default. Extract matching date for each table section based on the section's description. Yield only rows that contain table data, skipping table section description. Known table section headers (each section has a different matching date): - "Buques que efectuaron operaciones durante la noche del día anterior" - "Buques que efectuaron operaciones durante las 20 horas del viernes 16.3.18 hata el domingo 18.3.18" # noqa - "Buques que efectuaron operaciones durante el lunes 18.3.18" - "Ultima hora" Args: table (List[List[str]]): list of table rows from pdf Yields: List[str]: """ matching_date = None for idx, row in enumerate(table): # tabula stores string data as bytes by default row = [cell for cell in row] # try deciphering matching_date of subsequent rows if any('Buques que efectuaron' in cell for cell in row): raw_matching_date = ''.join(row) date_match = re.search(r'(\d{1,2}\.\d{1,2}\.\d{2})', raw_matching_date) # matching date is mentioned explictly in table section description if date_match: matching_date = to_isoformat(date_match.group(1)) logger.debug( 'Found matching date: {}'.format(matching_date)) # sometimes matching date is described implicitly in words in the pdf elif 'anterior' in raw_matching_date: matching_date = to_isoformat( str( date_parse(self.reported_date, dayfirst=False) - timedelta(days=1)), dayfirst=False, ) logger.debug( 'Found matching date: {}'.format(matching_date)) else: raise ValueError('Unable to find matching date: {}'.format( raw_matching_date)) elif any('ltima hora' in cell for cell in row): matching_date = self.reported_date logger.debug('Found matching date: {}'.format(matching_date)) # do not yield table section headers if not ('/EXPORT' in row or any('Buques que efectuaron' in cell for cell in row) or any('ltima hora' in cell for cell in row)): row.append('matching_date' if idx == 0 else matching_date) yield row
def iso_date(date_str): """ this method will make sure that dates are formatted properly as with isoformat :param date_str: :return: YYYY-MM-DD date formatted """ return date_parse(date_str).date().isoformat()
def date_query(expr, value): operator_func, value = parse_query_value(value) try: date_value = date_parse(value) except ValueError: date_value = datetime.datetime.now() return operator_func(expr, date_value)
def test_parse(self): """Test the parse_raw method.""" data = copy.deepcopy(TEST_CHECK) check = Check().parse(data) assert check.onfido_id == TEST_CHECK["id"] assert check.created_at == date_parse(TEST_CHECK["created_at"]) assert check.status == "in_progress" assert check.result is None
def get_launch_by_slug(launch_slug): launch = launches[launch_slug] data = get_weather(launch['lat'], launch['lng']) log.debug(data) interval = request.args.get("interval") if interval == None or interval == "daily": result = [] grouped = itertools.groupby( data["properties"]["periods"], lambda x: date_parse(x["startTime"]).date().isoformat(), ) for day, hours in grouped: wind_speed = [] wind_direction = [] for hour in hours: start = date_parse(hour["startTime"]) if 6 < start.hour < 16: wind = make_wind_dict(hour) wind_speed.append(wind["speed"]) wind_direction.append(wind["direction"]) try: speed_average = statistics.mean(wind_speed) direction_average = statistics.mean(wind_direction) except statistics.StatisticsError as _: pass finally: result.append( {day: make_time_unit_dict(speed_average, direction_average, launch)} ) return jsonify(result) elif interval == "hourly": result = [] for hour in data["properties"]["periods"]: wind = make_wind_dict(hour) result.append( { date_parse(hour["startTime"]).isoformat(): make_time_unit_dict( wind["speed"], wind["direction"], launch ) } ) return jsonify(result) else: return "Invalid interval", 400
def db_date(value, context): try: value = date_parse(value) except (ValueError, e): raise Invalid(str(e)) return value
def request_as_comment_dict(self, request): return { 'who': request.creator, 'when': date_parse(request.statehistory[0].when), 'id': '-1', 'parent': None, 'comment': request.description, }
def start_date(sync_db: sqlalchemy.engine.base.Engine, env_start_date: str) -> datetime: last_date: Optional[datetime] = last_sync_date(sync_db) if last_date is not None: return last_date + timedelta(days=1) if env_start_date == "": return datetime.today() return date_parse(env_start_date)
def filter_expiry_before(value): if value: try: date = date_parse(value, dayfirst=True).date() return Q(end_date__lte=date) except: pass return None
def create(self, date_str): ''' Convert string to datetime.datetime ''' try: return date_parse(date_str) except: return date_str
def dict_from_parsed(cls, parsed_point, dataset_id, training_selector): timestamp = date_parse(parsed_point[0]) return dict(created_at=datetime.datetime.now(), timestamp=timestamp, unit=parsed_point[1], value=float(parsed_point[2]), dataset_id=dataset_id, training=training_selector(timestamp))
def test_output(bucket, client, request): bucket().blob.return_value = key = mock.MagicMock() with mock_datetime_now(date_parse('2020/06/10 13:00'), datetime): output = get_blob_output(request) assert output.key_prefix == 'policies/xyz/2020/06/10/13' output.upload_file('resources.json', f"{output.key_prefix}/resources.json") key.upload_from_filename.assert_called_once()