def iterate(uid, start_date=None, end_date=None, class_name=None): date1 = (dtparse(start_date).replace( tzinfo=tmzn) if start_date else datetime.now(tz=tmzn)) date2 = (dtparse(end_date).replace(tzinfo=tmzn) if end_date else date1.replace(hour=23, minute=59, second=59)) class_list = [] for event in calendars[uid].walk("vevent"): if event["dtstart"].dt >= date1 and event["dtend"].dt <= date2: if not start_date: class_list.append(format_event(event)) break if class_name: class_name = [class_name] if isinstance(class_name, str) else class_name for c_name in class_name: if (fuzz.partial_ratio(c_name.lower(), event["summary"].lower()) > fuzz_threshold): class_list.append(format_event(event)) else: class_list.append(format_event(event)) return class_list
def message_to_values(self, key, value): payload = payload_parse(value) top_time = dtparse(value['time']).replace(tzinfo=pytz.utc) apply_start_end_filter(top_time, self.filters.get('start_date'), self.filters.get('end_date')) value['lat'] = float(value['lat']) value['lon'] = float(value['lon']) pt = Point(value['lon'], value['lat']) value['geom'] = from_shape(pt, srid=4326) if not value['values']: value['values'] = {} value['values']['location_quality'] = get_point_location_quality(pt) # All HSTORE values need to be strings value['values'] = { k: make_valid_string(x) for k, x in value['values'].items() } value['time'] = top_time.isoformat() if 'reftime' in value: value['reftime'] = dtparse( value['reftime']).replace(tzinfo=pytz.utc).isoformat() else: value['reftime'] = value['time'] value['payload'] = payload # Throw away non-column data value = self.match_columns(value) # Remove None to use the defaults defined in the table definition return key, {k: v for k, v in value.items() if v is not None}
def test_crtp_calculated_metadata_missing_time(self): axes = { 't': 'time', 'x': 'longitude', 'y': 'latitude', 'z': 'depth', } with ContiguousRaggedTrajectoryProfile(self.missing_time) as mmt: t = mmt.calculated_metadata(axes=axes) assert t.min_t == dtparse('2014-11-16 21:32:29.952500') assert t.max_t == dtparse('2014-11-17 07:59:08.398500') assert len(t.trajectories) == 1 traj = t.trajectories["UW157-20141116T211809"] assert np.isclose(traj.min_z, 0.47928014) assert np.isclose(traj.max_z, 529.68005) assert traj.min_t == dtparse('2014-11-16 21:32:29.952500') assert traj.max_t == dtparse('2014-11-17 07:59:08.398500') first_loc = traj.geometry.coords[0] assert np.isclose(first_loc[0], -124.681526638573) assert np.isclose(first_loc[1], 43.5022166666667) assert len(traj.profiles) == 13
def _parse_args(): opts = docopt(__doc__) if opts['transactions']: # from-date if opts['<from-date>'] is not None: from_date = dtparse(opts['<from-date>']).date() else: from_date = None # to-date if opts['<to-date>'] is not None: to_date = dtparse(opts['<to-date>']).date() else: to_date = None return { 'cmd': 'transactions', 'fmt': opts['--format'], 'token': opts['<token>'], 'from_date': from_date, 'to_date': to_date, 'output_file': opts['--output-file'], } elif opts['statement']: return { 'cmd': 'statement', 'token': opts['<token>'], 'year': opts['<year>'], 'statement_id': int(opts['<statement>']), 'fmt': opts['--format'], 'output_file': opts['--output-file'], }
def get_events(service, calendar_id, start, end, time_zone="America/New_York"): events_result = service.events().list(calendarId=calendar_id, timeMin=start, timeMax=end, maxResults=400, singleEvents=True, orderBy='startTime', timeZone=time_zone).execute() events = events_result.get('items', []) if not events: return [] else: out = [] for event in events: # start = event['start'].get('dateTime', event['start'].get('date')) # end = event['end'].get('dateTime', event['end'].get('date')) try: start = dtparse(event['start']['dateTime']) end = dtparse(event['end']['dateTime']) except: print(event['start']) curr = {'start': start, 'end': end, 'name': event['summary']} out.append(curr) return out
def test_imt_calculated_metadata(self): with IncompleteMultidimensionalTrajectory(self.single) as ncd: s = ncd.calculated_metadata() assert s.min_t == dtparse('1990-01-01 00:00:00') assert s.max_t == dtparse('1990-01-05 03:00:00') traj1 = s.trajectories["Trajectory1"] assert traj1.min_z == 0 assert traj1.max_z == 99 assert traj1.min_t == dtparse('1990-01-01 00:00:00') assert traj1.max_t == dtparse('1990-01-05 03:00:00') assert np.isclose(traj1.first_loc.x, -7.9336) assert np.isclose(traj1.first_loc.y, 42.00339) with IncompleteMultidimensionalTrajectory(self.multi) as ncd: m = ncd.calculated_metadata() assert m.min_t == dtparse('1990-01-01 00:00:00') assert m.max_t == dtparse('1990-01-02 12:00:00') assert len(m.trajectories) == 4 traj0 = m.trajectories["Trajectory0"] assert traj0.min_z == 0 assert traj0.max_z == 35 assert traj0.min_t == dtparse('1990-01-01 00:00:00') assert traj0.max_t == dtparse('1990-01-02 11:00:00') assert np.isclose(traj0.first_loc.x, -35.07884) assert np.isclose(traj0.first_loc.y, 2.15286) traj3 = m.trajectories["Trajectory3"] assert traj3.min_z == 0 assert traj3.max_z == 36 assert traj3.min_t == dtparse('1990-01-01 00:00:00') assert traj3.max_t == dtparse('1990-01-02 12:00:00') assert np.isclose(traj3.first_loc.x, -73.3026) assert np.isclose(traj3.first_loc.y, 1.95761)
def get_metric_statistics(self): namespace = self._get_param("Namespace") metric_name = self._get_param("MetricName") start_time = dtparse(self._get_param("StartTime")) end_time = dtparse(self._get_param("EndTime")) period = int(self._get_param("Period")) statistics = self._get_multi_param("Statistics.member") dimensions = self._get_multi_param("Dimensions.member") # Unsupported Parameters (To Be Implemented) unit = self._get_param("Unit") extended_statistics = self._get_param("ExtendedStatistics") # TODO: this should instead throw InvalidParameterCombination if not statistics and not extended_statistics: raise NotImplementedError( "Must specify either Statistics or ExtendedStatistics") datapoints = self.cloudwatch_backend.get_metric_statistics( namespace, metric_name, start_time, end_time, period, statistics, unit=unit, dimensions=dimensions, ) template = self.response_template(GET_METRIC_STATISTICS_TEMPLATE) return template.render(label=metric_name, datapoints=datapoints)
def list(self): service = build('calendar', 'v3', credentials=self.creds) now = dt.utcnow().isoformat() + 'Z' result = service.events().list(calendarId='primary', timeMin=now, maxResults=20, singleEvents=True, orderBy='startTime').execute() events = result.get('items', []) # 2019-11-05T10:00:00-08:00 items = [] for event in events: start = event['start'].get('dateTime', event['start'].get('date')) st_date = dt.strftime(dtparse(start), format='%m-%d-%Y') st_time = dt.strftime(dtparse(start), format='%I:%M%p') items.append({ "date": st_date, "time": st_time, "content": event['summary'] }) return items
def handle(self, *args, **options): sdate = None edate = None try: sdate = dtparse(options['start_date']) except ValueError as ve: self.stderr.write("Could not parse start-date '{}'\n".format(options['start_date'])) return try: edate = dtparse(options['end_date']) except ValueError as ve: self.stderr.write("Could not parse end-date '{}'\n".format(options['end_date'])) return if not options['all_stats'] and not options['format_stats'] and not options['card_stats']: self.stderr.write("No action taken. Specify all, formats, or cards.\n") fn = None if 'formatname' in options: fn = options['formatname'] if options['all_stats'] or options['format_stats']: self.stderr.write("formats...\n") FormatStat.calc_all(new_only=options['new_only'], start_date=sdate, end_date=edate, only_formatname=fn) if options['all_stats'] or options['card_stats']: self.stderr.write("cards...\n") FormatCardStat.calc_all(new_only=options['new_only'], start_date=sdate, end_date=edate, only_formatname=fn)
def describe_job_flows( self, job_flow_ids=None, job_flow_states=None, created_after=None, created_before=None, ): clusters = self.clusters.values() within_two_month = datetime.now(pytz.utc) - timedelta(days=60) clusters = [ c for c in clusters if c.creation_datetime >= within_two_month ] if job_flow_ids: clusters = [c for c in clusters if c.id in job_flow_ids] if job_flow_states: clusters = [c for c in clusters if c.state in job_flow_states] if created_after: created_after = dtparse(created_after) clusters = [ c for c in clusters if c.creation_datetime > created_after ] if created_before: created_before = dtparse(created_before) clusters = [ c for c in clusters if c.creation_datetime < created_before ] # Amazon EMR can return a maximum of 512 job flow descriptions return sorted(clusters, key=lambda x: x.id)[:512]
def get_measure_counts_for_dates(self, start_date, end_date): search_date = dtparse(start_date) search_date2 = dtparse(end_date) pat_cnt = {} for obn in self.observation_list: pat_cnt[obn] = sum((self.pat_dat[obn]['dt'] >= search_date) & (self.pat_dat[obn]['dt'] <= search_date2)) return pat_cnt
def test_call_volume_for_day(self): overview = CallVolumeOverview( q("time_received__gte=2015-01-01&time_received__lte=2015-01-01")) assert overview.bounds == {"min_time": dtparse("2015-01-01T09:00"), "max_time": dtparse('2015-01-01T12:30')} assert_list_equiv(overview.volume_by_date(), [{"date": dtparse("2015-01-01T09:00"), "volume": 1}, {"date": dtparse("2015-01-01T12:00"), "volume": 1}])
def valid_coupon(self, coupon): if not coupon.get('status', False): self.log.info('Coupon code disabled %s', coupon.get('code')) return False start = dtparse(coupon.get('start')).date() end = dtparse(coupon.get('end')).date() now = datetime.utcnow().date() self.log.info('Comparing coupon dates %s %s %s', start, end, now) return start < now and end > now
def message_to_values(self, key, value): payload = payload_parse(value) values = flatten(value) top_time = dtparse(values['timestamp']).replace(tzinfo=pytz.utc) apply_start_end_filter(top_time, self.filters.get('start_date'), self.filters.get('end_date')) top_level = { 'uid': values['imei'], 'gid': None, 'time': top_time.isoformat(), 'reftime': dtparse(values['navsat_fix_time']).replace( tzinfo=pytz.utc).isoformat(), 'lat': values['latitude'], 'lon': values['longitude'], 'z': None, 'payload': payload, } pt = Point(top_level['lon'], top_level['lat']) top_level['geom'] = from_shape(pt, srid=4326) skips = [ # No easy way to represent this as a flat dict. We can write a db view to extract this # data from the `payload` if required. 'data_segment_data_product_pipeline', 'data_segment_data_segment_data_product_pipeline' ] # Set additional values # Lat=91 and Lon=181 should be treated as bad location data values['location_quality'] = get_point_location_quality( pt, disallow_lon=[181], disallow_lat=[91]) values['mfr'] = 'numurus' # All HSTORE values need to be strings values = { k: make_valid_string(x) if x is not None else None for k, x in values.items() if k not in skips } fullvalues = {**top_level, 'values': {**values}} # Throw away non-column data fullvalues = self.match_columns(fullvalues) # Remove None to use the defaults defined in the table definition return key, {k: v for k, v in fullvalues.items() if v is not None}
def get_aggragation_for_dates_old(self, start_date, end_date): search_date = dtparse(start_date) search_date2 = dtparse(end_date) pat_ag = {} for obn in self.observation_list: pat_ag[obn] = self.pat_ag_func[obn](self.pat_dat[obn]['value'][ (self.pat_dat[obn]['dt'] >= search_date) & (self.pat_dat[obn]['dt'] <= search_date2)]) return pat_ag
def test_call_volume_for_month(self): overview = CallVolumeOverview( q("time_received__gte=2015-01-01&time_received__lte=2015-02-02")) results = overview.volume_by_date() assert overview.bounds == {"min_time": dtparse("2015-01-01T09:00"), "max_time": dtparse('2015-02-01T09:00')} assert_list_equiv(results, [{"date": dtparse("2015-01-01T00:00"), "volume": 2}, {"date": dtparse("2015-01-08T00:00"), "volume": 1}, {"date": dtparse("2015-02-01T00:00"), "volume": 1}])
def get_metric_data(self): start = dtparse(self._get_param("StartTime")) end = dtparse(self._get_param("EndTime")) queries = self._get_list_prefix("MetricDataQueries.member") results = self.cloudwatch_backend.get_metric_data( start_time=start, end_time=end, queries=queries ) template = self.response_template(GET_METRIC_DATA_TEMPLATE) return template.render(results=results)
def test_loading_metadata(archive_meta): assert "PROTEGE" == archive_meta.game_system_id assert "PROTĂ©GĂ© Test Campaign" == archive_meta.name assert "Alex Mair" == archive_meta.author assert dtparse("2016-04-20") == archive_meta.creation_date assert dtparse("2016-04-21") == archive_meta.revision_date assert "978-3-16-148410-0" == archive_meta.isbn assert "This is a test campaign for " \ "PROTĂ©GĂ©, whose main purpose is " \ "to ensure that dmclient has decent " \ "unit tests." == archive_meta.description
async def edit_event_time(ctx, client, args): CREDS = await get_credentials(ctx, client) service = build('calendar', 'v3', credentials=CREDS) summary = args[0].strip() starttime_arg = args[1].strip().lower() start_time = await set_time(ctx, starttime_arg) if len(start_time) < 1: return events = retrieve_all_events(service, UMARL_CALENDAR_ID) if not events: await ctx.send( embed=discord.Embed(description='No events exist on the calendar.', color=discord.Color.red())) return event_id, event_name = retrieve_event_id(summary, events) event = service.events().get(calendarId=UMARL_CALENDAR_ID, eventId=event_id).execute() if len(args) > 2: date_arg = args[2].strip() date_str = await check_and_format_date(ctx, date_arg) if len(date_str) < 1: return else: date_str = event['start']['dateTime'].split('T')[0] if len(args) > 3: duration = args[3].strip() if int(duration) < 15 or int(duration) > 1440: await ctx.send(embed=discord.Embed( description= "Invalid duration, please input a duration (in minutes) between 15 and 1440.", color=discord.Color.red())) return else: start_datetime = dtparse(event['start']['dateTime']) end_datetime = dtparse(event['end']['dateTime']) minutes_difference_timedelta = end_datetime - start_datetime duration = int(minutes_difference_timedelta.seconds / 60) event['start']['dateTime'] = date_str + 'T' + start_time event['end']['dateTime'] = await set_end_time(ctx, duration, date_str + 'T' + start_time) updated_event = service.events().update(calendarId=UMARL_CALENDAR_ID, eventId=event_id, body=event).execute() await ctx.send(embed=discord.Embed(description="Event time updated!", color=discord.Color.green()))
def GetCalendarEvents(): print('[Calendar API] Get last events') creds = None tokendir = os.path.join(rootdir, 'token.pickle') #Check if the user already logged if os.path.exists(tokendir): with open(tokendir, 'rb') as token: creds = pickle.load(token) # If the file doesn't exists, log the user if not creds or not creds.valid: if creds and creds.expired and creds.refresh_token: creds.refresh(Request()) else: flow = InstalledAppFlow.from_client_secrets_file( os.path.join(rootdir, 'credentials.json'), SCOPES) creds = flow.run_local_server(port=0) # Save the credentials for the next run with open(tokendir, 'wb') as token: pickle.dump(creds, token) service = build('calendar', 'v3', credentials=creds, cache_discovery=False) # Call the Calendar API now = datetime.datetime.utcnow().isoformat() + 'Z' print('[Calendar API] Getting the upcoming 7 events') events_result = service.events().list(calendarId=os.getenv('CALENDAR_ID'), timeMin=now, maxResults=7, singleEvents=True, orderBy='startTime').execute() events = events_result.get('items', []) return_events = [] if not events: print('No upcoming events found') for event in events: start = event['start'].get('dateTime', event['start'].get('date')) end = event['end'].get('dateTime', event['end'].get('date')) start_datetime = dtparse(start) end_datetime = dtparse(end) new_event = { "start_date": start_datetime, "end_date": end_datetime, "title": event["summary"] } return_events.append(new_event) return return_events
def test_get_creation(self): meta = utils.get_creation_attributes(history='DID THINGS') now = datetime.utcnow().replace(tzinfo=pytz.utc) assert (now - dtparse(meta['attributes']['date_created'])) < timedelta( minutes=1) assert (now - dtparse(meta['attributes']['date_issued'])) < timedelta( minutes=1) assert (now - dtparse( meta['attributes']['date_modified'])) < timedelta(minutes=1) assert 'DID THINGS' in meta['attributes']['history']
def get_final(self, result): photos = result[0]['Items'] for photo in photos: # Datetaken field is a string, e.g. "2014-03-17 01:55:38". raw_date = dtparse(photo['ImageDate']) photo['ImageDate'] = datetime.date(raw_date.year, raw_date.month, raw_date.day) raw_date = dtparse(photo['Created']) photo['Created'] = datetime.date(raw_date.year, raw_date.month, raw_date.day) yield photo if len(photos) == 0: raise sundew.StopPipeline
def parse_free_busy(free_busy): if 'busy' not in free_busy: return None busy_times = [] for busy_slot_dict in free_busy['busy']: start_time = dtparse(busy_slot_dict['start'], ignoretz=True) end_time = dtparse(busy_slot_dict['end'], ignoretz=True) busy_times.append(TimeSlot(start_time, end_time, buffer_time=0)) return busy_times
def is_date(string, fuzzy=False): """ Return whether the string can be interpreted as a date. :param string: str, string to check for date :param fuzzy: bool, ignore unknown tokens in string if True """ try: dtparse(string, fuzzy=fuzzy) return True except ValueError: return False
async def test_multiple_timezones_for_reserved_batch_session_predicate( mock_dt): mock_db_conn = MagicMock() mock_sched_ctx = MagicMock() mock_sess_ctx = MagicMock() mock_sess_ctx.session_type = SessionTypes.BATCH mock_sess_ctx.kernel_id = 'fake-kernel-id' now = '2020-06-29T00:00:00+00:00' mock_dt.now = MagicMock(return_value=dtparse(now)) # Start time is not yet reached (now < start time) start_time = '2020-06-29T00:00:01+00:00' mock_db_conn.scalar = AsyncMock(return_value=dtparse(start_time)) result = await check_reserved_batch_session(mock_db_conn, mock_sched_ctx, mock_sess_ctx) assert not result.passed, (now, start_time) # Start time is reached (now > start time) start_time = '2020-06-28T23:59:59+00:00' mock_db_conn.scalar = AsyncMock(return_value=dtparse(start_time)) result = await check_reserved_batch_session(mock_db_conn, mock_sched_ctx, mock_sess_ctx) assert result.passed, (now, start_time) # Start time is not yet reached by timezone (now < start time) # Note that 6/29 00:00 (UTC) < 6/29 00:00 (-09:00) == 6/29 09:00 (UTC) for i in range(1, 12): start_time = f'2020-06-29T00:00:00-{i:02d}:00' mock_db_conn.scalar = AsyncMock(return_value=dtparse(start_time)) result = await check_reserved_batch_session(mock_db_conn, mock_sched_ctx, mock_sess_ctx) assert not result.passed, (now, start_time) # Start time is reached by timezone (now > start time) # Note that 6/29 00:00 (UTC) > 6/29 00:00 (+09:00) == 6/28 15:00 (UTC) for i in range(1, 12): start_time = f'2020-06-29T00:00:00+{i:02d}:00' mock_db_conn.scalar = AsyncMock(return_value=dtparse(start_time)) result = await check_reserved_batch_session(mock_db_conn, mock_sched_ctx, mock_sess_ctx) assert result.passed, (now, start_time) # Should pass if start time is not specified (start immediately). mock_db_conn.scalar = AsyncMock(return_value=None) result = await check_reserved_batch_session(mock_db_conn, mock_sched_ctx, mock_sess_ctx) assert result.passed
def parse_start_end_times(base): times = {} reg = 'S1-COD_M([0-9S]*)_TN([0-9]*)_F(.*?)_S([0-9])_([0-9T]*)-([0-9T]*)-([0-9T]*)' match = re.search(reg, base) frame = match.group(1) track = match.group(2) slcpframe = match.group(3) subswath = match.group(4) start = dtparse(match.group(5)).strftime('%Y-%m-%dT%H:%M:%S') mid = dtparse(match.group(6)).strftime('%Y-%m-%dT%H:%M:%S') end = dtparse(match.group(7)).strftime('%Y-%m-%dT%H:%M:%S') times['starttime'] = start times['endtime'] = end return times
def get_list_of_events(service, n): now = datetime.now(timezone(settings.TIME_ZONE)).isoformat() # out_time_format = "%I:%M%p on %A, %B %d" out_date_format = "%Y-%m-%d" out_time_format = "%H:%M" events_result = service.events().list( calendarId='primary', timeMin=now, maxResults=n, singleEvents=True, orderBy='startTime').execute() events = events_result.get('items', []) listOfEvents = [] for event in events: summary = location = description = end_date = start_date = "" # print(f'\n\nDEBUG: {event}\n\n', file=sys.stderr) if 'start' in event: if 'date' in event['start']: start_date = event['start']['date'] if 'dateTime' in event['start']: start_date = event['start']['dateTime'] #formatted as "05:00PM on Friday, December 15" start_string_date = datetime.strftime(dtparse(start_date), format=out_date_format) start_string_time = datetime.strftime(dtparse(start_date), format=out_time_format) if 'end' in event: if 'date' in event['end']: end_date = event['end']['date'] if 'dateTime' in event['end']: end_date = event['end']['dateTime'] #formatted as "05:00PM on Friday, December 15" end_string_date = datetime.strftime(dtparse(end_date), format=out_date_format) end_string_time = datetime.strftime(dtparse(end_date), format=out_time_format) if 'location' in event: location = event['location'] if 'summary' in event: summary = event['summary'] if 'description' in event: description = event['description'] # Date and time are picked up better when we just pass the date time string as it is received from google # string = f'{summary} . {description} . {location} . from {start_date} . to {end_date}.' string = f'{summary} . {description} . {location} . from {start_string_date} {start_string_time}. to {end_string_date} {end_string_time}.' if settings.DEBUG: print(f'\n\nDEBUG: {string}\n\n', file=sys.stderr) listOfEvents.append((string, event)) # return a tuple of event return listOfEvents
def test_just_missing_time(self): with ContiguousRaggedTrajectoryProfile(self.missing_time) as mmt: t = mmt.calculated_metadata() assert t.min_t == dtparse('2014-11-16 21:32:29.952500') assert t.max_t == dtparse('2014-11-17 07:59:08.398500') assert len(t.trajectories) == 1 traj = t.trajectories["UW157-20141116T211809"] assert np.isclose(traj.min_z, 0.47928014) assert np.isclose(traj.max_z, 529.68005) assert traj.min_t == dtparse('2014-11-16 21:32:29.952500') assert traj.max_t == dtparse('2014-11-17 07:59:08.398500') assert np.isclose(traj.first_loc.x, -124.681526638573) assert np.isclose(traj.first_loc.y, 43.5022166666667) assert len(traj.profiles) == 13
def test_imt_calculated_metadata_single(self): filepath = os.path.join(os.path.dirname(__file__), 'resources', 'im-single.nc') with IncompleteMultidimensionalTrajectory(filepath) as ncd: s = ncd.calculated_metadata() assert s.min_t.round('S') == dtparse('1990-01-01 00:00:00') assert s.max_t.round('S') == dtparse('1990-01-05 03:00:00') traj1 = s.trajectories["Trajectory1"] assert traj1.min_z == 0 assert traj1.max_z == 99 assert traj1.min_t.round('S') == dtparse('1990-01-01 00:00:00') assert traj1.max_t.round('S') == dtparse('1990-01-05 03:00:00') first_loc = traj1.geometry.coords[0] assert np.isclose(first_loc[0], -7.9336) assert np.isclose(first_loc[1], 42.00339)
def extract_data_csv(data): datasets = defaultdict(dict) for dp in data: ts = dtparse(dp["point"]["start"]).replace(tzinfo = utc, microsecond = 0) name = dp["timeseriesDesc"]["metric"].split("/")[-1] # benchmark or single labels = dict((k.split("/")[-1], v) for k, v in dp["timeseriesDesc"]["labels"].iteritems()) kind = labels.get('kind', '?') # print("name: %s, kind: %s" % (name, kind)) if kind == "single": # special case for nb_sagemath, ... kind = name[3:] name = "instances" value = dp["point"].get("int64Value") # value must be a string! if value is None: dv = dp["point"].get("doubleValue") if name == "instances": value = '%d' % int(dv) else: value = '%f' % dv else: value = '%d' % value datasets[name]["timestamp"] = ts.isoformat() datasets[name][kind] = value csv = {} for name, data in datasets.iteritems(): # sort by kind, and timestamp in the front header, line = zip(*sorted(data.items(), key = lambda x : (x[0] != 'timestamp', x[0]))) csv[name] = (header, line) return csv
def main(): global screen global font pygame.init() # TODO: use display info # display_info = pygame.display.info() screen = pygame.display.set_mode((320,240)) font = pygame.font.Font(None, 40) now = datetime.datetime.now() today = now.date() one_month_ago = today - datetime.timedelta(days=30) if os.path.exists('db.json') and now.timestamp() - os.stat('db.json').st_mtime < 3000: pics = load_db() else: pics = refresh_db(progress_cb) clear_display() included = set() start_progress('Adding recent pictures', len(pics)) for i, (key,item) in enumerate(pics.items()): when = dtparse(item['mediaMetadata']['creationTime']) if when.date() >= one_month_ago: included.add(key) add_to_display(item) update_progress(i) added = 0 start_progress('Adding random pictures', num_pics) while added < num_pics: random_key = random.choice(list(pics.keys())) if random_key in included: continue add_to_display(pics[random_key]) added = added + 1 update_progress(added) pygame.display.quit() display_images()
def extract_data_csv(data): datasets = defaultdict(dict) for dp in data: ts = dtparse(dp["point"]["start"]).replace(tzinfo=utc, microsecond=0) name = dp["timeseriesDesc"]["metric"].split("/")[ -1] # benchmark or single labels = dict((k.split("/")[-1], v) for k, v in dp["timeseriesDesc"]["labels"].iteritems()) kind = labels.get('kind', '?') # print("name: %s, kind: %s" % (name, kind)) if kind == "single": # special case for nb_sagemath, ... kind = name[3:] name = "instances" value = dp["point"].get("int64Value") # value must be a string! if value is None: dv = dp["point"].get("doubleValue") if name == "instances": value = '%d' % int(dv) else: value = '%f' % dv else: value = '%d' % value datasets[name]["timestamp"] = ts.isoformat() datasets[name][kind] = value csv = {} for name, data in datasets.iteritems(): # sort by kind, and timestamp in the front header, line = zip( *sorted(data.items(), key=lambda x: (x[0] != 'timestamp', x[0]))) csv[name] = (header, line) return csv
def list_clusters(self, cluster_states=None, created_after=None, created_before=None, marker=None): max_items = 50 clusters = self.clusters.values() if cluster_states: clusters = [c for c in clusters if c.state in cluster_states] if created_after: created_after = dtparse(created_after) clusters = [c for c in clusters if c.creation_datetime > created_after] if created_before: created_before = dtparse(created_before) clusters = [c for c in clusters if c.creation_datetime < created_before] clusters = sorted(clusters, key=lambda x: x.id) start_idx = 0 if marker is None else int(marker) marker = None if len(clusters) <= start_idx + max_items else str(start_idx + max_items) return clusters[start_idx:start_idx + max_items], marker
def date_parse(date_string, as_date=True): from dateutil.parser import parse as dtparse class simple_utc(datetime.tzinfo): def tzname(self, **kwargs): return "UTC" def utcoffset(self, dt): return datetime.timedelta(0) date_string = date_string.strip().lower() if date_string == 'today': date = datetime.datetime.utcnow() elif date_string == 'yesterday': date = datetime.datetime.utcnow() - datetime.timedelta(days=1) else: date = dtparse(date_string) date_tzinfo = date.replace(tzinfo=simple_utc()) if as_date: date = date_tzinfo.date() else: date = date_tzinfo return date
def isodate(self, datestring): try: date_obj = dtparse(datestring) return date_obj.strftime("%Y-%m-%d") except ValueError as ve: sys.stderr.write("ERROR: Could not parse '{}'\n".format(datestring)) raise ve
def _datetime_parser(dct): for key, value in list(dct.items()): try: dct[key] = dtparse(value) except Exception: pass return dct
def nite2key(nite): if isinstance(nite,basestring): nite = dtparse(nite) for k,v in EPOCHS.items(): if v['tstart'] < nite < v['tstop']: return k msg = "Nite outside of EPOCHS: %s"%nite raise Exception(msg)
def describe_job_flows(self, job_flow_ids=None, job_flow_states=None, created_after=None, created_before=None): clusters = self.clusters.values() within_two_month = datetime.now(pytz.utc) - timedelta(days=60) clusters = [c for c in clusters if c.creation_datetime >= within_two_month] if job_flow_ids: clusters = [c for c in clusters if c.id in job_flow_ids] if job_flow_states: clusters = [c for c in clusters if c.state in job_flow_states] if created_after: created_after = dtparse(created_after) clusters = [c for c in clusters if c.creation_datetime > created_after] if created_before: created_before = dtparse(created_before) clusters = [c for c in clusters if c.creation_datetime < created_before] return sorted(clusters, key=lambda x: x.id)[:512]
def str_to_datetime(string, timezone=pytz.utc): """ Convert string to datetime.datetime :param string: {str} datetime in string :param timezone: {pytz.timezone} :return: {datetime.datetime} datetime """ dt = dtparse(string) return timezone.localize(dt) if dt is not None else None
def extract_data_yaml(dp): # yes, it's a bit stupid to extract this from here, # but that way it works in general name = dp["timeseriesDesc"]["metric"].split("/")[-1] labels = dict((k.split("/")[-1], v) for k, v in dp["timeseriesDesc"]["labels"].iteritems()) ts = dtparse(dp["point"]["start"]).replace(tzinfo = utc) value = dp["point"].get("int64Value", None) or dp["point"].get("doubleValue") entry = {"name": name, "timestamp": ts, "value": value} if labels: entry["labels"] = labels return entry
def get_job_metadata(job_id, api_key=None): """ Get job metadata from CrowdFlower. Args: external_job_id (str or int): CrowdFlower job ID api_key (str): CrowdFlower API key (see README) Returns: metadata (dict): all available metadata for this job created_at (datetime.datetime): time of job creation completed_at (datetime.datetime or None): time of job completion, if completed, else None """ api_key = api_key or os.environ["CROWDFLOWER_API_KEY"] metadata = get_job(job_id) created_at = dtparse(metadata.get('created_at')) completed_at = metadata.get('completed_at') completed_at = dtparse(completed_at) if completed_at else None return metadata, created_at, completed_at
def get_metric_statistics(self): namespace = self._get_param('Namespace') metric_name = self._get_param('MetricName') start_time = dtparse(self._get_param('StartTime')) end_time = dtparse(self._get_param('EndTime')) period = int(self._get_param('Period')) statistics = self._get_multi_param("Statistics.member") # Unsupported Parameters (To Be Implemented) unit = self._get_param('Unit') extended_statistics = self._get_param('ExtendedStatistics') dimensions = self._get_param('Dimensions') if unit or extended_statistics or dimensions: raise NotImplemented() # TODO: this should instead throw InvalidParameterCombination if not statistics: raise NotImplemented("Must specify either Statistics or ExtendedStatistics") datapoints = self.cloudwatch_backend.get_metric_statistics(namespace, metric_name, start_time, end_time, period, statistics) template = self.response_template(GET_METRIC_STATISTICS_TEMPLATE) return template.render(label=metric_name, datapoints=datapoints)
def str2dt(timestr): """Convert a string date to a real date. Args: timestr (str) - the datetime as a raw string. Returns: dateutil.parser.parse - the parsed datetime object. """ try: return dtparse(timestr) except (TypeError, ValueError): if timestr in ['None', 'null']: return None return timestr
def parseStatus( soup ) : """ Parse the status page soup into a dictionary of visits. """ import numpy as np from dateutil.parser import parse as dtparse # from dateutil.relativedelta import relativedelta import datetime today = datetime.datetime.today() visdict = {} trowlist = soup.findAll('tr') for trow in trowlist : rowtext = trow.getText().strip().split('\n') rowtext = np.array( rowtext )[ np.where( [ len(rt) for rt in rowtext ] )[0] ] if rowtext[0] == 'Visit' : continue visit=rowtext[0] status=rowtext[1] targets=rowtext[2].replace('ANY','') configs=rowtext[3] startdate = dtparse( '9999-09-09' ) enddate = dtparse( '9999-09-09') if status in ['Archived', 'Executed', 'Scheduled'] : startdate = dtparse( rowtext[4].split('-')[0] ) enddate = dtparse( rowtext[5].split('-')[0] ) elif status in ['Scheduling','Implementation'] : planwindowlist = rowtext[4:] for planwindow in planwindowlist : if '-' not in planwindow : continue try : startdate0 = dtparse( planwindow.split('-')[0] ) enddate0 = dtparse( planwindow.split('-')[1].split('(')[0] ) except : continue daystostart0 = (startdate0 - today ).days daystoend0 = (startdate0 - today ).days if daystostart0>=0 and daystoend0>=0 : startdate = startdate0 enddate = enddate0 break daystostart = ( startdate - today ).days daystoend = ( enddate - today ).days visdict[visit] = {'visit':visit, 'status':status, 'targets':targets, 'configs':configs, 'startdate':startdate, 'enddate':enddate, 'daystostart':daystostart, 'daystoend':daystoend, } return( visdict )
def parse_date_args(self): """Check for start [and end] date arguments passed to the spider and parse into date objects. If there is no start date, raise an exception. If there is no end date, default to today's date. """ # Check that a start argument was passed to the spider if hasattr(self, 'start'): # Convert the start date string into a date object self.date_range_start = dtparse(self.start, dayfirst=True).date() else: log.msg("No start date supplied", level=log.CRITICAL) raise Exception("No start date supplied") if hasattr(self, 'end'): self.date_range_end = dtparse(self.end, dayfirst=True).date() else: self.date_range_end = date.today() self.log("Scraping date range: %s ----> %s" % (self.date_range_start, self.date_range_end), level=log.WARNING)
def parse_start_url(self, response): # For now, let's SKIP cached results if 'cached' in response.flags: # REVISIT - in the future, maybe we pass on cached items? But maybe not... maybe that all belongs at the pipeline level pass try: if True or response.url.index('displaydeck.php'): deck = self.parse_deck(response) if deck is not None and len(deck) > 0: yield deck self.log('***** DECK IS {}'.format(str(deck))) td_o = dtparse(deck['tournament_date']) titem = TournamentItem(name=deck['tournament_name'], url=deck['tournament_url'], tournament_format=deck['deck_format'], start_date=td_o, end_date=td_o) yield titem except ValueError as ve: pass
def run(self): logger.debug("Starting") while True: lines = parse(getPage(),fieldwidths) for line in lines: #bla query try: cur = con.cursor() cur.execute("""SELECT * FROM repeaters WHERE Call = "%s"; """ % (line[3].lower())) data = cur.fetchone() except: logger.error("Unable to open DB") continue if isinstance(data, (list, tuple)): if re.match('[a-zA-Z0-9_]',line[2]): try: logging.debug("%s TS%s: %s (%s) at %s:%s %s-%s-%s on %s" % (line[3],int(line[6]),line[2],line[8][1:],line[0][10:12],line[0][8:10],line[0][6:8],line[0][4:6],line[0][0:4],line[7][2:])) except: logging.debug("Line error") continue value = "%s (%s)" % (line[2],line[8][1:]) publish.single("hytera/%s/usrTs%s" % (data[2], int(line[6])) , value, hostname=mqtt_host) publish.single("hytera/%s/tlkTs%s" % (data[2], int(line[6])) , line[7][2:], hostname=mqtt_host) dt=dtparse("%s UTC"%line[0]) try: dt = dt.astimezone(tz.tzlocal()) tsLog[int(line[6])-1][data[2]] = dt.strftime("%s") except: logger.error("astimezone broke") for key,value in tsLog[0].iteritems(): publish.single("hytera/%s/lastTs1" % (key), value, hostname=mqtt_host) for key,value in tsLog[1].iteritems(): publish.single("hytera/%s/lastTs2" % (key), value, hostname=mqtt_host) sleep(float(config.get("Users","sleeptime")))
def published(self): return dtparse(self._item['published']).astimezone(tzutc())
The Dark Energy Survey and Operations: Year 1 http://adsabs.harvard.edu/abs/2014SPIE.9149E..0VD """ __author__ = "Alex Drlica-Wagner" from collections import OrderedDict as odict from dateutil.parser import parse as dtparse # Official dates can (mostly) be found in the documentation. Here we # extend the epochs to be continuous across time and to extend into Y3. #https://opensource.ncsa.illinois.edu/confluence/display/DESDM/Y2A1+Observing+Epochs EPOCHS = odict([ (1, dict(year='SV',epoch='E1', tstart=dtparse('20120912'),tstop=dtparse('20121229'))), (2, dict(year='SV',epoch='E2', tstart=dtparse('20121229'),tstop=dtparse('20130830'))), (3, dict(year='Y1',epoch='E1', tstart=dtparse('20130831'),tstop=dtparse('20131129'))), (4, dict(year='Y1',epoch='E2', tstart=dtparse('20131129'),tstop=dtparse('20140815'))), (5, dict(year='Y2',epoch='E1', tstart=dtparse('20140815'),tstop=dtparse('20141130'))), # Currently Y2_E2 extends through Y3... (6, dict(year='Y2',epoch='E2', tstart=dtparse('20141130'),tstop=dtparse('20160801'))), ]) def nite2key(nite): if isinstance(nite,basestring):
token = isogeo.connect() # Windows 10 notifications class notif = ToastNotifier() # ------------ REAL START ---------------------------- latest_data_modified = isogeo.search(token, page_size=10, order_by="modified", whole_share=0, sub_resources=["events"] ) # parsing the previous date last_exe = dtparse(last_exe).strftime("%a %d %B %Y (%H:%M)") # last_exe = last_exe.decode("Latin1") # comparing total of metadats shared since last time now_total = latest_data_modified.get('total') if now_total > last_total: notif.show_toast("Isogeo - Total partage", "{} données ajoutées\ndepuis le {}" .format(now_total - last_total, last_exe), icon_path=r"img/favicon.ico", duration=10) elif now_total < last_total: notif.show_toast("Isogeo - Total partage", "{} données retirées\ndepuis le {}" .format(last_total - now_total, last_exe),
def handle(self, *args, **options): #logger = logging.getLogger(__name__) # the first (and only) arg should be a filename directory = options['inputdir'] deck_meta_only = options['deck_meta_only'] self.only_newer = options['only_newer'] self.start_time = timezone.now() if not os.access(directory, os.R_OK): sys.stderr.write("Cannot read directory '{}'.\n".format(directory)) return # let's figure out the last time we ran. sema_filename = join(directory, 'last_complete.semaphore') last_start_time = datetime(2000, 1, 1, 0, 0, 0, 0, self.localtime) try: sema_in = open(sema_filename, 'r') lst_str = sema_in.readline() last_start_time = dtparse(lst_str) last_start_time = last_start_time.replace(tzinfo=self.localtime) sema_in.close() except: sys.stderr.write("Cannot read the {} file. Proceeding anyway...\n".format(sema_filename)) onlyfiles = [f for f in listdir(directory) if isfile(join(directory, f))] for filename in onlyfiles: if filename.find('tournament_') > -1: # and False: # REVISIT - commented out for the moment! filehandler = open(join(directory, filename)) filetime = datetime.fromtimestamp(os.path.getmtime(join(directory, filename))) filetime = filetime.replace(tzinfo=self.localtime) if self.only_newer and filetime < last_start_time: continue jblob = json.load(filehandler) #{"url": "/en/content/2011-great-britain-national-championship", "tournament_format": null, "name": "Great Britain National Championship", "start_date": "2011-08-19"} if jblob['name'].lower().find('test deck') > -1: sys.stdout.write("Tournament: skipped test deck tournament: {}\n".format(jblob['name'])) pass else: turl = jblob['url'] # The StarCityGames tournaments will probably not have a URL at all if turl is None or len(turl) == 0: pass # Some of the Wizards-crawled tournaments will not have a fully-qualified URL elif turl.find('http') < 0: turl = 'http://magic.wizards.com' + turl format_string = jblob['tournament_format'] if format_string is None: format_string = 'Unknown' elif format_string.find('Modern') > -1: format_string = 'Modern' elif format_string.find('Standard') > -1: format_string = 'Standard' elif format_string.find('Commander') > -1: format_string = 'Commander' elif format_string.find('Pauper') > -1: format_string = 'Pauper' elif format_string.find('Legacy') > -1: format_string = 'Legacy' elif format_string.find('Tiny') > -1: format_string = 'TinyLeaders' elif format_string == '' and 'legacy' in jblob['name'].lower(): format_string = 'Legacy' db_format = Format.objects.filter( formatname__iexact=format_string, start_date__lte=self.isodate(jblob['start_date']), end_date__gt=self.isodate(jblob['end_date'])).first() if db_format is not None: db_tournament = Tournament.objects.filter( format=db_format, name__iexact=jblob['name'], start_date=self.isodate(jblob['start_date'])).first() if db_tournament is None: # let's make one! db_tournament = Tournament( format=db_format, name=jblob['name'], start_date=self.isodate(jblob['start_date']), end_date=self.isodate(jblob['end_date']), url=turl) db_tournament.save() sys.stdout.write("Tournament: created {}\n".format(jblob['name'])) else: if filetime > db_tournament.updated_at or not self.only_newer: # let's update!! if turl.find('/node/') < 0: db_tournament.url = turl sys.stdout.write("Tournament: updated {}\n".format(jblob['name'])) db_tournament.save() else: sys.stdout.write("Tournament: skipped updated with node {}\n".format(jblob['name'])) else: sys.stdout.write("Tournament: not newer... skipped {}\n".format(jblob['name'])) else: sys.stdout.write("Tournament: skipped no format '{}' found in db {}\n".format(format_string, jblob['name'])) tournament_url_re = re.compile('^(.+/coverage/([^/]+))/?') # Process the deck files for filename in onlyfiles: if filename.find('deck_') > -1: ###fqfilename = join(directory, filename) # print "last modified: %s" % time.ctime(getmtime(fqfilename)) filehandler = open(join(directory, filename)) filetime = datetime.fromtimestamp(os.path.getmtime(join(directory, filename))) filetime = filetime.replace(tzinfo=self.localtime) if self.only_newer and filetime < last_start_time: continue jblob = json.load(filehandler) # let's test if jblob is a set JSON if 'name' in jblob and 'mainboard_cards' in jblob: # TEMP FIX - Legacy decks did not havea format_name for a lot of crawling... if jblob['deck_format'] == '' and jblob['tournament_name'] and 'legacy' in jblob['tournament_name'].lower(): jblob['deck_format'] = 'Legacy' tourna = None # let's deal with decks that have valid tournament URLs if 'tournament_url' in jblob and jblob['tournament_url'] is not None and len(jblob['tournament_url']) > 0: # We are going to skip decks and tournaments that look like draft and sealed if jblob['tournament_url'].find('draft') > 0 or jblob['tournament_url'].find('sealed') > 0: # skip sys.stdout.write("Deck: skipped limited {}\n".format(jblob['name'])) continue # now find the tournament tu_match = tournament_url_re.search(jblob['tournament_url']) tournament_url = jblob['tournament_url'] tournament_name = '' if tu_match: tournament_url = tu_match.group(1) tournament_name = tu_match.group(2) # find a real tournament that we match to #sys.stdout.write("HERE with '{}' and '{}'\n".format(tournament_url, tournament_name)) tourna = Tournament.objects.filter(url=tournament_url).first() # exclude(format__formatname='Unknown').first() elif 'tournament_name' in jblob and 'deck_format' in jblob: # Let's look for tournaments by name and date instead tourna = Tournament.objects.filter(name=jblob['tournament_name'], format__formatname=jblob['deck_format']).first() else: # we just aren't going to have a tournament. pass # Let's see if we already have a deck for this. deck = None sys.stdout.write("Deck: {} - {} by {}\n".format(jblob['url'], jblob['name'], jblob['author'])) is_create = False # Should we skip writing this deck to the database? skip_deck = False # We are going to look at the "deckid" parameter that StarCityGames decks have and match on just that id. dukey_m = SCG_DECK_URL_KEY_RE.search(jblob['url']) if dukey_m: #sys.stdout.write("** MATCHED: {}\n".format(dukey_m.group(1))) deck = Deck.objects.filter(url__icontains='deckid={}'.format(dukey_m.group(1))).first() #sys.stdout.write("** MATCHED: deck is {}\n".format(str(deck))) else: if 'page_part' in jblob: if jblob['page_part'] == '0' or jblob['page_part'] == 0: deck = Deck.objects.filter(url=jblob['url']).first() if deck is None: deck = Deck.objects.filter(url='{}#deck{}'.format(jblob['url'], jblob['page_part'])).first() else: deck = Deck.objects.filter(url='{}#deck{}'.format(jblob['url'], jblob['page_part'])).first() else: deck = Deck.objects.filter(url=jblob['url']).first() if deck is None: # no deck, so let's add it! sys.stdout.write(" creating...\n") is_create = True deck = Deck(url=jblob['url'], visibility=Deck.HIDDEN) if 'page_part' in jblob: deck.url = '{}#deck{}'.format(jblob['url'], jblob['page_part']) deck.name = jblob['name'] deck.authorname = jblob['author'] if tourna is not None and tourna.format is not None: deck.format = tourna.format elif 'deck_format' in jblob: try: sys.stderr.write( "Format find: {} - {}\n".format(jblob['deck_format'], self.isodate(jblob['tournament_date']))) db_format_q = Format.objects.filter( formatname=jblob['deck_format'], start_date__lte=self.isodate( jblob['tournament_date']), end_date__gt=self.isodate( jblob['tournament_date'])).first() deck.format = db_format_q except ValueError: # Looks like we cannot find a valid format because we don't have a real # deck_format or a real tournament_date. sys.stdout.write( " skipped because a valid format cannot be found for '{}' on date '{}'\n".format( jblob['deck_format'], jblob['tournament_date'])) deck = None pass if deck is not None: try: deck.save() sys.stdout.write(" Created deck {}!\n".format(str(deck.id))) deck = Deck.objects.get(pk=deck.id) # reload except IntegrityError as ie: sys.stdout.write(" Could not create deck! Integrity error: {}\n".format(str(ie))) deck = None else: if filetime > deck.updated_at or not self.only_newer: # for kicks, let's update name and authorname, since it could be that the # scrapy crawler gets better at figuring those things out deck.name = jblob['name'] deck.authorname = jblob['author'] deck.url = jblob['url'] if 'page_part' in jblob: deck.url = '{}#deck{}'.format(jblob['url'], jblob['page_part']) deck.format = None try: db_format_q = Format.objects.filter( formatname=jblob['deck_format'], start_date__lte=self.isodate( jblob['tournament_date']), end_date__gt=self.isodate( jblob['tournament_date'])).first() deck.format = db_format_q except ValueError as ve: # Looks like we cannot find a valid format because we don't have a real # deck_format or a real tournament_date. sys.stdout.write( " updating '{}' even though could not find a valid format for '{}' on date {} - {}\n".format( jblob['name'], jblob['deck_format'], jblob['tournament_date'], str(ve))) deck.format = None deck.save() sys.stdout.write(" Updated!\n") else: skip_deck = True sys.stdout.write(" not newer... skipped {}\n".format(jblob['url'])) #sys.stdout.write("Deck: {} (db card count {})\n".format(jblob['name'], deck.get_card_count())) #sys.stdout.write(" URL: {}\n".format(jblob['url'])) if deck is not None and not skip_deck: # now we have to add cards to the deck... if is_create or not deck_meta_only: sys.stdout.write(" creating or updating deck list...\n") cardtext = '\n'.join(jblob['mainboard_cards']) if 'sideboard_cards' in jblob and jblob['sideboard_cards'] is not None and len(jblob['sideboard_cards']) > 0: cardtext = cardtext + '\nSB:' + '\nSB: '.join(jblob['sideboard_cards']) if 'commandzone_cards' in jblob and jblob[ 'commandzone_cards'] is not None and len(jblob['commandzone_cards']) > 0: cardtext = cardtext + '\nCZ:' + '\nCZ: '.join(jblob['commandzone_cards']) #sys.stdout.write(cardtext + "\n") # REVISIT - just updating them all for now. Shouldn't hurt since # set_cards_from_text deletes all of the current card associations, right? if True or deck.get_card_count() == 0: try: deck.set_cards_from_text(cardtext) except Deck.CardsNotFoundException as cnfe: for g in cnfe.cnfes: try: sys.stdout.write(" ERROR: Could not find card " + str(g.text) + " in file {}\n".format(filename)) except UnicodeEncodeError: sys.stdout.write( " ERROR: Could not find card BUT I CAN'T TELL YOU ABOUT IT BECAUSE UNICODE IN PYTHON SUCKS in {}\n".format(filename)) if tourna is not None and tourna.name is not None and tourna.name.lower().find('test deck') < 0: # Now we will associate the deck to a tournament td = TournamentDeck.objects.filter(deck=deck, tournament=tourna).first() place = 99999 if 'place' in jblob: place = jblob['place'] if td is None: td = TournamentDeck(deck=deck, tournament=tourna, place=place) td.save() sys.stdout.write(" TournamentDeck: created for {}\n".format(jblob['name'])) td = TournamentDeck.objects.filter(deck=deck, tournament=tourna).first() # RELOAD else: td.place = place td.save() sys.stdout.write(" TournamentDeck: updated for {}\n".format(jblob['name'])) else: sys.stdout.write(" TournamentDeck: skipped no valid tournament {}\n".format(jblob['name'])) sema = open(join(directory, 'last_complete.semaphore'), 'w') sema.write(str(self.start_time)) sema.write("\n") sema.close()
def updated(self): return dtparse(self._item['updated']).astimezone(tzutc())
config = ConfigParser.SafeConfigParser() config.read(settings_file) share_id = config.get('auth', 'app_id') share_token = config.get('auth', 'app_secret') # ------------ Real start ---------------- # instanciating the class isogeo = Isogeo(client_id=share_id, client_secret=share_token, lang="fr") token = isogeo.connect() # ------------ REAL START ---------------------------- latest_data_modified = isogeo.search(token, page_size=10, order_by="modified", whole_share=0, sub_resources=["events"] ) print("Last 10 data updated \nTitle | datetime\n\t description") for md in latest_data_modified.get("results"): title = md.get('title') evt_description = md.get("events")[0].get("description") print(str("___________________\n\n{} | {} \n\t {}"). format(title.encode("utf8"), dtparse(md.get("modified")[:19]).strftime("%a %d %B %Y"), evt_description.encode("utf8")))
def query(self, cpf, birth): """Queries a CPF at Receita Federal""" cpf = CPF(cpf) birth = dtparse(birth) if self.method == "crack": captcha = "" r = self.session.get(self.captcha_url, stream=True) if r.status_code == 200: with open("/tmp/teste.png", "wb") as f: for chunk in r.iter_content(1024): f.write(chunk) c = Captcha("deathbycaptcha") captcha = c.decode("/tmp/teste.png") else: return False payload = { "txtTexto_captcha_serpro_gov_br": captcha, "tempTxtCPF": cpf.format(), "tempTxtNascimento": birth.strftime("%d/%m/%Y"), "temptxtToken_captcha_serpro_gov_br": "", "temptxtTexto_captcha_serpro_gov_br": captcha, } r = self.session.post(self.post_url, data=payload) soup = bs(r.content, "lxml") obj = {} obj["death_year"] = None obj["death_message"] = None obj["cpf"] = cpf.format() obj["birth"] = birth.strftime("%Y/%m/%d") for span in soup.find_all("span", {"class": "clConteudoDados"}): text = span.text.strip() value = text.split(":")[1].strip() if text.startswith("Nome da Pessoa"): obj["name"] = value elif text.startswith("Situa") and "Cadastral" in text: obj["situation"] = value elif text.startswith("Data da Inscri"): obj["subscription_date"] = dtparse(value.replace("anterior a", "")).strftime("%Y/%m/%d") elif text.startswith("Digito Verificador"): obj["verification_digit"] = value elif "digo de controle do comprovante" in text: obj["control_code"] = value elif self.method == "bypass": # Inspired by http://hc0der.blogspot.com.br/2014/09/bypass-captcha-da-receita-federal.html token_string = "{}{}".format(cpf.cpf, birth.strftime("%d%m%Y")) token = hmac.new(self.bypass_secret.encode("utf-8"), token_string.encode("utf-8"), hashlib.sha1) headers = { "token": token.hexdigest(), "plataforma": "iPhone OS", "dispositivo": "iPhone", "aplicativo": "Pessoa FĂsica", "versao": "8.3", "versao_app": "4.1", "Content-Type": "application/x-www-form-urlencoded", } payload = "cpf={}&dataNascimento={}".format(cpf.cpf, birth.strftime("%d%m%Y")) r = self.session.post(self.bypass_url, headers=headers, data=payload, verify=False) data = r.json() obj = {} obj["cpf"] = cpf.format() obj["birth"] = birth.strftime("%Y/%m/%d") obj["name"] = data["nome"] obj["situation"] = data["descSituacaoCadastral"] obj["subscription_date"] = dtparse(data["dataIsncricao"].replace("anterior a", "")).strftime("%Y/%m/%d") obj["death_year"] = data["anoObito"] obj["death_message"] = data["mensagemObito"] obj["control_code"] = data["codigoControle"] obj["verification_digit"] = data["digitoVerificador"] obj["return_message"] = data.get("mensagemRetorno", None) return obj
def setUp(self): cs1 = CallSource.objects.create(call_source_id=1, descr='Citizen Initiated', code='C') cs2 = CallSource.objects.create(call_source_id=2, descr='Self Initiated', code='S') self.n1 = Nature.objects.create(nature_id=1, descr='Robbery') self.n2 = Nature.objects.create(nature_id=2, descr='Homicide') self.call1 = create_call(call_id=1, time_received='2014-01-15T9:00', nature=self.n1, call_source=cs1) self.call2 = create_call(call_id=2, time_received='2014-03-18T3:00', nature=self.n2, call_source=cs1) self.b1 = Beat.objects.create(beat_id=1, descr='B1') self.b2 = Beat.objects.create(beat_id=2, descr='B2') self.d1 = District.objects.create(district_id=1, descr='D1') self.d2 = District.objects.create(district_id=2, descr='D2') self.cu1 = CallUnit.objects.create(call_unit_id=1, descr='A1', beat=self.b1, district=self.d1) self.cu2 = CallUnit.objects.create(call_unit_id=2, descr='B2', beat=self.b2, district=self.d2) self.at1 = OfficerActivityType.objects.create( officer_activity_type_id=1, descr="IN CALL - CITIZEN INITIATED") self.at2 = OfficerActivityType.objects.create( officer_activity_type_id=2, descr="OUT OF SERVICE") self.at3 = OfficerActivityType.objects.create( officer_activity_type_id=3, descr="ON DUTY") self.a1 = create_officer_activity(activity_type=self.at1, time=dtparse( '2014-01-15T9:00'), call_unit=self.cu1, call=self.call1) self.a2 = create_officer_activity(activity_type=self.at1, time=dtparse( '2014-01-15T9:10'), call_unit=self.cu2, call=self.call1) self.a3 = create_officer_activity(activity_type=self.at1, time=dtparse( '2014-01-15T10:00'), call_unit=self.cu1, call=self.call2) self.a4 = create_officer_activity(activity_type=self.at1, time=dtparse( '2014-01-16T9:50'), call_unit=self.cu2, call=self.call2) self.a5 = create_officer_activity(activity_type=self.at2, time=dtparse( '2014-01-16T10:10'), call_unit=self.cu1, call=None) self.a6 = create_officer_activity(activity_type=self.at2, time=dtparse( '2014-01-18T9:00'), call_unit=self.cu2, call=None) # In order for this to be realistic, for every busy activity, # we need to have an on duty activity self.a7 = create_officer_activity(activity_type=self.at3, time=dtparse('2014-01-15T9:00'), call_unit=self.cu1, call=None) self.a8 = create_officer_activity(activity_type=self.at3, time=dtparse('2014-01-15T9:10'), call_unit=self.cu2, call=None) self.a9 = create_officer_activity(activity_type=self.at3, time=dtparse('2014-01-15T10:00'), call_unit=self.cu1, call=None) self.a10 = create_officer_activity(activity_type=self.at3, time=dtparse('2014-01-16T9:50'), call_unit=self.cu2, call=None) self.a11 = create_officer_activity(activity_type=self.at3, time=dtparse('2014-01-16T10:10'), call_unit=self.cu1, call=None) self.a12 = create_officer_activity(activity_type=self.at3, time=dtparse('2014-01-18T9:00'), call_unit=self.cu2, call=None)
def test_evaluates_one_activity(self): # Should return 1 busy activity (a6), 1 on_duty activity (a12) overview = OfficerActivityOverview(q('time__gte=2014-01-17')) over_time_results = overview.to_dict()['allocation_over_time'] correct_over_time_results = { str(dtparse('9:00').time()): { 'IN CALL - CITIZEN INITIATED': { 'avg_volume': 0, 'total': 0, 'freq': 1 }, 'IN CALL - SELF INITIATED': { 'avg_volume': 0, 'total': 0, 'freq': 1 }, 'IN CALL - DIRECTED PATROL': { 'avg_volume': 0, 'total': 0, 'freq': 1 }, 'OUT OF SERVICE': { 'avg_volume': 1.0, 'total': 1, 'freq': 1 }, 'ON DUTY': { 'avg_volume': 1.0, 'total': 1, 'freq': 1 }, 'PATROL': { 'avg_volume': 0.0, 'total': 0, 'freq': 1 } } } self.assertEqual(sorted(over_time_results.keys()), sorted(correct_over_time_results.keys())) self.assertEqual(sorted(over_time_results.values()), sorted(correct_over_time_results.values())) by_beat_results = overview.to_dict()['on_duty_by_beat'] correct_by_beat_results = [ {'beat_id': 2, 'beat': 'B2', 'on_duty': 1.0}, ] by_district_results = overview.to_dict()['on_duty_by_district'] correct_by_district_results = [ {'district_id': 2, 'district': 'D2', 'on_duty': 1.0}, ] for results, correct in ( (by_beat_results, correct_by_beat_results), (by_district_results, correct_by_district_results)): for d in correct: self.assertIn(d, results) for d in results: self.assertIn(d, correct)
def parse_printdeck(self, response): #self.log('Found deck at {}'.format(response.url)) deck = DeckItem(url=response.url) title_l = response.xpath('//span[contains(@class, "titletext")]/a/text()').extract() title = title_l[0] #self.log("TITLE: " + str(title)) deck['name'] = str(title) formatname = None place = 99999 body_poop = response.xpath('//body/text()').extract() for poop in body_poop: find_format = FORMAT_FIND_RE.search(poop) if find_format: formatname = find_format.group(1) place_format = PLACE_FIND_RE.search(poop) if place_format: place = place_format.group(1) #self.log("FORMAT: " + str(formatname)) deck['deck_format'] = str(formatname) #self.log("PLACE: " + str(place)) deck['place'] = str(place) authornames = response.xpath('//a[contains(@href, "p_first=")]/text()').extract() if len(authornames) > 0: #self.log("AUTHORNAME: " + str(authornames[0])) deck['author'] = str(authornames[0]) cities = response.xpath('//a[contains(@href, "city=")]/text()').extract() city = '' if len(cities) > 0: #self.log("CITY: " + str(cities[0])) city = str(cities[0]) states = response.xpath('//a[contains(@href, "state=")]/text()').extract() if len(states) > 0: #self.log("STATE: " + str(states[0])) pass countries = response.xpath('//a[contains(@href, "country=")]/text()').extract() if len(countries) > 0: #self.log("COUNTRY: " + str(countries[0])) pass start_dates = response.xpath('//a[contains(@href, "start_date=")]/@href').extract() if len(start_dates) > 0: sd_idx = start_dates[0].find('start_date=') if sd_idx > -1: start_date = start_dates[0][sd_idx + len('start_date='):sd_idx + len('start_date=') + len('xxxx-yy-zz')] #self.log("DATE: " + start_date) deck['tournament_date'] = start_date end_dates = response.xpath('//a[contains(@href, "end_date=")]/@href').extract() end_date = None if len(end_dates) > 0: sd_idx = end_dates[0].find('end_date=') if sd_idx > -1: end_date = end_dates[0][sd_idx + len('end_date='):sd_idx + len('end_date=') + len('xxxx-yy-zz')] url_base = 'http://sales.starcitygames.com/deckdatabase/deckshow.php?' format_part = '' if deck['deck_format'].lower() == 'standard': format_part = 't[T1]=1' elif deck['deck_format'].lower() == 'modern': format_part = 't[T3]=28' deck['tournament_url'] = url_base + '&'.join([format_part, 'city=' + city, 'start_date=' + deck['tournament_date'], 'end_date=' + end_date]) mainboard_lines = list() sideboard_lines = list() for span in response.xpath('//span'): class_attrs = span.xpath('@class').extract() if len(class_attrs) > 0 and class_attrs[0] == 'titletext': # skip continue kids = span.xpath('*|text()') in_sb = False cur_num = None for kid in kids: in_sb = in_sb or kid.extract().find('<strong>Sideboard') > -1 is_text_number_match = GET_NUMBER_RE.search(str(kid.extract())) if is_text_number_match: cur_num = is_text_number_match.group(1) else: ggg = kid.xpath('./text()').extract() booh = None if cur_num: booh = '{} {}'.format(str(cur_num), ggg[0].strip()) cur_num = None if in_sb: sideboard_lines.append(booh) else: mainboard_lines.append(booh) #self.log("MAIN: " + str(mainboard_lines)) #self.log("SIDE: " + str(sideboard_lines)) deck['mainboard_cards'] = mainboard_lines deck['sideboard_cards'] = sideboard_lines #self.log("DECK: " + str(deck)) yield deck td_o = dtparse(deck['tournament_date']) titem = TournamentItem(name='StarCityGames.com {} in {} {}'.format(deck['deck_format'], city, deck['tournament_date']), url=deck['tournament_url'], tournament_format=deck['deck_format'], tournament_date=td_o) yield titem