def iterate(uid, start_date=None, end_date=None, class_name=None):
date1 = (dtparse(start_date).replace(
tzinfo=tmzn) if start_date else datetime.now(tz=tmzn))
date2 = (dtparse(end_date).replace(tzinfo=tmzn)
if end_date else date1.replace(hour=23, minute=59, second=59))
class_list = []
for event in calendars[uid].walk("vevent"):
if event["dtstart"].dt >= date1 and event["dtend"].dt <= date2:
if not start_date:
class_list.append(format_event(event))
break
if class_name:
class_name = [class_name] if isinstance(class_name,
str) else class_name
for c_name in class_name:
if (fuzz.partial_ratio(c_name.lower(),
event["summary"].lower()) >
fuzz_threshold):
class_list.append(format_event(event))
else:
class_list.append(format_event(event))
return class_list
def message_to_values(self, key, value):
payload = payload_parse(value)
top_time = dtparse(value['time']).replace(tzinfo=pytz.utc)
apply_start_end_filter(top_time, self.filters.get('start_date'),
self.filters.get('end_date'))
value['lat'] = float(value['lat'])
value['lon'] = float(value['lon'])
pt = Point(value['lon'], value['lat'])
value['geom'] = from_shape(pt, srid=4326)
if not value['values']:
value['values'] = {}
value['values']['location_quality'] = get_point_location_quality(pt)
# All HSTORE values need to be strings
value['values'] = {
k: make_valid_string(x)
for k, x in value['values'].items()
}
value['time'] = top_time.isoformat()
if 'reftime' in value:
value['reftime'] = dtparse(
value['reftime']).replace(tzinfo=pytz.utc).isoformat()
else:
value['reftime'] = value['time']
value['payload'] = payload
# Throw away non-column data
value = self.match_columns(value)
# Remove None to use the defaults defined in the table definition
return key, {k: v for k, v in value.items() if v is not None}
def test_crtp_calculated_metadata_missing_time(self):
axes = {
't': 'time',
'x': 'longitude',
'y': 'latitude',
'z': 'depth',
}
with ContiguousRaggedTrajectoryProfile(self.missing_time) as mmt:
t = mmt.calculated_metadata(axes=axes)
assert t.min_t == dtparse('2014-11-16 21:32:29.952500')
assert t.max_t == dtparse('2014-11-17 07:59:08.398500')
assert len(t.trajectories) == 1
traj = t.trajectories["UW157-20141116T211809"]
assert np.isclose(traj.min_z, 0.47928014)
assert np.isclose(traj.max_z, 529.68005)
assert traj.min_t == dtparse('2014-11-16 21:32:29.952500')
assert traj.max_t == dtparse('2014-11-17 07:59:08.398500')
first_loc = traj.geometry.coords[0]
assert np.isclose(first_loc[0], -124.681526638573)
assert np.isclose(first_loc[1], 43.5022166666667)
assert len(traj.profiles) == 13
def _parse_args():
opts = docopt(__doc__)
if opts['transactions']:
# from-date
if opts['<from-date>'] is not None:
from_date = dtparse(opts['<from-date>']).date()
else:
from_date = None
# to-date
if opts['<to-date>'] is not None:
to_date = dtparse(opts['<to-date>']).date()
else:
to_date = None
return {
'cmd': 'transactions',
'fmt': opts['--format'],
'token': opts['<token>'],
'from_date': from_date,
'to_date': to_date,
'output_file': opts['--output-file'],
}
elif opts['statement']:
return {
'cmd': 'statement',
'token': opts['<token>'],
'year': opts['<year>'],
'statement_id': int(opts['<statement>']),
'fmt': opts['--format'],
'output_file': opts['--output-file'],
}
def get_events(service, calendar_id, start, end, time_zone="America/New_York"):
events_result = service.events().list(calendarId=calendar_id,
timeMin=start,
timeMax=end,
maxResults=400,
singleEvents=True,
orderBy='startTime',
timeZone=time_zone).execute()
events = events_result.get('items', [])
if not events:
return []
else:
out = []
for event in events:
# start = event['start'].get('dateTime', event['start'].get('date'))
# end = event['end'].get('dateTime', event['end'].get('date'))
try:
start = dtparse(event['start']['dateTime'])
end = dtparse(event['end']['dateTime'])
except:
print(event['start'])
curr = {'start': start, 'end': end, 'name': event['summary']}
out.append(curr)
return out
def test_imt_calculated_metadata(self):
with IncompleteMultidimensionalTrajectory(self.single) as ncd:
s = ncd.calculated_metadata()
assert s.min_t == dtparse('1990-01-01 00:00:00')
assert s.max_t == dtparse('1990-01-05 03:00:00')
traj1 = s.trajectories["Trajectory1"]
assert traj1.min_z == 0
assert traj1.max_z == 99
assert traj1.min_t == dtparse('1990-01-01 00:00:00')
assert traj1.max_t == dtparse('1990-01-05 03:00:00')
assert np.isclose(traj1.first_loc.x, -7.9336)
assert np.isclose(traj1.first_loc.y, 42.00339)
with IncompleteMultidimensionalTrajectory(self.multi) as ncd:
m = ncd.calculated_metadata()
assert m.min_t == dtparse('1990-01-01 00:00:00')
assert m.max_t == dtparse('1990-01-02 12:00:00')
assert len(m.trajectories) == 4
traj0 = m.trajectories["Trajectory0"]
assert traj0.min_z == 0
assert traj0.max_z == 35
assert traj0.min_t == dtparse('1990-01-01 00:00:00')
assert traj0.max_t == dtparse('1990-01-02 11:00:00')
assert np.isclose(traj0.first_loc.x, -35.07884)
assert np.isclose(traj0.first_loc.y, 2.15286)
traj3 = m.trajectories["Trajectory3"]
assert traj3.min_z == 0
assert traj3.max_z == 36
assert traj3.min_t == dtparse('1990-01-01 00:00:00')
assert traj3.max_t == dtparse('1990-01-02 12:00:00')
assert np.isclose(traj3.first_loc.x, -73.3026)
assert np.isclose(traj3.first_loc.y, 1.95761)
def get_metric_statistics(self):
namespace = self._get_param("Namespace")
metric_name = self._get_param("MetricName")
start_time = dtparse(self._get_param("StartTime"))
end_time = dtparse(self._get_param("EndTime"))
period = int(self._get_param("Period"))
statistics = self._get_multi_param("Statistics.member")
dimensions = self._get_multi_param("Dimensions.member")
# Unsupported Parameters (To Be Implemented)
unit = self._get_param("Unit")
extended_statistics = self._get_param("ExtendedStatistics")
# TODO: this should instead throw InvalidParameterCombination
if not statistics and not extended_statistics:
raise NotImplementedError(
"Must specify either Statistics or ExtendedStatistics")
datapoints = self.cloudwatch_backend.get_metric_statistics(
namespace,
metric_name,
start_time,
end_time,
period,
statistics,
unit=unit,
dimensions=dimensions,
)
template = self.response_template(GET_METRIC_STATISTICS_TEMPLATE)
return template.render(label=metric_name, datapoints=datapoints)
def list(self):
service = build('calendar', 'v3', credentials=self.creds)
now = dt.utcnow().isoformat() + 'Z'
result = service.events().list(calendarId='primary',
timeMin=now,
maxResults=20,
singleEvents=True,
orderBy='startTime').execute()
events = result.get('items', [])
# 2019-11-05T10:00:00-08:00
items = []
for event in events:
start = event['start'].get('dateTime', event['start'].get('date'))
st_date = dt.strftime(dtparse(start), format='%m-%d-%Y')
st_time = dt.strftime(dtparse(start), format='%I:%M%p')
items.append({
"date": st_date,
"time": st_time,
"content": event['summary']
})
return items
def handle(self, *args, **options):
sdate = None
edate = None
try:
sdate = dtparse(options['start_date'])
except ValueError as ve:
self.stderr.write("Could not parse start-date '{}'\n".format(options['start_date']))
return
try:
edate = dtparse(options['end_date'])
except ValueError as ve:
self.stderr.write("Could not parse end-date '{}'\n".format(options['end_date']))
return
if not options['all_stats'] and not options['format_stats'] and not options['card_stats']:
self.stderr.write("No action taken. Specify all, formats, or cards.\n")
fn = None
if 'formatname' in options:
fn = options['formatname']
if options['all_stats'] or options['format_stats']:
self.stderr.write("formats...\n")
FormatStat.calc_all(new_only=options['new_only'], start_date=sdate, end_date=edate, only_formatname=fn)
if options['all_stats'] or options['card_stats']:
self.stderr.write("cards...\n")
FormatCardStat.calc_all(new_only=options['new_only'], start_date=sdate, end_date=edate, only_formatname=fn)
def describe_job_flows(
self,
job_flow_ids=None,
job_flow_states=None,
created_after=None,
created_before=None,
):
clusters = self.clusters.values()
within_two_month = datetime.now(pytz.utc) - timedelta(days=60)
clusters = [
c for c in clusters if c.creation_datetime >= within_two_month
]
if job_flow_ids:
clusters = [c for c in clusters if c.id in job_flow_ids]
if job_flow_states:
clusters = [c for c in clusters if c.state in job_flow_states]
if created_after:
created_after = dtparse(created_after)
clusters = [
c for c in clusters if c.creation_datetime > created_after
]
if created_before:
created_before = dtparse(created_before)
clusters = [
c for c in clusters if c.creation_datetime < created_before
]
# Amazon EMR can return a maximum of 512 job flow descriptions
return sorted(clusters, key=lambda x: x.id)[:512]
def get_measure_counts_for_dates(self, start_date, end_date):
search_date = dtparse(start_date)
search_date2 = dtparse(end_date)
pat_cnt = {}
for obn in self.observation_list:
pat_cnt[obn] = sum((self.pat_dat[obn]['dt'] >= search_date)
& (self.pat_dat[obn]['dt'] <= search_date2))
return pat_cnt
def test_call_volume_for_day(self):
overview = CallVolumeOverview(
q("time_received__gte=2015-01-01&time_received__lte=2015-01-01"))
assert overview.bounds == {"min_time": dtparse("2015-01-01T09:00"),
"max_time": dtparse('2015-01-01T12:30')}
assert_list_equiv(overview.volume_by_date(),
[{"date": dtparse("2015-01-01T09:00"), "volume": 1},
{"date": dtparse("2015-01-01T12:00"), "volume": 1}])
def valid_coupon(self, coupon):
if not coupon.get('status', False):
self.log.info('Coupon code disabled %s', coupon.get('code'))
return False
start = dtparse(coupon.get('start')).date()
end = dtparse(coupon.get('end')).date()
now = datetime.utcnow().date()
self.log.info('Comparing coupon dates %s %s %s', start, end, now)
return start < now and end > now
def message_to_values(self, key, value):
payload = payload_parse(value)
values = flatten(value)
top_time = dtparse(values['timestamp']).replace(tzinfo=pytz.utc)
apply_start_end_filter(top_time, self.filters.get('start_date'),
self.filters.get('end_date'))
top_level = {
'uid':
values['imei'],
'gid':
None,
'time':
top_time.isoformat(),
'reftime':
dtparse(values['navsat_fix_time']).replace(
tzinfo=pytz.utc).isoformat(),
'lat':
values['latitude'],
'lon':
values['longitude'],
'z':
None,
'payload':
payload,
}
pt = Point(top_level['lon'], top_level['lat'])
top_level['geom'] = from_shape(pt, srid=4326)
skips = [
# No easy way to represent this as a flat dict. We can write a db view to extract this
# data from the `payload` if required.
'data_segment_data_product_pipeline',
'data_segment_data_segment_data_product_pipeline'
]
# Set additional values
# Lat=91 and Lon=181 should be treated as bad location data
values['location_quality'] = get_point_location_quality(
pt, disallow_lon=[181], disallow_lat=[91])
values['mfr'] = 'numurus'
# All HSTORE values need to be strings
values = {
k: make_valid_string(x) if x is not None else None
for k, x in values.items() if k not in skips
}
fullvalues = {**top_level, 'values': {**values}}
# Throw away non-column data
fullvalues = self.match_columns(fullvalues)
# Remove None to use the defaults defined in the table definition
return key, {k: v for k, v in fullvalues.items() if v is not None}
def get_aggragation_for_dates_old(self, start_date, end_date):
search_date = dtparse(start_date)
search_date2 = dtparse(end_date)
pat_ag = {}
for obn in self.observation_list:
pat_ag[obn] = self.pat_ag_func[obn](self.pat_dat[obn]['value'][
(self.pat_dat[obn]['dt'] >= search_date)
& (self.pat_dat[obn]['dt'] <= search_date2)])
return pat_ag
def test_call_volume_for_month(self):
overview = CallVolumeOverview(
q("time_received__gte=2015-01-01&time_received__lte=2015-02-02"))
results = overview.volume_by_date()
assert overview.bounds == {"min_time": dtparse("2015-01-01T09:00"),
"max_time": dtparse('2015-02-01T09:00')}
assert_list_equiv(results,
[{"date": dtparse("2015-01-01T00:00"), "volume": 2},
{"date": dtparse("2015-01-08T00:00"), "volume": 1},
{"date": dtparse("2015-02-01T00:00"), "volume": 1}])
def get_metric_data(self):
start = dtparse(self._get_param("StartTime"))
end = dtparse(self._get_param("EndTime"))
queries = self._get_list_prefix("MetricDataQueries.member")
results = self.cloudwatch_backend.get_metric_data(
start_time=start, end_time=end, queries=queries
)
template = self.response_template(GET_METRIC_DATA_TEMPLATE)
return template.render(results=results)
def test_loading_metadata(archive_meta):
assert "PROTEGE" == archive_meta.game_system_id
assert "PROTéGé Test Campaign" == archive_meta.name
assert "Alex Mair" == archive_meta.author
assert dtparse("2016-04-20") == archive_meta.creation_date
assert dtparse("2016-04-21") == archive_meta.revision_date
assert "978-3-16-148410-0" == archive_meta.isbn
assert "This is a test campaign for " \
"PROTéGé, whose main purpose is " \
"to ensure that dmclient has decent " \
"unit tests." == archive_meta.description
async def edit_event_time(ctx, client, args):
CREDS = await get_credentials(ctx, client)
service = build('calendar', 'v3', credentials=CREDS)
summary = args[0].strip()
starttime_arg = args[1].strip().lower()
start_time = await set_time(ctx, starttime_arg)
if len(start_time) < 1:
return
events = retrieve_all_events(service, UMARL_CALENDAR_ID)
if not events:
await ctx.send(
embed=discord.Embed(description='No events exist on the calendar.',
color=discord.Color.red()))
return
event_id, event_name = retrieve_event_id(summary, events)
event = service.events().get(calendarId=UMARL_CALENDAR_ID,
eventId=event_id).execute()
if len(args) > 2:
date_arg = args[2].strip()
date_str = await check_and_format_date(ctx, date_arg)
if len(date_str) < 1:
return
else:
date_str = event['start']['dateTime'].split('T')[0]
if len(args) > 3:
duration = args[3].strip()
if int(duration) < 15 or int(duration) > 1440:
await ctx.send(embed=discord.Embed(
description=
"Invalid duration, please input a duration (in minutes) between 15 and 1440.",
color=discord.Color.red()))
return
else:
start_datetime = dtparse(event['start']['dateTime'])
end_datetime = dtparse(event['end']['dateTime'])
minutes_difference_timedelta = end_datetime - start_datetime
duration = int(minutes_difference_timedelta.seconds / 60)
event['start']['dateTime'] = date_str + 'T' + start_time
event['end']['dateTime'] = await set_end_time(ctx, duration,
date_str + 'T' + start_time)
updated_event = service.events().update(calendarId=UMARL_CALENDAR_ID,
eventId=event_id,
body=event).execute()
await ctx.send(embed=discord.Embed(description="Event time updated!",
color=discord.Color.green()))
def GetCalendarEvents():
print('[Calendar API] Get last events')
creds = None
tokendir = os.path.join(rootdir, 'token.pickle')
#Check if the user already logged
if os.path.exists(tokendir):
with open(tokendir, 'rb') as token:
creds = pickle.load(token)
# If the file doesn't exists, log the user
if not creds or not creds.valid:
if creds and creds.expired and creds.refresh_token:
creds.refresh(Request())
else:
flow = InstalledAppFlow.from_client_secrets_file(
os.path.join(rootdir, 'credentials.json'), SCOPES)
creds = flow.run_local_server(port=0)
# Save the credentials for the next run
with open(tokendir, 'wb') as token:
pickle.dump(creds, token)
service = build('calendar', 'v3', credentials=creds, cache_discovery=False)
# Call the Calendar API
now = datetime.datetime.utcnow().isoformat() + 'Z'
print('[Calendar API] Getting the upcoming 7 events')
events_result = service.events().list(calendarId=os.getenv('CALENDAR_ID'),
timeMin=now,
maxResults=7,
singleEvents=True,
orderBy='startTime').execute()
events = events_result.get('items', [])
return_events = []
if not events:
print('No upcoming events found')
for event in events:
start = event['start'].get('dateTime', event['start'].get('date'))
end = event['end'].get('dateTime', event['end'].get('date'))
start_datetime = dtparse(start)
end_datetime = dtparse(end)
new_event = {
"start_date": start_datetime,
"end_date": end_datetime,
"title": event["summary"]
}
return_events.append(new_event)
return return_events
def test_get_creation(self):
meta = utils.get_creation_attributes(history='DID THINGS')
now = datetime.utcnow().replace(tzinfo=pytz.utc)
assert (now - dtparse(meta['attributes']['date_created'])) < timedelta(
minutes=1)
assert (now - dtparse(meta['attributes']['date_issued'])) < timedelta(
minutes=1)
assert (now - dtparse(
meta['attributes']['date_modified'])) < timedelta(minutes=1)
assert 'DID THINGS' in meta['attributes']['history']
def get_final(self, result):
photos = result[0]['Items']
for photo in photos:
# Datetaken field is a string, e.g. "2014-03-17 01:55:38".
raw_date = dtparse(photo['ImageDate'])
photo['ImageDate'] = datetime.date(raw_date.year, raw_date.month, raw_date.day)
raw_date = dtparse(photo['Created'])
photo['Created'] = datetime.date(raw_date.year, raw_date.month, raw_date.day)
yield photo
if len(photos) == 0:
raise sundew.StopPipeline
def parse_free_busy(free_busy):
if 'busy' not in free_busy:
return None
busy_times = []
for busy_slot_dict in free_busy['busy']:
start_time = dtparse(busy_slot_dict['start'], ignoretz=True)
end_time = dtparse(busy_slot_dict['end'], ignoretz=True)
busy_times.append(TimeSlot(start_time, end_time, buffer_time=0))
return busy_times
def is_date(string, fuzzy=False):
"""
Return whether the string can be interpreted as a date.
:param string: str, string to check for date
:param fuzzy: bool, ignore unknown tokens in string if True
"""
try:
dtparse(string, fuzzy=fuzzy)
return True
except ValueError:
return False
async def test_multiple_timezones_for_reserved_batch_session_predicate(
mock_dt):
mock_db_conn = MagicMock()
mock_sched_ctx = MagicMock()
mock_sess_ctx = MagicMock()
mock_sess_ctx.session_type = SessionTypes.BATCH
mock_sess_ctx.kernel_id = 'fake-kernel-id'
now = '2020-06-29T00:00:00+00:00'
mock_dt.now = MagicMock(return_value=dtparse(now))
# Start time is not yet reached (now < start time)
start_time = '2020-06-29T00:00:01+00:00'
mock_db_conn.scalar = AsyncMock(return_value=dtparse(start_time))
result = await check_reserved_batch_session(mock_db_conn, mock_sched_ctx,
mock_sess_ctx)
assert not result.passed, (now, start_time)
# Start time is reached (now > start time)
start_time = '2020-06-28T23:59:59+00:00'
mock_db_conn.scalar = AsyncMock(return_value=dtparse(start_time))
result = await check_reserved_batch_session(mock_db_conn, mock_sched_ctx,
mock_sess_ctx)
assert result.passed, (now, start_time)
# Start time is not yet reached by timezone (now < start time)
# Note that 6/29 00:00 (UTC) < 6/29 00:00 (-09:00) == 6/29 09:00 (UTC)
for i in range(1, 12):
start_time = f'2020-06-29T00:00:00-{i:02d}:00'
mock_db_conn.scalar = AsyncMock(return_value=dtparse(start_time))
result = await check_reserved_batch_session(mock_db_conn,
mock_sched_ctx,
mock_sess_ctx)
assert not result.passed, (now, start_time)
# Start time is reached by timezone (now > start time)
# Note that 6/29 00:00 (UTC) > 6/29 00:00 (+09:00) == 6/28 15:00 (UTC)
for i in range(1, 12):
start_time = f'2020-06-29T00:00:00+{i:02d}:00'
mock_db_conn.scalar = AsyncMock(return_value=dtparse(start_time))
result = await check_reserved_batch_session(mock_db_conn,
mock_sched_ctx,
mock_sess_ctx)
assert result.passed, (now, start_time)
# Should pass if start time is not specified (start immediately).
mock_db_conn.scalar = AsyncMock(return_value=None)
result = await check_reserved_batch_session(mock_db_conn, mock_sched_ctx,
mock_sess_ctx)
assert result.passed
def parse_start_end_times(base):
times = {}
reg = 'S1-COD_M([0-9S]*)_TN([0-9]*)_F(.*?)_S([0-9])_([0-9T]*)-([0-9T]*)-([0-9T]*)'
match = re.search(reg, base)
frame = match.group(1)
track = match.group(2)
slcpframe = match.group(3)
subswath = match.group(4)
start = dtparse(match.group(5)).strftime('%Y-%m-%dT%H:%M:%S')
mid = dtparse(match.group(6)).strftime('%Y-%m-%dT%H:%M:%S')
end = dtparse(match.group(7)).strftime('%Y-%m-%dT%H:%M:%S')
times['starttime'] = start
times['endtime'] = end
return times
def get_list_of_events(service, n):
now = datetime.now(timezone(settings.TIME_ZONE)).isoformat()
# out_time_format = "%I:%M%p on %A, %B %d"
out_date_format = "%Y-%m-%d"
out_time_format = "%H:%M"
events_result = service.events().list(
calendarId='primary',
timeMin=now,
maxResults=n, singleEvents=True,
orderBy='startTime').execute()
events = events_result.get('items', [])
listOfEvents = []
for event in events:
summary = location = description = end_date = start_date = ""
# print(f'\n\nDEBUG: {event}\n\n', file=sys.stderr)
if 'start' in event:
if 'date' in event['start']:
start_date = event['start']['date']
if 'dateTime' in event['start']:
start_date = event['start']['dateTime']
#formatted as "05:00PM on Friday, December 15"
start_string_date = datetime.strftime(dtparse(start_date), format=out_date_format)
start_string_time = datetime.strftime(dtparse(start_date), format=out_time_format)
if 'end' in event:
if 'date' in event['end']:
end_date = event['end']['date']
if 'dateTime' in event['end']:
end_date = event['end']['dateTime']
#formatted as "05:00PM on Friday, December 15"
end_string_date = datetime.strftime(dtparse(end_date), format=out_date_format)
end_string_time = datetime.strftime(dtparse(end_date), format=out_time_format)
if 'location' in event:
location = event['location']
if 'summary' in event:
summary = event['summary']
if 'description' in event:
description = event['description']
# Date and time are picked up better when we just pass the date time string as it is received from google
# string = f'{summary} . {description} . {location} . from {start_date} . to {end_date}.'
string = f'{summary} . {description} . {location} . from {start_string_date} {start_string_time}. to {end_string_date} {end_string_time}.'
if settings.DEBUG:
print(f'\n\nDEBUG: {string}\n\n', file=sys.stderr)
listOfEvents.append((string, event))
# return a tuple of event
return listOfEvents
def test_just_missing_time(self):
with ContiguousRaggedTrajectoryProfile(self.missing_time) as mmt:
t = mmt.calculated_metadata()
assert t.min_t == dtparse('2014-11-16 21:32:29.952500')
assert t.max_t == dtparse('2014-11-17 07:59:08.398500')
assert len(t.trajectories) == 1
traj = t.trajectories["UW157-20141116T211809"]
assert np.isclose(traj.min_z, 0.47928014)
assert np.isclose(traj.max_z, 529.68005)
assert traj.min_t == dtparse('2014-11-16 21:32:29.952500')
assert traj.max_t == dtparse('2014-11-17 07:59:08.398500')
assert np.isclose(traj.first_loc.x, -124.681526638573)
assert np.isclose(traj.first_loc.y, 43.5022166666667)
assert len(traj.profiles) == 13
def test_imt_calculated_metadata_single(self):
filepath = os.path.join(os.path.dirname(__file__), 'resources', 'im-single.nc')
with IncompleteMultidimensionalTrajectory(filepath) as ncd:
s = ncd.calculated_metadata()
assert s.min_t.round('S') == dtparse('1990-01-01 00:00:00')
assert s.max_t.round('S') == dtparse('1990-01-05 03:00:00')
traj1 = s.trajectories["Trajectory1"]
assert traj1.min_z == 0
assert traj1.max_z == 99
assert traj1.min_t.round('S') == dtparse('1990-01-01 00:00:00')
assert traj1.max_t.round('S') == dtparse('1990-01-05 03:00:00')
first_loc = traj1.geometry.coords[0]
assert np.isclose(first_loc[0], -7.9336)
assert np.isclose(first_loc[1], 42.00339)
def extract_data_csv(data):
datasets = defaultdict(dict)
for dp in data:
ts = dtparse(dp["point"]["start"]).replace(tzinfo = utc, microsecond = 0)
name = dp["timeseriesDesc"]["metric"].split("/")[-1] # benchmark or single
labels = dict((k.split("/")[-1], v) for k, v in dp["timeseriesDesc"]["labels"].iteritems())
kind = labels.get('kind', '?')
# print("name: %s, kind: %s" % (name, kind))
if kind == "single": # special case for nb_sagemath, ...
kind = name[3:]
name = "instances"
value = dp["point"].get("int64Value")
# value must be a string!
if value is None:
dv = dp["point"].get("doubleValue")
if name == "instances":
value = '%d' % int(dv)
else:
value = '%f' % dv
else:
value = '%d' % value
datasets[name]["timestamp"] = ts.isoformat()
datasets[name][kind] = value
csv = {}
for name, data in datasets.iteritems():
# sort by kind, and timestamp in the front
header, line = zip(*sorted(data.items(), key = lambda x : (x[0] != 'timestamp', x[0])))
csv[name] = (header, line)
return csv
def main():
global screen
global font
pygame.init()
# TODO: use display info
# display_info = pygame.display.info()
screen = pygame.display.set_mode((320,240))
font = pygame.font.Font(None, 40)
now = datetime.datetime.now()
today = now.date()
one_month_ago = today - datetime.timedelta(days=30)
if os.path.exists('db.json') and now.timestamp() - os.stat('db.json').st_mtime < 3000:
pics = load_db()
else:
pics = refresh_db(progress_cb)
clear_display()
included = set()
start_progress('Adding recent pictures', len(pics))
for i, (key,item) in enumerate(pics.items()):
when = dtparse(item['mediaMetadata']['creationTime'])
if when.date() >= one_month_ago:
included.add(key)
add_to_display(item)
update_progress(i)
added = 0
start_progress('Adding random pictures', num_pics)
while added < num_pics:
random_key = random.choice(list(pics.keys()))
if random_key in included:
continue
add_to_display(pics[random_key])
added = added + 1
update_progress(added)
pygame.display.quit()
display_images()
def extract_data_csv(data):
datasets = defaultdict(dict)
for dp in data:
ts = dtparse(dp["point"]["start"]).replace(tzinfo=utc, microsecond=0)
name = dp["timeseriesDesc"]["metric"].split("/")[
-1] # benchmark or single
labels = dict((k.split("/")[-1], v)
for k, v in dp["timeseriesDesc"]["labels"].iteritems())
kind = labels.get('kind', '?')
# print("name: %s, kind: %s" % (name, kind))
if kind == "single": # special case for nb_sagemath, ...
kind = name[3:]
name = "instances"
value = dp["point"].get("int64Value")
# value must be a string!
if value is None:
dv = dp["point"].get("doubleValue")
if name == "instances":
value = '%d' % int(dv)
else:
value = '%f' % dv
else:
value = '%d' % value
datasets[name]["timestamp"] = ts.isoformat()
datasets[name][kind] = value
csv = {}
for name, data in datasets.iteritems():
# sort by kind, and timestamp in the front
header, line = zip(
*sorted(data.items(), key=lambda x: (x[0] != 'timestamp', x[0])))
csv[name] = (header, line)
return csv
def list_clusters(self, cluster_states=None, created_after=None,
created_before=None, marker=None):
max_items = 50
clusters = self.clusters.values()
if cluster_states:
clusters = [c for c in clusters if c.state in cluster_states]
if created_after:
created_after = dtparse(created_after)
clusters = [c for c in clusters if c.creation_datetime > created_after]
if created_before:
created_before = dtparse(created_before)
clusters = [c for c in clusters if c.creation_datetime < created_before]
clusters = sorted(clusters, key=lambda x: x.id)
start_idx = 0 if marker is None else int(marker)
marker = None if len(clusters) <= start_idx + max_items else str(start_idx + max_items)
return clusters[start_idx:start_idx + max_items], marker
def date_parse(date_string, as_date=True):
from dateutil.parser import parse as dtparse
class simple_utc(datetime.tzinfo):
def tzname(self, **kwargs):
return "UTC"
def utcoffset(self, dt):
return datetime.timedelta(0)
date_string = date_string.strip().lower()
if date_string == 'today':
date = datetime.datetime.utcnow()
elif date_string == 'yesterday':
date = datetime.datetime.utcnow() - datetime.timedelta(days=1)
else:
date = dtparse(date_string)
date_tzinfo = date.replace(tzinfo=simple_utc())
if as_date:
date = date_tzinfo.date()
else:
date = date_tzinfo
return date
def isodate(self, datestring):
try:
date_obj = dtparse(datestring)
return date_obj.strftime("%Y-%m-%d")
except ValueError as ve:
sys.stderr.write("ERROR: Could not parse '{}'\n".format(datestring))
raise ve
def _datetime_parser(dct):
for key, value in list(dct.items()):
try:
dct[key] = dtparse(value)
except Exception:
pass
return dct
def nite2key(nite):
if isinstance(nite,basestring):
nite = dtparse(nite)
for k,v in EPOCHS.items():
if v['tstart'] < nite < v['tstop']:
return k
msg = "Nite outside of EPOCHS: %s"%nite
raise Exception(msg)
def describe_job_flows(self, job_flow_ids=None, job_flow_states=None, created_after=None, created_before=None):
clusters = self.clusters.values()
within_two_month = datetime.now(pytz.utc) - timedelta(days=60)
clusters = [c for c in clusters if c.creation_datetime >= within_two_month]
if job_flow_ids:
clusters = [c for c in clusters if c.id in job_flow_ids]
if job_flow_states:
clusters = [c for c in clusters if c.state in job_flow_states]
if created_after:
created_after = dtparse(created_after)
clusters = [c for c in clusters if c.creation_datetime > created_after]
if created_before:
created_before = dtparse(created_before)
clusters = [c for c in clusters if c.creation_datetime < created_before]
return sorted(clusters, key=lambda x: x.id)[:512]
def str_to_datetime(string, timezone=pytz.utc):
"""
Convert string to datetime.datetime
:param string: {str} datetime in string
:param timezone: {pytz.timezone}
:return: {datetime.datetime} datetime
"""
dt = dtparse(string)
return timezone.localize(dt) if dt is not None else None
def extract_data_yaml(dp):
# yes, it's a bit stupid to extract this from here,
# but that way it works in general
name = dp["timeseriesDesc"]["metric"].split("/")[-1]
labels = dict((k.split("/")[-1], v) for k, v in dp["timeseriesDesc"]["labels"].iteritems())
ts = dtparse(dp["point"]["start"]).replace(tzinfo = utc)
value = dp["point"].get("int64Value", None) or dp["point"].get("doubleValue")
entry = {"name": name, "timestamp": ts, "value": value}
if labels:
entry["labels"] = labels
return entry
def get_job_metadata(job_id, api_key=None):
"""
Get job metadata from CrowdFlower.
Args:
external_job_id (str or int): CrowdFlower job ID
api_key (str): CrowdFlower API key (see README)
Returns:
metadata (dict): all available metadata for this job
created_at (datetime.datetime): time of job creation
completed_at (datetime.datetime or None): time of job completion,
if completed, else None
"""
api_key = api_key or os.environ["CROWDFLOWER_API_KEY"]
metadata = get_job(job_id)
created_at = dtparse(metadata.get('created_at'))
completed_at = metadata.get('completed_at')
completed_at = dtparse(completed_at) if completed_at else None
return metadata, created_at, completed_at
def get_metric_statistics(self):
namespace = self._get_param('Namespace')
metric_name = self._get_param('MetricName')
start_time = dtparse(self._get_param('StartTime'))
end_time = dtparse(self._get_param('EndTime'))
period = int(self._get_param('Period'))
statistics = self._get_multi_param("Statistics.member")
# Unsupported Parameters (To Be Implemented)
unit = self._get_param('Unit')
extended_statistics = self._get_param('ExtendedStatistics')
dimensions = self._get_param('Dimensions')
if unit or extended_statistics or dimensions:
raise NotImplemented()
# TODO: this should instead throw InvalidParameterCombination
if not statistics:
raise NotImplemented("Must specify either Statistics or ExtendedStatistics")
datapoints = self.cloudwatch_backend.get_metric_statistics(namespace, metric_name, start_time, end_time, period, statistics)
template = self.response_template(GET_METRIC_STATISTICS_TEMPLATE)
return template.render(label=metric_name, datapoints=datapoints)
def str2dt(timestr):
"""Convert a string date to a real date.
Args:
timestr (str) - the datetime as a raw string.
Returns:
dateutil.parser.parse - the parsed datetime object.
"""
try:
return dtparse(timestr)
except (TypeError, ValueError):
if timestr in ['None', 'null']:
return None
return timestr
def parseStatus( soup ) :
""" Parse the status page soup into a dictionary of visits.
"""
import numpy as np
from dateutil.parser import parse as dtparse
# from dateutil.relativedelta import relativedelta
import datetime
today = datetime.datetime.today()
visdict = {}
trowlist = soup.findAll('tr')
for trow in trowlist :
rowtext = trow.getText().strip().split('\n')
rowtext = np.array( rowtext )[ np.where( [ len(rt) for rt in rowtext ] )[0] ]
if rowtext[0] == 'Visit' : continue
visit=rowtext[0]
status=rowtext[1]
targets=rowtext[2].replace('ANY','')
configs=rowtext[3]
startdate = dtparse( '9999-09-09' )
enddate = dtparse( '9999-09-09')
if status in ['Archived', 'Executed', 'Scheduled'] :
startdate = dtparse( rowtext[4].split('-')[0] )
enddate = dtparse( rowtext[5].split('-')[0] )
elif status in ['Scheduling','Implementation'] :
planwindowlist = rowtext[4:]
for planwindow in planwindowlist :
if '-' not in planwindow : continue
try :
startdate0 = dtparse( planwindow.split('-')[0] )
enddate0 = dtparse( planwindow.split('-')[1].split('(')[0] )
except :
continue
daystostart0 = (startdate0 - today ).days
daystoend0 = (startdate0 - today ).days
if daystostart0>=0 and daystoend0>=0 :
startdate = startdate0
enddate = enddate0
break
daystostart = ( startdate - today ).days
daystoend = ( enddate - today ).days
visdict[visit] = {'visit':visit,
'status':status,
'targets':targets,
'configs':configs,
'startdate':startdate,
'enddate':enddate,
'daystostart':daystostart,
'daystoend':daystoend,
}
return( visdict )
def parse_date_args(self):
"""Check for start [and end] date arguments passed to the spider
and parse into date objects.
If there is no start date, raise an exception.
If there is no end date, default to today's date.
"""
# Check that a start argument was passed to the spider
if hasattr(self, 'start'):
# Convert the start date string into a date object
self.date_range_start = dtparse(self.start, dayfirst=True).date()
else:
log.msg("No start date supplied", level=log.CRITICAL)
raise Exception("No start date supplied")
if hasattr(self, 'end'):
self.date_range_end = dtparse(self.end, dayfirst=True).date()
else:
self.date_range_end = date.today()
self.log("Scraping date range: %s ----> %s" % (self.date_range_start,
self.date_range_end),
level=log.WARNING)
def parse_start_url(self, response):
# For now, let's SKIP cached results
if 'cached' in response.flags:
# REVISIT - in the future, maybe we pass on cached items? But maybe not... maybe that all belongs at the pipeline level
pass
try:
if True or response.url.index('displaydeck.php'):
deck = self.parse_deck(response)
if deck is not None and len(deck) > 0:
yield deck
self.log('***** DECK IS {}'.format(str(deck)))
td_o = dtparse(deck['tournament_date'])
titem = TournamentItem(name=deck['tournament_name'],
url=deck['tournament_url'],
tournament_format=deck['deck_format'],
start_date=td_o,
end_date=td_o)
yield titem
except ValueError as ve:
pass
def run(self):
logger.debug("Starting")
while True:
lines = parse(getPage(),fieldwidths)
for line in lines:
#bla query
try:
cur = con.cursor()
cur.execute("""SELECT * FROM repeaters WHERE Call = "%s"; """ % (line[3].lower()))
data = cur.fetchone()
except:
logger.error("Unable to open DB")
continue
if isinstance(data, (list, tuple)):
if re.match('[a-zA-Z0-9_]',line[2]):
try:
logging.debug("%s TS%s: %s (%s) at %s:%s %s-%s-%s on %s" % (line[3],int(line[6]),line[2],line[8][1:],line[0][10:12],line[0][8:10],line[0][6:8],line[0][4:6],line[0][0:4],line[7][2:]))
except:
logging.debug("Line error")
continue
value = "%s (%s)" % (line[2],line[8][1:])
publish.single("hytera/%s/usrTs%s" % (data[2], int(line[6])) , value, hostname=mqtt_host)
publish.single("hytera/%s/tlkTs%s" % (data[2], int(line[6])) , line[7][2:], hostname=mqtt_host)
dt=dtparse("%s UTC"%line[0])
try:
dt = dt.astimezone(tz.tzlocal())
tsLog[int(line[6])-1][data[2]] = dt.strftime("%s")
except:
logger.error("astimezone broke")
for key,value in tsLog[0].iteritems():
publish.single("hytera/%s/lastTs1" % (key), value, hostname=mqtt_host)
for key,value in tsLog[1].iteritems():
publish.single("hytera/%s/lastTs2" % (key), value, hostname=mqtt_host)
sleep(float(config.get("Users","sleeptime")))
def published(self):
return dtparse(self._item['published']).astimezone(tzutc())
The Dark Energy Survey and Operations: Year 1
http://adsabs.harvard.edu/abs/2014SPIE.9149E..0VD
"""
__author__ = "Alex Drlica-Wagner"
from collections import OrderedDict as odict
from dateutil.parser import parse as dtparse
# Official dates can (mostly) be found in the documentation. Here we
# extend the epochs to be continuous across time and to extend into Y3.
#https://opensource.ncsa.illinois.edu/confluence/display/DESDM/Y2A1+Observing+Epochs
EPOCHS = odict([
(1, dict(year='SV',epoch='E1',
tstart=dtparse('20120912'),tstop=dtparse('20121229'))),
(2, dict(year='SV',epoch='E2',
tstart=dtparse('20121229'),tstop=dtparse('20130830'))),
(3, dict(year='Y1',epoch='E1',
tstart=dtparse('20130831'),tstop=dtparse('20131129'))),
(4, dict(year='Y1',epoch='E2',
tstart=dtparse('20131129'),tstop=dtparse('20140815'))),
(5, dict(year='Y2',epoch='E1',
tstart=dtparse('20140815'),tstop=dtparse('20141130'))),
# Currently Y2_E2 extends through Y3...
(6, dict(year='Y2',epoch='E2',
tstart=dtparse('20141130'),tstop=dtparse('20160801'))),
])
def nite2key(nite):
if isinstance(nite,basestring):
token = isogeo.connect()
# Windows 10 notifications class
notif = ToastNotifier()
# ------------ REAL START ----------------------------
latest_data_modified = isogeo.search(token,
page_size=10,
order_by="modified",
whole_share=0,
sub_resources=["events"]
)
# parsing the previous date
last_exe = dtparse(last_exe).strftime("%a %d %B %Y (%H:%M)")
# last_exe = last_exe.decode("Latin1")
# comparing total of metadats shared since last time
now_total = latest_data_modified.get('total')
if now_total > last_total:
notif.show_toast("Isogeo - Total partage",
"{} données ajoutées\ndepuis le {}"
.format(now_total - last_total, last_exe),
icon_path=r"img/favicon.ico",
duration=10)
elif now_total < last_total:
notif.show_toast("Isogeo - Total partage",
"{} données retirées\ndepuis le {}"
.format(last_total - now_total, last_exe),
def handle(self, *args, **options):
#logger = logging.getLogger(__name__)
# the first (and only) arg should be a filename
directory = options['inputdir']
deck_meta_only = options['deck_meta_only']
self.only_newer = options['only_newer']
self.start_time = timezone.now()
if not os.access(directory, os.R_OK):
sys.stderr.write("Cannot read directory '{}'.\n".format(directory))
return
# let's figure out the last time we ran.
sema_filename = join(directory, 'last_complete.semaphore')
last_start_time = datetime(2000, 1, 1, 0, 0, 0, 0, self.localtime)
try:
sema_in = open(sema_filename, 'r')
lst_str = sema_in.readline()
last_start_time = dtparse(lst_str)
last_start_time = last_start_time.replace(tzinfo=self.localtime)
sema_in.close()
except:
sys.stderr.write("Cannot read the {} file. Proceeding anyway...\n".format(sema_filename))
onlyfiles = [f for f in listdir(directory) if isfile(join(directory, f))]
for filename in onlyfiles:
if filename.find('tournament_') > -1: # and False: # REVISIT - commented out for the moment!
filehandler = open(join(directory, filename))
filetime = datetime.fromtimestamp(os.path.getmtime(join(directory, filename)))
filetime = filetime.replace(tzinfo=self.localtime)
if self.only_newer and filetime < last_start_time:
continue
jblob = json.load(filehandler)
#{"url": "/en/content/2011-great-britain-national-championship", "tournament_format": null, "name": "Great Britain National Championship", "start_date": "2011-08-19"}
if jblob['name'].lower().find('test deck') > -1:
sys.stdout.write("Tournament: skipped test deck tournament: {}\n".format(jblob['name']))
pass
else:
turl = jblob['url']
# The StarCityGames tournaments will probably not have a URL at all
if turl is None or len(turl) == 0:
pass
# Some of the Wizards-crawled tournaments will not have a fully-qualified URL
elif turl.find('http') < 0:
turl = 'http://magic.wizards.com' + turl
format_string = jblob['tournament_format']
if format_string is None:
format_string = 'Unknown'
elif format_string.find('Modern') > -1:
format_string = 'Modern'
elif format_string.find('Standard') > -1:
format_string = 'Standard'
elif format_string.find('Commander') > -1:
format_string = 'Commander'
elif format_string.find('Pauper') > -1:
format_string = 'Pauper'
elif format_string.find('Legacy') > -1:
format_string = 'Legacy'
elif format_string.find('Tiny') > -1:
format_string = 'TinyLeaders'
elif format_string == '' and 'legacy' in jblob['name'].lower():
format_string = 'Legacy'
db_format = Format.objects.filter(
formatname__iexact=format_string,
start_date__lte=self.isodate(jblob['start_date']),
end_date__gt=self.isodate(jblob['end_date'])).first()
if db_format is not None:
db_tournament = Tournament.objects.filter(
format=db_format,
name__iexact=jblob['name'],
start_date=self.isodate(jblob['start_date'])).first()
if db_tournament is None:
# let's make one!
db_tournament = Tournament(
format=db_format,
name=jblob['name'],
start_date=self.isodate(jblob['start_date']),
end_date=self.isodate(jblob['end_date']),
url=turl)
db_tournament.save()
sys.stdout.write("Tournament: created {}\n".format(jblob['name']))
else:
if filetime > db_tournament.updated_at or not self.only_newer:
# let's update!!
if turl.find('/node/') < 0:
db_tournament.url = turl
sys.stdout.write("Tournament: updated {}\n".format(jblob['name']))
db_tournament.save()
else:
sys.stdout.write("Tournament: skipped updated with node {}\n".format(jblob['name']))
else:
sys.stdout.write("Tournament: not newer... skipped {}\n".format(jblob['name']))
else:
sys.stdout.write("Tournament: skipped no format '{}' found in db {}\n".format(format_string, jblob['name']))
tournament_url_re = re.compile('^(.+/coverage/([^/]+))/?')
# Process the deck files
for filename in onlyfiles:
if filename.find('deck_') > -1:
###fqfilename = join(directory, filename)
# print "last modified: %s" % time.ctime(getmtime(fqfilename))
filehandler = open(join(directory, filename))
filetime = datetime.fromtimestamp(os.path.getmtime(join(directory, filename)))
filetime = filetime.replace(tzinfo=self.localtime)
if self.only_newer and filetime < last_start_time:
continue
jblob = json.load(filehandler)
# let's test if jblob is a set JSON
if 'name' in jblob and 'mainboard_cards' in jblob:
# TEMP FIX - Legacy decks did not havea format_name for a lot of crawling...
if jblob['deck_format'] == '' and jblob['tournament_name'] and 'legacy' in jblob['tournament_name'].lower():
jblob['deck_format'] = 'Legacy'
tourna = None
# let's deal with decks that have valid tournament URLs
if 'tournament_url' in jblob and jblob['tournament_url'] is not None and len(jblob['tournament_url']) > 0:
# We are going to skip decks and tournaments that look like draft and sealed
if jblob['tournament_url'].find('draft') > 0 or jblob['tournament_url'].find('sealed') > 0:
# skip
sys.stdout.write("Deck: skipped limited {}\n".format(jblob['name']))
continue
# now find the tournament
tu_match = tournament_url_re.search(jblob['tournament_url'])
tournament_url = jblob['tournament_url']
tournament_name = ''
if tu_match:
tournament_url = tu_match.group(1)
tournament_name = tu_match.group(2)
# find a real tournament that we match to
#sys.stdout.write("HERE with '{}' and '{}'\n".format(tournament_url, tournament_name))
tourna = Tournament.objects.filter(url=tournament_url).first() # exclude(format__formatname='Unknown').first()
elif 'tournament_name' in jblob and 'deck_format' in jblob:
# Let's look for tournaments by name and date instead
tourna = Tournament.objects.filter(name=jblob['tournament_name'], format__formatname=jblob['deck_format']).first()
else:
# we just aren't going to have a tournament.
pass
# Let's see if we already have a deck for this.
deck = None
sys.stdout.write("Deck: {} - {} by {}\n".format(jblob['url'], jblob['name'], jblob['author']))
is_create = False
# Should we skip writing this deck to the database?
skip_deck = False
# We are going to look at the "deckid" parameter that StarCityGames decks have and match on just that id.
dukey_m = SCG_DECK_URL_KEY_RE.search(jblob['url'])
if dukey_m:
#sys.stdout.write("** MATCHED: {}\n".format(dukey_m.group(1)))
deck = Deck.objects.filter(url__icontains='deckid={}'.format(dukey_m.group(1))).first()
#sys.stdout.write("** MATCHED: deck is {}\n".format(str(deck)))
else:
if 'page_part' in jblob:
if jblob['page_part'] == '0' or jblob['page_part'] == 0:
deck = Deck.objects.filter(url=jblob['url']).first()
if deck is None:
deck = Deck.objects.filter(url='{}#deck{}'.format(jblob['url'], jblob['page_part'])).first()
else:
deck = Deck.objects.filter(url='{}#deck{}'.format(jblob['url'], jblob['page_part'])).first()
else:
deck = Deck.objects.filter(url=jblob['url']).first()
if deck is None:
# no deck, so let's add it!
sys.stdout.write(" creating...\n")
is_create = True
deck = Deck(url=jblob['url'], visibility=Deck.HIDDEN)
if 'page_part' in jblob:
deck.url = '{}#deck{}'.format(jblob['url'], jblob['page_part'])
deck.name = jblob['name']
deck.authorname = jblob['author']
if tourna is not None and tourna.format is not None:
deck.format = tourna.format
elif 'deck_format' in jblob:
try:
sys.stderr.write(
"Format find: {} - {}\n".format(jblob['deck_format'], self.isodate(jblob['tournament_date'])))
db_format_q = Format.objects.filter(
formatname=jblob['deck_format'], start_date__lte=self.isodate(
jblob['tournament_date']), end_date__gt=self.isodate(
jblob['tournament_date'])).first()
deck.format = db_format_q
except ValueError:
# Looks like we cannot find a valid format because we don't have a real
# deck_format or a real tournament_date.
sys.stdout.write(
" skipped because a valid format cannot be found for '{}' on date '{}'\n".format(
jblob['deck_format'],
jblob['tournament_date']))
deck = None
pass
if deck is not None:
try:
deck.save()
sys.stdout.write(" Created deck {}!\n".format(str(deck.id)))
deck = Deck.objects.get(pk=deck.id) # reload
except IntegrityError as ie:
sys.stdout.write(" Could not create deck! Integrity error: {}\n".format(str(ie)))
deck = None
else:
if filetime > deck.updated_at or not self.only_newer:
# for kicks, let's update name and authorname, since it could be that the
# scrapy crawler gets better at figuring those things out
deck.name = jblob['name']
deck.authorname = jblob['author']
deck.url = jblob['url']
if 'page_part' in jblob:
deck.url = '{}#deck{}'.format(jblob['url'], jblob['page_part'])
deck.format = None
try:
db_format_q = Format.objects.filter(
formatname=jblob['deck_format'], start_date__lte=self.isodate(
jblob['tournament_date']), end_date__gt=self.isodate(
jblob['tournament_date'])).first()
deck.format = db_format_q
except ValueError as ve:
# Looks like we cannot find a valid format because we don't have a real
# deck_format or a real tournament_date.
sys.stdout.write(
" updating '{}' even though could not find a valid format for '{}' on date {} - {}\n".format(
jblob['name'],
jblob['deck_format'],
jblob['tournament_date'],
str(ve)))
deck.format = None
deck.save()
sys.stdout.write(" Updated!\n")
else:
skip_deck = True
sys.stdout.write(" not newer... skipped {}\n".format(jblob['url']))
#sys.stdout.write("Deck: {} (db card count {})\n".format(jblob['name'], deck.get_card_count()))
#sys.stdout.write(" URL: {}\n".format(jblob['url']))
if deck is not None and not skip_deck:
# now we have to add cards to the deck...
if is_create or not deck_meta_only:
sys.stdout.write(" creating or updating deck list...\n")
cardtext = '\n'.join(jblob['mainboard_cards'])
if 'sideboard_cards' in jblob and jblob['sideboard_cards'] is not None and len(jblob['sideboard_cards']) > 0:
cardtext = cardtext + '\nSB:' + '\nSB: '.join(jblob['sideboard_cards'])
if 'commandzone_cards' in jblob and jblob[
'commandzone_cards'] is not None and len(jblob['commandzone_cards']) > 0:
cardtext = cardtext + '\nCZ:' + '\nCZ: '.join(jblob['commandzone_cards'])
#sys.stdout.write(cardtext + "\n")
# REVISIT - just updating them all for now. Shouldn't hurt since
# set_cards_from_text deletes all of the current card associations, right?
if True or deck.get_card_count() == 0:
try:
deck.set_cards_from_text(cardtext)
except Deck.CardsNotFoundException as cnfe:
for g in cnfe.cnfes:
try:
sys.stdout.write(" ERROR: Could not find card " +
str(g.text) +
" in file {}\n".format(filename))
except UnicodeEncodeError:
sys.stdout.write(
" ERROR: Could not find card BUT I CAN'T TELL YOU ABOUT IT BECAUSE UNICODE IN PYTHON SUCKS in {}\n".format(filename))
if tourna is not None and tourna.name is not None and tourna.name.lower().find('test deck') < 0:
# Now we will associate the deck to a tournament
td = TournamentDeck.objects.filter(deck=deck, tournament=tourna).first()
place = 99999
if 'place' in jblob:
place = jblob['place']
if td is None:
td = TournamentDeck(deck=deck, tournament=tourna, place=place)
td.save()
sys.stdout.write(" TournamentDeck: created for {}\n".format(jblob['name']))
td = TournamentDeck.objects.filter(deck=deck, tournament=tourna).first() # RELOAD
else:
td.place = place
td.save()
sys.stdout.write(" TournamentDeck: updated for {}\n".format(jblob['name']))
else:
sys.stdout.write(" TournamentDeck: skipped no valid tournament {}\n".format(jblob['name']))
sema = open(join(directory, 'last_complete.semaphore'), 'w')
sema.write(str(self.start_time))
sema.write("\n")
sema.close()
def updated(self):
return dtparse(self._item['updated']).astimezone(tzutc())
config = ConfigParser.SafeConfigParser()
config.read(settings_file)
share_id = config.get('auth', 'app_id')
share_token = config.get('auth', 'app_secret')
# ------------ Real start ----------------
# instanciating the class
isogeo = Isogeo(client_id=share_id,
client_secret=share_token,
lang="fr")
token = isogeo.connect()
# ------------ REAL START ----------------------------
latest_data_modified = isogeo.search(token,
page_size=10,
order_by="modified",
whole_share=0,
sub_resources=["events"]
)
print("Last 10 data updated \nTitle | datetime\n\t description")
for md in latest_data_modified.get("results"):
title = md.get('title')
evt_description = md.get("events")[0].get("description")
print(str("___________________\n\n{} | {} \n\t {}").
format(title.encode("utf8"),
dtparse(md.get("modified")[:19]).strftime("%a %d %B %Y"),
evt_description.encode("utf8")))
def query(self, cpf, birth):
"""Queries a CPF at Receita Federal"""
cpf = CPF(cpf)
birth = dtparse(birth)
if self.method == "crack":
captcha = ""
r = self.session.get(self.captcha_url, stream=True)
if r.status_code == 200:
with open("/tmp/teste.png", "wb") as f:
for chunk in r.iter_content(1024):
f.write(chunk)
c = Captcha("deathbycaptcha")
captcha = c.decode("/tmp/teste.png")
else:
return False
payload = {
"txtTexto_captcha_serpro_gov_br": captcha,
"tempTxtCPF": cpf.format(),
"tempTxtNascimento": birth.strftime("%d/%m/%Y"),
"temptxtToken_captcha_serpro_gov_br": "",
"temptxtTexto_captcha_serpro_gov_br": captcha,
}
r = self.session.post(self.post_url, data=payload)
soup = bs(r.content, "lxml")
obj = {}
obj["death_year"] = None
obj["death_message"] = None
obj["cpf"] = cpf.format()
obj["birth"] = birth.strftime("%Y/%m/%d")
for span in soup.find_all("span", {"class": "clConteudoDados"}):
text = span.text.strip()
value = text.split(":")[1].strip()
if text.startswith("Nome da Pessoa"):
obj["name"] = value
elif text.startswith("Situa") and "Cadastral" in text:
obj["situation"] = value
elif text.startswith("Data da Inscri"):
obj["subscription_date"] = dtparse(value.replace("anterior a", "")).strftime("%Y/%m/%d")
elif text.startswith("Digito Verificador"):
obj["verification_digit"] = value
elif "digo de controle do comprovante" in text:
obj["control_code"] = value
elif self.method == "bypass":
# Inspired by http://hc0der.blogspot.com.br/2014/09/bypass-captcha-da-receita-federal.html
token_string = "{}{}".format(cpf.cpf, birth.strftime("%d%m%Y"))
token = hmac.new(self.bypass_secret.encode("utf-8"), token_string.encode("utf-8"), hashlib.sha1)
headers = {
"token": token.hexdigest(),
"plataforma": "iPhone OS",
"dispositivo": "iPhone",
"aplicativo": "Pessoa Física",
"versao": "8.3",
"versao_app": "4.1",
"Content-Type": "application/x-www-form-urlencoded",
}
payload = "cpf={}&dataNascimento={}".format(cpf.cpf, birth.strftime("%d%m%Y"))
r = self.session.post(self.bypass_url, headers=headers, data=payload, verify=False)
data = r.json()
obj = {}
obj["cpf"] = cpf.format()
obj["birth"] = birth.strftime("%Y/%m/%d")
obj["name"] = data["nome"]
obj["situation"] = data["descSituacaoCadastral"]
obj["subscription_date"] = dtparse(data["dataIsncricao"].replace("anterior a", "")).strftime("%Y/%m/%d")
obj["death_year"] = data["anoObito"]
obj["death_message"] = data["mensagemObito"]
obj["control_code"] = data["codigoControle"]
obj["verification_digit"] = data["digitoVerificador"]
obj["return_message"] = data.get("mensagemRetorno", None)
return obj
def setUp(self):
cs1 = CallSource.objects.create(call_source_id=1,
descr='Citizen Initiated', code='C')
cs2 = CallSource.objects.create(call_source_id=2,
descr='Self Initiated', code='S')
self.n1 = Nature.objects.create(nature_id=1, descr='Robbery')
self.n2 = Nature.objects.create(nature_id=2, descr='Homicide')
self.call1 = create_call(call_id=1,
time_received='2014-01-15T9:00',
nature=self.n1,
call_source=cs1)
self.call2 = create_call(call_id=2,
time_received='2014-03-18T3:00',
nature=self.n2,
call_source=cs1)
self.b1 = Beat.objects.create(beat_id=1, descr='B1')
self.b2 = Beat.objects.create(beat_id=2, descr='B2')
self.d1 = District.objects.create(district_id=1, descr='D1')
self.d2 = District.objects.create(district_id=2, descr='D2')
self.cu1 = CallUnit.objects.create(call_unit_id=1, descr='A1',
beat=self.b1, district=self.d1)
self.cu2 = CallUnit.objects.create(call_unit_id=2, descr='B2',
beat=self.b2, district=self.d2)
self.at1 = OfficerActivityType.objects.create(
officer_activity_type_id=1,
descr="IN CALL - CITIZEN INITIATED")
self.at2 = OfficerActivityType.objects.create(
officer_activity_type_id=2,
descr="OUT OF SERVICE")
self.at3 = OfficerActivityType.objects.create(
officer_activity_type_id=3,
descr="ON DUTY")
self.a1 = create_officer_activity(activity_type=self.at1,
time=dtparse(
'2014-01-15T9:00'),
call_unit=self.cu1,
call=self.call1)
self.a2 = create_officer_activity(activity_type=self.at1,
time=dtparse(
'2014-01-15T9:10'),
call_unit=self.cu2,
call=self.call1)
self.a3 = create_officer_activity(activity_type=self.at1,
time=dtparse(
'2014-01-15T10:00'),
call_unit=self.cu1,
call=self.call2)
self.a4 = create_officer_activity(activity_type=self.at1,
time=dtparse(
'2014-01-16T9:50'),
call_unit=self.cu2,
call=self.call2)
self.a5 = create_officer_activity(activity_type=self.at2,
time=dtparse(
'2014-01-16T10:10'),
call_unit=self.cu1,
call=None)
self.a6 = create_officer_activity(activity_type=self.at2,
time=dtparse(
'2014-01-18T9:00'),
call_unit=self.cu2,
call=None)
# In order for this to be realistic, for every busy activity,
# we need to have an on duty activity
self.a7 = create_officer_activity(activity_type=self.at3,
time=dtparse('2014-01-15T9:00'),
call_unit=self.cu1,
call=None)
self.a8 = create_officer_activity(activity_type=self.at3,
time=dtparse('2014-01-15T9:10'),
call_unit=self.cu2,
call=None)
self.a9 = create_officer_activity(activity_type=self.at3,
time=dtparse('2014-01-15T10:00'),
call_unit=self.cu1,
call=None)
self.a10 = create_officer_activity(activity_type=self.at3,
time=dtparse('2014-01-16T9:50'),
call_unit=self.cu2,
call=None)
self.a11 = create_officer_activity(activity_type=self.at3,
time=dtparse('2014-01-16T10:10'),
call_unit=self.cu1,
call=None)
self.a12 = create_officer_activity(activity_type=self.at3,
time=dtparse('2014-01-18T9:00'),
call_unit=self.cu2,
call=None)
def test_evaluates_one_activity(self):
# Should return 1 busy activity (a6), 1 on_duty activity (a12)
overview = OfficerActivityOverview(q('time__gte=2014-01-17'))
over_time_results = overview.to_dict()['allocation_over_time']
correct_over_time_results = {
str(dtparse('9:00').time()):
{
'IN CALL - CITIZEN INITIATED': {
'avg_volume': 0,
'total': 0,
'freq': 1
},
'IN CALL - SELF INITIATED': {
'avg_volume': 0,
'total': 0,
'freq': 1
},
'IN CALL - DIRECTED PATROL': {
'avg_volume': 0,
'total': 0,
'freq': 1
},
'OUT OF SERVICE': {
'avg_volume': 1.0,
'total': 1,
'freq': 1
},
'ON DUTY': {
'avg_volume': 1.0,
'total': 1,
'freq': 1
},
'PATROL': {
'avg_volume': 0.0,
'total': 0,
'freq': 1
}
}
}
self.assertEqual(sorted(over_time_results.keys()),
sorted(correct_over_time_results.keys()))
self.assertEqual(sorted(over_time_results.values()),
sorted(correct_over_time_results.values()))
by_beat_results = overview.to_dict()['on_duty_by_beat']
correct_by_beat_results = [
{'beat_id': 2, 'beat': 'B2', 'on_duty': 1.0},
]
by_district_results = overview.to_dict()['on_duty_by_district']
correct_by_district_results = [
{'district_id': 2, 'district': 'D2', 'on_duty': 1.0},
]
for results, correct in (
(by_beat_results, correct_by_beat_results),
(by_district_results, correct_by_district_results)):
for d in correct:
self.assertIn(d, results)
for d in results:
self.assertIn(d, correct)
def parse_printdeck(self, response):
#self.log('Found deck at {}'.format(response.url))
deck = DeckItem(url=response.url)
title_l = response.xpath('//span[contains(@class, "titletext")]/a/text()').extract()
title = title_l[0]
#self.log("TITLE: " + str(title))
deck['name'] = str(title)
formatname = None
place = 99999
body_poop = response.xpath('//body/text()').extract()
for poop in body_poop:
find_format = FORMAT_FIND_RE.search(poop)
if find_format:
formatname = find_format.group(1)
place_format = PLACE_FIND_RE.search(poop)
if place_format:
place = place_format.group(1)
#self.log("FORMAT: " + str(formatname))
deck['deck_format'] = str(formatname)
#self.log("PLACE: " + str(place))
deck['place'] = str(place)
authornames = response.xpath('//a[contains(@href, "p_first=")]/text()').extract()
if len(authornames) > 0:
#self.log("AUTHORNAME: " + str(authornames[0]))
deck['author'] = str(authornames[0])
cities = response.xpath('//a[contains(@href, "city=")]/text()').extract()
city = ''
if len(cities) > 0:
#self.log("CITY: " + str(cities[0]))
city = str(cities[0])
states = response.xpath('//a[contains(@href, "state=")]/text()').extract()
if len(states) > 0:
#self.log("STATE: " + str(states[0]))
pass
countries = response.xpath('//a[contains(@href, "country=")]/text()').extract()
if len(countries) > 0:
#self.log("COUNTRY: " + str(countries[0]))
pass
start_dates = response.xpath('//a[contains(@href, "start_date=")]/@href').extract()
if len(start_dates) > 0:
sd_idx = start_dates[0].find('start_date=')
if sd_idx > -1:
start_date = start_dates[0][sd_idx + len('start_date='):sd_idx + len('start_date=') + len('xxxx-yy-zz')]
#self.log("DATE: " + start_date)
deck['tournament_date'] = start_date
end_dates = response.xpath('//a[contains(@href, "end_date=")]/@href').extract()
end_date = None
if len(end_dates) > 0:
sd_idx = end_dates[0].find('end_date=')
if sd_idx > -1:
end_date = end_dates[0][sd_idx + len('end_date='):sd_idx + len('end_date=') + len('xxxx-yy-zz')]
url_base = 'http://sales.starcitygames.com/deckdatabase/deckshow.php?'
format_part = ''
if deck['deck_format'].lower() == 'standard':
format_part = 't[T1]=1'
elif deck['deck_format'].lower() == 'modern':
format_part = 't[T3]=28'
deck['tournament_url'] = url_base + '&'.join([format_part,
'city=' + city,
'start_date=' + deck['tournament_date'],
'end_date=' + end_date])
mainboard_lines = list()
sideboard_lines = list()
for span in response.xpath('//span'):
class_attrs = span.xpath('@class').extract()
if len(class_attrs) > 0 and class_attrs[0] == 'titletext':
# skip
continue
kids = span.xpath('*|text()')
in_sb = False
cur_num = None
for kid in kids:
in_sb = in_sb or kid.extract().find('<strong>Sideboard') > -1
is_text_number_match = GET_NUMBER_RE.search(str(kid.extract()))
if is_text_number_match:
cur_num = is_text_number_match.group(1)
else:
ggg = kid.xpath('./text()').extract()
booh = None
if cur_num:
booh = '{} {}'.format(str(cur_num), ggg[0].strip())
cur_num = None
if in_sb:
sideboard_lines.append(booh)
else:
mainboard_lines.append(booh)
#self.log("MAIN: " + str(mainboard_lines))
#self.log("SIDE: " + str(sideboard_lines))
deck['mainboard_cards'] = mainboard_lines
deck['sideboard_cards'] = sideboard_lines
#self.log("DECK: " + str(deck))
yield deck
td_o = dtparse(deck['tournament_date'])
titem = TournamentItem(name='StarCityGames.com {} in {} {}'.format(deck['deck_format'], city, deck['tournament_date']),
url=deck['tournament_url'],
tournament_format=deck['deck_format'],
tournament_date=td_o)
yield titem
