def insert_folders(folders: list):
""" Inserts list of folders into cache. Sets 'update' column to current date.
:param folders: list of raw dict-type folders
"""
if not folders:
return
session = db.Session()
for folder in folders:
logger.debug(folder)
# root folder has no name key
f_name = folder.get("name")
f = db.Folder(
folder["id"],
f_name,
iso_date.parse(folder["createdDate"]),
iso_date.parse(folder["modifiedDate"]),
folder["status"],
)
f.updated = datetime.utcnow()
session.merge(f)
try:
session.commit()
except IntegrityError:
logger.warning("Error inserting folders.")
session.rollback()
logger.info("Inserted/updated %d folders." % len(folders))
def parse(date, dayfirst=True):
'''Parse a `date` into a `FlexiDate`.
@param date: the date to parse - may be a string, datetime.date,
datetime.datetime or FlexiDate.
TODO: support for quarters e.g. Q4 1980 or 1954 Q3
TODO: support latin stuff like M.DCC.LIII
TODO: convert '-' to '?' when used that way
e.g. had this date [181-]
'''
if not date:
return None
if isinstance(date, FlexiDate):
return date
if isinstance(date, int):
return FlexiDate(year=date)
elif isinstance(date, datetime.datetime):
parser = PythonDateTimeParser()
return parser.parse(date)
elif isinstance(date, datetime.date):
parser = PythonDateParser()
return parser.parse(date)
else: # assuming its a string
parser = DateutilDateParser()
out = parser.parse(date, **{'dayfirst': dayfirst})
if out is not None:
return out
# msg = 'Unable to parse %s' % date
# raise ValueError(date)
val = 'UNPARSED: %s' % date
val = val.encode('ascii', 'ignore')
return FlexiDate(qualifier=val)
def save_photo(self, photo, album):
"""Save the given photo, if it's not already saved"""
# if album doesn't exist, create it
local_album = self.save_album(album)
if 'source' not in photo:
return None
# if photo doesn't exist, add it to album
local_photo, created = Photo.objects.get_or_create(
album=local_album,
external_id=photo['id'],
external_source='Facebook',
)
# update these fields no matter what
local_photo.name = photo.get('name', None)
local_photo.external_url = photo['link']
local_photo.created_time = parse(photo['created_time'])
local_photo.updated_time = parse(photo['updated_time'])
try:
local_photo.position = photo['position']
except Exception:
pass
local_photo.save()
# update photo no matter what
self.download_image(photo['source'], local_photo)
if local_photo.external_id == album['cover_photo']:
local_album.cover = local_photo
local_album.save()
return local_photo
def getCurrentProg(decodedcontent):
# Work out what's on NOW here
utcdatetime = datetime.now()
print("Current time is: " + str(utcdatetime))
for programme in decodedcontent['schedule']['day']['broadcasts']:
starttime = parse(programme['start'])
starttime = starttime.replace(tzinfo=None)
endtime = parse(programme['end'])
endtime = endtime.replace(tzinfo=None)
if (utcdatetime >= starttime) & (utcdatetime < endtime):
print("The current programme is '" + programme['programme']['display_titles']['title'] + "'")
pid = programme['programme']['pid']
#return pid
break
# TEMPORARY - Set Twitter filter
prog = re.sub(",","",programme['programme']['display_titles']['title'])
chan = decodedcontent['schedule']['service']['title']
prognospace = re.sub("\s+","",prog)
channospace = re.sub("\s+","",chan)
#totrack = {'track' : prog + "," + prognospace + "," + channospace + "," + '#' + channospace}
totrack = {'track' : "#" + prognospace + "," + '#' + channospace}
data = urllib.urlencode(totrack)
print("Ready to track '" + totrack['track'] + "'")
return [data,endtime]
def whitespace(options):
# clean events
Event.objects.filter(source="whitespace").delete()
soup = BeautifulSoup(urlopen("http://www.0x20.be/Main_Page").read())
for event in soup.ul('li'):
if event.text == 'More...':
continue
title = event.a.text
url = "http://www.0x20.be" + event.a["href"]
if "-" in event.b.text[:-1]:
start, end = map(lambda x: parse(x.strip()), event.b.text[:-1].split("-"))
else:
start = parse(event.b.text[:-1])
end = None
location = event('a')[1].text
Event.objects.create(
title=title,
source="whitespace",
url=url,
start=start,
end=end,
location=location.strip() if location else None
)
if not options["quiet"]:
print "Adding %s [%s] (%s)..." % (title.encode("Utf-8"), "whitespace", location.encode("Utf-8"))
def get_next_election(self, datafile=None, electiondate=None):
"""
Get next election. By default, will be relative to the current date.
:param datafile:
If datafile is specified, use instead of making an API call.
:param electiondate:
If electiondate is specified, gets the next election
after the specified date.
"""
if not electiondate:
today = datetime.datetime.now()
else:
today = dateutil_parser.parse(electiondate)
next_election = None
lowest_diff = None
for e in self.get_elections(datafile=datafile):
diff = (dateutil_parser.parse(e.electiondate) - today).days
if diff > 0:
if not lowest_diff and not next_election:
next_election = e
lowest_diff = diff
elif lowest_diff and next_election:
if diff < lowest_diff:
next_election = e
lowest_diff = diff
return next_election
def _parse_date(item):
try:
date_string = item['data']['date']
parsed_date = parser.parse(date_string, fuzzy=True, default=datetime.today())
except ValueError:
try:
date_chunks = date_string.split()
parsed_date = parser.parse(date_chunks[0], fuzzy=True, default=datetime.today())
except:
try:
alternate_date_string = item['meta']['parsedDate']
parsed_date = parser.parse(alternate_date_string, fuzzy=True, default=datetime.today())
except ValueError:
try:
date_chunks = alternate_date_string.split()
parsed_date = parser.parse(date_chunks[0], fuzzy=True, default=datetime.today())
except:
parsed_date = datetime.today()
return parsed_date
def unserialize(cls, data):
""" From JSON -> Activity object """
dataobj = data["object"]
obj_type = dataobj["objectType"].capitalize()
if "author" not in dataobj:
# author is not set for posted objects in inbox/major, so we add it
dataobj["author"] = data["actor"]
try:
objekt = getattr(cls._pump, obj_type)
obj = objekt.unserialize(dataobj)
except AttributeError:
obj = Unknown.unserialize(dataobj)
verb = data["verb"]
actor = cls._pump.Person.unserialize(data["actor"])
# generator is not always there (at least not for verb:'update' obj:Person)
generator = Generator.unserialize(data["generator"]) if "generator" in data else None
updated = parse(data["updated"])
url = data["url"]
published = parse(data["published"])
content = data["content"]
id = data["id"]
return cls(obj, verb, actor, generator, updated,
url, published, content, id)
def parse_gnats_item(hxs, scope=1):
item = GnatsItem()
item['number'] = get_text(hxs, '//*[@id="val_number"]')
if scope > 1:
item['number'] += '-%s' % scope
item['title'] = get_text(hxs, '//*[@id="val_synopsis"]')
item['responsible'] = get_text(hxs, '//*[@id="val_responsible_%s"]/a' % scope)
item['state'] = get_text(hxs, '//*[@id="val_state_%s"]' % scope)
item['reported_in'] = get_text(hxs, '//*[@id="val_reported-in"]')
item['submitter'] = get_text(hxs, '//*[@id="val_submitter-id"]')
item['category'] = get_text(hxs, '//*[@id="val_category"]/a')
item['level'] = get_text(hxs, '//*[@id="val_problem-level"]')
item['platform'] = get_text(hxs, '//*[@id="val_platform"]')
item['originator'] = get_text(hxs, '//*[@id="val_originator"]')
item['customer'] = get_text(hxs, '//*[@id="val_customer"]')
item['qa_owner'] = get_text(hxs, '//*[@id="val_systest-owner_%s"]/a' % scope)
item['ce_owner'] = get_text(hxs, '//*[@id="val_customer-escalation-owner"]/a')
item['dev_owner'] = get_text(hxs, '//*[@id="val_dev-owner_%s"]/a' % scope)
item['audit_trail'] = hxs.select('//*[@id="audit-trail"]').extract()[0].strip()
item['last_audit'] = get_text(hxs, '//*[@id="audit-trail"]//div[@class="section-contents"][last()]/pre')
item['arrived_at'] = parse(get_text(hxs, '//*[@id="val_arrival-date"]'))
item['modified_at'] = parse(get_text(hxs, '//*[@id="val_last-modified"]'))
item['crawled'] = True
item['worker'] = []
return item
def str2dt(time: datetime) -> np.ndarray:
"""
Converts times in string or list of strings to datetime(s)
Parameters
----------
time : str or datetime.datetime or numpy.datetime64
Results
-------
t : datetime.datetime
"""
if isinstance(time, datetime):
return time
elif isinstance(time, str):
return parse(time)
elif isinstance(time, np.datetime64):
return time.astype(datetime)
else: # some sort of iterable
try:
if isinstance(time[0], datetime):
return time
elif isinstance(time[0], np.datetime64):
return time.astype(datetime)
elif isinstance(time[0], str):
return [parse(t) for t in time]
except (IndexError, TypeError):
pass
# last resort--assume pandas/xarray
return time.values.astype('datetime64[us]').astype(datetime)
def __init__(self, site=None, variable=None, value_count=None,
begin_date_time_utc=None, end_date_time_utc=None,
source=None):
if not type(begin_date_time_utc) is datetime.datetime:
begin_date_time_utc = parse(begin_date_time_utc)
if not type(end_date_time_utc) is datetime.datetime:
end_date_time_utc = parse(end_date_time_utc)
if begin_date_time_utc.tzinfo is None:
begin_date_time_utc = begin_date_time_utc.replace(tzinfo=utc)
if end_date_time_utc.tzinfo is None:
end_date_time_utc = end_date_time_utc.replace(tzinfo=utc)
self.Site = site
self.Variable = variable
self.ValueCount = value_count
self.BeginDateTimeUTC = begin_date_time_utc
self.EndDateTimeUTC = end_date_time_utc
self.BeginDateTime = begin_date_time_utc
self.EndDateTime = end_date_time_utc
#SWIS data are all "Raw Data"
# though might have more than one QC level in the future
self.QualityControlLevelID = \
wof_base.QualityControlLevelTypes['RAW_DATA'][1]
self.QualityControlLevelCode = \
wof_base.QualityControlLevelTypes['RAW_DATA'][0]
self.Source = source
def _parse_line(self, line_str: str) -> list:
"""
parses a string into a list that can be read by the chamber
:param line_str:
:return: list of values.
"""
line = line_str.strip().split(",")
def f(v):
try:
return float(v)
except:
return v
if len(line) in (16, 13):
try:
return [
parser.parse("{} {}".format(line[0], line[1]), dayfirst=True),
*map(f, line[2:-1]),
parser.parse(line[-1])
]
except Exception as e:
traceback.print_exc()
else:
try:
return [
parser.parse(line[0], dayfirst=True),
*map(f, line[1:-1]),
parser.parse(line[-1])
]
except:
traceback.print_exc()
return list(map(f, line))
def test_parse_dates_custom_euroformat(self):
from dateutil.parser import parse
text = """foo,bar,baz
31/01/2010,1,2
01/02/2010,1,NA
02/02/2010,1,2
"""
parser = lambda d: parse(d, dayfirst=True)
df = read_csv(StringIO(text), skiprows=[0],
names=['time', 'Q', 'NTU'], index_col=0,
parse_dates=True, date_parser=parser,
na_values=['NA'])
exp_index = Index([datetime(2010, 1, 31), datetime(2010, 2, 1),
datetime(2010, 2, 2)], name='time')
expected = DataFrame({'Q' : [1, 1, 1], 'NTU' : [2, np.nan, 2]},
index=exp_index, columns=['Q', 'NTU'])
assert_frame_equal(df, expected)
parser = lambda d: parse(d, day_first=True)
self.assertRaises(Exception, read_csv,
StringIO(text), skiprows=[0],
names=['time', 'Q', 'NTU'], index_col=0,
parse_dates=True, date_parser=parser,
na_values=['NA'])
def is_report_ok(self, report):
if not report.header:
logger.error("header is empty!")
return False
if not report.items:
logger.error("report does not contain any item!")
return False
if 'session_name' not in report.header:
logger.error("header does not contain a 'session_name' item!")
return False
if 'meeting_number' not in report.header:
logger.error("header does not contain a 'meeting_number' item!")
return False
if 'meeting_date' not in report.header:
logger.error("header does not contain a 'meeting_date' item!")
return False
if not re.match(r'[\w-]+\ session', report.header['session_name']):
logger.error("invalid value for 'session_name': %s",
report.header['session_name'])
return False
if not re.match(r'\d+', report.header['meeting_number']):
logger.error("invalid value for 'meeting_number': %s",
report.header['meeting_number'])
return False
try:
from dateutil.parser import parse
parse(report.header['meeting_date'])
except ValueError as err:
logger.error("invalid value for 'meeting_date': '%s' (%s)",
report.header['meeting_date'], format(err))
return False
return True
def StringToDate(str_d, format='%Y-%m-%d', db=None):
"""Parse the string using the given format and return the date.
With StringToDate, it's best to have a fixed default format,
as it is easier for formulas to control the input date string than the
portal date format.
Pass `format=None` to allow StringToDate to guess.
"""
try:
if db:
format = db.datetime_format
if format:
dt = datetime.strptime(str_d, format)
else:
dt = parse(str_d)
except ValueError as e:
# XXX: Just let DateTime guess.
dt = parse(DateTime(str_d).ISO())
logger.info('StringToDate> %s, %s, %s, guessed: %s' % (
str(str_d),
format,
repr(e),
repr(dt)))
return DateTime(dt).toZone(TIMEZONE)
def _save_event(self, event):
"""Save the given event"""
uid = event['id']
try:
local_event = Event.objects.get(uid=uid)
except:
local_event = Event(uid=uid)
local_event.calendar = self.gcal
local_event.author = event['creator']
local_event.title = event['summary']
local_event.description = event.get('description', None)
local_event.location = event.get('location', None)
try:
local_event.start = parse(event['start']['dateTime'])
except:
local_event.start = parse(event['start']['date'])
try:
local_event.end = parse(event['end']['dateTime'])
except:
local_event.end = parse(event['end']['date'])
status = 'active'
if event['status'] == 'cancelled':
status = 'cancelled'
local_event.status = status
local_event.save()
return local_event
def convert_date(date_str):
# app.logger.debug(date_str)
#try:
date_str = date_str.replace(u" в ",u" ")
today_str = u"сегодня"
yesterday_str = u"вчера"
if today_str in date_str:
rowDate = date_str.replace(today_str,"")
date = parse(rowDate)
return date
elif yesterday_str in date_str:
rowDate = date_str.replace(yesterday_str,"")
date = parse(rowDate) - timedelta(days=1)
return date
else: #replace cyrillics in date and parse
# app.logger.debug("passed")
for k, v in rus_to_eng.items():
if date_str != date_str.replace(k,v):
rowDate = date_str.replace(k,v)
date = parse(rowDate)
return date
print("something is wrong with date")
return datetime.now(pytz.utc)
#except Exception as e:
# app.logger.debug(str(e))
return datetime.now(pytz.utc)
def test_Notification_eventtime(self):
datetimes = [parse('2016-08-02T10:40:40.014291-07:00').isoformat(),
parse('2016-07-29T14:26:58.519605-07:00').isoformat(),
parse('2016-07-29T14:28:25.793194-07:00').isoformat(),
parse('2016-07-29T14:28:25.793421-07:00').isoformat()]
for (actual, expected) in zip(self.notifications, datetimes):
self.assertEqual(actual.eventTime.isoformat(), expected)
def post(self, id = None): request = self.wsgi_request response = self.wsgi_response c = self.context param_dict = dict( title = request.params.title, conf_id = request.params.conf_id, desc = request.params.desc, venue = request.params.venue, talk_type = request.params.talk_type, start_date = parse(request.params.start_date), end_date = parse(request.params.end_date), duration = request.params.duration, speaker_title = request.params.speaker_title) content = param_dict session = Session(**param_dict) session = session.save() conf = Conference.get(session.conf_id) if conf: conf.add_session(session) self.set_body(session.to_json()) response.headers['content-type'] = 'application/json' return self.render()
def _process_json(response_body):
"""
Returns a UwPassword objects
"""
data = json.loads(response_body)
uwpassword = UwPassword(uwnetid=data["uwNetID"],
kerb_status=data["kerbStatus"],
interval=None,
last_change=None,
last_change_med=None,
expires_med=None,
interval_med=None,
minimum_length=int(data["minimumLength"]),
time_stamp=parse(data["timeStamp"]),)
if "lastChange" in data:
uwpassword.last_change = parse(data["lastChange"])
if "interval" in data:
uwpassword.interval = timeparse(data["interval"])
if "lastChangeMed" in data:
uwpassword.last_change_med = parse(data["lastChangeMed"])
if "expiresMed" in data:
uwpassword.expires_med = parse(data["expiresMed"])
if "intervalMed" in data:
uwpassword.interval_med = timeparse(data["intervalMed"])
if "netidStatus" in data:
netid_status = []
for status in data["netidStatus"]:
netid_status.append(status)
uwpassword.netid_status = netid_status
return uwpassword
def get(self, itemId):
ffrom, to = self.get_argument('from', None), self.get_argument('to', None)
filt = {'item_id': ObjectId(itemId), '_id': {
'$gt': ObjectId.from_datetime(parse(ffrom) if ffrom else (self.now - datetime.timedelta(1))),
'$lte': ObjectId.from_datetime(parse(to) if to else self.now)
}}
self.write(dumps(alfred.db.values.find(filt)))
def test_get_sprint_dates(self):
date_ref = parse("2015-09-04-T-09:00")
expect_d1 = parse("2015-09-10-T-09:00")
expect_d2 = parse("2015-09-23-T-09:00")
d1, d2 = utils.get_sprint_dates(date_ref)
self.assertEquals(d1, expect_d1)
self.assertEquals(d2, expect_d2)
def test_non_conflict_schedule():
print(
'Test non-conflicting requests: Agent2 and Agent1 live in harmony',
now)
sch_man = ScheduleManager(60, now=now)
ag1 = ('Agent1', 'Task1',
(['campus/building/rtu1', parse('2013-11-27 12:00:00'),
parse('2013-11-27 12:15:00')],
['campus/building/rtu2', parse('2013-11-27 12:00:00'),
parse('2013-11-27 13:00:00')],
['campus/building/rtu3', parse('2013-11-27 12:45:00'),
parse('2013-11-27 13:00:00')],),
PRIORITY_LOW_PREEMPT,
now)
now2 = now + timedelta(minutes=55)
ag2 = ('Agent2', 'Task2',
(['campus/building/rtu1', parse('2013-11-27 12:30:00'),
parse('2013-11-27 13:00:00')],),
PRIORITY_HIGH,
now2)
result1, event_time1 = verify_add_task(sch_man, *ag1)
success1, data1, info_string1 = result1
assert all((success1, not data1, not info_string1,
event_time1 == parse('2013-11-27 12:00:00')))
result2, event_time2 = verify_add_task(sch_man, *ag2)
success2, data2, info_string2 = result2
assert success2
assert not data2
assert info_string2 == ''
assert event_time2 == parse('2013-11-27 12:30:00')
def test_conflict_override_fail_on_running_agent():
print(
'Test conflicting requests: Agent2 fails to override running Agent1',
now)
sch_man = ScheduleManager(60, now=now)
ag1 = ('Agent1', 'Task1',
(['campus/building/rtu1', parse('2013-11-27 12:00:00'),
parse('2013-11-27 12:35:00')],),
PRIORITY_LOW,
now)
ag2 = ('Agent2', 'Task2',
(['campus/building/rtu1', parse('2013-11-27 12:30:00'),
parse('2013-11-27 13:00:00')],),
PRIORITY_HIGH,
now + timedelta(minutes=45))
result1, event_time1 = verify_add_task(sch_man, *ag1)
success1, data1, info_string1 = result1
assert all((success1, not data1, not info_string1,
event_time1 == parse('2013-11-27 12:00:00')))
result2, event_time2 = verify_add_task(sch_man, *ag2)
success2, data2, info_string2 = result2
conflicts2 = data2
assert not success2
assert conflicts2 == {'Agent1': {'Task1': [
['campus/building/rtu1', '2013-11-27 12:00:00',
'2013-11-27 12:35:00']]}}
def test_conflict_override_error():
print('Test conflicting requests: '
'Agent2 fails to override running Agent1 '
'because of non high priority.',
now)
sch_man = ScheduleManager(60, now=now)
ag1 = ('Agent1', 'Task1',
(['campus/building/rtu1', parse('2013-11-27 12:00:00'),
parse('2013-11-27 12:35:00')],),
PRIORITY_LOW_PREEMPT,
now)
ag2 = ('Agent2', 'Task2',
(['campus/building/rtu1', parse('2013-11-27 12:30:00'),
parse('2013-11-27 13:00:00')],),
PRIORITY_LOW,
now)
result1, event_time1 = verify_add_task(sch_man, *ag1)
success1, data1, info_string1 = result1
assert all((success1, not data1, not info_string1,
event_time1 == parse('2013-11-27 12:00:00')))
result2, event_time2 = verify_add_task(sch_man, *ag2)
success2, data2, info_string2 = result2
conflicts2 = data2
assert not success2
assert conflicts2 == {'Agent1': {'Task1': [
['campus/building/rtu1', '2013-11-27 12:00:00',
'2013-11-27 12:35:00']]}}
def getVmEvents():
event_filter = _eventMapper(request.args)
if request.args.get('stream') != 'true':
start_time = parser.parse(request.args.get('start_time', datetime(1970, 1, 1).isoformat()))
stop_time = parser.parse(request.args.get('end_time', datetime.utcnow().isoformat()))
elements = int(request.args.get('max_events', 10))
def generator():
events = app.eventStore.get(start_time, stop_time, elements)
for event in events:
if 'all' in event_filter or event["event_type"] in event_filter:
yield json.dumps(event, default=_datetime_serial) + '\n'
return Response(generator(), mimetype='application/json')
def stream(environ, start_response):
start_response('200 OK', [('Content-Type', 'application/json')])
def generator():
body = queue.Queue()
app.eventBroker.subscribe(body)
try:
for item in body:
if 'all' in event_filter or item["event_type"] in event_filter:
yield json.dumps(item, default=_datetime_serial) + '\n'
except Exception as e:
app.eventBroker.unsubscribe(body)
raise e
except GeneratorExit as e:
app.eventBroker.unsubscribe(body)
raise e
return generator()
return stream
def api_timeline(request):
response = {}
userId = int(request.REQUEST.get('userId', '0'))
forumId = int(request.REQUEST.get('forumId', '0')) or int(request.REQUEST.get('groupId', '0'))
unit = request.REQUEST.get('unit', '0')
startDate = str(request.REQUEST.get('startDate', ''))
endDate = str(request.REQUEST.get('endDate', ''))
response["earliestDate"] = Annotation.objects.filter(forum=forumId).order_by('created_at')[0].created_at.ctime()
response["latestDate"] = Annotation.objects.filter(forum=forumId).order_by('-created_at')[0].created_at.ctime()
response["timeline"] = []
# Postgres specific
byunit_select = {unit: "DATE_TRUNC('" + unit + "', created_at)"}
annotations = Annotation.objects.filter(forum=forumId)
if len(startDate) > 0:
annotations = annotations.filter(created_at__gte=parser.parse(startDate))
if len(endDate) > 0:
annotations = annotations.exclude(created_at__gte=parser.parse(endDate))
items = annotations.extra(select=byunit_select).values(unit).annotate(count=Count('id')).order_by(unit)
for item in items:
time_item = {}
time_item[unit] = item[unit].ctime()
time_item["count"] = item["count"]
response["timeline"].append(time_item)
return HttpResponse(json.dumps(response), mimetype='application/json')
def test_conflict_override_success_running_agent2():
print('Test conflicting requests: '
'Agent2 overrides running Agent1 which has more than one device',
now)
sch_man = ScheduleManager(60, now=now)
ag1 = ('Agent1', 'Task1',
(['campus/building/rtu1', parse('2013-11-27 12:00:00'),
parse('2013-11-27 12:15:00')],
['campus/building/rtu2', parse('2013-11-27 12:00:00'),
parse('2013-11-27 13:00:00')],
['campus/building/rtu3', parse('2013-11-27 12:45:00'),
parse('2013-11-27 13:00:00')],),
PRIORITY_LOW_PREEMPT,
now)
now2 = now + timedelta(minutes=55)
ag2 = ('Agent2', 'Task2',
(['campus/building/rtu3', parse('2013-11-27 12:30:00'),
parse('2013-11-27 13:00:00')],),
PRIORITY_HIGH,
now2)
result1, event_time1 = verify_add_task(sch_man, *ag1)
success1, data1, info_string1 = result1
assert all((success1, not data1, not info_string1,
event_time1 == parse('2013-11-27 12:00:00')))
result2, event_time2 = verify_add_task(sch_man, *ag2)
success2, data2, info_string2 = result2
assert success2
assert data2 == {('Agent1', 'Task1')}
assert info_string2 == ''
assert event_time2 == parse('2013-11-27 12:26:00')
def _query_datetime_range(start_time=None, end_time=None, isoformat=True, **time_delta):
"""
Get a PyMongo query expression for datetime
:param start_time: (str) Query start time (inclusive) in isoformat (YYYY-MM-DDTHH:MM:SS.mmmmmm).
Default is 30 days before current time.
:param end_time: (str) Query end time (exclusive) in isoformat (YYYY-MM-DDTHH:MM:SS.mmmmmm).
Default is current time.
:param isoformat: (bool) If ruturned Pymongo query uses isoformat for datetime. Default is True.
:param time_delta: (dict) Time difference to calculate start_time from end_time. Accepts arguments in python
datetime.timedelta function. time_delta and start_time can not be given at the same time. Default is 30 days.
:return: (dict) A Mongodb query expression for a datetime range.
"""
if start_time and time_delta:
raise SyntaxError("Can't specify start_time and time_delta at the same time!")
if end_time:
end_time = parser.parse(end_time)
else:
end_time = datetime.utcnow()
if not start_time:
if not time_delta:
time_delta = {"days":30}
start_time = end_time-timedelta(**time_delta)
else:
start_time = parser.parse(start_time)
if start_time > end_time:
raise ValueError("query_start should be earlier than query_end!")
if isoformat:
return {"$gte":start_time.isoformat(), "$lt":end_time.isoformat()}
else:
return {"$gte":start_time, "$lt":end_time}
def create_difference(new_csv_path):
logger.debug("new_csv_path {}".format(new_csv_path))
filename = os.path.basename(new_csv_path)
logger.debug("filename {}".format(filename))
date = parser.parse(filename[:8]).date()
csvs = os.listdir(os.path.dirname(new_csv_path))
csvs = [csv for csv in csvs if "all_current" in csv]
csvs = [{"file":this_file, "date":parser.parse(this_file[:8]).date()} for this_file in csvs]
newlist = sorted(csvs, key=lambda k: k['date'], reverse=True)
older_dates = [d for d in newlist if d["date"]<date]
if len(older_dates)>0:
most_recent_previous_file = older_dates[0]["file"]
else:
return None
most_recent_previous_file_full = os.path.join(os.path.dirname(new_csv_path),most_recent_previous_file)
#Now we just find the diff
new = pd.read_csv(new_csv_path)
old = pd.read_csv(most_recent_previous_file_full,usecols=["hash"])
diff = new[-new["hash"].isin(old["hash"].unique())]
diff_file_path = os.path.join(os.path.dirname(new_csv_path),filename[:8] + "__diff_from_" + most_recent_previous_file[:8]+".csv")
diff.to_csv(diff_file_path, index=False)
return diff_file_path
def getArticleURLS(source, args):
#Create filename where everything is stored eventually. Doing str(int()) so the time is rounded off
filename = source+'_ARTICLES'+'_YEAR_'+str(args.scrapeYear)+'_MONTH_'+str(args.scrapeMonth)+'_'+str(int(time.time()))+'.csv'
urls = []
currentPage = 1
print currentPage
hasNextPage = True
outOfRange = False
while hasNextPage and not outOfRange:
print 'setting dict'
#Parse HTML, invoke config (x)paths
tree = parsedHTML(resultsConfig(currentPage)[source]['pageURL'])
print("#################")
print tree.xpath(resultsConfig(currentPage)[source]['itemXpath'])
items = tree.xpath(resultsConfig(currentPage)[source]['itemXpath'])
print 'looping over items'
print len(items)
#For every item on the search results page...
for item in items:
#Here we invoke the correct Xpaths from the config dict above
#Not every results page correctly displays datetime in result, so if it's not here
#do the check when fetching the articles. Else, if its ordered by date just terminate if the current article date is < the year youre scraping
if resultsConfig(currentPage)[source]['dateOnPage'] and resultsConfig(currentPage)[source]['dateOrdered'] and args.scrapeYear:
articleDate = dparser.parse(item.xpath(resultsConfig(currentPage)[source]['dateXpath'])[0].get('datetime'), fuzzy=True)
#If we already see that the article date is not from a year we want to collect (eg if from 2014 and 2015 was specified)
#then we just terminate the while loop. Only works one way, as articles are ordered by date, so can only do if smaller
yearToScrape =int(args.scrapeYear)
if articleDate.year < yearToScrape:
outOfRange = True
#Note that it then just terminates on the next page (since there is no 'break' statement for the while loop)
articleURL = item.xpath(resultsConfig(currentPage)[source]['urlXpath'])[0].get('href')
#Some websites have relative URL pointers, so prefix the base URL
if '://' not in articleURL:
articleURL = resultsConfig(currentPage)[source]['baseURL']+articleURL
#Urlfilter hack to prevent video/audio/gadfly pages from being visited (mostly bloomberg)
#These pages have custom xpath structures, so not even bothering collecting them
urlFilters = ['/videos/','/audio/','/gadfly/','/features/','/press-releases/']
#If any of the above strings is in the url, pass writing it, else write it
if any(urlFilter in articleURL for urlFilter in urlFilters):
pass
else:
urls.append(articleURL)
#If there are less items in the results than the resultsPerPage param, we assume this is the last page
if len(items) < resultsConfig(currentPage)[source]['resultsPerPage']:
hasNextPage = False
#Increase page number by 1 for the next iteration of the while loop
currentPage += 1
#Once all URLs for the page have been collected, go visit the actual articles
#Do this here so it doesn't first collect too many URLs that are useless afterwards
collectArticles(urls, source, args, filename)
#Reinitialize URLS array again for next loop
urls = []
def _jinja2_filter_datetime(date, fmt=None):
date = parser.parse(date)
native = date.replace(tzinfo=None)
format = "%d %b %y, %H:%M"
return native.strftime(format)
else:
ee = re.findall('\nFrom: (\S+@\S+)\n', header)
if len(ee) == 1:
email = ee[0]
email = email.lower().strip()
email = email.replace("<","")
# Find sent_at date
dd = re.findall('\nDate: .*, (.*)\n', header)
if len(dd) == 1:
date = dd[0]
date = date[:26] # hard counted to be the first 26 characters
# Try to parse the date using
try:
sent_at = parse(date)
except:
print('Cannot parse: ', date)
fail = fail + 1
if fail > 5:
break
continue
# Find subject
hh = re.findall('\nSubject: (.*)\n', header)
if len(hh) == 1:
subject = hh[0]
subject = subject.lower().strip()
# Reset fail counter:
fail = 0
def parse_time_str(time_str):
return parse(time_str)
def process(self, **kwargs):
if self.PARAM_CONFIG in kwargs:
config_id = kwargs[self.PARAM_CONFIG]['pk']
else:
config_id = kwargs[self.PARAM_CONFIG_ID]
if self.PARAM_USER in kwargs:
user_ids = {kwargs[self.PARAM_USER]['pk']}
else:
user_ids = kwargs.get(self.PARAM_USER_IDS)
run_date = kwargs.get(self.PARAM_RUN_DATE)
run_date_specified = run_date is not None
if isinstance(run_date, str):
run_date = parse(run_date)
run_date = run_date or datetime.datetime.now(
tz=tzlocal.get_localzone())
run_even_if_not_enabled = bool(
kwargs.get(self.PARAM_RUN_EVEN_IF_NOT_ENABLED))
config = DocumentDigestConfig.objects \
.filter(pk=config_id).select_related('for_role', 'for_user').first() # type: DocumentDigestConfig
if not config:
self.log_error('{1} not found: #{0}'.format(
config_id, DocumentDigestConfig.__name__))
return
if not config.enabled and not run_even_if_not_enabled:
self.log_info('{1} #{0} is disabled.'.format(
config_id, DocumentDigestConfig.__name__))
return
tz_msg = ' at timezone {0}'.format(
run_date.tzname()) if run_date_specified else ''
self.log_info(
'Rendering and sending {what} #{pk} ({doc_filter}) for date "{run_date}" to {n} users{tz_msg}'
.format(what=DocumentDigestConfig.__name__,
pk=config.pk,
doc_filter=config.documents_filter,
n=len(user_ids),
run_date=run_date,
tz_msg=tz_msg))
if user_ids:
users_qr = User.objects.filter(pk__in=user_ids)
elif config.for_role_id is not None:
users_qr = User.objects.filter(role_id=config.for_role_id)
elif config.for_user_id is not None:
users_qr = User.objects.get(pk=config.for_user_id)
else:
self.log_error(
'{what} #{config_id} specifies neither for_user nor for_role.'.
format(what=DocumentDigestConfig.__name__,
config_id=config.pk))
return
log = CeleryTaskLogger(self)
for user in users_qr: # type: User
if config.for_user_id != user.id and (
config.for_role_id is None
or config.for_role_id != user.role_id):
self.log_error(
'{what} #{what_id} is not applicable for user {user_name} (#{user_id})'
.format(what=DocumentDigestConfig.__name__,
what_id=config.pk,
user_name=user.get_full_name(),
user_id=user.pk))
continue
try:
digest = render_digest(config=config,
dst_user=user,
run_date=run_date)
if digest:
digest.send(log)
except Exception as e:
self.log_error(
f'Unable to send {RenderedDigest}.\n'
f'Config: #{config.pk}\n'
f'Dst user: {user.get_full_name()} #{user.pk}\n'
f'Run date: {run_date}',
exc_info=e)
apod_base = 'http://apod.nasa.gov/apod/'
apod_archive_url = apod_base + 'archivepix.html'
apod_encoding = 'latin-1'
archive_soup = absolute_soup(scraperwiki.scrape(apod_archive_url), apod_encoding, apod_base)
archive_links = archive_soup.find_all(href=re.compile('ap[0-9]+\.html'))
for archive_link in archive_links:
page_soup = absolute_soup(scraperwiki.scrape(archive_link['href']), apod_encoding, apod_base)
# URL
url = archive_link['href']
# Date
date_raw = archive_link.previous_sibling[:-3]
date = parser.parse(date_raw).strftime('%Y-%m-%d')
# Title
title = archive_link.text
# Explanation
page_html = str(page_soup) # The raw HTML, but with links turned absolute.
explanation_ugly = re.search('<(b|(h3))>.*?Explanation.*?</(b|(h3))>\s*(.*?)\s*(</p>)?<p>', page_html, re.DOTALL | re.IGNORECASE).group(5)
explanation = oneline(explanation_ugly)
# Picture URL. Check that there actually is a picture, as NASA sometimes
# publishes videos instead.
picture_link = page_soup.find(href=re.compile(apod_base + 'image/'))
if picture_link:
picture_url = picture_link['href']
picture_found = True
def __init__(self, date=None):
if date is not None:
self.dateToMatch = dparser.parse(date).date()
else:
self.dateToMatch = None
# 建立資料庫連線
conn = sqlite3.connect('market_price.sqlite')
# 寫入LOG File
dt = datetime.datetime.now()
print("##############################################")
print("## Nvesto 上市櫃公司 ##")
print("## 券商每日進出分點資料讀取 ##")
print("## ##")
print("## datetime: " + str(dt) + " ##")
print("##############################################")
str_date = str(dt)
str_date = parser.parse(str_date).strftime("%Y%m%d")
name = "GET_BROKER_TRADING_LOG_" + str_date + ".txt"
file = open(name, 'a', encoding = 'UTF-8')
tStart = time.time()#計時開始
file.write("\n\n\n*** LOG datetime " + str(datetime.datetime.now()) + " ***\n")
#登入網站
Login_nvesto = Login_nvesto()
#print(Login_nvesto)
if Login_nvesto == True:
print("Nvesto網站登入成功. \n")
else:
file.write("$$$ Nvesto網站登入失敗，程式中止. $$$")
def run(self, conf, args, plugins):
if 'subcommand' in args:
if args.subcommand == 'info':
print("Not implemented yet")
elif args.subcommand == "intel":
# Start with MISP and OTX to get Intelligence Reports
print('###################### %s ###################' %
args.DOMAIN)
passive_dns = []
urls = []
malware = []
files = []
# OTX
otx_e = plugins['otx'].test_config(conf)
if otx_e:
print('[+] Downloading OTX information....')
try:
otx = OTXv2(conf["AlienVaultOtx"]["key"])
res = otx.get_indicator_details_full(
IndicatorTypes.DOMAIN, unbracket(args.DOMAIN))
otx_pulses = res["general"]["pulse_info"]["pulses"]
# Get Passive DNS
if "passive_dns" in res:
for r in res["passive_dns"]["passive_dns"]:
passive_dns.append({
"ip":
r['hostname'],
"first":
parse(r["first"]).astimezone(pytz.utc),
"last":
parse(r["last"]).astimezone(pytz.utc),
"source":
"OTX"
})
if "url_list" in res:
for r in res["url_list"]["url_list"]:
if "result" in r:
urls.append({
"date":
parse(r["date"]),
"url":
r["url"],
"ip":
r["result"]["urlworker"]["ip"] if "ip"
in r["result"]["urlworker"] else "",
"source":
"OTX"
})
else:
urls.append({
"date": parse(r["date"]),
"url": r["url"],
"ip": "",
"source": "OTX"
})
except AttributeError:
print('OTX crashed ¯\_(ツ)_/¯')
# UrlScan
us = UrlScan()
print('[+] Downloading UrlScan information....')
res = us.search(args.DOMAIN)
for r in res['results']:
urls.append({
"date":
parse(r["task"]["time"]).astimezone(pytz.utc),
"url":
r["page"]["url"],
"ip":
r["page"]["ip"],
"source":
"UrlScan"
})
# CIRCL
circl_e = plugins['circl'].test_config(conf)
if circl_e:
print('[+] Downloading CIRCL passive DNS information....')
x = pypdns.PyPDNS(basic_auth=(conf['Circl']['user'],
conf['Circl']['pass']))
res = x.query(unbracket(args.DOMAIN))
for answer in res:
passive_dns.append({
"ip":
answer['rdata'],
"first":
answer['time_first'].astimezone(pytz.utc),
"last":
answer['time_last'].astimezone(pytz.utc),
"source":
"CIRCL"
})
# BinaryEdge
be_e = plugins['binaryedge'].test_config(conf)
if be_e:
print('[+] Downloading BinaryEdge information....')
try:
be = BinaryEdge(conf['BinaryEdge']['key'])
res = be.domain_dns(unbracket(args.DOMAIN))
for d in res['events']:
if "A" in d:
for a in d['A']:
passive_dns.append({
"ip":
a,
"first":
parse(d['updated_at']).astimezone(
pytz.utc),
"last":
parse(d['updated_at']).astimezone(
pytz.utc),
"source":
"BinaryEdge"
})
except BinaryEdgeException:
print(
'You need a paid BinaryEdge subscription for this request'
)
# RobTex
print('[+] Downloading Robtex information....')
rob = Robtex()
res = rob.get_pdns_domain(args.DOMAIN)
for d in res:
if d['rrtype'] in ['A', 'AAAA']:
passive_dns.append({
'first':
d['time_first_o'].astimezone(pytz.utc),
'last':
d['time_last_o'].astimezone(pytz.utc),
'ip':
d['rrdata'],
'source':
'Robtex'
})
# PT
pt_e = plugins['pt'].test_config(conf)
if pt_e:
try:
pt_osint = {}
ptout = False
print('[+] Downloading Passive Total information....')
client = DnsRequest(conf['PassiveTotal']['username'],
conf['PassiveTotal']['key'])
raw_results = client.get_passive_dns(
query=unbracket(args.DOMAIN))
if "results" in raw_results:
for res in raw_results["results"]:
passive_dns.append({
"first":
parse(res["firstSeen"]).astimezone(
pytz.utc),
"last":
parse(res["lastSeen"]).astimezone(
pytz.utc),
"ip":
res["resolve"],
"source":
"PT"
})
if "message" in raw_results:
if "quota_exceeded" in raw_results["message"]:
print("PT quota exceeded")
ptout = True
if not ptout:
client2 = EnrichmentRequest(
conf["PassiveTotal"]["username"],
conf["PassiveTotal"]['key'])
# Get OSINT
# TODO: add PT projects here
pt_osint = client2.get_osint(
query=unbracket(args.DOMAIN))
# Get malware
raw_results = client2.get_malware(
query=unbracket(args.DOMAIN))
if "results" in raw_results:
for r in raw_results["results"]:
malware.append({
'hash':
r["sample"],
'date':
parse(r['collectionDate']),
'source':
'PT (%s)' % r["source"]
})
except requests.exceptions.ReadTimeout:
print("PT: Time Out")
# VT
vt_e = plugins['vt'].test_config(conf)
if vt_e:
if conf["VirusTotal"]["type"] != "public":
print('[+] Downloading VT information....')
vt = PrivateApi(conf["VirusTotal"]["key"])
res = vt.get_domain_report(unbracket(args.DOMAIN))
if "results" in res:
if "resolutions" in res['results']:
for r in res["results"]["resolutions"]:
passive_dns.append({
"first":
parse(r["last_resolved"]).astimezone(
pytz.utc),
"last":
parse(r["last_resolved"]).astimezone(
pytz.utc),
"ip":
r["ip_address"],
"source":
"VT"
})
if "undetected_downloaded_samples" in res[
'results']:
for r in res['results'][
'undetected_downloaded_samples']:
files.append({
'hash':
r['sha256'],
'date':
parse(r['date'])
if 'date' in r else '',
'source':
'VT'
})
if "undetected_referrer_samples" in res['results']:
for r in res['results'][
'undetected_referrer_samples']:
files.append({
'hash':
r['sha256'],
'date':
parse(r['date'])
if 'date' in r else '',
'source':
'VT'
})
if "detected_downloaded_samples" in res['results']:
for r in res['results'][
'detected_downloaded_samples']:
malware.append({
'hash': r['sha256'],
'date': parse(r['date']),
'source': 'VT'
})
if "detected_referrer_samples" in res['results']:
for r in res['results'][
'detected_referrer_samples']:
if "date" in r:
malware.append({
'hash': r['sha256'],
'date': parse(r['date']),
'source': 'VT'
})
if "detected_urls" in res['results']:
for r in res['results']['detected_urls']:
urls.append({
'date':
parse(r['scan_date']).astimezone(
pytz.utc),
'url':
r['url'],
'ip':
'',
'source':
'VT'
})
else:
vt_e = False
tg_e = plugins['threatgrid'].test_config(conf)
if tg_e:
try:
print('[+] Downloading Threat Grid....')
tg = ThreatGrid(conf['ThreatGrid']['key'])
res = tg.search_samples(unbracket(args.DOMAIN),
type='domain')
already = []
if 'items' in res:
for r in res['items']:
if r['sample_sha256'] not in already:
d = parse(r['ts'])
d = d.replace(tzinfo=None)
malware.append({
'hash': r["sample_sha256"],
'date': d,
'source': 'ThreatGrid'
})
already.append(r['sample_sha256'])
except ThreatGridError as e:
print("Failed to connect to Threat Grid: %s" %
e.message)
# TODO: Add MISP
print('----------------- Intelligence Report')
if otx_e:
if len(otx_pulses):
print('OTX:')
for p in otx_pulses:
print(' -%s (%s - %s)' %
(p['name'], p['created'][:10],
"https://otx.alienvault.com/pulse/" +
p['id']))
else:
print('OTX: Not found in any pulse')
if pt_e:
if "results" in pt_osint:
if len(pt_osint["results"]):
if len(pt_osint["results"]) == 1:
if "name" in pt_osint["results"][0]:
print(
"PT: %s %s" %
(pt_osint["results"][0]["name"],
pt_osint["results"][0]["sourceUrl"]))
else:
print(
"PT: %s" %
(pt_osint["results"][0]["sourceUrl"]))
else:
print("PT:")
for r in pt_osint["results"]:
if "name" in r:
print("-%s %s" %
(r["name"], r["sourceUrl"]))
else:
print("-%s" % (r["sourceUrl"]))
else:
print("PT: Nothing found!")
else:
print("PT: Nothing found!")
if len(malware) > 0:
print('----------------- Malware')
for r in sorted(malware, key=lambda x: x["date"]):
print("[%s] %s %s" % (r["source"], r["hash"],
r["date"].strftime("%Y-%m-%d")))
if len(files) > 0:
print('----------------- Files')
for r in files:
if r['date'] != '':
print("[%s] %s (%s)" %
(r["source"], r["hash"],
r["date"].strftime("%Y-%m-%d")))
else:
print("[%s] %s" % (
r["source"],
r["hash"],
))
if len(urls) > 0:
print('----------------- Urls')
for r in sorted(urls,
key=lambda x: x["date"],
reverse=True):
print("[%s] %s - %s %s" %
(r["source"], r["url"], r["ip"],
r["date"].strftime("%Y-%m-%d")))
# TODO: add ASN + location info here
if len(passive_dns) > 0:
print('----------------- Passive DNS')
for r in sorted(passive_dns,
key=lambda x: x["first"],
reverse=True):
print("[+] %-40s (%s -> %s)(%s)" %
(r["ip"], r["first"].strftime("%Y-%m-%d"),
r["last"].strftime("%Y-%m-%d"), r["source"]))
else:
self.parser.print_help()
else:
self.parser.print_help()
def on_rec_analysis_message(self, topic, headers, message, matched):
'''Subscribe to device data and assemble data set to pass
to applications.
'''
obj = jsonapi.loads(message[0])
if isinstance(obj, list):
obj = obj[0]
dev_list = topic.split('/')
device_or_subdevice = dev_list[-2]
device_id = [
dev for dev in self._master_devices
if dev == device_or_subdevice
]
subdevice_id = [
dev for dev in self._master_subdevices
if dev == device_or_subdevice
]
if not device_id and not subdevice_id:
return
if isinstance(device_or_subdevice, unicode):
device_or_subdevice = (
device_or_subdevice.decode('utf-8').encode('ascii'))
def agg_subdevice(obj):
sub_obj = {}
for key, value in obj.items():
sub_key = ''.join([key, '_', device_or_subdevice])
sub_obj[sub_key] = value
if len(dev_list) > 5:
self._subdevice_values.update(sub_obj)
self._needed_subdevices.remove(device_or_subdevice)
else:
self._device_values.update(sub_obj)
self._needed_devices.remove(device_or_subdevice)
return
# The below if statement is used to distinguish between unit/all
# and unit/sub-device/all
if (device_or_subdevice not in self._needed_devices
and device_or_subdevice not in self._needed_subdevices):
_log.error("Warning device values already present, "
"reinitializing")
self._initialize_devices()
agg_subdevice(obj)
if self._should_run_now():
field_names = {}
self._device_values.update(self._subdevice_values)
for k, v in self._device_values.items():
field_names[k.lower() if isinstance(k, str) else k] = v
if not converter.initialized and \
conv_map is not None:
converter.setup_conversion_map(map_names, field_names)
if from_file:
_timestamp = parse(headers.get('Date'), fuzzy=True)
self.received_input_datetime = _timestamp
else:
_timestamp = datetime.now()
self.received_input_datetime = datetime.utcnow()
obj = converter.process_row(field_names)
results = app_instance.run(_timestamp, obj)
# results = app_instance.run(
# dateutil.parser.parse(self._subdevice_values['Timestamp'],
# fuzzy=True), self._subdevice_values)
self._process_results(results)
self._initialize_devices()
else:
needed = deepcopy(self._needed_devices)
needed.extend(self._needed_subdevices)
_log.info("Still need {} before running.".format(needed))
def clean_data(self, value, field):
"""
Clean the data based on the field type.
"""
field_type = field.get_internal_type()
if field_type in ['CharField', 'EmailField',
'URLField', 'SlugField']:
if not value:
value = ''
if len(value) > field.max_length:
# truncate the value to ensure its length <= max_length
value = value[:field.max_length]
if field.name == 'time_zone':
if value not in pytz.all_timezones:
if value in self.t4_timezone_map_keys:
value = self.t4_timezone_map[value]
try:
value = field.to_python(value)
except exceptions.ValidationError:
if field.has_default():
value = field.get_default()
else:
value = ''
elif field_type == 'BooleanField':
try:
if value in [True, 1, 'TRUE']:
value = True
value = field.to_python(value)
except exceptions.ValidationError:
value = False
elif field_type == 'DateField':
if value:
value = dparser.parse(value)
try:
value = field.to_python(value)
except exceptions.ValidationError:
pass
if not value:
if not field.null:
value = date
elif field_type == 'DateTimeField':
if value:
try:
value = dparser.parse(value)
except:
value = None
try:
value = field.to_python(value)
except exceptions.ValidationError:
pass
if not value:
if value == '':
value = None
if not field.null:
value = datetime.now()
elif field_type == 'DecimalField':
try:
value = field.to_python(value)
except exceptions.ValidationError:
value = Decimal(0)
elif field_type == 'IntegerField':
try:
value = int(value)
except:
value = 0
elif field_type == 'FloatField':
try:
value = float(value)
except:
value = 0
elif field_type == 'ForeignKey':
orignal_value = value
# assume id for foreign key
try:
value = int(value)
except:
value = None
if value:
try:
model = field.related.parent_model()
except AttributeError:
model = field.related.model
[value] = model.objects.filter(pk=value)[:1] or [None]
if not value and not field.null:
# if the field doesn't allow null, grab the first one.
try:
model = field.related.parent_model()
except AttributeError:
model = field.related.model
[value] = model.objects.all().order_by('id')[:1] or [None]
return value
import csv
from dateutil.parser import parse
from dateutil import tz
from ics import Calendar, Event
from pytz import timezone
import sys
import re
from bs4 import BeautifulSoup
add_default_tz = lambda x, tzinfo: x.replace(tzinfo=x.tzinfo or tzinfo)
MDT = tz.gettz("US/Mountain")
parsed_dt = lambda s: add_default_tz(parse(s), MDT)
def find_sessions(soup):
"Parse HTML file and extract session info"
sessions = soup.findAll('div', class_="agenda-item")
for session in sessions:
date = session.find('div', {"class": "time"}).next_element.next_sibling
title = session.find('h3', {"class": "title"}).text
speakers = session.findAll('div', {"class": "speaker"})
abstract = session.find('span', {"class": "excerpt"}).text
yield {
'begin': parsed_dt(date.split('-')[0]),
'end': parsed_dt(re.sub(r'\d\d:\d\d - ', '', date)),
'title': title,
'author': ', '.join([speaker.next.text for speaker in speakers]),
'abstract': abstract
}
def csv2fund(filename):
"""
@summary converts a csv file to a fund with the given starting value
@param filename: csv file to open and convert
@param start_val: starting value for the portfolio
@return fund : time series containing fund value over time
@return leverage : time series containing fund value over time
@return slippage : value of slippage over the csv time
@return commissions : value of slippage over the csv time
"""
reader=csv.reader(open(filename,'rU'), delimiter=',')
reader.next()
symbols=[]
dates=[]
for row in reader:
if not(row[0] in symbols):
if not(row[0]=="cash" or row[0] == ''):
symbols.append(row[0])
if not(dp.parse(row[3]) in dates):
dates.append(dp.parse(row[3]))
print symbols
reader=csv.reader(open(filename,'rU'), delimiter=',')
reader.next()
if not("_CASH" in symbols):
symbols.append("_CASH")
vals=numpy.zeros([len(dates),len(symbols)])
share_table=pandas.DataFrame(index=dates, columns=symbols, data=vals)
share_table["_CASH"]=0
# share_table["_CASH"].ix[0]=start_val
commissions=0
for row in reader:
date=dp.parse(row[3])
order_type=row[2]
sym = row[0]
if order_type != 'Deposit Cash':
price = float(row[5])
shares=float(row[4])
commission=float(row[7])
if order_type=="Deposit Cash":
cash = float(row[6])
share_table["_CASH"].ix[date]=share_table.ix[date]["_CASH"]+ cash
if order_type=="Buy":
share_table.ix[date][sym]+=shares
commissions=commissions+float(commission)
share_table["_CASH"].ix[date]=share_table.ix[date]["_CASH"]-float(price)*float(shares)-float(commission)
if order_type=="Sell":
share_table[sym].ix[date]+=-1*shares
commissions=commissions+float(commission)
share_table["_CASH"].ix[date]=share_table.ix[date]["_CASH"]+float(price)*float(shares)-float(commission)
if order_type=="Sell Short":
share_table[sym].ix[date]+=-1*shares
commissions=commissions+float(commission)
share_table["_CASH"].ix[date]=share_table.ix[date]["_CASH"]+float(price)*float(shares)-float(commission)
if order_type=="Buy to Cover":
share_table.ix[date][sym]+=shares
commissions=commissions+float(commission)
share_table["_CASH"].ix[date] = share_table.ix[date]["_CASH"]-float(price)*float(shares)-float(commission)
share_table = share_table.cumsum()
time_index = sorted(share_table.index)
column_index = sorted(share_table.columns)
share_table = share_table.reindex(index=time_index, columns=column_index)
i_start_cash = share_table["_CASH"].ix[0]
print i_start_cash
return [share_table, commissions, i_start_cash]
currentsignature = makeSignature(currentmeta)
currentproposal = currentmeta['experiment_identifier']
print('FD:%s:%d:%s:%s' % (root, year, inst, postfix))
print('PROP:%d:%d:%s ' % (numor, 0, currentmeta['experiment_identifier']))
print('DS:%d:%d:%s ' % (numor, 0, currentsignature))
previousmeta = currentmeta
tdiffsig = currentsignature
prevtxtsig = txtSignature(currentmeta)
for numor, dfile in fliter:
try:
meta = readMetaData(dfile)
meta['numor'] = numor
except Exception as err:
print('Failed to read %s with %s' % (dfile, str(err)))
signature = makeSignature(meta)
etime = parser.parse(previousmeta['end_time'])
stime = parser.parse(meta['start_time'])
diff = (stime - etime).total_seconds()
txtsig = txtSignature(meta)
# if(diff > 4000):
# print('PREDICT:%d' % (numor))
print('TDIFF:%d:%d:%d:%f:%d' %(numor-1,numor,levenshtein(tdiffsig,signature),\
diff,levenshtein(prevtxtsig,txtsig)))
etime = parser.parse(meta['end_time'])
print('ELAPSED:%d:%f' % (numor, (etime - stime).total_seconds()))
previousmeta = meta
tdiffsig = signature
prevtxtsig = txtsig
mdiff = levenshtein(currentsignature, signature)
if meta['experiment_identifier'] != currentproposal:
print('PROP:%d:%d:%s ' % (numor, mdiff, meta['experiment_identifier']))
def convert_timezone(s, from_zone, to_zone):
"""Converts a datetime iso string (datetime.isoformat()) from one zone to another"""
if isinstance(s, str):
s = parser.parse(s)
s = s.replace(tzinfo=from_zone)
return s.astimezone(to_zone).isoformat()
def analyze_transactions(filename, plot_name, share_table, show_transactions=False):
"""
@summary computes various statistics for given filename and appends to report assumed via plot_name
@param filename: file of past transactions
@param plot_name: name of report
"""
html_file = open("./"+plot_name+"/report-"+plot_name+".html","a")
html_file.write("<pre>\n\nTransaction Statistics\n")
#calc stats
#first pass
reader=csv.reader(open(filename,'rU'), delimiter=',')
reader.next()
prev=0
first=1
diffs=[]
volume=0
start=0
sold=0
bought=0
end=0
rets=[]
efficiencies=[]
holds=[]
commissions=[]
# slippage=[]
buy_dates=[] #matrix of when stocks were bought (used for matches)
for row in reader:
weighted_ret=0
weighted_e=0
weighted_hold=0
num_stocks=0
volume+=1
if(row[7]!=''):
commissions.append(float(row[7]))
# slippage.append(float(row[8]))
if first:
#add na for first trade efficiency
start=dp.parse(row[3])
first=0
prev=dp.parse(row[3])
else:
if row[2] == "Buy":
bought=bought+float(row[5])
buy_dates.append({"date":dp.parse(row[3]),"stock":row[0],"amount":row[4],"price":float(row[5])})
elif row[2] == "Sell":
#sold at price
sold=sold+float(row[5])
#get number of stocks for this sell
stocks=float(row[4])
#try and match trade (grab first date of stocks)
for date in buy_dates:
#while stocks are left
if(stocks>0):
#match a date
if(date["stock"]==row[1]):
stocks_sold=0
#use as many stocks from date as necessary
leftover=float(date["amount"])-stocks
if(leftover>0):
date["amount"]=leftover
stocks_sold=stocks
#compute stats
temp_e=calculate_efficiency(date["date"], dp.parse(row[3]), row[0])
weighted_ret=(weighted_ret*num_stocks+(float(row[5])/date["price"])*stocks_sold)/(num_stocks+stocks_sold)
weighted_hold=(weighted_hold*num_stocks+(dp.parse(row[3])-date["date"]).days*stocks_sold)/(num_stocks+stocks_sold)
weighted_e=(weighted_e*num_stocks+temp_e*stocks_sold)/(num_stocks+stocks_sold)
num_stocks=num_stocks+stocks_sold
break
else:
stocks_sold=float(date["amount"])
stocks=stocks-stocks_sold
#compute stats
temp_e=calculate_efficiency(date["date"], dp.parse(row[3]), row[0])
weighted_ret=(weighted_ret*num_stocks+(float(row[5])/date["price"])*stocks_sold)/(num_stocks+stocks_sold)
weighted_hold=(weighted_hold*num_stocks+(dp.parse(row[3])-date["date"]).days*stocks_sold)/(num_stocks+stocks_sold)
weighted_e=(weighted_e*num_stocks+temp_e*stocks_sold)/(num_stocks+stocks_sold)
num_stocks=num_stocks+stocks_sold
date["stock"]="DONE"
#buy_dates.remove(date)
#elif row[2] == "Sell Short":
#do nothing
#elif row[2] == "Buy to Cover":
#do nothing
# elif row[2] == "Deposit Cash":
if(prev!=dp.parse(row[3])):
diffs.append(dp.parse(row[3])-prev)
prev=dp.parse(row[3])
end=prev
holds.append(weighted_hold)
efficiencies.append(weighted_e)
rets.append(weighted_ret*100)
avg_period=sum(diffs, dt.timedelta(0))/len(diffs)
avg_hold=_ignore_zeros_average(holds)
t=sold/(bought+sold)
turnover=t/(end-start).days
efficiency=_ignore_zeros_average(efficiencies)
avg_com=_ignore_zeros_average(commissions)
# avg_slip=_ignore_zeros_average(slippage)
avg_ret=_ignore_zeros_average(rets)
#print stats
html_file.write("\nNumber of trades: %10d" % volume)
html_file.write("\nAverage Trading Period: %10s" % str(avg_period).split(",")[0])
html_file.write("\nAverage Position Hold: %5d days" % avg_hold)
html_file.write("\nAverage Daily Turnover: %%%9.4f" % (turnover*100))
html_file.write("\nAverage Trade Efficiency: %%%9.4f" % (efficiency*100))
html_file.write("\nAverage Commissions: %10d" % avg_com)
# html_file.write("\nAverage Slippage: %10d" % avg_slip)
html_file.write("\nAverage Return: %%%9.4f\n\n" % avg_ret)
html_file.write("Positions by Date\n")
for date in share_table.index:
html_file.write("\nPosition for day: "+str(date).split()[0])
html_file.write("\n")
i=0
for item in share_table.ix[date]:
if(item>0):
html_file.write("%15s" % str(share_table.columns[i]))
i=i+1
html_file.write("\n")
for item in share_table.ix[date]:
if(item>0):
html_file.write("%15s" % str(item))
html_file.write("\n\nTransactions for day:\n")
reader=csv.reader(open(filename,'rU'), delimiter=',')
a=0
cash=0
if(show_transactions):
for row in reader:
if a==0:
html_file.write(" Date | ")
html_file.write(" Name | ")
html_file.write(" Type | ")
html_file.write(" Price | ")
html_file.write(" Shares | ")
html_file.write("Commission | ")
# html_file.write(" Slippage | ")
html_file.write("OnHand Cash| ")
html_file.write("Efficiency | ")
html_file.write(" Returns ")
html_file.write(" | ")
html_file.write("\n")
a=1
else:
compute=False
if(len(str(row[3]).split())>2):
compute=(dt.datetime.strptime(str(row[3]),"%b %d, %Y")==date)
else:
compute=(str(row[3])==str(date))
if(compute):
var=row[2]
if var == "Cash Deposit":
cash=cash+float(row[6])
var="Deposit"
elif var == "Cash Withdraw":
cash=cash-float(row[6])
var="Withdraw"
else:
cash=cash-float(row[6])
var=var.split(" ")[0]
html_file.write("%12s | " % str(row[3].split(':')[0]))
html_file.write("%10s | " % str(row[0]))
html_file.write("%10s | " % str(var))
html_file.write("%10s | " % str(row[5]))
html_file.write("%10s | " % str(row[4]))
html_file.write("%10s | " % str(row[7]))
html_file.write("%10s | " % str(row[8]))
html_file.write("%10s | " % str(round(cash,2)))
html_file.write(" %%%9.2f | " % (efficiencies[a-1]*100))
html_file.write(" %%%9.2f " % (rets[a-1]))
a=a+1
html_file.write(" | ")
html_file.write("\n")
html_file.close()
isBadDay = {}
temperature = {}
temperatureHour = {}
wind = {}
windHour = {}
with open(os.path.join(csvDataPath, '1348559.csv'), 'r') as f:
f_csv = csv.reader(f)
headers = next(f_csv)
counter = 0
for row in f_csv:
print(counter)
counter += 1
# date 05
currentDate = row[5]
date = parse(currentDate)
dateString = date.strftime(dateTimeMode)
hour = date.hour
# 天气情况
if dateString not in isBadDay:
isBadDay[dateString] = 0
weatherCondition = row[9]
if weatherCondition != '':
isBadDay[dateString] = 1
# 气温
if dateString not in temperature:
temperature[dateString] = []
temperatureHour[dateString] = [NO_TEM for e in range(24)]
if row[15] != '':
try:
temperature[dateString].append(float(row[15].replace('s', '')))
def parse(self, value):
if isinstance(value, int) or re.match(r'\d+', value):
value = datetime.now() + timedelta(seconds=int())
elif isinstance(value, str):
value = parser.parse(value)
return generate(value, accept_naive=True)
def _runnable(self, interval):
from dateutil import parser
stream = self._container.stats(stream=True, decode=True)
cpu_old = {}
network_old = {}
for raw in stream:
if self._stopper.isSet():
break
stats = {}
stats['info'] = self.get_info()
if stats['info']['status'] in ('running', 'paused'):
stats['read'] = parser.parse(raw['read'])
cpu_stats = {}
try:
cpu_new = {}
cpu_new['total'] = raw['cpu_stats']['cpu_usage'][
'total_usage']
cpu_new['system'] = raw['cpu_stats']['system_cpu_usage']
# Compatibility wih older Docker API
if 'online_cpus' in raw['cpu_stats']:
cpu_stats['online_cpus'] = raw['cpu_stats'][
'online_cpus']
else:
cpu_stats['online_cpus'] = len(
raw['cpu_stats']['cpu_usage']['percpu_usage']
or [])
except KeyError as e:
# raw do not have CPU information
_LOGGER.info(
"Cannot grab CPU usage for container {} ({})".format(
self._container.id, e))
_LOGGER.debug(raw)
else:
if cpu_old:
cpu_delta = float(cpu_new['total'] - cpu_old['total'])
system_delta = float(cpu_new['system'] -
cpu_old['system'])
cpu_stats['total'] = round(0.0, PRECISION)
if cpu_delta > 0.0 and system_delta > 0.0:
cpu_stats['total'] = round(
(cpu_delta / system_delta) *
float(cpu_stats['online_cpus']) * 100.0,
PRECISION)
cpu_old = cpu_new
memory_stats = {}
try:
memory_stats['usage'] = raw['memory_stats']['usage']
memory_stats['limit'] = raw['memory_stats']['limit']
memory_stats['max_usage'] = raw['memory_stats'][
'max_usage']
except (KeyError, TypeError) as e:
# raw_stats do not have MEM information
_LOGGER.info(
"Cannot grab MEM usage for container {} ({})".format(
self._container.id, e))
_LOGGER.debug(raw)
else:
memory_stats['usage_percent'] = round(
float(memory_stats['usage']) /
float(memory_stats['limit']) * 100.0, PRECISION)
network_stats = {}
try:
network_new = {}
_LOGGER.debug("Found network stats: {}".format(
raw["networks"]))
network_stats['total_tx'] = 0
network_stats['total_rx'] = 0
for if_name, data in raw["networks"].items():
_LOGGER.debug(
"Stats for interface {} -> up {} / down {}".format(
if_name, data["tx_bytes"], data["rx_bytes"]))
network_stats['total_tx'] += data["tx_bytes"]
network_stats['total_rx'] += data["rx_bytes"]
network_new = {
'read': stats['read'],
'total_tx': network_stats['total_tx'],
'total_rx': network_stats['total_rx'],
}
except KeyError as e:
# raw_stats do not have NETWORK information
_LOGGER.info(
"Cannot grab NET usage for container {} ({})".format(
self._container.id, e))
_LOGGER.debug(raw)
else:
if network_old:
tx = network_new['total_tx'] - network_old['total_tx']
rx = network_new['total_rx'] - network_old['total_rx']
tim = (network_new['read'] -
network_old['read']).total_seconds()
network_stats['speed_tx'] = round(
float(tx) / tim, PRECISION)
network_stats['speed_rx'] = round(
float(rx) / tim, PRECISION)
network_old = network_new
stats['cpu'] = cpu_stats
stats['memory'] = memory_stats
stats['network'] = network_stats
else:
stats['cpu'] = {}
stats['memory'] = {}
stats['network'] = {}
self._notify(stats)
time.sleep(interval)
def getDateFromTimeString(timeString):
return parse(timeString).strftime(dateTimeMode)
def test_to_datetime_utc(self):
from dateutil.parser import parse
arr = np.array([parse('2012-06-13T01:39:00Z')], dtype=object)
result = to_datetime(arr, utc=True)
self.assert_(result.tz is pytz.utc)
def date_parse(dates):
return [parse(timestamp).time() for timestamp in dates]
def test_func(proxies):
headers = {
'Content-Type': "application/json"
}
if API_KEY_ID:
headers['authorization'] = get_api_key_header_val(API_KEY)
try:
if USERNAME:
res = requests.get(SERVER, auth=(USERNAME, PASSWORD), verify=INSECURE, headers=headers)
else:
res = requests.get(SERVER, verify=INSECURE, headers=headers)
if res.status_code >= 400:
try:
res.raise_for_status()
except requests.exceptions.HTTPError as e:
if HTTP_ERRORS.get(res.status_code) is not None:
# if it is a known http error - get the message form the preset messages
return_error("Failed to connect. "
"The following error occurred: {}".format(HTTP_ERRORS.get(res.status_code)))
else:
# if it is unknown error - get the message from the error itself
return_error("Failed to connect. The following error occurred: {}".format(str(e)))
except requests.exceptions.RequestException as e:
return_error("Failed to connect. Check Server URL field and port number.\nError message: " + str(e))
# build general Elasticsearch class
es = elasticsearch_builder(proxies)
if demisto.params().get('isFetch'):
# check the existence of all necessary fields for fetch
fetch_params_check()
try:
# test if FETCH_INDEX exists
test_query_to_fetch_incident_index(es)
# test if TIME_FIELD in index exists
response = test_time_field_query(es)
# try to get response from FETCH_QUERY - if exists check the time field from that query
temp = test_fetch_query(es)
if temp:
response = temp
# get the value in the time field
hit_date = str(response.get('hits', {}).get('hits')[0].get('_source').get(str(TIME_FIELD)))
# if not a timestamp test the conversion to datetime object
if 'Timestamp' not in TIME_METHOD:
parse(str(hit_date))
# test timestamp format and conversion to date
else:
test_timestamp_format(hit_date)
timestamp_to_date(hit_date)
except ValueError as e:
return_error("Inserted time format is incorrect.\n" + str(e) + '\n' + TIME_FIELD + ' fetched: ' + hit_date)
else:
# check that we can reach any indexes in the supplied server URL
test_general_query(es)
demisto.results('ok')
def date_parse(text, format_='%x'):
formatted_date = datetime.date(1988, 7, 16).strftime(format_)
dayfirst = formatted_date.index('16') == 0
monthfirst = formatted_date.index('7') <= 1
yearfirst = not dayfirst and not monthfirst
return parse(text, dayfirst=dayfirst, yearfirst=yearfirst, ignoretz=True)
def convert_dates(dt):
spl = parse(dt, dayfirst=False)
return str(spl.year) + "-" + add_st(str(spl.month)) + "-" + add_st(
str(spl.day))
def isodate_to_jsdate(isodate):
pydate = dp.parse(isodate)
unixdate = mktime(pydate.timetuple())
jsdate = unixdate * 1000
return jsdate
# Draw the heatmap with the mask and correct aspect ratio
sn.heatmap(corr,
mask=mask,
cmap=cmap,
vmax=.3,
center=0,
square=True,
linewidths=.5,
cbar_kws={"shrink": .5})
#Correlation_Matrix.show()
### Feature Engineering
dataset["first_open"] = [
parser.parse(row_data) for row_data in dataset["first_open"]
]
dataset["enrolled_date"] = [
parser.parse(row_data) if isinstance(row_data, str) else row_data
for row_data in dataset["enrolled_date"]
]
dataset["difference"] = (dataset.enrolled_date -
dataset.first_open).astype('timedelta64[h]')
Response_Figure = plt.figure()
plt.hist(dataset["difference"].dropna(), color='#3F5D7D')
plt.title('Distribution of Time Since Enrolled')
#Response_Figure.show()
plt.hist(dataset["difference"].dropna(), color='#3F5D7D', range=[0, 100])
plt.title('Distribution of Time Since Enrolled')
def parse_date(date_obj):
return parse(date_obj)
def handle_join_date(self, value):
return parse(value).date()
import pandas as pd
import os, sys, re
import numpy as np
import argparse
from collections import OrderedDict
from dateutil.parser import parse
os.getcwd()
from datetime import datetime, date
days_diff = 2
covid_data_path = "/Users/itaybd/covid19/COVID-19/new_covid/"
test_path = "COVID-19/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_deaths_US.csv"
"https://github.com/CSSEGISandData/COVID-19.git"
parse("2/5/2019", dayfirst=True)
def dates_expr(d):
return len(re.findall('[0-9]+', d)) >= 2
def add_st(s):
if len(s) == 1:
return "0" + s
return s
"""
def convert_dates(dt):
spl = dt.split("/")
return spl[2]+"-"+add_st(spl[0])+"-"+add_st(spl[1])
