def main():
argv = docopt(USAGE)
if argv["--start"]:
start = pacific_to_utc(argv["--start"])
elif argv["--last"]:
start = datetime.strftime(
datetime.utcnow() - timedelta(minutes=int(argv["--last"])), "%Y-%m-%dT%H:%M:%SZ")
else:
start = datetime.strftime(datetime.utcnow() - timedelta(days=7), "%Y-%m-%dT%H:%M:%SZ")
if argv["--end"]:
end = pacific_to_utc(argv["--end"])
else:
end = datetime.strftime(datetime.utcnow(),"%Y-%m-%dT%H:%M:%SZ")
pager = PagerDuty(argv['<subdomain>'], argv['<api-token>'], argv['<policy>'])
for command in ['all','wakeups','flakes']:
if argv[command]:
incidents = pager.do_list(command, argv['--no-thurs'], since=start, until=end)
incident_list = list(incidents)
(incident_list_prod, incident_list_staging)=segregation(incident_list)
if incidents:
if argv['--email']:
email_output(incident_list_prod, incident_list_staging, argv['--top'])
elif argv['--top']:
pprint_rankings(top(incident_list_prod, argv['--top']))
pprint_rankings(top(incident_list_staging, argv['--top']))
else:
pprint_incidents(incident_list_prod)
pprint_incidents(incident_list_staging)
if argv['mtr']:
print pager.get_mtr(since=start, until=end)
def call(self, function, params=None):
self.requestPerMinute += 1
now = datetime.utcnow()
if self.requestPerMinute >= self.requestLimit:
waittime = 60 - now.second
logging.warning("Limit for request per minute exceeded. Waiting for: {0} sec.".format(waittime))
time.sleep(waittime)
now = datetime.utcnow()
if self.checkMinute != now.minute:
self.requestPerMinute = 0
self.checkMinute = now.minute
payload = ''
try:
p = "" if not params else '?' + "&".join(
["{key}={value}".format(key=k, value=v) for (k, v) in params.iteritems()])
url = "{base}.{func}{params}".format(base=self.baseConfig["url"], func=function, params=p)
logging.debug("{0} {1} API call:{2}".format(self.checkMinute, self.requestPerMinute, url))
request = urllib2.Request(url, None, self.baseConfig["headers"])
stream = urllib2.urlopen(request)
payload = stream.read()
data = json.loads(payload)
if isinstance(data, dict) and 'ruid' in data:
logging.error('Api call failed with error: {0} Code: {1}'.format(data['message'], data['code']))
return None
return data
except Exception as e:
logging.error('Error: {0} Context: {1}'.format(e, payload))
return None
def _update_session_last_used(self):
if self.key in ApiSession.sessionDict:
ApiSession.sessionDict[self.key]["last_used"] = \
datetime.utcnow()
else:
ApiSession.sessionDict[self.key] = \
{'api': self, 'last_used': datetime.utcnow()}
def run(self, **kwargs):
logger.info("TASK :: alarm_dispatcher")
# Select Alarm where date_start_notice >= now() - 60 minutes and <= now() + 5 minutes
start_time = datetime.utcnow().replace(tzinfo=utc) + relativedelta(minutes=-60)
end_time = datetime.utcnow().replace(tzinfo=utc) + relativedelta(minutes=+5)
alarm_list = Alarm.objects.filter(date_start_notice__range=(start_time, end_time),
status=ALARM_STATUS.PENDING).order_by('date_start_notice')
# Browse all the Alarm found
for obj_alarm in alarm_list:
# Check if there is an existing Event
if obj_alarm.event:
obj_alarm.status = ALARM_STATUS.IN_PROCESS
obj_alarm.save()
second_towait = obj_alarm.get_time_diff()
# If second_towait negative then set to 0 to be run directly
if second_towait <= 0:
perform_alarm.delay(obj_alarm.event, obj_alarm)
else:
# Call the Alarm in the future
perform_alarm.apply_async(
args=[obj_alarm.event, obj_alarm], countdown=second_towait)
else:
logger.error("There is no Event attached to this Alarm: %d" % obj_alarm.id)
## Mark the Alarm as ERROR
obj_alarm.status = ALARM_STATUS.FAILURE
obj_alarm.save()
def latest(self):
""" Return newest record (limited to last week). """
return self.query_legacy(
datetime.utcnow() - timedelta(7),
datetime.utcnow(),
time_near=datetime.utcnow()
)
def valid_admin_cookie(cookie):
if g.read_only_mode:
return (False, None)
# parse the cookie
try:
first_login, last_request, hash = cookie.split(',')
except ValueError:
return (False, None)
# make sure it's a recent cookie
try:
first_login_time = datetime.strptime(first_login, COOKIE_TIMESTAMP_FORMAT)
last_request_time = datetime.strptime(last_request, COOKIE_TIMESTAMP_FORMAT)
except ValueError:
return (False, None)
cookie_age = datetime.utcnow() - first_login_time
if cookie_age.total_seconds() > g.ADMIN_COOKIE_TTL:
return (False, None)
idle_time = datetime.utcnow() - last_request_time
if idle_time.total_seconds() > g.ADMIN_COOKIE_MAX_IDLE:
return (False, None)
# validate
expected_cookie = c.user.make_admin_cookie(first_login, last_request)
return (constant_time_compare(cookie, expected_cookie),
first_login)
def _get_group(self, cr, uid, orderpoint, context=None):
"""
Will return the groups and the end dates of the intervals of the purchase calendar
that need to be executed now.
If a purchase calendar is defined, it should give the
:return [(date, group)]
"""
#Check if orderpoint has last execution date and calculate if we need to calculate again already
calendar_obj = self.pool.get("resource.calendar")
att_obj = self.pool.get("resource.calendar.attendance")
group = False
context = context or {}
context['no_round_hours'] = True
date = False
now_date = self._convert_to_tz(cr, uid, datetime.utcnow(), context=context)
res_intervals = []
if orderpoint.purchase_calendar_id and orderpoint.purchase_calendar_id.attendance_ids:
if orderpoint.last_execution_date:
new_date = datetime.strptime(orderpoint.last_execution_date, DEFAULT_SERVER_DATETIME_FORMAT)
else:
new_date = datetime.utcnow()
# Convert to timezone of user
new_date = self._convert_to_tz(cr, uid, new_date, context=context)
intervals = calendar_obj._schedule_days(cr, uid, orderpoint.purchase_calendar_id.id, 1, new_date, compute_leaves=True, context=context)
for interval in intervals:
# If last execution date, interval should start after it in order not to execute the same orderpoint twice
# TODO: Make the interval a little bigger
if (orderpoint.last_execution_date and (interval[0] > new_date and interval[0] < now_date)) or (not orderpoint.last_execution_date and interval[0] < now_date and interval[1] > now_date):
group = att_obj.browse(cr, uid, interval[2], context=context).group_id.id
date = interval[1]
res_intervals += [(date, group), ]
else:
return [(now_date, None)]
return res_intervals
def __init__(self, title=None, authors=None,
publisher=None, publication_date=datetime.utcnow(),
types=[], content_url=None, valid=True,
updated_by=None):
"""
Constructs a new Publication instance.
Arguments:
title (string): The publication title.
authors (Contact[]): The list of authors.
publisher (string): The publisher of the publication.
publication_date (DateTime): The date of publication.
types (OrgTypeEnum[]): The types associated with the publication.
content_url (string): The URL for the publication.
valid (boolean): The validity of the publication.
updated_by (ObjectId): The object id of the publication's last updater.
"""
self.title = title
self.authors = authors
self.publisher = publisher
self.publication_date = publication_date
self.types = types
self.content_url = content_url
self.valid = valid
self.last_updated = datetime.utcnow()
self.updated_by = updated_by
def home(req):
form = GetCustomerForm(req.GET)
# The data provided on input
cust_id = req.GET.get('cust_id')
client_type = req.GET.get('client_type')
if cust_id:
# What to invoke the service with
request = {'cust_id':cust_id}
# When was the service invoked
before = datetime.utcnow()
# Invoke the service with the user-provided input using a selected client
if client_type == 'AnyServiceInvoker':
response = req.client_any.invoke('customer.get1', request)
else:
response = req.client_json.invoke(request)
# How long we waited for the response, in milliseconds
time = (datetime.utcnow() - before).total_seconds() * 1000
else:
time, response = None, None
return TemplateResponse(req, 'customer.html',
{'time':time, 'form':form, 'response':response})
def run_task(self, **kwargs):
Session.merge(self.task)
self.task.start_time = datetime.utcnow()
self.task.ident = threading.get_ident()
self.task.status = TaskStatus.running.value
Session.merge(self.task)
Session.commit()
try:
self.run_function(**kwargs)
self.task.log = self.log.messages
self.task.end_time = datetime.utcnow()
self.task.status = TaskStatus.finished.value
self.task.result = TaskResult.success.value
self.task = Session.merge(self.task)
Session.commit()
except Exception as e:
self.task.log = self.log.messages
self.task.tb = traceback.format_exc()
self.task.end_time = datetime.utcnow()
self.task.status = TaskStatus.finished.value
self.task.result = TaskResult.fail.value
self.task = Session.merge(self.task)
Session.commit()
defect = jira.defect_for_exception(
"Background Task Error: {}".format(
self.task.name),
e, tb=traceback.format_exc(),
username=self.task.username)
self.task.defect_ticket = defect.key
self.task = Session.merge(self.task)
Session.commit()
finally:
Session.remove()
def page(self):
changemsg = []
if cherrypy.session.id != cherrypy.session.originalid:
if cherrypy.session.originalid is None:
changemsg.append(
'Created new session because no session id was given.')
if cherrypy.session.missing:
changemsg.append(
'Created new session due to missing '
'(expired or malicious) session.')
if cherrypy.session.regenerated:
changemsg.append('Application generated a new session.')
try:
expires = cherrypy.response.cookie['session_id']['expires']
except KeyError:
expires = ''
return page % {
'sessionid': cherrypy.session.id,
'changemsg': '<br>'.join(changemsg),
'respcookie': cherrypy.response.cookie.output(),
'reqcookie': cherrypy.request.cookie.output(),
'sessiondata': list(cherrypy.session.items()),
'servertime': (
datetime.utcnow().strftime('%Y/%m/%d %H:%M') + ' UTC'
),
'serverunixtime': calendar.timegm(datetime.utcnow().timetuple()),
'cpversion': cherrypy.__version__,
'pyversion': sys.version,
'expires': expires,
}
def test_delete_index(self):
mother_case_id = uuid.uuid4().hex
_submit_case_block(
True, mother_case_id, user_id='user1', owner_id='owner1', case_type='mother',
case_name='mother', date_modified=datetime.utcnow()
)
child_case_id = uuid.uuid4().hex
_submit_case_block(
True, child_case_id, user_id='user1', owner_id='owner1', case_type='child',
case_name='child', date_modified=datetime.utcnow(), index={
'mom': ('mother', mother_case_id)
}
)
case = self.casedb.get_case(child_case_id)
self.assertEqual(len(case.indices), 1)
_submit_case_block(
False, child_case_id, user_id='user1', date_modified=datetime.utcnow(), index={
'mom': ('mother', '')
}
)
case = self.casedb.get_case(child_case_id)
self.assertEqual(len(case.indices), 0)
def test_update_case(self):
case_id = uuid.uuid4().hex
opened_on = datetime.utcnow()
_submit_case_block(
True, case_id, user_id='user1', owner_id='owner1', case_type='demo',
case_name='create_case', date_modified=opened_on, update={
'dynamic': '123'
}
)
modified_on = datetime.utcnow()
_submit_case_block(
False, case_id, user_id='user2', owner_id='owner2',
case_name='update_case', date_modified=modified_on, date_opened=opened_on, update={
'dynamic': '1234'
}
)
case = self.casedb.get_case(case_id)
self.assertEqual(case.owner_id, 'owner2')
self.assertEqual(case.name, 'update_case')
self.assertEqual(coerce_to_datetime(case.opened_on), coerce_to_datetime(opened_on))
self.assertEqual(case.opened_by, 'user1')
self.assertEqual(case.modified_on, modified_on)
self.assertEqual(case.modified_by, 'user2')
self.assertTrue(case.server_modified_on > modified_on)
self.assertFalse(case.closed)
self.assertIsNone(case.closed_on)
self.assertEqual(case.dynamic_case_properties()['dynamic'], '1234')
def download_and_commit_uids(self, crispin_client, uids):
start = datetime.utcnow()
raw_messages = crispin_client.uids(uids)
if not raw_messages:
return
new_uids = set()
with self.syncmanager_lock:
with session_scope() as db_session:
account = Account.get(self.account_id, db_session)
folder = Folder.get(self.folder_id, db_session)
raw_messages = self.__deduplicate_message_object_creation(
db_session, raw_messages, account)
if not raw_messages:
return 0
for msg in raw_messages:
uid = self.create_message(db_session, account, folder,
msg)
if uid is not None:
db_session.add(uid)
db_session.commit()
new_uids.add(uid)
log.info('Committed new UIDs',
new_committed_message_count=len(new_uids))
# If we downloaded uids, record message velocity (#uid / latency)
if self.state == "initial" and len(new_uids):
self._report_message_velocity(datetime.utcnow() - start,
len(new_uids))
if self.is_first_message:
self._report_first_message()
self.is_first_message = False
self.saved_uids.update(new_uids)
def test_update_index(self):
mother_case_id = uuid.uuid4().hex
_submit_case_block(
True, mother_case_id, user_id='user1', owner_id='owner1', case_type='mother',
case_name='mother', date_modified=datetime.utcnow()
)
child_case_id = uuid.uuid4().hex
_submit_case_block(
True, child_case_id, user_id='user1', owner_id='owner1', case_type='child',
case_name='child', date_modified=datetime.utcnow(), index={
'mom': ('mother', mother_case_id)
}
)
case = self.casedb.get_case(child_case_id)
self.assertEqual(case.indices[0].identifier, 'mom')
_submit_case_block(
False, child_case_id, user_id='user1', date_modified=datetime.utcnow(), index={
'mom': ('other_mother', mother_case_id)
}
)
case = self.casedb.get_case(child_case_id)
self.assertEqual(case.indices[0].referenced_type, 'other_mother')
def attach_data(self, data, original_filename=None, username=None, attachment_id=None,
media_meta=None):
"""
This creates the auxmedia attachment with the downloaded data.
"""
self.last_modified = datetime.utcnow()
if not attachment_id:
attachment_id = self.file_hash
if not self._attachments or attachment_id not in self._attachments:
if not getattr(self, '_id'):
# put attchment blows away existing data, so make sure an id has been
# assigned to this guy before we do it. this is the expected path
self.save()
else:
# this should only be files that had attachments deleted while the bug
# was in effect, so hopefully we will stop seeing it after a few days
logging.error('someone is uploading a file that should have existed for multimedia %s' % self._id)
self.put_attachment(data, attachment_id, content_type=self.get_mime_type(data, filename=original_filename))
new_media = AuxMedia()
new_media.uploaded_date = datetime.utcnow()
new_media.attachment_id = attachment_id
new_media.uploaded_filename = original_filename
new_media.uploaded_by = username
new_media.checksum = self.file_hash
if media_meta:
new_media.media_meta = media_meta
self.aux_media.append(new_media)
self.save()
return True
def weblate_context(request):
'''
Context processor to inject various useful variables into context.
'''
return {
'version': weblate.VERSION,
'weblate_url': URL_BASE % weblate.VERSION,
'donate_url': URL_DONATE % weblate.VERSION,
'site_title': appsettings.SITE_TITLE,
'site_url': get_site_url(),
'offer_hosting': appsettings.OFFER_HOSTING,
'demo_server': appsettings.DEMO_SERVER,
'enable_avatars': appsettings.ENABLE_AVATARS,
'current_date': datetime.utcnow().strftime('%Y-%m-%d'),
'current_year': datetime.utcnow().strftime('%Y'),
'current_month': datetime.utcnow().strftime('%m'),
'current_url': request.get_full_path(),
'mt_enabled': appsettings.MACHINE_TRANSLATION_ENABLED,
'hooks_enabled': appsettings.ENABLE_HOOKS,
'registration_open': appsettings.REGISTRATION_OPEN,
}
def test_serialize(self):
"""Objects are serialized to JSON-compatible objects"""
def epoch(obj):
"""Convert to JS Epoch time"""
return int(time.mktime(obj.timetuple())) * 1000
types = [('test', str, 'test'),
(pd.Timestamp('2013-06-08'), int,
epoch(pd.Timestamp('2013-06-08'))),
(datetime.utcnow(), int, epoch(datetime.utcnow())),
(1, int, 1),
(1.0, float, 1.0),
(np.float32(1), float, 1.0),
(np.int32(1), int, 1),
(np.float64(1), float, 1.0),
(np.int64(1), int, 1)]
for puts, pytype, gets in types:
nt.assert_equal(Data.serialize(puts), gets)
class BadType(object):
"""Bad object for type warning"""
test_obj = BadType()
with nt.assert_raises(LoadError) as err:
Data.serialize(test_obj)
nt.assert_equals(err.exception.message,
'cannot serialize index of type BadType')
def get_user_rank(user, subreddit):
"""Returns the user's rank in the subreddit."""
sr_name = subreddit.display_name.lower()
# fetch mod/contrib lists if necessary
cached = False
if sr_name in get_user_rank.moderator_cache:
cache_age = datetime.utcnow() - get_user_rank.cache_time[sr_name]
if cache_age < timedelta(hours=1):
cached = True
if not cached:
get_user_rank.cache_time[sr_name] = datetime.utcnow()
mod_list = set()
for mod in subreddit.get_moderators():
mod_list.add(mod.name)
get_user_rank.moderator_cache[sr_name] = mod_list
contrib_list = set()
try:
for contrib in subreddit.get_contributors():
contrib_list.add(contrib.name)
except HTTPError as e:
if e.response.status_code != 404:
raise
get_user_rank.contributor_cache[sr_name] = contrib_list
if user.name in get_user_rank.moderator_cache[sr_name]:
return 'moderator'
elif user.name in get_user_rank.contributor_cache[sr_name]:
return 'contributor'
else:
return 'user'
def getAccessToken(self):
poyntTokenUrl = self.apiHost + "/token"
currentDatetime = datetime.utcnow()
expiryDatetime = datetime.utcnow() + timedelta(seconds=300)
payload = {
'exp': expiryDatetime,
'iat': currentDatetime,
'iss': self.applicationId,
'sub': self.applicationId,
'aud': 'https://services.poynt.net',
'jti': str(uuid.uuid4())
}
encodedJWT = jwt.encode(payload, self.rsaPrivateKey, algorithm='RS256')
#print encodedJWT
payload = {'grantType':'urn:ietf:params:oauth:grant-type:jwt-bearer', 'assertion':encodedJWT}
print "Obtaining AccessToken using self-signed JWT:"
code, jsonObj = self._sendFormPostRequest(poyntTokenUrl, payload, {})
#r = requests.post(poyntTokenUrl, data=payload, headers=headers)
#prettyPrint(r.json())
if code == requests.codes.ok:
self.accessToken = jsonObj['accessToken']
self.tokenType = jsonObj['tokenType']
self.refreshToken = jsonObj['refreshToken']
return True
else:
print "*** FAILED TO OBTAIN ACCESS TOKEN ***"
return False
def enqueue_job(self, job):
"""
Move a scheduled job to a queue. In addition, it also does puts the job
back into the scheduler if needed.
"""
self.log.debug('Pushing {0} to {1}'.format(job.id, job.origin))
interval = job.meta.get('interval', None)
repeat = job.meta.get('repeat', None)
# If job is a repeated job, decrement counter
if repeat:
job.meta['repeat'] = int(repeat) - 1
job.enqueued_at = datetime.utcnow()
job.save()
queue = self.get_queue_for_job(job)
queue.push_job_id(job.id)
self.connection.zrem(self.scheduled_jobs_key, job.id)
if interval:
# If this is a repeat job and counter has reached 0, don't repeat
if repeat is not None:
if job.meta['repeat'] == 0:
return
self.connection._zadd(self.scheduled_jobs_key,
to_unix(datetime.utcnow()) + int(interval),
job.id)
def add_history(self, source, description=None, first_seen=None, last_seen=None, active=False):
last_seen = last_seen or datetime.utcnow()
first_seen = first_seen or datetime.utcnow()
# Do we have to extend current active record ?
if active:
active_history = self.get_active(description)
if active_history and last_seen > active_history.last_seen:
active_history.last_seen = last_seen
self.save(validate=False)
return self
# Do we have to extend an inactive record ?
else:
index, overlapping_history = self._get_overlapping(description, first_seen, last_seen)
if overlapping_history:
if source not in overlapping_history.sources:
overlapping_history.sources.append(source)
overlapping_history.first_seen = min(overlapping_history.first_seen, first_seen)
overlapping_history.last_seen = max(overlapping_history.last_seen, last_seen)
self.save(validate=False)
return self
# Otherwise, just create a new record
return self.modify(
push__history=LinkHistory(
description=description,
first_seen=first_seen or datetime.utcnow(),
last_seen=last_seen or datetime.utcnow(),
active=active,
sources=[source]))
def testSimpleIntervals(self):
td_days = timedelta(days=7)
time_start = datetime.utcnow() - timedelta(days=365)
time_end = datetime.utcnow()
tree = IntervalNode(time_start.toordinal(), time_end.toordinal())
int_end = time_start + td_days
int_start = time_start
counter = 0
while int_end < time_end:
tree.insert(int_start.toordinal(), int_end.toordinal(), other="week %d" % counter)
int_start = int_end
int_end = int_end+td_days
# print "inserting week: %d" % counter
counter = counter + 1
td_hours = timedelta(hours=4)
start_check = time_start
check_time = time_start
day_count = 0
hour_count = 0
self.node_hits = 0
while check_time < time_end:
def report_schedule(node):
if node.other is not None:
self.node_hits += 1
tree.intersect(check_time.toordinal(), check_time.toordinal(), report_schedule)
check_time = check_time + td_hours
if check_time > start_check+td_days:
#print "node hits per interval: %d" % self.node_hits
self.node_hits = 0
start_check = check_time
def get_next_job(self, suites, components, arches, checks):
NAMESPACE.machine.last_ping = datetime.utcnow()
if self.__class__.shutdown_request:
return None
arches = [x for x in arches if x not in ["source", "all"]]
job = NAMESPACE.session.query(Job).join(Job.source).join(Source.group_suite).filter(
~Job.depedencies.any(),
Job.dose_report == None,
Job.assigned_at == None,
Job.finished_at == None,
Job.failed.is_(None),
GroupSuite.suite.has(Suite.name.in_(suites)),
Source.component.has(Component.name.in_(components)),
(Job.arch.has(Arch.name.in_(arches)) |
(Job.arch.has(Arch.name.in_(["source", "all"])) &
Source.affinity.has(Arch.name.in_(arches)))),
Job.check.has(Check.name.in_(checks)),
).order_by(
Job.assigned_count.asc(),
Source.uploaded_at.asc(),
).first()
if job is None:
return None
job.assigned_count += 1
job.assigned_at = datetime.utcnow()
job.builder = NAMESPACE.machine
emit('start', 'job', job.debilize())
return job.debilize()
def start(self):
""" Begins the job by kicking off all tasks with no dependencies. """
if not self.state.allow_start:
raise DagobahError('job cannot be started in its current state; ' +
'it is probably already running')
self.snapshot = deepcopy(self.graph)
is_valid, reason = self.validate(self.snapshot)
if not is_valid:
raise DagobahError(reason)
# don't increment if the job was run manually
if self.cron_iter and datetime.utcnow() > self.next_run:
self.next_run = self.cron_iter.get_next(datetime)
self.run_log = {'job_id': self.job_id,
'name': self.name,
'parent_id': self.parent.dagobah_id,
'log_id': self.backend.get_new_log_id(),
'start_time': datetime.utcnow(),
'tasks': {}}
self._set_status('running')
for task in self.tasks.itervalues():
task.reset()
for task_name in self.ind_nodes(self.snapshot):
self._put_task_in_run_log(task_name)
self.tasks[task_name].start()
self._commit_run_log()
def bulk_comment_insert(Session, commentdata, thread_id):
"""
Bulk insert of multiple comments into the database for a given thread id (e.g. 117 not ew38xn)
:param Session: sql alchemy session
:param commentdata: list of comment dictionaries
:param thread_id: thread id (e.g. 117 not ew38xn)
:return:
"""
# init db session
session = Session()
# iterate through all comments and add them to the session
for comment in commentdata:
newcomment = Comment(
link_id=thread_id,
name=comment['name'],
parent_id=comment['parent_id'],
score=comment['score'],
created_utc=comment['created_utc'],
author=comment['author'],
body=comment['body'],
body_html=comment['body_html'],
lastchecked=datetime.utcnow(),
lastmodified=datetime.utcnow()
)
# add to session and commit
session.add(newcomment)
# commit all changes
session.commit()
def get(name, filename):
url = BASE_URL + filename
allowed = robot.is_allowed(USER_AGENT, url)
delay = robot.get_crawl_delay(USER_AGENT)
if delay is None:
delay = 0.1
if allowed:
sleep(delay)
print("Downloading " + name + "...", flush=True, end="")
try:
if not allowed:
raise Exception("Not allowed to download " + url)
start = datetime.utcnow()
r = session.get(url)
duration = (datetime.utcnow() - start).total_seconds()
r.raise_for_status()
print(" " + size_fmt(len(r.text)) + " in " + time_fmt(duration) + " (" + size_fmt(len(r.text) / duration) + "/s)")
except Exception as e:
print(" " + str(e))
raise
if r.headers["Content-Type"] == "text/plain":
r.encoding = "UTF-8"
return r.text
def create_or_login(resp):
session['openid'] = resp.identity_url
user_model = User()
user = user_model.find({'email' : resp.email})[0]
if user is None:
user_data = structure = {
"nick" : resp.nickname or resp.fullname,
"fullname" : resp.fullname,
"email" : resp.email,
"favourites" : [],
"roles" : [],
"reputation" : 1,
"logins" : [{
'ip' : request.remote_addr,
'date_loggedin' : datetime.utcnow().isoformat(),
}],
"about" : "",
"language" : resp.language,
"timezone" : resp.timezone
}
user_data["_id"] = user_model.insert(user_data)
session["userid"] = str(user_data["_id"])
app.logger.log(json.dumps(session))
else:
user_model.update({"_id" : user["_id"]}, {"$push" : {"logins" : {
"ip" : request.remote_addr,
"date_loggedin" : datetime.utcnow().isoformat(),
}}})
session["userid"] = str(user["_id"])
return redirect(url_for("bp_word.home"))
def run(self, **kwargs):
logger.info("TASK :: event_dispatcher")
# List all the events where event.start > NOW() - 12 hours and status = EVENT_STATUS.PENDING
start_from = datetime.utcnow().replace(tzinfo=utc) - timedelta(hours=12)
start_to = datetime.utcnow().replace(tzinfo=utc)
event_list = Event.objects.filter(start__gte=start_from, start__lte=start_to, status=EVENT_STATUS.PENDING)
for obj_event in event_list:
try:
# Get and perform alarm
obj_alarm = Alarm.objects.get(event=obj_event)
perform_alarm.delay(obj_event, obj_alarm)
except ObjectDoesNotExist:
pass
# Check if need to create a sub event in the future
next_occurrence = obj_event.get_next_occurrence()
print "next_occurrence"
print next_occurrence
if next_occurrence:
# The result of get_next_occurrences help to create the next event
new_event = obj_event.copy_event(next_occurrence)
# Copy the alarm link to the event
alarm_list = Alarm.objects.filter(event=obj_event)
for obj_alarm in alarm_list:
obj_alarm.copy_alarm(new_event)
# Mark the event as COMPLETED
obj_event.status = EVENT_STATUS.COMPLETED
obj_event.save()
def home(request):
"""This is home page of the site."""
username = None
language_code = request.LANGUAGE_CODE
# we need to get testimonials in the users language
all_testimonials = Testimonial.objects.filter(language=language_code)
context = {
'testimonial': choice(all_testimonials),
}
# we also need to get problems counter
counter = ProblemCounter.objects.get(pk=1)
# lets see if we need to increase counter
if counter.next_update > datetime.utcnow().replace(tzinfo=utc):
# not yet
context.update({
'counter': counter.count,
})
else:
# time to increase counter and set new time
new_counter = counter.count + 1
new_update_time = datetime.utcnow().replace(tzinfo=utc) + timedelta(hours=choice(range(4, 16)))
counter.count = new_counter
counter.next_update = new_update_time
counter.save()
context.update({
'counter': new_counter,
})
return render(request, 'mainsite/index.html', context)
start = start_ts.strftime('%Y-%m-%dT%H:%M:%SZ')
gap = _format_interval(interval)
if start_was_none:
start_ts = _clamp_date(interval, start_ts)
start = start_ts.strftime('%Y-%m-%dT%H:%M:%SZ')
stats_max = end
stats_min = start
else:
start = start_ts.strftime('%Y-%m-%dT%H:%M:%SZ')
elif stat_facet['max'] == 'NOW':
is_date = True
domain_ms = _get_interval_duration(stat_facet['min'])
interval = _get_interval(domain_ms, SLOTS)
nb_slot = domain_ms / interval['ms']
gap = _format_interval(interval)
end_ts = datetime.utcnow()
end_ts_clamped = _clamp_date(interval, end_ts)
end_ts = _get_next_interval(interval, end_ts_clamped, end_ts_clamped != end_ts)
start_ts = _remove_duration(interval, nb_slot, end_ts)
stats_max = end = end_ts.strftime('%Y-%m-%dT%H:%M:%SZ')
stats_min = start = start_ts.strftime('%Y-%m-%dT%H:%M:%SZ')
properties.update({
'min': stats_min,
'max': stats_max,
'start': start,
'end': end,
'gap': gap,
'slot': SLOTS,
'canRange': True,
'isDate': is_date,
def send(self, request, **kwargs):
"""
Send a given PreparedRequest.
:rtype: requests.Response
"""
# Set defaults that the hooks can utilize to ensure they always have
# the correct parameters to reproduce the previous request.
kwargs.setdefault('stream', self.stream)
kwargs.setdefault('verify', self.verify)
kwargs.setdefault('cert', self.cert)
kwargs.setdefault('proxies', self.proxies)
# It's possible that users might accidentally send a Request object.
# Guard against that specific failure case.
if isinstance(request, Request):
raise ValueError('You can only send PreparedRequests.')
# Set up variables needed for resolve_redirects and dispatching of hooks
allow_redirects = kwargs.pop('allow_redirects', True)
stream = kwargs.get('stream')
hooks = request.hooks
# Resolve URL in redirect cache, if available.
if allow_redirects:
checked_urls = set()
while request.url in self.redirect_cache:
checked_urls.add(request.url)
new_url = self.redirect_cache.get(request.url)
if new_url in checked_urls:
break
request.url = new_url
# Get the appropriate adapter to use
adapter = self.get_adapter(url=request.url)
# Start time (approximately) of the request
start = datetime.utcnow()
# Send the request
r = adapter.send(request, **kwargs)
# Total elapsed time of the request (approximately)
r.elapsed = datetime.utcnow() - start
# Response manipulation hooks
r = dispatch_hook('response', hooks, r, **kwargs)
# Persist cookies
if r.history:
# If the hooks create history then we want those cookies too
for resp in r.history:
extract_cookies_to_jar(self.cookies, resp.request, resp.raw)
extract_cookies_to_jar(self.cookies, request, r.raw)
# Redirect resolving generator.
gen = self.resolve_redirects(r, request, **kwargs)
# Resolve redirects if allowed.
history = [resp for resp in gen] if allow_redirects else []
# Shuffle things around if there's history.
if history:
# Insert the first (original) request at the start
history.insert(0, r)
# Get the last request made
r = history.pop()
r.history = history
if not stream:
r.content
return r
from falcon import testing
import pytest
import json
from datetime import datetime, timedelta
import sys, os
myPath = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, myPath + '/../')
from app import api
JWT_SECRET = 'secret'
JWT_ALGORITHM = 'HS256'
JWT_EXP_DELTA_SECONDS = 86400
token = jwt.encode(dict(
user_id = int(1),
exp = datetime.utcnow() + timedelta(seconds=int(JWT_EXP_DELTA_SECONDS))
), JWT_SECRET, algorithm = JWT_ALGORITHM).decode('UTF-8')
customer1 = {
'id': 1,
'name': 'Ronaldo',
'dob': '1991-01-08'
}
@pytest.fixture()
def client():
# Assume the hypothetical `myapp` package has a function called
# `create()` to initialize and return a `falcon.API` instance.
return testing.TestClient(api)
def _update_subscription(self, subscription):
""" Update subscription settings """
db = current.db
s3db = current.s3db
pe_id = subscription["pe_id"]
# Save filters
filter_id = subscription["filter_id"]
filters = subscription.get("filters")
if filters:
ftable = s3db.pr_filter
if not filter_id:
success = ftable.insert(pe_id=pe_id, query=filters)
filter_id = success
else:
success = db(ftable.id == filter_id).update(query=filters)
if not success:
return None
# Save subscription settings
stable = s3db.pr_subscription
subscription_id = subscription["id"]
frequency = subscription["frequency"]
if not subscription_id:
success = stable.insert(pe_id=pe_id,
filter_id=filter_id,
notify_on=subscription["notify_on"],
frequency=frequency,
method=subscription["method"])
subscription_id = success
else:
success = db(stable.id == subscription_id).update(
pe_id=pe_id,
filter_id=filter_id,
notify_on=subscription["notify_on"],
frequency=frequency,
method=subscription["method"])
if not success:
return None
# Save subscriptions
rtable = s3db.pr_subscription_resource
subscribe = subscription.get("subscribe")
if subscribe:
from datetime import datetime, timedelta
now = datetime.utcnow()
resources = subscription["resources"]
subscribed = {}
timestamps = {}
if resources:
for r in resources:
subscribed[(r.resource, r.url)] = r.id
timestamps[r.id] = (r.last_check_time, r.next_check_time)
intervals = s3db.pr_subscription_check_intervals
interval = timedelta(minutes=intervals.get(frequency, 0))
keep = set()
fk = '''{"subscription_id": %s}''' % subscription_id
for new in subscribe:
resource, url = new["resource"], new["url"]
if (resource, url) not in subscribed:
# Restore subscription if previously unsubscribed, else
# insert new record
unsubscribed = {
"deleted": True,
"deleted_fk": fk,
"resource": resource,
"url": url
}
rtable.update_or_insert(_key=unsubscribed,
deleted=False,
deleted_fk=None,
subscription_id=subscription_id,
resource=resource,
url=url,
last_check_time=now,
next_check_time=None)
else:
# Keep it
record_id = subscribed[(resource, url)]
last_check_time, next_check_time = timestamps[record_id]
data = {}
if not last_check_time:
# Someone has tampered with the timestamps, so
# we need to reset them and start over
last_check_time = now
data["last_check_time"] = last_check_time
due = last_check_time + interval
if next_check_time != due:
# Time interval has changed
data["next_check_time"] = due
if data:
db(rtable.id == record_id).update(**data)
keep.add(record_id)
# Unsubscribe all others
unsubscribe = set(subscribed.values()) - keep
db(rtable.id.belongs(unsubscribe)).update(deleted=True,
deleted_fk=fk,
subscription_id=None)
# Update subscription
subscription["id"] = subscription_id
subscription["filter_id"] = filter_id
return subscription
def get_utcnow():
"""
Used to make it easier to mock utcnow() in the tests.
"""
return datetime.utcnow()
def antennaS11(argv):
"""
Acquire OSL and antenna traces through a receiver to produce
a calibrated antenna S11 measurement.
"""
# Set default parameter values
result = True
parser = argparse.ArgumentParser(description= 'Acquire OSL and antenna traces through the receiver to produce a calibrated antenna S11 measurement.')
parser.add_argument('folder', help='Output folder to save traces.')
parser.add_argument('-m', '--message', nargs=1, help='Store a comment with the measurement')
parser.add_argument('-e', '--external', help='Apply correction from lab measurement of external OSL. Specify calibration folder')
parser.add_argument('-f', '--freq', nargs='?', default=75e6, type=float, help='Frequency (Hz) for normalizing the external cal fits')
parser.add_argument('-n', '--npoly', nargs='?', default=10, type=int, help='Number of polynomial terms in external cal fits')
parser.add_argument('-p', '--plot', action='store_true', help='Plot the antenna S11')
parser.add_argument('-r', '--revisit', action='store_true', help='Use existing data in specified folder rather than acquire new traces')
args = parser.parse_args()
print args
# ---------------------------------------------------------------------------
# Get the EDGES configuration
# ---------------------------------------------------------------------------
# Get an instance of the EDGES class
d = edges.edges()
# Read the environment sensors
values = d.getEnvironmentSensors()
# Read the sensor labels from the EDGES settings
labels = ( d.settings.get('Environment', 'sensor1_label'),
d.settings.get('Environment', 'sensor2_label'),
d.settings.get('Environment', 'sensor3_label'),
'Rack Temperature' )
# Read the sensor units from the EDGES settings
units = ( d.settings.get('Environment', 'sensor1_unit'),
d.settings.get('Environment', 'sensor2_unit'),
d.settings.get('Environment', 'sensor3_unit'),
'K' )
# Read the installation settings
site = d.settings.get('Installation', 'site')
instrument = d.settings.get('Installation', 'instrument')
# Set the subdir from the specified folder
subdir = '{}/{}/s11/{}'.format(site, instrument, args.folder)
gammas = None
print('')
if args.revisit:
# -------------------------------------------------------------------------
# Load existing traces for additional work
# -------------------------------------------------------------------------
# Look for trace files in the folder
files = sorted(glob.glob('{}/{}/*_input1.s1p'.format(d.getDataDir(), subdir)))
if len(files) > 1:
print('Found multiple trace sets, using first set\n')
# Get the timestamp
timestamp = d.parseFileDateString(os.path.split(files[0])[1][:-11])
print('Using existing traces from: {}\n'.format(timestamp))
# Get a set of trace files
inputfiles = []
inputfiles.append(files[0])
inputfiles.append('{}_input2.s1p'.format(files[0][:-11]))
inputfiles.append('{}_input3.s1p'.format(files[0][:-11]))
inputfiles.append('{}_input4.s1p'.format(files[0][:-11]))
# Load the trace files
for index, name in enumerate(inputfiles):
r, f = vna.s1p_read(name)
if gammas == None:
gammas = np.zeros((len(f),4), dtype=np.complex128)
gammas[:,index] = r
print('Successfully loaded traces.\n')
else:
# -------------------------------------------------------------------------
# Connect to VNA and acquire new traces
# -------------------------------------------------------------------------
# Get the current time
timestamp = datetime.utcnow()
print '\nTimestamp: {}'.format(timestamp)
print('\nReading and writing environmental sensor data...\n')
# Print the labels and values to stdout
for l, v, u in zip(labels, values, units):
print("{}: {:.2f} {}".format(l.rjust(25), v, u))
# Concatenate label text and units
fullLabels = ["{} [{}]".format(l, u) for l, u in zip(labels, units)]
# Write to EDGES file record
result = d.writeRecord(subdir, 'sensors', timestamp, values, fullLabels, breaklevel=5)
print("")
# Start the VNA measurements
session = d.startVNASession()
# Loop over each OSL+Antenna input
inputs = {1, 2, 3, 4}
for i in inputs:
print('\nAcquiring input {}...'.format(i))
# Acquire the trace and save it to the subdir
data = d.getVNATrace(i, session)
# Write to file
output_path = d.getFullPath(subdir, 'input{}.s1p'.format(i), timestamp, level=5)
with open (output_path, "w") as outfile:
outfile.write(data)
# Add to our array of traces
r, f = vna.s1p_read_from_text(data)
if gammas == None:
gammas = np.zeros((len(f),4), dtype=np.complex128)
gammas[:,i-1] = r
# Close the VNA session
d.stopVNASession(session)
## end if args.revisit
# write comment if there is one
if args.message:
result = d.writeText(subdir, 'message', timestamp, args.message[0], 'Message', breaklevel=5)
print('\nMessage: ' + args.message[0] + '\n')
if gammas == None:
return False
# ---------------------------------------------------------------------------
# Apply internal OSL calibration
# ---------------------------------------------------------------------------
print('Applying internal OSL correction...\n')
# Correct the antenna S11 using the OSL inputs
gammas_internal_corrected = vna.internal_s11_correction(gammas[:,0], gammas[:,1], gammas[:,2], gammas[:,3])
# Save internally calibrated antenna S11
output_path = d.getFullPath(subdir, 'antenna_s11.s1p', timestamp, level=5)
vna.s1p_write(output_path, f, gammas_internal_corrected, 0)
# ---------------------------------------------------------------------------
# Apply external OSL calibration (optional)
# ---------------------------------------------------------------------------
if args.external != None:
print('Applying (optional) external OSL correction...\n')
bExternalCorrection = True
folder_calib = args.external
subdir_calib = '{}/{}/s11_calibration/{}'.format(site, instrument, folder_calib)
# The usual internal OSL standards measured at time of external calibration
int_open, f_int = vna.s1p_read('{}/{}/internal_open.s1p'.format(d.getDataDir(), subdir_calib))
int_short, f_int = vna.s1p_read('{}/{}/internal_short.s1p'.format(d.getDataDir(), subdir_calib))
int_load, f_int = vna.s1p_read('{}/{}/internal_load.s1p'.format(d.getDataDir(), subdir_calib))
# External sources measured through input 4
ext_open, f_ext = vna.s1p_read('{}/{}/external_open.s1p'.format(d.getDataDir(), subdir_calib))
ext_short, f_ext = vna.s1p_read('{}/{}/external_short.s1p'.format(d.getDataDir(), subdir_calib))
ext_load, f_ext = vna.s1p_read('{}/{}/external_load.s1p'.format(d.getDataDir(), subdir_calib))
resistance_of_match = np.genfromtxt('{}/{}/resistance_of_match.txt'.format(d.getDataDir(), subdir_calib), comments='#')
# Apply the correction
gammas_external_corrected = vna.external_s11_correction(
f, gammas_internal_corrected,
f_ext, int_open, int_short, int_load,
ext_open, ext_short, ext_load,
resistance_of_match,
f_norm=args.freq, N_poly_terms=args.npoly)[0]
# Save externally calibrated antenna S11
output_path = d.getFullPath(subdir, 'calibrated_{}_antenna_s11.s1p'.format(folder_calib), timestamp, level=5)
vna.s1p_write(output_path, f, gammas_external_corrected, 0)
# ---------------------------------------------------------------------------
# Make output plot and save to PNG
# ---------------------------------------------------------------------------
if args.plot:
# Plot
f1 = plt.figure(num=1, figsize=(14, 6))
plt.subplot(1,2,1)
plt.plot(f/1e6, 20 * np.log10(np.abs(gammas_internal_corrected)))
if args.external != None:
plt.plot(f/1e6, 20 * np.log10(np.abs(gammas_external_corrected)))
ax = plt.gca()
ax.set_yticks(np.arange(-20,0,1))
plt.xlim([40, 200])
plt.ylim([-20, 0])
plt.grid()
plt.xlabel('frequency [MHz]')
plt.ylabel('magnitude [dB]')
plt.subplot(1,2,2)
plt.plot(f/1e6, 180/np.pi * np.unwrap(np.angle(gammas_internal_corrected)))
if args.external != None:
plt.plot(f/1e6, 180/np.pi * np.unwrap(np.angle(gammas_external_corrected)))
plt.xlim([40, 200])
plt.ylim([-800, 20])
plt.grid()
plt.xlabel('frequency [MHz]')
plt.ylabel('phase [deg]')
# Save plot
output_plot = d.getFullPath(subdir, 'antenna_s11.png', timestamp, level=5)
plt.savefig(output_plot, bbox_inches='tight')
plt.close(f1)
# Display plot
eog_command = 'eog {} &'.format(output_plot)
subprocess.call(eog_command, shell = True)
if result:
print('Success.\n')
else:
print('Failed.\n')
return result
def ingest(objectid,
dsets_file,
grq_update_url,
dataset_processed_queue,
prod_path,
job_path,
dry_run=False,
force=False):
"""Run dataset ingest."""
logger.info("#" * 80)
logger.info("datasets: %s" % dsets_file)
logger.info("grq_update_url: %s" % grq_update_url)
logger.info("dataset_processed_queue: %s" % dataset_processed_queue)
logger.info("prod_path: %s" % prod_path)
logger.info("job_path: %s" % job_path)
logger.info("dry_run: %s" % dry_run)
logger.info("force: %s" % force)
# get dataset
if os.path.isdir(prod_path):
local_prod_path = prod_path
else:
local_prod_path = get_remote_dav(prod_path)
if not os.path.isdir(local_prod_path):
raise RuntimeError("Failed to find local dataset directory: %s" %
local_prod_path)
# dataset name
pname = os.path.basename(local_prod_path)
# dataset file
dataset_file = os.path.join(local_prod_path, '%s.dataset.json' % pname)
# get dataset json
with open(dataset_file) as f:
dataset = json.load(f)
logger.info("Loaded dataset JSON from file: %s" % dataset_file)
# check minimum requirements for dataset JSON
logger.info("Verifying dataset JSON...")
verify_dataset(dataset)
logger.info("Dataset JSON verfication succeeded.")
# get version
version = dataset['version']
# recognize
r = Recognizer(dsets_file, local_prod_path, objectid, version)
# get ipath
ipath = r.currentIpath
# get extractor
extractor = r.getMetadataExtractor()
if extractor is not None:
match = SCRIPT_RE.search(extractor)
if match: extractor = match.group(1)
logger.info("Configured metadata extractor: %s" % extractor)
# metadata file
metadata_file = os.path.join(local_prod_path, '%s.met.json' % pname)
# metadata seed file
seed_file = os.path.join(local_prod_path, 'met.json')
# metadata file already here
if os.path.exists(metadata_file):
with open(metadata_file) as f:
metadata = json.load(f)
logger.info("Loaded metadata from existing file: %s" % metadata_file)
else:
if extractor is None:
logger.info(
"No metadata extraction configured. Setting empty metadata.")
metadata = {}
else:
logger.info("Running metadata extractor %s on %s" %
(extractor, local_prod_path))
m = check_output([extractor, local_prod_path])
logger.info("Output: %s" % m)
# generate json to update metadata and urls
metadata = json.loads(m)
# set data_product_name
metadata['data_product_name'] = objectid
# merge with seed metadata
if os.path.exists(seed_file):
with open(seed_file) as f:
seed = json.load(f)
metadata.update(seed)
logger.info("Loaded seed metadata from file: %s" % seed_file)
# write it out to file
with open(metadata_file, 'w') as f:
json.dump(metadata, f, indent=2)
logger.info("Wrote metadata to %s" % metadata_file)
# delete seed file
if os.path.exists(seed_file):
os.unlink(seed_file)
logger.info("Deleted seed file %s." % seed_file)
# add context
context_file = os.path.join(local_prod_path, '%s.context.json' % pname)
if os.path.exists(context_file):
with open(context_file) as f:
context = json.load(f)
logger.info("Loaded context from existing file: %s" % context_file)
else:
context = {}
metadata['context'] = context
# set metadata and dataset groups in recognizer
r.setDataset(dataset)
r.setMetadata(metadata)
# get level
level = r.getLevel()
# get type
dtype = r.getType()
# get publish path
pub_path_url = r.getPublishPath()
# get publish urls
pub_urls = [i for i in r.getPublishUrls()]
# get S3 profile name and api keys for dataset publishing
s3_secret_key, s3_access_key = r.getS3Keys()
s3_profile = r.getS3Profile()
# set osaka params
osaka_params = {}
# S3 profile takes precedence over explicit api keys
if s3_profile is not None:
osaka_params['profile_name'] = s3_profile
else:
if s3_secret_key is not None and s3_access_key is not None:
osaka_params['aws_access_key_id'] = s3_access_key
osaka_params['aws_secret_access_key'] = s3_secret_key
# get browse path and urls
browse_path = r.getBrowsePath()
browse_urls = r.getBrowseUrls()
# get S3 profile name and api keys for browse image publishing
s3_secret_key_browse, s3_access_key_browse = r.getS3Keys("browse")
s3_profile_browse = r.getS3Profile("browse")
# set osaka params for browse
osaka_params_browse = {}
# S3 profile takes precedence over explicit api keys
if s3_profile_browse is not None:
osaka_params_browse['profile_name'] = s3_profile_browse
else:
if s3_secret_key_browse is not None and s3_access_key_browse is not None:
osaka_params_browse['aws_access_key_id'] = s3_access_key_browse
osaka_params_browse['aws_secret_access_key'] = s3_secret_key_browse
# get pub host and path
logger.info("Configured pub host & path: %s" % (pub_path_url))
# check scheme
if not osaka.main.supported(pub_path_url):
raise RuntimeError("Scheme %s is currently not supported." %
urlparse(pub_path_url).scheme)
# upload dataset to repo; track disk usage and start/end times of transfer
prod_dir_usage = get_disk_usage(local_prod_path)
tx_t1 = datetime.utcnow()
if dry_run:
logger.info("Would've published %s to %s" %
(local_prod_path, pub_path_url))
else:
publish_dataset(local_prod_path,
pub_path_url,
params=osaka_params,
force=force)
tx_t2 = datetime.utcnow()
# add metadata for all browse images and upload to browse location
imgs_metadata = []
imgs = glob('%s/*browse.png' % local_prod_path)
for img in imgs:
img_metadata = {'img': os.path.basename(img)}
small_img = img.replace('browse.png', 'browse_small.png')
if os.path.exists(small_img):
small_img_basename = os.path.basename(small_img)
if browse_path is not None:
this_browse_path = os.path.join(browse_path,
small_img_basename)
if dry_run:
logger.info("Would've uploaded %s to %s" %
(small_img, browse_path))
else:
logger.info("Uploading %s to %s" %
(small_img, browse_path))
osaka.main.put(small_img,
this_browse_path,
params=osaka_params_browse,
noclobber=False)
else:
small_img_basename = None
img_metadata['small_img'] = small_img_basename
tooltip_match = BROWSE_RE.search(img_metadata['img'])
if tooltip_match: img_metadata['tooltip'] = tooltip_match.group(1)
else: img_metadata['tooltip'] = ""
imgs_metadata.append(img_metadata)
# sort browse images
browse_sort_order = r.getBrowseSortOrder()
if isinstance(browse_sort_order,
types.ListType) and len(browse_sort_order) > 0:
bso_regexes = [re.compile(i) for i in browse_sort_order]
sorter = {}
unrecognized = []
for img in imgs_metadata:
matched = None
for i, bso_re in enumerate(bso_regexes):
if bso_re.search(img['img']):
matched = img
sorter[i] = matched
break
if matched is None: unrecognized.append(img)
imgs_metadata = [sorter[i] for i in sorted(sorter)]
imgs_metadata.extend(unrecognized)
# save dataset metrics on size and transfer
tx_dur = (tx_t2 - tx_t1).total_seconds()
prod_metrics = {
'ipath': ipath,
'url': urlparse(pub_path_url).path,
'path': local_prod_path,
'disk_usage': prod_dir_usage,
'time_start': tx_t1.isoformat() + 'Z',
'time_end': tx_t2.isoformat() + 'Z',
'duration': tx_dur,
'transfer_rate': prod_dir_usage / tx_dur
}
# set update json
ipath = r.currentIpath
update_json = {
'id': objectid,
'objectid': objectid,
'metadata': metadata,
'urls': pub_urls,
'browse_urls': browse_urls,
'images': imgs_metadata,
'dataset': ipath.split('/')[1],
'ipath': ipath,
'system_version': version,
'dataset_level': level,
'dataset_type': dtype,
}
update_json.update(dataset)
#logger.info("update_json: %s" % pformat(update_json))
# custom index specified?
index = r.getIndex()
if index is not None: update_json['index'] = index
# update GRQ
if isinstance(update_json['metadata'],
types.DictType) and len(update_json['metadata']) > 0:
#logger.info("update_json: %s" % pformat(update_json))
if dry_run:
logger.info("Would've indexed doc at %s: %s" %
(grq_update_url,
json.dumps(update_json, indent=2, sort_keys=True)))
else:
res = index_dataset(grq_update_url, update_json)
logger.info("res: %s" % res)
update_json['grq_index_result'] = res
# finish if dry run
if dry_run: return (prod_metrics, update_json)
# create PROV-ES JSON file for publish processStep
prod_prov_es_file = os.path.join(
local_prod_path, '%s.prov_es.json' % os.path.basename(local_prod_path))
pub_prov_es_bn = "publish.prov_es.json"
if os.path.exists(prod_prov_es_file):
pub_prov_es_file = os.path.join(local_prod_path, pub_prov_es_bn)
prov_es_info = {}
with open(prod_prov_es_file) as f:
try:
prov_es_info = json.load(f)
except Exception, e:
tb = traceback.format_exc()
raise (RuntimeError("Failed to load PROV-ES from %s: %s\n%s" %
(prod_prov_es_file, str(e), tb)))
log_publish_prov_es(prov_es_info, pub_prov_es_file, local_prod_path,
pub_urls, prod_metrics, objectid)
# upload publish PROV-ES file
osaka.main.put(pub_prov_es_file,
os.path.join(pub_path_url, pub_prov_es_bn),
params=osaka_params,
noclobber=False)
def _fetch(self, url, payload=None, method='GET'):
"""
Wrapper to request object
"""
# Prepare our headers:
headers = {
'User-Agent': self.app_id,
'Accept': 'application/json',
'Authorization': 'Bearer ' + self.token,
}
if payload:
# Only set our header payload if it's defined
headers['Content-Type'] = 'application/json'
# Default content response object
content = {}
# Update our URL
url = '{}/{}'.format(GITTER_API_URL, url)
# Some Debug Logging
self.logger.debug('Gitter {} URL: {} (cert_verify={})'.format(
method,
url, self.verify_certificate))
if payload:
self.logger.debug('Gitter Payload: {}' .format(payload))
# By default set wait to None
wait = None
if self.ratelimit_remaining == 0:
# Determine how long we should wait for or if we should wait at
# all. This isn't fool-proof because we can't be sure the client
# time (calling this script) is completely synced up with the
# Gitter server. One would hope we're on NTP and our clocks are
# the same allowing this to role smoothly:
now = datetime.utcnow()
if now < self.ratelimit_reset:
# We need to throttle for the difference in seconds
# We add 0.5 seconds to the end just to allow a grace
# period.
wait = (self.ratelimit_reset - now).total_seconds() + 0.5
# Always call throttle before any remote server i/o is made
self.throttle(wait=wait)
# fetch function
fn = requests.post if method == 'POST' else requests.get
try:
r = fn(
url,
data=payload,
headers=headers,
verify=self.verify_certificate,
)
if r.status_code != requests.codes.ok:
# We had a problem
status_str = \
NotifyGitter.http_response_code_lookup(r.status_code)
self.logger.warning(
'Failed to send Gitter {} to {}: '
'{}error={}.'.format(
method,
url,
', ' if status_str else '',
r.status_code))
self.logger.debug(
'Response Details:\r\n{}'.format(r.content))
# Mark our failure
return (False, content)
try:
content = loads(r.content)
except (TypeError, ValueError):
# ValueError = r.content is Unparsable
# TypeError = r.content is None
content = {}
try:
self.ratelimit_remaining = \
int(r.headers.get('X-RateLimit-Remaining'))
self.ratelimit_reset = datetime.utcfromtimestamp(
int(r.headers.get('X-RateLimit-Reset')))
except (TypeError, ValueError):
# This is returned if we could not retrieve this information
# gracefully accept this state and move on
pass
except requests.RequestException as e:
self.logger.warning(
'Exception received when sending Gitter {} to {}: '.
format(method, url))
self.logger.debug('Socket Exception: %s' % str(e))
# Mark our failure
return (False, content)
return (True, content)
def execute(chain_config, notification_enabled, relay_from_block_num,
apply_relay_from_block_num, relay_ignore_sec_threshold):
""" 未完了の入出金処理の検出
"""
logger.info(
'Relay from block: {relay_from_block_num}, '
'Apply Relay from block: {apply_relay_from_block_num}'
.format(relay_from_block_num=relay_from_block_num,
apply_relay_from_block_num=apply_relay_from_block_num))
# Web3プロバイダーの生成
provider_from = Web3.HTTPProvider(chain_config['chainRpcUrlFrom'])
provider_to = Web3.HTTPProvider(chain_config['chainRpcUrlTo'])
# Relayイベントの取得
relay_events = contract.get_relay_event_logs(
provider_from,
chain_config['bridgeContractAddressFrom'],
relay_from_block_num)
# ApplyRelayイベントの取得
apply_relay_events = contract.get_apply_relay_event_logs(
provider_to,
chain_config['bridgeContractAddressTo'],
apply_relay_from_block_num)
# applyRelayが実行済みのRelayイベントの抽出
completed_relay_tx_hashes = set()
for apply_relay_event in apply_relay_events:
parsed_apply_relay_event = contract.parse_apply_relay_event_log(
apply_relay_event)
completed_relay_tx_hashes.add(parsed_apply_relay_event['relayTxHash'])
# applyRelayが未実行のRelayイベントの抽出
pending_relays = []
for relay_event in relay_events:
parsed_relay_event = contract.parse_relay_event_log(relay_event)
# NOTICE 直近のRelayイベントはバッチで処理されるまでにタイムラグがあるため無視する
if (datetime.utcnow().timestamp() - parsed_relay_event['timestamp']) \
< relay_ignore_sec_threshold:
continue
if parsed_relay_event['txHash'] not in completed_relay_tx_hashes:
pending_relays.append(parsed_relay_event)
# 未完了の入出金処理が存在するか
if len(pending_relays) > 0:
logger.info(
'{pending_relay_count} pending relays were detected.'
.format(pending_relay_count=len(pending_relays)))
for pending_relay in pending_relays:
logger.info(
'[RelayEvent] timestamp={timestamp}, blockNum={blockNumber}, '
'txHash={txHash}, sender={sender}, recipient={recipient}, '
'amount={amount}, fee={fee}'.format(**pending_relay))
# Slackに通知
if notification_enabled:
notification.notify_pending_relays(
chain_config['isDeposit'],
pending_relays,
relay_from_block_num,
apply_relay_from_block_num)
else:
logger.info("No pending relay was detected.")
class NotifyGitter(NotifyBase):
"""
A wrapper for Gitter Notifications
"""
# The default descriptive name associated with the Notification
service_name = 'Gitter'
# The services URL
service_url = 'https://gitter.im/'
# All pushover requests are secure
secure_protocol = 'gitter'
# A URL that takes you to the setup/help of the specific protocol
setup_url = 'https://github.com/caronc/apprise/wiki/Notify_gitter'
# Allows the user to specify the NotifyImageSize object
image_size = NotifyImageSize.XY_32
# Gitter does not support a title
title_maxlen = 0
# Gitter is kind enough to return how many more requests we're allowed to
# continue to make within it's header response as:
# X-RateLimit-Reset: The epoc time (in seconds) we can expect our
# rate-limit to be reset.
# X-RateLimit-Remaining: an integer identifying how many requests we're
# still allow to make.
request_rate_per_sec = 0
# For Tracking Purposes
ratelimit_reset = datetime.utcnow()
# Default to 1
ratelimit_remaining = 1
# Default Notification Format
notify_format = NotifyFormat.MARKDOWN
# Define object templates
templates = (
'{schema}://{token}:{targets}/',
)
# Define our template tokens
template_tokens = dict(NotifyBase.template_tokens, **{
'token': {
'name': _('Token'),
'type': 'string',
'regex': (r'[a-z0-9]{40}', 'i'),
'private': True,
'required': True,
},
'targets': {
'name': _('Rooms'),
'type': 'list:string',
},
})
# Define our template arguments
template_args = dict(NotifyBase.template_args, **{
'image': {
'name': _('Include Image'),
'type': 'bool',
'default': False,
'map_to': 'include_image',
},
'to': {
'alias_of': 'targets',
},
})
def __init__(self, token, targets, include_image=False, **kwargs):
"""
Initialize Gitter Object
"""
super(NotifyGitter, self).__init__(**kwargs)
try:
# The personal access token associated with the account
self.token = token.strip()
except AttributeError:
# Token was None
msg = 'No API Token was specified.'
self.logger.warning(msg)
raise TypeError(msg)
if not VALIDATE_TOKEN.match(self.token):
msg = 'The Personal Access Token specified ({}) is invalid.' \
.format(token)
self.logger.warning(msg)
raise TypeError(msg)
# Parse our targets
self.targets = parse_list(targets)
# Used to track maping of rooms to their numeric id lookup for
# messaging
self._room_mapping = None
# Track whether or not we want to send an image with our notification
# or not.
self.include_image = include_image
def send(self, body, title='', notify_type=NotifyType.INFO, **kwargs):
"""
Perform Gitter Notification
"""
# error tracking (used for function return)
has_error = False
# Set up our image for display if configured to do so
image_url = None if not self.include_image \
else self.image_url(notify_type)
if image_url:
body = '\n{}'.format(image_url, body)
# Create a copy of the targets list
targets = list(self.targets)
if self._room_mapping is None:
# Populate our room mapping
self._room_mapping = {}
postokay, response = self._fetch(url='rooms')
if not postokay:
return False
# Response generally looks like this:
# [
# {
# noindex: False,
# oneToOne: False,
# avatarUrl: 'https://path/to/avatar/url',
# url: '/apprise-notifications/community',
# public: True,
# tags: [],
# lurk: False,
# uri: 'apprise-notifications/community',
# lastAccessTime: '2019-03-25T00:12:28.144Z',
# topic: '',
# roomMember: True,
# groupId: '5c981cecd73408ce4fbbad2f',
# githubType: 'REPO_CHANNEL',
# unreadItems: 0,
# mentions: 0,
# security: 'PUBLIC',
# userCount: 1,
# id: '5c981cecd73408ce4fbbad31',
# name: 'apprise/community'
# }
# ]
for entry in response:
self._room_mapping[entry['name'].lower().split('/')[0]] = {
# The ID of the room
'id': entry['id'],
# A descriptive name (useful for logging)
'uri': entry['uri'],
}
if len(targets) == 0:
# No targets specified
return False
while len(targets):
target = targets.pop(0).lower()
if target not in self._room_mapping:
self.logger.warning(
'Failed to locate Gitter room {}'.format(target))
# Flag our error
has_error = True
continue
# prepare our payload
payload = {
'text': body,
}
# Our Notification URL
notify_url = 'rooms/{}/chatMessages'.format(
self._room_mapping[target]['id'])
# Perform our query
postokay, response = self._fetch(
notify_url, payload=dumps(payload), method='POST')
if not postokay:
# Flag our error
has_error = True
return not has_error
def _fetch(self, url, payload=None, method='GET'):
"""
Wrapper to request object
"""
# Prepare our headers:
headers = {
'User-Agent': self.app_id,
'Accept': 'application/json',
'Authorization': 'Bearer ' + self.token,
}
if payload:
# Only set our header payload if it's defined
headers['Content-Type'] = 'application/json'
# Default content response object
content = {}
# Update our URL
url = '{}/{}'.format(GITTER_API_URL, url)
# Some Debug Logging
self.logger.debug('Gitter {} URL: {} (cert_verify={})'.format(
method,
url, self.verify_certificate))
if payload:
self.logger.debug('Gitter Payload: {}' .format(payload))
# By default set wait to None
wait = None
if self.ratelimit_remaining == 0:
# Determine how long we should wait for or if we should wait at
# all. This isn't fool-proof because we can't be sure the client
# time (calling this script) is completely synced up with the
# Gitter server. One would hope we're on NTP and our clocks are
# the same allowing this to role smoothly:
now = datetime.utcnow()
if now < self.ratelimit_reset:
# We need to throttle for the difference in seconds
# We add 0.5 seconds to the end just to allow a grace
# period.
wait = (self.ratelimit_reset - now).total_seconds() + 0.5
# Always call throttle before any remote server i/o is made
self.throttle(wait=wait)
# fetch function
fn = requests.post if method == 'POST' else requests.get
try:
r = fn(
url,
data=payload,
headers=headers,
verify=self.verify_certificate,
)
if r.status_code != requests.codes.ok:
# We had a problem
status_str = \
NotifyGitter.http_response_code_lookup(r.status_code)
self.logger.warning(
'Failed to send Gitter {} to {}: '
'{}error={}.'.format(
method,
url,
', ' if status_str else '',
r.status_code))
self.logger.debug(
'Response Details:\r\n{}'.format(r.content))
# Mark our failure
return (False, content)
try:
content = loads(r.content)
except (TypeError, ValueError):
# ValueError = r.content is Unparsable
# TypeError = r.content is None
content = {}
try:
self.ratelimit_remaining = \
int(r.headers.get('X-RateLimit-Remaining'))
self.ratelimit_reset = datetime.utcfromtimestamp(
int(r.headers.get('X-RateLimit-Reset')))
except (TypeError, ValueError):
# This is returned if we could not retrieve this information
# gracefully accept this state and move on
pass
except requests.RequestException as e:
self.logger.warning(
'Exception received when sending Gitter {} to {}: '.
format(method, url))
self.logger.debug('Socket Exception: %s' % str(e))
# Mark our failure
return (False, content)
return (True, content)
def url(self):
"""
Returns the URL built dynamically based on specified arguments.
"""
# Define any arguments set
args = {
'format': self.notify_format,
'overflow': self.overflow_mode,
'image': 'yes' if self.include_image else 'no',
'verify': 'yes' if self.verify_certificate else 'no',
}
return '{schema}://{token}/{targets}/?{args}'.format(
schema=self.secure_protocol,
token=NotifyGitter.quote(self.token, safe=''),
targets='/'.join(
[NotifyGitter.quote(x, safe='') for x in self.targets]),
args=NotifyGitter.urlencode(args))
@staticmethod
def parse_url(url):
"""
Parses the URL and returns enough arguments that can allow
us to substantiate this object.
"""
results = NotifyBase.parse_url(url)
if not results:
# We're done early as we couldn't load the results
return results
results['token'] = NotifyGitter.unquote(results['host'])
# Get our entries; split_path() looks after unquoting content for us
# by default
results['targets'] = NotifyGitter.split_path(results['fullpath'])
# Support the 'to' variable so that we can support targets this way too
# The 'to' makes it easier to use yaml configuration
if 'to' in results['qsd'] and len(results['qsd']['to']):
results['targets'] += NotifyGitter.parse_list(results['qsd']['to'])
# Include images with our message
results['include_image'] = \
parse_bool(results['qsd'].get('image', False))
return results
def ping(self):
self.last_seen = datetime.utcnow()
db.session.add(self)
def now_as_float():
return datetime_to_epoch(datetime.utcnow())
def _set_time_start(cls, deployment_history):
if not deployment_history.time_start:
deployment_history.time_start = datetime.utcnow()
def update_user(self, sailthru_vars, email, site=None, new_user=False, send_welcome_email=False):
"""
Adds/updates Sailthru profile information for a user.
Args:
sailthru_vars(dict): User profile information to pass as 'vars' to Sailthru
email(str): User email address
new_user(boolean): True if new registration
send_welcome_email(boolean): True if a welcome email should be sent
Returns:
None
"""
email_config = EmailMarketingConfiguration.current()
if not email_config.enabled:
return
sailthru_client = SailthruClient(email_config.sailthru_key, email_config.sailthru_secret)
try:
sailthru_response = sailthru_client.api_post("user",
_create_email_user_param(sailthru_vars, sailthru_client,
email, new_user, email_config,
site=site))
except SailthruClientError as exc:
log.error("Exception attempting to add/update user %s in Sailthru - %s", email, unicode(exc))
raise self.retry(exc=exc,
countdown=email_config.sailthru_retry_interval,
max_retries=email_config.sailthru_max_retries)
if not sailthru_response.is_ok():
error = sailthru_response.get_error()
log.error("Error attempting to add/update user in Sailthru: %s", error.get_message())
if _retryable_sailthru_error(error):
raise self.retry(countdown=email_config.sailthru_retry_interval,
max_retries=email_config.sailthru_max_retries)
return
if send_welcome_email and email_config.sailthru_welcome_template and is_default_site(site) and not \
sailthru_vars.get('is_enterprise_learner'):
scheduled_datetime = datetime.utcnow() + timedelta(seconds=email_config.welcome_email_send_delay)
try:
sailthru_response = sailthru_client.api_post(
"send",
{
"email": email,
"template": email_config.sailthru_welcome_template,
"schedule_time": scheduled_datetime.strftime('%Y-%m-%dT%H:%M:%SZ')
}
)
except SailthruClientError as exc:
log.error("Exception attempting to send welcome email to user %s in Sailthru - %s", email, unicode(exc))
raise self.retry(exc=exc,
countdown=email_config.sailthru_retry_interval,
max_retries=email_config.sailthru_max_retries)
if not sailthru_response.is_ok():
error = sailthru_response.get_error()
log.error("Error attempting to send welcome email to user in Sailthru: %s", error.get_message())
if _retryable_sailthru_error(error):
raise self.retry(countdown=email_config.sailthru_retry_interval,
max_retries=email_config.sailthru_max_retries)
def patch_utcnow(test):
utcnow = test.patch(publication, "datetime").utcnow
ref = utcnow.return_value = datetime.utcnow()
return ref
def before_request():
if current_user.is_authenticated:
current_user.last_seen = datetime.utcnow()
db.session.commit()
def create_message(type):
message = {}
message['type'] = type
message['hostname'] = socket.gethostname()
message['time'] = datetime.utcnow()
return message
def _set_time_end(cls, deployment_history):
if not deployment_history.time_end:
deployment_history.time_end = datetime.utcnow()
def unix_utc_now():
d = datetime.utcnow()
_unix = calendar.timegm(d.utctimetuple())
return _unix
def index():
return render_template('index.html',
current_time=datetime.utcnow(),
name=session.get("name"))
def get_file_meta(worker_name,
path,
cliargs,
reindex_dict,
statsembeded=False):
"""This is the get file meta data function.
It scrapes file meta and ignores files smaller
than minsize Bytes, newer than mtime
and in excluded_files. Returns file meta dict.
"""
try:
# check if stats embeded in path
if statsembeded:
metadata = path[1]
fullpath = path[0]
else:
fullpath = path
filename = os.path.basename(fullpath)
# check if file is in exluded_files list
if file_excluded(filename):
return None
extension = os.path.splitext(filename)[1][1:].strip().lower()
if statsembeded:
# get embeded stats from path
mode, ino, dev, nlink, uid, gid, size, atime, mtime, ctime, blocks = metadata
else:
# use lstat to get meta and not follow sym links
s = os.lstat(fullpath)
mode, ino, dev, nlink, uid, gid, size, atime, mtime, ctime = s
blocks = s.st_blocks
# Are we storing file size or on disk size
if cliargs['sizeondisk']:
size = blocks * cliargs['blocksize']
# Skip files smaller than minsize cli flag
if size < cliargs['minsize']:
return None
# Convert time in days (mtime cli arg) to seconds
time_sec = cliargs['mtime'] * 86400
file_mtime_sec = time.time() - mtime
if time_sec < 0:
# Only process files modified less than x days ago
if file_mtime_sec > (time_sec * -1):
return None
else:
# Only process files modified at least x days ago
if file_mtime_sec < time_sec:
return None
# convert times to utc for es
mtime_utc = datetime.utcfromtimestamp(mtime).isoformat()
atime_utc = datetime.utcfromtimestamp(atime).isoformat()
ctime_utc = datetime.utcfromtimestamp(ctime).isoformat()
# get owner and group names
owner, group = get_owner_group_names(uid, gid, cliargs)
# create md5 hash of file using metadata filesize and mtime
filestring = str(size) + str(mtime)
filehash = hashlib.md5(filestring.encode('utf-8')).hexdigest()
# get time
indextime_utc = datetime.utcnow().isoformat()
# get absolute path of parent directory
parentdir = os.path.abspath(os.path.join(fullpath, os.pardir))
# create file metadata dictionary
filemeta_dict = {
"filename": filename,
"extension": extension,
"path_parent": parentdir,
"filesize": size,
"owner": owner,
"group": group,
"last_modified": mtime_utc,
"last_access": atime_utc,
"last_change": ctime_utc,
"hardlinks": nlink,
"inode": str(ino),
"filehash": filehash,
"tag": "",
"tag_custom": "",
"dupe_md5": "",
"worker_name": worker_name,
"indexing_date": indextime_utc,
"_type": "file"
}
# check plugins for adding extra meta data to filemeta_dict
for plugin in plugins:
try:
# check if plugin is for file doc
mappings = {'mappings': {'file': {'properties': {}}}}
plugin.add_mappings(mappings)
filemeta_dict.update(plugin.add_meta(fullpath))
except KeyError:
pass
# add any autotags to filemeta_dict
if cliargs['autotag'] and len(config['autotag_files']) > 0:
filemeta_dict = auto_tag(filemeta_dict, 'file', mtime, atime,
ctime)
# add cost per gb to filemeta_dict
if cliargs['costpergb']:
filemeta_dict = cost_per_gb(filemeta_dict, fullpath, mtime, atime,
ctime, 'file')
# search for and copy over any existing tags from reindex_dict
for sublist in reindex_dict['file']:
if sublist[0] == fullpath:
filemeta_dict['tag'] = sublist[1]
filemeta_dict['tag_custom'] = sublist[2]
break
except (OSError, IOError) as e:
warnings.warn("OS/IO Exception caused by: %s" % e)
return False
except Exception as e:
warnings.warn("Exception caused by: %s" % e)
return False
return filemeta_dict
def get(self):
never = datetime(1900, 1, 1)
user = self.current_user
def sort_key(token):
return (token.last_activity or never, token.created or never)
now = datetime.utcnow()
api_tokens = []
for token in sorted(user.api_tokens, key=sort_key, reverse=True):
if token.expires_at and token.expires_at < now:
self.db.delete(token)
self.db.commit()
continue
api_tokens.append(token)
# group oauth client tokens by client id
# AccessTokens have expires_at as an integer timestamp
now_timestamp = now.timestamp()
oauth_tokens = defaultdict(list)
for token in user.oauth_tokens:
if token.expires_at and token.expires_at < now_timestamp:
self.log.warning("Deleting expired token")
self.db.delete(token)
self.db.commit()
continue
if not token.client_id:
# token should have been deleted when client was deleted
self.log.warning("Deleting stale oauth token for %s",
user.name)
self.db.delete(token)
self.db.commit()
continue
oauth_tokens[token.client_id].append(token)
# get the earliest created and latest last_activity
# timestamp for a given oauth client
oauth_clients = []
for client_id, tokens in oauth_tokens.items():
created = tokens[0].created
last_activity = tokens[0].last_activity
for token in tokens[1:]:
if token.created < created:
created = token.created
if last_activity is None or (
token.last_activity
and token.last_activity > last_activity):
last_activity = token.last_activity
token = tokens[0]
oauth_clients.append({
'client':
token.client,
'description':
token.client.description or token.client.identifier,
'created':
created,
'last_activity':
last_activity,
'tokens':
tokens,
# only need one token id because
# revoking one oauth token revokes all oauth tokens for that client
'token_id':
tokens[0].api_id,
'token_count':
len(tokens),
})
# sort oauth clients by last activity, created
def sort_key(client):
return (client['last_activity'] or never, client['created']
or never)
oauth_clients = sorted(oauth_clients, key=sort_key, reverse=True)
html = self.render_template('token.html',
api_tokens=api_tokens,
oauth_clients=oauth_clients)
self.finish(html)
def register(show_spinner=False) -> str:
params = Params()
params.put("SubscriberInfo", HARDWARE.get_subscriber_info())
IMEI = params.get("IMEI", encoding='utf8')
HardwareSerial = params.get("HardwareSerial", encoding='utf8')
dongle_id = params.get("DongleId", encoding='utf8')
needs_registration = None in (IMEI, HardwareSerial, dongle_id)
# create a key for auth
# your private key is kept on your device persist partition and never sent to our servers
# do not erase your persist partition
if not os.path.isfile(PERSIST+"/comma/id_rsa.pub"):
needs_registration = True
cloudlog.warning("generating your personal RSA key")
mkdirs_exists_ok(PERSIST+"/comma")
assert os.system("openssl genrsa -out "+PERSIST+"/comma/id_rsa.tmp 2048") == 0
assert os.system("openssl rsa -in "+PERSIST+"/comma/id_rsa.tmp -pubout -out "+PERSIST+"/comma/id_rsa.tmp.pub") == 0
os.rename(PERSIST+"/comma/id_rsa.tmp", PERSIST+"/comma/id_rsa")
os.rename(PERSIST+"/comma/id_rsa.tmp.pub", PERSIST+"/comma/id_rsa.pub")
if needs_registration:
if show_spinner:
spinner = Spinner()
spinner.update("registering device")
# Create registration token, in the future, this key will make JWTs directly
with open(PERSIST+"/comma/id_rsa.pub") as f1, open(PERSIST+"/comma/id_rsa") as f2:
public_key = f1.read()
private_key = f2.read()
# Block until we get the imei
imei1, imei2 = None, None
while imei1 is None and imei2 is None:
try:
imei1, imei2 = HARDWARE.get_imei(0), HARDWARE.get_imei(1)
except Exception:
cloudlog.exception("Error getting imei, trying again...")
time.sleep(1)
serial = HARDWARE.get_serial()
params.put("IMEI", imei1)
params.put("HardwareSerial", serial)
backoff = 0
while True:
try:
register_token = jwt.encode({'register': True, 'exp': datetime.utcnow() + timedelta(hours=1)}, private_key, algorithm='RS256')
cloudlog.info("getting pilotauth")
resp = api_get("v2/pilotauth/", method='POST', timeout=15,
imei=imei1, imei2=imei2, serial=serial, public_key=public_key, register_token=register_token)
if resp.status_code in (402, 403):
cloudlog.info(f"Unable to register device, got {resp.status_code}")
dongle_id = UNREGISTERED_DONGLE_ID
else:
dongleauth = json.loads(resp.text)
dongle_id = dongleauth["dongle_id"]
break
except Exception:
cloudlog.exception("failed to authenticate")
backoff = min(backoff + 1, 15)
time.sleep(backoff)
if show_spinner:
spinner.close()
if dongle_id:
params.put("DongleId", dongle_id)
set_offroad_alert("Offroad_UnofficialHardware", dongle_id == UNREGISTERED_DONGLE_ID)
return dongle_id
def scrape_tree_meta(paths, cliargs, reindex_dict):
global worker
tree_dirs = []
tree_files = []
totalcrawltime = 0
statsembeded = False
path_count = 0
for path in paths:
path_count += 1
starttime = time.time()
if not cliargs['dirsonly']:
root, dirs, files = path
else:
root, dirs = path
files = []
if path_count == 1:
if type(root) is tuple:
statsembeded = True
# check if stats embeded in data from diskover tree walk client or crawlapi
if statsembeded:
root_path = root[0]
dmeta = get_dir_meta(worker,
root,
cliargs,
reindex_dict,
statsembeded=True)
else:
root_path = root
dmeta = get_dir_meta(worker,
root_path,
cliargs,
reindex_dict,
statsembeded=False)
if dmeta == "sametimes":
# fetch meta data for directory and all it's files (doc sources) from index2 since
# directory times haven't changed
dir_source, files_source = get_metadata(root_path, cliargs)
datenow = datetime.utcnow().isoformat()
for file_source in files_source:
# update indexed at time
file_source['indexing_date'] = datenow
# update worker name
file_source['worker_name'] = worker
tree_files.append(('file', file_source))
if dir_source:
# update indexed at time
dir_source['indexing_date'] = datenow
# update worker name
dir_source['worker_name'] = worker
# update crawl time
elapsed = time.time() - starttime
dir_source['crawl_time'] = round(elapsed, 6)
tree_dirs.append(dir_source)
totalcrawltime += elapsed
# get meta off disk since times different in Redis than on disk
elif dmeta:
# no files in batch, get them with scandir
if cliargs['dirsonly']:
for entry in scandir(root):
if entry.is_file(
follow_symlinks=False) and not file_excluded(
entry.name):
files.append(entry.name)
filecount = 0
for file in files:
if statsembeded:
fmeta = get_file_meta(worker,
file,
cliargs,
reindex_dict,
statsembeded=True)
else:
fmeta = get_file_meta(worker,
os.path.join(root_path, file),
cliargs,
reindex_dict,
statsembeded=False)
if fmeta:
tree_files.append(fmeta)
filecount += 1
# update crawl time=
elapsed = time.time() - starttime
dmeta['crawl_time'] = round(elapsed, 6)
# check for empty dirs and dirsonly cli arg
if cliargs['indexemptydirs']:
tree_dirs.append(dmeta)
elif not cliargs['indexemptydirs'] and (len(dirs) > 0
or filecount > 0):
tree_dirs.append(dmeta)
totalcrawltime += elapsed
# check if doc count is more than es chunksize and bulk add to es
if len(tree_dirs) + len(tree_files) >= config['es_chunksize']:
es_bulk_add(worker, tree_dirs, tree_files, cliargs, totalcrawltime)
del tree_dirs[:]
del tree_files[:]
totalcrawltime = 0
# bulk add to es
if len(tree_dirs) > 0 or len(tree_files) > 0:
es_bulk_add(worker, tree_dirs, tree_files, cliargs, totalcrawltime)
def _genThreadId(self):
time_stamp = datetime.utcnow().isoformat()
return hashlib.md5(time_stamp).hexdigest()
def get_dir_meta(worker_name, path, cliargs, reindex_dict, statsembeded=False):
"""This is the get directory meta data function.
It gets directory metadata and returns dir meta dict.
It checks if meta data is in Redis and compares times
mtime and ctime on disk compared to Redis and if same
returns sametimes string.
"""
try:
if statsembeded:
metadata = path[1]
dirpath = path[0]
# get directory meta embeded in path
mode, ino, dev, nlink, uid, gid, size, atime, mtime, ctime = metadata
else:
# get directory meta using lstat
dirpath = path
mode, ino, dev, nlink, uid, gid, size, atime, mtime, ctime = os.lstat(
dirpath)
# convert times to utc for es
mtime_utc = datetime.utcfromtimestamp(mtime).isoformat()
atime_utc = datetime.utcfromtimestamp(atime).isoformat()
ctime_utc = datetime.utcfromtimestamp(ctime).isoformat()
if cliargs['index2']:
# check if directory times cached in Redis
redis_dirtime = redis_conn.get(
base64.encodestring(dirpath.encode('utf-8', errors='ignore')))
if redis_dirtime:
cached_times = float(redis_dirtime.decode('utf-8'))
# check if cached times are the same as on disk
current_times = float(mtime + ctime)
if cached_times == current_times:
return "sametimes"
# get time now in utc
indextime_utc = datetime.utcnow().isoformat()
# get owner and group names
owner, group = get_owner_group_names(uid, gid, cliargs)
filename = os.path.basename(dirpath)
parentdir = os.path.abspath(os.path.join(dirpath, os.pardir))
dirmeta_dict = {
"filename": filename,
"path_parent": parentdir,
"filesize": 0,
"items": 1, # 1 for itself
"items_files": 0,
"items_subdirs": 0,
"last_modified": mtime_utc,
"last_access": atime_utc,
"last_change": ctime_utc,
"hardlinks": nlink,
"inode": str(ino),
"owner": owner,
"group": group,
"tag": "",
"tag_custom": "",
"crawl_time": 0,
"change_percent_filesize": "",
"change_percent_items": "",
"change_percent_items_files": "",
"change_percent_items_subdirs": "",
"costpergb": "",
"worker_name": worker_name,
"indexing_date": indextime_utc,
"_type": "directory"
}
# check plugins for adding extra meta data to dirmeta_dict
for plugin in plugins:
try:
# check if plugin is for directory doc
mappings = {'mappings': {'directory': {'properties': {}}}}
plugin.add_mappings(mappings)
dirmeta_dict.update(plugin.add_meta(dirpath))
except KeyError:
pass
# add any autotags to dirmeta_dict
if cliargs['autotag'] and len(config['autotag_dirs']) > 0:
dirmeta_dict = auto_tag(dirmeta_dict, 'directory', mtime, atime,
ctime)
# search for and copy over any existing tags from reindex_dict
for sublist in reindex_dict['directory']:
if sublist[0] == dirpath:
dirmeta_dict['tag'] = sublist[1]
dirmeta_dict['tag_custom'] = sublist[2]
break
except (OSError, IOError) as e:
warnings.warn("OS/IO Exception caused by: %s" % e)
return False
except Exception as e:
warnings.warn("Exception caused by: %s" % e)
raise
# cache directory times in Redis, encode path (key) using base64
if config['redis_cachedirtimes'] == 'true':
redis_conn.set(base64.encodestring(
dirpath.encode('utf-8', errors='ignore')),
mtime + ctime,
ex=config['redis_dirtimesttl'])
return dirmeta_dict
def _get_models_dict():
"""Downloads the ditionary of models definitions.
File will be re-downloaded every 30 minutes (upon request) to update
the models definitions from repository.
Returns
-------
models_dict : dict
Dictionary with models definitions. Each key is an available model.
Each model entry is defined by:
- "model", path to the script with model definition
- "state", path to the archive containing the pre-saved model state
- "model_md5", md5 checksum of model definition
- "state_md5", md5 checksum of pre-saved model state
"""
# The `.last_update` contains the last time MODELS_DICT_FILE
# has been download. Read the last update time if this file is available.
# Otherwise the file will be created later
last_update_path = fm.join(SECML_MODELS_DIR, '.last_update')
last_update_format = "%d %m %Y %H:%M" # Specific format to avoid locale
current_datetime = datetime.utcnow() # UTC datetime to avoid locale
update_models_dict = None # Trigger flag for model definitions update
if fm.file_exist(MODELS_DICT_PATH):
update_models_dict = True # By default, trigger update
if fm.file_exist(last_update_path):
try:
with open(last_update_path) as fp:
last_update = \
datetime.strptime(fp.read(), last_update_format)
# Compute the threshold for triggering an update
last_update_th = last_update + timedelta(minutes=30)
except ValueError as e:
# Error occurred while parsing the last update date from file
# Clean it and re-create later. Definitions update stays True
_logger.debug(e) # Log the error for debug purposes
_logger.debug("Removing `{:}`".format(last_update_path))
fm.remove_file(last_update_path)
else:
# Do not trigger update if last update threshold is not passed
if current_datetime < last_update_th:
update_models_dict = False
if update_models_dict is not False:
# if update_models_dict is None means that models dict is not available
# if it is True means that an update has been triggered
# Either cases, we need to download the data and extract it
try: # Catch download errors
# Download definitions from current version's branch first,
# then from master branch
_dl_data_versioned(MODELS_DICT_FILE, SECML_MODELS_DIR)
except Exception as e:
if update_models_dict is None:
# If update_models_dict is still None, means that models dict
# is not available, so we propagate the error. Otherwise pass
raise e
_logger.debug(e) # Log the error for debug purposes
_logger.debug("Error when updating the models definitions. "
"Using the last available ones...")
else: # No error raised during download process
# Check if file has been correctly downloaded
if not fm.file_exist(MODELS_DICT_PATH):
raise RuntimeError(
'Something wrong happened while downloading the '
'models definitions. Please try again.')
# Update or create the "last update" file
with open(last_update_path, "w") as fp:
fp.write(current_datetime.strftime(last_update_format))
with open(MODELS_DICT_PATH) as fp:
return json.loads(fp.read())
from airflow import DAG
from datetime import datetime, timedelta
from airflow.contrib.operators.kubernetes_pod_operator import KubernetesPodOperator
from airflow.operators.dummy_operator import DummyOperator
default_args = {
'owner': 'airflow',
'depends_on_past': False,
'start_date': datetime.utcnow(),
'email': ['*****@*****.**'],
'email_on_failure': False,
'email_on_retry': False,
'retries': 1,
'retry_delay': timedelta(minutes=5)
}
dag = DAG('kubernetes_sample',
default_args=default_args,
schedule_interval=timedelta(minutes=1))
start = DummyOperator(task_id='run_this_first', dag=dag)
passing = KubernetesPodOperator(namespace='default',
image="Python:3.6",
cmds=["Python", "-c"],
arguments=["print('hello world')"],
labels={"foo": "bar"},
name="passing-test",
task_id="passing-task",
get_logs=True,
dag=dag)
def do(self, password):
self._time = datetime.utcnow()
self.page.generar_extraccion(self._transferencia.montos,
str(self._transferencia.observaciones),
password)
return self
def upgrade():
# Rename score table to answer_criterion_score
try:
# expected foreign key to follow naming conventions
with op.batch_alter_table('score',
naming_convention=convention) as batch_op:
# drop the fk before altering the column
batch_op.drop_constraint('fk_score_assignment_id_assignment',
'foreignkey')
batch_op.drop_constraint('fk_score_answer_id_answer', 'foreignkey')
batch_op.drop_constraint('fk_score_criterion_id_criterion',
'foreignkey')
except sa.exc.InternalError:
# if not, it is likely this name
with op.batch_alter_table('score') as batch_op:
# drop the fk before altering the column
batch_op.drop_constraint('score_ibfk_1', 'foreignkey')
batch_op.drop_constraint('score_ibfk_2', 'foreignkey')
batch_op.drop_constraint('score_ibfk_3', 'foreignkey')
op.rename_table('score', 'answer_criterion_score')
with op.batch_alter_table('answer_criterion_score',
naming_convention=convention) as batch_op:
# create the fk
batch_op.create_foreign_key(
'fk_answer_criterion_score_assignment_id_assignment',
'assignment', ['assignment_id'], ['id'],
ondelete="CASCADE")
batch_op.create_foreign_key(
'fk_answer_criterion_score_answer_id_answer',
'answer', ['answer_id'], ['id'],
ondelete="CASCADE")
batch_op.create_foreign_key(
'fk_answer_criterion_score_criterion_id_criterion',
'criterion', ['criterion_id'], ['id'],
ondelete="CASCADE")
# create new answer_score table
answer_score_table = op.create_table(
'answer_score',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('assignment_id', sa.Integer(), nullable=False),
sa.Column('answer_id', sa.Integer(), nullable=False),
sa.Column('scoring_algorithm',
EnumType(ScoringAlgorithm),
nullable=True),
sa.Column('score', sa.Float(), nullable=False),
sa.Column('variable1', sa.Float(), nullable=True),
sa.Column('variable2', sa.Float(), nullable=True),
sa.Column('rounds', sa.Integer(), nullable=False),
sa.Column('wins', sa.Integer(), nullable=False),
sa.Column('loses', sa.Integer(), nullable=False),
sa.Column('opponents', sa.Integer(), nullable=False),
sa.Column('modified_user_id', sa.Integer(), nullable=True),
sa.Column('modified', sa.DateTime(), nullable=False),
sa.Column('created_user_id', sa.Integer(), nullable=True),
sa.Column('created', sa.DateTime(), nullable=False),
sa.ForeignKeyConstraint(['answer_id'], ['answer.id'],
ondelete='CASCADE'),
sa.ForeignKeyConstraint(['assignment_id'], ['assignment.id'],
ondelete='CASCADE'),
sa.ForeignKeyConstraint(['created_user_id'], ['user.id'],
ondelete='SET NULL'),
sa.ForeignKeyConstraint(['modified_user_id'], ['user.id'],
ondelete='SET NULL'),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('answer_id'),
mysql_charset='utf8',
mysql_collate='utf8_unicode_ci',
mysql_engine='InnoDB')
op.create_index(op.f('ix_answer_score_score'),
'answer_score', ['score'],
unique=False)
# migrate aggregate data from answer_criteria_score to answer_score
answer_criterion_score_table = sa.table(
'answer_criterion_score', sa.column('id', sa.Integer),
sa.Column('assignment_id', sa.Integer),
sa.Column('answer_id', sa.Integer),
sa.Column('criterion_id', sa.Integer),
sa.Column('scoring_algorithm', EnumType(ScoringAlgorithm)),
sa.Column('score', sa.Float), sa.Column('variable1', sa.Float),
sa.Column('variable2', sa.Float), sa.Column('rounds', sa.Integer),
sa.Column('wins', sa.Integer), sa.Column('loses', sa.Integer),
sa.Column('opponents', sa.Integer))
connection = op.get_bind()
answer_scores = {}
for answer_criterion_score in connection.execute(
answer_criterion_score_table.select()):
answer_scores.setdefault(
answer_criterion_score.answer_id,
{
'assignment_id': answer_criterion_score.assignment_id,
'answer_id': answer_criterion_score.answer_id,
'scoring_algorithm': answer_criterion_score.scoring_algorithm,
'rounds': answer_criterion_score.rounds,
'wins': answer_criterion_score.wins,
'loses': answer_criterion_score.loses,
'opponents': answer_criterion_score.opponents,
# to be averaged
'score': [],
'variable1': [],
'variable2': []
})
answer_scores[answer_criterion_score.answer_id]['score'].append(
answer_criterion_score.score)
if answer_criterion_score.variable1:
answer_scores[
answer_criterion_score.answer_id]['variable1'].append(
answer_criterion_score.variable1)
if answer_criterion_score.variable2:
answer_scores[
answer_criterion_score.answer_id]['variable2'].append(
answer_criterion_score.variable2)
for answer_id, score in answer_scores.items():
average_score = sum(score['score']) / float(len(
score['score'])) if len(score['score']) else 0
average_variable1 = sum(score['variable1']) / float(
len(score['variable1'])) if len(score['variable1']) else None
average_variable2 = sum(score['variable2']) / float(
len(score['variable2'])) if len(score['variable2']) else None
connection.execute(answer_score_table.insert().values(
assignment_id=score['assignment_id'],
answer_id=score['answer_id'],
scoring_algorithm=score['scoring_algorithm'],
rounds=score['rounds'],
wins=score['wins'],
loses=score['loses'],
opponents=score['opponents'],
score=average_score,
variable1=average_variable1,
variable2=average_variable2,
modified=datetime.utcnow(),
created=datetime.utcnow()))
