async def test_run_task_timeout(context): """`run_task` raises `ScriptWorkerTaskException` and kills the process after exceeding `task_max_timeout`. """ temp_dir = os.path.join(context.config['work_dir'], "timeout") context.config['task_script'] = ( sys.executable, TIMEOUT_SCRIPT, temp_dir ) # With shorter timeouts we hit issues with the script not managing to # create all 6 files context.config['task_max_timeout'] = 5 pre = arrow.utcnow().timestamp with pytest.raises(ScriptWorkerTaskException): await swtask.run_task(context, noop_to_cancellable_process) post = arrow.utcnow().timestamp # I don't love these checks, because timing issues may cause this test # to be flaky. However, I don't want a non- or long- running test to pass. # Did this run at all? assert post - pre >= 5 # Did this run too long? e.g. did it exit on its own rather than killed # If this is set too low (too close to the timeout), it may not be enough # time for kill_proc, kill_pid, and the `finally` block to run assert post - pre < 10 # Did the script generate the expected output? files = {} for path in glob.glob(os.path.join(temp_dir, '*')): files[path] = (time.ctime(os.path.getmtime(path)), os.stat(path).st_size) print("{} {}".format(path, files[path])) for path in glob.glob(os.path.join(temp_dir, '*')): print("Checking {}...".format(path)) assert files[path] == (time.ctime(os.path.getmtime(path)), os.stat(path).st_size) assert len(list(files.keys())) == 6 # Did we clean up? assert context.proc is None
def post(self): '''make a new paste''' if not 'paste' in request.form: return json.dumps({'error':'paste required'}) if 'api_key' in request.form: user = database.ApiKey.objects(key=request.form['api_key']).first() if user: user = user.user else: return json.dumps({'error': 'invalid api_key'}) paste = database.Paste() paste.name = random_string() #deduplicate paste name while database.Paste.objects(name=paste.name).first(): paste.name = random_string() paste.paste = request.form['paste'] paste.language = request.form['language'] if 'language' in request.form else None #TODO: autodetect language here paste.user = user paste.digest = sha1(paste.paste.encode('utf-8')).hexdigest() paste.time = arrow.utcnow().datetime if 'expiration' in request.form: #expiration needs to be a time in seconds, >0 try: seconds = int(request.form['expiration']) if seconds < 0: return json.dumps({'error': 'cannot expire in the past'}) if seconds > 0: paste.expire = arrow.utcnow().replace(seconds=+seconds) except ValueError: return json.dumps({'error': 'invalid expiration format, should be number of seconds'}) paste.save() #domain is optional, no validation done. if you feel like using one of the alternatives (vomitb.in, not-pasteb.in), set the domain before sending the paste return json.dumps({'success': 1, 'url': 'https://{domain}/{name}'.format(domain=request.form['domain'] if 'domain' in request.form else 'zifb.in', name=paste.name)})
def upgrade(): """ Add next_check and last_check columns to the projects table. """ op.add_column( "projects", sa.Column( "last_check", sa.TIMESTAMP(timezone=True), default=arrow.utcnow().datetime, server_default=sa.func.current_timestamp(), ), ) op.add_column( "projects", sa.Column( "next_check", sa.TIMESTAMP(timezone=True), default=arrow.utcnow().datetime, server_default=sa.func.current_timestamp(), ), ) op.create_index( op.f("ix_projects_last_check"), "projects", ["last_check"], unique=False ) op.create_index( op.f("ix_projects_next_check"), "projects", ["next_check"], unique=False )
def format_between(tp1, tp2): t1 = arrow.get(tp1) t2 = arrow.get(tp2) now = arrow.utcnow() today = arrow.utcnow().replace(hour=0, minute=0, second=0, microsecond=0) if t2 < now: return "Päättynyt" elif t1 < now < t2: left = t2 - now l_hours = int(left.total_seconds() / timedelta(hours=1).total_seconds()) l_minutes = int((left.total_seconds() - timedelta(hours=l_hours).total_seconds()) / 60) if l_hours == 0: return "Menossa, aikaa jäljellä {} minuuttia".format(l_minutes) else: return "Menossa, aikaa jäljellä {} tuntia ja {} minuuttia".format(l_hours, l_minutes) elif now < t1 < (today + timedelta(days=3)): return "Alkaa {} ja päättyy {}.".format( format_single_helper(t1), format_single_helper(t2)) else: return "Alkaa {} ja päättyy {}.".format( t1.to(settings.TIME_ZONE).format("DD.MM.YYYY klo. HH:mm", locale='fi_FI'), t2.to(settings.TIME_ZONE).format("DD.MM.YYYY klo. HH:mm", locale='fi_FI'))
def get_url_with_papi_access(self, url, type, cookies=None, timeout=None, saveToDb=True): papiaccess = IndexerApiAccess(indexer=self.indexer, type=type, url=url, time=arrow.utcnow().datetime) try: papiaccess.username = request.authorization.username if request.authorization is not None else None except RuntimeError: #Is thrown when we're searching which is run in a thread. When downloading NFOs or whatever this will work pass indexerStatus = None try: time_before = arrow.utcnow() response = self.get(url, cookies=cookies, timeout=timeout) response.raise_for_status() time_after = arrow.utcnow() papiaccess.response_time = (time_after - time_before).seconds * 1000 + ((time_after - time_before).microseconds / 1000) papiaccess.response_successful = True indexerStatus = self.handle_indexer_success(saveIndexerStatus=saveToDb) except RequestException as e: self.error("Error while connecting to URL %s: %s" % (url, str(e))) papiaccess.error = "Connection failed: %s" % removeSensitiveData(str(e)) response = None indexerStatus = self.handle_indexer_failure("Connection failed: %s" % removeSensitiveData(str(e)), saveIndexerStatus=saveToDb) finally: if saveToDb: papiaccess.save() return response, papiaccess, indexerStatus
def test_should_get_builds_statuses(app, f_projects): p = f_projects[0] assert ProjectLogic.should_fetch_builds_statuses_for_project(get_fresh_p(p)) is False wp = get_fresh_p(p) wp.build_triggered_on = arrow.utcnow() app.db.session.add(wp) app.db.session.commit() assert ProjectLogic.should_fetch_builds_statuses_for_project(get_fresh_p(p)) is True bs = BuildStatus( repo_name="foobar", build_id="12345", href="http://example.com/afasdfgt", status="pending" ) # ProjectLogic.get_or_create_build_info_from_bs(p, bs) app.db.session.commit() assert ProjectLogic.should_fetch_builds_statuses_for_project(get_fresh_p(p)) is False wp = get_fresh_p(p) wp.build_triggered_on = arrow.utcnow() app.db.session.add(wp) app.db.session.commit() assert ProjectLogic.should_fetch_builds_statuses_for_project(get_fresh_p(p)) is True
def test_get_projects_to_fetch_builds_status(app, f_projects): p = f_projects[0] assert len(ProjectLogic.get_projects_to_fetch_builds_status()) == 0 wp = get_fresh_p(p) wp.build_triggered_on = arrow.utcnow() app.db.session.add(wp) app.db.session.commit() p_list = ProjectLogic.get_projects_to_fetch_builds_status() assert len(p_list) == 1 assert p_list[0].id == p.id bs = BuildStatus( repo_name="foobar", build_id="12345", href="http://example.com/afasdfgt", status="pending" ) # ProjectLogic.get_or_create_build_info_from_bs(p, bs) app.db.session.commit() assert len(ProjectLogic.get_projects_to_fetch_builds_status()) == 0 wp = get_fresh_p(p) wp.build_triggered_on = arrow.utcnow() app.db.session.add(wp) app.db.session.commit() p_list = ProjectLogic.get_projects_to_fetch_builds_status() assert len(p_list) == 1 assert p_list[0].id == p.id
def map_cis_fields(options, csr): """ MAP issuer options to DigiCert CIS fields/options. :param options: :param csr: :return: """ if not options.get('validity_years'): if not options.get('validity_end'): options['validity_end'] = arrow.utcnow().replace(years=current_app.config.get('DIGICERT_DEFAULT_VALIDITY', 1)) options['validity_years'] = determine_validity_years(options['validity_end']) else: options['validity_end'] = arrow.utcnow().replace(years=options['validity_years']) data = { "profile_name": current_app.config.get('DIGICERT_CIS_PROFILE_NAME'), "common_name": options['common_name'], "additional_dns_names": get_additional_names(options), "csr": csr, "signature_hash": signature_hash(options.get('signing_algorithm')), "validity": { "valid_to": options['validity_end'].format('YYYY-MM-DD') }, "organization": { "name": options['organization'], "units": [options['organizational_unit']] } } return data
def _test_purge_expire(self, models): policy = self._get_policy(purge_interval=0.1) back_datetime = arrow.utcnow().replace(hours=-1).datetime models.GreylistEntry.create_entry(key='1.1.1.1', expire_time=back_datetime) green = gevent.spawn(policy.task_purge_expire, run_once=True) gevent.joinall([green], timeout=1.1) #gevent.kill(green) self.assertEquals(self._model_count(models.GreylistEntry), 0) back_datetime = arrow.utcnow().replace(hours=-25).datetime models.GreylistEntry.create_entry(key='1.1.1.1', timestamp=back_datetime) green = gevent.spawn(policy.task_purge_expire, run_once=True) gevent.joinall([green], timeout=1.1) #gevent.kill(green) self.assertEquals(self._model_count(models.GreylistEntry), 0) back_datetime = arrow.utcnow().replace(hours=-10).datetime #models.GreylistEntry(key='1.1.1.1', timestamp=back_datetime, protocol="").save() models.GreylistEntry.create_entry(key='1.1.1.1', timestamp=back_datetime) green = gevent.spawn(policy.task_purge_expire, run_once=True) gevent.joinall([green], timeout=1.1) #gevent.kill(green) self.assertEquals(self._model_count(models.GreylistEntry), 1)
def test_all_filters(self): meta = json.loads(INPUT1) # true case meta['SentTimestamp'] = str(arrow.utcnow().timestamp) self.assertTrue(self.filters.checkAllFilters(meta)) # fail sending site meta['SendingSite'] = 'TEST3' self.assertFalse(self.filters.checkAllFilters(meta)) # fail payload type meta = json.loads(INPUT1) meta['PayloadType'] = 'Report' self.assertFalse(self.filters.checkAllFilters(meta)) # fail payload format meta = json.loads(INPUT1) meta['PayloadFormat'] = 'OtherPDF' self.assertFalse(self.filters.checkAllFilters(meta)) # fail sensitivity meta = json.loads(INPUT1) meta['DataSensitivity'] = 'OUO' self.assertFalse(self.filters.checkAllFilters(meta)) # fail restrictions meta = json.loads(INPUT1) meta['SharingRestrictions'] = 'AMBER' self.assertFalse(self.filters.checkAllFilters(meta)) # fail recon policy meta = json.loads(INPUT1) meta['ReconPolicy'] = 'Touch' self.assertFalse(self.filters.checkAllFilters(meta)) # fail file age meta = json.loads(INPUT1) meta['SentTimestamp'] = str(arrow.utcnow().shift(months=-2, seconds=-10).timestamp) self.assertFalse(self.filters.checkAllFilters(meta))
def process_tuple(self, tup): _user_id, _feature_id, _feature = tup.values # Output log to file. # log_bolt_feature = "\n[%s] Features %s for user %s: \n" % (arrow.utcnow(), _feature_id, _user_id) + \ # "- Total duration:\t%s\n" \ # "- Start time:\t%s\n" \ # "- End time:\t%s\n" \ # "- Most motion:\t%s\n" \ # "- motion prob:\t%s\n" \ # "- Most location lv1:\t%s\n" \ # "- location lv1 prob:\t%s\n" \ # "- Most location lv2:\t%s\n" \ # "- location lv2 prob:\t%s\n" \ # "- Max speed:\t%s\n" \ # "- Min speed:\t%s\n" \ # "- Average speed:\t%s\n" % _feature # log.debug(log_bolt_feature) temp = random.choice(self.event_list) temp["user"]["objectId"] = _user_id temp["timestamp"] = arrow.utcnow().timestamp temp["startTime"],temp["endTime"] = _feature[1], _feature[2] log.debug("hi") log.debug(str(arrow.utcnow())) log.debug(save_event(temp))
def allSlots(): if caching.get('all_slots', False): #we have a cached all_slots cached_time = caching.get('time_all_slots', arrow.utcnow().replace(hours=-1)) if cached_time > arrow.utcnow().replace(minutes=-5): #use cache! print("cached all_slots") return caching['all_slots'] else: #timing is off! print('timing!') #else print('uncached!') try: master_sheet = gc.open_by_key(MASTER_KEY).sheet1 all_slots = master_sheet.get_all_records() #all_slots = r.json()['result'] except Exception as e: return {"Error":e} caching['all_slots'] = all_slots caching['time_all_slots'] = arrow.utcnow() return all_slots
def parse_date(self, agetd, entry): m = self.age_pattern.search(agetd.text) days = None hours = None if m: days = int(m.group("days1")) hours = int(m.group("days2")) * 2.4 else: p = re.compile(r"(?P<hours>\d+) hours?") m = p.search(agetd.text) if m: days = 0 hours = int(m.group("hours")) if hours is not None: pubdate = arrow.utcnow().replace(days=-days, hours=-1) # hours because of timezone change below if hours > 0: pubdate = pubdate.replace(hours=-hours) pubdate = pubdate.to("+01:00") # nzbindex server time, I guess? entry.epoch = pubdate.timestamp entry.pubdate_utc = str(pubdate) entry.age_days = (arrow.utcnow() - pubdate).days entry.age = str(entry.age_days) + "d" entry.age_precise = True # Precise to 2.4 hours, should be enough for duplicate detection entry.pubDate = pubdate.format("ddd, DD MMM YYYY HH:mm:ss Z") else: self.error("Found no age info in %s" % str(agetd)) raise IndexerResultParsingRowException("Unable to parse age")
def get_records(): import sqlite3 from_date_str = request.args.get('from', time.strftime( "%Y-%m-%d 00:00")) # Get the from date value from the URL to_date_str = request.args.get('to', time.strftime( "%Y-%m-%d %H:%M")) # Get the to date value from the URL timezone = request.args.get('timezone', 'Etc/UTC') # This will return a string, if field range_h exists in the request range_h_form = request.args.get('range_h', '') range_h_int = "nan" # initialise this variable with not a number print "REQUEST:" print request.args try: range_h_int = int(range_h_form) except: print "range_h_form not a number" print "Received from browser: %s, %s, %s, %s" % (from_date_str, to_date_str, timezone, range_h_int) # Validate date before sending it to the DB if not validate_date(from_date_str): from_date_str = time.strftime("%Y-%m-%d 00:00") if not validate_date(to_date_str): # Validate date before sending it to the DB to_date_str = time.strftime("%Y-%m-%d %H:%M") print '2. From: %s, to: %s, timezone: %s' % (from_date_str, to_date_str, timezone) # Create datetime object so that we can convert to UTC from the browser's # local time from_date_obj = datetime.datetime.strptime(from_date_str, '%Y-%m-%d %H:%M') to_date_obj = datetime.datetime.strptime(to_date_str, '%Y-%m-%d %H:%M') # If range_h is defined, we don't need the from and to times if isinstance(range_h_int, int): arrow_time_from = arrow.utcnow().replace(hours=-range_h_int) arrow_time_to = arrow.utcnow() from_date_utc = arrow_time_from.strftime("%Y-%m-%d %H:%M") to_date_utc = arrow_time_to.strftime("%Y-%m-%d %H:%M") from_date_str = arrow_time_from.to(timezone).strftime("%Y-%m-%d %H:%M") to_date_str = arrow_time_to.to(timezone).strftime("%Y-%m-%d %H:%M") else: # Convert datetimes to UTC so we can retrieve the appropriate records # from the database from_date_utc = arrow.get(from_date_obj, timezone).to( 'Etc/UTC').strftime("%Y-%m-%d %H:%M") to_date_utc = arrow.get(to_date_obj, timezone).to( 'Etc/UTC').strftime("%Y-%m-%d %H:%M") conn = sqlite3.connect('/var/www/lab_app/lab_app.db') curs = conn.cursor() curs.execute("SELECT * FROM temperatures WHERE rDateTime BETWEEN ? AND ? AND sensorID = 'Ambient'", (from_date_utc.format('YYYY-MM-DD HH:mm'), to_date_utc.format('YYYY-MM-DD HH:mm'))) temperatures = curs.fetchall() curs.execute("SELECT * FROM humidities WHERE rDateTime BETWEEN ? AND ? AND sensorID = 'Ambient'", (from_date_utc.format('YYYY-MM-DD HH:mm'), to_date_utc.format('YYYY-MM-DD HH:mm'))) humidities = curs.fetchall() conn.close() return [temperatures, humidities, timezone, from_date_str, to_date_str]
def sitemap(): pages = [] for p in Post.published(): pages.append({'url': url_for('.post', slug=p.slug, _external=True), 'last_modified': p.utc_pub_date, }) # FIXME: Be lazy with the semistatic pages and categories for the moment awhile_ago = arrow.utcnow().replace(days=-10) for cat in Category.query.all(): pages.append({'url': url_for('.category', slug=cat.title.lower(), _external=True), 'last_modified': awhile_ago, }) for page in ('about', 'projects'): pages.append({'url': url_for('.semistatic', page=page, _external=True), 'last_modified': awhile_ago, }) posts = (Post.query .filter(Post.utc_pub_date < arrow.utcnow().datetime) .filter_by(status=Post.PUBLISHED).count()) last_pub_date = Post.published().order_by(Post.utc_pub_date.desc())[0].utc_pub_date for archive_page in xrange(posts / POSTS_PER_ARCHIVE_PAGE): pages.append({'url': url_for('.archive', page=archive_page + 1, _external=True), 'last_modified': last_pub_date, }) return render_template('sitemap.xml', pages=pages)
def handle_indexer_failure(self, reason=None, disable_permanently=False, saveIndexerStatus=True): # Escalate level by 1. Set disabled-time according to level so that with increased level the time is further in the future try: indexer_status = self.indexer.status.get() except IndexerStatus.DoesNotExist: indexer_status = IndexerStatus(indexer=self.indexer) if indexer_status.level == 0: indexer_status.first_failure = arrow.utcnow() indexer_status.latest_failure = arrow.utcnow() indexer_status.reason = reason # Overwrite the last reason if one is set, should've been logged anyway if disable_permanently: indexer_status.disabled_permanently = True self.info("Disabling indexer permanently until reenabled by user because the authentication failed") else: indexer_status.level = min(len(self.disable_periods) - 1, indexer_status.level + 1) indexer_status.disabled_until = arrow.utcnow().replace(minutes=+self.disable_periods[indexer_status.level]) self.info( "Disabling indexer temporarily due to access problems. Will be reenabled %s" % indexer_status.disabled_until.humanize() ) if saveIndexerStatus: self.saveIndexerStatus(indexer_status) return indexer_status
def test_publication_date(minimal_record_model, depid_pid, legacyjson_v1): """Test publication date.""" for k in ['publication_date', 'embargo_date']: minimal_record_model[k] = arrow.utcnow().date() - timedelta(days=1) obj = legacyjson_v1.transform_record( depid_pid, minimal_record_model)['metadata'] assert arrow.get(obj[k]).date() <= arrow.utcnow().date()
def post(self, catalog, name, title, summary, tags, content): """ 创建文章 $input: catalog?str: 目录 title?str: 标题 name?str&optional: 名称 summary?str&optional: 摘要 tags: - str content?str: 内容 $output: id?str: ID """ if not name: name = title[:32] if not summary: summary = content[:160] id = "/".join([g.user["id"], catalog, name]) resp = db.run(r.table("article").insert({ "id": id, "author": g.user["id"], "catalog": catalog, "name": name, "title": title, "summary": summary, "tags": tags, "content": content, "date_create": arrow.utcnow().datetime, "date_modify": arrow.utcnow().datetime })) if resp["errors"]: abort(400, "Conflict", "创建失败: %s" % resp["first_error"]) return {"id": id}
def initialize(self): default_vector = lambda: [ # Length of duration arrow.utcnow().timestamp, # Start time of feature arrow.utcnow().timestamp, # End time of feature arrow.utcnow().timestamp, # Most possible motion type "", # Most possible motion prob -1.0, # Most possible location Lv1 type "", # Most possible location Lv1 prob -1.0, # Most possible location Lv2 type "", # Most possible location Lv2 prob -1.0, # Max speed float("inf"), # Min speed -1.0, # Average speed -1.0 ] self.feature_vector = defaultdict(default_vector)
def refresh_user_timeline(self): if self._last_refreshed < arrow.utcnow().replace(seconds=-120): since_id = self._user_timeline[0].id if self._user_timeline else None statuses = self._api.user_timeline(self.id, since_id=since_id) statuses.extend(self._user_timeline) self._user_timeline = statuses self._last_refreshed = arrow.utcnow()
def init_repo_data(): import arrow import uuid session = DBSession() for i in range(1, 10): f = FileRouteModule() f.id = int(i) f.src_path = '/Users/Kevin/workspace/distributionfile/data' f.src_name = i f.src_extension = 'txt' f.retry_times = 0 f.retry_interval = 0 f.create_dt = arrow.utcnow().to('local').naive f.update_dt = arrow.utcnow().to('local').naive f.valid_from = arrow.utcnow().to('local').naive f.valid_to = arrow.utcnow().to('local').replace(days=365).naive f.tar_path = 'tmp/' f.tar_name = '{file}_{YMD}'.format(file=i, YMD='{YMD}') f.transtype = 'SFTP' f.ftpname = '10.8.4.116' session.add(f) f = session.query(FileRouteModule).get(9) f.tar_name = '9_{YM}' session.commit() session.close()
def sync(watson): """ Get the frames from the server and push the new ones. The URL of the server and the User Token must be defined via the `watson config` command. Example: \b $ watson config backend.url http://localhost:4242 $ watson config backend.token 7e329263e329 $ watson sync Received 42 frames from the server Pushed 23 frames to the server """ last_pull = arrow.utcnow() pulled = watson.pull() click.echo("Received {} frames from the server".format(len(pulled))) pushed = watson.push(last_pull) click.echo("Pushed {} frames to the server".format(len(pushed))) watson.last_sync = arrow.utcnow() watson.save()
def modify_workflow_json(workflow): tasks = {} for task in workflow['tasks']: cur_task = Viewer_Task(task) tasks[cur_task.name] = vars(cur_task) workflow['tasks'] = tasks submitted_time = arrow.get(workflow['submitted_time']).timestamp cur_time = arrow.utcnow().timestamp if workflow['completed_time'] == 'null': completed_time = arrow.utcnow().timestamp else: completed_time = arrow.get(workflow['completed_time']).timestamp calc = Decimal(completed_time - submitted_time)/Decimal(60.0) duration = max(float(calc.quantize(Decimal('0e-1'), rounding=decimal.ROUND_HALF_UP)), 0.0) workflow['duration'] = duration calc = Decimal(cur_time - submitted_time)/Decimal(60.0) time_before_now = max(float(calc.quantize(Decimal('0e-3'), rounding=decimal.ROUND_HALF_UP)), 0.0) workflow['time_before_now'] = time_before_now workflow['startTime'] = arrow.get(workflow['submitted_time']).to('US/Mountain').format('MM-DD-YYYY HH:mm:ss') workflow['modified'] = True return workflow
def test_receipts_report(self): self.create_tariff("default_tariff", default=True) customer_info = self.create_customer_by_self("*****@*****.**") db.session.commit() customer_id = customer_info["customer_id"] self.admin_client.customer.update_balance(customer_id, "100", "test withdraw for test mode", "RUB") self.admin_client.customer.update(customer_id, detailed_info={"passport_series_number": "1234 567 890", "passport_issued_by": "UFMS Russia", "passport_issued_date": "2013-01-01"}) customer_db = Customer.get_by_id(customer_id) customer_db.confirm_email() self.admin_client.customer.make_prod(customer_id) self.admin_client.customer.update_balance(customer_id, "100", "test withdraw for prod mode", "RUB") start = utcnow().datetime - timedelta(days=30) end = utcnow().datetime + timedelta(hours=1) report = self.get_report("receipts", start, end, "csv") self.assertTrue(report) self.assertGreater(report.count(b";"), 5) report = [row for row in report.split(b"\r\n") if row] self.assertEqual(len(report), 2) # header + balance update after prod self.get_report("receipts", start, end, "tsv")
def filter_by_rule(): from_time = request.args.get("start_time", arrow.utcnow().replace(days=-7)) to_time = request.args.get("end_time", arrow.utcnow()) start = int(request.args.get("from", 0)) end = int(request.args.get("size", 100)) start_time = int(arrow.get(from_time).float_timestamp * 1000) end_time = int(arrow.get(to_time).float_timestamp * 1000) false_positive = request.args.get("false_positive", "false") sort_order = ElasticSearchHelpers.create_sort(True) time_filter = ElasticSearchHelpers.create_timestamp_filter(start_time, end_time) query_filter = ElasticSearchHelpers.create_query_string_filter("false_positive:" + false_positive) try: query = ElasticSearchHelpers.create_elasticsearch_filtered_query(filtered_query=query_filter, timestamp_filter=time_filter, sort_order=sort_order) datastore = get_data_store() params = dict(from_=start) params["size"] = end params["_source"] = "check_id" results = datastore.search(query=query, params=params) rules = set() for result in results["hits"]["hits"]: rule = result["_source"]["check_id"] rules .add(rule) # sets are not JSON serializable response = make_response(json.dumps(list(rules))) response.headers["Content-Type"] = "application/json" return response except DataStoreException: return "Failed to retrieve commits", 500
def test_store_elasticsearch_tokens_groups3(store, indicator): t = store.store.tokens.create({ 'username': '******', 'groups': ['staff'], 'write': True }) t2 = store.store.tokens.create({ 'username': '******', 'groups': ['staff2'], 'read': True, }) i = store.handle_indicators_create(t['token'], { 'indicator': 'example.com', 'group': 'staff', 'provider': 'example.com', 'tags': ['test'], 'itype': 'fqdn', 'lasttime': arrow.utcnow().datetime, 'reporttime': arrow.utcnow().datetime }, flush=True) assert i i = store.handle_indicators_search(t2['token'], {'itype': 'fqdn'}) i = json.loads(i) assert len(i) == 0 i = store.handle_indicators_search(t2['token'], {'indicator': 'example.com'}) i = json.loads(i) assert len(i) == 0
def last_metrics(cls): last_24_hours = arrow.utcnow().replace(hours=-24).datetime objects = cls.select().where(cls.timestamp >= last_24_hours) count = objects.count() if count == 0: return last_1_hour = arrow.utcnow().replace(hours=-1).datetime accepted = cls.select(fn.Sum(cls.accepts)).where(cls.timestamp >= last_24_hours) rejected = cls.select(fn.Sum(cls.rejects)).where(cls.timestamp >= last_24_hours) delay = cls.select(fn.Avg(cls.delay)).where(cls.timestamp >= last_24_hours, cls.accepts>=0, cls.delay>=0) metrics = { 'count': count, 'accepted': accepted or 0, 'rejected': rejected or 0, 'delay': delay or 0.0, 'abandoned': objects.filter(cls.accepts==0, cls.timestamp<=last_1_hour).count(), #'count_accepts': objects.filter(accepts__gte=1).count(), } metrics['requests'] = metrics['accepted'] + metrics['rejected'] return metrics
def validate_when(self, value): when = arrow.get(value).replace(year=arrow.utcnow().year) if when < arrow.utcnow(): when = arrow.get(value).replace(year=arrow.utcnow().year + 1) return when.date()
def test_expires_arrow_to_timestamp(self): """ Converting expiration arrow instance to timestamp """ utc = arrow.utcnow() msk = arrow.utcnow().to('Europe/Moscow') timestamp = calendar.timegm(datetime.utcnow().utctimetuple()) self.assertEqual(timestamp, time.expires_to_timestamp(utc)) self.assertEqual(timestamp, time.expires_to_timestamp(msk))
def present_staff(self): """Fetch all present staff members""" resolutions = Resolution.query.join(Inquiry).filter( Resolution.resolved_at >= arrow.utcnow().replace(hours=-3), Inquiry.queue_id == self.id).all() staff = set() for resolution in resolutions: user = User.query.get(resolution.user_id) user.resolution = resolution ns = [res.resolved_at - res.created_at for res in Resolution.query.filter( Resolution.resolved_at >= arrow.utcnow().replace(hours=-6), Resolution.user_id == user.id )] if ns: total = ns[0] for n in ns[1:]: total = n + total user.average = total/len(ns) else: user.average = 'n/a' current = Resolution.query.filter_by(user_id=user.id, resolved_at=None).first() user.status = 'free' if not current else 'busy' staff.add(user) return staff
def fetch_price(country_code='NI', session=None): """Requests the most recent known power prices in Nicaragua grid Arguments: country_code (optional) -- ignored, only information for Nicaragua is returned session (optional) -- request session passed in order to re-use an existing session Return: A list of dictionaries in the form: [ { 'countryCode': 'FR', 'currency': EUR, 'datetime': '2017-01-01T01:00:00Z', 'price': 0.0, 'source': 'mysource.com' }, { 'countryCode': 'FR', 'currency': EUR, 'datetime': '2017-01-01T00:00:00Z', 'price': 0.0, 'source': 'mysource.com' } ] """ requests_obj = session or requests.session() response = requests_obj.get(PRICE_URL) response.encoding = 'utf-8' prices_html = response.text now_local_time = arrow.utcnow().to(TIMEZONE) midnight_local_time = arrow.utcnow().to(TIMEZONE).replace(hour=0, minute=0, second=0, microsecond=0) hours_text = prices_html.split('<br />') data = [] for hour_data in hours_text: if not hour_data: # there is usually an empty item at the end of the list, ignore it continue # hour_data is like "Hora 13: 84.72" hour = int(extract_text(hour_data, 'Hora ', ':')) price = float(extract_text(hour_data, ' ')) price_date = midnight_local_time.replace(hour=hour) if price_date > now_local_time: # data for previous day is also included price_date = price_date.replace(days=-1) data.append({ 'countryCode': country_code, 'datetime': price_date.datetime, 'currency': 'USD', 'price': price, 'source': 'cndc.org.ni' }) return data
async def clean_wf_cache(db: Database): cutoff = arrow.utcnow().timestamp - 86500 await db[db.db_nam].WarframeCache.delete_many({'Created': {'$lt': cutoff}})
def GetDayOffset(self, day): dow = -(int(arrow.utcnow().to("Europe/Dublin").format("d")) - 1) + day if int(arrow.utcnow().to("Europe/Dublin").format("d")) > 5: dow += 7 day_arw = arrow.utcnow().to("Europe/Dublin").shift(days=+dow) return day_arw
async def tomorrow(self, ctx): """Classes for tomorrow.""" day_arw = arrow.utcnow().to("Europe/Dublin").shift(days=+1) embed = self.GetDayArwSchedEmbed(day_arw) await ctx.send(embed=embed)
def get_filename(self): """ Returns the file name for the solves export. """ return self.filename_template.format( username=self.username, date=arrow.utcnow().format('YYYY-MM-DD_HH-mm'))
def is_disabled_by_date(cls): is_disabled_expr = sa.sql.and_( cls.disabled_utc.isnot(None), cls.disabled_utc <= arrow.utcnow(), ) return sa_sql.case([(is_disabled_expr, sa.true())], else_=sa.false())
def is_disabled_by_date(self): return self.disabled_utc is not None and self.disabled_utc <= arrow.utcnow( )
def utcnow(): return arrow.utcnow().datetime
def parse(expression, now=None, tz='UTC', type=None, roundDown=True): ''' the main meat and potatoes of this this whole thing takes our datemath expression and does our date math :param expression - the datemath expression :param now - what time is now; when will now be then? soon :param type - if we are dealing with a arrow or datetime object :param roundDown - wether or not we should round up or round down on this. default is roundDown=True, which means if it was 12:00:00, `/d` would be '00:00:00', and with roundDown=False, `/d` would be '29:59:59' ''' if now is None: now = arrow.utcnow() if debug: print("Orig Expression: {0}".format(expression)) math = '' time = '' if 'UTC' not in tz: if debug: print("will now convert tz to {0}".format(tz)) now = now.to(tz) if expression == 'now': if debug: print("Now, no dm: {0}".format(now)) if type: return getattr(now, type) else: return now elif re.match('\d{10,}', str(expression)): if debug: print('found an epoch timestamp') if len(str(expression)) == 13: raise DateMathException('Unable to parse epoch timestamps in millis, please convert to the nearest second to continue - i.e. 1451610061 / 1000') ts = arrow.get(int(expression)) ts = ts.replace(tzinfo=tz) return ts elif expression.startswith('now'): ''' parse our standard "now+1d" kind of queries ''' math = expression[3:] time = now if debug: print('now expression: {0}'.format(now)) else: ''' parse out datemath with date, ex "2015-10-20||+1d" ''' if '||' in expression: timestamp, math = expression.split('||') time = parseTime(timestamp, tz) elif expression.startswith(('+', '-', '/')): ''' this catches expressions that don't start with 'now' but we are assume are 'now', such as '+1h', '-2/w', '/1d', '+2h-2m', etc ''' math = expression time = now else: math = '' time = parseTime(expression, tz) if not math or math == '': rettime = time rettime = evaluate(math, time, tz, roundDown) if type: return getattr(rettime, type) else: return rettime
import sys import logging import arrow import eventProducer topic = 'ihr_atlas_probe_discolog' # End time startTime = arrow.utcnow() startTime.replace(microsecond=0, second=0) endTime = startTime.shift(days=1) # Logging FORMAT = '%(asctime)s %(processName)s %(message)s' logging.basicConfig(format=FORMAT, filename='ihr-kafka-disco-data-fetch.log', level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S') logging.info("Started: %s" % sys.argv) ep = eventProducer.EventProducer(topic) # Get one day of live data ep.startLive(endTime.timestamp)
def test_close_1050(): here = arrow.utcnow() there = here.shift(hours=+75) assert there.isoformat() == close_time(1050, 1000, here)
def run(self, token): self.start_time = arrow.utcnow().timestamp current_time = datetime.datetime.now().time() current_time.isoformat() self.log.info('Sending Client Startup Signal...') super().run(token)
def test_close_0(): here = arrow.utcnow() there = here.shift(hours=+1) print(here.isoformat()) print(close_time(0, 600, here)) assert there.isoformat() == close_time(0, 600, here)
else: val = float(-val) elif re.match('[a-zA-Z]+', char): now = calculate(now, val, unitMap(char)) else: raise DateMathException(''''{}' is not a valid timeunit for expression: '{}' '''.format(char, expression)) i += 1 if debug: print("Fin: {0}".format(now)) if debug: print('\n\n') return now if __name__ == "__main__": if debug: print('NOW: {0}'.format(arrow.utcnow())) if debug: print('\n\n') #parse('now-1h') #parse('now+12h') #parse('now+1h') #parse('now+1h+1m') #parse('now+1h/d') #parse('now-2d/d') #parse('2012-01-01||+1M/d') #parse('now+1w/w') #parse('+1d/d') #parse('/h') #parse('/d') #parse('2014-11-18||+1M/M') #parse('2014-11-18||+1M/M+1h') #parse('2014-11-18||/w')
def shift_by_some(): here = arrow.utcnow() there = here.shift(hours=+1) there = there.shift(minutes=+27) assert there.isoformat() == shift_start_by(here.isoformat(), 1.45)
def test_close_210(): here = arrow.utcnow() there = here.shift(hours=+13) there = there.shift(minutes=+30) assert there.isoformat() == close_time(210, 200, here)
print("Got database") collection = db.dated print("Using sample collection") except Exception as err: print("Failed") print(err) sys.exit(1) # # Insertions: I commented these out after the first # run successfuly inserted them # record = { "type": "dated_memo", "date": arrow.utcnow().naive, "text": "This is a sample memo" } post_id = collection.insert_one(record).inserted_id record = { "type": "dated_memo", "date": arrow.utcnow().replace(days=+1).naive, "text": "Sample one day later" } post_id = collection.insert_one(record).inserted_id # # Read database --- May be useful to see what is in there, # even after you have a working 'insert' operation in the flask app, # but they aren't very readable. If you have more than a couple records,
def test_open_0(): here = arrow.utcnow().isoformat() assert here == open_time(0, 600, here)
def get_data(self, siteID, start_date, end_date, method, metrics): self.form_data.update({ 'body': { 'siteID': siteID, 'start_date': start_date, 'end_date': end_date, 'method': method, 'metrics': metrics } }) res = requests.post(self.get_data_url, data=bytes(json.dumps(self.form_data), encoding='utf-8')) data = res.json() data = data["body"]["data"][0]["result"]["items"][1][0] return data if __name__ == '__main__': username, password, token = ('xxxx', 'xxxx', 'xxxxxxxx') #TODO bdtj = BDTJ(username, password, token) today = arrow.utcnow().to('local').shift().format('YYYYMMDD') yesterday = arrow.utcnow().to('local').shift(days=-1).format('YYYYMMDD') method = 'overview/getTimeTrendRpt' metrics = 'pv_count,visitor_count,ip_count,bounce_ratio,avg_visit_time' for siteID in bdtj.siteIDs.values(): data = bdtj.get_data(siteID, yesterday, today, method, metrics) print(data)
def shift_by_0(): here = arrow.utcnow() assert here.isoformat() == shift_start_by(here.isoformat(), 0)
def markdown(self, disclose=False): """ Return the markdown representation of the flaw Thought for generating reports, mainly PDF-based """ # pylint: disable = line-too-long return_str = "" return_str += "# Vulnerability advisory: " + str(self.title) + "\n" return_str += "## General" + "\n" return_str += str(self.description) + "\n" return_str += "\n" return_str += "| Item | Value |" + "\n" return_str += "| ---- | ----- |" + "\n" return_str += "| RVD ID |" + str(self.id) + "|" + "\n" return_str += "| title |" + str(self.title) + "|" + "\n" return_str += "| type |" + str(self.type) + "|" + "\n" return_str += "| cwe |" + str(self.cwe) + "|" + "\n" return_str += "| cve |" + str(self.cve) + "|" + "\n" return_str += "| keywords |" + str(self.keywords) + "|" + "\n" return_str += "| vendor |" + str(self.vendor) + "|" + "\n" return_str += "| system |" + str(self.system) + "|" + "\n" return_str += "\n" # return_str += "\newpage" # severity return_str += "## Severity" + "\n" return_str += "\n" return_str += "| Item | Value |" + "\n" return_str += "| ---- | ----- |" + "\n" return_str += "| rvss-score | " + str(self.rvss_score) + " |" + "\n" return_str += "| rvss-vector | " + str(self.rvss_vector) + " |" + "\n" return_str += ( "| severity-description | " + str(self.severity_description) + " |" + "\n" ) return_str += "| cvss-score | " + str(self.cvss_score) + " |" + "\n" return_str += "| cvss-vector | " + str(self.cvss_vector) + " |" + "\n" # return_str += "\n" return_str += "\\newpage" # flaw return_str += "## The flaw" + "\n" return_str += ( "This section describes de flaw in more detail and \ captures relevant elements of it. For full understanding of the \ taxonomy used for its categorization, refer to \ [our taxonomy](https://github.com/aliasrobotics/RVD/blob/master/docs/TAXONOMY.md)" + "\n" ) return_str += "\n" return_str += "| Item | Value |" + "\n" return_str += "| ---- | ----- |" + "\n" return_str += "| phase | " + str(self.phase) + " |" + "\n" return_str += "| specificity | " + str(self.specificity) + " |" + "\n" return_str += ( "| architectural-location | " + str(self.architectural_location) + " |" + "\n" ) return_str += "| application | " + str(self.application) + " |" + "\n" return_str += "| subsystem | " + str(self.subsystem) + " |" + "\n" return_str += "| package | " + str(self.package) + " |" + "\n" return_str += "| languages | " + str(self.languages) + " |" + "\n" return_str += "| date-detected | " + str(self.date_detected) + " |" + "\n" return_str += "| detected-by | " + str(self.detected_by) + " |" + "\n" return_str += ( "| detected-by-method | " + str(self.detected_by_method) + " |" + "\n" ) return_str += ( "| date-reported | " + str(arrow.utcnow().format("YYYY-MM-DD")) + " |" + "\n" ) return_str += "| reported-by | " + str(self.reported_by) + " |" + "\n" return_str += ( "| reported-by-relationship | " + str(self.reported_by_relationship) + " |" + "\n" ) return_str += "| issue | " + str(self.issue) + " |" + "\n" return_str += "| links | " + str(self.links) + " |" + "\n" return_str += "| reproducibility | " + str(self.reproducibility) + " |" + "\n" return_str += "| trace | " + str(self.trace) + " |" + "\n" return_str += "| reproduction | " + str(self.reproduction) + " |" + "\n" return_str += ( "| reproduction-image | " + str(self.reproduction_image) + " |" + "\n" ) # additional_fields - flaw for key in self.additional_fields.keys(): if isinstance(self.additional_fields[key], dict): if key == "flaw": for key2 in self.additional_fields[key].keys(): return_str += ( "| " + (key2) + " | " + str(self.additional_fields[key][key2]) + " | " + "\n" ) return_str += "\\newpage" + "\n" # exploitation if disclose: return_str += "## Exploitation" + "\n" return_str += "\n" return_str += "| Item | Value |" + "\n" return_str += "| ---- | ----- |" + "\n" return_str += ( "| description | " + str(self.description_exploitation) + "|" + "\n" ) return_str += ( "| exploitation-image | " + str(self.exploitation_image) + "|" + "\n" ) return_str += ( "| exploitation-vector | " + str(self.exploitation_vector) + "|" + "\n" ) # TODO: make this prettier by separating this field from the table # and into a dedicate yaml section of verbatim return_str += ( "| exploitation-recipe | " + str(self.exploitation_recipe) + "|" + "\n" ) # additional_fields - exploitation for key in self.additional_fields.keys(): if isinstance(self.additional_fields[key], dict): if key == "exploitation": for key2 in self.additional_fields[key].keys(): return_str += ( "| " + (key2) + " | " + str(self.additional_fields[key][key2]) + " | " + "\n" ) else: return_str += "## Exploitation" + "\n" return_str += "\n" return_str += "| Item | Value |" + "\n" return_str += "| ---- | ----- |" + "\n" return_str += "| description | Not disclosed |" + "\n" return_str += "| exploitation-image | Not disclosed |" + "\n" return_str += "| exploitation-vector | Not disclosed |" + "\n" return_str += "| exploitation-recipe | Not disclosed |" + "\n" return_str += "\\newpage" + "\n" # mitigation if disclose: return_str += "## Mitigation" + "\n" return_str += "\n" return_str += "| Item | Value |" + "\n" return_str += "| ---- | ----- |" + "\n" return_str += ( "| description | " + str(self.description_mitigation) + "|" + "\n" ) return_str += "| pull-request | " + str(self.pull_request) + "|" + "\n" # additional_fields - mitigation for key in self.additional_fields.keys(): if isinstance(self.additional_fields[key], dict): if key == "mitigation": for key2 in self.additional_fields[key].keys(): return_str += ( "| " + (key2) + " | " + str(self.additional_fields[key][key2]) + " | " + "\n" ) else: return_str += "## Mitigation" + "\n" return_str += "\n" return_str += "| Item | Value |" + "\n" return_str += "| ---- | ----- |" + "\n" return_str += "| description | Not disclosed |" + "\n" return_str += "| pull-request | Not disclosed |" + "\n" return_str += "\n" # # additional_fields (others) # return_str += '## Mitigation' + "\n" # return_str += "\n" # return_str += '| Item | Value |' + "\n" # return_str += '| ---- | ----- |' + "\n" # for key in self.additional_fields.keys(): # if key in ['mitigation', 'exploitation', 'flaw']: # the ones contemplated above with additional_fields # continue # if isinstance(self.additional_fields[key], dict): # for key2 in self.additional_fields[key].keys(): # return_str +="| " + (key2) + " | " + str(self.additional_fields[key][key2]) + " | " + "\n" # else: # return_str +="| " + (key2) + " | " + str(self.additional_fields[key]) + " | " + "\n" # return_str += "\n" return return_str
async def execute(self, payload): """ Runs necessary checks and executes the command function. :type payload: sigma.core.mechanics.payload.CommandPayload """ if self.bot.ready: if payload.msg.guild: delete_command_message = payload.settings.get( 'delete_commands') if delete_command_message: try: await payload.msg.delete() except (discord.Forbidden, discord.NotFound): pass override = check_filter_perms(payload.msg, payload.settings, 'arguments') if await self.check_black_args(payload.settings, payload.args): if not any([ payload.msg.author.guild_permissions.administrator, override ]): await self.respond_with_emote(payload.msg, '🛡') return if not self.bot.cfg.dsc.bot and payload.msg.author.id != self.bot.user.id: self.log.warning(f'{payload.msg.author.name} tried using me.') return if not self.cd.is_cooling(payload.msg): if not await self.bot.cool_down.on_cooldown( f'{self.name}_core', payload.msg.author): await self.update_cooldown(payload.msg.author) perms, guild_perms = await self.check_permissions(payload) exec_timestamp = arrow.utcnow().float_timestamp self.log_command_usage(payload.msg, payload.args, exec_timestamp) self.cd.set_cooling(payload.msg) if perms.permitted: if guild_perms.permitted: requirements = CommandRequirements( self, payload.msg) if requirements.reqs_met: try: executed = False last_error = None client_os_broken_tries = 0 while client_os_broken_tries < 3 and not executed: try: await self.add_detailed_stats( payload, exec_timestamp) await getattr( self.command, self.name)(self, payload) executed = True except CancelledError: pass except aiohttp.ClientOSError as err: last_error = err client_os_broken_tries += 1 await asyncio.sleep(1) if not executed: raise last_error await add_cmd_stat(self) await self.add_usage_sumarum(payload.msg) self.bot.command_count += 1 cmd_ev_pld = CommandEventPayload( self.bot, self, payload) event_task = self.bot.queue.event_runner( 'command', cmd_ev_pld) self.bot.loop.create_task(event_task) except self.get_exception() as e: error = SigmaError(self, e) await error.error_handler(payload) else: await self.respond_with_emote(payload.msg, '📝') await self.send_requirements_error( requirements, payload) else: self.log.warning( 'ACCESS DENIED: This module or command is not allowed in this location.' ) await self.respond_with_emote(payload.msg, '🔒') else: perms.log_unpermitted() await self.respond_with_emote(payload.msg, '⛔') if perms.response: try: await payload.msg.channel.send( embed=perms.response) except (discord.Forbidden, discord.NotFound): pass else: await self.respond_with_emote(payload.msg, '❄') else: await self.respond_with_emote(payload.msg, '🕙')
def habiticaRegister(username, email, password, our_uuid): user_dict = {} #if user is already in e-mission db, try to load user data if edb.get_habitica_db().find({'user_id': our_uuid}).count() == 1: try: result = habiticaProxy(our_uuid, 'GET', '/api/v3/user', None) user_dict = result.json() logging.debug("parsed json from GET habitica user = %s" % user_dict) #if it fails, then user is in db but not in Habitica, so needs to create new account #FIX! Still need to test if this will throw an error correctly except urllib2.HTTPError: user_dict = newHabiticaUser(username, email, password, our_uuid) edb.get_habitica_db().update({"user_id": our_uuid}, { "$set": { 'metrics_data': { 'last_timestamp': arrow.utcnow().timestamp, 'bike_count': 0, 'walk_count': 0 }, 'habitica_username': username, 'habitica_password': password, 'habitica_id': user_dict['data']['_id'], 'habitica_token': user_dict['data']['apiToken'], 'habitica_group_id': None } }, upsert=True) if user_dict['data']['party']['_id']: edb.get_habitica_db().update({"user_id": our_uuid}, { "$set": { 'habitica_group_id': user_dict['data']['party']['_id'] } }, upsert=True) #now we have the user data in user_dict, so check if db is correct #Fix! should prob check here if our db is right, if it's in group, etc #if user is not in db, try to log in using email and password else: try: login_url = url + '/api/v3/user/auth/local/login' user_request = { 'username': username, 'email': email, 'password': password } logging.debug("About to login %s" % user_request) login_response = requests.post(login_url, json=user_request) if login_response.status_code == 401: user_dict = newHabiticaUser(username, email, password, our_uuid) else: logging.debug("habitica http response from login = %s" % login_response) user_auth = json.loads(login_response.text) logging.debug("parsed json from habitica has keys = %s" % user_auth) #login only returns user auth headers, so now get authenticated user and put it in user_dict auth_headers = { 'x-api-user': user_auth['data']['id'], 'x-api-key': user_auth['data']['apiToken'] } get_user_url = url + '/api/v3/user' result = requests.request('GET', get_user_url, headers=auth_headers, json={}) logging.debug("result = %s" % result) result.raise_for_status() user_dict = result.json() user_dict['data']['apiToken'] = user_auth['data']['apiToken'] logging.debug("parsed json from GET habitica user = %s" % user_dict) #if it fails, then user is also not in Habitica, so needs to create new account and put it in user_dict #FIX!! throw except only if u returns a 401 error except: pass logging.debug("habitica user to be created in our db = %s" % user_dict['data']) #Now save new user (user_dict) to our db #Since we are randomly generating the password, we store it in case users #want to access their Habitica account from the browser #Need to create a way from them to retrieve username/password #metrics_data is used to calculate points based on km biked/walked #last_timestamp is the last time the user got points, and bike/walk_count are the leftover km habitica_user_table = edb.get_habitica_db() habitica_user_table.insert({ 'user_id': our_uuid, 'metrics_data': { 'last_timestamp': arrow.utcnow().timestamp, 'bike_count': 0, 'walk_count': 0 }, 'habitica_username': username, 'habitica_password': password, 'habitica_id': user_dict['data']['id'], 'habitica_token': user_dict['data']['apiToken'], 'habitica_group_id': None }) #Since we have a new user in our db, create its default habits (walk, bike) setup_default_habits(our_uuid) #And invite new user to a group, or retrieve group id if user is already in one group_id = setup_party(our_uuid) edb.get_habitica_db().update({"user_id": our_uuid}, {"$set": { 'habitica_group_id': group_id }}, upsert=True) user_dict['habitica_group_id'] = group_id return user_dict
def export_to_cve(self, filepath, version, mode, next_identifier): """ Export flaw (self) to CVE JSON format in filepath :param filepath string, full path of the destiny file :param version int, version of CVE JSON, only 4 supported for now :param mode string, public, reserved or reject :returns None """ if mode != "public": raise NotImplementedError if version == 4: file = open(filepath, "w") file.write("{\n") # CVE_data_meta file.write(' "CVE_data_meta": {\n') file.write(' "ASSIGNER": "*****@*****.**",\n') file.write( ' "DATE_PUBLIC": "' + str(arrow.utcnow().format("YYYY-MM-DDTHH:mm:ss ZZ")) + '",\n' ) file.write(' "ID": "' + str(next_identifier) + '",\n') file.write(' "STATE": "PUBLIC",\n') file.write(' "TITLE": "' + str(self.title) + '"\n') file.write(" },\n") # affects file.write(' "affects": {\n') file.write(' "vendor": {\n') file.write(' "vendor_data": [\n') file.write(" {\n") file.write(' "product": {\n') file.write(' "product_data": [\n') file.write(" {\n") file.write( ' "product_name": "' + str( self.system.split(",")[0].split(":")[0] ) # pick only one element for now, TODO: redo + '",\n' ) file.write(' "version": {\n') file.write(' "version_data": [\n') file.write(" {\n") if ":" in self.system.split(",")[0]: file.write( ' "version_value": "' + str(self.system.split(",")[0].split(":")[1]) + '"\n' ) else: file.write( ' "version_value": ""\n' ) file.write(" }\n") file.write(" ]\n") file.write(" }\n") file.write(" }\n") file.write(" ]\n") file.write(" },\n") file.write( ' "vendor_name": "' + str(self.vendor.split(",")[0]) + '"\n' ) file.write(" }\n") file.write(" ]\n") file.write(" }\n") file.write(" },\n") # credit file.write(' "credit": [\n') file.write(" {\n") file.write(' "lang": "eng",\n') file.write(' "value": "' + str(self.detected_by) + '"\n') file.write(" }\n") file.write(" ],\n") # format file.write(' "data_format": "MITRE",\n') file.write(' "data_type": "CVE",\n') file.write(' "data_version": "4.0",\n') # description file.write(' "description": {\n') file.write(' "description_data": [\n') file.write(" {\n") file.write(' "lang": "eng",\n') file.write(' "value": "' + str(self.description) + '"\n') file.write(" }\n") file.write(" ]\n") file.write(" },\n") # generator file.write(' "generator": {\n') file.write(' "engine": "Robot Vulnerability Database (RVD)"\n') file.write(" },\n") # impact file.write(' "impact": {\n') file.write(' "cvss": {\n') file.write( ' "attackComplexity": "' + str(self.cvss_vector_extract(self.cvss_vector, "AC")) + '",\n' ) file.write( ' "attackVector": "' + str(self.cvss_vector_extract(self.cvss_vector, "AV")) + '",\n' ) file.write( ' "availabilityImpact": "' + str(self.cvss_vector_extract(self.cvss_vector, "A")) + '",\n' ) file.write(' "baseScore": ' + str(self.cvss_score) + ",\n") file.write( ' "baseSeverity": "' + str(self.cvss_vector_extract(self.cvss_vector, "severity")) + '",\n' ) file.write( ' "confidentialityImpact": "' + str(self.cvss_vector_extract(self.cvss_vector, "C")) + '",\n' ) file.write( ' "integrityImpact": "' + str(self.cvss_vector_extract(self.cvss_vector, "I")) + '",\n' ) file.write( ' "privilegesRequired": "' + str(self.cvss_vector_extract(self.cvss_vector, "PR")) + '",\n' ) file.write( ' "scope": "' + str(self.cvss_vector_extract(self.cvss_vector, "S")) + '",\n' ) file.write( ' "userInteraction": "' + str(self.cvss_vector_extract(self.cvss_vector, "UI")) + '",\n' ) file.write(' "vectorString": "' + str(self.cvss_vector) + '",\n') file.write(' "version": "3.0"\n') file.write(" }\n") file.write(" },\n") # problem-type file.write(' "problemtype": {\n') file.write(' "problemtype_data": [\n') file.write(" {\n") file.write(' "description": [\n') file.write(" {\n") file.write(' "lang": "eng",\n') file.write(' "value": "' + str(self.cwe) + '"\n') file.write(" }\n") file.write(" ]\n") file.write(" }\n") file.write(" ]\n") file.write(" },\n") # references file.write(' "references": {\n') for i in range(len(self.links)): file.write(' "reference_data": [\n') file.write(" {\n") file.write(' "name": "' + str(self.links[i]) + '",\n') file.write(' "refsource": "CONFIRM",\n') file.write(' "url": "' + str(self.links[i]) + '"\n') file.write(" }\n") if i == (len(self.links) - 1): file.write(" ]\n") else: file.write(" ],\n") file.write(" },\n") # source file.write(' "source": {\n') file.write(' "defect": [\n') file.write(' "RVD#' + str(self.id) + '"\n') file.write(" ],\n") file.write(' "discovery": "EXTERNAL"\n') file.write(" }\n") # end file.write("}\n") file.close() else: raise NotImplementedError
def requires(self): start = arrow.get('20090601', 'YYYYMMDD').ceil('week') end = arrow.utcnow().ceil('week') for batch in arrow.Arrow.range('week', start, end): batch_file = join(SPL_BATCH_DIR, batch.format('YYYYMMDD') + '.ids') yield SPL2JSON(batch_file)
def utcnow_datetime_aware(): """Returns timezone-aware datetime for the current UTC moment""" return arrow.utcnow().datetime
def scanfile(infile, ovrx={}, mforce=False): """ Read audio file attributes, tags, and metadata for @infile """ dasc = {} # get file parts xvreal = infile tdir, xv = os.path.split(xvreal) xvbase, xvext = os.path.splitext(xv) # get base & parent dir names tdir_base = os.path.split(tdir)[1] tdir_parent = os.path.split(os.path.split(tdir)[0])[1] logthis("Examining file:", suffix=xv, loglevel=LL.INFO) tstatus('scanfile', event='start', filename=xv) # Get xattribs fovr = {} fovr.update(ovrx) fovr.update(parse_xattr_overrides(xvreal)) if 'ignore' in fovr: logthis("File has 'ignore' flag set via override; skipping", loglevel=LL.INFO) return None # Get file path information dasc['dpath'] = {'base': tdir_base, 'parent': tdir_parent, 'full': tdir} dasc['fpath'] = { 'real': xvreal, 'base': xvbase, 'file': xv, 'ext': xvext.replace('.', '') } # Stat, Extended Attribs, Ownership dasc['stat'] = util.dstat(xvreal) dasc['owner'] = { 'user': util.getuser(dasc['stat']['uid']), 'group': util.getgroup(dasc['stat']['gid']) } # Modification key (MD5 of inode number + mtime + filesize) mkey_id = util.getmkey(dasc['stat']) dasc['mkey_id'] = mkey_id dasc['status'] = 'new' # Get mediainfo & build codec string minfo = util.mediainfo(xvreal, config, format_lower=False) try: if minfo['audio'][0]['format'] == "mpeg audio": if minfo['audio'][0]['format'].endswith('3'): acodec = "MP3" elif minfo['audio'][0]['format'].endswith('2'): acodec = "MP2" else: acodec = "MPEG Audio" if 'format_profile' in minfo['audio'][0]: acodec += " " + minfo['audio'][0]['format_profile'] else: acodec = "" if minfo['general']['format'] != minfo['audio'][0]['format']: acodec = minfo['general']['format'] + " " acodec += minfo['audio'][0]['format'] if 'format_profile' in minfo['audio'][0]: acodec += " " + minfo['audio'][0]['format_profile'] except Exception as e: logexc(e, "Failed to determine codec") acodec = "Unknown" try: if 'bit_depth' in minfo['audio'][0]: bitdepth = minfo['audio'][0]['bit_depth'] else: bitdepth = 16 except Exception as e: logexc(e, "Failed to determine bit depth; assuming default 16-bit") bitdepth = 16 # Open file with Mutagen try: mf = mutagen.File(xvreal) except OSError as e: logexc(e, "Failed to open file") return None except Exception as e: logexc(e, "Failed to parse tag data from file") return None # Determine MIME type try: amime = mf.mime[0] except: try: amime = mimetypes.guess_type(xvreal)[0] except: amime = None if amime is None: logthis("Failed to auto-detect MIME type for file:", suffix=xvreal, loglevel=LL.WARNING) amime = "audio/x-" + xvext.lower() logthis("Detected MIME-Type:", suffix=amime, loglevel=LL.DEBUG) # Set default subsong info dasc['subsong'] = { 'index': 1, 'start_time': 0.0, 'duration': None, 'cue': None } # Check for tag data tags = mf.tags if tags: tdate = parse_album_date(tags) if tdate is not None: tyear = tdate.format("YYYY") tstamp = tdate.timestamp else: tyear = None tstamp = None dasc['tags'] = { 'artist': get_best_tag(tags, ('ARTIST', 'TPE1', 'TOPE', 'Author')), 'album': get_best_tag(tags, ('ALBUM', 'TALB', 'TOAL', 'WM/AlbumTitle')), 'title': get_best_tag(tags, ('TITLE', 'TIT2', 'TIT3', 'Title')), 'year': tyear, 'timestamp': tstamp, 'genre': get_best_tag(tags, ('GENRE', 'TCON', 'WM/Genre')), 'tracknum': parse_tracknum(tags, forceInt=True), 'trackstr': parse_tracknum(tags, forceInt=False), 'disc': get_best_tag(tags, ('DISCNUMBER', 'DISC', 'TPOS')), 'album_artist': get_best_tag(tags, ('ALBUMARTIST', 'TXXX:ALBUM ARTIST', 'TXXX:ALBUM_ARTIST', 'WM/AlbumArtist')) } dasc['alltags'] = get_all_tags_once(tags) dasc['format'] = { 'format': acodec, 'mime': amime, 'channels': get_info_safe(mf.info, 'channels', 2), 'sampling_rate': get_info_safe(mf.info, 'sample_rate', 44100), 'encoding_settings': minfo['audio'][0].get('encoding_settings'), 'writing_library': minfo['audio'][0].get('writing_library'), 'bitrate': get_info_safe(mf.info, 'bitrate', 1411000), 'bit_depth': bitdepth, 'length': get_info_safe(mf.info, 'length') } if dasc['tags']['album_artist'] is None: dasc['tags']['album_artist'] = dasc['tags']['artist'] else: dasc['tags'] = {} dasc['alltags'] = {} dasc['format'] = {} logthis("** NO DATA **", loglevel=LL.WARNING) # Record last time this entry was updated (UTC) last_up = arrow.utcnow().timestamp logthis("last_updated =", suffix=last_up, loglevel=LL.DEBUG) dasc['last_updated'] = last_up # Check for matching cue file... subsongs = [] cuepath = dasc['dpath']['full'] + '/' + dasc['fpath']['base'] + '.cue' if os.path.exists(cuepath): cue = parse_cue_file(cuepath) if cue: # Set subsong CUE sheet source file dasc['subsong']['cue'] = dasc['fpath']['base'] + '.cue' dasc['subsong']['cue_realpath'] = cuepath logthis("Using CUE sheet for subsong indexing", suffix=dasc['fpath']['base'] + '.cue', loglevel=LL.VERBOSE) for ssindex, ssdata in cue['tracks'].items(): try: if cue['tracks'].get(ssindex + 1): ssduration = cue['tracks'][ssindex + 1]['index'][1][ 0] - cue['tracks'][ssindex]['index'][1][0] else: ssduration = dasc['format']['length'] - cue['tracks'][ ssindex]['index'][1][0] except Exception as e: logexc(e, "Failed to calculate subsong duration") ssduration = 0.0 tsubsong = clone_master_track(dasc, ssdata, ssindex, ssduration) logthis( u"[SUBSONG] Title: {title} / Track: {tracknum} ({trackstr}) / Artist: {artist} / Album: {album} / AlbumArtist: {album_artist} / Year: {year}" .format(**tsubsong['tags']), loglevel=LL.DEBUG) subsongs.append(tsubsong) if len(subsongs) == 0: logthis( u"Title: {title} / Track: {tracknum} ({trackstr}) / Artist: {artist} / Album: {album} / AlbumArtist: {album_artist} / Year: {year}" .format(**dasc['tags']), loglevel=LL.DEBUG) return dasc else: return subsongs
def scheduler(broker=None): """ Creates a task from a schedule at the scheduled time and schedules next run """ if not broker: broker = get_broker() db.close_old_connections() try: for s in Schedule.objects.exclude(repeats=0).filter( next_run__lt=timezone.now()): args = () kwargs = {} # get args, kwargs and hook if s.kwargs: try: # eval should be safe here because dict() kwargs = eval('dict({})'.format(s.kwargs)) except SyntaxError: kwargs = {} if s.args: args = ast.literal_eval(s.args) # single value won't eval to tuple, so: if type(args) != tuple: args = (args, ) q_options = kwargs.get('q_options', {}) if s.hook: q_options['hook'] = s.hook # set up the next run time if not s.schedule_type == s.ONCE: next_run = arrow.get(s.next_run) while True: if s.schedule_type == s.MINUTES: next_run = next_run.replace(minutes=+(s.minutes or 1)) elif s.schedule_type == s.HOURLY: next_run = next_run.replace(hours=+1) elif s.schedule_type == s.DAILY: next_run = next_run.replace(days=+1) elif s.schedule_type == s.WEEKLY: next_run = next_run.replace(weeks=+1) elif s.schedule_type == s.MONTHLY: next_run = next_run.replace(months=+1) elif s.schedule_type == s.QUARTERLY: next_run = next_run.replace(months=+3) elif s.schedule_type == s.YEARLY: next_run = next_run.replace(years=+1) if Conf.CATCH_UP or next_run > arrow.utcnow(): break s.next_run = next_run.datetime s.repeats += -1 # send it to the cluster q_options['broker'] = broker q_options['group'] = q_options.get('group', s.name or s.id) kwargs['q_options'] = q_options s.task = django_q.tasks.async_task(s.func, *args, **kwargs) # log it if not s.task: logger.error( _('{} failed to create a task from schedule [{}]').format( current_process().name, s.name or s.id)) else: logger.info( _('{} created a task from schedule [{}]').format( current_process().name, s.name or s.id)) # default behavior is to delete a ONCE schedule if s.schedule_type == s.ONCE: if s.repeats < 0: s.delete() continue # but not if it has a positive repeats s.repeats = 0 # save the schedule s.save() except Exception as e: logger.error(e)
async def givecookie(cmd, message, args): if message.mentions: target = message.mentions[0] author_stamp = arrow.get(message.author.created_at).float_timestamp current_stamp = arrow.utcnow().float_timestamp time_diff = current_stamp - author_stamp if time_diff > 2592000: if message.author.id != target.id: if not target.bot: if not cmd.bot.cool_down.on_cooldown( cmd.name, message.author): upgrade_file = cmd.db[ cmd.db.db_cfg.database].Upgrades.find_one( {'UserID': message.author.id}) if upgrade_file is None: cmd.db[cmd.db.db_cfg.database].Upgrades.insert_one( {'UserID': message.author.id}) upgrade_file = {} cookie_coll = cmd.db[cmd.db.db_cfg.database].Cookies base_cooldown = 3600 if 'stamina' in upgrade_file: stamina = upgrade_file['stamina'] else: stamina = 0 cooldown = int(base_cooldown - ((base_cooldown / 100) * (stamina * 0.2))) file_check = cookie_coll.find_one( {'UserID': target.id}) if not file_check: cookies = 0 data = {'UserID': target.id, 'Cookies': 0} cookie_coll.insert_one(data) else: cookies = file_check['Cookies'] cookies += 1 cookie_coll.update_one({'UserID': target.id}, {'$set': { 'Cookies': cookies }}) cmd.bot.cool_down.set_cooldown(cmd.name, message.author, cooldown) title = f'🍪 You gave a cookie to {target.display_name}.' response = discord.Embed(color=0xd99e82, title=title) else: timeout_seconds = cmd.bot.cool_down.get_cooldown( cmd.name, message.author) if timeout_seconds > 60: timeout_seconds = arrow.utcnow( ).timestamp + timeout_seconds timeout = arrow.get(timeout_seconds).humanize() else: timeout = f'in {timeout_seconds} seconds' timeout_title = f'🕙 You can give another cookie {timeout}.' response = discord.Embed(color=0x696969, title=timeout_title) else: response = discord.Embed( color=0xBE1931, title=f'❗ Bots don\'t eat cookies.') else: response = discord.Embed( color=0xBE1931, title=f'❗ Nope, can\'t give cookies to yourself.') else: response = discord.Embed( color=0xBE1931, title=f'❗ Sorry, your account is too young to give cookies.') else: response = discord.Embed(color=0xBE1931, title=f'❗ No user targeted.') await message.channel.send(embed=response)
def _searchdate(self, timezone, arrow=None): if not arrow: arrow = utcnow() return arrow.to(timezone).format('YYYY-MM-DD')