def get_all_pull_requests(r): '''Gets all pull requests for the repo r, regardless of whether they are open or closed''' for p in r.get_pulls(state='all'): created_at_ist = p.created_at.replace(tzinfo=tzutc()).astimezone(tzlocal()).ctime() if p.closed_at is not None: closed_at_ist = p.closed_at.replace(tzinfo=tzutc()).astimezone(tzlocal()).ctime() else: closed_at_ist = None pull_requests[p.id] = { 'user': p.user.name, 'username' : p.user.login, 'created_at': created_at_ist, 'closed_at': closed_at_ist, 'additions':p.additions, 'deletions':p.deletions, 'changed_files': p.changed_files, 'review_comments' : p.review_comments, 'merged' : p.is_merged() } if p.user.name is None or p.user.name.strip() == '': pull_requests[p.id]['user'] = p.user.login return pull_requests
def process_bind_param(self, value, engine): if value is not None: if value.tzinfo is None: # TODO: do we want to assume that unqualified datetimes are UTC? return value.replace(tzinfo=tzutc()) else: return value.astimezone(tzutc())
def test_verify_certificate_with_invalid_date_range(self): start = datetime.utcnow() + relativedelta(months=2) start = start.replace(tzinfo=tzutc()) environ = self.make_environ( '/C=MX', '/C=MX', start=start ) assert not verify_certificate( environ, 'SSL_CLIENT_VERIFY', 'SSL_CLIENT_V_START', 'SSL_CLIENT_V_END' ) end = datetime.utcnow() + relativedelta(days=-5) end = end.replace(tzinfo=tzutc()) environ = self.make_environ( '/C=MX', '/C=MX', end=end ) assert not verify_certificate( environ, 'SSL_CLIENT_VERIFY', 'SSL_CLIENT_V_START', 'SSL_CLIENT_V_END' )
def _make_offset_aware(self, mapping=None): # root if mapping is None: mapping = self naive_types = (datetime.datetime, datetime.time) for key, value in mapping.iteritems(): new_key, new_value = key, value # timezone naive key if isinstance(key, naive_types) and key.tzinfo is None: new_key = key.replace(tzinfo=tzutc()) del mapping[key] # timezone naive value if isinstance(value, naive_types) and value.tzinfo is None: new_value = value.replace(tzinfo=tzutc()) # list elif isinstance(value, list) and len(value) and \ any([isinstance(v, naive_types) for v in value]): new_value = list() for v in value: if isinstance(v, naive_types): v = v.replace(tzinfo=tzutc()) new_value.append(v) # recursive value elif isinstance(value, dict): new_value = self._make_offset_aware(value) mapping[new_key] = new_value return mapping
def test_is_rhic_lookup_task_expired(self): # Create a valid, in_progress task task_a = RHICLookupTask(uuid="11a1aa11-a11a-1a11-111a-a11111111111", completed=False) task_a.save() self.assertFalse(identity.is_rhic_lookup_task_expired(task_a)) # Create a completed task task_b = RHICLookupTask(uuid="11a1aa11-a11a-1a11-111a-a22222222222", completed=True, initiated=datetime.now(tzutc()), modified=datetime.now(tzutc())) task_b.save() self.assertFalse(identity.is_rhic_lookup_task_expired(task_b)) # Create a timedout incomplete task cfg = config.get_rhic_serve_config_info() timeout_in_minutes = cfg["single_rhic_lookup_timeout_in_minutes"] expired_time = datetime.now(tzutc()) - timedelta(minutes=timeout_in_minutes+1) task_c = RHICLookupTask(uuid="11a1aa11-a11a-1a11-111a-a333333333333", completed=False, initiated=expired_time) task_c.save() self.assertTrue(identity.is_rhic_lookup_task_expired(task_c)) # Create a completed expired task expired_hours = cfg["single_rhic_lookup_cache_unknown_in_hours"] expired_time = datetime.now(tzutc()) - timedelta(hours=expired_hours+1) task_d = RHICLookupTask(uuid="11a1aa11-a11a-1a11-111a-a444444444444", completed=True, modified=expired_time) task_d.save() self.assertTrue(identity.is_rhic_lookup_task_expired(task_d))
def test_purge_expired_rhic_lookups(self): cfg = config.get_rhic_serve_config_info() # Create a valid, in_progress task task_a = RHICLookupTask(uuid="11a1aa11-a11a-1a11-111a-a11111111111", completed=False) task_a.save() # Create a completed task task_b = RHICLookupTask(uuid="11a1aa11-a11a-1a11-111a-a22222222222", completed=True, initiated=datetime.now(tzutc()), modified=datetime.now(tzutc())) task_b.save() # Create a timedout incomplete task timeout_in_minutes = cfg["single_rhic_lookup_timeout_in_minutes"] expired_time = datetime.now(tzutc()) - timedelta(minutes=timeout_in_minutes+1) task_c = RHICLookupTask(uuid="11a1aa11-a11a-1a11-111a-a333333333333", completed=False, initiated=expired_time) task_c.save() # Create a completed expired task expired_hours = cfg["single_rhic_lookup_cache_unknown_in_hours"] expired_time = datetime.now(tzutc()) - timedelta(hours=expired_hours+1) task_d = RHICLookupTask(uuid="11a1aa11-a11a-1a11-111a-a444444444444", completed=True, modified=expired_time) task_d.save() identity.purge_expired_rhic_lookups() found = RHICLookupTask.objects() self.assertEquals(len(found), 2) for f in found: self.assertTrue(f.uuid in [task_a.uuid, task_b.uuid]) self.assertTrue(f.uuid not in [task_c.uuid, task_d.uuid])
def process_value_type(self, sentinel, value): if self.vtype == 'normalize': return sentinel, value.strip().lower() elif self.vtype == 'integer': try: v = int(value.strip()) except ValueError: v = 0 elif self.vtype == 'size': try: return sentinel, len(value) except TypeError: return sentinel, 0 elif self.vtype == 'age': if not isinstance(sentinel, datetime): sentinel = datetime.now(tz=tzutc()) - timedelta(sentinel) if not isinstance(value, datetime): value = parse(value) # Reverse the age comparison, we want to compare the value being # greater than the sentinel typically. Else the syntax for age # comparisons is intuitively wrong. return value, sentinel # Allows for expiration filtering, for events in the future as opposed # to events in the past which age filtering allows for. elif self.vtype == 'expiration': if not isinstance(sentinel, datetime): sentinel = datetime.now(tz=tzutc()) + timedelta(sentinel) if not isinstance(value, datetime): value = parse(value) return sentinel, value
def _parse_tzstr(self, tzstr, zero_as_utc=True): if tzstr == b'Z' or tzstr == b'z': return tz.tzutc() if len(tzstr) not in {3, 5, 6}: raise ValueError('Time zone offset must be 1, 3, 5 or 6 characters') if tzstr[0:1] == b'-': mult = -1 elif tzstr[0:1] == b'+': mult = 1 else: raise ValueError('Time zone offset requires sign') hours = int(tzstr[1:3]) if len(tzstr) == 3: minutes = 0 else: minutes = int(tzstr[(4 if tzstr[3:4] == self._TIME_SEP else 3):]) if zero_as_utc and hours == 0 and minutes == 0: return tz.tzutc() else: if minutes > 59: raise ValueError('Invalid minutes in time zone offset') if hours > 23: raise ValueError('Invalid hours in time zone offset') return tz.tzoffset(None, mult * (hours * 60 + minutes) * 60)
def is_rhic_lookup_task_expired(current_task): cfg = config.get_rhic_serve_config_info() if not current_task.completed: # Task is in progress, ensure that it's initiated time is within timeout range timeout_in_minutes = cfg["single_rhic_lookup_timeout_in_minutes"] threshold = current_task.initiated + timedelta(minutes=timeout_in_minutes) if not threshold.tzinfo: threshold = pytz.UTC.localize(threshold) if threshold < datetime.now(tzutc()): _LOG.info("Task has timed out, threshold was: %s. Task = <%s>" % (threshold, current_task)) # Current time is greater than the threshold this task had to stay alive # It is expired return True else: # Task has completed, check if it's within cached time boundaries valid_hours = cfg["single_rhic_lookup_cache_unknown_in_hours"] modified = current_task.modified if not modified.tzinfo: modified = pytz.UTC.localize(modified) threshold = datetime.now(tzutc()) - timedelta(hours=valid_hours) if modified < threshold: _LOG.info("Cached task has expired, threshold was: %s. Task = <%s>" % (threshold, current_task)) # Task was modified more than # hours ago # It is expired return True return False
def save(cls, data): """ Save an entity in DB. :param dict data: Item from the 'data' section of the dictionary that was received from the Facebook API request /posts/ :return: Nothing """ record = cls.get_or_insert( data["id"], from_ = data.get("from", {}), to = data.get("to", {}), message = data.get("message", ""), message_tags = data.get("message_tags", {}), picture = data.get("picture", ""), link = data.get("link", ""), actions = data.get("actions", {}), type = data.get("type", ""), status_type = data.get("status_type", ""), # gae date issue solving: # http://hype-free.blogspot.ru/2013/02/converting-datetime-to-utc-in-python.html created_time = parse(data["created_time"]).astimezone(tz.tzutc()).replace(tzinfo=None), updated_time = parse(data["updated_time"]).astimezone(tz.tzutc()).replace(tzinfo=None), shares = data.get("shares", {}), likes = data.get("likes", {}), ) return None
def get_replication_info(self, instance): if self.check_instance_is_master(instance=instance): return 0 with self.pymongo() as client: replSetGetStatus = client.admin.command('replSetGetStatus') primary_opttime = None for member in replSetGetStatus['members']: if member['stateStr'] == 'PRIMARY': primary_opttime = member['optimeDate'].replace(tzinfo=tz.tzutc()).astimezone(tz.tzlocal()) if primary_opttime is None: raise Exception("There is not any Primary in the Replica Set") instance_opttime = None for member in replSetGetStatus['members']: if member["name"] == "{}:{}".format(instance.address, instance.port): instance_opttime = member['optimeDate'].replace(tzinfo=tz.tzutc()).astimezone(tz.tzlocal()) instance_member = member if instance_opttime is None: raise Exception("Could not find the instance in the Replica Set") delay = primary_opttime - instance_opttime seconds_delay = delay.days * 24 * 3600 + delay.seconds LOG.info("The instance {} is {} seconds behind Primary".format(instance, seconds_delay)) if seconds_delay == 0 and instance_member["stateStr"] not in ["PRIMARY", "SECONDARY"]: LOG.info("The instance {} is 0 seconds behind Primary, but it is not Secondary. It is {}".format(instance, instance_member["stateStr"])) return 100000 return seconds_delay
def _parse(self, text): lines = text.split("\n") issued = datetime.strptime(lines[3], "%I%M %p %Z %a %b %d %Y") if 'CST' in lines[3]: self.issued = issued.replace(tzinfo=SPCSWO._CST).astimezone(tzutc()) elif 'CDT'in lines[3]: self.issued = issued.replace(tzinfo=SPCSWO._CDT).astimezone(tzutc()) match = re.search(r"([\d]{6})Z \- ([\d]{6})Z", lines[5]) valid_start_str, valid_end_str = match.groups() valid_start = datetime.strptime(valid_start_str, "%d%H%M").replace(tzinfo=tzutc()) valid_end = datetime.strptime(valid_end_str, "%d%H%M").replace(tzinfo=tzutc()) if valid_end < valid_start: valid_end += ((valid_start - valid_end) + timedelta(days=1)) valid_len = valid_end - valid_start self.valid_start = valid_start.replace(year=issued.year, month=issued.month) self.valid_end = self.valid_start + valid_len products = re.findall(r"\.\.\. ([A-Z ]+) \.\.\.", text) prods = {} for prod in products: match = re.search("\.\.\. %s \.\.\.([\w\d\s\.]+)\&\&" % prod, text, re.S) cont_str = match.groups()[0].strip() prods[prod] = SPCSWOContours(prod, cont_str, self._conus) return prods
def as_dict(self): #time_fmt='%c' time_utc = "%Y-%m-%d %H:%M:%S %Z%z" d = { 'created':self.created.replace(tzinfo=tz.tzutc()).strftime(time_utc), 'last_modified':self.last_modified.replace(tzinfo=tz.tzutc()).strftime(time_utc) }
def test_start_end(self): """ LogFile: test .start and .end property work correctly """ logfile = LogFile(self.file_year_rollover) assert logfile.start == datetime(2013, 12, 30, 00, 13, 01, 661000, tzutc()) assert logfile.end == datetime(2014, 01, 02, 23, 27, 11, 720000, tzutc())
def _assert_default_entity(self, entity): ''' Asserts that the entity passed in matches the default entity. ''' self.assertEqual(entity.age, 39) self.assertEqual(entity.sex, 'male') self.assertEqual(entity.name, 'John Doe') self.assertEqual(entity.married, True) self.assertEqual(entity.deceased, False) self.assertFalse(hasattr(entity, "optional")) self.assertFalse(hasattr(entity, "aquarius")) self.assertEqual(entity.ratio, 3.1) self.assertEqual(entity.evenratio, 3.0) self.assertEqual(entity.large, 933311100) self.assertEqual(entity.Birthday, datetime(1973, 10, 4, tzinfo=tzutc())) self.assertEqual(entity.birthday, datetime(1970, 10, 4, tzinfo=tzutc())) self.assertIsInstance(entity.binary, EntityProperty) self.assertEqual(entity.binary.type, EdmType.BINARY) self.assertEqual(entity.binary.value, b'binary') self.assertIsInstance(entity.other, EntityProperty) self.assertEqual(entity.other.type, EdmType.INT32) self.assertEqual(entity.other.value, 20) self.assertIsInstance(entity.clsid, EntityProperty) self.assertEqual(entity.clsid.type, EdmType.GUID) self.assertEqual(entity.clsid.value, 'c9da6455-213d-42c9-9a79-3e9149a57833') self.assertTrue(hasattr(entity, "Timestamp")) self.assertIsInstance(entity.Timestamp, datetime) self.assertIsNotNone(entity.etag)
def get_docs_from_xml(self, root): docs = [] for channel in root: for item in channel.findall("item"): new_doc = Document() new_doc.title = item.find("title").text or "" new_doc.download_date = datetime.now(tz.tzutc()) new_doc.publish_date = dateparser.parse(item.find("pubDate").text, "") or new_doc.download_date if new_doc.publish_date.tzinfo is None or self.force_timezone: new_doc.publish_date=new_doc.publish_date.replace(tzinfo=self.timezone) new_doc.publish_date = new_doc.publish_date.astimezone(tz.tzutc()) new_doc.source_url = item.find("link").text or "" new_doc.original_summary = strip_html(item.find("description").text or "") if item.find("guid"): new_doc.guid = hashlib.md5(item.find("guid").encode('utf-8')).hexdigest() else: new_doc.guid = hashlib.md5(new_doc.source_url.encode('utf-8')).hexdigest() new_doc.provider = self.name if new_doc.guid not in self.processed_guids: self.processed_guids[new_doc.guid] = True self.document_count += 1 docs.append(new_doc) return docs
def datetime_to_serial_date(dt): """ Convert datetime.datetime object to a serial date. """ if dt.tzname() is None: dt = dt.replace(tzinfo=tz.tzutc()) origin = datetime.datetime(1899, 12, 30, tzinfo=tz.tzutc()) delta = dt - origin return float(delta.days) + (float(delta.seconds) / 86400.0)
def setup(self): """ get start end end date of logfile before starting to parse. """ if self.mlogfilter.is_stdin: # assume this year (we have no other info) now = datetime.now() self.startDateTime = datetime(now.year, 1, 1, tzinfo=tzutc()) self.endDateTime = datetime(MAXYEAR, 12, 31, tzinfo=tzutc()) else: logfiles = self.mlogfilter.args['logfile'] self.startDateTime = min([lf.start+timedelta(hours=self.mlogfilter.args['timezone'][i]) for i, lf in enumerate(logfiles)]) self.endDateTime = max([lf.end+timedelta(hours=self.mlogfilter.args['timezone'][i]) for i, lf in enumerate(logfiles)]) # now parse for further changes to from and to datetimes dtbound = DateTimeBoundaries(self.startDateTime, self.endDateTime) self.fromDateTime, self.toDateTime = dtbound(self.mlogfilter.args['from'] or None, self.mlogfilter.args['to'] or None) # define start_limit for mlogfilter's fast_forward method self.start_limit = self.fromDateTime # for single logfile, get file seek position of `to` datetime if len(self.mlogfilter.args['logfile']) == 1 and not self.mlogfilter.is_stdin: if self.mlogfilter.args['to'] != "end": # fast forward, get seek value, then reset file logfile = self.mlogfilter.args['logfile'][0] logfile.fast_forward(self.toDateTime) self.seek_to = logfile.filehandle.tell() logfile.filehandle.seek(0) else: self.seek_to = -1 else: self.seek_to = False
def iso8601_to_rostime(iso): """Converts ISO 8601 time to ROS Time. Args: iso: ISO 8601 encoded string. Returns: std_msgs/Time. """ # Convert to datetime in UTC. t = dateutil.parser.parse(iso) if not t.utcoffset(): t = t.replace(tzinfo=tzutc()) # Convert to time from epoch in UTC. epoch = datetime.utcfromtimestamp(0) epoch = epoch.replace(tzinfo=tzutc()) dt = t - epoch # Create ROS message. time = Time() time.data.secs = int(dt.total_seconds()) time.data.nsecs = dt.microseconds * 1000 return time
def test_check_for_alert_with_no_matched_zones(self): """Alerts are only valid if the availability zone in the history matches an availability zone in the subscription""" history = [{u'Timestamp': datetime.datetime(2015, 12, 31, 22, 13, 43, tzinfo=tzutc()), u'ProductDescription': 'Windows', u'InstanceType': 'g2.2xlarge', u'SpotPrice': '0.105200', u'AvailabilityZone': 'us-east-1d'}, {u'Timestamp': datetime.datetime(2015, 12, 31, 21, 56, 18, tzinfo=tzutc()), u'ProductDescription': 'Windows', u'InstanceType': 'g2.2xlarge', u'SpotPrice': '0.104400', u'AvailabilityZone': 'us-east-1d'}] just_1a = {'name': 'Just 1a', 'threshold':'0.05', 'region':'us-east-1', 'zone': 'us-east-1a', 'instance_type':'g2.2xlarge', 'product':'Windows', 'user':'******', 'last_alert':'Under'} result = alert.check_for_alert(history, just_1a) assert result is None, 'There should not be an alert for Just 1a'
def test_log_new_events_prints_correct_event(self, mock_describe_events): mock_describe_events.return_value = { "StackEvents": [ { "Timestamp": datetime.datetime( 2016, 3, 15, 14, 2, 0, 0, tzinfo=tzutc() ), "LogicalResourceId": "id-2", "ResourceType": "type-2", "ResourceStatus": "resource-status" }, { "Timestamp": datetime.datetime( 2016, 3, 15, 14, 1, 0, 0, tzinfo=tzutc() ), "LogicalResourceId": "id-1", "ResourceType": "type-1", "ResourceStatus": "resource", "ResourceStatusReason": "User Initiated" } ] } self.stack.most_recent_event_datetime = ( datetime.datetime(2016, 3, 15, 14, 0, 0, 0, tzinfo=tzutc()) ) self.stack._log_new_events()
def arrow(date=None, tz=None): def _tz_now(tzinfo): date = datetime.utcnow() date = date.replace(tzinfo=_tz.tzutc()) return date.astimezone(tzinfo) date_expr = None tz_expr = None if date is None: if tz is None: date_expr = datetime.utcnow() tz_expr = TimeZone(_tz.tzutc()) else: tz_expr = TimeZone(tz) date_expr = _tz_now(tz_expr.tzinfo) else: if tz is None: try: tz_expr = TimeZone(date) date_expr = _tz_now(tz_expr.tzinfo) except: date_expr = date tz_expr = TimeZone(_tz.tzutc()) else: date_expr = date tz_expr = tz return Arrow(date_expr, tz_expr)
def create_call(number,time,*args): timezone = args[0] if len(args) > 0 and len(args[0]) > 2 else None message = args[1] if len(args) > 1 else None call = Call(None) number_obj = Number(number) now = datetime.datetime.utcnow().replace(tzinfo=tz.tzutc()) if not number_obj.exists(): number_obj.create() if timezone: number_obj.set("tz",timezone) dt = parse(time) if dt.tzinfo == None: try: stored_timezone = tz.gettz(number_obj.get("tz")) if tz.gettz(number_obj.get("tz")) else tz.gettz(timezone) dt = dt.replace(tzinfo=stored_timezone) except TypeError: dt = dt.replace(tzinfo=tz.gettz(timezone)) else: dt = dt.replace(tzinfo=tz.gettz(timezone)) if dt.tzinfo == None: dt = dt.replace(tzinfo=tz.tzutc()) if dt < now: dt = dt + datetime.timedelta(days=1) resp = twilio.twiml.Response() resp.message("Alarm created for %s" % (dt.strftime("%c %Z"))) dt = dt.astimezone(tz.tzutc()) call.create(number,dt,message) return str(resp)
def get_records(self, lastrun="1970-01-01"): """Get records from given date :param lastrun: date to get records from (optional). If CONF.extract_from is present, it will be used instead of the lastrun parameter. If CONF.extract_to is present, it will be used instead of the extract_to parameter """ extract_from = CONF.extract_from or lastrun extract_to = CONF.extract_to or datetime.datetime.utcnow() if isinstance(extract_from, six.string_types): extract_from = dateutil.parser.parse(extract_from) if isinstance(extract_to, six.string_types): extract_to = dateutil.parser.parse(extract_to) if extract_from.tzinfo is None: extract_from.replace(tzinfo=tz.tzutc()) if extract_to.tzinfo is None: extract_to.replace(tzinfo=tz.tzutc()) if self.records is None: self._extract(extract_from, extract_to) return self.records
def QA_SU_save_binance(frequency): symbol_list = QA_fetch_symbol() col = QASETTING.client.binance[frequency] col.create_index( [("symbol", pymongo.ASCENDING), ("start_time", pymongo.ASCENDING)],unique=True) end = datetime.datetime.now(tzutc()) end += FREQUANCY_DICT.get(frequency) for index, symbol_info in enumerate(symbol_list): QA_util_log_info('The {} of Total {}'.format (symbol_info['symbol'], len(symbol_list))) QA_util_log_info('DOWNLOAD PROGRESS {} '.format(str( float(index / len(symbol_list) * 100))[0:4] + '%') ) ref = col.find({"symbol": symbol_info['symbol']}).sort("start_time", -1) if ref.count() > 0: start_stamp = ref.next()['close_time'] / 1000 start_time = datetime.datetime.fromtimestamp(start_stamp, tz=tzutc()) QA_util_log_info('UPDATE_SYMBOL {} Trying updating {} from {} to {}'.format( frequency, symbol_info['symbol'], start_time, end)) else: start_time = BINANCE_MIN_DATE QA_util_log_info('NEW_SYMBOL {} Trying downloading {} from {} to {}'.format( frequency, symbol_info['symbol'], start_time, end)) data = QA_fetch_kline(symbol_info['symbol'], time.mktime(start_time.utctimetuple()), time.mktime(end.utctimetuple()), frequency) if data is None: QA_util_log_info('SYMBOL {} from {} to {} has no data'.format( symbol_info['symbol'], start_time, end)) continue col.insert_many(data)
def QA_SU_save_bitmex(frequency): symbol_list = QA_fetch_bitmex_symbols(active=True) symbol_list = symbol_list col = QASETTING.client.bitmex[frequency] col.create_index( [("symbol", pymongo.ASCENDING), ("timestamp", pymongo.ASCENDING)], unique=True) end = datetime.datetime.now(tzutc()) + relativedelta(days=-1, hour=0, minute=0, second=0, microsecond=0) for index, symbol_info in enumerate(symbol_list): QA_util_log_info('The {} of Total {}'.format (symbol_info['symbol'], len(symbol_list))) QA_util_log_info('DOWNLOAD PROGRESS {} '.format(str( float(index / len(symbol_list) * 100))[0:4] + '%') ) ref = col.find({"symbol": symbol_info['symbol']}).sort("timestamp", -1) if ref.count() > 0: start_stamp = ref.next()['timestamp'] / 1000 start_time = datetime.datetime.fromtimestamp(start_stamp+1,tz=tzutc()) QA_util_log_info('UPDATE_SYMBOL {} Trying updating {} from {} to {}'.format( frequency, symbol_info['symbol'], start_time, end)) else: start_time = symbol_info.get('listing', "2018-01-01T00:00:00Z") start_time = parse(start_time) QA_util_log_info('NEW_SYMBOL {} Trying downloading {} from {} to {}'.format( frequency, symbol_info['symbol'], start_time, end)) data = QA_fetch_bitmex_kline(symbol_info['symbol'], start_time, end, frequency) if data is None: QA_util_log_info('SYMBOL {} from {} to {} has no data'.format( symbol_info['symbol'], start_time, end)) continue col.insert_many(data)
def testOutlook3(self): self.portal.invokeFactory('Folder', id='test_calendar') cal = self.portal.test_calendar self.file = open(join(_prefix, 'data', 'outlook3.ics'), 'r') self.items = fromFile(cal, self.file) self.file.close() item = self.items[0][1] attendees = item.getAttendees() self.failUnless(attendees) values = ( '"Sidnei da Silva" <*****@*****.**>', '"George Alan Runyan" <*****@*****.**>', '"Toby Roberts" <*****@*****.**>', '"Andy McKay" <*****@*****.**>') for value in values: self.failUnless(value in attendees, (value, attendees)) self.assertEquals(item.Title(), 'Test Event: Weekly Meeting') self.assertEquals(item.getLocation(), 'Enfold Systems') self.assertEquals(item.Description().strip(), '') self.assertEquals(toTime(item.start_date), '15:30:00') self.assertEquals(toTime(item.end_date), '16:00:00') self.assertEquals(item.start_date, datetime.datetime(2005, 10, 21, 15, 30, tzinfo=tzutc())) self.assertEquals(item.end_date, datetime.datetime(2005, 10, 21, 16, 0, tzinfo=tzutc())) self.assertEquals(tuple(item.Subject()), ())
def create_candlepin_pool_json(pool_uuid=None, account_id=None, product_id=None, product_name=None): if not pool_uuid: pool_uuid = "unit_test_pool_id" if not account_id: account_id = "1" if not product_id: product_id = "unit_test_product_id" if not product_name: product_name = "unit_test_product_name" entry = { "objects": [ { "uuid": pool_uuid, "account": account_id, "active": True, "product_id": product_id, "product_name": product_name, "created": datetime.now(tzutc()), "start_date": datetime.now(tzutc()), "end_date": datetime.now(tzutc()), "updated": datetime.now(tzutc()), "quantity": 1, } ] } return entry
def _parse_date(self, str_value): """ Converts a string to a float representing miliseconds since the epoch. Since the elasticsearch date parser is not run on search_after, the date must be converted to ms since the epoch as that is how the dates are stored in the elasticsearch index. """ # Dates like "2017" can also be cast as floats so if a number is less # than 9999 it is assumed to be a year and not ms since the epoch. try: date = float(str_value) if date < 9999: raise ValueError("This is not in the form ms since the epoch.") return date except ValueError: try: date = parse(str_value, default=DEFAULT_DATE) # If timezone isn't specified assume it's utc. if not date.tzinfo: date = date.replace(tzinfo=tz.tzutc()) epoch = dt.utcfromtimestamp(0).replace(tzinfo=tz.tzutc()) return (date - epoch).total_seconds() * 1000.0 except ValueError: pass
def _assert_merged_entity(self, entity): ''' Asserts that the entity passed in matches the default entity merged with the updated entity. ''' self.assertEqual(entity.age, 'abc') self.assertEqual(entity.sex, 'female') self.assertEqual(entity.sign, 'aquarius') self.assertEqual(entity.married, True) self.assertEqual(entity.deceased, False) self.assertEqual(entity.sign, 'aquarius') self.assertEqual(entity.ratio, 3.1) self.assertEqual(entity.evenratio, 3.0) self.assertEqual(entity.large, 933311100) self.assertEqual(entity.Birthday, datetime(1973, 10, 4, tzinfo=tzutc())) self.assertEqual(entity.birthday, datetime(1991, 10, 4, tzinfo=tzutc())) self.assertIsInstance(entity.other, EntityProperty) self.assertEqual(entity.other.type, EdmType.INT32) self.assertEqual(entity.other.value, 20) self.assertIsInstance(entity.clsid, EntityProperty) self.assertEqual(entity.clsid.type, EdmType.GUID) self.assertEqual(entity.clsid.value, 'c9da6455-213d-42c9-9a79-3e9149a57833') self.assertTrue(hasattr(entity, "Timestamp")) self.assertIsNotNone(entity.etag)
def get_date_age(date): return (datetime.now(tz=tzutc()) - parser.parse(date)).days
def QA_SU_save_bitmex_day(frequency='1d', ui_log=None, ui_progress=None): """ 获取 bitmex K线 日线数据,统一转化字段保存数据为 crypto_asset_day """ market = 'bitmex' symbol_list = QA_fetch_crypto_asset_list(market=market) col = DATABASE.crypto_asset_day col.create_index([('market', pymongo.ASCENDING), ("symbol", pymongo.ASCENDING), ("date_stamp", pymongo.ASCENDING)], unique=True) end = datetime.datetime.now(tzutc()) QA_util_log_info( 'Starting DOWNLOAD PROGRESS of day Klines from bitmex... ', ui_log=ui_log, ui_progress=ui_progress) for index in range(len(symbol_list)): symbol_info = symbol_list.iloc[index] QA_util_log_info('The "{}" #{} of total in {}'.format( symbol_info['symbol'], index, len(symbol_list)), ui_log=ui_log, ui_progress=ui_progress) QA_util_log_info('DOWNLOAD PROGRESS {} '.format( str(float(index / len(symbol_list) * 100))[0:4] + '%'), ui_log=ui_log, ui_progress=ui_progress) query_id = { "symbol": symbol_info['symbol'], 'market': symbol_info['market'] } ref = col.find(query_id).sort('date_stamp', -1) if (col.count_documents(query_id) > 0): start_stamp = ref.next()['date_stamp'] start_time = datetime.datetime.fromtimestamp(start_stamp + 1, tz=tzutc()) QA_util_log_info( 'UPDATE_SYMBOL "{}" Trying updating "{}" from {} to {}'.format( symbol_info['symbol'], Bitmex2QA_FREQUENCY_DICT[frequency], QA_util_timestamp_to_str(start_time), QA_util_timestamp_to_str(end)), ui_log=ui_log, ui_progress=ui_progress) else: start_time = symbol_info.get('listing', "2018-01-01T00:00:00Z") start_time = parse(start_time) QA_util_log_info( 'NEW_SYMBOL "{}" Trying downloading "{}" from {} to {}'.format( symbol_info['symbol'], Bitmex2QA_FREQUENCY_DICT[frequency], QA_util_timestamp_to_str(start_time), QA_util_timestamp_to_str(end)), ui_log=ui_log, ui_progress=ui_progress) data = QA_fetch_bitmex_kline( symbol_info['symbol'], QA_util_datetime_to_Unix_timestamp(start_time), QA_util_datetime_to_Unix_timestamp(end), frequency, callback_func=QA_SU_save_data_bitmex_callback) if data is None: QA_util_log_info('SYMBOL "{}" from {} to {} has no data'.format( symbol_info['symbol'], QA_util_timestamp_to_str(start_time), QA_util_timestamp_to_str(end)), ui_log=ui_log, ui_progress=ui_progress) continue QA_util_log_info( 'DOWNLOAD PROGRESS of day Klines from bitmex accomplished ', ui_log=ui_log, ui_progress=ui_progress)
def _isoformat(value): return value and value.astimezone(tzutc()).isoformat()
from dateutil.tz import tzutc from mock import patch, call import pytest from pynamodb.constants import UTC, DATETIME_FORMAT from inpynamodb.models import Model from pynamodb.attributes import ( BinarySetAttribute, BinaryAttribute, NumberSetAttribute, NumberAttribute, UnicodeAttribute, UnicodeSetAttribute, UTCDateTimeAttribute, BooleanAttribute, LegacyBooleanAttribute, MapAttribute, MapAttributeMeta, ListAttribute, JSONAttribute, DEFAULT_ENCODING, NUMBER, STRING, STRING_SET, NUMBER_SET, BINARY_SET, BINARY, BOOLEAN, _get_value_for_deserialize) UTC = tzutc() class AttributeTestModel(Model): class Meta: host = 'http://localhost:8000' table_name = 'test' binary_attr = BinaryAttribute() binary_set_attr = BinarySetAttribute() number_attr = NumberAttribute() number_set_attr = NumberSetAttribute() unicode_attr = UnicodeAttribute() unicode_set_attr = UnicodeSetAttribute() datetime_attr = UTCDateTimeAttribute() bool_attr = BooleanAttribute()
from sqlalchemy import (Table, Column, Integer, String, ForeignKey, Float, DateTime, Boolean, ) from sqlalchemy.sql import join, select, func, label from sqlalchemy.orm import mapper, relationship, column_property from sqlalchemy.ext.declarative import declarative_base from dateutil.parser import parse from dateutil.tz import tzutc import wof.models as wof_base Base = declarative_base() # Instantiate zome useful time zones utc = tzutc() def init_model(db_session): Base.query = db_session.query_property() #TODO: Andy, please check param_to_medium_dict = { 'water_ph': wof_base.SampleMediumTypes.SURFACE_WATER, 'water_y_velocity': wof_base.SampleMediumTypes.SURFACE_WATER, 'water_x_velocity': wof_base.SampleMediumTypes.SURFACE_WATER, 'water_temperature': wof_base.SampleMediumTypes.SURFACE_WATER, 'upward_water_velocity': wof_base.SampleMediumTypes.SURFACE_WATER, 'water_turbidity': wof_base.SampleMediumTypes.SURFACE_WATER, 'water_total_dissolved_salts': wof_base.SampleMediumTypes.SURFACE_WATER, 'seawater_salinity': wof_base.SampleMediumTypes.SURFACE_WATER, 'northward_water_velocity': wof_base.SampleMediumTypes.SURFACE_WATER,
import json import datetime from dateutil.tz import tzutc class DateEncoder(json.JSONEncoder): def default(self, obj): if isinstance(obj, datetime.datetime): return obj.strftime('%Y-%m-%d %H:%M:%S') elif isinstance(obj, datetime.date): return obj.strftime("%Y-%m-%d") else: return json.JSONEncoder.default(self, obj) if __name__ == '__main__': data = {"name": "Tom", "birthday": datetime.datetime(2019, 11, 20, 7, 34, 15, tzinfo = tzutc())} print(type(data)) print(json.dumps(data, cls=DateEncoder))
def fillListViewTracks(self): """Fills list with tracks""" files = glob(self.DATAPATH + '/search/temp/*') for f in files: if os.path.isfile(f): os.remove(f) #If Windows if sys.platform.startswith('win'): #Get drive from user select drive = self.getDrive() #If not selected than C:, that should be always present if drive is None: #drive = "C:/" Very dangerous feature. Reads all GPX from the C: drive. It can take a lot of time. #removed QgsMessageLog.logMessage( u"Nebyl vybrán žádný disk. Nebudu hledat data.", "Patrac") return #TODO - do it better to handle another devices than Garmin self.path = drive[:-1] + '/' # For Linux is path set just for testing purposes # TODO - change to have real connected devices else: drive = self.getDriveLinux() if drive is None: #drive = "C:/" Very dangerous feature. Reads all GPX from the C: drive. It can take a lot of time. #removed QgsMessageLog.logMessage( u"Nebyl vybrán žádný disk. Nebudu hledat data.", "Patrac") return self.path = drive #for f in glob.iglob('E:/Garmin/GPX/*/*.gpx'): # generator, search immediate subdirectories i = 0 for root, dirnames, filenames in os.walk(self.path): for f in fnmatch.filter(filenames, '*.gpx'): #for f in iglob(self.path, recursive=True): #copyfile(f, self.DATAPATH + '/search/gpx/' + SECTOR + '/' + os.path.basename(f)) #First copy original file to search/gpx/ directory #shutil.copyfile(os.path.join(root, f.decode('utf8')), self.DATAPATH + u'/search/gpx/' + os.path.basename(f.decode('utf8'))) #Then copy the same file to search/tem/ directory and name it according to position in list shutil.copyfile( os.path.join(root, f), self.DATAPATH + '/search/temp/' + str(i) + '.gpx') #Notice size of list.csv listSize = 0 if i > 0: listSize = os.path.getsize(self.DATAPATH + '/search/temp/list.csv') #Run transformation to get time extent of the GPX #Extent is added to search/temp/list.csv if sys.platform.startswith('win'): # QgsMessageLog.logMessage(str(f), "Patrac") p = subprocess.Popen( (self.pluginPath + '/xslt/run_xslt_extent.bat', self.pluginPath, self.DATAPATH + '/search/temp/' + str(i) + '.gpx', self.DATAPATH + '/search/temp/list.csv')) p.wait() else: # QgsMessageLog.logMessage(str(f), "Patrac") p = subprocess.Popen( ('bash', self.pluginPath + '/xslt/run_xslt_extent.sh', self.pluginPath, self.DATAPATH + '/search/temp/' + str(i) + '.gpx', self.DATAPATH + '/search/temp/list.csv')) p.wait() i = i + 1 listSizeAfterXSLT = os.path.getsize(self.DATAPATH + '/search/temp/list.csv') if listSize == listSizeAfterXSLT: #Something bad happend with XSLT #Add empty row to /search/temp/list.csv' listFile = open(self.DATAPATH + '/search/temp/list.csv', 'w+') listFile.write(";\n") listFile.close() #if some GPX were found if os.path.isfile(self.DATAPATH + '/search/temp/list.csv'): self.listViewModel = QStandardItemModel() from_zone = tz.tzutc() to_zone = tz.tzlocal() #Loop via GPX tracks with open(self.DATAPATH + '/search/temp/list.csv') as fp: for cnt, line in enumerate(fp): track = u'Track ' + str(cnt) + ' ' items = line.split(';') start = '' end = '' #This is some workatound, becaouse the list can contain more than one time information for each track if len(items[0]) > 30: items2 = items[0].split(' ') start = items2[0] else: start = items[0] if len(items[1]) > 30: items2 = items[1].split(' ') end = items2[len(items2) - 1] else: end = items[1] if len(start) > 10 and len(end) > 10: #Convert to local time zone from UTC start_local = self.iso_time_to_local(start) end_local = self.iso_time_to_local(end) track += '(' + start_local + ' <-> ' + end_local + ')' item = QStandardItem(track) #check = Qt.Checked if randint(0, 1) == 1 else Qt.Unchecked #item.setCheckState(check) item.setCheckable(True) self.listViewModel.appendRow(item) else: item = QStandardItem("Another Type of GPX") item.setCheckable(False) self.listViewModel.appendRow(item) #print("Line {}: {}".format(cnt, line)) self.listViewTracks.setModel(self.listViewModel) else: QgsMessageLog.logMessage(u"Nebyl nalezen žádný záznam:", "Patrac")
from sipsimple.configuration.settings import SIPSimpleSettings from sipsimple.payloads import caps, pidf, prescontent, rpid from sipsimple.payloads import cipid from sipsimple.threading.green import run_in_green_thread from sipsimple.util import ISOTimestamp from blink.configuration.datatypes import IconDescriptor, FileURL, PresenceState from blink.configuration.settings import BlinkSettings from blink.resources import IconManager, Resources from blink.util import run_in_gui_thread del cipid # this only needs to be imported to register its namespace and extensions __all__ = ['PresenceManager', 'PendingWatcherDialog'] epoch = datetime.fromtimestamp(0, tzutc()) class BlinkPresenceState(object): def __init__(self, account): self.account = account @property def online_state(self): blink_settings = BlinkSettings() state = blink_settings.presence.current_state.state note = blink_settings.presence.current_state.note state = 'offline' if state == 'Invisible' else state.lower()
def test_26_periodictask(self): current_utc_time = datetime(2018, 3, 4, 5, 6, 8) with mock.patch('privacyidea.models.datetime') as mock_dt: mock_dt.utcnow.return_value = current_utc_time task1 = PeriodicTask("task1", False, "0 5 * * *", ["localhost"], "some.module", 2, { "key1": "value2", "KEY2": True, "key3": u"öfføff", }) task2 = PeriodicTask("some other task", True, "0 6 * * *", ["localhost"], "some.other.module", 1, { "foo": "bar" }) self.assertEqual(PeriodicTask.query.filter_by(name="task1").one(), task1) self.assertEqual(PeriodicTask.query.filter_by(name="some other task").one(), task2) self.assertEqual(PeriodicTaskOption.query.filter_by(periodictask_id=task1.id, key="KEY2").one().value, "True") # Values are converted to strings self.assertEqual(task1.get(), { "id": task1.id, "name": "task1", "active": False, "interval": "0 5 * * *", # we get a timezone-aware datetime here "last_update": current_utc_time.replace(tzinfo=tzutc()), "nodes": ["localhost"], "taskmodule": "some.module", "ordering": 2, "options": { "key1": "value2", "KEY2": "True", "key3": u"öfføff", }, "last_runs": {}}) # register a run task1.set_last_run("localhost", datetime(2018, 3, 4, 5, 6, 7)) # assert we can update the task later_utc_time = current_utc_time + timedelta(seconds=1) with mock.patch('privacyidea.models.datetime') as mock_dt: mock_dt.utcnow.return_value = later_utc_time PeriodicTask("task one", True, "0 8 * * *", ["localhost", "otherhost"], "some.module", 3, { "KEY2": "value number 2", "key 4": 1234 }, id=task1.id) # the first run for otherhost task1.set_last_run("otherhost", datetime(2018, 8, 9, 10, 11, 12)) result = PeriodicTask.query.filter_by(name="task one").one().get() self.assertEqual(result, { "id": task1.id, "active": True, "name": "task one", "interval": "0 8 * * *", "last_update": later_utc_time.replace(tzinfo=tzutc()), "nodes": ["localhost", "otherhost"], "taskmodule": "some.module", "ordering": 3, "options": {"KEY2": "value number 2", "key 4": "1234"}, "last_runs": { "localhost": datetime(2018, 3, 4, 5, 6, 7, tzinfo=tzutc()), "otherhost": datetime(2018, 8, 9, 10, 11, 12, tzinfo=tzutc()), } }) # assert all old options are removed self.assertEqual(PeriodicTaskOption.query.filter_by(periodictask_id=task1.id, key="key3").count(), 0) # the second run for localhost task1.set_last_run("localhost", datetime(2018, 3, 4, 5, 6, 8)) result = PeriodicTask.query.filter_by(name="task one").one().get() self.assertEqual(result, { "id": task1.id, "active": True, "name": "task one", "interval": "0 8 * * *", "last_update": later_utc_time.replace(tzinfo=tzutc()), "nodes": ["localhost", "otherhost"], "taskmodule": "some.module", "ordering": 3, "options": {"KEY2": "value number 2", "key 4": "1234"}, "last_runs": { "localhost": datetime(2018, 3, 4, 5, 6, 8, tzinfo=tzutc()), "otherhost": datetime(2018, 8, 9, 10, 11, 12, tzinfo=tzutc()), } }) # remove "localhost", assert the last run is removed PeriodicTask("task one", True, "0 8 * * *", ["otherhost"], "some.module", 4, {"foo": "bar"}, id=task1.id) self.assertEqual(PeriodicTaskOption.query.filter_by(periodictask_id=task1.id).count(), 1) self.assertEqual(PeriodicTaskLastRun.query.filter_by(periodictask_id=task1.id).one().node, "otherhost") # naive timestamp in the database self.assertEqual(PeriodicTaskLastRun.query.filter_by(periodictask_id=task1.id).one().timestamp, datetime(2018, 8, 9, 10, 11, 12, tzinfo=None)) self.assertEqual(PeriodicTaskLastRun.query.filter_by(periodictask_id=task1.id).one().aware_timestamp, datetime(2018, 8, 9, 10, 11, 12, tzinfo=tzutc())) # remove the tasks, everything is removed task1.delete() self.assertEqual(PeriodicTaskOption.query.count(), 1) # from task2 self.assertEqual(PeriodicTaskLastRun.query.count(), 0) task2.delete() self.assertEqual(PeriodicTaskOption.query.count(), 0)
def QA_SU_save_bitmex_min(frequency='1m', ui_log=None, ui_progress=None): """ 获取 bitmex K线 分钟线数据,统一转化字段保存数据为 crypto_asset_min """ market = 'bitmex' symbol_list = QA_fetch_crypto_asset_list(market=market) col = DATABASE.crypto_asset_min col.create_index([('market', pymongo.ASCENDING), ("symbol", pymongo.ASCENDING), ('time_stamp', pymongo.ASCENDING), ('date_stamp', pymongo.ASCENDING)]) col.create_index([('market', pymongo.ASCENDING), ("symbol", pymongo.ASCENDING), ("type", pymongo.ASCENDING), ('time_stamp', pymongo.ASCENDING)], unique=True) end = datetime.datetime.now(tzutc()) QA_util_log_info( 'Starting DOWNLOAD PROGRESS of min Klines from bitmex... ', ui_log=ui_log, ui_progress=ui_progress) for index in range(len(symbol_list)): symbol_info = symbol_list.iloc[index] QA_util_log_info('The "{}" #{} of total in {}'.format( symbol_info['symbol'], index, len(symbol_list)), ui_log=ui_log, ui_progress=ui_progress) QA_util_log_info('DOWNLOAD PROGRESS {} '.format( str(float(index / len(symbol_list) * 100))[0:4] + '%'), ui_log=ui_log, ui_progress=ui_progress) query_id = { "symbol": symbol_info['symbol'], 'market': symbol_info['market'], 'type': Bitmex2QA_FREQUENCY_DICT[frequency] } ref = col.find(query_id).sort('time_stamp', -1) if (col.count_documents(query_id) > 0): start_stamp = ref.next()['time_stamp'] start_time = datetime.datetime.fromtimestamp(start_stamp + 1, tz=tzutc()) QA_util_log_info( 'UPDATE_SYMBOL "{}" Trying updating "{}" from {} to {}'.format( symbol_info['symbol'], Bitmex2QA_FREQUENCY_DICT[frequency], QA_util_timestamp_to_str(start_time), QA_util_timestamp_to_str(end)), ui_log=ui_log, ui_progress=ui_progress) # 查询到 Kline 缺漏,点抓取模式,按缺失的时间段精确请求K线数据 missing_data_list = QA_util_find_missing_kline( symbol_info['symbol'], Bitmex2QA_FREQUENCY_DICT[frequency], market='bitmex')[::-1] else: start_time = symbol_info.get('listing', "2018-01-01T00:00:00Z") start_time = parse(start_time) QA_util_log_info( 'NEW_SYMBOL "{}" Trying downloading "{}" from {} to {}'.format( symbol_info['symbol'], Bitmex2QA_FREQUENCY_DICT[frequency], QA_util_timestamp_to_str(start_time), QA_util_timestamp_to_str(end)), ui_log=ui_log, ui_progress=ui_progress) miss_kline = pd.DataFrame( [[ QA_util_datetime_to_Unix_timestamp(start_time), QA_util_datetime_to_Unix_timestamp(end), '{} 到 {}'.format( start_time, end) ]], columns=['expected', 'between', 'missing']) missing_data_list = miss_kline.values if len(missing_data_list) > 0: # 查询确定中断的K线数据起止时间,缺分时数据,补分时数据 expected = 0 between = 1 missing = 2 reqParams = {} for i in range(len(missing_data_list)): reqParams['from'] = missing_data_list[i][expected] reqParams['to'] = missing_data_list[i][between] if (reqParams['from'] > (QA_util_datetime_to_Unix_timestamp() + 3600)): # 出现“未来”时间,一般是默认时区设置错误造成的 raise Exception( 'A unexpected \'Future\' timestamp got, Please check self.missing_data_list_func param \'tzlocalize\' set. More info: {:s}@{:s} at {:s} but current time is {}' .format( symbol_info['symbol'], frequency, QA_util_print_timestamp(reqParams['from']), QA_util_print_timestamp( QA_util_datetime_to_Unix_timestamp()))) QA_util_log_info( 'Fetch "{:s}" slices "{:s}" kline:{:s} to {:s}'.format( symbol_info['symbol'], frequency, QA_util_timestamp_to_str( missing_data_list[i][expected])[2:16], QA_util_timestamp_to_str( missing_data_list[i][between])[2:16])) data = QA_fetch_bitmex_kline_min( symbol_info['symbol'], start_time=reqParams['from'], end_time=reqParams['to'], frequency=frequency, callback_func=QA_SU_save_data_bitmex_callback) if data is None: QA_util_log_info( 'SYMBOL "{}" from {} to {} has no MORE data'.format( symbol_info['symbol'], QA_util_timestamp_to_str(start_time), QA_util_timestamp_to_str(end))) continue QA_util_log_info( 'DOWNLOAD PROGRESS of min Klines from bitmex accomplished ', ui_log=ui_log, ui_progress=ui_progress)
import csv from datetime import datetime from dateutil import tz from matplotlib import pyplot as plt # --- start config --- DATA_PATH = '/mnt/storage/speedtest-log/' TIME_ROW = 3 DOWNLOAD_ROW = 6 UPLOAD_ROW = 7 DELIMITER = ',' TIME_FORMAT = '%Y-%m-%dT%H:%M:%S.%fZ' FROM_TIMEZONE = tz.tzutc() TO_TIMEZONE = tz.tzlocal() TMP_IMAGE_FOLDER = 'images' OUTPUT = 'speedtest.avi' FPS = 1 FIG_DPI = 300 # --- end config --- def plot(download, upload, file_name, image_folder, title='', dpi=100): fig = plt.figure(dpi=dpi) fig.suptitle(title) hours = range(24)
def datetime_to_fedoratime(datetime): # format a date-time in a format fedora can handle # make sure time is in UTC, since the only time-zone notation Fedora seems able to handle is 'Z' utctime = datetime.astimezone(tzutc()) return utctime.strftime( '%Y-%m-%dT%H:%M:%S') + '.%03d' % (utctime.microsecond / 1000) + 'Z'
def main(): argument_spec = aci_argument_spec() argument_spec.update( aaa_password=dict(type='str', no_log=True), aaa_password_lifetime=dict(type='int'), aaa_password_update_required=dict(type='bool'), aaa_user=dict(type='str', required=True, aliases=['name' ]), # Not required for querying all objects clear_password_history=dict(type='bool'), description=dict(type='str', aliases=['descr']), email=dict(type='str'), enabled=dict(type='bool'), expiration=dict(type='str'), expires=dict(type='bool'), first_name=dict(type='str'), last_name=dict(type='str'), phone=dict(type='str'), state=dict(type='str', default='present', choices=['absent', 'present', 'query']), ) module = AnsibleModule( argument_spec=argument_spec, supports_check_mode=True, required_if=[ ['state', 'absent', ['aaa_user']], ['state', 'present', ['aaa_user']], ['expires', True, ['expiration']], ], ) aci = ACIModule(module) if not HAS_DATEUTIL: module.fail_json(msg='dateutil required for this module') aaa_password = module.params['aaa_password'] aaa_password_lifetime = module.params['aaa_password_lifetime'] aaa_password_update_required = aci.boolean( module.params['aaa_password_update_required']) aaa_user = module.params['aaa_user'] clear_password_history = module.params['clear_password_history'] description = module.params['description'] email = module.params['email'] enabled = aci.boolean(module.params['enabled'], 'active', 'inactive') expires = aci.boolean(module.params['expires']) first_name = module.params['first_name'] last_name = module.params['last_name'] phone = module.params['phone'] state = module.params['state'] expiration = module.params['expiration'] if expiration is not None and expiration != 'never': try: expiration = aci.iso8601_format( dateutil.parser.parse(expiration).replace(tzinfo=tzutc())) except Exception as e: module.fail_json(msg="Failed to parse date format '%s', %s" % (module.params['expiration'], e)) aci.construct_url(root_class=dict( aci_class='aaaUser', aci_rn='userext/user-{0}'.format(aaa_user), filter_target='eq(aaaUser.name, "{0}")'.format(aaa_user), module_object=aaa_user, ), ) aci.get_existing() if state == 'present': aci.payload( aci_class='aaaUser', class_config=dict( accountStatus=enabled, clearPwdHistory=clear_password_history, email=email, expiration=expiration, expires=expires, firstName=first_name, lastName=last_name, name=aaa_user, phone=phone, pwd=aaa_password, pwdLifeTime=aaa_password_lifetime, pwdUpdateRequired=aaa_password_update_required, ), ) aci.get_diff(aci_class='aaaUser') aci.post_config() elif state == 'absent': aci.delete_config() aci.exit_json()
# ---------------------------------------------------------------- # Time zones # ---------------------------------------------------------------- TIMEZONES = [ None, "UTC", "US/Eastern", "Asia/Tokyo", "dateutil/US/Pacific", "dateutil/Asia/Singapore", "+01:15", "-02:15", "UTC+01:15", "UTC-02:15", tzutc(), tzlocal(), FixedOffset(300), FixedOffset(0), FixedOffset(-300), timezone.utc, timezone(timedelta(hours=1)), timezone(timedelta(hours=-1), name="foo"), ] TIMEZONE_IDS = [repr(i) for i in TIMEZONES] @td.parametrize_fixture_doc(str(TIMEZONE_IDS)) @pytest.fixture(params=TIMEZONES, ids=TIMEZONE_IDS) def tz_naive_fixture(request): """
def get_utc_time(): return datetime.datetime.now(tzlocal()).astimezone(tzutc())
from dateutil import tz from keras.callbacks import EarlyStopping from keras.models import Sequential from keras.layers import Dense, Dropout import multiprocessing as mp from threading import Thread ALG_LEN = 3 names = ["Logistic Regression", "Linear SVM", "Neural Net"] topic_name = 'raw_nmea_numeric' from_zone = tz.tzutc() to_zone = tz.tzlocal() def main(file): # Create figure for plotting fig = plt.figure(figsize=(7, 3)) axes = [] for i in range(1, ALG_LEN + 1): axes.append(fig.add_subplot(1, 3, i)) plt.subplots_adjust(top=0.8, bottom=0.25, left=0.17, hspace=1.1,
def test_state_machine_get_execution_history_contains_expected_success_events_when_started( ): expected_events = [ { "timestamp": datetime(2020, 1, 1, 0, 0, 0, tzinfo=tzutc()), "type": "ExecutionStarted", "id": 1, "previousEventId": 0, "executionStartedEventDetails": { "input": "{}", "inputDetails": { "truncated": False }, "roleArn": _get_default_role(), }, }, { "timestamp": datetime(2020, 1, 1, 0, 0, 10, tzinfo=tzutc()), "type": "PassStateEntered", "id": 2, "previousEventId": 0, "stateEnteredEventDetails": { "name": "A State", "input": "{}", "inputDetails": { "truncated": False }, }, }, { "timestamp": datetime(2020, 1, 1, 0, 0, 10, tzinfo=tzutc()), "type": "PassStateExited", "id": 3, "previousEventId": 2, "stateExitedEventDetails": { "name": "A State", "output": "An output", "outputDetails": { "truncated": False }, }, }, { "timestamp": datetime(2020, 1, 1, 0, 0, 20, tzinfo=tzutc()), "type": "ExecutionSucceeded", "id": 4, "previousEventId": 3, "executionSucceededEventDetails": { "output": "An output", "outputDetails": { "truncated": False }, }, }, ] client = boto3.client("stepfunctions", region_name=region) sm = client.create_state_machine( name="test-state-machine", definition=simple_definition, roleArn=_get_default_role(), ) execution = client.start_execution(stateMachineArn=sm["stateMachineArn"]) execution_history = client.get_execution_history( executionArn=execution["executionArn"]) execution_history["events"].should.have.length_of(4) execution_history["events"].should.equal(expected_events)
def _create_list_objs(self, entity_factory, list_scopes): """Create and return list of objects used entity factory and UI data (list of scopes UI text elements {"header": "item", ...} remapped to list of dicts {"attr": "value", ...}). Return list of created objects. """ list_factory_objs = [ entity_factory().obj_inst() for _ in xrange(len(list_scopes)) ] list_scopes_with_upper_keys = [ StringMethods.dict_keys_to_upper_case(scope) for scope in list_scopes ] list_scopes_to_convert = StringMethods.exchange_dicts_items( transform_dict=Representation.remap_collection(), dicts=list_scopes_with_upper_keys, is_keys_not_values=True) # convert and represent values in scopes for scope in list_scopes_to_convert: # convert u'None', u'No person' to None type StringMethods.update_dicts_values(scope, ["None", "No person"], None) for key, val in scope.iteritems(): if val: if key in ["mandatory", "verified"]: # convert u'false', u'true' like to Boolean scope[key] = StringMethods.get_bool_value_from_arg(val) if key in ["updated_at", "created_at"]: # UI like u'08/20/2017' to date=2017-08-20, timetz=00:00:00 datetime_val = parser.parse(val) if str(datetime_val.time()) != "00:00:00": # UI like u'08/20/2017 07:30:45 AM +03:00' to date=2017-08-20, # timetz=04:30:45+00:00 if 'tzinfo', else: # CSV like u'08-20-2017 04:30:45' to date=2017-08-20, # timetz=04:30:45+00:00 datetime_val = (datetime_val.astimezone( tz=tz.tzutc()) if datetime_val.tzinfo else datetime_val.replace( tzinfo=tz.tzutc())) scope[key] = datetime_val if (key == "comments" and isinstance(val, list) and all( isinstance(comment, dict) for comment in val)): # extract datetime from u'(Creator) 08/20/2017 07:30:45 AM +03:00' scope[key] = [{ k: (parser.parse( re.sub(regex.TEXT_W_PARENTHESES, Symbols.BLANK, v)).astimezone(tz=tz.tzutc()) if k == "created_at" else v) for k, v in comment.iteritems() } for comment in val] # convert multiple values to list of strings and split if need it if (key in Representation.people_attrs_names and not isinstance(val, list)): # split Tree View values if need 'Ex1, Ex2 F' to ['Ex1', 'Ex2 F'] # Info Widget values will be represent by internal methods scope[key] = val.split(", ") # convert 'slug' from CSV for snapshoted objects u'*23eb72ac-4d9d' if (key == "slug" and (self.obj_name in objects.ALL_SNAPSHOTABLE_OBJS) and Symbols.STAR in val): scope[key] = val.replace(Symbols.STAR, Symbols.BLANK) return [ factory_obj.update_attrs(is_allow_none=True, **scope) for scope, factory_obj in zip(list_scopes_to_convert, list_factory_objs) ]
def additional_facility_details(request): data = {} from_zone = tz.tzutc() to_zone = tz.gettz('Asia/Kolkata') sid = transaction.savepoint() booking_detail_no = int(datetime.now().strftime('%Y%m%d%H%M%S')) try: data_list = [] equipment_name_list = [] facility_list = [] hall_equipment_list = [] hall_equipment_rate = [] hall_equipment_dict = {} print "request in | hallbookingapp | view | hallbooking_landing.py | additional_facility_details" user_type = request.GET.get('user_type_val') from_hour = request.GET.get('from_hour') to_hour = request.GET.get('to_hour') from_minute = request.GET.get('from_minute') to_minute = request.GET.get('to_minute') hall_id = request.GET.get('hall_id') from_date = request.GET.get('fromdate') to_date = request.GET.get('todate') from_period = request.GET.get('from_period') to_period = request.GET.get('to_period') company_name = request.GET.get('companyindividualname') date_value = request.GET.get('date_value') event_date = datetime.now() hall_detail_id = HallDetail.objects.get(id=hall_id) hall_facility_list = hall_detail_id.hall_equipment.all() hall_equipment_obj = HallEquipment.objects.filter( hall_detail=hall_detail_id.id) for hall_equipment in hall_equipment_obj: if user_type == 'nm': hall_equipment_val = float(hall_equipment.non_member_charges) else: hall_equipment_val = float(hall_equipment.member_charges) hall_equipment_dict = { 'facility_list': str(hall_equipment.hall_functioning_equipment.equipment_name), 'hall_rate': float(hall_equipment_val), 'from_hour': str(from_hour), 'from_minute': str(from_minute), 'to_minute': str(to_minute), 'to_hour': str(to_hour), 'from_period': str(from_period), 'to_period': str(to_period), } hall_equipment_list.append(hall_equipment_dict) length = len(from_date) if length: local_date = datetime.strptime(str(from_date), '%d/%m/%Y') local_from_time = '' local_to_time = '' if from_period == "PM": if from_hour == '12': local_from_time = time(int(from_hour), int(from_minute)) else: local_from_time = time( int(from_hour) + 12, int(from_minute)) else: if int(from_hour) == 12: local_from_time = time(int(00), int(from_minute)) else: local_from_time = time(int(from_hour), int(from_minute)) if to_period == "PM": if to_hour == '12': local_to_time = time(int(to_hour), int(to_minute)) else: local_to_time = time(int(to_hour) + 12, int(to_minute)) else: if int(to_hour) == 12: local_to_time = time(int(00), int(to_minute)) else: local_to_time = time(int(to_hour), int(to_minute)) from_time = datetime.strptime(str(local_from_time), '%H:%M:%S') to_time = datetime.strptime(str(local_to_time), '%H:%M:%S') f_time = datetime.strftime(from_time, '%H,%M') t_time = datetime.strftime(to_time, '%H,%M') f_time = datetime.strptime(f_time, '%H,%M').time() t_time = datetime.strptime(t_time, '%H,%M').time() from_date_time = datetime.combine(local_date.date(), f_time) to_date_time = datetime.combine(local_date.date(), t_time) date_time_dict = {} date_time_dict.setdefault((from_date_time.date()), []).append(from_date_time.time()) date_time_dict.setdefault((to_date_time.date()), []).append(to_date_time.time()) hall_obj = HallDetail.objects.get(id=request.GET.get('hall_id')) for key, value in date_time_dict.iteritems(): utc_to_date = datetime.combine(key, value[1]) utc_from_date = datetime.combine(key, value[0]) utc_to_date = utc_to_date.replace(tzinfo=from_zone) final_to_date = utc_to_date.astimezone(to_zone) utc_from_date = utc_from_date.replace(tzinfo=from_zone) final_from_date = utc_from_date.astimezone(to_zone) # Payment Calculation hall_rent_dict = {} if utc_from_date.date() in [ item.holiday_date for item in Holiday.objects.filter( holiday_date__gte=utc_from_date.date(), status=True, is_deleted=False, is_booking_available=True) ]: week_day = '6' else: week_day = utc_from_date.strftime('%w') minutes = int( ceil((final_to_date - final_from_date).seconds / 60.0)) if request.GET.get('user_type_val') == 'm': member_obj = UserDetail.objects.get( id=request.POST.get('company_list')) if member_obj.valid_invalid_member: hall_rent_dict = get_total_hall_rent( week_day, minutes, hall_obj, True) else: hall_rent_dict = get_total_hall_rent( week_day, minutes, hall_obj, False) else: hall_rent_dict = get_total_hall_rent( week_day, minutes, hall_obj, False) data = { 'success': 'true', 'hall_name': hall_detail_id.hall_name, 'hall_equipment': hall_equipment_list, 'event_nature': str(request.GET.get('NatureoftheEvent')), 'event_date': event_date.strftime('%I:%M %p'), 'booking_from_time': from_date, 'hall_detail_id': hall_detail_id.id, 'booking_to_time': to_date, 'company_name_val': company_name, 'hall_rent': hall_rent_dict['total_rent'], 'date_value': date_value } return HttpResponse(json.dumps(data), content_type='application/json') except Exception, e: print "Responce Out | hallbookingapp | view | hallbooking_landing.py | additional_facility_details", str( traceback.print_exc()) return HttpResponse(json.dumps(data), content_type='application/json')
import datetime import re from dateutil import parser from dateutil import tz from googlecloudsdk.core.util import times_data try: from dateutil import tzwin # pylint: disable=g-import-not-at-top, Windows except ImportError: tzwin = None LOCAL = tz.tzlocal() # The local timezone. UTC = tz.tzutc() # The UTC timezone. def GetTimeZone(name): """Returns a datetime.tzinfo object for name. Args: name: A timezone name string, None for the local timezone. Returns: A datetime.tzinfo object for name, local timezone if name is unknown. """ if name in ('UTC', 'Z'): return UTC if name in ('LOCAL', 'L'): return LOCAL
def python_value(self, value): return arrow.get(value, tzinfo=tzutc()) if value is not None else None
def load_series( self, train_start_date: datetime, train_end_date: datetime, tag_list: List[SensorTag], dry_run: Optional[bool] = False, **kwargs, ): """ See GordoBaseDataProvider for documentation """ if dry_run: raise NotImplementedError( "Dry run for IrocReader is not implemented") if not tag_list: logger.warning( "Iroc reader called with empty tag_list, returning none") return if train_end_date < train_start_date: raise ValueError( f"Iroc reader called with train_end_date: {train_end_date} before train_start_date: {train_start_date}" ) base_paths_from_assets = list( map(lambda tag: self.base_path_from_asset(tag.asset), tag_list)) if len(set(base_paths_from_assets)) != 1: raise ValueError( "Iroc reader found either more than one asset or no asset from the tag list provided" ) elif None in base_paths_from_assets: raise ValueError( "Iroc reader could not associate some tags to an asset.") base_path = base_paths_from_assets[0] # We query with an extra day on both sides since the way the files are # organized in the datalake does not account for timezones, so some timestamps # are in the wrong files all_base_paths = ( f"{base_path}/{t.year:0>4d}/{t.month:0>2d}/{t.day:0>2d}/" for t in pd.date_range( start=train_start_date.astimezone(tz.tzutc()) - pd.Timedelta("1D"), end=train_end_date.astimezone(tz.tzutc()) + pd.Timedelta("1D"), freq="D", )) fetched_tags = self._fetch_all_iroc_files_from_paths( all_base_paths, train_start_date, train_end_date, tag_list) if len(fetched_tags) < 0: raise ValueError( f"Found no data for tags {tag_list} in the daterange {train_start_date} to " f"{train_end_date}") concatted = pd.concat(fetched_tags, copy=False) if len(concatted.columns) != len(tag_list): raise ValueError( f"Did not find data for all tags, the missing tags are " f"{set(to_list_of_strings(tag_list))-set(concatted.columns)}") for col in concatted.columns: withouth_na = concatted[col].dropna() withouth_na.sort_index(inplace=True) yield withouth_na
def utc_now(): # utcnow() doesn't include a tzinfo. return datetime.datetime.utcnow().replace(tzinfo=tz.tzutc())
def calendar_events(request, calendar_slug): """ JSON events feed class conforming to the JQuery FullCalendar and jquery-week-calendar CalEvent standard. [1]: http://code.google.com/p/jquery-week-calendar/ [2]: http://arshaw.com/fullcalendar Corresponds to: http://arshaw.com/fullcalendar/docs/#calevent-objects """ calendar = get_object_or_404(Calendar, slug=calendar_slug) start = request.GET.get('start', None) end = request.GET.get('end', None) if settings.USE_TZ: start = start and datetime.datetime.fromtimestamp(int(start), tzutc()) end = end and datetime.datetime.fromtimestamp(int(end), tzutc()) else: start = start and datetime.datetime.fromtimestamp(int(start)) end = end and datetime.datetime.fromtimestamp(int(end)) events = GET_EVENTS_FUNC(request, calendar) period = Period(events, start, end) cal_events = [] for o in period.get_occurrences(): audience_bits = [ x for x in o.event.appropriate_for.get_set_bits() if x in VALID_AUDIENCES ] audiences = [AUDIENCE_TYPES[x]['name'][0] for x in audience_bits] if o.event.all_day: start = o.start.date().isoformat() diff = o.end - o.start end = o.start.date() + datetime.timedelta(days=diff.days) end = end.isoformat() else: start = o.start.isoformat() end = o.end.isoformat() occurrence_id = encode_occurrence(o) cal_event = { 'id': occurrence_id, 'allDay': o.event.all_day, 'event_id': o.event.pk, 'start': start, 'end': end, 'title': "%s %s" % ("".join(audiences), o.title), 'description': o.description, 'delete_url': "%s?id=%s&action=cancel" % (reverse('ajax_edit_event', kwargs={'calendar_slug': calendar_slug}), o.event.pk), 'delete_occurrence_url': "%s?id=%s&action=cancel" % (reverse('ajax_edit_event', kwargs={'calendar_slug': calendar_slug}), occurrence_id), 'edit_url': reverse('admin:events_event_change', args=(o.event.pk, )), 'update_url': reverse('ajax_edit_event', kwargs={'calendar_slug': calendar_slug}), 'update_occurrence_url': "%s?id=%s" % (reverse('ajax_edit_event', kwargs={'calendar_slug': calendar_slug}), occurrence_id), 'repeating_id': o.event.rule_id, 'repeating_name': getattr(o.event.rule, "name", ""), 'repeats': o.event.rule is not None, 'audiences': audiences, } cal_events.append(cal_event) return JSONResponse(cal_events)
def when(self, value: str) -> None: timestamp = datetime.fromisoformat(value) if timestamp.tzinfo is None: self._when = timestamp.replace(tzinfo=tzutc()) self._when = timestamp.astimezone(tzutc())
def test_rsvp_last_viewed_updated(self): guest = save_guest(EXAMPLE_GUEST_1) self.client.get(f"/rsvp/{guest.id}", follow_redirects=True) edited_guest = get_guest(guest.id) self.assertAlmostEqual(edited_guest.last_viewed, datetime.datetime.now(tzutc()), delta=datetime.timedelta(seconds=5))
def convert_repr_rest_to_ui(obj): """Convert object's attributes from REST to UI like representation.""" def convert_attr_val_repr_dict_to_unicode(attr_name, attr_value): """Convert attribute value from dictionary to unicode representation (get value by key from dictionary 'attr_value' where key determine according to 'attr_name'). """ if isinstance(attr_value, dict): converted_attr_value = attr_value if attr_name in Representation.people_attrs_names + [ "created_by", "modified_by" ]: converted_attr_value = unicode(attr_value.get("email")) if attr_name in [ "custom_attribute_definitions", "program", "audit", "mapped_objects" ]: converted_attr_value = ( unicode(attr_value.get("title")) if attr_name != "custom_attribute_definitions" else { attr_value.get("id"): attr_value.get("title").upper() }) if attr_name in ["custom_attribute_values"]: converted_attr_value = { attr_value.get("custom_attribute_id"): attr_value.get("attribute_value") } if obj_attr_name == "comments": converted_attr_value = { k: (parser.parse(v).replace( tzinfo=tz.tzutc()) if k == "created_at" and isinstance(v, unicode) else v) for k, v in attr_value.iteritems() if k in ["modified_by", "created_at", "description"] } if attr_name == "assertions": for name, assertion_id in ControlEntity.ASSERTIONS.iteritems( ): if assertion_id == attr_value["id"]: converted_attr_value = name return converted_attr_value origin_obj = copy.deepcopy(obj) for obj_attr_name in obj.__dict__.keys(): # 'Ex', u'Ex', 1, None to 'Ex', u'Ex', 1, None obj_attr_value = getattr(obj, obj_attr_name) # REST like u'08-20-2017T04:30:45' to date=2017-08-20, # timetz=04:30:45+00:00 if (obj_attr_name in ["updated_at", "created_at"] and isinstance(obj_attr_value, unicode)): obj_attr_value = (parser.parse(obj_attr_value).replace( tzinfo=tz.tzutc())) if isinstance(obj_attr_value, dict) and obj_attr_value: # "modified_by" {"type": "Person", "id": x} to u'*****@*****.**' # todo: deprecated? if obj_attr_name == "modified_by": from lib.service import rest_service obj_attr_value = getattr( rest_service.ObjectsInfoService().get_obj( obj=Representation.repr_dict_to_obj( obj_attr_value)), "email") # {'name': u'Ex1', 'type': u'Ex2', ...} to u'Ex1' else: obj_attr_value = convert_attr_val_repr_dict_to_unicode( obj_attr_name, obj_attr_value) # [el1, el2, ...] or [{item1}, {item2}, ...] to [u'Ex1, u'Ex2', ...] if (isinstance(obj_attr_value, list) and all( isinstance(item, dict) for item in obj_attr_value)): obj_attr_value = [ convert_attr_val_repr_dict_to_unicode( obj_attr_name, item) for item in obj_attr_value ] setattr(obj, obj_attr_name, obj_attr_value) # merge "custom_attribute_definitions" and "custom_attribute_values" obj_cas_attrs_names = [ "custom_attributes", "custom_attribute_definitions", "custom_attribute_values" ] if set(obj_cas_attrs_names).issubset(obj.__dict__.keys()): cas_def = obj.custom_attribute_definitions cas_val = obj.custom_attribute_values # form CAs values of CAs definitions exist but CAs values not, or CAs # definitions have different then CAs values lengths if (cas_def and (not cas_val or (isinstance(cas_def and cas_val, list)) and len(cas_def) != len(cas_val))): from lib.entities.entities_factory import ( CustomAttributeDefinitionsFactory) cas_val_dicts_keys = ([_.keys()[0] for _ in cas_val] if isinstance( cas_val, list) else [None]) _cas_val = [ { k: v } for k, v in CustomAttributeDefinitionsFactory. generate_ca_title_id([ Representation.repr_dict_to_obj(cad) for cad in origin_obj.custom_attribute_definitions ]).iteritems() if k not in cas_val_dicts_keys ] cas_val = _cas_val if not cas_val else cas_val + _cas_val cas_def_dict = (dict( [_def.iteritems().next() for _def in cas_def]) if (isinstance(cas_def, list) and all( isinstance(_def, dict) for _def in cas_def)) else None) cas_val_dict = (dict( [_val.iteritems().next() for _val in cas_val]) if (isinstance(cas_def, list) and all( isinstance(_def, dict) for _def in cas_def)) else None) cas = StringMethods.merge_dicts_by_same_key( cas_def_dict, cas_val_dict) if obj.custom_attributes: cas.update(obj.custom_attributes) if cas in [{None: None}, {}]: cas = None setattr(obj, "custom_attributes", cas) return obj
def convert_obj_repr_from_rest_to_ui(obj): """Convert object's attributes from REST to UI like representation.""" def convert_attr_value_from_dict_to_unicode(attr_name, attr_value): """Convert attribute value from dictionary to unicode representation (get value by key from dictionary 'attr_value' where key determine according to 'attr_name'). """ if isinstance(attr_value, dict): converted_attr_value = attr_value if attr_name in [ "contact", "manager", "owners", "assessor", "creator", "verifier", "created_by", "modified_by", "Assessor", "Creator", "Verifier" ]: converted_attr_value = unicode(attr_value.get("email")) if attr_name in [ "custom_attribute_definitions", "program", "audit", "objects_under_assessment" ]: converted_attr_value = ( unicode(attr_value.get("title")) if attr_name != "custom_attribute_definitions" else { attr_value.get("id"): attr_value.get("title").upper() }) if attr_name in ["custom_attribute_values"]: converted_attr_value = { attr_value.get("custom_attribute_id"): attr_value.get("attribute_value") } if obj_attr_name == "comments": converted_attr_value = { k: (parser.parse(v).replace( tzinfo=tz.tzutc()) if k == "created_at" and isinstance(v, unicode) else v) for k, v in attr_value.iteritems() if k in ["modified_by", "created_at", "description"] } return converted_attr_value origin_obj = copy.deepcopy(obj) for obj_attr_name in obj.__dict__.keys(): # 'Ex', u'Ex', 1, None to 'Ex', u'Ex', 1, None obj_attr_value = (obj.assignees.get(obj_attr_name.title()) if ( obj_attr_name in ["assessor", "creator", "verifier"] and "assignees" in obj.__dict__.keys()) else getattr( obj, obj_attr_name)) # REST like u'08-20-2017T04:30:45' to date=2017-08-20, # timetz=04:30:45+00:00 if (obj_attr_name in ["updated_at", "created_at"] and isinstance(obj_attr_value, unicode)): obj_attr_value = (parser.parse(obj_attr_value).replace( tzinfo=tz.tzutc())) if isinstance(obj_attr_value, dict) and obj_attr_value: # to "assignees" = {"Assessor": [], "Creator": [], "Verifier": []} if obj_attr_name == "assignees": obj_attr_value = { k: ([ convert_attr_value_from_dict_to_unicode(k, _v) for _v in v ] if isinstance(v, list) else convert_attr_value_from_dict_to_unicode(k, v)) for k, v in obj_attr_value.iteritems() if k in ["Assessor", "Creator", "Verifier"] } # "modified_by" {"type": "Person", "id": x} to u'*****@*****.**' if obj_attr_name == "modified_by": from lib.service import rest_service obj_attr_value = getattr( rest_service.ObjectsInfoService().get_obj( obj=Entity.convert_dict_to_obj_repr( obj_attr_value)), "email") # {'name': u'Ex1', 'type': u'Ex2', ...} to u'Ex1' else: obj_attr_value = convert_attr_value_from_dict_to_unicode( obj_attr_name, obj_attr_value) # [el1, el2, ...] or [{item1}, {item2}, ...] to [u'Ex1, u'Ex2', ...] if (isinstance(obj_attr_value, list) and all( isinstance(item, dict) for item in obj_attr_value)): obj_attr_value = [ convert_attr_value_from_dict_to_unicode( obj_attr_name, item) for item in obj_attr_value ] setattr(obj, obj_attr_name, obj_attr_value) # merge "custom_attribute_definitions" and "custom_attribute_values" obj_cas_attrs_names = [ "custom_attributes", "custom_attribute_definitions", "custom_attribute_values" ] if set(obj_cas_attrs_names).issubset(obj.__dict__.keys()): cas_def = obj.custom_attribute_definitions cas_val = obj.custom_attribute_values # form CAs values of CAs definitions exist but CAs values not, or CAs # definitions have different then CAs values lengths if (cas_def and (not cas_val or (isinstance(cas_def and cas_val, list)) and len(cas_def) != len(cas_val))): from lib.entities.entities_factory import ( CustomAttributeDefinitionsFactory) cas_val_dicts_keys = ([_.keys()[0] for _ in cas_val] if isinstance( cas_val, list) else [None]) _cas_val = [ { k: v } for k, v in CustomAttributeDefinitionsFactory. generate_ca_values(list_ca_def_objs=origin_obj. custom_attribute_definitions, is_none_values=True).iteritems() if k not in cas_val_dicts_keys ] cas_val = _cas_val if not cas_val else cas_val + _cas_val cas_def_dict = (dict( [_def.iteritems().next() for _def in cas_def]) if (isinstance(cas_def, list) and all( isinstance(_def, dict) for _def in cas_def)) else { None: None }) cas_val_dict = (dict( [_val.iteritems().next() for _val in cas_val]) if (isinstance(cas_def, list) and all( isinstance(_def, dict) for _def in cas_def)) else { None: None }) cas = string_utils.merge_dicts_by_same_key( cas_def_dict, cas_val_dict) setattr(obj, "custom_attributes", cas) return obj
def test_metadata2asset_simple1(schema_dir): data = metadata2asset({ "contentSize": 69105, "digest": "783ad2afe455839e5ab2fa659861f58a423fd17f", "digest_type": "sha1", "encodingFormat": "application/x-nwb", "nwb_version": "2.2.5", "experiment_description": "experiment_description1", "experimenter": ("experimenter1", ), "identifier": "identifier1", "institution": "institution1", "keywords": ["keyword1", "keyword 2"], "lab": "lab1", "related_publications": ("related_publications1", ), "session_description": "session_description1", "session_id": "session_id1", "session_start_time": datetime(2017, 4, 15, 12, 0, tzinfo=tzutc()), "age": None, "date_of_birth": None, "genotype": None, "sex": None, "species": None, "subject_id": None, "number_of_electrodes": 0, "number_of_units": 0, "nd_types": [], "tissue_sample_id": "tissue42", }) assert data == AssetMeta.unvalidated( schemaVersion="1.0.0-rc1", identifier="identifier1", name=None, description=None, contributor=None, about=None, studyTarget=None, license=None, protocol=None, ethicsApproval=None, keywords=["keyword1", "keyword 2"], acknowledgement=None, access=[ AccessRequirements( status=AccessType.Open, email=None, contactPoint=None, description=None, embargoedUntil=None, ) ], url=None, repository="https://dandiarchive.org/", relatedResource=None, wasGeneratedBy=None, contentSize=69105, encodingFormat="application/x-nwb", digest=Digest(value="783ad2afe455839e5ab2fa659861f58a423fd17f", cryptoType=DigestType.sha1), path=None, dataType=None, sameAs=None, modality=None, measurementTechnique=None, variableMeasured=None, wasDerivedFrom=[ BioSample(identifier="tissue42", assayType=None, anatomy=None, wasDerivedFrom=None) ], wasAttributedTo=[], contentUrl=None, ) # We need to convert `data` to a `dict` this way instead of with `.dict()` # so that enums will be converted to strings. data_as_dict = json.loads(data.json(exclude_unset=True, exclude_none=True)) validate_asset_json(data_as_dict, schema_dir)
def when(self) -> str: if self._when.tzinfo is None: return self._when.replace(tzinfo=tzutc()).isoformat(timespec='seconds') return self._when.astimezone(tzutc()).isoformat(timespec='seconds')