def _get_aging_factor(self, lease, time): """Returns an aging factor for the preemptability score This is a convenience function that can be used to "age" a preemptability score (allowing leases that have been submitted long ago to avoid preemption). The method returns a factor between 0 and 1 that can be multiplied by the score, reducing the score based on the lease's "age". Currently, this method uses a hard-coded horizon of 31 days (any lease older than 7 days cannot be preempted, and leases less than 7 days are assigned a factor proportional to their age) Arguments: lease -- Lease that is going to be preempted time -- Time at which preemption would take place """ # TODO: Make horizon configurable horizon = time - DateTimeDelta(7) if lease.submit_time <= horizon: return -1 else: seconds = (time - lease.submit_time).seconds horizon_seconds = DateTimeDelta(31).seconds return float(horizon_seconds - seconds) / horizon_seconds
class ExpireDateMixin(object): """ A mixin for providing expire dates. """ expire_attr = 'expire_date' expire_dates = [None, now() - DateTimeDelta(10), now() + DateTimeDelta(10)] def __init__(self): parent = super(ExpireDateMixin, self) if hasattr(parent, '__init__'): getattr(parent, '__init__')() self.register_attr(self.expire_attr, self.get_expire_date) def get_expire_date(self, ident): """ Get the expire_date for a given ident. @type ident: str @param ident: The 'ident' or 'id of an item @rtype: mx.DateTime.DateTime or NoneType @return: An expire date or None """ if ident in getattr(self, 'items'): return getattr(self, 'items')[ident][self.expire_attr] return self._get_cyclic_value(ident, self.expire_dates)
def is_active(self, date=Date(*time.localtime()[:3])): # IVR 2009-04-29 Lars Gustav Gudbrandsen requested on 2009-04-22 that # all employment-related info should be considered active 14 days # prior to the actual start day. # Jazz 2011-03-23: In order to avoid revoking affiliations/priviledges # for people changing role/position at UiO we should let the # affiliation be valid for a few days after it has been revoked # in the authoritative source if self.start: return ((self.start - DateTimeDelta(14) <= date) and ((not self.end) or (date <= self.end + DateTimeDelta(3)))) return ((not self.end) or (date <= self.end + DateTimeDelta(3)))
class BasicPersonSource(BaseDataSource): birth_dates = [ now(), now() - DateTimeDelta(365 * 100), now() - DateTimeDelta(356 * 20) ] genders = ['M', 'F', None] birth_date_attr = 'birth_date' gender_attr = 'gender' def __init__(self): parent = super(BasicPersonSource, self) if hasattr(parent, '__init__'): getattr(parent, '__init__')() self.register_attr(self.birth_date_attr, self.get_birth_date) self.register_attr(self.gender_attr, self.get_gender) def get_gender(self, ident): """ Get the gender for a given ident. @type ident: str @param ident: The 'ident' or 'id of an item @rtype: str or NoneType @return: One of the values in self.genders """ if ident in getattr(self, 'items'): return getattr(self, 'items')[ident][self.expire_attr] return self._get_cyclic_value(ident, self.genders) def get_birth_date(self, ident): """ Get the expire_date for a given ident. @type ident: str @param ident: The 'ident' or 'id of an item @rtype: mx.DateTime.DateTime or NoneType @return: One of the values in self.birth_dates """ if ident in getattr(self, 'items'): return getattr(self, 'items')[ident][self.expire_attr] return self._get_cyclic_value(ident, self.birth_dates)
def delete_stale_events(cl_events, db): """Remove all events of type cl_events older than GRACE_PERIOD. cl_events is an iterable listing change_log event types that we want expunged. These events cannot require any state change in Cerebrum (other than their own deletion). It is the caller's responsibility to check that this is so. """ if not isinstance(cl_events, (list, tuple, set)): cl_events = [ cl_events, ] const = Factory.get("Constants")() typeset_request = ", ".join(str(const.ChangeType(x)) for x in cl_events) logger.debug("Deleting stale requests: %s", typeset_request) for event in db.get_log_events(types=cl_events): tstamp = event["tstamp"] timeout = cereconf.GRACE_PERIOD try: params = pickle.loads(event["change_params"]) if params['timeout'] is not None: timeout = DateTimeDelta(params['timeout']) logger.debug('Timeout set to %s for %s', (now() + timeout).strftime('%Y-%m-%d'), event['change_id']) if timeout > cereconf.MAX_INVITE_PERIOD: logger.warning('Too long timeout (%s) for for %s', timeout.strftime('%Y-%m-%d'), event['change_id']) timeout = cereconf.MAX_INVITE_PERIOD except KeyError: pass if now() - tstamp <= timeout: continue logger.debug("Deleting stale event %s (@%s) for entity %s (id=%s)", str(const.ChangeType(event["change_type_id"])), event["tstamp"].strftime("%Y-%m-%d"), fetch_name(event["subject_entity"], db), event["subject_entity"]) db.remove_log_event(event["change_id"]) db.commit() logger.debug("Deleted all stale requests: %s", typeset_request)
def delete_stale_events(cl_events, db): """Remove all events of type cl_events older than GRACE_PERIOD. cl_events is an iterable listing change_log event types that we want expunged. These events cannot require any state change in Cerebrum (other than their own deletion). It is the caller's responsibility to check that this is so. """ if not isinstance(cl_events, (list, tuple, set)): cl_events = [cl_events, ] clconst = Factory.get("CLConstants")() typeset_request = ", ".join(str(clconst.ChangeType(x)) for x in cl_events) logger.debug("Deleting stale requests: %s", typeset_request) for event in db.get_log_events(types=cl_events): tstamp = event["tstamp"] timeout = cereconf.GRACE_PERIOD try: params = json.loads(event["change_params"]) if params['timeout'] is not None: timeout = DateTimeDelta(params['timeout']) logger.debug('Timeout set to %s for %s', (now() + timeout).strftime('%Y-%m-%d'), event['change_id']) if timeout > cereconf.MAX_INVITE_PERIOD: logger.warning('Too long timeout (%s) for for %s', timeout.strftime('%Y-%m-%d'), event['change_id']) timeout = cereconf.MAX_INVITE_PERIOD except KeyError: pass if now() - tstamp <= timeout: continue logger.debug("Deleting stale event %s (@%s) for entity %s (id=%s)", str(clconst.ChangeType(event["change_type_id"])), event["tstamp"].strftime("%Y-%m-%d"), fetch_name(event["subject_entity"], db), event["subject_entity"]) db.remove_log_event(event["change_id"]) db.commit() logger.debug("Deleted all stale requests: %s", typeset_request)
def sync_group(affil, gname, descr, mtype, memb, recurse=True): logger.debug( ("sync_group(parent:'%s'; groupname:'%s'; description:'%s'; " + "membertype:'%s'; members:'%s'; recurse:'%s')") % (affil, gname, descr, mtype, memb.keys(), recurse)) if mtype == constants.entity_group: # memb has group_name as keys members = {} for tmp_gname in memb.keys(): grp = get_group(tmp_gname) members[int(grp.entity_id)] = 1 else: # memb has account_id as keys members = memb.copy() if affil is not None: AffiliatedGroups.setdefault(affil, {})[gname] = 1 try: group = get_group(gname) except Errors.NotFoundError: group = Factory.get('Group')(db) group.clear() group.populate( creator_id=group_creator, visibility=constants.group_visibility_all, name=gname, description=descr, group_type=constants.group_type_unknown, ) group.write_db() else: # Update description if it has changed if group.description != descr: group.description = descr group.write_db() if group.is_expired(): # Extend the group's life by 6 months from mx.DateTime import now, DateTimeDelta group.expire_date = now() + DateTimeDelta(6 * 30) group.write_db() # Make sure the group is listed for export to LMS if not group.has_spread(constants.spread_lms_group): group.add_spread(constants.spread_lms_group) for member in group.search_members(group_id=group.entity_id, member_type=mtype, member_filter_expired=False): member = int(member["member_id"]) if members.has_key(member): del members[member] else: logger.debug("sync_group(): Deleting member %d" % member) group.remove_member(member) for member in members.keys(): group.add_member(member)
def group_invite_user(self, inviter, group, email, timeout=3): """ This method sets up an event that will allow a user to join a group. @type inviter: self.account_class @param inviter: The account that is setting up the invitation @type group: self.group_class @param group_name: The group that should be joined @type email: str @param : The email adrress of the user that is invited @type timeout: int @param timeout: The number of days until the confirmation key expires. @rtype: dict @return: A dictionary with the following keys: 'confirmation_key': <str> The code that is used to confirm the invitation 'match_user': <str> A username, if a user exists with that email-address 'match_user_email': An e-mailaddress. Not sure why? """ ac = self.account_class(self.db) assert hasattr(inviter, 'entity_id') assert hasattr(group, 'entity_id') timeout = DateTimeDelta(int(timeout)) if timeout.day < 1: raise CerebrumError('Timeout too short (%d)' % timeout.day) if (timeout > cereconf.MAX_INVITE_PERIOD): raise CerebrumError("Timeout too long (%d)" % timeout.day) ret = {'confirmation_key': self.vhutils.setup_event_request( group.entity_id, self.co.va_group_invitation, params={'inviter_id': inviter.entity_id, 'group_id': group.entity_id, 'invitee_mail': email, 'timeout': timeout.day,}, change_by=inviter.entity_id)} # check if e-mail matches a valid username try: ac.find_by_name(email) ret['match_user'] = ac.account_name if ac.np_type in (self.co.fedaccount_type, self.co.virtaccount_type): ret['match_user_email'] = ac.get_email_address() except NotFoundError: pass return ret
def calc_bin_span(self, days): """ Given a desired bin span in days, calculate number of bins that comes closest. """ assert self.span if (days % 7) == 0: # If user asked for a bin width that is a multiple of 7 days, shift # bins to align with weeks. self.first += RelativeDateTime(weekday=(Monday,0)) self.span = self.last - self.first self.bin_span = DateTimeDelta(days) self.n_bins = int(math.ceil(self.span / self.bin_span)) if self.n_bins > self.max_bins: raise Error('Too many bins (%d, maximum is %d) - make bins bigger, or limit date range' % (self.n_bins, self.max_bins))
def collect_warnable_accounts(database): """Collect all FA/VA that are about to expire. Both VA and FA are collected as candidates. """ account = Factory.get("Account")(database) const = Factory.get("Constants")() return set( x["account_id"] # collect all accounts for x in account.list(filter_expired=True) # ... that are VA/FA if (x["np_type"] in (const.virtaccount_type, const.fedaccount_type) and # ... and have an expire date x["expire_date"] and # ... and that expire date is within the right window DateTimeDelta(0) <= x["expire_date"] - now() <= get_config("EXPIRE_WARN_WINDOW")))
def get_printings(original_story, first_publication_date): printings = Story.objects.filter(story_version__base_story =\ original_story).filter(issue__publication_date__gt = "")\ .order_by('issue__publication_date') if len(first_publication_date) <= 4 or 'Q' in first_publication_date: first_printings = printings.filter(issue__publication_date__icontains\ = first_publication_date[0:4]) else: end_date = ParseDateTimeUTC(first_publication_date[0:7]) end_date += DateTimeDelta(120) end_date = end_date.strftime("%Y-%m") first_printings = printings.filter(issue__publication_date__gte \ = first_publication_date[0:7]).filter(issue__publication_date__lte \ = end_date) if not first_printings: first_printings = \ printings.filter(issue__publication_date__icontains \ = first_publication_date[0:4]) return printings, first_printings
def adjustDateTime(adjustDelta, dateTime): """ sometimes a DateTime object has to be adjusted, when illegal components were specified adjustDateTime corrects DateTime object with specifications obtained from fixTimeComponents() input: adjustFlags is tuple/list of flags for adjusting ( increaseDay, increaseHour, increaseMin ) output: corrected DateTime object """ # Increase day dateTime += DateTimeDelta(adjustDelta[0]) # Increase hour dateTime += TimeDelta(adjustDelta[1]) # Increase minute dateTime += DateTimeDeltaFromSeconds(60.0 * adjustDelta[2]) return dateTime
def handle_match(self, match): """Handles cases where the log_regex is matched.""" message = match.group('message') if not self.status_regex.match(message): return date_time = match.group('date_time') date_time, ms = date_time.split(',') date_time = DateTime.strptime(date_time, self.date_time_fmt) ms = DateTimeDelta(0, 0, 0, int(ms) / 1000.0) date_time = date_time + ms if message.startswith('LOADING_IMAGE'): name = message[message.find(':') + 2:] self.last_import = Import(date_time, name) self.imports.append(self.last_import) elif not hasattr(self, 'last_import') or self.last_import is None: return elif message.startswith('LOADED_IMAGE'): self.last_import.setid_end = date_time elif message.startswith('BEGIN_POST_PROCESS'): self.last_import.post_process_start = date_time elif message.startswith('END_POST_PROCESS'): self.last_import.post_process_end = date_time elif message.startswith('BEGIN_SAVE_TO_DB'): self.last_import.save_to_db_start = date_time elif message.startswith('END_SAVE_TO_DB'): self.last_import.save_to_db_end = date_time elif message.startswith('IMPORT_OVERLAYS'): self.last_import.overlays_start = date_time elif message.startswith('IMPORT_THUMBNAILING'): self.last_import.thumbnailing_start = date_time elif message.startswith('IMPORT_DONE'): self.last_import.end = date_time self.last_import = None elif message.startswith('DATASET_STORED'): self.last_series = Series(date_time) self.last_import.series.append(self.last_series) elif message.startswith('DATA_STORED'): self.last_import.series[-1].end = date_time elif message.startswith('IMPORT_STEP'): self.last_series.planes.append(Plane(date_time))
def test_phonenumberchange_fresh_person(self): """Fresh phone numbers should still be available for new persons""" cereconf.INDIVIDUATION_PHONE_TYPES['system_fs']['types']['contact_mobile_phone']['delay'] = 7 cereconf.INDIVIDUATION_PHONE_TYPES['system_fs']['types']['contact_private_mobile']['delay'] = 7 ou = Factory.get('OU')(self.db) co = Factory.get('Constants')(self.db) ou.find_stedkode(0, 0, 0, 0) pe = self.createPerson(first_name='Miss', last_name='Test2', studnr='008') ac = self.createAccount(pe, 'mstest2') pe.populate_affiliation(source_system=co.system_fs, ou_id=ou.entity_id, affiliation=co.affiliation_student, status=co.affiliation_status_student_aktiv) pe.write_db() pe.populate_contact_info(source_system=co.system_fs, type=co.contact_mobile_phone, value='98012345') pe.write_db() self.db.commit() # hack the create_date in change_log print self.db.execute(""" UPDATE [:table schema=cerebrum name=change_log] SET tstamp = :tid WHERE subject_entity = :s_id AND change_type_id = :ct_id""", {'s_id': pe.entity_id, 'ct_id': co.person_create, 'tid': now() - DateTimeDelta(5),}) d = self.client.callRemote('generate_token', id_type="externalid_studentnr", ext_id='008', username='******', phone_no='98012345', browser_token='a') d.addCallback(self.assertEquals, 'true') return d
def CheckDateTypes(self): dt = DateTime(2002, 6, 15) dtd = DateTimeDelta(0, 0, 0, 1) self.cur.execute("create table test (t timestamp)") self.cur.execute("insert into test(t) values (?)", (dt, )) self.cur.execute("select t from test") res = self.cur.fetchone() self.failUnlessEqual( dt, res.t, "DateTime object should have been %s, was %s" % (repr(dt), repr(res.t))) self.cur.execute("drop table test") self.cur.execute("create table test(i interval)") self.cur.execute("insert into test(i) values (?)", (dtd, )) self.cur.execute("select i from test") res = self.cur.fetchone() self.failUnlessEqual( dtd, res.i, "DateTimeDelta object should have been %s, was %s" % (repr(dtd), repr(res.i)))
def disable_account(account_id, db): """Disable account corresponding to account_id. Once disabled, an account is just a placeholder for the username. It has no other value/associated attributes. NB! We keep entity_contact_info around, since we may want to know at least some human-'relatable' info about the account after it's disabled """ account = get_account(account_id, db) if not account: return # If the account has already been deleted, we cannot disable it any # further. if account.is_deleted(): return delete_common(account.entity_id, db) account.expire_date = now() - DateTimeDelta(1) account.write_db() logger.debug("Disabled account %s (id=%s)", account.account_name, account.entity_id)
from logilab.common.db import get_dbapi_compliant_module from maay.dbentity import ScoredDocument, Document, FileInfo, \ DBEntity, DocumentProvider, DocumentScore, Word, Node, Result from maay.texttool import normalizeText, WORDS_RGX, MAX_STORED_SIZE IntegrityError = None class MaayAuthenticationError(Exception): """raised on db authentication failure""" ANONYMOUS_AVATARID = '__anonymous__' WEB_AVATARID = '__may__' ONE_HOUR = DateTimeDelta(0, 1) class IQuerier(Interface): """defines the High-Level interface to Maay SQL database""" def findDocuments(query): """returns all Documents matching <query>""" def getFileInformations(filename): """returns a list of FileInfo's instances according to DB's content """ def getIndexedFiles(): """returns a list of indexed file names as strings """
def run(self): self.parse_options() if self.opt.file != None: lease_elem = ET.parse(self.opt.file).getroot() # If a relative starting time is used, replace for an # absolute starting time. exact = lease.find("start/exact") if exact != None: exact_time = exact.get("time") exact.set("time", str(self.__absolute_time(exact_time))) lease_xml_str = ET.tostring(lease_elem) else: if self.opt.preemptible == None: preemptible = False else: preemptible = self.opt.preemptible capacity = Capacity([constants.RES_CPU, constants.RES_MEM]) capacity.set_quantity(constants.RES_CPU, int(self.opt.cpu) * 100) capacity.set_quantity(constants.RES_MEM, int(self.opt.mem)) requested_resources = dict([(i + 1, capacity) for i in range(self.opt.numnodes)]) if self.opt.duration == haizea_request_lease.DURATION_UNLIMITED: # This is an interim solution (make it run for a century). # TODO: Integrate concept of unlimited duration in the lease datastruct duration = DateTimeDelta(36500) else: duration = ISO.ParseTimeDelta(self.opt.duration) if self.opt.start == haizea_request_lease.START_NOW: lease = Lease(lease_id=None, submit_time=None, requested_resources=requested_resources, start=Timestamp(Timestamp.NOW), duration=Duration(duration), deadline=None, preemptible=preemptible, software=DiskImageSoftwareEnvironment( self.opt.vmimage, self.opt.vmimagesize), state=None) elif self.opt.start == haizea_request_lease.START_BESTEFFORT: lease = Lease(lease_id=None, submit_time=None, requested_resources=requested_resources, start=Timestamp(Timestamp.UNSPECIFIED), duration=Duration(duration), deadline=None, preemptible=preemptible, software=DiskImageSoftwareEnvironment( self.opt.vmimage, self.opt.vmimagesize), state=None) else: start = self.__absolute_time(self.opt.start) lease = Lease(lease_id=None, submit_time=None, requested_resources=requested_resources, start=Timestamp(start), duration=Duration(duration), deadline=None, preemptible=preemptible, software=DiskImageSoftwareEnvironment( self.opt.vmimage, self.opt.vmimagesize), state=None) lease_xml_str = ET.tostring(lease.to_xml()) server = self.create_rpc_proxy(self.opt.server) try: lease_id = server.create_lease(lease_xml_str) print "Lease submitted correctly." print "Lease ID: %i" % lease_id except xmlrpclib.Fault, err: print >> sys.stderr, "XMLRPC fault: %s" % err.faultString if self.opt.debug: raise
def __init__(self, opennebula_vm): # If there is no HAIZEA parameter, the default is to treat the # request as an immediate request with unlimited duration if not opennebula_vm.template.has_key(OpenNebulaHaizeaVM.HAIZEA_PARAM): self.start = OpenNebulaHaizeaVM.HAIZEA_START_NOW self.duration = OpenNebulaHaizeaVM.HAIZEA_DURATION_UNLIMITED self.preemptible = OpenNebulaHaizeaVM.HAIZEA_PREEMPTIBLE_NO self.group = None else: self.start = opennebula_vm.template[OpenNebulaHaizeaVM.HAIZEA_PARAM][OpenNebulaHaizeaVM.HAIZEA_START] self.duration = opennebula_vm.template[OpenNebulaHaizeaVM.HAIZEA_PARAM][OpenNebulaHaizeaVM.HAIZEA_DURATION] self.preemptible = opennebula_vm.template[OpenNebulaHaizeaVM.HAIZEA_PARAM][OpenNebulaHaizeaVM.HAIZEA_PREEMPTIBLE] if opennebula_vm.template[OpenNebulaHaizeaVM.HAIZEA_PARAM].has_key(OpenNebulaHaizeaVM.HAIZEA_GROUP): self.group = opennebula_vm.template[OpenNebulaHaizeaVM.HAIZEA_PARAM][OpenNebulaHaizeaVM.HAIZEA_GROUP] else: self.group = None self.submit_time = UNIX2DateTime(opennebula_vm.stime) # Create Timestamp object if self.start == OpenNebulaHaizeaVM.HAIZEA_START_NOW: self.start = Timestamp(Timestamp.NOW) elif self.start == OpenNebulaHaizeaVM.HAIZEA_START_BESTEFFORT: self.start = Timestamp(Timestamp.UNSPECIFIED) elif self.start[0] == "+": # Relative time self.start = Timestamp(round_datetime(self.submit_time + ISO.ParseTime(self.start[1:]))) else: self.start = Timestamp(ISO.ParseDateTime(self.start)) # Create Duration object if self.duration == OpenNebulaHaizeaVM.HAIZEA_DURATION_UNLIMITED: # This is an interim solution (make it run for a century). # TODO: Integrate concept of unlimited duration in the lease datastruct self.duration = Duration(DateTimeDelta(36500)) else: self.duration = Duration(ISO.ParseTimeDelta(self.duration)) self.preemptible = (self.preemptible == OpenNebulaHaizeaVM.HAIZEA_PREEMPTIBLE_YES) self.capacity = Capacity([constants.RES_CPU, constants.RES_MEM, constants.RES_DISK]) # CPU # CPUs in VMs are not reported the same as in hosts. # THere are two template values: CPU and VCPU. # CPU reports the percentage of the CPU needed by the VM. # VCPU, which is optional, reports how many CPUs are needed. cpu = int(float(opennebula_vm.template["CPU"]) * 100) if opennebula_vm.template.has_key("VCPU"): ncpu = int(opennebula_vm.template["VCPU"]) else: ncpu = 1 self.capacity.set_ninstances(constants.RES_CPU, ncpu) for i in range(ncpu): self.capacity.set_quantity_instance(constants.RES_CPU, i+1, cpu) # Memory. Unlike hosts, memory is reported directly in MBs self.capacity.set_quantity(constants.RES_MEM, int(opennebula_vm.template["MEMORY"])) self.one_id = opennebula_vm.id
def runtest(lmaManager=None, lma_view=None, HDFmanagers=None): # colormap = get_cmap('gist_yarg_r') colormap = get_cmap('gist_earth') density_maxes = [] total_counts = [] all_t = [] for delta_minutes in minute_intervals: time_delta = DateTimeDelta(0, 0, delta_minutes, 0) n_frames = int(ceil((end_time - start_time) / time_delta)) n_cols = 6 n_rows = int(ceil(float(n_frames) / n_cols)) w, h = figaspect(float(n_rows) / n_cols) xedge = np.arange(b.x[0], b.x[1] + dx, dx) yedge = np.arange(b.y[0], b.y[1] + dy, dy) x_range = b.x[1] - b.x[0] y_range = b.y[1] - b.y[0] min_count, max_count = 1, max_count_baseline * delta_minutes f = figure(figsize=(w, h)) p = small_multiples_plot(fig=f, rows=n_rows, columns=n_cols) p.label_edges(True) for ax in p.multiples.flat: ax.yaxis.set_major_formatter(kilo_formatter) ax.xaxis.set_major_formatter(kilo_formatter) for i in range(n_frames): frame_start = start_time + i * time_delta frame_end = frame_start + time_delta b.sec_of_day = (frame_start.abstime, frame_end.abstime) b.t = (frame_start, frame_end) do_plot = False flash_extent_density = True density = None if source_density == True: lmaManager.refresh(b) lma_view.transformed.cache_is_old() x, y, t = lma_view.transformed['x', 'y', 't'] density, edges = np.histogramdd((x, y), bins=(xedge, yedge)) do_plot = True else: for lmaManager in HDFmanagers: # yes, loop through every file every time and reselect data. # so wrong, yet so convenient. h5 = lmaManager.h5file if flash_extent_density == False: lmaManager.refresh(b) lma_view = AcuityView(DataSelection( lmaManager.data, b), mapProj, bounds=b) # lma_view.transformed.cache_is_old() x, y, t = lma_view.transformed['x', 'y', 't'] if x.shape[0] > 1: do_plot = True break else: # assume here that the bounds sec_of_day day is the same as # the dataset day t0, t1 = b.sec_of_day # events = getattr(h5.root.events, lmaManager.table.name)[:] # flashes = getattr(h5.root.flashes, lmaManager.table.name)[:] event_dtype = getattr(h5.root.events, lmaManager.table.name)[0].dtype events_all = getattr(h5.root.events, lmaManager.table.name)[:] flashes = getattr(h5.root.flashes, lmaManager.table.name) def event_yielder(evs, fls): these_events = [] for fl in fls: if ((fl['n_points'] > 9) & (t0 < fl['start']) & (fl['start'] <= t1)): these_events = evs[evs['flash_id'] == fl['flash_id']] if len(these_events) != fl['n_points']: print('not giving all ', fl['n_points'], ' events? ', these_events.shape) for an_ev in these_events: yield an_ev # events = np.fromiter((an_ev for an_ev in ( events_all[events_all['flash_id'] == fl['flash_id']] # for fl in flashes if ( # (fl['n_points']>9) & (t0 < fl['start']) & (fl['start'] <= t1) # ) # ) ), dtype=event_dtype) events = np.fromiter(event_yielder( events_all, flashes), dtype=event_dtype) # print events['flash_id'].shape ### Flash extent density ### x, y, z = mapProj.fromECEF(*geoProj.toECEF( events['lon'], events['lat'], events['alt'])) # Convert to integer grid coordinate bins # 0 1 2 3 # | | | | | # -1.5 0.0 1.5 3.0 4.5 if x.shape[0] > 1: density, edges = extent_density( x, y, events['flash_id'].astype('int32'), b.x[0], b.y[0], dx, dy, xedge, yedge) do_plot = True break # print 'density values: ', density.min(), density.max() if do_plot == True: # need some data # density,edges = np.histogramdd((x,y), bins=(xedge,yedge)) density_plot = p.multiples.flat[i].pcolormesh( xedge, yedge, np.log10(density.transpose()), vmin=-0.2, vmax=np.log10(max_count), cmap=colormap) label_string = frame_start.strftime('%H%M:%S') text_label = p.multiples.flat[i].text( b.x[0] - pad + x_range * .01, b.y[0] - pad + y_range * .01, label_string, color=(0.5, ) * 3, size=6) density_plot.set_rasterized(True) density_maxes.append(density.max()) total_counts.append(density.sum()) all_t.append(frame_start) print(label_string, x.shape, density.max(), density.sum()) color_scale = ColorbarBase(p.colorbar_ax, cmap=density_plot.cmap, norm=density_plot.norm, orientation='horizontal') # color_scale.set_label('count per pixel') color_scale.set_label('log10(count per pixel)') # moving reference frame correction. all panels will have same limits, based on time of last frame view_dt = 0.0 # (frame_start - t0).seconds x_ctr = x0 + view_dt * u y_ctr = y0 + view_dt * v view_x = (x_ctr - view_dx / 2.0 - pad, x_ctr + view_dx / 2.0 + pad) view_y = (y_ctr - view_dy / 2.0 - pad, y_ctr + view_dy / 2.0 + pad) # view_x = (b.x[0]+view_dt*u, b.x[1]+view_dt*u) # view_y = (b.y[0]+view_dt*v, b.y[1]+view_dt*v) # print 'making timeseries', # time_series = figure(figsize=(16,9)) # ts_ax = time_series.add_subplot(111) # ts_ax.plot_date(mx2num(all_t),total_counts,'-', label='total sources', tz=tz) # ts_ax.plot_date(mx2num(all_t),density_maxes,'-', label='max pixel', tz=tz) # ts_ax.xaxis.set_major_formatter(time_series_x_fmt) # ts_ax.legend() # time_filename = 'out/LMA-timeseries_%s_%5.2fkm_%5.1fs.pdf' % (start_time.strftime('%Y%m%d_%H%M%S'), dx/1000.0, time_delta.seconds) # time_series.savefig(time_filename) # print ' ... done' print('making multiples', end=' ') p.multiples.flat[0].axis(view_x + view_y) filename = 'out/LMA-density_%s_%5.2fkm_%5.1fs.pdf' % ( start_time.strftime('%Y%m%d_%H%M%S'), dx / 1000.0, time_delta.seconds) f.savefig(filename, dpi=150) print(' ... done') f.clf() return events
class VirtGroup(Group_class, EntityContactInfo, EntityTrait): """VirtGroup extension of VirtHome. This class tailors Cerebrum.Group:Group to VirtHome's needs. TBD: Do we need to make a special version of *_member() functions that check the member type? (i.e. can a virtgroup have any other members than VA/FA?) """ DEFAULT_GROUP_LIFETIME = DateTimeDelta(180) def __init__(self, *rest, **kw): self.__super.__init__(*rest, **kw) self.legal_chars = set(string.letters + string.digits + " .@-") # end __init__ def populate(self, creator_id, name, description): """Populate a VirtGroup's instance, subject to some constraints.""" assert not self.illegal_name(name), "Invalid group name %s" % (name, ) assert description.strip() self.__super.populate( creator_id=creator_id, visibility=self.const.group_visibility_all, name=name, description=description, group_type=self.const.group_type_unknown, ) #self.expire_date = now() + self.DEFAULT_GROUP_LIFETIME # end populate def set_group_resource(self, url): """Unconditionally reassign a new URL to this group. If URL is None (or ''), we'll remove whatever URL was in the database. """ # Check the URL's validity before doing anything to the database. self.verify_group_url(url) resources = self.get_contact_info(self.const.system_virthome, self.const.virthome_group_url) if resources: # There can be at most one URL... r = resources[0] # If the old value matches the new one, there is nothing to do. if r["contact_value"] == url: return # If the old value doesn't match the new one, we delete the old one # first. Helps us avoid multiple URLs for one group. self.delete_contact_info(self.const.system_virthome, self.const.virthome_group_url) if url: self.add_contact_info(self.const.system_virthome, self.const.virthome_group_url, url) # end set_group_resource def verify_group_url(self, url): """Check that the URL at least looks sane. We allow empty/None values here. """ if not url: return True resource = urlparse.urlparse(url) if resource.scheme not in ( "http", "https", "ftp", ): raise ValueError("Invalid url for group <%s>: <%s>" % (self.group_name, url)) return True # end verify_group_url def illegal_name(self, name): """Return a string with error message if groupname is illegal""" if not name.strip(): return "Group name is empty" if (name.startswith(" ") or name.endswith(" ")): return "Group name cannot start/end with space" if any(x not in self.legal_chars for x in name): return "Illegal character in group name" if name.count("@") != 1: return "Group name is missing a realm" tmp_name, tmp_realm = name.split("@") if tmp_realm != cereconf.VIRTHOME_REALM: return "Wrong realm <%s> for VirtGroup <%s>" return False
def process_affiliations(employment_file, person_file, use_fok, people_to_ignore=None): """Parse employment_file and determine all affiliations. There are roughly 3 distinct parts: #. Cache all the affiliations in Cerebrum #. Scan the file and compare the file data with the cache. When there is a match, remove the entry from the cache. #. Remove from Cerebrum whatever is left in the cache (once we are done with the file, the cache contains those entries that were in Cerebrum """ expired = load_expired_employees(file(person_file), use_fok, logger) # First we cache all existing affiliations. It's a mapping person-id => # mapping (ou-id, affiliation) => status. affiliation_cache = cache_db_affiliations() person_cache = dict() def person_cacher(empid): ret = person_cache.get(empid, NotSet) if ret is NotSet: ret = person_cache[empid] = get_person(empid) return ret for tpl in make_employment_iterator(file(employment_file), use_fok, logger): if not tpl.valid(): logger.debug( "Ignored invalid entry for person while " "processing affiliation: «%s»", tpl.sap_ansattnr) continue if people_to_ignore and tpl.sap_ansattnr in people_to_ignore: logger.debug("Invalid person with sap_id=%s", tpl.sap_ansattnr) continue if tpl.sap_ansattnr in expired: logger.debug( "Person sap_id=%s is no longer an employee; " "all employment info will be ignored", tpl.sap_ansattnr) continue # is the entry within a valid time frame? # The shift by 180 days has been requested by UiA around 2007-03-27 if not (tpl.start_date - DateTimeDelta(180) <= today() <= tpl.end_date): logger.debug("Entry %s has wrong timeframe (start: %s, end: %s)", tpl, tpl.start_date, tpl.end_date) continue ou_id = get_ou_id(tpl.sap_ou_id) if ou_id is None: logger.warn( "Cannot map SAP OU %s to Cerebrum ou_id (employment " "for person sap_id=%s).", tpl.sap_ou_id, tpl.sap_ansattnr) continue person = person_cacher(tpl.sap_ansattnr) if person is None: logger.warn("Cannot map SAP ansattnr %s to cerebrum person_id", tpl.sap_ansattnr) continue (affiliation, affiliation_status) = sap_employment2affiliation(tpl.lonnstittel) synchronize_affiliations(affiliation_cache, person, ou_id, affiliation, affiliation_status) # We are done with fetching updates from file. # Need to write persons for p in person_cache.values(): if p is None: continue logger.info("Writing cached affs for person id:%s", p.entity_id) p.write_db() # All the affiliations left in the cache exist in Cerebrum, but NOT in the # datafile. Thus delete them! remove_affiliations(affiliation_cache)
_make_timezone( LocalTimezone, ('LOCAL', ), ) # ------------------------------------------------- # STANDARD (NON-DAYLIGHT) TIMEZONES # # The following timezones do not implement daylight # saving time. Thus they are Timezone instances, # and not TimezoneDST instances. # good old UTC _make_timezone(Timezone, ('UTC', 'GMT', 'UCT', 'Universal', 'Greenwich'), offset='-0000', offset_delta=DateTimeDelta(0, 0, 0)) # Argentina and Uruguay (GMT-3) _make_timezone(Timezone, ( 'Argentina', 'ar', 'Uruguay', 'Uruguai', 'uy', ), offset='-0300', offset_delta=DateTimeDelta(0, 0, -180)) # GMT-4 for LatAms _make_timezone( Timezone,
def _make_offset_delta(tup): delta = DateTimeDelta(0, int(tup[1]), int(tup[2])) if tup[0] == '-': delta = -1 * delta return delta
def get_matching_accs(db): """ Get defunct account data. This function searches the database for accounts where: - account is not expired - account is owned by a person with no affiliations - account has been quarantined for > 1 year :return generator: A generator that yields dicts with account and quarantine data """ ac = Factory.get('Account')(db) pe = Factory.get('Person')(db) co = Factory.get('Constants')(db) def _u(db_value): if db_value is None: return text_type('') if isinstance(db_value, bytes): return db_value.decode(db.encoding) return text_type(db_value) def _row_to_quar(row): """ list_entity_quarantines row to dict """ return { 'q_type': text_type(co.Quarantine(row['quarantine_type'])), 'q_desc': _u(row['description']), 'q_date': text_type(row['start_date'].strftime('%Y-%m-%d')), } logger.debug('caching personal accounts ...') owner_type = co.entity_person accounts = ac.search(owner_type=owner_type) logger.info('found %d accounts with owner_type=%r', len(accounts), text_type(owner_type)) logger.debug('caching account homedirs ...') acc2disk = dict( (r['account_id'], r['path']) for r in ac.list_account_home()) logger.info('found %d accounts assigned to a disk', len(acc2disk)) logger.debug('caching active account quarantines ...') acc2quar = defaultdict(list) for q in ac.list_entity_quarantines(only_active=True, entity_types=co.entity_account): acc2quar[q['entity_id']].append(q) logger.info('found quarantines for %d accounts', len(acc2quar)) logger.debug('caching person names ...') person2name = dict( (r['person_id'], r['name']) for r in pe.search_person_names(name_variant=co.name_full, source_system=co.system_cached)) logger.info('found full names for %d persons', len(person2name)) # Add person_id to the list if the person has an affiliation logger.debug('caching person affiliations ...') person_has_affs = set((r['person_id'] for r in pe.list_affiliations())) logger.info('found %d persons with affiliations', len(person_has_affs)) for acc in accounts: # Is the account owner still affiliated? if acc['owner_id'] in person_has_affs: continue for quar in acc2quar[acc['account_id']]: if (quar['start_date'] + DateTimeDelta(365)) < now(): break else: # loop terminated wihtout finding a 'quar' -- i.e. no active # quarantine older than one year continue yield { 'account_name': _u(acc['name']), 'full_name': _u(person2name.get(acc['owner_id'])) or u'(not set)', 'disk_path': _u(acc2disk.get(acc['account_id'])) or u'(not set)', 'q_type': text_type(co.Quarantine(quar['quarantine_type'])), 'q_desc': _u(quar['description']), 'q_date': text_type(quar['start_date'].strftime('%Y-%m-%d')), }