def calc_delay(self, cr, uid, vals): """ Calculates delay of work order. @return: Delay """ code_lst = [] time_lst = [] code_ids = self.pool.get('mrp_operations.operation.code').search(cr, uid, [('id','=',vals['code_id'])]) code = self.pool.get('mrp_operations.operation.code').browse(cr, uid, code_ids)[0] oper_ids = self.search(cr,uid,[('production_id','=',vals['production_id']),('workcenter_id','=',vals['workcenter_id'])]) oper_objs = self.browse(cr,uid,oper_ids) for oper in oper_objs: code_lst.append(oper.code_id.start_stop) time_lst.append(oper.date_start) code_lst.append(code.start_stop) time_lst.append(vals['date_start']) diff = 0 for i in range(0,len(code_lst)): if code_lst[i] == 'pause' or code_lst[i] == 'done' or code_lst[i] == 'cancel': if not i: continue if code_lst[i-1] not in ('resume','start'): continue a = datetime.strptime(time_lst[i-1],'%Y-%m-%d %H:%M:%S') b = datetime.strptime(time_lst[i],'%Y-%m-%d %H:%M:%S') diff += (b-a).days * 24 diff += (b-a).seconds / float(60*60) return diff
def day_is_complete(day_str): # Return true if there are evenly spaced logs throughout the day day_start = datetime.strptime(day_str + " 00:00", "%Y-%m-%d %H:%M") day_end = datetime.strptime(day_str + " 23:59", "%Y-%m-%d %H:%M") arp_logs = ArpLog.objects.filter(runtime__gt=day_start, runtime__lt=day_end).order_by('runtime') print(arp_logs.count()) return True
def json_date_to_datetime(dateraw): cy = datetime.isocalendar(date.today())[0] try: newdate = datetime.strptime(dateraw + str(cy), '%b %d%Y') except ValueError: newdate = datetime.strptime(dateraw, '%m/%d/%y') return newdate
def get_data(var_id, row, col): base_url = ('http://dapds00.nci.org.au/thredds/dodsC/rr9/Climate/eMAST/' 'ANUClimate/0_01deg/v1m0_aus/mon/land/%s/e_01/1970_2012/' % (var_id)) emast_id = "eMAST_ANUClimate_mon_tmax_v1m0" start_date = "1970-01-01" stop_date = "2000-12-31" current = datetime.strptime(start_date, "%Y-%m-%d") stop = datetime.strptime(stop_date, "%Y-%m-%d") tmax = [] dates = [] while current < stop: if current.month < 10: month = "0%s" % (current.month) else: month = "%s" % (current.month) year = current.year url = "%s%s_%s%s.nc" % (base_url, emast_id, year, month) dataset = open_url(url) variable = dataset['air_temperature'] #print variable[0,2000:2005,2000:2005].array[:] tmax.append(variable[0,2000:2005,2000:2005].array[:][0][0][0]) dates.append(current) current += relativedelta(months=1) f = open("tmax_%d_%d.txt" % (row, col), "w") for i in xrange(tmax): f >> tmax f.close()
def search(self, serie, numero, remitente, destinatario, sucursal, fecha): flat = serie or numero or remitente or destinatario or fecha tz = timezone.get_current_timezone() if flat: if fecha: "si se ingreso fecha" date = datetime.strptime(fecha, "%d/%m/%Y") end_date = timezone.make_aware(date, tz) start_date = end_date - timedelta(days=7) else: date = datetime.strptime("01/10/2015", "%d/%m/%Y") end_date = timezone.now() start_date = timezone.make_aware(date, tz) else: end_date = timezone.now() start_date = end_date - timedelta(days=7) busqueda = self.annotate( saldo=F('depositslip__total_amount')-F('amount') ).filter( depositslip__serie__icontains=serie, depositslip__number__icontains=numero, depositslip__sender__full_name__icontains=remitente, depositslip__addressee__full_name__icontains=destinatario, depositslip__state='2', depositslip__destination=sucursal, depositslip__created__range=(start_date, end_date) ) return busqueda
def _compute_day(self, cr, uid, ids, fields, args, context=None): """ @param cr: the current row, from the database cursor, @param uid: the current user’s ID for security checks, @param ids: List of Openday’s IDs @return: difference between current date and log date @param context: A standard dictionary for contextual values """ res = {} for issue in self.browse(cr, uid, ids, context=context): for field in fields: res[issue.id] = {} duration = 0 ans = False hours = 0 if field in ['day_open']: if issue.date_open: date_create = datetime.strptime(issue.create_date, "%Y-%m-%d %H:%M:%S") date_open = datetime.strptime(issue.date_open, "%Y-%m-%d %H:%M:%S") ans = date_open - date_create elif field in ['day_close']: if issue.date_closed: date_create = datetime.strptime(issue.create_date, "%Y-%m-%d %H:%M:%S") date_close = datetime.strptime(issue.date_closed, "%Y-%m-%d %H:%M:%S") ans = date_close - date_create if ans: duration = float(ans.days) res[issue.id][field] = abs(float(duration)) return res
def _newInstance(self, name, value): types = self._getPropertyType(name) propertyType = types[0] isArray = types[1] if propertyType == bool: if isArray: return [x for x in value[value.keys()[0]]] else: return value elif propertyType == datetime: format = "%Y-%m-%d %H:%M:%S" if isArray: return [datetime.strptime(x, format) for x in value[value.keys()[0]]] else: return datetime.strptime(value, format) elif propertyType == str: if isArray: return [x.encode("utf-8") for x in value[value.keys()[0]]] else: #like taobao.simba.rpt.adgroupbase.get, response.rpt_adgroup_base_list is a json string,but will be decode into a list via python json lib if not isinstance(value,str): #the value should be a json string return value return value.encode("utf-8") else: if isArray: return [propertyType(x) for x in value[value.keys()[0]]] else: return propertyType(value)
def valid_admin_cookie(cookie): if g.read_only_mode: return (False, None) # parse the cookie try: first_login, last_request, hash = cookie.split(',') except ValueError: return (False, None) # make sure it's a recent cookie try: first_login_time = datetime.strptime(first_login, COOKIE_TIMESTAMP_FORMAT) last_request_time = datetime.strptime(last_request, COOKIE_TIMESTAMP_FORMAT) except ValueError: return (False, None) cookie_age = datetime.utcnow() - first_login_time if cookie_age.total_seconds() > g.ADMIN_COOKIE_TTL: return (False, None) idle_time = datetime.utcnow() - last_request_time if idle_time.total_seconds() > g.ADMIN_COOKIE_MAX_IDLE: return (False, None) # validate expected_cookie = c.user.make_admin_cookie(first_login, last_request) return (constant_time_compare(cookie, expected_cookie), first_login)
def __parser_creditcard_transaction(row, report): if len(row) != 27: report['lines_with_wrong_size'].append(row) raise ValueError('The expected parameter count for CreditCardTransaction parser is 27', len(row)) transaction = { 'order_key': UUID(row[1]), 'order_reference': row[2], 'merchant_key': UUID(row[3]), 'merchant_name': row[4], 'transaction_key': UUID(row[5]), 'transaction_key_to_acquirer': row[6], 'transaction_reference': row[7], 'creditcard_brand': row[8], 'creditcard_number': row[9], 'installment_count': int(row[10]) if len(row[10].strip()) > 0 else 0, 'acquirer_name': row[11], 'status': row[12], 'amount_in_cents': long(row[13]) if len(row[13].strip()) > 0 else long(0), 'iata_amount_in_cents': long(row[14]) if len(row[14].strip()) > 0 else long(0), 'authorization_code': row[15], 'transaction_identifier': row[16], 'unique_sequential_number': row[17], 'captured_amount_in_cents': long(row[19]) if len(row[19].strip()) > 0 else long(0), 'authorized_amount_in_cents': long(row[18]) if len(row[18].strip()) > 0 else long(0), 'voided_amount_in_cents': long(row[20]) if len(row[20].strip()) > 0 else long(0), 'refunded_amount_in_cents': long(row[21]) if len(row[21].strip()) > 0 else long(0), 'acquirer_authorization_return_code': row[22], 'authorized_date': datetime.strptime(row[23], "%Y-%m-%dT%H:%M:%S") if len(row[23].strip()) > 0 else None, 'captured_date': datetime.strptime(row[24], "%Y-%m-%dT%H:%M:%S") if len(row[24].strip()) > 0 else None, 'voided_date': datetime.strptime(row[25], "%Y-%m-%dT%H:%M:%S") if len(row[25].strip()) > 0 else None, 'last_probe_date': datetime.strptime(row[26], "%Y-%m-%dT%H:%M:%S") if len(row[26].strip()) > 0 else None } report['creditcard_transaction_collection'].append(transaction)
def check_rotation_needed(tree): """ Browse the master keys and check their creation date to display a warning if older than 6 months (it's time to rotate). """ show_rotation_warning = False six_months_ago = datetime.utcnow()-timedelta(days=183) if 'kms' in tree['sops']: for entry in tree['sops']['kms']: # check if creation date is older than 6 months if 'created_at' in entry: d = datetime.strptime(entry['created_at'], '%Y-%m-%dT%H:%M:%SZ') if d < six_months_ago: show_rotation_warning = True if 'pgp' in tree['sops']: for entry in tree['sops']['pgp']: # check if creation date is older than 6 months if 'created_at' in entry: d = datetime.strptime(entry['created_at'], '%Y-%m-%dT%H:%M:%SZ') if d < six_months_ago: show_rotation_warning = True if show_rotation_warning: print("INFO: the data key on this document is over 6 months old. " "Considering rotating it with $ sops -r <file> ", file=sys.stderr)
def _parse_datetime_value(value): # timezones are not supported and are assumed UTC if value[-1] == 'Z': value = value[:-1] value_len = len(value) if value_len in (8, 10): value = datetime.strptime(value, '%Y-%m-%d').replace( tzinfo=timezone.utc, ) return [value, value + timedelta(days=1)] elif value[4] == '-': try: value = datetime.strptime(value, '%Y-%m-%dT%H:%M:%S').replace( tzinfo=timezone.utc, ) except ValueError: value = datetime.strptime(value, '%Y-%m-%dT%H:%M:%S.%f').replace( tzinfo=timezone.utc, ) else: value = datetime.utcfromtimestamp(float(value)).replace( tzinfo=timezone.utc, ) return [value - timedelta(minutes=5), value + timedelta(minutes=6)]
def sendSubscriber(request): results= {} if request.method == 'POST': subName = request.POST.get('subName') subArrival = request.POST.get('subArrival') subArrival = datetime.strptime(subArrival, "%d.%m.%Y") subDeparture= request.POST.get('subDeparture') subDeparture = datetime.strptime(subDeparture, "%d.%m.%Y") subSwitch = request.POST.get('subSwitch') subOther = request.POST.get('subOther'); s = Subscriber() s.name = subName s.arrival = subArrival s.departure = subDeparture s.switch = subSwitch s.other = subOther s.save() results['subName'] = subName results['subArrival'] = subArrival.strftime("%d.%m.%Y") results['subDeparture']=subDeparture.strftime("%d.%m.%Y") results['subSwitch'] = subSwitch results['subOther'] = subOther j = json.dumps(results) return HttpResponse(j, content_type='application/json')
def testformat(s): """Parse a date and returns a dict. The dict contains the date string, format, and an error message if the date cannot be parsed. """ error = '' try: d = datetime.strptime(s, '%Y-%m-%d') dateformat = 'date' except: try: d = datetime.strptime(s, '%Y-%m-%dT%H:%M:%SZ') dateformat = 'datetime' except: dateformat = 'unknown' if dateformat == 'unknown': try: d = dateutil.parser.parse(s) # s = d.strftime("%Y-%m-%dT%H:%M:%SZ") # dateformat = 'datetime' s = d.strftime("%Y-%m-%d") dateformat = 'date' except: error = 'Could not parse date "' + s + '" into a valid format.' if error == '': response = {'text': s, 'format': dateformat} else: response = {'text': s, 'format': 'unknown', 'error': error} return response
def get_params_from_request(self): """ Get params from request GET """ # designer designer = self.request.GET.get('designer') designer = designer if designer else 'all' # status status = self.request.GET.get('status') status = status if status else 'all' # style style = self.request.GET.get('style') style = style if style else 'all' # age group age_group = self.request.GET.get('age-group') age_group = age_group if age_group else 'all' # created time try: created_from = self.request.GET.get('created-from') created_from_dt = datetime.strptime(created_from, '%m/%d/%Y') except: created_from_dt = None try: created_to = self.request.GET.get('created-to') created_to_dt = datetime.strptime(created_to, '%m/%d/%Y') except: created_to_dt = None return {'status': status, 'style': style, 'age_group': age_group, 'created_from': created_from_dt, 'created_to': created_to_dt, 'designer': designer}
def pesquisarFluxos(request): if request.method == 'POST': pessoaBusca = request.POST.get('pessoaBusca') dataBuscaInicio = datetime.strptime(request.POST.get('dataBuscaInicio', ''), '%d/%m/%Y %H:%M:%S') dataBuscaFim = datetime.strptime(request.POST.get('dataBuscaFinal', ''), '%d/%m/%Y %H:%M:%S') nome = Pessoa.objects.filter(id=pessoaBusca) pessoas = Pessoa.objects.all().order_by('nome') totalreceber = 0 totalpagar = 0 try: sql = "select * from caixas_conta where pessoa_id like %s and data >= '%s' and data <= '%s'" % (pessoaBusca, dataBuscaInicio, dataBuscaFim) contas = Conta.objects.raw(sql) for item in contas: if item.tipo == 'E': totalreceber = totalreceber + item.valor else: totalpagar = totalpagar + item.valor total = totalreceber - totalpagar except: contas = [Conta(descricao='erro')] return render(request, 'fluxos/listarFluxos.html', {'contas': contas, 'nome':nome, 'pessoas': pessoas, 'totalreceber':totalreceber,'totalpagar':totalpagar, 'total':total})
def init(self, cr, uid=1): """ This view will be used in dashboard The reason writing this code here is, we need to check date range from today to first date of fiscal year. """ pool_obj_fy = self.pool['account.fiscalyear'] today = time.strftime('%Y-%m-%d') fy_id = pool_obj_fy.find(cr, uid, exception=False) LIST_RANGES = [] if fy_id: fy_start_date = pool_obj_fy.read(cr, uid, fy_id, ['date_start'])['date_start'] fy_start_date = datetime.strptime(fy_start_date, '%Y-%m-%d') last_month_date = datetime.strptime(today, '%Y-%m-%d') - relativedelta(months=1) while (last_month_date > fy_start_date): LIST_RANGES.append(today + " to " + last_month_date.strftime('%Y-%m-%d')) today = (last_month_date- relativedelta(days=1)).strftime('%Y-%m-%d') last_month_date = datetime.strptime(today, '%Y-%m-%d') - relativedelta(months=1) LIST_RANGES.append(today +" to " + fy_start_date.strftime('%Y-%m-%d')) cr.execute('delete from temp_range') for range in LIST_RANGES: self.pool['temp.range'].create(cr, uid, {'name':range}) cr.execute(""" create or replace view report_aged_receivable as ( select id,name from temp_range )""")
def calc_elapsed_time(params): f=params['inputfile'] o = '%s_elapsedTime.txt' %(params['inputfile'][:-4]) o1 = open(o,'w') tot=len(open(f,'r').readlines()) count=1 while count <=tot-1: line = linecache.getline(f,count) linenext=linecache.getline(f,count+1) if params['debug'] is True: print line[:-2] print linenext[:-2] region1=line.split()[5] region2=linenext.split()[5] if region1 != region2: if params['debug'] is True: print '<--------------------------------->Next region..' count = count + 1 continue t1=datetime.strptime(line.split()[2],'%Y-%m-%dT%H:%M:%S-0500') t2=datetime.strptime(linenext.split()[2],'%Y-%m-%dT%H:%M:%S-0500') price=line.split()[1] if params['debug'] is True: print (t1-t2).seconds o1.write('%s\t%s\t%f\n' %(region1,price,(t1-t2).seconds)) count = count + 1
def print_report(self, cr, uid, ids, context=None): """ To get the date and print the report @param self: The object pointer. @param cr: A database cursor @param uid: ID of the user currently logged in @param context: A standard dictionary @return : retrun report """ if context is None: context = {} datas = {'ids': context.get('active_ids', [])} res = self.read(cr, uid, ids, ['date_start', 'date_end', 'location_ids','show_details'], context=context) res = res and res[0] or {} # get timedelta between user timezone and UTC utc_time = datetime.now() user_time = fields.datetime.context_timestamp(cr, uid, utc_time).replace(tzinfo=None) user_timedelta = utc_time - user_time res['date_start2'] = (datetime.strptime(res['date_start']+' 00:00:00',tools.DEFAULT_SERVER_DATETIME_FORMAT) + user_timedelta).strftime(tools.DEFAULT_SERVER_DATETIME_FORMAT) res['date_end2'] = (datetime.strptime(res['date_end']+' 23:59:59',tools.DEFAULT_SERVER_DATETIME_FORMAT) + user_timedelta).strftime(tools.DEFAULT_SERVER_DATETIME_FORMAT) datas['form'] = res if res.get('id',False): datas['ids']=[res['id']] return self.pool['report'].get_action(cr, uid, [], 'pos_sales_summary.report_summaryofsales', data=datas, context=context)
def validate_config(self,config): print 'VALIDATE CONFIG' if not config: return config try: config_obj = json.loads(config) if 'api_version' in config_obj: try: int(config_obj['api_version']) except ValueError: raise ValueError('api_version must be an integer') if 'default_tags' in config_obj: if not isinstance(config_obj['default_tags'],list): raise ValueError('default_tags must be a list') if 'default_groups' in config_obj: if not isinstance(config_obj['default_groups'],list): raise ValueError('default_groups must be a list') # Check if default groups exist context = {'model':model,'user':c.user} for group_name in config_obj['default_groups']: try: group = get_action('group_show')(context,{'id':group_name}) except NotFound,e: raise ValueError('Default group not found') if 'default_extras' in config_obj: if not isinstance(config_obj['default_extras'],dict): raise ValueError('default_extras must be a dictionary') if 'from' in config_obj: try: datetime.strptime(config_obj['from'], '%Y-%m-%dT%H:%M:%SZ') except ValueError: raise ValueError("Incorrect date format, should be yyyy-mm-ddThh:mm:ssZ ") # int(config_obj['from']) if 'until' in config_obj: try: datetime.strptime(config_obj['until'], '%Y-%m-%dT%H:%M:%SZ') except ValueError: raise ValueError("Incorrect date format, should be yyyy-mm-ddThh:mm:ssZ ") #if 'vocabulary' in config_obj: # if config_obj['vocabulary'] != 'metashare' and config_obj['vocabulary'] != 'olac' and config_obj['vocabulary'] !='cmdi': # raise ValueError("Incorrect vocabulary, please choose between metashare, olac and cmdi") #else: # raise ValueError("Please provide a vocabulary, you can choose between metashare, olac and cmdi") if 'user' in config_obj: # Check if user exists context = {'model':model,'user':c.user} try: user = get_action('user_show')(context,{'id':config_obj.get('user')}) except NotFound,e: raise ValueError('User not found')
def event_adjecents(request, start_date, start_time, end_date, end_time, event=None, locations=None): """Returns the events which take place adjacent to the to-be-planned event.""" start = datetime.strptime("%s %s" % (start_date, start_time), "%d-%m-%Y %H:%M") - timedelta(minutes=15) end = datetime.strptime("%s %s" % (end_date, end_time), "%d-%m-%Y %H:%M") + timedelta(minutes=15) realstart = datetime.strptime("%s %s" % (start_date, start_time), "%d-%m-%Y %H:%M") realend = datetime.strptime("%s %s" % (end_date, end_time), "%d-%m-%Y %H:%M") # Haal alle conflicting events op met een kwartier speling aan beide # einden. Haal vervolgens de de echte conflicting events eruit, zodat de # adjacent events overblijven. if locations: locations = Location.objects.filter(pk__in=locations) events = Event.objects.none() adjevents = Event.objects.none() for location in locations: events |= Event.conflicting_events(realstart, realend, location) adjevents |= Event.conflicting_events(start, end, location) else: events = Event.conflicting_events(realstart, realend) adjevents = Event.conflicting_events(start, end) if event: events = events.exclude(pk=event) result = [] for event in adjevents: if event not in events: result.append(model_to_dict(event)) return result
def load_movies(): """Load movies from u.item into database.""" print "Movie" Movie.query.delete() for row in open("seed_data/u.item"): row = row.rstrip().split('|') movie_id = row[0] #eliminate date at end of title title_temp = row[1].split("(") title = title_temp[0].rstrip() # make sure data doesn't suck try: released_at = datetime.strptime(row[2], "%d-%b-%Y") except: released_at = datetime.strptime('01-Jan-1900', "%d-%b-%Y") imdb_url = row[3] movie = Movie(movie_id=movie_id, title=title, released_at=released_at, imdb_url=imdb_url) db.session.add(movie) db.session.commit()
def hotspotsRange(start_time, stop_time, location, **kwargs): ''' Run ofver a range of timesteps at 5 minute intervals in between ''' start = datetime.strptime(start_time, '%Y%m%d.%H%M%S') stop = datetime.strptime(stop_time, '%Y%m%d.%H%M%S') kwargs.update({'task_id': hotspotsRange.request.id}) subtasks = [ send_task("cybercomq.gis.hotspotpysal.hotspots", args=(ts,location), kwargs=kwargs, queue="gis", track_started=True).task_id for ts in date_range(start,stop) ] return subtasks
def getConferenceSessionsByFilters(self, request): """Query for the sessions that are not specified type and are not running after specified time """ # first get the sessions that don't have the specified type # then get the sessions that don't run past the specified time type_f = request.websafeType print type_f print '------------------------------' time_f = request.websafeTime print time_f print '------------------------------' eq_f = request.websafeOperator print eq_f print '------------------------------' q = ConferenceSession.query(ConferenceSession.type != type_f) print q sessions=[] for sess in q: if sess.start_time is not None: if eq_f == '<': if sess.start_time < datetime.strptime(time_f, '%H:%M').time(): # add the session to the sessions sessions.append(sess) elif eq_f == '=': if sess.start_time == datetime.strptime(time_f, '%H:%M').time(): # add the session to the sessions sessions.append(sess) elif eq_f == '>': if sess.start_time > datetime.strptime(time_f, '%H:%M').time(): # add the session to the sessions sessions.append(sess) return ConferenceSessionForms( items=[self._copySessionToForm(s) for s in sessions] )
def calc_period_prec(self, cr, uid, fiscal_year, context={}): fiscalyear_obj = self.pool['account.fiscalyear'] date_start = ( str(datetime.strptime(fiscal_year.date_start, '%Y-%m-%d') - relativedelta(years=1))[:10]) date_stop = ( str(datetime.strptime(fiscal_year.date_stop, '%Y-%m-%d') - relativedelta(years=1))[:10]) id_fy = fiscalyear_obj.search( cr, uid, [('date_start', '=', date_start), ('date_stop', '=', date_stop)]) if not id_fy: raise osv.except_osv( _('Error!'), _('Not find the previous exercise for period %s - %s' % ( date_start, date_stop))) # Anno Successivo date_start = ( str(datetime.strptime(fiscal_year.date_start, '%Y-%m-%d') + relativedelta(years=1))[:10]) date_stop = ( str(datetime.strptime(fiscal_year.date_stop, '%Y-%m-%d') + relativedelta(years=1))[:10]) id_fy1 = fiscalyear_obj.search( cr, uid, [('date_start', '=', date_start), ('date_stop', '=', date_stop)]) if not id_fy1: raise osv.except_osv( _('Error!'), _('Not find the next exercise for period %s - %s' % ( date_start, date_stop))) return id_fy[0], id_fy1[0]
def is_date(s, f): try: datetime.strptime(s, f) except ValueError: return False else: return True
def __validateArgs(): #=============================================================================== if len(sys.argv) < 5: print "python",sys.argv[0], "CIK ALIAS SINCE UNTIL" print "where CIK: one platform client key" print " ALIAS: dataport alias" print " SINCE: MM/DD/YYYY" print " UNTIL: MM/DD/YYYY" sys.exit(1) cik, alias, since, until = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4] if len(cik) != 40: print "Invalid cik" sys.exit(1) since = since + " 00:00:00" until = until + " 23:59:59" try: start = datetime.strptime(since, "%m/%d/%Y %H:%M:%S") end = datetime.strptime(until, "%m/%d/%Y %H:%M:%S") except ValueError as err: print "Invalid time format." sys.exit(1) start_timestamp = int(time.mktime(start.timetuple())) end_timestamp = int(time.mktime(end.timetuple())) if start_timestamp > end_timestamp: print "SINCE must not be greater than UNTIL" sys.exit(1) return cik, alias, start_timestamp, end_timestamp
def test_podcast_file_sync(self): # download only one podcast episode with utils.temp_podcast(self.client, archive_type='soundcloud', max_allowed=1) as podcast: url = urls.soundcloud_track_list(podcast['broadcast_id'], self.client.soundcloud_client_id) httpretty.register_uri(httpretty.GET, url, body=json.dumps(soundcloud_one_track.DATA)) self.client.episode_sync() episode_list = self.client.episode_list(only_files=False) with utils.temp_audio_file() as mp3_body: utils.mock_mp3_download(episode_list[0]['download_url'], mp3_body) self.client.podcast_file_sync() episode_list = self.client.episode_list() self.assert_not_none(episode_list[0]['file_path']) first_episode_date = episode_list[0]['date'] # add an additional, newer podcast, make sure things are deleted url = urls.soundcloud_track_list(podcast['broadcast_id'], self.client.soundcloud_client_id) httpretty.register_uri(httpretty.GET, url, body=json.dumps(soundcloud_two_tracks.DATA)) self.client.episode_sync() episode_list = self.client.episode_list(only_files=False) with utils.temp_audio_file() as mp3_body: utils.mock_mp3_download(episode_list[1]['download_url'], mp3_body) self.client.podcast_file_sync() # make sure 2 episodes in db, but only 1 with a file path episode_list = self.client.episode_list() self.assert_not_none(episode_list[0]['file_path']) all_episodes = self.client.episode_list(only_files=False) self.assertNotEqual(len(episode_list), len(all_episodes)) second_episode_date = episode_list[0]['date'] self.assertTrue(datetime.strptime(second_episode_date, self.client.datetime_output_format) > datetime.strptime(first_episode_date, self.client.datetime_output_format))
def _process_validation_times(self, x509): """ PyOpenSSL uses a kooky date format that *usually* parses out quite easily but on the off chance that it's not in UTC, a lot of work needs to be done. """ valid_from = x509.get_notBefore() valid_until = x509.get_notAfter() try: self.valid_from = pytz.UTC.localize(datetime.strptime( valid_from, self.TIME_FORMAT )) except ValueError: self.valid_from = self._process_nonstandard_time(valid_from) try: self.valid_until = pytz.UTC.localize(datetime.strptime( valid_until, self.TIME_FORMAT )) except ValueError: self.valid_until = self._process_nonstandard_time(valid_until)
def onchange_fiscalyear(self, cr, uid, ids, fiscalyear_id=False, context=None): print_state = 'draft' date_move_line_from = date_move_line_from_view = False date_move_line_to = False if fiscalyear_id: print_state = 'print' fiscalyear_data = self._get_account_fiscalyear_data(cr, uid, ids, fiscalyear_id) #set values today_date = date.today() date_start = datetime.strptime(fiscalyear_data.date_start,"%Y-%m-%d").date() date_stop = datetime.strptime(fiscalyear_data.date_stop,"%Y-%m-%d").date() #set date_move_line_from if fiscalyear_data.date_last_print: date_last_print = datetime.strptime(fiscalyear_data.date_last_print,"%Y-%m-%d").date() date_move_line_from = date_move_line_from_view = (date_last_print+timedelta(days=1)).__str__() if date_last_print == date_stop: date_move_line_from = date_move_line_from_view = date_start.__str__() print_state = 'printed' else: date_move_line_from = date_move_line_from_view = date_start.__str__() #set date_move_line_to if today_date > date_stop: date_move_line_to = date_stop.__str__() else: date_move_line_to = (today_date-timedelta(days=1)).__str__() return {'value': { 'date_move_line_from': date_move_line_from, 'date_move_line_from_view': date_move_line_from_view, 'date_move_line_to': date_move_line_to, 'print_state': print_state, } }
def _compute_board_amount(self, sequence, residual_amount, amount_to_depr, undone_dotation_number, posted_depreciation_line_ids, total_days, depreciation_date): amount = 0 if sequence == undone_dotation_number: amount = residual_amount else: if self.method == 'linear': amount = amount_to_depr / (undone_dotation_number - len(posted_depreciation_line_ids)) if self.prorata: amount = amount_to_depr / self.method_number if sequence == 1: if self.method_period % 12 != 0: date = datetime.strptime(self.date, '%Y-%m-%d') month_days = calendar.monthrange(date.year, date.month)[1] days = month_days - date.day + 1 amount = (amount_to_depr / self.method_number) / month_days * days else: days = (self.company_id.compute_fiscalyear_dates(depreciation_date)['date_to'] - depreciation_date).days + 1 amount = (amount_to_depr / self.method_number) / total_days * days elif self.method == 'degressive': amount = residual_amount * self.method_progress_factor if self.prorata: if sequence == 1: if self.method_period % 12 != 0: date = datetime.strptime(self.date, '%Y-%m-%d') month_days = calendar.monthrange(date.year, date.month)[1] days = month_days - date.day + 1 amount = (residual_amount * self.method_progress_factor) / month_days * days else: days = (self.company_id.compute_fiscalyear_dates(depreciation_date)['date_to'] - depreciation_date).days + 1 amount = (residual_amount * self.method_progress_factor) / total_days * days return amount
def create_new_database(self, dbname=None, client_id=None, partner_id=None, user_id=None, notify_user=False, trial=False, support_team_id=None): self.ensure_one() db_count = self.env['saas_portal.client'].search_count([('partner_id', '=', partner_id), ('state', '=', 'open'), ('plan_id', '=', self.id)]) if self.maximum_allowed_db_per_partner != 0 and db_count >= self.maximum_allowed_db_per_partner: raise MaximumDBException server = self.server_id if not server: server = self.env['saas_portal.server'].get_saas_server() server.action_sync_server() vals = {'name': dbname or self.generate_dbname()[0], 'server_id': server.id, 'plan_id': self.id, 'partner_id': partner_id, 'trial': trial, 'support_team_id': support_team_id, } client = None if client_id: vals['client_id'] = client_id client = self.env['saas_portal.client'].search([('client_id', '=', client_id)]) vals = self._new_database_vals(vals)[0] if client: client.write(vals) else: client = self.env['saas_portal.client'].create(vals) client_id = client.client_id scheme = server.request_scheme port = server.request_port if user_id: owner_user = self.env['res.users'].browse(user_id) else: owner_user = self.env.user owner_user_data = { 'user_id': owner_user.id, 'login': owner_user.login, 'name': owner_user.name, 'email': owner_user.email, } trial_expiration_datetime = datetime.strptime(client.create_date, DEFAULT_SERVER_DATETIME_FORMAT) + timedelta(hours=self.expiration) # for trial state = { 'd': client.name, 'e': trial and trial_expiration_datetime or client.create_date, 'r': '%s://%s:%s/web' % (scheme, client.name, port), 'owner_user': owner_user_data, 't': client.trial, } if self.template_id: state.update({'db_template': self.template_id.name}) scope = ['userinfo', 'force_login', 'trial', 'skiptheuse'] url = server._request_server(path='/saas_server/new_database', scheme=scheme, port=port, state=state, client_id=client_id, scope=scope,)[0] res = requests.get(url, verify=(self.server_id.request_scheme == 'https' and self.server_id.verify_ssl)) if res.status_code != 200: # TODO /saas_server/new_database show more details here raise exceptions.Warning('Error %s' % res.status_code) data = simplejson.loads(res.text) params = { 'state': data.get('state'), 'access_token': client.oauth_application_id._get_access_token(user_id, create=True), } url = '{url}?{params}'.format(url=data.get('url'), params=werkzeug.url_encode(params)) # send email if notify_user: template = self.env.ref('saas_portal.email_template_create_saas') client.message_post_with_template(template.id, composition_mode='comment') if trial: client.expiration_datetime = trial_expiration_datetime client.send_params_to_client_db() client.server_id.action_sync_server() return {'url': url, 'id': client.id, 'client_id': client_id}
def calculate_integrated_sliding2(self, integration_factor, output_images): file_temp = h5py.File(self.results_folder + "/mask.hdf5", 'r') dset = file_temp["Mask"] mask = dset[...] file_temp.close() new_clean_data_list = [] for index, element in enumerate(self.clean_data_list): if index == len(self.clean_data_list) - 1: break new_clean_data_list.append( [self.clean_data_list[index], self.clean_data_list[index + 1]]) n = len(new_clean_data_list) - (len(new_clean_data_list) % integration_factor) new_clean_data_list = new_clean_data_list[:n] groups_list = [ new_clean_data_list[i:i + integration_factor] for i in range(0, len(new_clean_data_list), integration_factor) ] integrated_sliding2_results_folder = self.results_folder + "/integrated_sliding2_factor_%d" % ( integration_factor) if not os.path.exists(integrated_sliding2_results_folder): os.makedirs(integrated_sliding2_results_folder) integrated_sliding2_len = len(groups_list) - 1 mag_mean = np.zeros(integrated_sliding2_len) phase_mean = np.zeros(integrated_sliding2_len) phase_mean_sum = np.zeros(integrated_sliding2_len) phase_std_dev = np.zeros(integrated_sliding2_len) date_values = [] vmax = np.pi * self.lambda_d vmin = -1 * vmax for enum, group in enumerate(groups_list): if enum == len(groups_list) - 1: break for index, element in enumerate(groups_list[enum]): data0 = self.sar_collection.find_one({'take': str(element[0])}) file_temp0 = h5py.File(data0['route'], 'r') dset0 = file_temp0["Complex_image"] data1 = self.sar_collection.find_one({'take': str(element[1])}) file_temp1 = h5py.File(data1['route'], 'r') dset1 = file_temp1["Complex_image"] if index == 0: Imagen = np.sqrt(dset0[...] * np.conj(dset1[...])) date_begin = data0['date'] time_begin = data0['time'] else: Imagen += np.sqrt(dset0[...] * np.conj(dset1[...])) file_temp0.close() file_temp1.close() if index == len(group) - 1: date = data1['date'] time = data1['time'] Imagen /= integration_factor phase = np.angle(Imagen) magnitude = np.absolute(Imagen) masked_angle = np.ma.masked_where(mask == 0, phase) masked_magnitude = np.ma.masked_where(mask == 0, magnitude) masked_plot = np.ma.masked_where(mask == 1, magnitude) mag_mean[enum] = masked_magnitude.mean() phase_mean[enum] = masked_angle.mean() * self.lambda_d phase_mean_sum[enum] = np.sum(phase_mean) phase_std_dev[enum] = np.std(masked_angle) * self.lambda_d date_values.append( datetime.strptime(''.join((date_begin, time_begin)), ''.join( (date_format, time_format)))) if output_images: fig = plt.figure(1) plt.title( "Integration factor: {integration_factor} \n From {date_begin} {time_begin} \n to {date} {time} (UTC)" .format(integration_factor=integration_factor, date_begin=date_begin, time_begin=time_begin, date=date, time=time), fontsize=11) plt.ylabel('Range (m)', fontsize=12) plt.xlabel('Cross-range (m)', fontsize=12) if enum == 0: im = plt.imshow( self.lambda_d * phase, cmap='jet', aspect='auto', extent=[self.xi, self.xf, self.yi, self.yf], vmin=vmin, vmax=vmax) cbar = plt.colorbar(im, orientation='vertical', format='%.2f') cbar.ax.set_title('Displacement \n (mm)', fontsize=10) aux = int( (self.yf - (self.xf * np.tan(48.0 * np.pi / 180.0))) * Imagen.shape[0] / (self.yf - self.yi)) mask_aux = np.zeros(Imagen.shape) count = 0 nposx = Imagen.shape[1] nposy = Imagen.shape[0] for k in range(nposy): if k >= (aux + 1): mask_aux[k, 0:count] = 1 mask_aux[k, nposx - count - 1:nposx - 1] = 1 count = count + 1 masked_values = np.ma.masked_where(mask_aux == 0, mask_aux) plt.imshow(masked_values, cmap='binary', aspect='auto', extent=[self.xi, self.xf, self.yi, self.yf]) plt.imshow(masked_plot, cmap='binary', aspect='auto', extent=[self.xi, self.xf, self.yi, self.yf]) if enum > 0: im = plt.imshow( self.lambda_d * phase, cmap='jet', aspect='auto', extent=[self.xi, self.xf, self.yi, self.yf], vmin=vmin, vmax=vmax) plt.imshow(masked_values, cmap='binary', aspect='auto', extent=[self.xi, self.xf, self.yi, self.yf]) plt.imshow(masked_plot, cmap='binary', aspect='auto', extent=[self.xi, self.xf, self.yi, self.yf]) plt.savefig(integrated_sliding2_results_folder + "/groups_%d_%d.png" % (enum, enum + 1)) fig = plt.figure(1) fig.clear() fig = plt.figure(figsize=(10.0, 6.0)) plt.subplot(221) plt.title(r'$\overline{\Delta r}\/\/vs\/\/time$', fontsize=16) plt.ylabel(r'$\overline{\Delta r}\/\/(mm)$', fontsize=16) plt.xlabel(r'$time$', fontsize=16) plt.plot(date_values, phase_mean) ax = plt.gca() ax.set_ylim( [-(np.amax(phase_mean) * 2.0), (np.amax(phase_mean) * 2.0)]) ax.xaxis.set_major_formatter(DateFormatter('%d/%m %H:%M')) labels = ax.get_xticklabels() plt.setp(labels, rotation=30, fontsize=10) plt.subplot(222) plt.title(r'$\overline{\Delta r_{acc}}\/\/(mm)\/\/vs\/\/time$', fontsize=16) plt.ylabel(r'$\overline{\Delta r}\/\/(acc)$', fontsize=16) plt.xlabel(r'$time$', fontsize=16) plt.plot(date_values, phase_mean_sum) ax = plt.gca() ax.xaxis.set_major_formatter(DateFormatter('%d/%m %H:%M')) labels = ax.get_xticklabels() plt.setp(labels, rotation=30, fontsize=10) sub = plt.subplot(223) plt.title(r'$\sigma_{\Delta r}\/\/vs\/\/time$', fontsize=16) plt.ylabel(r'$\sigma_{\Delta r}\/\/(mm)$', fontsize=16) plt.xlabel(r'$time$', fontsize=16) plt.plot(date_values, phase_std_dev) ax = plt.gca() ax.set_ylim([0.0, np.amax(phase_std_dev) * 1.2]) ax.xaxis.set_major_formatter(DateFormatter('%d/%m %H:%M')) labels = ax.get_xticklabels() plt.setp(labels, rotation=30, fontsize=10) plt.subplot(224) plt.title(r'$\overline{mag}\/\/vs\/\/time$', fontsize=16) plt.ylabel(r'$\overline{mag}$', fontsize=16) plt.xlabel(r'$time$', fontsize=16) #plt.plot(date_values, mag_mean * 2 * 1e6) plt.plot(date_values, mag_mean) ax = plt.gca() ax.xaxis.set_major_formatter(DateFormatter('%d/%m %H:%M')) #ax.set_ylim([0.0 , np.amax(mag_mean * 2 * 1e6) * 1.2]) ax.set_ylim([0.0, np.amax(mag_mean) * 1.2]) labels = ax.get_xticklabels() plt.setp(labels, rotation=30, fontsize=10) plt.tight_layout() plt.savefig(integrated_sliding2_results_folder + "/statistical_report.png") print "Done!"
def calculate_integrated_sliding(self, integration_factor): file_temp = h5py.File(self.results_folder + "/mask.hdf5", 'r') dset = file_temp["Mask"] mask = dset[...] file_temp.close() n = len(self.clean_data_list) - (len(self.clean_data_list) % integration_factor) self.clean_data_list = self.clean_data_list[:n] groups_list = [ self.clean_data_list[i:i + integration_factor] for i in range(0, len(self.clean_data_list), integration_factor) ] integrated_sliding_results_folder = self.results_folder + "/integrated_sliding_factor_%d" % ( integration_factor) if not os.path.exists(integrated_sliding_results_folder): os.makedirs(integrated_sliding_results_folder) integrated_sliding_len = len(groups_list) - 1 mag_mean = np.zeros(integrated_sliding_len) phase_mean = np.zeros(integrated_sliding_len) phase_mean_sum = np.zeros(integrated_sliding_len) phase_std_dev = np.zeros(integrated_sliding_len) date_values = [] fig = plt.figure(1) vmax = np.pi * self.lambda_d vmin = -1 * vmax for enum, group in enumerate(groups_list): if enum == len(groups_list) - 1: break for index, element in enumerate(groups_list[enum]): data = self.sar_collection.find_one({'take': str(element)}) file_temp = h5py.File(data['route'], 'r') dset = file_temp["Complex_image"] if index == 0: Imagen_master = dset[...] file_temp.close() continue Imagen_master += dset[...] file_temp.close() Imagen_master /= integration_factor for index, element in enumerate(groups_list[enum + 1]): data = self.sar_collection.find_one({'take': str(element)}) file_temp = h5py.File(data['route'], 'r') dset = file_temp["Complex_image"] if index == 0: Imagen_slave = dset[...] file_temp.close() continue Imagen_slave += dset[...] file_temp.close() if index == len(group) - 1: date = (data['date']) time = (data['time']) Imagen_slave /= integration_factor phase = np.angle(Imagen_master * np.conj(Imagen_slave)) magnitude = np.absolute(Imagen_master) masked_angle = np.ma.masked_where(mask == 0, phase) masked_magnitude = np.ma.masked_where(mask == 0, magnitude) mag_mean[enum] = masked_magnitude.mean() phase_mean[enum] = masked_angle.mean() * self.lambda_d phase_mean_sum[enum] = np.sum(phase_mean) phase_std_dev[enum] = np.std(masked_angle) * self.lambda_d date_values.append( datetime.strptime(''.join((date, time)), ''.join( (date_format, time_format)))) #fig.suptitle("Image %d and %d" %(take, take + 1)) plt.ylabel('Range (m)', fontsize=14) plt.xlabel('Cross-range (m)', fontsize=14) if enum == 0: im = plt.imshow(self.lambda_d * phase, cmap='jet', aspect='auto', extent=[self.xi, self.xf, self.yi, self.yf], vmin=vmin, vmax=vmax) cbar = plt.colorbar(im, orientation='vertical', format='%.2f') im = plt.imshow(self.lambda_d * phase, cmap='jet', aspect='auto', extent=[self.xi, self.xf, self.yi, self.yf], vmin=vmin, vmax=vmax) plt.savefig(integrated_sliding_results_folder + "/groups_%d_%d_takes%s_%s.png" % (enum, enum + 1, tuple(groups_list[enum]), tuple(groups_list[enum + 1]))) fig.clear() fig = plt.figure(figsize=(15.0, 8.0)) plt.subplot(221) plt.title(r'$\overline{\Delta r}\/\/vs\/\/time$', fontsize=16) plt.ylabel(r'$\overline{\Delta r}\/\/(mm)$', fontsize=16) plt.xlabel(r'$time$', fontsize=16) plt.plot(date_values, phase_mean) ax = plt.gca() ax.set_ylim( [-(np.amax(phase_mean) * 2.0), (np.amax(phase_mean) * 2.0)]) ax.xaxis.set_major_formatter(DateFormatter('%d/%m %H:%M')) labels = ax.get_xticklabels() plt.setp(labels, rotation=30, fontsize=10) plt.subplot(222) plt.title(r'$\overline{\Delta r}\/\/(acc)\/\/vs\/\/time$', fontsize=16) plt.ylabel(r'$\overline{\Delta r}\/\/(acc)$', fontsize=16) plt.xlabel(r'$time$', fontsize=16) plt.plot(date_values, phase_mean_sum) ax = plt.gca() ax.xaxis.set_major_formatter(DateFormatter('%d/%m %H:%M')) labels = ax.get_xticklabels() plt.setp(labels, rotation=30, fontsize=10) sub = plt.subplot(223) plt.title(r'$\sigma_{\Delta r}\/\/vs\/\/time$', fontsize=16) plt.ylabel(r'$\sigma_{\Delta r}\/\/(mm)$', fontsize=16) plt.xlabel(r'$time$', fontsize=16) plt.plot(date_values, phase_std_dev) ax = plt.gca() ax.set_ylim([0.0, np.amax(phase_std_dev) * 1.2]) ax.xaxis.set_major_formatter(DateFormatter('%d/%m %H:%M')) labels = ax.get_xticklabels() plt.setp(labels, rotation=30, fontsize=10) plt.subplot(224) plt.title(r'$\overline{mag}\/\/vs\/\/time\/\/(normalized)$', fontsize=16) plt.ylabel(r'$\overline{mag}$', fontsize=16) plt.xlabel(r'$time$', fontsize=16) plt.plot(date_values, mag_mean * 2) ax = plt.gca() ax.set_ylim([0.0, np.amax(mag_mean * 2) * 1.2]) ax.xaxis.set_major_formatter(DateFormatter('%d/%m %H:%M')) labels = ax.get_xticklabels() plt.setp(labels, rotation=30, fontsize=10) plt.tight_layout() plt.savefig(integrated_sliding_results_folder + "/statistical_report.png") print "Done!"
def clean_data_report(self, threshold): self.clean_data_list = [] file_temp = h5py.File(self.results_folder + "/mask.hdf5", 'r') dset = file_temp["Mask"] mask = dset[...] file_temp.close() clean_data_set = set() for take in range(1, self.ntakes): data = self.sar_collection.find_one({'take': str(take)}) file_temp = h5py.File(data['route'], 'r') dset = file_temp["Complex_image"] Imagen_master = dset[...] file_temp.close() data = self.sar_collection.find_one({'take': str(take + 1)}) date = (data['date']) time = (data['time']) file_temp = h5py.File(data['route'], 'r') dset = file_temp["Complex_image"] Imagen_slave = dset[...] file_temp.close() phase = np.angle(Imagen_master * np.conj(Imagen_slave)) masked_angle = np.ma.masked_where(mask == 0, phase) phase_std_dev = np.std(masked_angle) * self.lambda_d if phase_std_dev < threshold: #clean_data_list.append([take, take +1]) clean_data_set.add(take) clean_data_set.add(take + 1) self.clean_data_list = list(clean_data_set) clean_results_folder = self.results_folder + "/clean_data" if not os.path.exists(clean_results_folder): os.makedirs(clean_results_folder) report_len = len(self.clean_data_list) - 1 mag_mean = np.zeros(report_len) phase_mean = np.zeros(report_len) phase_mean_sum = np.zeros(report_len) phase_std_dev = np.zeros(report_len) date_values = [] fig = plt.figure(1) vmax = np.pi * self.lambda_d vmin = -1 * vmax for index, element in enumerate(self.clean_data_list): if index == len(self.clean_data_list) - 1: break master = self.clean_data_list[index] slave = self.clean_data_list[index + 1] print "Processing take %d and %d." % (master, slave) data = self.sar_collection.find_one({'take': str(master)}) file_temp = h5py.File(data['route'], 'r') dset = file_temp["Complex_image"] Imagen_master = dset[...] file_temp.close() data = self.sar_collection.find_one({'take': str(slave)}) date = (data['date']) time = (data['time']) file_temp = h5py.File(data['route'], 'r') dset = file_temp["Complex_image"] Imagen_slave = dset[...] file_temp.close() phase = np.angle(Imagen_master * np.conj(Imagen_slave)) magnitude = np.absolute(Imagen_master) masked_angle = np.ma.masked_where(mask == 0, phase) masked_magnitude = np.ma.masked_where(mask == 0, magnitude) mag_mean[index] = masked_magnitude.mean() phase_mean[index] = masked_angle.mean() * self.lambda_d phase_mean_sum[index] = np.sum(phase_mean) phase_std_dev[index] = np.std(masked_angle) * self.lambda_d date_values.append( datetime.strptime(''.join((date, time)), ''.join( (date_format, time_format)))) fig.clear() fig = plt.figure(figsize=(15.0, 8.0)) plt.subplot(221) plt.title(r'$\overline{\Delta r}\/\/vs\/\/time$', fontsize=16) plt.ylabel(r'$\overline{\Delta r}\/\/(mm)$', fontsize=16) plt.xlabel(r'$time$', fontsize=16) plt.plot(date_values, phase_mean) ax = plt.gca() ax.set_ylim( [-(np.amax(phase_mean) * 2.0), (np.amax(phase_mean) * 2.0)]) ax.xaxis.set_major_formatter(DateFormatter('%d/%m %H:%M')) labels = ax.get_xticklabels() plt.setp(labels, rotation=30, fontsize=10) plt.subplot(222) plt.title(r'$\overline{\Delta r}\/\/(acc)\/\/vs\/\/time$', fontsize=16) plt.ylabel(r'$\overline{\Delta r}\/\/(acc)$', fontsize=16) plt.xlabel(r'$time$', fontsize=16) plt.plot(date_values, phase_mean_sum) ax = plt.gca() ax.xaxis.set_major_formatter(DateFormatter('%d/%m %H:%M')) labels = ax.get_xticklabels() plt.setp(labels, rotation=30, fontsize=10) sub = plt.subplot(223) plt.title(r'$\sigma_{\Delta r}\/\/vs\/\/time$', fontsize=16) plt.ylabel(r'$\sigma_{\Delta r}\/\/(mm)$', fontsize=16) plt.xlabel(r'$time$', fontsize=16) plt.plot(date_values, phase_std_dev) ax = plt.gca() ax.set_ylim([0.0, np.amax(phase_std_dev) * 1.2]) ax.xaxis.set_major_formatter(DateFormatter('%d/%m %H:%M')) labels = ax.get_xticklabels() plt.setp(labels, rotation=30, fontsize=10) plt.subplot(224) plt.title(r'$\overline{mag}\/\/vs\/\/time\/\/(normalized)$', fontsize=16) plt.ylabel(r'$\overline{mag}$', fontsize=16) plt.xlabel(r'$time$', fontsize=16) plt.plot(date_values, mag_mean) ax = plt.gca() ax.xaxis.set_major_formatter(DateFormatter('%d/%m %H:%M')) labels = ax.get_xticklabels() plt.setp(labels, rotation=30, fontsize=10) plt.tight_layout() plt.savefig(clean_results_folder + "/statistical_report.png") print "Done!"
def process_data(self, xi, xf, yi, yf, dx, dy, R0=0.0, ifft_fact=8, win=False): #grid extension: [(xi, xf), (yi, yf)] #grid resolution: dx and dy self.xi = xi self.xf = xf self.yi = yi self.yf = yf self.dx = dx self.dy = dy results_folder = "/home/andre/sar_processed_data/imaging/" + self.collection_name if not os.path.exists(results_folder): os.makedirs(results_folder) processed_data_db = self.db_client['sar_processed_data'] sar_processed_data_collection = processed_data_db[self.collection_name] if self.algorithm == "range_migration": range_res = 2**11 cross_range_res = 2**13 rows = int((max(cross_range_res, self.nx) - self.nx) / 2) cols = int((max(range_res, self.nfre) - self.nfre) / 2) cols_left = cols cols_right = cols rows_up = rows rows_down = rows if not (self.nx % 2) == 0: rows_down = rows + 1 if not (self.nfre % 2) == 0: cols_left = cols + 1 rs = (self.yf + self.yi) / 2.0 #rs = 100.0 s0 = self.calculate_matched_filter(rs, range_res, cross_range_res) for take in range(1, self.ntakes + 1): print "Processing %d out of %d." % (take, self.ntakes) s21 = np.empty([self.nx, self.nfre], dtype=np.complex64) data = self.sar_collection.find({'take_number': str(take + 1)}) for position in range(self.nx): print "Take %d, position %d" % (take, position) data_real = literal_eval( data[position]['data']['data_real']) data_imag = literal_eval( data[position]['data']['data_imag']) s21[position, :] = np.array( data_real)[0] + 1j * np.array(data_imag)[0] if win: s21 = self.hanning(s21) s21 = np.pad(s21, [[rows_up, rows_down], [cols_left, cols_right]], 'constant', constant_values=0) dt = json.loads(data[self.nx - 1]['datetime'], object_hook=json_util.object_hook) date = str(dt.date()) time = str(dt.time().strftime("%H:%M:%S")) self.rm_algorithm(s21, s0, take, date, time) if self.algorithm == "terrain_mapping": self.nposx = int(np.ceil( (xf - xi) / dx) + 1) #number of positions axis x self.nposy = int(np.ceil( (yf - yi) / dy) + 1) #number of positions axis y self.xf = self.xi + self.dx * (self.nposx - 1 ) #recalculating x final position self.yf = self.yi + self.dy * (self.nposy - 1 ) #recalculating y final position self.npos = self.nposx * self.nposy #total of positions self.nr = 2**int(np.ceil(np.log2( self.nfre * ifft_fact))) #calculate a number of ranges, #considering the zero padding n = np.arange(self.nr) #final number of ranges B = self.df * self.nr #final bandwidth dr = c0 / (2 * B) #recalculate resolution, considering ifft_fact self.rn = dr * n #range vector self.R = dr * self.nr #for the period verb for the interp xa = self.xai + self.dax * np.arange( self.nx) #antenna positions vector xn = self.xi + self.dx * np.arange(self.nposx) #grid vector axis x yn = self.yi + self.dy * np.arange(self.nposy) #grid vector axis y sar_processed_data_collection.delete_many({}) Rnk = self.calculate_Rnk( xn, yn, xa) #vector of distance from the antenna positions to the grid Rnk_folder = results_folder + "/Rnk.hdf5" f = h5py.File(Rnk_folder, 'w') dset = f.create_dataset("Rnk", (Rnk.shape), dtype=np.float32) dset[...] = Rnk f.close() post = { "type": "parameters", "xi": self.xi, "xf": self.xf, "yi": self.yi, "yf": self.yf, "dx": self.dx, "dy": self.dy, "fi": self.fre_min, "ff": self.fre_max, "nfre": self.nfre, "ifft_fact": ifft_fact, "window": win, "Rnk_folder": Rnk_folder } sar_processed_data_collection.insert(post) parameters = sar_processed_data_collection.find_one( {'type': 'parameters'}) file_temp = h5py.File(parameters['Rnk_folder'], 'r') dset = file_temp["Rnk"] Rnk = dset[...] file_temp.close() starting_take = 1 ''' if sar_processed_data_collection.find().count() is not 0: parameters = sar_processed_data_collection.find_one({'type' : 'parameters'}) if (self.xi == parameters['xi']) and (self.xf == parameters['xf']) and (self.yi == parameters['yi']) and (self.yf == parameters['yf']): file_temp = h5py.File(parameters['Rnk_folder'], 'r') dset = file_temp["Rnk"] Rnk = dset[...] file_temp.close() starting_take = len(self.sar_collection.find().distinct('take')) else: sar_processed_data_collection.delete_many({}) if sar_processed_data_collection.find().count() is 0: Rnk = self.calculate_Rnk(xn, yn ,xa) #vector of distance from the antenna positions to the grid Rnk_folder = results_folder + "/Rnk.hdf5" f = h5py.File(Rnk_folder, 'w') dset = f.create_dataset("Rnk", (Rnk.shape), dtype = np.float32) dset[...] = Rnk f.close() post = {"type": "parameters", "xi": self.xi, "xf": self.xf, "yi": self.yi, "yf": self.yf, "dx": self.dx, "dy": self.dy, "ifft_fact": ifft_fact, "window": win, "Rnk_folder": Rnk_folder} sar_processed_data_collection.insert(post) post = None ''' for take in range(starting_take, self.ntakes + 1): print "Processing %d out of %d." % (take, self.ntakes) s21 = np.empty([self.nx, self.nfre], dtype=np.complex64) data = self.sar_collection.find({'take_number': str(take)}) #if data.count() < self.nx: # continue for position in range(self.nx): data_real = literal_eval( data[position]['data']['data_real']) data_imag = literal_eval( data[position]['data']['data_imag']) s21[position, :] = np.array( data_real)[0] + 1j * np.array(data_imag)[0] if win: s21 = s21 * np.hanning(s21.shape[1]) s21 = s21 * np.hanning(s21.shape[0])[:, np.newaxis] dt = datetime.strptime(data[self.nx - 1]['datetime'], "%Y-%m-%d %H:%M:%S.%f") date = str(dt.date()) time = str(dt.time().strftime("%H:%M:%S")) self.tm_algorithm(s21, Rnk, take, date, time, results_folder, sar_processed_data_collection)
def calculate_sliding(self, threshold, output_images): self.threshold = threshold print "Processing data..." correlation = self.calculate_correlation() aux = int((self.yf - (self.xf * np.tan(48.0 * np.pi / 180.0))) * correlation.shape[0] / (self.yf - self.yi)) mask_aux = np.zeros(correlation.shape) count = 0 nposx = correlation.shape[1] nposy = correlation.shape[0] for k in range(nposy): if k >= (aux + 1): mask_aux[k, 0:count] = 1 mask_aux[k, nposx - count - 1:nposx - 1] = 1 count = count + 1 masked_values = np.ma.masked_where(mask_aux == 0, mask_aux) fig = plt.figure(1) plt.title("Complex correlation magnitude", fontsize=11) im = plt.imshow(np.absolute(correlation), cmap='jet', aspect='auto', extent=[self.xi, self.xf, self.yi, self.yf], vmin=0, vmax=1) plt.imshow(masked_values, cmap='Greys', aspect='auto', extent=[self.xi, self.xf, self.yi, self.yf]) plt.colorbar(im, orientation='vertical', format='%.1f') plt.savefig(self.results_folder + "/complex_correlation_mag.png") fig.clear() mask = np.absolute(correlation) mask[mask < threshold] = 0 mask[mask >= threshold] = 1 vmax = np.pi * self.lambda_d vmin = -1 * vmax mag_mean = np.zeros(self.ntakes - 1) phase_mean = np.zeros(self.ntakes - 1) phase_mean_sum = np.zeros(self.ntakes - 1) phase_std_dev = np.zeros(self.ntakes - 1) date_values = [] fig = plt.figure(1) file_temp = h5py.File(self.results_folder + "/mask.hdf5", 'w') dset = file_temp.create_dataset("Mask", (mask.shape), dtype=np.uint8) dset[...] = mask file_temp.close() for take in range(1, self.ntakes): print "Processing take %d and %d." % (take, take + 1) data = self.sar_collection.find_one({'take': str(take)}) file_temp = h5py.File(data['route'], 'r') dset = file_temp["Complex_image"] Imagen_master = dset[...] file_temp.close() data = self.sar_collection.find_one({'take': str(take + 1)}) date = (data['date']) time = (data['time']) file_temp = h5py.File(data['route'], 'r') dset = file_temp["Complex_image"] Imagen_slave = dset[...] file_temp.close() phase = np.angle(Imagen_master * np.conj(Imagen_slave)) magnitude = np.absolute(Imagen_master) masked_angle = np.ma.masked_where(mask == 0, phase) masked_magnitude = np.ma.masked_where(mask == 0, magnitude) mag_mean[take - 1] = masked_magnitude.mean() phase_mean[take - 1] = masked_angle.mean() * self.lambda_d phase_mean_sum[take - 1] = np.sum(phase_mean) phase_std_dev[take - 1] = np.std(masked_angle) * self.lambda_d date_values.append( datetime.strptime(''.join((date, time)), ''.join( (date_format, time_format)))) if output_images: fig.suptitle("Image %d and %d" % (take, take + 1)) plt.ylabel('Range (m)', fontsize=14) plt.xlabel('Cross-range (m)', fontsize=14) if take == 1: im = plt.imshow( self.lambda_d * phase, cmap='jet', aspect='auto', extent=[self.xi, self.xf, self.yi, self.yf], vmin=vmin, vmax=vmax) cbar = plt.colorbar(im, orientation='vertical', format='%.2f') im = plt.imshow(self.lambda_d * phase, cmap='jet', aspect='auto', extent=[self.xi, self.xf, self.yi, self.yf], vmin=vmin, vmax=vmax) plt.savefig(self.results_folder + "/take%d_%d.png" % (take, (take + 1))) fig.clear() fig = plt.figure(figsize=(10.0, 6.0)) plt.subplot(221) plt.title(r'$\overline{\Delta r}\/\/vs\/\/time$', fontsize=16) plt.ylabel(r'$\overline{\Delta r}\/\/(mm)$', fontsize=16) plt.xlabel(r'$time$', fontsize=16) plt.plot(date_values, phase_mean) ax = plt.gca() ax.set_ylim( [-(np.amax(phase_mean) * 2.0), (np.amax(phase_mean) * 2.0)]) ax.xaxis.set_major_formatter(DateFormatter('%d/%m %H:%M')) labels = ax.get_xticklabels() plt.setp(labels, rotation=30, fontsize=10) plt.subplot(222) plt.title(r'$\overline{\Delta r_{acc}}\/\/(mm)\/\/vs\/\/time$', fontsize=16) plt.ylabel(r'$\overline{\Delta r_{acc}}$', fontsize=16) plt.xlabel(r'$time$', fontsize=16) plt.plot(date_values, phase_mean_sum) ax = plt.gca() ax.xaxis.set_major_formatter(DateFormatter('%d/%m %H:%M')) labels = ax.get_xticklabels() plt.setp(labels, rotation=30, fontsize=10) sub = plt.subplot(223) plt.title(r'$\sigma_{\Delta r}\/\/vs\/\/time$', fontsize=16) plt.ylabel(r'$\sigma_{\Delta r}\/\/(mm)$', fontsize=16) plt.xlabel(r'$time$', fontsize=16) plt.plot(date_values, phase_std_dev) ax = plt.gca() ax.set_ylim([0.0, np.amax(phase_std_dev) * 1.2]) ax.xaxis.set_major_formatter(DateFormatter('%d/%m %H:%M')) labels = ax.get_xticklabels() plt.setp(labels, rotation=30, fontsize=10) plt.subplot(224) plt.title(r'$\overline{mag}\/\/vs\/\/time$', fontsize=16) plt.ylabel(r'$\overline{mag}$', fontsize=16) plt.xlabel(r'$time$', fontsize=16) plt.plot(date_values, mag_mean) ax = plt.gca() ax.set_ylim([0.0, np.amax(mag_mean) * 1.2]) ax.xaxis.set_major_formatter(DateFormatter('%d/%m %H:%M')) labels = ax.get_xticklabels() plt.setp(labels, rotation=30, fontsize=10) plt.tight_layout() plt.savefig(self.results_folder + "/statistical_report.png") print "Done!"
def date(value): """Parse a valid looking date in the format YYYY-mm-dd""" date = datetime.strptime(value, "%Y-%m-%d") if date.year < 1900: raise ValueError(u"Year must be >= 1900") return date
def convert_timestamp(folder: str, ext: str = ".log") -> None: for file in listdir(folder): if path.splitext(file)[1] != ext: continue date_time = datetime.strptime(file, "%Y%m%d-%H%M%S{0}".format(ext)) rename(path.join(folder, file), path.join(folder, str(date_time.timestamp()) + ext))
def _dbs_dob_matches(record, applicant_date_of_birth): return datetime.strptime(applicant_date_of_birth, '%Y-%m-%d') == datetime.strptime( record['date_of_birth'], '%Y-%m-%d')
def _get_purchase_schedule_date(self): procurement_date_planned = datetime.strptime( self.date_planned, DEFAULT_SERVER_DATETIME_FORMAT) schedule_date = (procurement_date_planned - relativedelta(days=self.company_id.po_lead)) return schedule_date
def parse(self, response): self.driver.get(response.url) time.sleep(3) name = self.driver.find_element_by_css_selector('.startup_name').text profile_url = str(response.url) company_website = self.driver.find_element_by_css_selector('.startup_website').get_attribute("href") try: location = self.driver.find_element_by_css_selector('.startup_location').text except: location = '' tags = self.driver.find_element_by_css_selector('.startup_market').text founding_day = 1 try: founding_month = self.driver.find_element_by_css_selector('.startup_found_month').text founding_year = self.driver.find_element_by_css_selector('.startup_found_year').text datetime_string = "{0} {1}, {2}".format(founding_day, founding_month, founding_year) founding_date = datetime.strptime(datetime_string, '%d %B, %Y') founding_date = founding_date.strftime('%Y-%m-%d') except: founding_date = '' try: social_urls = self.driver.find_element_by_xpath("//div[@class='col-md-5 socials pdt text-right']") social_urls = social_urls.find_elements_by_tag_name('a') social_urls_list = [] for social_url in social_urls: social_url_href = social_urls_list.append(social_url.get_attribute("href")) social_urls_string = ", ".join(social_urls_list) except Exception as e: print(e) social_urls_string = '' try: main_description = self.driver.find_element_by_css_selector('.profile-desc-text') description = main_description.text html_description = main_description.get_attribute('innerHTML') soup = BeautifulSoup(html_description, 'lxml') try: p_tag = soup.select('p') raw_p_tag = str(p_tag[0]).split('<br/>')[0] cleanr = re.compile('<.*?>') short_description = re.sub(cleanr, '', raw_p_tag) except: short_description = '' except: description, short_description = '', '' try: #founder_part = self.driver.find_element_by_xpath("//div[@class='row team team-member-parent']") team = self.driver.find_elements_by_xpath("//div[@class='desc']") founders = [] for i in range(0,len(team)): statement = team[i].find_element_by_xpath("//span[@class='profile-text block role']").text.lower() if 'founder' in statement: founder = team[i].find_element_by_xpath("//span[@class='item-label bold member-name']").text founders.append(founder) founders = ''.join(founders) except Exception as e: founders = '' emails = is_email_phone(description, 'email') phones = is_email_phone(description, 'phone') emp_r = '' for lin in social_urls_list: reg = re.compile('\d+') if 'linkedin.com' in lin: self.driver.get(lin) try: data_for_login = yaml_loader('settings.yaml') log_email = data_for_login['Info']['LinkedIn_email'] log_pass = data_for_login['Info']['LinkedIn_password'] log =self.driver.find_element_by_link_text('Sign in') log.click() email =self.driver.find_element_by_id('login-email') email.send_keys(log_email) password =self.driver.find_element_by_id('login-password') password.send_keys(log_pass) password.send_keys(Keys.ENTER) except Exception as e: print(e) try: employee_range = self.driver.find_element_by_css_selector( '.link-without-visited-state.inline-block.ember-view').text emp_r = reg.findall(employee_range) if len(emp_r) > 1: emp_r= str(emp_r[0]) + str(emp_r[1]) else: emp_r = emp_r[0] except: pass yield {"company_name":name,"company_url":profile_url, "company_website_url":company_website, "location":location,"tags":tags, "founding_date":founding_date, "founders": founders,"employee_range":emp_r, "urls":social_urls_string,"emails":emails, "phones":phones,"description_short":short_description,"description":description}
def ymd_to_jd(self, time_str, fmt='%Y%m%d'): dt = datetime.strptime(time_str, fmt) tt = dt.timetuple() return tt.tm_yday
def parse(self, proxy=False): result = [] # avoiding blocks headers = { 'user-agent': feed.UserAgent_random().lstrip(), 'referer': 'https://www.google.com/search?newwindow=1&q='+self.href } if proxy != False: proxyDict = { "http": "http://" + proxy, "https": "https://" + proxy, } else: proxyDict = {} # custom ранобэ.рф API import if self.href.find('http://xn--80ac9aeh6f.xn--p1ai/') != -1: request = f"https://xn--80ac9aeh6f.xn--p1ai/api/v2/books/{ self.href[31:-1] }/chapters" request = requests.get(request).json() # (request, headers=headers, proxies=proxyDict) for each in request['items']: # ignoring payed chapters if each['availabilityStatus'] == 'free': result.append(feedUpdate( name=each["title"], href="http://xn--80ac9aeh6f.xn--p1ai"+each["url"], datetime=datetime.strptime(each["publishTime"], '%Y-%m-%d %H:%M:%S'), title=self.title)) # custom instagram import if self.href.find('https://www.instagram.com/') != -1: if not randint(0, 100) == 0: return [] try: request = requests.get(self.href, headers=headers, proxies=proxyDict) request = BeautifulSoup(request.text, "html.parser") for each in request.find_all('script'): data = 'window._sharedData = ' if each.text.find(data) != -1: # preparing JSON data = each.text.find(data) + len(data) # data start position data = each.text[data:-1] # -1 is for removing ; in the end data = json.loads(data) # selecting data from JSON data = data['entry_data']['ProfilePage'][0]['graphql'] data = data['user']['edge_owner_to_timeline_media']['edges'] # parsing data from JSON for each in data: # avoiding errors caused by empty titles try: result_name = each['node']['edge_media_to_caption']['edges'][0]['node']['text'] except IndexError: result_name = 'no title' result.append(feedUpdate( name=result_name, href="http://instragram.com/p/"+each['node']['shortcode'], datetime=datetime.fromtimestamp(each['node']['taken_at_timestamp']), title=self.title)) except (KeyError, requests.exceptions.ProxyError, requests.exceptions.SSLError) as err: return [] # custom RSS YouTube converter (link to feed has to be converted manually) elif self.href.find('https://www.youtube.com/channel/') != -1: self.href_title = self.href[:] # 32 = len('https://www.youtube.com/channel/') # 7 = len('/videos') self.href = "https://www.youtube.com/feeds/videos.xml?channel_id=" + self.href[32:-7] result = feed.parse(self) # custom RSS readmanga converter (link to feed has to be converted manually to simplify feed object creation) elif self.href.find('http://readmanga.me/') != -1 and self.href.find('readmanga.me/rss/manga') == -1 and self.href_title == None: # 20 = len('http://readmanga.me/') self.href = "feed://readmanga.me/rss/manga?name=" + self.href[20:] result = feed.parse(self) # custom RSS mintmanga converter (link to feed has to be converted manually to simplify feed object creation) elif self.href.find('http://mintmanga.com/') != -1 and self.href.find('mintmanga.com/rss/manga') == -1 and self.href_title == None: # 21 = len('http://mintmanga.com/') self.href = "feed://mintmanga.com/rss/manga?name=" + self.href[21:] result = feed.parse(self) # custom RSS deviantart converter (link to feed has to be converted manually to simplify feed object creation) elif self.href.find('https://www.deviantart.com/') != -1: self.href_title = self.href[:] # 27 = len('https://www.deviantart.com/') # 9 = len('/gallery/') self.href = self.href[27:-9] self.href = "http://backend.deviantart.com/rss.xml?q=gallery%3A" + self.href result = feed.parse(self) # custom fantasy-worlds.org loader elif self.href.find('https://fantasy-worlds.org/series/') != -1: strainer = SoupStrainer('div', attrs={'class': 'rightBlock'}) request = requests.get(self.href, headers=headers, proxies=proxyDict) request = BeautifulSoup(request.text, "html.parser", parse_only=strainer) for each in request.find('ul').find('li').find('ul').find('li').find('ul').find_all('li'): result.append(feedUpdate( name=f"{self.title} {each.text[:each.text.find(' // ')]}", href=each.find('a')['href'], datetime=datetime.now(), # <=== fake date title=self.title)) # custom pikabu import elif self.href.find('pikabu.ru/@') != -1: # try: strainer = SoupStrainer('div', attrs={'class': 'stories-feed__container'}) request = requests.get(self.href, headers=headers, proxies=proxyDict) request = BeautifulSoup(request.text, "html.parser", parse_only=strainer) for each in request.find_all('article'): try: result_datetime = each.find('time')['datetime'][:-3]+"00" result_datetime = datetime.strptime(result_datetime, '%Y-%m-%dT%H:%M:%S%z') result.append(feedUpdate( name=each.find('h2', {'class': "story__title"}).find('a').getText(), href=each.find('h2', {'class': "story__title"}).find('a')['href'], datetime=result_datetime, title=self.title)) except (TypeError, AttributeError) as err: # advertisement, passing as no need to save it pass # except (requests.exceptions.ConnectionError, requests.exceptions.SSLError) as err: # # failed connection, hope it works from time to time # return [] # # custom fanserials parser # elif self.href.find('http://fanserial.net/') != -1 and self.filter is not None: # strainer = SoupStrainer('ul', attrs={'id': 'episode_list'}) # # request = requests.get(self.href, headers=headers, proxies=proxyDict) # request = BeautifulSoup(request.text, "html.parser", parse_only=strainer) # print(request) # # for each in request.find_all('li'): # print(each) # result_href = '' # for each_span in each.find('div').find('div', attrs={'class': 'serial-translate'}).find_all('span'): # result_href = 'http://fanserial.tv' + each_span.find('a').get('href') # # result.append(feedUpdate( # name=each.find('div', attrs={'class': 'field-description'}).find('a').text, # href=result_href, # datetime=datetime.now(), # <=== fake date # title=self.title)) # default RSS import else: proxyDict = urllib.request.ProxyHandler(proxyDict) request = feedparser.parse(self.href, request_headers=headers, handlers=[proxyDict]) for each in request["items"]: # HREF RESULT if self.title == "Expresso": result_href = each["summary"] start = result_href.find('https://expres.co/') end = result_href.find('"') result_href = result_href[start:end] else: result_href = each["links"][0]["href"] # DATE RESULT: parsing dates if "published" in each: result_datetime = each["published"] elif "updated" in each: result_datetime = each["updated"] else: print(f"result_datetime broke for { self.title }") tzinfos = {'PDT': gettz("America/Los_Angeles"), 'PST': gettz("America/Juneau")} result_datetime = parser.parse(result_datetime, tzinfos=tzinfos) # APPEND RESULT result.append(feedUpdate( name=each["title_detail"]["value"], href=result_href, datetime=result_datetime, title=self.title)) # universal postfixes result_filtered = [] for each in result: # FILTERING: passing item cycle if filter does not match if self.filter is not None: if each.name.find(self.filter) == -1 or each.href.find(self.filter) == -1: continue # DATETIME fixes # fix timezone unaware # if each.datetime.tzinfo is not None and each.datetime.tzinfo.utcoffset(each.datetime) is not None: # each_dt = localtime(each.datetime) # each.datetime = datetime(each_dt.year, each_dt.month, each_dt.day, # each_dt.hour, each_dt.minute, each_dt.second) # if each.datetime.tzinfo is not None and each.datetime.tzinfo.utcoffset(each.datetime) is not None: # print("!!!! WARNING !!!!") # # add DELAY # if type(self.delay) is not type(None): # each.datetime += timedelta(hours=self.delay) # NAME fixes each.name = ' '.join(each.name.split()) each.name = each.name[:140] # SQLite does not support max-length # extra symbols if each.title == 'Shadman': each.name = each.name[:each.name.find('(')-1] elif each.title == 'Apple' and each.name[-len('Apple'):] == 'Apple': # - symbol can be a variety of different symbols # 8 = len(' - Apple') each.name = each.name[:-8] elif each.title == 'LastWeekTonight': end = each.name.find(': Last Week Tonight with John Oliver (HBO)') if end != -1: each.name = each.name[:end] result_filtered.append(each) return result_filtered
usage() except getopt.GetoptError, error: usage("ERROR: " + str(error)) command_offset = 3 if "-p" in sys.argv[:4]: command_offset += 2 command = " ".join(sys.argv[command_offset:]) if command: input.test_params["command"] = command if input.param("parallel", True): # workaround for a python2.6 bug of using strptime with threads datetime.strptime("30 Nov 00", "%d %b %y") RemoteJob().parallel_remote(input) else: RemoteJob().sequential_remote(input) if __name__ == "__main__": main() def create_log_file(log_config_file_name, log_file_name, level): tmpl_log_file = open("jython.logging.conf") log_file = open(log_config_file_name, "w") log_file.truncate() for line in tmpl_log_file: newline = line.replace("@@LEVEL@@", level)
def listaFila(request): if has_permission(request.user, 'permissao_unidade'): unidade = UnidadeSaude.objects.filter(users=request.user) filas = [None] aux_filas = [None] hoje = date.today() consultas = None autorizacoes = None exames = None objetos_aux = None if unidade: if request.method == 'POST': data = request.POST.get('data', None) dataFinal = request.POST.get('dataFinal', None) objeto = request.POST.get('objeto', None) if objeto == 'Consultas': obj_aux = 'Consultas' consultas = unidade[0].consultas.all() objetos = unidade[0].consultas.filter( data=hoje, create_fila=True).order_by('hora', 'data') elif objeto == 'Autorizacoes': obj_aux = 'Autorizacoes' autorizacoes = unidade[0].autorizacoes.all() objetos = unidade[0].autorizacoes.filter( data=hoje, create_fila=True).order_by('hora', 'data') elif objeto == 'Exames': obj_aux = 'Exames' exames = unidade[0].exames.all() objetos = unidade[0].exames.filter( data=hoje, create_fila=True).order_by('hora', 'data') context = {'objeto': obj_aux} if objetos: context['objetos'] = objetos else: context['msg_alert'] = 'Não possui Filas para hoje' if data: try: if consultas: objetos_aux = unidade[0].consultas.filter( data__range=[ datetime.strptime(data, '%Y-%m-%d'), datetime.strptime(dataFinal, '%Y-%m-%d') ], create_fila=True).order_by('hora', 'data') elif autorizacoes: objetos_aux = unidade[0].autorizacoes.filter( data__range=[ datetime.strptime(data, '%Y-%m-%d'), datetime.strptime(dataFinal, '%Y-%m-%d') ], create_fila=True).order_by('hora', 'data') elif exames: objetos_aux = unidade[0].exames.filter( data__range=[ datetime.strptime(data, '%Y-%m-%d'), datetime.strptime(dataFinal, '%Y-%m-%d') ], create_fila=True).order_by('hora', 'data') if objetos_aux: context['objetos'] = objetos_aux context['msg_alert'] = None else: context[ 'msg_busca'] = 'Nada encontrado para esses parâmetros!' except ValueError: context['msg_busca'] = 'Informe uma data válida!' return render(request, 'lista_fila.html', {'context': context}) return render(request, 'lista_fila.html') else: context = {'msg_error': 'Indisponivel Acessar Essa Área'} return render(request, 'home.html', {'context': context}) else: context = {'msg_error': 'Impossivel Acessar Essa Área'} return render(request, 'home_usuario.html', {'context': context})
def main(): logging.config.fileConfig(fname=os.path.join('config', 'log.config'), disable_existing_loggers=False) # Get the logger specified in the file f_handler = logging.FileHandler(os.path.join('logs', 'generate_fred.log')) f_handler.setLevel(logging.DEBUG) log = logging.getLogger(__name__) f_format = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') f_handler.setFormatter(f_format) log.addHandler(f_handler) parser = argparse.ArgumentParser() parser.add_argument('-p', '--prob-minimum', default=0.75, dest='prob_minimum', help='Minimum probability to be considered as fire grids') parser.add_argument('-s', '--start-date', dest='start_date_str', help='Start date') parser.add_argument('-e', '--end-date', dest='end_date_str', help='End date') parser.add_argument('--max-sunglint-angle', dest='max_sunglint_angle', default=12.0, type=float, help='Sun glint angle is defined as angle between reflected solar path \ and satellite view angle. If hotspot has sun glint angle less than the max, \ the hotspot is rejected.') parser.add_argument('-a', '--alpha', dest='alpha', default=0.5, help='Alpha weight for polar orbiting satellites') parser.add_argument('--ahi-hotspot-folder', dest='ahi_folder', default=os.path.join('..', 'data', 'raw', 'hotspots', 'ahi'), help='AHI hotspot folder') parser.add_argument('--viirs-hotspot-folder', dest='viirs_folder', default=os.path.join('..', 'data', 'raw', 'hotspots', 'viirs'), help='VIIRS hotspot folder') parser.add_argument('--modis-hotspot-folder', dest='modis_folder', default=os.path.join('..', 'data', 'raw', 'hotspots', 'modis'), help='MODIS hotspot folder') parser.add_argument('-o', '--output-folder', dest="out_file_path", default=os.path.join('..', 'data'), help="Specify output folder") parser.add_argument('-g', '--grid', dest="grid_shp", default=os.path.join('..', 'references', 'shapefile', '2km_grid_ASEAN_land_clipped.shp'), help="Specify grid .shp file") parser.add_argument('-n', '--name', dest="prefix_name", default='LATE_2km_', help="Prefix for output file names") args = parser.parse_args() log.debug(args) alpha = args.alpha prob_minimum = args.prob_minimum max_sunglint_angle = args.max_sunglint_angle with open(os.path.join('config', 'config.json'), "r") as read_file: json_config = json.load(read_file) clipping_box = json_config['parameters']['clipping_box'] gamma = json_config['parameters']['gamma'] sat_resolution_meter = json_config['sat_resolution_meter'] shapefile_path = json_config['shapefile']['path'] bounding_box = json_config['plotting']['bounding_box'] dpi = json_config['plotting']['dpi'] model_fpath = json_config['model']['path'] h8_ml_model_fpath = json_config['h8_ml_model']['path'] h8_predict_features = json_config['h8_ml_model']['predict_features'] low_prob_thres = json_config['parameters']['low_prob_thres'] med_prob_thres = json_config['parameters']['med_prob_thres'] #read in grid shapefile try: df_grid = geopandas.read_file(args.grid_shp) df_grid.crs = {'init': 'epsg:3857'} log.debug(args.grid_shp + ' loaded successfully!') except Exception as e: log.error(args.grid_shp + ' cannot be loaded !') exit() start_date = datetime.strptime(args.start_date_str, "%Y-%m-%d %H:%M") date_process = start_date end_date = datetime.strptime(args.end_date_str, "%Y-%m-%d %H:%M") for days in range(0, 31): geo_hs = geohotspot.GeoHotspot() log.info('Reading hotspot .txt files') jp1_files = '../data/raw/hotspots/JP1_LATE_APR_SEP_DAY_2019.txt' geo_hs.get_hs_from_csv_db(file_path=jp1_files, sat_name='JP1_LATE') npp_files = '../data/raw/hotspots/NPP_LATE_APR_SEP_DAY_2019.csv' geo_hs.get_hs_from_csv_db(file_path=npp_files, sat_name='NPP_LATE') while date_process <= (end_date + timedelta(days=1)): h8_files = os.path.join(args.ahi_folder, "H08_*" + date_process.strftime('%Y%m%d_0') #h8_files = os.path.join(args.ahi_folder, "H08_*" + date_process.strftime('%Y%m%d_01') + "*_L2WLFbet_FLDK.06001_06001.csv") geo_hs.parse_jaxa_hotspot_txt(file_path=h8_files) modis_files = os.path.join(args.modis_folder, "*14." + date_process.strftime('%y%j') + "*.txt") geo_hs.parse_modis_mod14_txt(file_path=modis_files) date_process = date_process + timedelta(days=1) # remove hotspots outside of clipping area geo_hs.clip_hotspot(clipping_box) # reject hotspots due to sun glint #geo_hs.reject_sunglint_hs('Himawari-8/9', max_sunglint_angle) geo_df = geo_hs.hs_df.copy() geo_df['polar_weight'] = 0.0 geo_df['aqua_weight'] = 0.0 geo_df['terra_weight'] = 0.0 geo_df['n20_weight'] = 0.0 geo_df['npp_weight'] = 0.0 geo_df['geo_weight'] = 0.0 geo_df['weight'] = 0.0 geo_df['confidence'] = geo_df['confidence'].fillna(0) geo_df.astype({'geo_weight': 'float64', 'polar_weight': 'float64', 'weight': 'float64', 'confidence': 'float64'}) geo_df['date'] = pd.to_datetime(geo_df['date'], format="%d/%m/%Y %H:%M:%S") # selects period of interest geo_df = geo_df[(geo_df['date'] >= start_date) & (geo_df['date'] <= end_date)] log.debug(geo_df['date'].unique()) log.debug(geo_df[['satellite', 'date']].groupby(['satellite']).count()) try: h8_ml_model = load(h8_ml_model_fpath) log.debug('Loaded trained H8 ML model from ' + h8_ml_model_fpath) log.debug(f'Model pipeline: {h8_ml_model}') geo_df.loc[geo_df['satellite'] == 'Himawari-8/9', 'geo_weight'] = h8_ml_model.predict_proba( geo_df.loc[geo_df['satellite'] == 'Himawari-8/9', h8_predict_features])[:, 1] log.info('Added in probabilities using H8 Gradient Boosting Model.') except Exception as e: log.exception(e) geo_df.loc[geo_df['satellite'] == 'TERRA', 'terra_weight'] = \ geo_df.loc[geo_df['satellite'] == 'TERRA', 'confidence'] / 100.0 geo_df.loc[geo_df['satellite'] == 'AQUA', 'aqua_weight'] = \ geo_df.loc[geo_df['satellite'] == 'AQUA', 'confidence'] / 100.0 geo_df.loc[geo_df['satellite'] == 'JP1_LATE', 'n20_weight'] = \ geo_df.loc[geo_df['satellite'] == 'JP1_LATE', 'confidence'] / 100.0 geo_df.loc[geo_df['satellite'] == 'NPP_LATE', 'npp_weight'] = \ geo_df.loc[geo_df['satellite'] == 'NPP_LATE', 'confidence'] / 100.0 # count number of Himawari observations geo_obs_count = int((end_date - start_date).seconds / 600) # normalize the weight for Himawari geo_df['geo_weight'] = geo_df['geo_weight'] / geo_obs_count # assign zero weight for hotspots due to sun glint #sun_glint_flag = (geo_df['sunglint_angle'] <= max_sunglint_angle) & (geo_df['satellite'] == "Himawari-8/9") #geo_df.loc[sun_glint_flag, 'geo_weight'] = 0.0 num_polar_sat = len(geo_df.loc[geo_df['satellite'] != 'Himawari-8/9', 'satellite'].unique()) if num_polar_sat >= 1: geo_df['polar_weight'] = (1.0 / num_polar_sat) * (geo_df['confidence'] / 100.0) geo_df['weight'] = ((1 - alpha) * geo_df['geo_weight']) + (alpha * geo_df['polar_weight']) #round to 4 decimals to save storage geo_df = geo_df.round(8) try: gdf = geopandas.GeoDataFrame(geo_df, geometry=geopandas.points_from_xy(geo_df.lon, geo_df.lat)) log.debug('Created geopandas DataFrame') except Exception as e: log.exception(e) # transform to mercator epsg 3857 gdf.crs = {'init': 'epsg:4326'} gdf_merc = gdf.to_crs({'init': 'epsg:3857'}) gdf_merc.reset_index(inplace=True, drop=True) gdf_merc['x'] = gdf_merc['geometry'].x gdf_merc['y'] = gdf_merc['geometry'].y for key, value in sat_resolution_meter.items(): gdf_merc.loc[gdf_merc['satellite'] == key, 'resolution_meter'] = value try: interim_file_path = os.path.join(args.out_file_path, 'interim') os.makedirs(interim_file_path, exist_ok=True) except Exception as e: log.exception(e) log.warning(interim_file_path + ' directory cannot be created!') try: processed_file_path = os.path.join(args.out_file_path, 'processed') os.makedirs(processed_file_path, exist_ok=True) except Exception as e: log.exception(e) log.warning(processed_file_path + ' directory cannot be created!') try: hotspot_json = os.path.join(interim_file_path, args.prefix_name + 'hotspot_' + end_date.strftime('%Y%m%d') + '.geojson') gdf_merc.to_file(hotspot_json, driver='GeoJSON') log.info(hotspot_json + ' is saved successfully.') except Exception as e: log.exception(e) log.warning(hotspot_json + ' export warning!') # create polygon for index, row in gdf_merc.iterrows(): gdf_merc['geometry'].iloc[index] = get_poly_box(row['x'], row['y'], row['resolution_meter']) try: hotspot_polygon_json = os.path.join(interim_file_path, args.prefix_name + 'hotspot_polygon_' + end_date.strftime('%Y%m%d') + '.geojson') gdf_merc.round(4) gdf_merc.to_file(hotspot_polygon_json, driver='GeoJSON') log.info(hotspot_polygon_json + ' is saved successfully.') except Exception as e: log.exception(e) log.warning(hotspot_polygon_json + ' export warning!') try: log.debug('Processing grid sjoin...') df_grid_joined = geopandas.sjoin(df_grid, gdf_merc, op='intersects') grid_weight_total = df_grid_joined[['id', 'polar_weight', 'geo_weight', 'weight', 'terra_weight', 'aqua_weight', 'n20_weight', 'npp_weight']].groupby(['id']).sum() # grid_weight_total = df_grid_joined[['id', 'polar_weight', 'geo_weight', 'weight']].groupby(['id']).sum() grid_geometry = df_grid_joined[['id', 'geometry']].groupby(['id']).first() processed_grid = pd.merge(grid_weight_total, grid_geometry, on='id') processed_grid_gpd = geopandas.GeoDataFrame(processed_grid) processed_grid_gpd.crs = {'init': 'epsg:3857'} log.debug('Processing grid completed.') except Exception as e: log.exception(e) log.error('Unable to process grid sjoin!') # try: # trained_model = pickle.load(open(model_fpath, 'rb')) # log.debug('Loaded trained model from ' + model_fpath) # processed_grid_gpd['prob'] = trained_model.predict_proba(processed_grid_gpd['weight']. # values.astype(float).reshape(-1,1))[:, 1] # print (processed_grid_gpd.head()) # log.info('Probabilities filled using trained model.') # except Exception as e: # log.exception(e) # try: hotspot_grid_json = os.path.join(processed_file_path, args.prefix_name + 'hotspot_grid_' + end_date.strftime('%Y%m%d') + '.geojson') processed_grid_gpd.to_file(hotspot_grid_json, driver='GeoJSON') log.info(processed_grid_gpd + ' is saved successfully.') except Exception as e: log.warning(hotspot_grid_json + ' export warning!') # # try: # ann_file = os.path.join(processed_file_path, args.prefix_name + 'hotspot_grid_' # + end_date.strftime('%Y%m%d') + '.ann') # save_fred_grid_meteor_ann(processed_grid_gpd, ann_file, low_prob_thres, med_prob_thres) # log.info(ann_file + ' is saved successfully.') # except Exception as e: # log.exception(e) # log.warning(ann_file + ' cannot be saved!') start_date = start_date + timedelta(days=1) date_process = start_date end_date = end_date + timedelta(days=1)
def _parseMember(self, text): data = np.array(text.split('\r\n')) data_idxs = [] new_record = False begin_idx = 0 member_name = data[0] dates = [] # Figure out the indices for the data chunks for i in range(len(data)): if "STID" in data[i]: # Here is information about the record spl = data[i].split() if spl[2].strip() == "STNM": station = "" # The bufkit file has a blank space for the station name wmo_id = spl[4] dates.append(datetime.strptime(spl[7], '%y%m%d/%H%M')) else: station = spl[2] wmo_id = spl[5] dates.append(datetime.strptime(spl[8], '%y%m%d/%H%M')) slat = float(data[i+1].split()[2]) slon = float(data[i+1].split()[5]) selv = float(data[i+1].split()[8]) stim = float(data[i+2].split()[2]) if data[i].find('HGHT') >= 0 and new_record == False: # we've found a new data chunk new_record = True begin_idx = i+1 elif 'STID' in data[i] and new_record == True: # We've found the end of the data chunk new_record = False data_idxs.append((begin_idx, i-1)) elif 'STN' in data[i] and new_record == True: # We've found the end of the last data chunk of the file new_record = False data_idxs.append((begin_idx, i)) elif new_record == True: continue ##print data[i] data_idxs = data_idxs[1:] # Make arrays to store the data profiles = [] # Parse out the profiles for i in range(len(data_idxs)): data_stuff = data[data_idxs[i][0]: data_idxs[i][1]] profile_length = len(data[data_idxs[i][0]: data_idxs[i][1]])//2 hght = np.zeros((profile_length,), dtype=float) pres = np.zeros((profile_length,), dtype=float) tmpc = np.zeros((profile_length,), dtype=float) dwpc = np.zeros((profile_length,), dtype=float) wdir = np.zeros((profile_length,), dtype=float) wspd = np.zeros((profile_length,), dtype=float) omeg = np.zeros((profile_length,), dtype=float) for j in np.arange(0, profile_length * 2, 2): if len(data_stuff[j+1].split()) == 1: hght[j // 2] = float(data_stuff[j+1].split()[0]) else: hght[j // 2] = float(data_stuff[j+1].split()[1]) tmpc[j // 2] = float(data_stuff[j].split()[1]) dwpc[j // 2] = float(data_stuff[j].split()[3]) pres[j // 2] = float(data_stuff[j].split()[0]) wspd[j // 2] = float(data_stuff[j].split()[6]) wdir[j // 2] = float(data_stuff[j].split()[5]) omeg[j // 2] = float(data_stuff[j].split()[7]) prof = profile.create_profile(profile='raw', pres=pres, hght=hght, tmpc=tmpc, dwpc=dwpc, wdir=wdir, wspd=wspd, omeg=omeg, location=station, date=dates[i], latitude=slat) profiles.append(prof) return member_name, profiles, dates
def extract_date(s, loader_context): date_format = loader_context['date_format'] d = datetime.strptime(s, date_format) return d.strftime('%d/%m/%Y')
from datetime import datetime from datetime import timedelta with open("2018day4.txt", 'r') as file: data = {y : z for y,z in [x.strip('[').split('] ') for x in file.read().splitlines()]} guards = {} for k,v in sorted(data.items()): if 'Guard' in v: guard = int(v.split(' ')[1].strip('#')) if guard not in guards: guards[guard] = [] elif 'falls' in v: sleep_date = datetime.strptime(k, '%Y-%m-%d %H:%M') elif 'wakes' in v: wake_date = datetime.strptime(k, '%Y-%m-%d %H:%M') while sleep_date != wake_date: guards[guard].append(int(str(sleep_date).split(':')[1])) sleep_date += timedelta(minutes=1) mx = max(guards.items(), key = lambda x: len(x[1])) print(mx[0] * max(mx[1], key = lambda x: mx[1].count(x))) nmx = max(guards.items(), key = lambda x: max(x[1].count(i) for i in range(60))) print(nmx[0] * max(nmx[1], key = lambda x: nmx[1].count(x)))
def check_model(self, datas, model, model_name=''): """ Keys description in model - fields with 'required' key : * not required are fill with '' if they are not given * required doesn't accept '' value - 'type' key : if you provide a type, this type is check first before other keys - 'date' key : accepted type for date : str or datetime (convert one to other) - 'min_size' and 'max_size' : this is used for str or unicode - 'min_number' and 'max_number' : this is used for int or float keys examples: model = { 'my_field': {'max_size': 35, 'required': True}, 'my_date': {'date': '%d/%m/%Y'}, "weight": {'max_number': 50, 'type': float}, } Carefull cases considered in this script (search each 'case' string in source code below): Case 1/ key in model with not in datas : => no check but with a default value '' (only for display) Case 2/ key in datas but with a False value (string with no value : empty or Null value according to database) : => no check but with a default value '' Case 3/ data == 0.0 or 0 which is considered like False but is not : => check but convert in string """ if model_name: model_name = '(model: ' + model_name + ')' for field, definition in model.items(): #check type before all other checks if requested in model if 'type' in definition and field in datas: self.check_type(field, [definition['type']], datas[field]) to_check = self.must_be_checked(datas, field) for key, val in definition.items(): if to_check: data = datas[field] size = self.evaluate_size_according_to_type(data) if key == 'max_size': self.check_type(field, [str, unicode], data) if size > val: raise InvalidSize( "Max size for field '%s' is " "%s : %s given\nCurrent field value : %s" % (field, val, size, data)) elif key == 'min_size': self.check_type(field, [str, unicode], data) if size < val: raise InvalidSize( "Min size for field '%s' is " "%s : %s given\nCurrent field value : %s" % (field, val, size, data)) elif key == 'min_number': self.check_type(field, [int, float], data) if size < val: raise InvalidSize("Min number for field '%s' is " "%s : %s given" % (field, val, size)) elif key == 'max_number': self.check_type(field, [int, float], data) if size > val: raise InvalidSize("Max number for field '%s' is " "%s : %s given" % (field, val, size)) elif key == 'in' and data not in val: raise InvalidValueNotInList( "field '%s' with value '%s' must belong " "to this list %s" % (field, data, val)) elif key == 'date': self.check_type(field, [str, datetime], data) if isinstance(data, datetime): try: datas[field] = datetime.strftime(data, val) except: raise InvalidType( "The date '%s' must be in the format '%s'" % (data, val)) elif isinstance(data, str): try: datas[field] = \ datetime.strptime(data, val).strftime(val) except: raise InvalidType( "The date '%s' must be in the format '%s'" % (data, val)) elif key == 'numeric': #TODO : to end self.check_type(field, [int, float], data) datas[field] = val % data data = '' else: if key == 'required' and val is True: raise InvalidMissingField( "Required field '%s' is missing %s" % (field, model_name)) else: # must have an empty value to be called # in python template (mako, jinja2, etc) if field not in datas: #case 1/ datas[field] = '' elif type(datas[field]) == bool: #case 2/ datas[field] = '' # case 3/ if type(datas[field]) in [int, float]: datas[field] = str(datas[field]) return datas
def main(): if len(sys.argv) >= 2: input_filename = sys.argv[1] else: print('You have to specify an input filename as a first argument.') return if len(sys.argv) >= 3: input_id = sys.argv[2] else: print('You have to specify student id as a second argument.') return with open(input_filename, newline='') as csvfile: reader = csv.DictReader(csvfile) # for row in reader: # print(row) input_data = None if input_id == 'average': points_sum = {} count = {} for row in reader: for key, value in row.items(): if key == 'student': continue if key not in points_sum: points_sum[key] = 0.0 points_sum[key] += float(value) if key not in count: count[key] = 0 count[key] += 1 input_data = {} for key, value in points_sum.items(): input_data[key] = value/count[key] input_data['student'] = 'average' #print(input_data) else: for row in reader: if row['student'] != input_id: continue else: input_data = row raw_points = [] points_by_date = [] raw_data_exercise = {} raw_points_per_date = {} for key, value in input_data.items(): if key == 'student': continue else: parsed = key.strip().replace(' ', '').split('/') # print(parsed) output_key = parsed[1] if output_key not in raw_data_exercise: raw_data_exercise[output_key] = 0 raw_data_exercise[output_key] += float(value) if parsed[0] not in raw_points_per_date: raw_points_per_date[parsed[0]] = 0 raw_points_per_date[parsed[0]] += float(value) for key, value in raw_data_exercise.items(): raw_points.append(value) for key, points in raw_points_per_date.items(): points_by_date.append((key, points)) # print(points_by_date) dates = [] points = [] for key, value in sorted(points_by_date, key=lambda k: k[0]): dates.append(datetime.strptime(key, '%Y-%m-%d').date().toordinal()) points.append(value) for i in range(1, len(points)): points[i] = points[i - 1] + points[i] start_date = datetime.strptime('2018-9-17', '%Y-%m-%d').date().toordinal() dates = numpy.array([date - start_date for date in dates]) # print(dates, points) regression_slope = numpy.linalg.lstsq([[date] for date in dates], points, rcond=-1)[0][0] output = { 'passed': numpy.count_nonzero(numpy.array(raw_points)), 'median': numpy.median(numpy.array(raw_points)), 'mean': numpy.mean(numpy.array(raw_points)), 'total': sum(raw_points), 'regression slope': regression_slope, } if regression_slope != 0: output['date 16'] = date.fromordinal(math.floor((16.0 / regression_slope) + start_date)).__str__() output['date 20'] = date.fromordinal(math.floor((20.0 / regression_slope) + start_date)).__str__() print(json.dumps(output, indent=4))
def myparser(x): return datetime.strptime(x, '%b %d, %Y @ %H:%M:%S.%f')
def get_unix_time(time_str): return datetime.strptime(time_str, '%Y-%m-%d %H:%M:%S')
def get_date_time(date_time): datetime_object = datetime.strptime(date_time, '%b %d %Y %I:%M%p') return datetime_object
def return_modify_date(self, platform, file): if platform not in self.platforms: print('Plataforma do jogo Streets Of Rage 4 não foi encontrada.') else: createDate = datetime.strptime(ctime(stat(file).st_mtime), '%a %b %d %H:%M:%S %Y').date() return createDate
def day_range(start_date_string, end_date_string): start_date = datetime.strptime(start_date_string, ISODATE).date() end_date = datetime.strptime(end_date_string, ISODATE).date() delta = end_date - start_date for i in range(delta.days + 1): yield (start_date + timedelta(i)).isoformat()
def retrieve(self, request, pk=None, format=None): """Get all metrics for a pod Arguments: request {[Django request]} -- The request object Keyword Arguments: pk {string} -- Name of the pod format {string} -- Output format to use (default: {None}) Returns: Json -- Object containing all metrics for the pod """ q = Q() since = self.request.query_params.get("since", None) if since is not None: since = datetime.strptime(since, "%Y-%m-%dT%H:%M:%S.%fZ") since = pytz.utc.localize(since) q &= Q(date__gte=since) summarize = self.request.query_params.get("summarize", None) if summarize is not None: summarize = int(summarize) metric_filter = self.request.query_params.get("metric_filter", None) if metric_filter: q &= Q(name=metric_filter) last_n = self.request.query_params.get("last_n", None) if last_n: last_n = int(last_n) metric_type = self.request.query_params.get("metric_type", "pod") if metric_type == "pod": pod = KubePod.objects.filter(name=pk).first() metrics = pod.metrics else: run = ModelRun.objects.get(pk=pk) metrics = run.metrics if request.accepted_renderer.format != "zip": # generate json result = self.__format_result(metrics, q, summarize, last_n) return Response(result, status=status.HTTP_200_OK) result_file = io.BytesIO() with zipfile.ZipFile(result_file, mode="w", compression=zipfile.ZIP_DEFLATED) as zf: if metric_type == "run": run = ModelRun.objects.get(pk=pk) pods = run.pods.all() filename = secure_filename(run.name) since = run.created_at until = run.finished_at q &= Q(date__gte=since) if until: q &= Q(date__lte=until) zf = self.__format_zip_result(metrics, q, summarize, last_n, "result", zf) try: task_result = metrics.get(name="TaskResult @ 0") with io.StringIO() as task_result_file: task_result_file.write(task_result.value) zf.writestr("official_result.txt", task_result_file.getvalue()) except ObjectDoesNotExist: pass for pod in pods: pod_metrics = pod.metrics zf = self.__format_zip_result(pod_metrics, q, summarize, last_n, pod.name, zf) else: zf = self.__format_zip_result(metrics, q, summarize, last_n, "result", zf) pod = KubePod.objects.filter(name=pk).first() filename = secure_filename(pod.name) zf.close() response = Response(result_file.getvalue(), status=status.HTTP_200_OK) response[ "content-disposition"] = "attachment; " "filename=metrics_{}.zip".format( filename) return response
from datetime import datetime from dateutil.rrule import rrule, DAILY from pymongo import MongoClient from dotenv import load_dotenv load_dotenv() MONGO_URL = os.environ.get("MONGO_URL") SINCE_DATE = os.environ.get("SINCE_DATE") UNTIL_DATE = os.environ.get("UNTIL_DATE") mongo_client = MongoClient(MONGO_URL) db = mongo_client.get_default_database() if SINCE_DATE: since_date = datetime.strptime(SINCE_DATE, "%Y-%m-%d") else: latest_msg = db.company_daily_messages.find_one({}, sort=[("date", -1)]) if latest_msg: since_date = latest_msg["date"] else: since_date = datetime.now() if UNTIL_DATE: until_date = datetime.strptime(UNTIL_DATE, "%Y-%m-%d") else: until_date = datetime.now() print(f"since_date: {since_date}") print(f"until_date: {until_date}")
def mtime(self): # assert self.exists() stat = self._stat() mtime = self.mtime_re.search(stat).group(1) date = datetime.strptime(mtime, "%Y-%m-%d %H:%M:%S.%f") return date.timestamp()
def extract_date_available_market(self, start_, end_, trd_cal_=trading_calendar()): """ Match NYSE trading calendar days with input days. """ #define start and endDate as datetime startDate=datetime.strptime(start_,'%Y-%m-%d') endDate=datetime.strptime(end_,'%Y-%m-%d') #check if both are the same dates if startDate == endDate: #assign single startdate as date() only list_pre = [startDate.date()] #find min nearest date to set a benchmark date = min( trd_cal_, key= lambda x: abs(x - list_pre[0]) ) #check if nearest is the same as input date if date == list_pre[0]: idx = [trd_cal_.index(date)] return [trd_cal_[idx[0]].strftime('%Y-%m-%d')] #if not is the same else: print("No trading days at {}".format( startDate.date()) ) sys.exit() #if input dates are different else: #check nearest date to start date date = min( trd_cal_, key=lambda x: abs(x - startDate.date()) ) idx_1 = trd_cal_.index(date) #check nearest date to end date date = min( trd_cal_, key=lambda x: abs(x - endDate.date()) ) idx_2 = trd_cal_.index(date) #make range of dates resulted_dates_range = trd_cal_[idx_1:idx_2+1] #check if range of dates is less than 1 (no dates) if len(resulted_dates_range)<1: print("No trading days in {} to {}".format( startDate.date(), endDate.date()) ) sys.exit() #if there is at least one trading day else: return [result_date_.strftime('%Y-%m-%d') for result_date_ in resulted_dates_range]