def process_data(self):
# fill empty share class with NA
self.data['crsp']['SHRCLS'] = self.data['crsp']['SHRCLS'].fillna('NA')
# add market cap column to crsp
self.data['crsp'].loc[:, 'mcap'] = np.abs(
self.data['crsp']['PRC']) * self.data['crsp']['SHROUT']
# add the month end dates
vs = [
utils.last_day_of_month(pd.to_datetime(uu))
for uu in self.data['crsp'].date.unique()
]
me_date_map = dict(zip(self.data['crsp'].date.unique(), vs))
self.data['crsp'].loc[:, 'me_date'] = [
me_date_map[uu] for uu in self.data['crsp'].date.values
]
# redefine the datadate : move one quarter forward
all_dates = list(
set(self.data['link'].datadate.values).union(
self.data['compustat'].datadate.values))
new_dates = [
utils.last_day_of_month(pd.Timestamp(x) + pd.DateOffset(days=80))
for x in all_dates
]
date_map = dict(zip(all_dates, new_dates))
self.data['link'].loc[:, 'datadate'] = [
date_map[uu] for uu in self.data['link'].datadate.values
]
self.data['compustat'].loc[:, 'datadate'] = [
date_map[uu] for uu in self.data['compustat'].datadate.values
]
def data_for_country(self, country):
last_day = last_day_of_month(self.year, self.month)
query = Q(community_worker__country=country)
query &= Q(created_date__lte=last_day)
submissions = devices.models.SubmissionWorkerDevice.objects.medicines_submissions.filter(query).count()
disbursements = models.Disbursement.objects.prior_to(self.year, self.month).filter(country=country)
cost = disbursements.aggregate(Sum('amount'))['amount__sum']
if cost and submissions:
return cost/submissions
return 0.0
def open_file(self, filename, force_date = None):
self._file_list = list()
ds = None
if os.linesep in filename:
# already list of files (date range)
return filename.split(os.linesep)
mtype = mimetypes.guess_type(filename)[0]
if mtype is None or 'xml' not in mtype:
# assuming text file containing list of VFR files
try:
f = open(filename)
i = 0
lines = f.read().splitlines()
for line in lines:
if len(line) < 1 or line.startswith('#'):
continue # skip empty or commented lines
if not line.startswith('http://') and \
not line.startswith('20'):
# determine date if missing
if not force_date:
if line.startswith('ST_Z'):
date = yesterday()
else:
date = last_day_of_month()
else:
date = force_date
line = date + '_' + line
if not line.endswith('.xml.gz'):
# add extension if missing
line += '.xml.gz'
if not os.path.exists(line):
if not line.startswith('http://'):
line = 'http://vdp.cuzk.cz/vymenny_format/soucasna/' + line
line = download_vfr(line)
self._file_list.append(line)
i += 1
VfrLogger.msg("%d VFR files will be processed..." % len(self._file_list))
except IOError:
raise VfrError("Unable to read '%s'" % filename)
f.close()
else:
# single VFR file
self._file_list.append(filename)
return self._file_list
def dashboardReportUser():
month = request.args.get('month')
year = request.args.get('year')
auth_header = request.headers.get('Authorization')
if auth_header:
auth_token = auth_header.split(" ")[1]
else:
auth_token = ''
response = jsonify({'message': 'Provide a valid auth token.'}), 401
if auth_token:
secret_key = os.environ.get('SECRET_KEY', '')
user = decode_auth_token(secret_key, auth_token)
data = {}
today = dt.datetime.today().date()
if user != 401:
if year is None:
year = today.year
if month:
start = dt.datetime(year=int(year), month=int(month),
day=1).date()
else:
start = dt.datetime(year=int(year), month=today.month,
day=1).date()
end = last_day_of_month(start)
totalOfficeHourUserYear = countOfficeHourUserYear(
mongoClient, user['user_id'], today.year)
totalOfficeHourUserMonthly = countOfficeHourUserMonthly(
mongoClient, user['user_id'], str(start), str(end))
totalReportUserYear = countReportUserYear(mongoClient,
user['user_id'],
today.year)
totalReportUserMonthly = countReportUserMonthly(
mongoClient, user['user_id'], str(start), str(end))
array_report_user = arrayReportUser(totalReportUserYear,
totalReportUserMonthly)
array_office_hour_user = arrayOfficeHourUser(
totalOfficeHourUserYear, totalOfficeHourUserMonthly)
data.update(array_report_user)
data.update(array_office_hour_user)
response = jsonify(message="success", data=data, status=200)
return response
def load_mcap_breakpoints(self):
# FAMA FRENCH NYSE ME breakpoints
ff_me_breakpoints = pd.read_csv(
'{loc}FamaFrench_ME_Breakpoints_csv.zip'.format(
loc=self.data_location),
skiprows=[0],
header=-1,
names=['date', 'count'] + [5 * x
for x in np.arange(1, 21)]).iloc[:-1]
dts = [
utils.last_day_of_month(pd.Timestamp('%s-%s-01' % (x[:4], x[4:6])))
for x in ff_me_breakpoints.date.values
]
ff_me_breakpoints.loc[:, 'date'] = dts
self.mcap_breakpoints = ff_me_breakpoints[['date', 20]]
self.mcap_breakpoints.columns = ['mcap_date', 'breakpt']
self.mcap_breakpoints.loc[:, 'breakpt'] = self.mcap_breakpoints[
'breakpt'] * 1000 # to be consistent with the 'mcap' column
def parse_cmd(argv, flags, params, optdir):
get_opt(argv, flags, params, optdir)
if optdir['list']:
if not optdir['dbname']:
raise getopt.GetoptError("--dbname required")
return 0
filename = optdir.get('filename', None)
date = optdir.get('date', None)
ftype = optdir.get('ftype', None)
# check required options
if not filename and not ftype:
raise getopt.GetoptError("--file or --type required")
if filename and ftype:
raise getopt.GetoptError("--file and --type are mutually exclusive")
if ftype and not date:
if ftype.startswith('ST_Z'):
date_list = [yesterday()]
else:
date_list = [last_day_of_month()]
elif ftype and date and ':' in date:
if ftype.startswith('ST_Z'):
date_list = get_date_interval(date)
else:
raise getopt.GetoptError("Date interval is valid only for '--type ST_ZXXX'")
else:
date_list = [date]
if optdir['overwrite'] and optdir.get('append', False):
raise getopt.GetoptError("--append and --overwrite are mutually exclusive")
if optdir['layer']:
optdir['layer'] = optdir['layer'].split(',')
if filename:
# is file a valid VFR file
filename = check_file(filename)
else: # --date & --type
flist = []
base_url = "http://vdp.cuzk.cz/vymenny_format/"
if ftype != 'ST_UVOH':
base_url += "soucasna/"
else:
base_url += "specialni/"
for d in date_list:
fname = "%s_%s.xml.gz" % (d, ftype)
url = base_url + fname
if os.path.exists(fname):
# use existing file
flist.append(fname)
else:
# download file
flist.append(download_vfr(url))
if not flist:
raise getopt.GetoptError("Empty date range")
filename = os.linesep.join(flist)
if not filename:
raise getopt.GetoptError("Invalid input file")
return filename
def prior_to(self, year, month):
last_day = last_day_of_month(year, month)
query = Q(date__lte=last_day)
return super(PriorToManager, self).get_query_set().filter(query)
def data_for_country(self, country):
last_day = last_day_of_month(self.year, self.month)
query = Q(community_worker__country=country)
query &= Q(created_date__lte=last_day)
submissions = devices.models.SubmissionWorkerDevice.objects.medicines_submissions.filter(query).count()
return submissions
def dashboardAttendanceUser():
month = request.args.get('month')
year = request.args.get('year')
auth_header = request.headers.get('Authorization')
if auth_header:
auth_token = auth_header.split(" ")[1]
else:
auth_token = ''
response = jsonify({'message': 'Provide a valid auth token.'}), 401
if auth_token:
secret_key = os.environ.get('SECRET_KEY', '')
user = decode_auth_token(secret_key, auth_token)
data = {}
if user != 401:
today = dt.datetime.today().date()
if year is None:
year = today.year
if month:
start = dt.datetime(year=int(year), month=int(month),
day=1).date()
else:
month = today.month
start = dt.datetime(year=int(year), month=int(month),
day=1).date()
end = last_day_of_month(start)
dateRange = np.arange(np.datetime64(start),
np.datetime64(end + timedelta(days=1)),
dtype='datetime64[D]')
listBusday = np.busdaycalendar(holidays=dateRange,
weekmask=busmask_names)
listWeekend = np.busdaycalendar(holidays=dateRange,
weekmask=weekmask_names)
listHoliday = getListHoliday(mongoClient, np, start.year,
start.month)
get_data_redis = redis_client.get(
keys_redis(user['user_id'], 'attendances'))
parse_query_date = dt.datetime.strptime(
str(year) + '-' + str(month), '%Y-%m').date()
if get_data_redis and parse_query_date < dt.datetime.today(
).replace(day=1).date():
start_date = dt.datetime.strptime(
str(start) + ' 00:00:00', '%Y-%m-%d %H:%M:%S')
end_date = dt.datetime.strptime(
str(end) + ' 23:59:59', '%Y-%m-%d %H:%M:%S')
list_presence = [
data for data in json.loads(get_data_redis) if
start_date <= parse_datetime(data['startDate']) <= end_date
]
else:
list_presence = getPresence(mongoClient, user['user_id'],
str(start), str(end))
# Delete working days if there are holidays
listBusday = np.array(
list(
filter(lambda x: x not in listHoliday,
listBusday.holidays)))
latePresence = len([
data for data in list_presence if parse_datetime(
data['startDate']).date() not in listWeekend.holidays and
max_time_presence < parse_datetime(data['startDate']).time()
])
permit = len([
data for data in list_presence
if data['message'] in ['CUTI', 'SAKIT', 'IZIN']
])
totalWfh = len(
[data for data in list_presence if data['location'] == 'WFH'])
totalWfo = len(
[data for data in list_presence if data['location'] == 'WFO'])
totalPerjadin = len([
data for data in list_presence
if data['location'] == 'PERJADIN'
])
presence = len([
data for data in list_presence
if parse_datetime(data['startDate']).date() in listBusday
])
presenceWeekend = len([
data for data in list_presence if parse_datetime(
data['startDate']).date() in listWeekend.holidays
])
busDays = len(listBusday)
weekEnd = len(listWeekend.holidays)
#check no precene from today
if parse_query_date >= dt.datetime.today().replace(day=1).date():
dateRangeFromNow = np.arange(np.datetime64(start),
np.datetime64(today +
timedelta(days=1)),
dtype='datetime64[D]')
listBusdayFromNow = np.busdaycalendar(
holidays=dateRangeFromNow, weekmask=busmask_names)
listBusdayFromNow = np.array(
list(
filter(lambda x: x not in listHoliday,
listBusdayFromNow.holidays)))
noPresence = len(listBusdayFromNow) - presence
else:
noPresence = busDays - presence
precentagePresence = round(
float(presence - permit) / float(busDays) * 100, 2)
precentageLatePresence = round(
float(latePresence) / float(busDays) * 100, 2)
precentagePermit = round(float(permit) / float(busDays) * 100, 2)
precentageNoPresence = round(
float(noPresence) / float(busDays) * 100, 2)
precentagePresenceWeekend = round(
float(presenceWeekend) / float(weekEnd) * 100, 2)
precentageWfo = round(float(totalWfo) / float(busDays) * 100, 2)
precentageWfh = round(float(totalWfh) / float(busDays) * 100, 2)
precentagePerjadin = round(
float(totalPerjadin) / float(busDays) * 100, 2)
array_presence = arrayPresence(noPresence, precentageNoPresence,
presence - permit,
precentagePresence, presenceWeekend,
precentagePresenceWeekend,
latePresence,
precentageLatePresence)
array_permit = arrayPermit(permit, precentagePermit)
array_location_user = arrayLocationUser(totalWfo, precentageWfo,
totalWfh, precentageWfh,
totalPerjadin,
precentagePerjadin)
data.update(array_presence)
data.update(array_permit)
data.update(array_location_user)
response = jsonify(message="success", data=data, status=200)
return response