def test_log_parser():
p = PlotLogParser()
p.buffer = 0
p.size = 0
p.buckets = 0
p.num_threads = 0
assert p.size == 0
assert p.buffer == 0
assert p.buckets == 0
assert p.num_threads == 0
with open('./tests/assets/plot.logfile') as f:
lines = f.readlines()
p.feed(lines)
assert p.phase == 4
assert p.total_time == 39945.08
assert p.tmp_dir == '/farm/yards/901'
assert p.tmp2_dir == '/farm/yards/901'
assert p.pool_key == '0b6f2b9428744d5062a2073e14b3ca9896a71f7ca9850bdcb285f26108fb19f610c788d47e4830c4c7abfa7611e00168'
assert p.farmer_key == '93222af1a0f7b2ff39f98eb87c1b609fea797798302a60d1f1d6e5152cfdce12c260325d78446e7b8758101b64f43bd5'
assert p.size == 32
assert p.buffer == 4000
assert p.buckets == 128
assert p.num_threads == 4
assert p.start_time == pendulum.local(2021, 4, 4, 19, 0, 50)
assert p.complete_time == pendulum.local(2021, 4, 5, 6, 6, 35)
assert p.target_path == "/farm/wagons/801/plot-k32-2021-04-04-19-00-3eb8a37981de1cc76187a36ed947ab4307943cf92967a7e166841186c7899e24.plot"
assert p.final_dir == '/farm/wagons/801'
def cleanDate(date):
#date contains a date in string, we will remove whitespaces and non-printable chars
pattern_hour = re.compile("[0-9][0-9]:[0-9][0-9]")
pattern_manana = re.compile("Mañana [0-9][0-9]:[0-9][0-9]")
pattern_comienza = re.compile("Comienza en [0-9][0-9]")
pattern_date = re.compile(
"[0-9][0-9] [a-z][a-z][a-z] [0-9][0-9]:[0-9][0-9]")
if not date.isprintable():
date = cleanBothEnds(date)
today = datetime.datetime.now()
#If only game time appears
if pattern_hour.match(date):
hour, minute = date.split(":")
date = str(
pendulum.local(today.year, today.month, today.day, int(hour),
int(minute)))
elif pattern_comienza.match(date):
date = str(pendulum.local(today.year, today.month, today.day))
elif pattern_manana.match(date):
hour, minute = date.split(" ")[-1].split(":")
date = str(
pendulum.local(today.year, today.month, today.day + 1,
int(hour), int(minute)))
elif pattern_date.match(date):
day, month, game_time = date.split(" ")
hour, minute = game_time.split(":")
date = str(
pendulum.local(today.year, int(aliases[month][0]), int(day),
int(hour), int(minute)))
#date = "Hoy a las " + date
else:
date = str(pendulum.now())
return date
def _on_finished(date, time):
if not date or not time:
return
datetime = pendulum.local(date.year(), date.month(), date.day(),
time.hour(), time.minute(),
time.second())
self.mount_workspace(workspace_fs.workspace_id, datetime)
def get_time_zone():
datetime_object = datetime.now()
tz_type = pendulum.local(int(datetime_object.year),
int(datetime_object.month),
int(datetime_object.day))
local_tz_name = tz_type.timezone.name
print(local_tz_name)
return local_tz_name
print(dt.timezone.name)
# create a new datetime using a specified timezone
dt_wtz = pendulum.datetime(2021, 11, 3, tz="America/New_York")
print(dt_wtz)
print(dt_wtz.timezone.name)
# conver the time to another time zone
dt_watz = dt.in_timezone("Europe/Paris")
print(dt_watz)
print(dt_watz.timezone.name)
# create a new datetime using the now() func
# dt = pendulum.now()
dt = pendulum.now("Europe/London")
print(dt)
print(dt.timezone.name)
# create a new datetime using local timezone
dt = pendulum.local(2021, 11, 7)
print(dt)
print(dt.timezone.name)
# create a new datetime using today, tomorrow, yesterday
dates = [pendulum.today(), pendulum.tomorrow(), pendulum.yesterday()]
print(", ".join(map(str, dates)))
# create a new datetime using system timestamp
dt = pendulum.from_timestamp(time.time())
print(dt)
def test_local():
local = pendulum.local(2018, 2, 2, 12, 34, 56, 123456)
assert_datetime(local, 2018, 2, 2, 12, 34, 56, 123456)
assert local.timezone_name == "America/Toronto"
print(isinstance(dt1, datetime))
print(dt1.timezone.name)
# TODO: convert the time to another time zone
dt2 = dt1.in_timezone('Europe/Paris')
print(dt2)
# TODO: create a new datetime using the now() function
dt3 = pendulum.now('Europe/London')
print(dt3)
print(dt3.timezone.name)
# TODO: Use the local function function
here = pendulum.local(2020, 7, 28)
print(here)
print(here.timezone.name)
# TODO: Use today, tomorrow, yesterday
today = pendulum.today()
tomorrow = pendulum.tomorrow()
yest = pendulum.yesterday('America/New_York')
print(today)
print(tomorrow)
print(yest)
# TODO: create a datetime from a system timestamp
t = time.time()
dt4 = pendulum.from_timestamp(t)
print(dt4)
这个如何查看呢？
"""
from airflow.models.dag import DAG
from airflow.operators.bash import BashOperator
from airflow.operators.python import PythonOperator
from airflow.operators.dummy import DummyOperator
from airflow.utils.task_group import TaskGroup
import pendulum
import sys
from datetime import datetime, timedelta
# [START global variable]
# DEFAULT_START_DATE = pendulum.yesterday()
DEFAULT_START_DATE = pendulum.local(2010, 1, 1).to_date_string()
DEFAULT_END_DATE = pendulum.yesterday().to_date_string()
PROJECT_ROOT = "/home/work"
PROJECT_WAREHOUSE = f"{PROJECT_ROOT}/azkaban_warehouse"
PROJECT_ODS = f"{PROJECT_WAREHOUSE}/ods"
PROJECT_DWB = f"{PROJECT_WAREHOUSE}/dwb"
PROJECT_DWS = f"{PROJECT_WAREHOUSE}/dws"
PROJECT_SHARE = f"{PROJECT_WAREHOUSE}/share"
COMMAND_PYTHON3 = "/home/work/app/python3/bin/python3"
COMMAND_DATAX = "python /home/work/app/datax/bin/datax.py"
RESULT_STATE = "success"
params = {
def test_returns_task_schedules(self) -> None:
from pendulum import DateTime as Pendulum, Duration, local, parse
from camcops_server.cc_modules.cc_taskindex import (
PatientIdNumIndexEntry,
TaskIndexEntry,
)
from camcops_server.cc_modules.cc_taskschedule import (
PatientTaskSchedule,
TaskSchedule,
TaskScheduleItem,
)
from camcops_server.tasks.bmi import Bmi
schedule1 = TaskSchedule()
schedule1.group_id = self.group.id
schedule1.name = "Test 1"
self.dbsession.add(schedule1)
schedule2 = TaskSchedule()
schedule2.group_id = self.group.id
self.dbsession.add(schedule2)
self.dbsession.commit()
item1 = TaskScheduleItem()
item1.schedule_id = schedule1.id
item1.task_table_name = "phq9"
item1.due_from = Duration(days=0)
item1.due_by = Duration(days=7)
self.dbsession.add(item1)
item2 = TaskScheduleItem()
item2.schedule_id = schedule1.id
item2.task_table_name = "bmi"
item2.due_from = Duration(days=0)
item2.due_by = Duration(days=8)
self.dbsession.add(item2)
item3 = TaskScheduleItem()
item3.schedule_id = schedule1.id
item3.task_table_name = "phq9"
item3.due_from = Duration(days=30)
item3.due_by = Duration(days=37)
self.dbsession.add(item3)
item4 = TaskScheduleItem()
item4.schedule_id = schedule1.id
item4.task_table_name = "gmcpq"
item4.due_from = Duration(days=30)
item4.due_by = Duration(days=38)
self.dbsession.add(item4)
self.dbsession.commit()
patient = self.create_patient()
idnum = self.create_patient_idnum(
patient_id=patient.id,
which_idnum=self.nhs_iddef.which_idnum,
idnum_value=TEST_NHS_NUMBER,
)
PatientIdNumIndexEntry.index_idnum(idnum, self.dbsession)
server_patient = self.create_patient(as_server_patient=True)
_ = self.create_patient_idnum(
patient_id=server_patient.id,
which_idnum=self.nhs_iddef.which_idnum,
idnum_value=TEST_NHS_NUMBER,
as_server_patient=True,
)
schedule_1 = PatientTaskSchedule()
schedule_1.patient_pk = server_patient.pk
schedule_1.schedule_id = schedule1.id
schedule_1.settings = {
"bmi": {"bmi_key": "bmi_value"},
"phq9": {"phq9_key": "phq9_value"},
}
schedule_1.start_datetime = local(2020, 7, 31)
self.dbsession.add(schedule_1)
schedule_2 = PatientTaskSchedule()
schedule_2.patient_pk = server_patient.pk
schedule_2.schedule_id = schedule2.id
self.dbsession.add(schedule_2)
bmi = Bmi()
self.apply_standard_task_fields(bmi)
bmi.id = 1
bmi.height_m = 1.83
bmi.mass_kg = 67.57
bmi.patient_id = patient.id
bmi.when_created = local(2020, 8, 1)
self.dbsession.add(bmi)
self.dbsession.commit()
self.assertTrue(bmi.is_complete())
TaskIndexEntry.index_task(
bmi, self.dbsession, indexed_at_utc=Pendulum.utcnow()
)
self.dbsession.commit()
proquint = server_patient.uuid_as_proquint
# For type checker
assert proquint is not None
assert self.other_device.name is not None
self.req.fake_request_post_from_dict(
{
TabletParam.CAMCOPS_VERSION: MINIMUM_TABLET_VERSION,
TabletParam.DEVICE: self.other_device.name,
TabletParam.OPERATION: Operations.GET_TASK_SCHEDULES,
TabletParam.PATIENT_PROQUINT: proquint,
}
)
response = client_api(self.req)
reply_dict = get_reply_dict_from_response(response)
self.assertEqual(
reply_dict[TabletParam.SUCCESS], SUCCESS_CODE, msg=reply_dict
)
task_schedules = json.loads(reply_dict[TabletParam.TASK_SCHEDULES])
self.assertEqual(len(task_schedules), 2)
s = task_schedules[0]
self.assertEqual(s[TabletParam.TASK_SCHEDULE_NAME], "Test 1")
schedule_items = s[TabletParam.TASK_SCHEDULE_ITEMS]
self.assertEqual(len(schedule_items), 4)
phq9_1_sched = schedule_items[0]
self.assertEqual(phq9_1_sched[TabletParam.TABLE], "phq9")
self.assertEqual(
phq9_1_sched[TabletParam.SETTINGS], {"phq9_key": "phq9_value"}
)
self.assertEqual(
parse(phq9_1_sched[TabletParam.DUE_FROM]), local(2020, 7, 31)
)
self.assertEqual(
parse(phq9_1_sched[TabletParam.DUE_BY]), local(2020, 8, 7)
)
self.assertFalse(phq9_1_sched[TabletParam.COMPLETE])
self.assertFalse(phq9_1_sched[TabletParam.ANONYMOUS])
bmi_sched = schedule_items[1]
self.assertEqual(bmi_sched[TabletParam.TABLE], "bmi")
self.assertEqual(
bmi_sched[TabletParam.SETTINGS], {"bmi_key": "bmi_value"}
)
self.assertEqual(
parse(bmi_sched[TabletParam.DUE_FROM]), local(2020, 7, 31)
)
self.assertEqual(
parse(bmi_sched[TabletParam.DUE_BY]), local(2020, 8, 8)
)
self.assertTrue(bmi_sched[TabletParam.COMPLETE])
self.assertFalse(bmi_sched[TabletParam.ANONYMOUS])
phq9_2_sched = schedule_items[2]
self.assertEqual(phq9_2_sched[TabletParam.TABLE], "phq9")
self.assertEqual(
phq9_2_sched[TabletParam.SETTINGS], {"phq9_key": "phq9_value"}
)
self.assertEqual(
parse(phq9_2_sched[TabletParam.DUE_FROM]), local(2020, 8, 30)
)
self.assertEqual(
parse(phq9_2_sched[TabletParam.DUE_BY]), local(2020, 9, 6)
)
self.assertFalse(phq9_2_sched[TabletParam.COMPLETE])
self.assertFalse(phq9_2_sched[TabletParam.ANONYMOUS])
# GMCPQ
gmcpq_sched = schedule_items[3]
self.assertTrue(gmcpq_sched[TabletParam.ANONYMOUS])
def test_local():
local = pendulum.local(2018, 2, 2, 12, 34, 56, 123456)
assert_datetime(local, 2018, 2, 2, 12, 34, 56, 123456)
assert local.timezone_name == "America/Toronto"
import pendulum
dt = pendulum.datetime(2019, 5, 1)
print(type(dt))
pendulum.datetime(2019, 5, 1, tz="Asia/Seoul")
dt = pendulum.local(2019, 5, 1)
print(dt.timezone.name)
now = pendulum.now()
print(now)
naive = pendulum.naive(2019, 5, 1)
print(naive.timezone)
dt = pendulum.parse('2019-05-01T22:00:00')
print(dt)
pendulum.set_locale('ko')
print(pendulum.now().add(years=1).diff_for_humans())
dt = pendulum.parse('2019-05-01T22:00:00')
print(dt.year)
print(dt.month)
print(dt.day)
print(dt.hour)
print(dt.minute)
print(dt.second)
print(dt.microsecond)
print(isinstance(dt_us, datetime))
print(dt_us.timezone_name)
# TODO: convert the time to another time zone
d2 = dt1.in_timezone("Asia/Singapore")
print(d2)
# TODO: create a new datetime using the now() function
dt3 = pendulum.now()
print(dt3)
# TODO: Use the local function function
here = pendulum.local(2020, 12, 31)
print(here)
print(here.timezone.name)
# TODO: Use today, tomorrow, yesterday
today = pendulum.today()
tomorrow = pendulum.tomorrow()
yesterday = pendulum.yesterday("America/New_York")
print(today)
print(tomorrow)
print(yesterday)
# TODO: create a datetime from a system timestamp