def __init__(self, circle, fcp, totalsize=0):
BaseTask.__init__(self, circle)
self.circle = circle
self.fcp = fcp
self.totalsize = totalsize
# cache
self.fd_cache = LRU(512)
# failed
self.failed = {}
self.failcnt = 0
# debug
self.d = {"rank": "rank %s" % circle.rank}
self.logger = utils.getLogger(__name__)
# reduce
self.vsize = 0
assert len(circle.workq) == 0
if self.circle.rank == 0:
print("\nChecksum verification ...")
def __init__(self, companies, **kwargs):
BaseTask.__init__(self, companies, **kwargs)
self.actions = ['news//last//5']
self.action_keys = {
'news//last//5':
['datetime', 'headline', 'source', 'url', 'summary']
}
def __init__(self,
circle,
fcp,
total_chunks,
totalsize=0,
signature=False):
BaseTask.__init__(self, circle)
self.circle = circle
self.fcp = fcp
self.totalsize = totalsize
self.signature = signature
if self.signature:
self.bfsign = BFsignature(total_chunks)
# cache
self.fd_cache = LRU(512)
# failed
self.failed = {}
self.failcnt = 0
# debug
self.d = {"rank": "rank %s" % circle.rank}
self.logger = utils.getLogger(__name__)
# reduce
self.vsize = 0
assert len(circle.workq) == 0
if self.circle.rank == 0:
print("\nChecksum verification ...")
def __init__(self, circle, fcp, total_chunks, totalsize=0,signature=False):
BaseTask.__init__(self, circle)
self.circle = circle
self.fcp = fcp
self.totalsize = totalsize
self.signature = signature
if self.signature:
self.bfsign = BFsignature(total_chunks)
# failed
self.failed = {}
self.failcnt = 0
# debug
self.d = {"rank": "rank %s" % circle.rank}
self.logger = utils.getLogger(__name__)
# reduce
self.vsize = 0
assert len(circle.workq) == 0
if self.circle.rank == 0:
print("\nChecksum verification ...")
def __init__(self, companies, **kwargs):
BaseTask.__init__(self, companies, **kwargs)
self.actions = ['chart//1d']
self.action_keys = {
'chart//1d': [
'minute', 'volume', 'changeOverTime', 'high', 'low', 'average',
'marketNumberOfTrades'
]
}
def __init__(self, companies, **kwargs):
BaseTask.__init__(self, companies, **kwargs)
self.actions = ['company']
self.action_keys = {
'company': [
'symbol', 'companyName', 'exchange', 'industry', 'description',
'sector'
]
}
def __init__(self, user, args=None):
BaseTask.__init__(self)
self.args = args
print 'in AppointmentReminder:: self.args: %s; type(self.args):%s' % (self.args, type(self.args))
if isinstance(user, sms.User):
self.user = user
self.patient = Patient.objects.get(address=self.user.identity)
else:
raise ValueError('unknown type given for user: %s' % user)
# this is awkward for messaging (please refer to the spec).
# a cleaner solution is two reminder tasks like: 'Reminder' and 'Cancellable Reminder'
# below you must support both response (ok and cancel) which would allow a user to cancel when you don't want them to...
# m1
if 'nocancel' in self.args:
q1 = render_to_string('tasks/appts/reminder_nocancel.html', {'patient': self.patient, 'args': self.args})
else:
q1 = render_to_string('tasks/appts/reminder.html', {'patient': self.patient, 'args': self.args})
r1 = sms.Response('ok', match_regex=r'ok', label='ok')
r2 = sms.Response('cancel', match_regex=r'cancel|no', callback=self.cancel, label='cancel')
r_stop = sms.Response('stop', match_regex=r'stop', label='stop', callback=self.stopped)
# the initial reminder message
m1 = sms.Message(
q1,
[r1, r2, r_stop],
label='appt', retries=AppointmentReminder.RETRY_COUNT, timeout=AppointmentReminder.RETRY_TIMEOUT,
no_match_callback=self.unparseable_response)
# user ok'd appointment
m2 = sms.Message(
'See you soon.', [],
label='appt')
# user cancelled appointment
m3 = sms.Message(
'Thank you for letting us know. Remember to also actually cancel your appointment with your care provider.', [],
label='appt')
# user wants all messages stopped
m_stop = sms.Message(
'Ok, stopping messages now. Thank you for participating.', [],
label='appt')
self.graph = { m1: [m2, m3, m_stop],
m2: [],
m3: [],
m_stop: [] }
# self.interaction = sms.Interaction(graph=self.graph, initialnode=m1, label='interaction')
super(AppointmentReminder, self).setinteraction(graph=self.graph, initialnode=m1, label='interaction')
def __init__(self, circle, src, dest,
treewalk=None,
totalsize=0,
hostcnt=0,
prune=False,
verify=False,
resume=False,
workq=None):
BaseTask.__init__(self, circle)
self.circle = circle
self.treewalk = treewalk
self.totalsize = totalsize
self.prune = prune
self.workq = workq
self.resume = resume
self.checkpoint_file = None
self.checkpoint_db = None
self.src = src
self.dest = os.path.abspath(dest)
self.rfd_cache = LRU(oflimit, callback = self.cb_close_fd)
self.wfd_cache = LRU(oflimit, callback = self.cb_close_fd)
self.cnt_filesize_prior = 0
self.cnt_filesize = 0
self.blocksize = 1024 * 1024
self.chunksize = 1024 * 1024
# debug
self.d = {"rank": "rank %s" % circle.rank}
self.wtime_started = MPI.Wtime()
self.wtime_ended = None
self.workcnt = 0 # this is the cnt for the enqued items
self.reduce_items = 0 # this is the cnt for processed items
if self.treewalk:
log.debug("treewalk files = %s" % treewalk.flist, extra=self.d)
# fini_check
self.fini_cnt = Counter()
# verify
self.verify = verify
self.use_store = False
if self.verify:
self.chunksums_mem = []
self.chunksums_buf = []
# checkpointing
self.checkpoint_interval = sys.maxsize
self.checkpoint_last = MPI.Wtime()
if self.circle.rank == 0:
print("Start copying process ...")
def __init__(self, circle, src, dest=None, preserve=False, force=False):
BaseTask.__init__(self, circle)
self.d = {"rank": "rank %s" % circle.rank}
self.circle = circle
self.src = src
self.dest = dest
self.force = force
self.interval = 10 # progress report
# For now, I am setting the option
# TODO: should user allowed to meddle this?
self.sizeonly = False
self.checksum = False
# to be fixed
self.optlist = [] # files
self.opt_dir_list = [] # dirs
self.sym_links = 0
self.follow_sym_links = False
self.workdir = os.getcwd()
self.tempdir = os.path.join(self.workdir, ".pcircle")
if not os.path.exists(self.tempdir):
os.mkdir(self.tempdir)
if G.use_store:
self.dbname = "%s/fwalk.%s" % (self.tempdir, circle.rank)
self.flist = DbStore(self.dbname)
self.flist_buf = []
else:
self.flist = []
self.src_flist = self.flist
# hold unlinkable dest directories
# we have to do the --fix-opt at the end
self.dest_dirs = []
self.cnt_dirs = 0
self.cnt_files = 0
self.cnt_filesize = 0
self.last_cnt = 0
self.skipped = 0
self.last_reduce_time = MPI.Wtime()
# reduce
self.reduce_items = 0
self.time_started = MPI.Wtime()
self.time_ended = None
def __init__(self, namespace=None, cursor=None):
self.namespace = namespace if namespace else Namespace()
self.status_error = NO_ERROR
self.cursor = cursor
self.map_func_task = dict() if not hasattr(
self, 'map_func_task') else self.map_func_task
self.map_func_task[BaseTask()] = self.task_run
def __init__(self, user, args=None):
BaseTask.__init__(self)
self.args = args
print "in appointmentrequest: self.args: %s; type(self.args):%s" % (self.args, type(self.args))
if isinstance(user, sms.User):
self.user = user
self.patient = Patient.objects.get(address=self.user.identity)
else:
raise ValueError("unknown type given for user: %s" % user)
# m1
# resolves to:
# Hi {{ patient.first_name }}. Please schedule a {{ args.appt_type }}.
# Reply with a time (like 10/1/2012 16:30:00), or 'stop' to quit.
q1 = render_to_string("tasks/appts/request.html", {"patient": self.patient, "args": self.args})
r1 = sms.Response("stop", match_regex=r"stop|STOP", label="r1", callback=self.appointment_cancelled_alert)
# r2 = sms.Response('8/30/2012 16:30:00', r'\d+/\d+/\d+\s\d+:\d+:\d+', label='datetime', callback=self.schedule_reminders)
r2 = sms.Response(
"8/30/2012 16:30:00",
match_callback=AppointmentRequest.match_date,
label="r2",
callback=self.schedule_reminders,
)
m1 = sms.Message(q1, [r1, r2], label="m1")
# m2
q2 = "Ok, stopping messages now. Thank you for participating."
m2 = sms.Message(q2, [], label="m2")
# m3
# resolves to:
# Great, we set up 3 appt. reminders and a followup for you.
q3 = render_to_string("tasks/appts/rescheduled.html", {"args": self.args})
m3 = sms.Message(q3, [], label="m3")
# define a super class with .restore() in it. below, user will call createGraph(), createInteraction()
# which remember handles to graph and interaction. when .restore() is called it just updates the node we're at searching with the label.
graph = {m1: [m2, m3], m2: [], m3: []}
# set self.graph
self.graph = graph
# set self.interaction
# self.interaction = sms.Interaction(graph=self.graph, initialnode=m1, label='interaction')
super(AppointmentRequest, self).setinteraction(graph=self.graph, initialnode=m1, label="interaction")
def __init__(self, user, args=None):
BaseTask.__init__(self)
self.args = args
print 'in AppointmentReminder:: self.args: %s; type(self.args):%s' % (self.args, type(self.args))
if isinstance(user, sms.User):
self.user = user
self.patient = Patient.objects.get(address=self.user.identity)
else:
raise ValueError('unknown type given for user: %s' % user)
# this is awkward for messaging (please refer to the spec).
# a cleaner solution is two reminder tasks like: 'Reminder' and 'Cancellable Reminder'
# below you must support both response (ok and cancel) which would allow a user to cancel when you don't want them to...
# m1
if 'nocancel' in self.args:
# resolves to:
# {% load parse_date %}Your {{ args.appt_type }} is approaching. Reply 'ok' to confirm.
q1 = render_to_string('tasks/appts/reminder_nocancel.html', {'patient': self.patient, 'args': self.args})
else:
# resolves to:
#{% load parse_date %}Your {{ args.appt_type }} is approaching. Reply 'cancel' to cancel it or 'ok' to confirm.
q1 = render_to_string('tasks/appts/reminder.html', {'patient': self.patient, 'args': self.args})
r1 = sms.Response('ok', match_regex=r'ok|OK|Ok', label='r1')
r2 = sms.Response('cancel', match_regex=r'cancel|no', label='r2', callback=self.cancel)
m1 = sms.Message(q1, [r1,r2], label='m1')
# m2
q2 = 'See you soon.'
m2 = sms.Message(q2, [], label='m2')
# m3
q3 = 'Ok, canceling appointment as you requested.'
m3 = sms.Message(q3, [], label='m3')
self.graph = { m1: [m2, m3],
m2: [],
m3: [] }
# self.interaction = sms.Interaction(graph=self.graph, initialnode=m1, label='interaction')
super(AppointmentReminder, self).setinteraction(graph=self.graph, initialnode=m1, label='interaction')
def __init__(self, circle, src, dest=None, preserve=False, force=False):
BaseTask.__init__(self, circle)
self.d = {"rank": "rank %s" % circle.rank}
self.circle = circle
self.src = src
self.dest = dest
self.force = force
self.use_store = False
self.interval = 10 # progress report
# For now, I am setting the option
# TODO: should user allowed to meddle this?
self.sizeonly = False
self.checksum = False
# to be fixed
self.optlist = [] # files
self.opt_dir_list = [] # dirs
self.sym_links = 0
self.follow_sym_links = False
self.flist = []
self.flist_buf = []
# hold unlinkable dest directories
# we have to do the --fix-opt at the end
self.dest_dirs = []
self.cnt_dirs = 0
self.cnt_files = 0
self.cnt_filesize = 0
self.last_cnt = 0
self.skipped = 0
self.last_reduce_time = MPI.Wtime()
# reduce
self.reduce_items = 0
self.time_started = MPI.Wtime()
self.time_ended = None
def __init__(self, circle, treewalk, chunksize, totalsize=0):
BaseTask.__init__(self, circle)
self.circle = circle
self.treewalk = treewalk
self.totalsize = totalsize
self.workcnt = 0
self.chunkq = []
self.chunksize = chunksize
# debug
self.d = {"rank": "rank %s" % circle.rank}
self.wtime_started = MPI.Wtime()
self.wtime_ended = None
# reduce
self.vsize = 0
self.logger = utils.getLogger(__name__)
if self.circle.rank == 0:
print("Start parallel checksumming ...")
def __init__(self, user, args=None):
# why is this here?
BaseTask.__init__(self)
self.args = args
print "in mediabus: self.args: %s; type(self.args):%s" % (self.args, type(self.args))
self.user = user
if "message" not in self.args.keys():
raise Exception("No message in args")
message = args["message"]
firstword = int(str(message).lower().split(None, 1)[0])
textrest = ""
try:
textrest = str(message).split(None, 1)[1]
except:
textrest = ""
# Store to DB
mbrl = MediaBusRawLog(identity=user.identity, message_number=firstword, message_text=textrest)
mbrl.save()
# Thank you message
m_thank = sms.Message(
"Thank you for your input! Gracias por su participacion!",
[],
label="thankyouresponse",
retries=0,
timeout=0,
)
# define a super class with .restore() in it. below, user will call createGraph(), createInteraction()
# which remember handles to graph and interaction. when .restore() is called it just updates the node we're at searching with the label.
self.graph = {m_thank: []}
super(mediabus, self).setinteraction(graph=self.graph, initialnode=m_thank, label="mediabusinteraction")
def __init__(self,
namespace=None,
migration=None,
preparation=None,
rollback=None):
self.namespace = namespace if namespace else Namespace()
self.status_error = NO_ERROR
self.migration = migration
self.preparation = preparation
self.rollback = rollback
self.map_func_task = dict() if not hasattr(
self, 'map_func_task') else self.map_func_task
self.map_func_task[BaseTask()] = self.task_run
def __init__(self, user, args=None):
BaseTask.__init__(self)
self.args = args
print 'in appointmentrequest: self.args: %s; type(self.args):%s' % (self.args, type(self.args))
if isinstance(user, sms.User):
self.user = user
self.patient = Patient.objects.get(address=self.user.identity)
else:
raise ValueError('unknown type given for user: %s' % user)
# (old) r2 = sms.Response('8/30/2010 16:30:00', r'\d+/\d+/\d+\s\d+:\d+:\d+', label='datetime', callback=self.reschedule_reminder)
r_feedback = sms.Response('feedback', match_regex=r'(feedback)', label='feedback', callback=self.store_feedback)
r_missed = sms.Response('missed', match_regex=r'(missed)', label='missed', callback=self.missed_appt)
# message asking them for feedback after their appointment
m1 = sms.Message(
render_to_string('tasks/appts/followup.html', {'patient': self.patient, 'args': self.args}),
[r_feedback, r_missed],
label='resp', retries=AppointmentFollowup.RETRY_COUNT, timeout=AppointmentFollowup.RETRY_TIMEOUT,
no_match_callback=self.unparseable_response)
# message sent when user provides feedback in the form 'feedback <my message here>'
m_thanks = sms.Message('Thank you for your feedback.', [], label='thanks')
# message sent when user missed their appointment
m_missed = sms.Message('You will receive a new request to reschedule within a few days.', [], label='missed')
self.graph = { m1: [m_thanks, m_missed],
m_thanks: [],
m_missed: [] }
# self.interaction = sms.Interaction(self.graph, m1, self.__class__.__name__ + '_interaction')
super(AppointmentFollowup, self).setinteraction(graph=self.graph, initialnode=m1, label='interaction')
def __init__(self, circle, treewalk, chunksize, totalsize=0, totalfiles=0):
BaseTask.__init__(self, circle)
self.circle = circle
self.treewalk = treewalk
self.totalsize = totalsize
self.totalfiles = totalfiles
self.total_chunks = 0
self.workcnt = 0
#self.chunkq = []
self.chunksize = chunksize
# debug
self.d = {"rank": "rank %s" % circle.rank}
self.wtime_started = MPI.Wtime()
self.wtime_ended = None
# reduce
self.vsize = 0
self.vsize_prior = 0
self.logger = utils.getLogger(__name__)
if self.circle.rank == 0:
print("Start parallel checksumming ...")
def __init__(self, user, args=None):
BaseTask.__init__(self)
self.args = args
print 'in appointmentrequest: self.args: %s; type(self.args):%s' % (self.args, type(self.args))
if isinstance(user, sms.User):
self.user = user
self.patient = Patient.objects.get(address=self.user.identity)
else:
raise ValueError('unknown type given for user: %s' % user)
# m1
# resolves to:
# How was your {{ args.appt_type }}? Reply with 'comment' and feedback; or a time (10/1/2020 16:30:00) to reschedule.
q1 = render_to_string('tasks/appts/followup.html', {'patient': self.patient, 'args': self.args})
r1 = sms.Response('comment', match_regex=r'comment|COMMENT', label='comment', callback=self.store_feedback)
#r2 = sms.Response('8/30/2010 16:30:00', r'\d+/\d+/\d+\s\d+:\d+:\d+', label='datetime', callback=self.reschedule_reminder)
r2 = sms.Response('8/30/2010 16:30:00', match_callback=AppointmentFollowup.match_date, label='datetime', callback=self.reschedule_reminder)
m1 = sms.Message(q1, [r1,r2], label='m1')
# m2
q2 = 'Thank you for your feedback.'
m2 = sms.Message(q2, [], label='m2')
# m3
q3 = 'Ok, you will be sent a reminder to reschedule as you have requested.'
m3 = sms.Message(q3, [], label='m3')
self.graph = { m1: [m2, m3],
m2: [],
m3: [] }
# self.interaction = sms.Interaction(self.graph, m1, self.__class__.__name__ + '_interaction')
super(AppointmentFollowup, self).setinteraction(graph=self.graph, initialnode=m1, label='interaction')
def __init__(self, user, args=None):
BaseTask.__init__(self)
self.args = args
print 'in appointmentrequest: self.args: %s; type(self.args):%s' % (self.args, type(self.args))
if isinstance(user, sms.User):
self.user = user
self.patient = Patient.objects.get(address=self.user.identity)
else:
raise ValueError('unknown type given for user: %s' % user)
# (old) r2 = sms.Response('8/30/2012 16:30:00', r'\d+/\d+/\d+\s\d+:\d+:\d+', label='datetime', callback=self.schedule_reminders)
# first we set up all the expected responses
# idealy this'd be directly before each message, but most of these responses are repeated in this case
# r_cancel = sms.Response('cancel', match_regex=r'cancel|no', label='cancel', callback=self.appointment_cancelled_alert)
r_stop = sms.Response('stop', match_regex=r'cancel|n|no|stop', label='stop', callback=self.appointment_stopped_alert)
r_stalling = sms.Response('ok', match_regex=r'ok', label='stalling')
r_date_past = sms.Response('yesterday at 3pm', match_callback=AppointmentRequest.match_past_date, label='datepast')
r_valid_appt = sms.Response('today at 3pm', match_callback=AppointmentRequest.match_date_and_time, callback=self.checking_valid_appt, label='datetime')
r_appt_confirmed = sms.Response('ok', match_regex=r'ok', label='confirmed', callback=self.schedule_reminders)
# message sent asking the user to schedule an appt. and to text us back the date and time
m_initial = sms.Message(
render_to_string('tasks/appts/request.html', {'patient': self.patient, 'args': self.args}),
[r_stop, r_stalling, r_date_past, r_valid_appt],
label='remind', retries=AppointmentRequest.RETRY_COUNT, timeout=AppointmentRequest.RETRY_TIMEOUT,
no_match_callback=self.unparseable_response)
# message sent when the user doesn't want reminders for whatever reason
# FAISAL: this is currently disabled -- 'no', 'stop', and 'cancel' are all treated as full stops
## m_cancel = sms.Message(
## 'Ok, you will not be sent any additional reminders for this appointment.', [],
## label='remind')
# message sent when the user decides to stop
m_stop = sms.Message(
'Ok, stopping all messages now. Thank you for participating.', [],
label='remind')
# message sent when the user delays us by saying 'ok'; we prompt them for more info when they're ready
# this replicates the expected responses of the initial node because it basically is the initial node
m_stalling = sms.Message(
'Please respond with both the date and the time of the appointment when you\'ve scheduled it, or \'stop\' if you don\'t want to schedule one now.',
[r_stop, r_stalling, r_date_past, r_valid_appt],
label='remind', retries=AppointmentRequest.RETRY_COUNT, timeout=AppointmentRequest.RETRY_TIMEOUT)
# message sent when the user enters a date that's already past; we tell them what they entered and reprompt them
# this replicates the expected responses of the initial node because it basically is the initial node
m_date_past = sms.Message(
"returned by custom msg callback",
[r_stop, r_stalling, r_date_past, r_valid_appt],
label='remind', custom_message_callback=self.date_past_callback,
retries=AppointmentRequest.RETRY_COUNT, timeout=AppointmentRequest.RETRY_TIMEOUT)
# FAISAL: left to demonstrate no match callback parameter
# m1 = sms.Message(q1, [r1,r2], label='m1', no_match_callback=self.no_match_test)
# message sent when the user sends us a valid date and time
# we wait for confirmation, and if it comes we set up the appointment
m_valid_appt = sms.Message(
"returned by custom msg callback",
[r_stop, r_appt_confirmed, r_date_past, r_valid_appt],
label='remind', custom_message_callback=self.valid_appt_msg_callback)
# message sent when the user confirms their valid appointment
# the response that leads here actually sets up the appointment
m_appt_confirmed = sms.Message(
render_to_string('tasks/appts/response_confirmed.html', {'patient': self.patient, 'args': self.args}), [],
label='remind')
# define a super class with .restore() in it. below, user will call createGraph(), createInteraction()
# which remember handles to graph and interaction. when .restore() is called it just updates the node we're at searching with the label.
self.graph = { m_initial: [m_stop, m_stalling, m_date_past, m_valid_appt],
m_stop: [],
m_stalling: [m_stop, m_stalling, m_date_past, m_valid_appt],
m_date_past: [m_stop, m_stalling, m_date_past, m_valid_appt],
m_valid_appt: [m_stop, m_appt_confirmed, m_date_past, m_valid_appt],
m_appt_confirmed: []
}
super(AppointmentRequest, self).setinteraction(graph=self.graph, initialnode=m_initial, label='interaction')
def __init__(self, companies, **kwargs):
BaseTask.__init__(self, companies, **kwargs)
self.actions = ['price']
def __init__(self, circle, src, dest,
treewalk=None,
totalsize=0,
hostcnt=0,
prune=False,
verify=False,
resume=False,
workq=None):
BaseTask.__init__(self, circle)
self.circle = circle
self.treewalk = treewalk
self.totalsize = totalsize
self.prune = prune
self.workq = workq
self.resume = resume
self.checkpoint_file = None
self.checkpoint_db = None
self.src = src
self.dest = os.path.abspath(dest)
# cache, keep the size conservative
# TODO: we need a more portable LRU size
if hostcnt != 0:
max_ofile, _ = resource.getrlimit(resource.RLIMIT_NOFILE)
procs_per_host = self.circle.size / hostcnt
self._read_cache_limit = ((max_ofile - 64) / procs_per_host) / 3
self._write_cache_limit = ((max_ofile - 64) / procs_per_host) * 2 / 3
if self._read_cache_limit <= 0 or self._write_cache_limit <= 0:
self._read_cache_limit = 1
self._write_cache_limit = 8
self.rfd_cache = LRU(self._read_cache_limit)
self.wfd_cache = LRU(self._write_cache_limit)
self.cnt_filesize_prior = 0
self.cnt_filesize = 0
self.blocksize = 1024 * 1024
self.chunksize = 1024 * 1024
# debug
self.d = {"rank": "rank %s" % circle.rank}
self.wtime_started = MPI.Wtime()
self.wtime_ended = None
self.workcnt = 0 # this is the cnt for the enqued items
self.reduce_items = 0 # this is the cnt for processed items
if self.treewalk:
log.debug("treewalk files = %s" % treewalk.flist, extra=self.d)
# fini_check
self.fini_cnt = Counter()
# verify
self.verify = verify
self.use_store = False
if self.verify:
self.chunksums_mem = []
self.chunksums_buf = []
# checkpointing
self.checkpoint_interval = sys.maxsize
self.checkpoint_last = MPI.Wtime()
if self.circle.rank == 0:
print("Start copying process ...")
def __init__(self, companies, **kwargs):
BaseTask.__init__(self, companies, **kwargs)
self.actions = ['chart','stats']
self.action_keys = {'chart': ['date','high','low','volume','changePercent','open','close'],'stats': ['marketcap','beta','week52high','week52low','week52change','shortInterest','shortDate','float','consensusEPS','day200MovingAvg','day50MovingAvg','institutionPercent','insiderPercent','shortRatio','month6ChangePercent','month3ChangePercent','month1ChangePercent','day5ChangePercent']}
self.average_keys = ['close','change','spread','volume']
self.periods = [5,10,20]
def __init__(self, circle, src, dest,
treewalk=None,
totalsize=0,
hostcnt=0,
prune=False,
verify=False,
resume=False,
workq=None):
BaseTask.__init__(self, circle)
self.circle = circle
self.treewalk = treewalk
self.totalsize = totalsize
self.prune = prune
self.workq = workq
self.resume = resume
self.checkpoint_file = None
self.src = src
self.dest = os.path.abspath(dest)
# cache, keep the size conservative
# TODO: we need a more portable LRU size
if hostcnt != 0:
max_ofile, _ = resource.getrlimit(resource.RLIMIT_NOFILE)
procs_per_host = self.circle.size / hostcnt
self._read_cache_limit = ((max_ofile - 64) / procs_per_host) / 3
self._write_cache_limit = ((max_ofile - 64) / procs_per_host) * 2 / 3
if self._read_cache_limit <= 0 or self._write_cache_limit <= 0:
self._read_cache_limit = 1
self._write_cache_limit = 8
self.rfd_cache = LRU(self._read_cache_limit)
self.wfd_cache = LRU(self._write_cache_limit)
self.cnt_filesize_prior = 0
self.cnt_filesize = 0
self.blocksize = 1024 * 1024
self.chunksize = 1024 * 1024
# debug
self.d = {"rank": "rank %s" % circle.rank}
self.wtime_started = MPI.Wtime()
self.wtime_ended = None
self.workcnt = 0 # this is the cnt for the enqued items
self.reduce_items = 0 # this is the cnt for processed items
if self.treewalk:
log.debug("treewalk files = %s" % treewalk.flist, extra=self.d)
# fini_check
self.fini_cnt = Counter()
# verify
self.verify = verify
self.chunksums = []
# checkpointing
self.checkpoint_interval = sys.maxsize
self.checkpoint_last = MPI.Wtime()
if self.circle.rank == 0:
print("Start copying process ...")