def rlinput(prompt, prefill='', oneline=False, ctxkey=''):
"""
Get user input with readline editing support.
"""
sentinel = ''
if prefill is None:
prefill = ''
def only_once(text):
""" generator for startup hook """
readline.insert_text(text)
yield
while True:
yield
savedhist = NamedTemporaryFile()
readline.write_history_file(savedhist.name)
ctxhistname = ".tl" + ctxkey + "history"
ctxhistfile = os.path.join(G.ProjectFolder, ctxhistname)
try:
readline.clear_history()
except AttributeError:
print "This readline doesn't support clear_history()"
raise
savedcompleter = readline.get_completer()
try:
ulines = uniqify(ctxhistfile)
readline.read_history_file(ctxhistfile)
readline.set_completer(HistoryCompleter(ulines).complete)
except IOError:
pass
readline.parse_and_bind('tab: complete')
saveddelims = readline.get_completer_delims()
readline.set_completer_delims('') ## No delims. Complete entire lines.
readline.set_completion_display_matches_hook(match_display_hook)
gen = only_once(prefill)
readline.set_startup_hook(gen.next)
try:
if oneline:
edited = raw_input(prompt)
else:
print prompt
edited = "\n".join(iter(raw_input, sentinel))
if edited.endswith(r'%%'):
## Invoke external editor
edited = external_edit(edited[0:-2])
return edited
finally:
## Restore readline state
readline.write_history_file(ctxhistfile)
readline.clear_history()
readline.read_history_file(savedhist.name)
savedhist.close()
readline.set_completer(savedcompleter)
readline.set_completer_delims(saveddelims)
readline.set_startup_hook()
def __init__(self, name, title, files, stype, cfg, trees,
cuts = DEFAULT_TRUE, weights = '1.0', scale = 1.0, lumi = 1.0,
channel = '', roottree = 'HWWTree'):
assert(type(name) is str)
assert(type(title) is str)
assert(type(cuts) is str)
assert(type(weights) is str)
assert(type(scale) is float)
self.__root = roottree # << the root tree (all other trees will be friends)
self.__trees = None # << list of TChain's controlled by this sample
self.__tree_names = utils.uniqify(utils.listify(trees) + [self.__root])
self.__name = name # << name of the sample
self.__title = title # << title (used in legends, &c.)
self.__cuts = cuts.replace(' ','') # << cuts that define sample selection
self.__cuts_base = self.__cuts
self.__applied_cuts = None
self.__weights = weights.replace(' ','') # << event weights
self.__weights_base = self.__weights
self.__scale = scale # << product of scale factors
self.__counters = {} # << counters maintained by this sample
self.__histograms = {} # << histograms produced by this sample
self.__internals = [] # << other internal objects to keep
self.__locked = False # << whether the sample is mutable (ie., can produce histograms)
self.__selections = {} # << named selections of this sample
self.__uid = utils.uuid()
samplebase.__init__(self, files, lumi, stype, cfg, channel, title)
def searchPhone(self, message_number):
print 'processing message num: ' + str(message_number)
try:
response, message_data = self.imap.fetch(message_number, '(BODY.PEEK[HEADER])')
except:
print "Exception in HEADER"
return False
raw_message = message_data[0][1] # message_data, the data structure returned by imaplib, encodes some data re: the request type
header = HeaderParser().parsestr(raw_message)
if header['Content-Type'] is not None and 'multipart' in header['Content-Type']:
print "INcorrect content type"
return False # right now we're just skipping any multipart messages. this needs to be rewritten to parse the text parts of said messgs.
try:
response, message_data = self.imap.fetch(message_number, '(BODY.PEEK[TEXT])')
except:
print "Exception in TEXT"
return False
text_payload = message_data[0][1]
found_digits = number_re.findall(text_payload)
if found_digits != []:
print "Message %d has numbers." % num(message_number)
print found_digits
### need to cast the Date header into a MySQL object.
ts = header['Date']
print 'header date: ' + str(ts)
if rfc822.parsedate_tz(ts) is not None: #making sure the date header is not empty
ts_tuple = rfc822.parsedate_tz(ts)
#perhaps in the future we can intead set the ts_tuple to (0,0,0,0,0,0,0) and interpret it in the UI as 'no date header'. assuming that is actually the problem.
#otherwise, we're setting it to the date of the most recently received email... and this could get awkward. #TODO: fix this once the UI is ready.
ts_python_datetime = datetime.datetime(*(ts_tuple[0:6]))
ts_mysql_datetime = ts_python_datetime.isoformat(' ')
print 'about to insert into the database'
### sometimes it fails due to unicode issues
print 'about to parse name and email from header'
print 'header: ' + str(header['From'])
try:
name, email = email_re.match(header['From']).groups()[1:3]
except:
print "Unexpected error:", sys.exc_info()[0]
return False
print 'parsing name and email from FROM header: ' + str(name) + ', ' + str(email)
try:
m = Message(
sender=header['From'][:255],
recipient=header['To'][:255],
sender_name=str(name)[:255],
sender_email=email[:255],
subject=header['Subject'][:255],
date=ts_mysql_datetime,
payload=str(text_payload[:65534])
)
m.save()
except:
return False
pure_digits = uniqify(map(''.join, found_digits)) # the phone number regexp will create lists like ['','650','555','1212']. this collapses the list into a string.
print 'We found pure digits: ' + str(pure_digits)
for phone_number in pure_digits:
if len(str(phone_number)) > 7: # for now, we want numbers with area codes only.
print phone_number
PhoneNumber(value=phone_number, message=m).save()
def primeFactorsStd(n): primes = primeFactors(n) return map(lambda x : (x,primes.count(x)) , uniqify(primes))
def search_phone(self):
print 'processing message num: ' + str(self.number)
try:
response, message_data = self.imap.fetch(self.number, '(BODY.PEEK[HEADER])')
except:
print "Exception in HEADER"
return False
raw_message = message_data[0][1] # message_data, the data structure returned by imaplib, encodes some data re: the request type
header = HeaderParser().parsestr(raw_message)
if header['Content-Type'] is not None and 'multipart' in header['Content-Type']:
print "INcorrect content type"
return False # right now we're just skipping any multipart messages. this needs to be rewritten to parse the text parts of said messgs.
try:
response, message_data = self.imap.fetch(self.number, '(BODY.PEEK[TEXT])')
except:
print "Exception in TEXT"
return False
text_payload = message_data[0][1]
found_digits = number_re.findall(text_payload)
found_digits += number_re.findall(header['Subject'])
if len(found_digits) > 0:
print "Message %d has numbers." % num(self.number)
print found_digits
### need to cast the Date header into a MySQL object.
ts = header['Date']
print 'header date: ' + str(ts)
if parse(ts) is not None: #making sure the date header is not empty
date = parse(ts)
print 'about to insert into the database'
### sometimes it fails due to unicode issues
print 'about to parse name and email from header'
print 'header: ' + str(header['From'])
try:
name, email = email_re.match(header['From']).groups()[1:3]
except:
print "Unexpected error:", sys.exc_info()[0]
return False
print 'parsing name and email from FROM header: ' + str(name) + ', ' + str(email)
try:
m = Message(
user=self.user,
sender=header['From'][:255],
recipient=header['To'][:255],
sender_name=str(name)[:255],
sender_email=email[:255],
subject=header['Subject'][:255],
date_add=date,
payload=str(text_payload[:65534])
)
m.save()
except Exception as e:
print "Can't save", "test", e
pure_digits = uniqify(map(''.join, found_digits)) # the phone number regexp will create lists like ['','650','555','1212']. this collapses the list into a string.
print 'We found pure digits: ' + str(pure_digits)
for phone_number in pure_digits:
if len(str(phone_number)) > 7: # for now, we want numbers with area codes only.
print phone_number
PhoneNumber(value=phone_number, message=m, user=self.user, mailbox=self.mailbox).save()
self.found_phones += 1
def searchPhone(self, message_number):
print 'processing message num: ' + str(message_number)
try:
response, message_data = self.imap.fetch(message_number,
'(BODY.PEEK[HEADER])')
except:
print "Exception in HEADER"
return False
raw_message = message_data[0][
1] # message_data, the data structure returned by imaplib, encodes some data re: the request type
header = HeaderParser().parsestr(raw_message)
if header['Content-Type'] is not None and 'multipart' in header[
'Content-Type']:
print "INcorrect content type"
return False # right now we're just skipping any multipart messages. this needs to be rewritten to parse the text parts of said messgs.
try:
response, message_data = self.imap.fetch(message_number,
'(BODY.PEEK[TEXT])')
except:
print "Exception in TEXT"
return False
text_payload = message_data[0][1]
found_digits = number_re.findall(text_payload)
if found_digits != []:
print "Message %d has numbers." % num(message_number)
print found_digits
### need to cast the Date header into a MySQL object.
ts = header['Date']
print 'header date: ' + str(ts)
if rfc822.parsedate_tz(
ts) is not None: #making sure the date header is not empty
ts_tuple = rfc822.parsedate_tz(ts)
#perhaps in the future we can intead set the ts_tuple to (0,0,0,0,0,0,0) and interpret it in the UI as 'no date header'. assuming that is actually the problem.
#otherwise, we're setting it to the date of the most recently received email... and this could get awkward. #TODO: fix this once the UI is ready.
ts_python_datetime = datetime.datetime(*(ts_tuple[0:6]))
ts_mysql_datetime = ts_python_datetime.isoformat(' ')
print 'about to insert into the database'
### sometimes it fails due to unicode issues
print 'about to parse name and email from header'
print 'header: ' + str(header['From'])
try:
name, email = email_re.match(header['From']).groups()[1:3]
except:
print "Unexpected error:", sys.exc_info()[0]
return False
print 'parsing name and email from FROM header: ' + str(
name) + ', ' + str(email)
try:
m = Message(sender=header['From'][:255],
recipient=header['To'][:255],
sender_name=str(name)[:255],
sender_email=email[:255],
subject=header['Subject'][:255],
date=ts_mysql_datetime,
payload=str(text_payload[:65534]))
m.save()
except:
return False
pure_digits = uniqify(
map(''.join, found_digits)
) # the phone number regexp will create lists like ['','650','555','1212']. this collapses the list into a string.
print 'We found pure digits: ' + str(pure_digits)
for phone_number in pure_digits:
if len(str(phone_number)
) > 7: # for now, we want numbers with area codes only.
print phone_number
PhoneNumber(value=phone_number, message=m).save()