def seed_twitter(gate=None):
NER = readSeedIndex(LINKEDIN, False)
if gate is not None:
notProcessed = False
ignoreProcess = False
cnt = 0
Found = False
while (not Found) and cnt <= len(NER):
x = NER[cnt]
Found = x['firstName'].upper().find(
gate) >= 0 or x['lastName'].upper().find(gate) >= 0
cnt = cnt + 1
NER = NER[cnt:]
while True:
create_profiles_idx_from_twitter_search(NER)
NER = readSeedIndex("linkedin")
def process_twitter(gate='@davecowden'):
entities = readSeedIndex(TWITTER_IN)
notProcessed = False
ignoreProcessed = False
logging.getLogger().log(logging.CRITICAL,
"starting to process %s profiles " % len(entities))
for aBucket in entities:
notProcessed = aBucket['userName'] == gate
if notProcessed or ignoreProcessed:
ignoreProcessed = True
process_twitter_profile(aBucket)
else:
logging.getLogger().log(logging.CRITICAL,
"ignoring bucket %s" % aBucket)
while True:
entities = readSeedIndex("twitter_in")
for aBucket in entities:
process_twitter_profile(aBucket)
def main_from_site():
while True:
import time
links = readSeedIndex(LINKEDIN_INPUT)
for link in links:
try:
process_linkedin_profile_from_site(link)
except Exception as ex:
print ex
continue
def clean_re_read():
links = readSeedIndex(LINKEDIN_INPUT, False)
logging.getLogger().log(logging.CRITICAL,
"number of links from linkedin %s" % len(links))
for link in links:
try:
process_linkedin_profile_from_site(link)
except Exception as ex:
print ex
continue
def process_twitter_clean(gate='@ShowdownJoe'):
entities = readSeedIndex(TWITTER_IN, False)
notProcessed = False
ignoreProcessed = False
logging.getLogger().log(logging.CRITICAL,
"starting to process %s profiles " % len(entities))
for aBucket in entities:
notProcessed = aBucket['userName'] == gate
if notProcessed or ignoreProcessed:
ignoreProcessed = True
process_twitter_profile(aBucket)
else:
logging.getLogger().log(logging.CRITICAL,
"ignoring bucket %s" % aBucket)
def clean_read_meetup( gate = {u'lastName': u'Levinshtein', u'firstName': u'Gal'}):
NER = readSeedIndex(LINKEDIN,True)
notProcessed = False
ignoreProcessed = False
logging.getLogger().log(logging.CRITICAL,"starting to process %s profiles " % len(NER))
for a_ner in NER:
notProcessed = a_ner == gate
if notProcessed or ignoreProcessed:
ignoreProcessed = True
create_profiles_idx_from_meetup_search(a_ner)
else:
logging.getLogger().log(logging.CRITICAL,"ignoring %s" %a_ner)
logging.getLogger().log(logging.CRITICAL,"completed processing, last element %s" % a_ner)
main()
def clean_read(start_at_link='https://www.linkedin.com/in/kursadd'):
links = readSeedIndex(LINKEDIN_INPUT, False)
logging.getLogger().log(logging.CRITICAL,
"number of links from linkedin %s" % len(links))
notProcessed = False
ignoreProcessed = False
for link in links:
try:
notProcessed = start_at_link == link
if notProcessed or ignoreProcessed:
process_linkedin_profile_from_site(link)
ignoreProcessed = True
else:
logging.getLogger().log(logging.CRITICAL,
"ignoring link %s" % link)
except Exception as ex:
print ex
continue
def re_process_twitter_clean():
entities = readSeedIndex(TWITTER_IN, False)
logging.getLogger().log(logging.CRITICAL,
"starting to process %s profiles " % len(entities))
for aBucket in entities:
process_twitter_profile(aBucket)
def main():
while True:
import time
NER = readSeedIndex(LINKEDIN,False)
for a_ner in NER:
create_profiles_idx_from_meetup_search(a_ner)