def collect_all_apps():
urlGen = UrlGen()
allCategories = urlGen.get_test_categories()
appsList = parseapp.parse_all_apps(allCategories, None)
cc = 0
for i in appsList:
i.package = utils.get_package(i.link)
app = db.search_partappinfo(i.package)
dev = db.search_developer(i.company, i.package)
cc = cc + 1
if dev is None: #maybe this package is not relative with dev.
pass
else:
db.update_devinfo(dev.company, utils.getdate(), dev.status,
i.package, i.company, i.company_link)
#print i.package
if app is None:
db.write_appinfo(i.rank, i.title, i.package,
i.link, i.company, i.company_link, i.desc,
utils.getdate(), i.category, i.icon, i.icon_small)
else:
i.date = utils.getdate()
db.check_append_appchangelog_info(i, app)
db.update_appinfo(i.rank, i.title, i.package,
i.link, i.company, i.company_link, i.desc,
utils.getdate(), i.category, i.icon,
i.icon_small)
return appsList
def save_app_to_list(appslist):
cc = 0
date = utils.getdate()
#if len(appslist) > 0:
# print appslist[0].category
for i in appslist:
i.package = utils.get_package(i.link)
app = db.search_partappinfo(i.package)
dev = db.search_developer(i.company, i.package)
cc += 1
if dev is None: #maybe this package is not relative with dev.
pass
else:
db.update_devinfo(dev.company, date, dev.status, i.package,
i.company, i.company_link)
#if dev is not None and cmp(dev.date, date) != 0:
# print "update dev:", i.company, i.company_link, dev.date, date
if app is None:
db.write_appinfo(i.rank, i.title, i.package,
i.link, i.company, i.company_link, i.desc,
utils.getdate(), i.category, i.icon, i.icon_small)
else:
i.date = date
db.check_append_appchangelog_info(i, app)
db.update_appinfo(i.rank, i.title, i.package,
i.link, i.company, i.company_link, i.desc,
utils.getdate(), i.category, i.icon,
i.icon_small)
def update_app_developer():
index = 0
date = utils.getdate()
devsList = db.get_specail_devslist(date)
print "update_app_developer dev len=", devsList.__len__()
for dev in devsList:
app = db.search_appinfo(dev.package)
if app is not None:
db.update_devinfo(dev.company, utils.getdate(), dev.status,
app.package, app.company, app.company_link)
if cmp(dev.company, app.company) != 0:
print "update dev:", index, dev.company, app.package
index += 1
def _gen_resultado_estudio(self, estudio_clinico_resultado):
return OKW.K_RESULTADO_ESTUDIO(
FE_REALIZACION=getdate(),
DS_DESCRIPCION='DS_DESCRIPCION %d' % self._icount,
DS_ORIGEN='DS_ORIGEN %d' % self._icount,
K_ESTUDIO_CLINICO_RESULTADO = estudio_clinico_resultado,
)
def _clona_nota(self, nota):
nueva = self._set_defaults(clona(nota), False)
nueva.FL_DETALLE_EXPEDIENTE = 0
nueva.FL_USUARIO = nota.FL_USUARIO
nueva.CL_USUARIO = nota.CL_USUARIO
nueva.UI_EXPEDIENTE = Guid.NewGuid()
nueva.FE_CAPTURA = getdate().AddMinutes(self._icount)
return nueva
def check_apps_suspend():
appsList = db.get_specail_old_appslist(utils.getdate())
index = 0
print "check_apps_suspend app old len=", len(appsList)
for app in appsList:
if app.title.find(const._SUSPEND) == -1:
status = utils.get_httpstatuscode(app.link.strip('\n'))
if status != 200:
print("title:%s %s suspend!" % (app.title, app.link))
if app.title.find(const._SUSPEND) == -1:
app.title = const._SUSPEND + "[" + utils.getdate(
) + "]" + app.title
db.update_apptitle(app.title, app.package)
db.update_appdate(utils.getdate(), app.package)
else:
db.update_appdate(utils.getdate(), app.package)
#print ("index = %d %s status:%d" % (index, app.title, status))
index = index + 1
def check_app_developer_suspend():
index = 0
date = utils.getdate()
yesterday = utils.get_yesterday_date()
#date='2018-11'
devsList = db.get_specail_devslist_ex(date, yesterday)
print "check_app_developer_suspend dev len=", devsList.__len__()
for dev in devsList:
if cmp(dev.date, date) != 0 and cmp(dev.status, const.OK_STATUS) == 0:
#if dev.status != SUSPEND_STATUS:
req = utils.get_httpstatus_request(dev.company_link.strip('\n'))
if req.content is not None:
error = req.content.find(
'We\'re sorry, the requested URL was not found on this server.'
)
if error == -1:
dev_status = const.OK_STATUS
#print ("index = %d %s status:%s" % (index, dev.company, dev_status))
else:
dev_status = const.SUSPEND_STATUS
print("dev:%s %s suspend!" %
(dev.company, dev.company_link))
appsList = db.get_specail_appslist_bydev(dev.company)
for app in appsList:
if app.title.find(const._SUSPEND) == -1:
app.title = const._SUSPEND + "[" + utils.getdate(
) + "]" + app.title
db.update_apptitle(app.title, app.package)
print("\t [dead now]title:%s %s suspend!" %
(app.title, app.link))
else:
print("\t [dead ago]title:%s %s suspend!" %
(app.title, app.link))
elif req.status_code == 404:
dev_status = const.SUSPEND_STATUS
db.update_devinfo_simple(dev.company, utils.getdate(), dev_status,
dev.package)
index = index + 1
def _gen_diagnostico(self, diag):
return OKW.K_DIAGNOSTICO_PACIENTE(
FL_DIAGNOSTICO=diag,
FL_SEVERIDAD_DIAGNOSTICO=1,
FL_ESTATUS_DIAGNOSTICO=2,
FE_DIAGNOSTICO=getdate(),
FL_PACIENTE_CITA=0,
FG_ACCION_QX=False,
FG_ACCION_INCAPACIDAD=False,
FG_ACCION_REFERENCIA=False,
FG_ACCION_CONTRARREFERENCIA=False,
FG_REPLICADO=False,
)
def write_app(row):
global cc
i = AppDetail()
i.rank = row[0]
i.title = row[1]
i.link = row[2]
i.package = row[3]
i.company = row[4]
i.company_link = row[5]
i.desc = row[6]
i.date = row[7]
i.category = row[8]
i.icon = row[9]
i.icon_small = row[10]
date = utils.getdate()
app = db.search_partappinfo(i.package)
dev = db.search_developer(i.company, i.package)
cc += 1
if dev is None: #maybe this package is not relative with dev.
pass
else:
db.update_devinfo(dev.company, date, dev.status, i.package, i.company,
i.company_link)
if app is None:
db.write_appinfo(i.rank, i.title, i.package,
i.link, i.company, i.company_link, i.desc,
utils.getdate(), i.category, i.icon, i.icon_small)
'''
else:
db.check_append_appchangelog_info(i, app)
db.update_appinfo(i.rank, i.title, i.package,
i.link, i.company, i.company_link,
i.desc, utils.getdate(), i.category,
i.icon, i.icon_small)
'''
print "write app:", cc, i.title, i.package
def check_new_developer():
index = 0
date = utils.getdate()
appsList = db.get_specail_new_appslist(date)
print "check_new_developer app new len=", len(appsList)
for app in appsList:
dev = db.search_developer(app.company, app.package)
needupdate = 0
if dev is None:
dev = db.search_developer_pkg(app.package)
if dev is None:
db.write_developer(app.company, app.company_link, date,
app.package)
print "insert:", index, app.company, app.company_link, app.package
needupdate += 1
if needupdate == 0:
db.update_devinfo(dev.company, utils.getdate(), dev.status,
app.package, app.company, app.company_link)
#print dev.company, app.package
pass
index = index + 1
def _gen_medicacion(self):
# Evitamos el límite de 13 argumentos x función en ipy
m = OKW.K_MEDICACION()
m.FL_PRESENTACION = 379
m.NO_CANTIDAD = 1
m.NO_FRECUENCIA = 24
m.NO_DURACION = 60
m.DS_OBSERVACIONES = 'DS_OBSERVACIONES %d' % self._icount
m.FG_SUSPENDIDO = False
m.FE_INICIAR = getdate()
m.FG_HOSPITALIZADO = False
m.FG_FUERA_CUADRO = False
m.FG_DOSIS = False
m.FL_USUARIO = self.fl_usuario
m.FL_UNIDAD_TIEMPO_FRECUENCIA = 4
m.FL_UNIDAD_TIEMPO_DURACION = 2
m.FG_CONTRAREFERENCIA = False
m.FE_HASTA = getdate().AddDays(60)
m.FG_POST_HOSPITALARIA = False
return m
def _set_defaults(self, entidad, with_fl_paciente=True):
now = getdate()
defaults = dict(
FE_CREACION=now,
FE_ULTMOD=now,
FL_PACIENTE=self.fl_paciente,
FL_USUARIO=self.fl_usuario,
CL_USUARIO=self.cl_login,
NB_PROGRAMA=_PROGRAMA,
FG_REPLICADO=False,
FE_SYNC_EXPEDIENTE=_FE_SYNC_DEFAULT,
)
for (p, v) in defaults.iteritems():
if p == 'FL_PACIENTE' and with_fl_paciente == False:
continue
safe_set_prop(entidad, p, v)
return entidad
def write_app_changelog(row):
global cc
i = AppDetail()
i.rank = row[0]
i.title = row[1]
#i.link = row[2]
i.package = row[2]
i.company = row[3]
i.desc = row[4]
i.date = row[5]
i.category = row[6]
cc += 1
date = utils.getdate()
app = db.search_partappinfo(i.package)
if app is not None and cc == 1:
i.link = app.link
i.company_link = app.company_link
if i.title.find("new!!") != -1 or i.company.find(
"new!!") != -1 or i.desc.find("new!!") != -1:
db.write_appchangelogInfo_ex(i)
print "update appchangelog :", cc, i.title, i.package, i.company, i.desc
def force_check_suspended_developer():
index = 0
date = utils.getdate()
devsList = db.get_specail_devslist("xxxx")
print "force_check_suspended_developer dev len=", devsList.__len__()
for dev in devsList:
if dev.status == const.SUSPEND_STATUS:
req = utils.get_httpstatus_request(dev.company_link.strip('\n'))
if req.content is not None:
error = req.content.find(
'We\'re sorry, the requested URL was not found on this server.'
)
#print("dev:%s %s %s, %s, suspend!" % (dev.company, dev.company_link, error, req.content.__len__()))
if error == -1:
dev_status = const.OK_STATUS
print("index = %d %s status:%s" %
(index, dev.company, dev_status))
else:
dev_status = const.SUSPEND_STATUS
elif req.status_code == 404:
dev_status = const.SUSPEND_STATUS
#db.update_devinfo_simple(dev.company, utils.getdate(), dev_status, dev.package)
index = index + 1
devsList = db.get_specail_devslist(date)
print "update_app_developer dev len=", devsList.__len__()
for dev in devsList:
app = db.search_appinfo(dev.package)
if app is not None:
db.update_devinfo(dev.company, utils.getdate(), dev.status,
app.package, app.company, app.company_link)
if cmp(dev.company, app.company) != 0:
print "update dev:", index, dev.company, app.package
index += 1
def update_app_developer_fast():
index = 0
devsDict = db.search_not_same_dev()
print "update_app_developer_fast dev len=", devsDict.__len__()
for key, value in devsDict.items():
dev = value
db.update_devinfo_link(dev, key)
print "update dev:", index, dev.company, dev.package
index += 1
print "update_app_developer_fast:", utils.getdate()
update_app_developer_fast()
#print "rescan_developers:", utils.getdate()
#suspend_developer_rescan.rescan_developers()
#update_app_developer() #we should update developer first, becase developer maybe changed.
#check_new_developer()
def load_data(folderpath):
"""
Open CSV files in a folder and index the data.
"""
files = os.listdir(folderpath)
for f in files:
fullname = folderpath + f
if os.path.isfile(fullname):
with open(fullname, 'rb') as csvfile:
eventsreader = csv.reader(csvfile,
delimiter=',',
quotechar='"')
for line, row in enumerate(eventsreader):
if line == 0:
for column in range(0, len(row)):
if row[column] == "TimeCreated":
TIME_IDX = column
elif row[column] == "Id":
ID_IDX = column
elif row[column] == "MachineName":
MACHINENAME_IDX = column
elif row[column] == "TargetUserName":
TARGETUSERNAME_IDX = column
elif row[column] == "SubjectUserName":
SUBJECTUSERNAME_IDX = column
elif row[column] == "LogonType":
LOGONTYPE_IDX = column
elif row[column] == "ProcessName":
PROCESSNAME_IDX = column
elif row[column] == "IpAddress":
IP_IDX = column
elif row[column] == "WorkstationName":
WORKSTATIONAME_IDX = column
elif row[column] == "ServiceName":
SERVICENAME_IDX = column
elif row[column] == "ImagePath":
IMAGEPATH_IDX = column
elif row[column] == "ServiceType":
SERVICETYPE_IDX = column
elif row[column] == "Status":
STATUS_IDX = column
elif row[column] == "SubStatus":
SUBSTATUS_IDX = column
elif row[column] == "TaskName":
TASKNAME_IDX = column
elif row[column] == "TaskContent":
TASKCONTENT_IDX = column
elif row[column] == "Message":
MESSAGGE_IDX = column
elif row[column] == "ShareName":
SHARENAME_IDX = column
elif row[column] == "PrivilegeList":
PRIVILEGE_IDX = column
print "reading ", f, " line:", line
#4624 Authentication event
if row[ID_IDX] == "4624":
strtime = row[TIME_IDX]
time = getdate(strtime)
hostname = row[MACHINENAME_IDX]
username = row[TARGETUSERNAME_IDX].lower()
logontype = getlogontype(str(row[LOGONTYPE_IDX]))
processname = row[PROCESSNAME_IDX]
sourceip = row[IP_IDX]
sourcehost = row[WORKSTATIONAME_IDX]
host, created_host = Host.objects.get_or_create(
hostname=hostname)
user, created_user = User.objects.get_or_create(
username=username)
if (logontype == "Network"
or logontype == "RemoteInteractive") and (
sourceip != "-" and ":" not in sourceip):
source, created_source = SourceIp.objects.get_or_create(
sourceip=sourceip, hostname=sourcehost)
event4624, createdevent = Event_4624.objects.get_or_create(
time=time,
host=host,
user=user,
logontype=logontype,
processname=processname,
sourcehost=sourcehost,
sourceip=source)
else:
try:
event4624, createdevent = Event_4624.objects.get_or_create(
time=time,
host=host,
user=user,
logontype=logontype,
processname=processname,
sourcehost=sourcehost)
except:
pass
if createdevent:
print "new 4624 created for host " + hostname
# 4625 Authentication event
if row[ID_IDX] == "4625":
strtime = row[TIME_IDX]
time = getdate(strtime)
hostname = row[MACHINENAME_IDX]
username = row[TARGETUSERNAME_IDX].lower()
logontype = getlogontype(str(row[LOGONTYPE_IDX]))
processname = row[PROCESSNAME_IDX]
sourceip = row[IP_IDX]
sourcehost = row[WORKSTATIONAME_IDX]
status = row[STATUS_IDX]
substatus = row[SUBSTATUS_IDX]
host, created_host = Host.objects.get_or_create(
hostname=hostname)
user, created_user = User.objects.get_or_create(
username=username)
#new
if (logontype == "Network"
or logontype == "RemoteInteractive") and (
sourceip != "-" and ":" not in sourceip):
source, created_source = SourceIp.objects.get_or_create(
sourceip=sourceip, hostname=sourcehost)
event4625, createdevent = Event_4625.objects.get_or_create(
time=time,
host=host,
user=user,
logontype=logontype,
processname=processname,
sourcehost=sourcehost,
sourceip=source,
status=status,
substatus=substatus)
else:
event4625, createdevent = Event_4625.objects.get_or_create(
time=time,
host=host,
user=user,
logontype=logontype,
processname=processname,
sourcehost=sourcehost,
status=status,
substatus=substatus)
if createdevent:
print "new 4625 created for host " + hostname
# 4776 Local Authentication Event
if row[ID_IDX] == "4776":
strtime = row[TIME_IDX]
time = getdate(strtime)
hostname = row[MACHINENAME_IDX]
username = row[TARGETUSERNAME_IDX].lower()
status = row[STATUS_IDX]
host, created_host = Host.objects.get_or_create(
hostname=hostname)
user, created_user = User.objects.get_or_create(
username=username)
event4776, createdevent = Event_4776.objects.get_or_create(
time=time, host=host, user=user, status=status)
if createdevent:
print "new 4776 created for host " + hostname
# 5140 File Share Event
if row[ID_IDX] == "5140":
strtime = row[TIME_IDX]
time = getdate(strtime)
hostname = row[MACHINENAME_IDX]
username = row[SUBJECTUSERNAME_IDX].lower()
sharename = row[SHARENAME_IDX]
host, created_host = Host.objects.get_or_create(
hostname=hostname)
user, created_user = User.objects.get_or_create(
username=username)
event5140, createdevent = Event_5140.objects.get_or_create(
time=time,
host=host,
user=user,
sharename=sharename)
if createdevent:
print "new 5140 created for host " + hostname
# 4672 Privileged Auth
if row[ID_IDX] == "4672":
strtime = row[TIME_IDX]
time = getdate(strtime)
hostname = row[MACHINENAME_IDX]
username = row[SUBJECTUSERNAME_IDX].lower()
privilegelist = row[PRIVILEGE_IDX]
host, created_host = Host.objects.get_or_create(
hostname=hostname)
user, created_user = User.objects.get_or_create(
username=username)
event4672, createdevent = Event_4672.objects.get_or_create(
time=time,
host=host,
user=user,
privilegelist=privilegelist)
if createdevent:
print "new 4672 created for host " + hostname
# 7045 New Service created event
elif row[ID_IDX] == "7045":
strtime = row[TIME_IDX]
time = getdate(strtime)
hostname = row[MACHINENAME_IDX]
servicename = row[SERVICENAME_IDX]
imagepath = row[IMAGEPATH_IDX]
servicetype = row[SERVICETYPE_IDX]
service, service_created = Service.objects.get_or_create(
servicename=servicename,
imagepath=imagepath,
servicetype=servicetype)
host, created_host = Host.objects.get_or_create(
hostname=hostname)
event7045, createdevent = Event_7045.objects.get_or_create(
time=time, host=host, service=service)
if createdevent:
print "new 7045 created for " + hostname
# 4698 New Scheduled Task event
elif row[ID_IDX] == "4698":
strtime = row[TIME_IDX]
time = getdate(strtime)
hostname = row[MACHINENAME_IDX]
taskname = row[TASKNAME_IDX]
taskdetails = row[TASKCONTENT_IDX]
#arguments = re.findall('<Arguments>(.*?)</Arguments>', taskdetails, re.DOTALL)[0]
#cmdline = re.findall('<Command>(.*?)</Command>', taskdetails, re.DOTALL)[0]
try:
arguments = find_between(taskdetails,
"<Arguments>",
"</Arguments>")
except:
arguments = ""
try:
cmdline = find_between(taskdetails, "<Command>",
"</Command>")
except:
cmdline = ""
host, created_host = Host.objects.get_or_create(
hostname=hostname)
task, taskcreated = Task.objects.get_or_create(
taskname=taskname,
imagepath=cmdline,
arguments=arguments)
event4698, createdevent = Event_4698.objects.get_or_create(
time=time, host=host, task=task)
if createdevent:
print "new 4698 created for " + hostname
# WMI event
elif row[ID_IDX] == "2":
strtime = row[TIME_IDX]
time = getdate(strtime)
hostname = row[MACHINENAME_IDX]
wmidetails = row[MESSAGGE_IDX]
operationid = re.findall('GroupOperationId = (.*?);',
wmidetails, re.DOTALL)[0]
operation = re.findall('Operation = (.*?);',
wmidetails, re.DOTALL)[0]
host, created_host = Host.objects.get_or_create(
hostname=hostname)
WmiEvent_2.objects.create(time=time,
host=host,
operationid=operationid,
operation=operation)
print "new Wmi Event 2 created for " + hostname
def _gen_somatometria(self):
return SOMATOM.K_SOMATOMETRIA(
FE_MEDICION=getdate(),
OBSERVACIONES='OBSERVACIONES %d' % self._icount,
)
def _update_registro(self, entidad):
safe_set_prop(entidad, 'FE_ULTMOD', getdate())
safe_set_prop(entidad, 'FG_REPLICADO', False)
def _clona_resultado(self, resultado):
nueva = self._set_defaults(clona(resultado))
nueva.UI_RESULTADO = Guid.NewGuid()
nueva.FE_AGREGADO = getdate().AddMinutes(self._icount)
return nueva
def _gen_evento_prescripcion(self):
return OKW.K_EVENTO_PRESCRIPCION(
DS_EVENTO='DS_EVENTO prueba %d' % self._icount,
FE_PRESCRIPCION=getdate(),
)
